/src/python/WMCore/WMSpec/Steps/Templates/StageOut.py

https://github.com/PerilousApricot/WMCore
Python | 96 lines | 33 code | 11 blank | 52 comment | 1 complexity | 3445795b53c35e8bd1e06b6f290059be MD5 | raw file
  1. #!/usr/bin/env python
  2. """
  3. _StageOut_
  4. Template for a StageOut Step
  5. """
  6. from WMCore.WMSpec.Steps.Template import Template
  7. from WMCore.WMSpec.Steps.Template import CoreHelper
  8. from WMCore.WMSpec.ConfigSectionTree import nodeName
  9. class StageOutStepHelper(CoreHelper):
  10. """
  11. _StageOutStepHelper_
  12. Add API calls and helper methods to the basic WMStepHelper to specialise
  13. for StageOut tasks
  14. """
  15. def addFile(self, infile, outfile, stageOut = True):
  16. """
  17. Enqueues(sp?) a file to the StageOut step
  18. infile must be a LOCAL FILENAME. No file:/// in front
  19. outfile must be a LFN which will be mapped to the remote
  20. """
  21. target = self.data.files.section_("file%i" % self.data.filecount)
  22. target.input = infile
  23. target.output = outfile
  24. target.stageOut = stageOut
  25. self.data.filecount += 1
  26. def disableRetries(self):
  27. """
  28. handy for testing, without the 10 minute retry loop
  29. """
  30. self.data.retryCount = 1
  31. self.data.retryDelay = 0
  32. def disableStraightToMerge(self):
  33. """
  34. _disableStraightToMerge_
  35. Disable straight to merge for this step.
  36. """
  37. if hasattr(self.data.output, "minMergeSize"):
  38. delattr(self.data.output, "minMergeSize")
  39. return
  40. def setMinMergeSize(self, minMergeSize, maxMergeEvents):
  41. """
  42. _setMinMergeSize_
  43. Set the mininum size for promoting a file to merged status.
  44. """
  45. self.data.output.minMergeSize = minMergeSize
  46. self.data.output.maxMergeEvents = maxMergeEvents
  47. return
  48. def minMergeSize(self):
  49. """
  50. _minMergeSize_
  51. Retrieve the minimum size for promoting a file to merged status. If
  52. straight to merge is disabled -1 will be returned.
  53. """
  54. return getattr(self.data.output, "minMergeSize", -1)
  55. class StageOut(Template):
  56. """
  57. _StageOut_
  58. Tools for creating a template StageOut Step
  59. """
  60. def install(self, step):
  61. stepname = nodeName(step)
  62. step.stepType = "StageOut"
  63. step.section_("files")
  64. step.filecount = 0
  65. step.retryCount = 3
  66. step.retryDelay = 300
  67. def helper(self, step):
  68. """
  69. _helper_
  70. Wrap the WMStep provided in the CMSSW helper class that
  71. includes the ability to add and manipulate the details
  72. of a CMSSW workflow step
  73. """
  74. return StageOutStepHelper(step)