/src/python/WMComponent/WorkQueueManager/WorkQueueManager.py

https://github.com/dmwm/WMCore
Python | 80 lines | 44 code | 16 blank | 20 comment | 2 complexity | 0b460c7927111e209fdeb1ab09f71217 MD5 | raw file
  1. #!/usr/bin/env python
  2. """
  3. WorkQueuemanager component
  4. Runs periodic tasks for WorkQueue
  5. """
  6. from __future__ import print_function
  7. import threading
  8. from WMComponent.WorkQueueManager.WorkQueueManagerCleaner import WorkQueueManagerCleaner
  9. from WMComponent.WorkQueueManager.WorkQueueManagerLocationPoller import WorkQueueManagerLocationPoller
  10. from WMComponent.WorkQueueManager.WorkQueueManagerReqMgrPoller import WorkQueueManagerReqMgrPoller
  11. from WMComponent.WorkQueueManager.WorkQueueManagerWMBSFileFeeder import WorkQueueManagerWMBSFileFeeder
  12. from WMComponent.WorkQueueManager.WorkQueueManagerWorkPoller import WorkQueueManagerWorkPoller
  13. from WMCore.Agent.Harness import Harness
  14. from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig, queueConfigFromConfigObject
  15. class WorkQueueManager(Harness):
  16. """WorkQueuemanager component
  17. Runs periodic tasks for WorkQueue
  18. """
  19. def __init__(self, config):
  20. # call the base class
  21. Harness.__init__(self, config)
  22. self.config = queueConfigFromConfigObject(config)
  23. def preInitialization(self):
  24. print("WorkQueueManager.preInitialization")
  25. # Add event loop to worker manager
  26. myThread = threading.currentThread()
  27. pollInterval = self.config.WorkQueueManager.pollInterval
  28. dataLocationInterval = getattr(self.config.WorkQueueManager, "dataLocationInterval", 1 * 60 * 60)
  29. ### Global queue special functions
  30. if self.config.WorkQueueManager.level == 'GlobalQueue':
  31. # Get work from ReqMgr, report back & delete finished requests
  32. myThread.workerThreadManager.addWorker(
  33. WorkQueueManagerReqMgrPoller(
  34. queueFromConfig(self.config),
  35. getattr(self.config.WorkQueueManager,
  36. 'reqMgrConfig', {})
  37. ),
  38. pollInterval)
  39. ### local queue special function
  40. elif self.config.WorkQueueManager.level == 'LocalQueue':
  41. # pull work from parent queue
  42. myThread.workerThreadManager.addWorker(
  43. WorkQueueManagerWorkPoller(queueFromConfig(self.config),
  44. self.config),
  45. pollInterval)
  46. # inject acquired work into wmbs
  47. myThread.workerThreadManager.addWorker(
  48. WorkQueueManagerWMBSFileFeeder(queueFromConfig(self.config),
  49. self.config),
  50. pollInterval)
  51. ### general functions
  52. # Data location updates
  53. myThread.workerThreadManager.addWorker(
  54. WorkQueueManagerLocationPoller(queueFromConfig(self.config),
  55. self.config),
  56. dataLocationInterval)
  57. # Clean finished work & apply end policies
  58. myThread.workerThreadManager.addWorker(
  59. WorkQueueManagerCleaner(queueFromConfig(self.config),
  60. self.config),
  61. pollInterval)
  62. return