/xbmc/utils/JobManager.cpp

http://github.com/xbmc/xbmc · C++ · 423 lines · 332 code · 55 blank · 36 comment · 61 complexity · e0c91755c2479c44defc2376bd7c9519 MD5 · raw file

  1. /*
  2. * Copyright (C) 2005-2018 Team Kodi
  3. * This file is part of Kodi - https://kodi.tv
  4. *
  5. * SPDX-License-Identifier: GPL-2.0-or-later
  6. * See LICENSES/README.md for more information.
  7. */
  8. #include "JobManager.h"
  9. #include "threads/SingleLock.h"
  10. #include "utils/XTimeUtils.h"
  11. #include "utils/log.h"
  12. #include <algorithm>
  13. #include <functional>
  14. #include <stdexcept>
  15. bool CJob::ShouldCancel(unsigned int progress, unsigned int total) const
  16. {
  17. if (m_callback)
  18. return m_callback->OnJobProgress(progress, total, this);
  19. return false;
  20. }
  21. CJobWorker::CJobWorker(CJobManager *manager) : CThread("JobWorker")
  22. {
  23. m_jobManager = manager;
  24. Create(true); // start work immediately, and kill ourselves when we're done
  25. }
  26. CJobWorker::~CJobWorker()
  27. {
  28. // while we should already be removed from the job manager, if an exception
  29. // occurs during processing that we haven't caught, we may skip over that step.
  30. // Thus, before we go out of scope, ensure the job manager knows we're gone.
  31. m_jobManager->RemoveWorker(this);
  32. if(!IsAutoDelete())
  33. StopThread();
  34. }
  35. void CJobWorker::Process()
  36. {
  37. SetPriority( GetMinPriority() );
  38. while (true)
  39. {
  40. // request an item from our manager (this call is blocking)
  41. CJob *job = m_jobManager->GetNextJob(this);
  42. if (!job)
  43. break;
  44. bool success = false;
  45. try
  46. {
  47. success = job->DoWork();
  48. }
  49. catch (...)
  50. {
  51. CLog::Log(LOGERROR, "%s error processing job %s", __FUNCTION__, job->GetType());
  52. }
  53. m_jobManager->OnJobComplete(success, job);
  54. }
  55. }
  56. void CJobQueue::CJobPointer::CancelJob()
  57. {
  58. CJobManager::GetInstance().CancelJob(m_id);
  59. m_id = 0;
  60. }
  61. CJobQueue::CJobQueue(bool lifo, unsigned int jobsAtOnce, CJob::PRIORITY priority)
  62. : m_jobsAtOnce(jobsAtOnce), m_priority(priority), m_lifo(lifo)
  63. {
  64. }
  65. CJobQueue::~CJobQueue()
  66. {
  67. CancelJobs();
  68. }
  69. void CJobQueue::OnJobComplete(unsigned int jobID, bool success, CJob *job)
  70. {
  71. CSingleLock lock(m_section);
  72. // check if this job is in our processing list
  73. Processing::iterator i = find(m_processing.begin(), m_processing.end(), job);
  74. if (i != m_processing.end())
  75. m_processing.erase(i);
  76. // request a new job be queued
  77. QueueNextJob();
  78. }
  79. void CJobQueue::CancelJob(const CJob *job)
  80. {
  81. CSingleLock lock(m_section);
  82. Processing::iterator i = find(m_processing.begin(), m_processing.end(), job);
  83. if (i != m_processing.end())
  84. {
  85. i->CancelJob();
  86. m_processing.erase(i);
  87. return;
  88. }
  89. Queue::iterator j = find(m_jobQueue.begin(), m_jobQueue.end(), job);
  90. if (j != m_jobQueue.end())
  91. {
  92. j->FreeJob();
  93. m_jobQueue.erase(j);
  94. }
  95. }
  96. bool CJobQueue::AddJob(CJob *job)
  97. {
  98. CSingleLock lock(m_section);
  99. // check if we have this job already. If so, we're done.
  100. if (find(m_jobQueue.begin(), m_jobQueue.end(), job) != m_jobQueue.end() ||
  101. find(m_processing.begin(), m_processing.end(), job) != m_processing.end())
  102. {
  103. delete job;
  104. return false;
  105. }
  106. if (m_lifo)
  107. m_jobQueue.push_back(CJobPointer(job));
  108. else
  109. m_jobQueue.push_front(CJobPointer(job));
  110. QueueNextJob();
  111. return true;
  112. }
  113. void CJobQueue::QueueNextJob()
  114. {
  115. CSingleLock lock(m_section);
  116. if (m_jobQueue.size() && m_processing.size() < m_jobsAtOnce)
  117. {
  118. CJobPointer &job = m_jobQueue.back();
  119. job.m_id = CJobManager::GetInstance().AddJob(job.m_job, this, m_priority);
  120. m_processing.push_back(job);
  121. m_jobQueue.pop_back();
  122. }
  123. }
  124. void CJobQueue::CancelJobs()
  125. {
  126. CSingleLock lock(m_section);
  127. for_each(m_processing.begin(), m_processing.end(), [](CJobPointer& jp) { jp.CancelJob(); });
  128. for_each(m_jobQueue.begin(), m_jobQueue.end(), [](CJobPointer& jp) { jp.FreeJob(); });
  129. m_jobQueue.clear();
  130. m_processing.clear();
  131. }
  132. bool CJobQueue::IsProcessing() const
  133. {
  134. return CJobManager::GetInstance().m_running && (!m_processing.empty() || !m_jobQueue.empty());
  135. }
  136. bool CJobQueue::QueueEmpty() const
  137. {
  138. CSingleLock lock(m_section);
  139. return m_jobQueue.empty();
  140. }
  141. CJobManager &CJobManager::GetInstance()
  142. {
  143. static CJobManager sJobManager;
  144. return sJobManager;
  145. }
  146. CJobManager::CJobManager()
  147. {
  148. m_jobCounter = 0;
  149. m_running = true;
  150. m_pauseJobs = false;
  151. }
  152. void CJobManager::Restart()
  153. {
  154. CSingleLock lock(m_section);
  155. if (m_running)
  156. throw std::logic_error("CJobManager already running");
  157. m_running = true;
  158. }
  159. void CJobManager::CancelJobs()
  160. {
  161. CSingleLock lock(m_section);
  162. m_running = false;
  163. // clear any pending jobs
  164. for (unsigned int priority = CJob::PRIORITY_LOW_PAUSABLE; priority <= CJob::PRIORITY_DEDICATED; ++priority)
  165. {
  166. for_each(m_jobQueue[priority].begin(), m_jobQueue[priority].end(), [](CWorkItem& wi) { wi.FreeJob(); });
  167. m_jobQueue[priority].clear();
  168. }
  169. // cancel any callbacks on jobs still processing
  170. for_each(m_processing.begin(), m_processing.end(), [](CWorkItem& wi) { wi.Cancel(); });
  171. // tell our workers to finish
  172. while (m_workers.size())
  173. {
  174. lock.Leave();
  175. m_jobEvent.Set();
  176. std::this_thread::yield(); // yield after setting the event to give the workers some time to die
  177. lock.Enter();
  178. }
  179. }
  180. unsigned int CJobManager::AddJob(CJob *job, IJobCallback *callback, CJob::PRIORITY priority)
  181. {
  182. CSingleLock lock(m_section);
  183. if (!m_running)
  184. return 0;
  185. // increment the job counter, ensuring 0 (invalid job) is never hit
  186. m_jobCounter++;
  187. if (m_jobCounter == 0)
  188. m_jobCounter++;
  189. // create a work item for this job
  190. CWorkItem work(job, m_jobCounter, priority, callback);
  191. m_jobQueue[priority].push_back(work);
  192. StartWorkers(priority);
  193. return work.m_id;
  194. }
  195. void CJobManager::CancelJob(unsigned int jobID)
  196. {
  197. CSingleLock lock(m_section);
  198. // check whether we have this job in the queue
  199. for (unsigned int priority = CJob::PRIORITY_LOW_PAUSABLE; priority <= CJob::PRIORITY_DEDICATED; ++priority)
  200. {
  201. JobQueue::iterator i = find(m_jobQueue[priority].begin(), m_jobQueue[priority].end(), jobID);
  202. if (i != m_jobQueue[priority].end())
  203. {
  204. delete i->m_job;
  205. m_jobQueue[priority].erase(i);
  206. return;
  207. }
  208. }
  209. // or if we're processing it
  210. Processing::iterator it = find(m_processing.begin(), m_processing.end(), jobID);
  211. if (it != m_processing.end())
  212. it->m_callback = NULL; // job is in progress, so only thing to do is to remove callback
  213. }
  214. void CJobManager::StartWorkers(CJob::PRIORITY priority)
  215. {
  216. CSingleLock lock(m_section);
  217. // check how many free threads we have
  218. if (m_processing.size() >= GetMaxWorkers(priority))
  219. return;
  220. // do we have any sleeping threads?
  221. if (m_processing.size() < m_workers.size())
  222. {
  223. m_jobEvent.Set();
  224. return;
  225. }
  226. // everyone is busy - we need more workers
  227. m_workers.push_back(new CJobWorker(this));
  228. }
  229. CJob *CJobManager::PopJob()
  230. {
  231. CSingleLock lock(m_section);
  232. for (int priority = CJob::PRIORITY_DEDICATED; priority >= CJob::PRIORITY_LOW_PAUSABLE; --priority)
  233. {
  234. // Check whether we're pausing pausable jobs
  235. if (priority == CJob::PRIORITY_LOW_PAUSABLE && m_pauseJobs)
  236. continue;
  237. if (m_jobQueue[priority].size() && m_processing.size() < GetMaxWorkers(CJob::PRIORITY(priority)))
  238. {
  239. // pop the job off the queue
  240. CWorkItem job = m_jobQueue[priority].front();
  241. m_jobQueue[priority].pop_front();
  242. // add to the processing vector
  243. m_processing.push_back(job);
  244. job.m_job->m_callback = this;
  245. return job.m_job;
  246. }
  247. }
  248. return NULL;
  249. }
  250. void CJobManager::PauseJobs()
  251. {
  252. CSingleLock lock(m_section);
  253. m_pauseJobs = true;
  254. }
  255. void CJobManager::UnPauseJobs()
  256. {
  257. CSingleLock lock(m_section);
  258. m_pauseJobs = false;
  259. }
  260. bool CJobManager::IsProcessing(const CJob::PRIORITY &priority) const
  261. {
  262. CSingleLock lock(m_section);
  263. if (m_pauseJobs)
  264. return false;
  265. for(Processing::const_iterator it = m_processing.begin(); it < m_processing.end(); ++it)
  266. {
  267. if (priority == it->m_priority)
  268. return true;
  269. }
  270. return false;
  271. }
  272. int CJobManager::IsProcessing(const std::string &type) const
  273. {
  274. int jobsMatched = 0;
  275. CSingleLock lock(m_section);
  276. if (m_pauseJobs)
  277. return 0;
  278. for(Processing::const_iterator it = m_processing.begin(); it < m_processing.end(); ++it)
  279. {
  280. if (type == std::string(it->m_job->GetType()))
  281. jobsMatched++;
  282. }
  283. return jobsMatched;
  284. }
  285. CJob *CJobManager::GetNextJob(const CJobWorker *worker)
  286. {
  287. CSingleLock lock(m_section);
  288. while (m_running)
  289. {
  290. // grab a job off the queue if we have one
  291. CJob *job = PopJob();
  292. if (job)
  293. return job;
  294. // no jobs are left - sleep for 30 seconds to allow new jobs to come in
  295. lock.Leave();
  296. bool newJob = m_jobEvent.WaitMSec(30000);
  297. lock.Enter();
  298. if (!newJob)
  299. break;
  300. }
  301. // ensure no jobs have come in during the period after
  302. // timeout and before we held the lock
  303. CJob *job = PopJob();
  304. if (job)
  305. return job;
  306. // have no jobs
  307. RemoveWorker(worker);
  308. return NULL;
  309. }
  310. bool CJobManager::OnJobProgress(unsigned int progress, unsigned int total, const CJob *job) const
  311. {
  312. CSingleLock lock(m_section);
  313. // find the job in the processing queue, and check whether it's cancelled (no callback)
  314. Processing::const_iterator i = find(m_processing.begin(), m_processing.end(), job);
  315. if (i != m_processing.end())
  316. {
  317. CWorkItem item(*i);
  318. lock.Leave(); // leave section prior to call
  319. if (item.m_callback)
  320. {
  321. item.m_callback->OnJobProgress(item.m_id, progress, total, job);
  322. return false;
  323. }
  324. }
  325. return true; // couldn't find the job, or it's been cancelled
  326. }
  327. void CJobManager::OnJobComplete(bool success, CJob *job)
  328. {
  329. CSingleLock lock(m_section);
  330. // remove the job from the processing queue
  331. Processing::iterator i = find(m_processing.begin(), m_processing.end(), job);
  332. if (i != m_processing.end())
  333. {
  334. // tell any listeners we're done with the job, then delete it
  335. CWorkItem item(*i);
  336. lock.Leave();
  337. try
  338. {
  339. if (item.m_callback)
  340. item.m_callback->OnJobComplete(item.m_id, success, item.m_job);
  341. }
  342. catch (...)
  343. {
  344. CLog::Log(LOGERROR, "%s error processing job %s", __FUNCTION__, item.m_job->GetType());
  345. }
  346. lock.Enter();
  347. Processing::iterator j = find(m_processing.begin(), m_processing.end(), job);
  348. if (j != m_processing.end())
  349. m_processing.erase(j);
  350. lock.Leave();
  351. item.FreeJob();
  352. }
  353. }
  354. void CJobManager::RemoveWorker(const CJobWorker *worker)
  355. {
  356. CSingleLock lock(m_section);
  357. // remove our worker
  358. Workers::iterator i = find(m_workers.begin(), m_workers.end(), worker);
  359. if (i != m_workers.end())
  360. m_workers.erase(i); // workers auto-delete
  361. }
  362. unsigned int CJobManager::GetMaxWorkers(CJob::PRIORITY priority)
  363. {
  364. static const unsigned int max_workers = 5;
  365. if (priority == CJob::PRIORITY_DEDICATED)
  366. return 10000; // A large number..
  367. return max_workers - (CJob::PRIORITY_HIGH - priority);
  368. }