PageRenderTime 61ms CodeModel.GetById 26ms RepoModel.GetById 1ms app.codeStats 0ms

/src/GearmanManager/GearmanManager.php

http://github.com/brianlmoon/GearmanManager
PHP | 1213 lines | 685 code | 216 blank | 312 comment | 159 complexity | 5246df19a4bdd9ae877962e0afdc148e MD5 | raw file
  1. <?php
  2. namespace GearmanManager;
  3. /**
  4. *
  5. * PHP script for managing PHP based Gearman workers
  6. *
  7. * Copyright (c) 2010, Brian Moon
  8. * All rights reserved.
  9. *
  10. * Redistribution and use in source and binary forms, with or without
  11. * modification, are permitted provided that the following conditions are met:
  12. *
  13. * * Redistributions of source code must retain the above copyright notice,
  14. * this list of conditions and the following disclaimer.
  15. * * Redistributions in binary form must reproduce the above copyright
  16. * notice, this list of conditions and the following disclaimer in the
  17. * documentation and/or other materials provided with the distribution.
  18. * * Neither the name of Brian Moon nor the names of its contributors may be
  19. * used to endorse or promote products derived from this software without
  20. * specific prior written permission.
  21. *
  22. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32. * POSSIBILITY OF SUCH DAMAGE.
  33. *
  34. */
  35. declare(ticks = 1);
  36. error_reporting(E_ALL | E_STRICT);
  37. /**
  38. * Class that handles all the process management
  39. */
  40. abstract class GearmanManager {
  41. /**
  42. * Log levels can be enabled from the command line with -v, -vv, -vvv
  43. */
  44. const LOG_LEVEL_INFO = 1;
  45. const LOG_LEVEL_PROC_INFO = 2;
  46. const LOG_LEVEL_WORKER_INFO = 3;
  47. const LOG_LEVEL_DEBUG = 4;
  48. const LOG_LEVEL_CRAZY = 5;
  49. /**
  50. * Default config section name
  51. */
  52. const DEFAULT_CONFIG = "GearmanManager";
  53. /**
  54. * Defines job priority limits
  55. */
  56. const MIN_PRIORITY = -5;
  57. const MAX_PRIORITY = 5;
  58. /**
  59. * Holds the worker configuration
  60. */
  61. protected $config = array();
  62. /**
  63. * Boolean value that determines if the running code is the parent or a child
  64. */
  65. protected $isparent = true;
  66. /**
  67. * When true, workers will stop look for jobs and the parent process will
  68. * kill off all running children
  69. */
  70. protected $stop_work = false;
  71. /**
  72. * The timestamp when the signal was received to stop working
  73. */
  74. protected $stop_time = 0;
  75. /**
  76. * The filename to log to
  77. */
  78. protected $log_file;
  79. /**
  80. * Holds the resource for the log file
  81. */
  82. protected $log_file_handle;
  83. /**
  84. * Flag for logging to syslog
  85. */
  86. protected $log_syslog = false;
  87. /**
  88. * Verbosity level for the running script. Set via -v option
  89. */
  90. protected $verbose = 0;
  91. /**
  92. * The array of running child processes
  93. */
  94. protected $children = array();
  95. /**
  96. * The array of jobs that have workers running
  97. */
  98. protected $jobs = array();
  99. /**
  100. * The PID of the running process. Set for parent and child processes
  101. */
  102. protected $pid = 0;
  103. /**
  104. * The PID of the parent process, when running in the forked helper.
  105. */
  106. protected $parent_pid = 0;
  107. /**
  108. * PID file for the parent process
  109. */
  110. protected $pid_file = "";
  111. /**
  112. * PID of helper child
  113. */
  114. protected $helper_pid = 0;
  115. /**
  116. * The user to run as
  117. */
  118. protected $user = null;
  119. /**
  120. * If true, the worker code directory is checked for updates and workers
  121. * are restarted automatically.
  122. */
  123. protected $check_code = false;
  124. /**
  125. * Holds the last timestamp of when the code was checked for updates
  126. */
  127. protected $last_check_time = 0;
  128. /**
  129. * When forking helper children, the parent waits for a signal from them
  130. * to continue doing anything
  131. */
  132. protected $wait_for_signal = false;
  133. /**
  134. * Directory where worker functions are found
  135. */
  136. protected $worker_dir = "";
  137. /**
  138. * Number of workers that do all jobs
  139. */
  140. protected $do_all_count = 0;
  141. /**
  142. * Maximum time a worker will run
  143. */
  144. protected $max_run_time = 3600;
  145. /**
  146. * +/- number of seconds workers will delay before restarting
  147. * this prevents all your workers from restarting at the same
  148. * time which causes a connection stampeded on your daemons
  149. * So, a max_run_time of 3600 and worker_restart_splay of 600 means
  150. * a worker will self restart between 3600 and 4200 seconds after
  151. * it is started.
  152. *
  153. * This does not affect the time the parent process waits on children
  154. * when shutting down.
  155. */
  156. protected $worker_restart_splay = 600;
  157. /**
  158. * Maximum number of jobs this worker will do before quitting
  159. */
  160. protected $max_job_count = 0;
  161. /**
  162. * Maximum job iterations per worker
  163. */
  164. protected $max_runs_per_worker = null;
  165. /**
  166. * Number of times this worker has run a job
  167. */
  168. protected $job_execution_count = 0;
  169. /**
  170. * Servers that workers connect to
  171. */
  172. protected $servers = array();
  173. /**
  174. * List of functions available for work
  175. */
  176. protected $functions = array();
  177. /**
  178. * Function/Class prefix
  179. */
  180. protected $prefix = "";
  181. /**
  182. * Creates the manager and gets things going
  183. *
  184. */
  185. public function __construct() {
  186. if (!function_exists("posix_kill")) {
  187. $this->show_help("The function posix_kill was not found. Please ensure POSIX functions are installed");
  188. }
  189. if (!function_exists("pcntl_fork")) {
  190. $this->show_help("The function pcntl_fork was not found. Please ensure Process Control functions are installed");
  191. }
  192. $this->pid = getmypid();
  193. /**
  194. * Parse command line options. Loads the config file as well
  195. */
  196. $this->getopt();
  197. /**
  198. * Register signal listeners
  199. */
  200. $this->register_ticks();
  201. /**
  202. * Load up the workers
  203. */
  204. $this->load_workers();
  205. if (empty($this->functions)) {
  206. $this->log("No workers found");
  207. posix_kill($this->pid, SIGUSR1);
  208. exit();
  209. }
  210. /**
  211. * Validate workers in the helper process
  212. */
  213. $this->fork_me("validate_workers");
  214. $this->log("Started with pid $this->pid", GearmanManager::LOG_LEVEL_PROC_INFO);
  215. /**
  216. * Start the initial workers and set up a running environment
  217. */
  218. $this->bootstrap();
  219. $this->process_loop();
  220. /**
  221. * Kill the helper if it is running
  222. */
  223. if (isset($this->helper_pid)) {
  224. posix_kill($this->helper_pid, SIGKILL);
  225. }
  226. $this->log("Exiting");
  227. }
  228. protected function process_loop() {
  229. /**
  230. * Main processing loop for the parent process
  231. */
  232. while (!$this->stop_work || count($this->children)) {
  233. $status = null;
  234. /**
  235. * Check for exited children
  236. */
  237. $exited = pcntl_wait( $status, WNOHANG );
  238. /**
  239. * We run other children, make sure this is a worker
  240. */
  241. if (isset($this->children[$exited])) {
  242. /**
  243. * If they have exited, remove them from the children array
  244. * If we are not stopping work, start another in its place
  245. */
  246. if ($exited) {
  247. $worker = $this->children[$exited]['job'];
  248. unset($this->children[$exited]);
  249. $code = pcntl_wexitstatus($status);
  250. if ($code === 0) {
  251. $exit_status = "exited";
  252. } else {
  253. $exit_status = $code;
  254. }
  255. $this->child_status_monitor($pid, $child["job"], $exit_status);
  256. if (!$this->stop_work) {
  257. $this->start_worker($worker);
  258. }
  259. }
  260. }
  261. if ($this->stop_work && time() - $this->stop_time > 60) {
  262. $this->log("Children have not exited, killing.", GearmanManager::LOG_LEVEL_PROC_INFO);
  263. $this->stop_children(SIGKILL);
  264. } else {
  265. /**
  266. * If any children have been running 150% of max run time, forcibly terminate them
  267. */
  268. if (!empty($this->children)) {
  269. foreach ($this->children as $pid => $child) {
  270. if (!empty($child['start_time']) && time() - $child['start_time'] > $this->max_run_time * 1.5) {
  271. $this->child_status_monitor($pid, $child["job"], "killed");
  272. posix_kill($pid, SIGKILL);
  273. }
  274. }
  275. }
  276. }
  277. /**
  278. * php will eat up your cpu if you don't have this
  279. */
  280. usleep(50000);
  281. }
  282. }
  283. /**
  284. * Handles anything we need to do when we are shutting down
  285. *
  286. */
  287. public function __destruct() {
  288. if ($this->isparent) {
  289. if (!empty($this->pid_file) && file_exists($this->pid_file)) {
  290. if (!unlink($this->pid_file)) {
  291. $this->log("Could not delete PID file", GearmanManager::LOG_LEVEL_PROC_INFO);
  292. }
  293. }
  294. }
  295. }
  296. /**
  297. * Parses the command line options
  298. *
  299. */
  300. protected function getopt() {
  301. $opts = getopt("ac:dD:h:Hl:o:p:P:u:v::w:r:x:Z");
  302. if (isset($opts["H"])) {
  303. $this->show_help();
  304. }
  305. if (isset($opts["c"])) {
  306. $this->config['file'] = $opts['c'];
  307. }
  308. if (isset($this->config['file'])) {
  309. if (file_exists($this->config['file'])) {
  310. $this->parse_config($this->config['file']);
  311. }
  312. else {
  313. $this->show_help("Config file {$this->config['file']} not found.");
  314. }
  315. }
  316. /**
  317. * command line opts always override config file
  318. */
  319. if (isset($opts['P'])) {
  320. $this->config['pid_file'] = $opts['P'];
  321. }
  322. if (isset($opts["l"])) {
  323. $this->config['log_file'] = $opts["l"];
  324. }
  325. if (isset($opts['a'])) {
  326. $this->config['auto_update'] = 1;
  327. }
  328. if (isset($opts['w'])) {
  329. $this->config['worker_dir'] = $opts['w'];
  330. }
  331. if (isset($opts['x'])) {
  332. $this->config['max_worker_lifetime'] = (int)$opts['x'];
  333. }
  334. if (isset($opts['r'])) {
  335. $this->config['max_runs_per_worker'] = (int)$opts['r'];
  336. }
  337. if (isset($opts['D'])) {
  338. $this->config['count'] = (int)$opts['D'];
  339. }
  340. if (isset($opts['t'])) {
  341. $this->config['timeout'] = $opts['t'];
  342. }
  343. if (isset($opts['h'])) {
  344. $this->config['host'] = $opts['h'];
  345. }
  346. if (isset($opts['p'])) {
  347. $this->prefix = $opts['p'];
  348. } elseif (!empty($this->config['prefix'])) {
  349. $this->prefix = $this->config['prefix'];
  350. }
  351. if (isset($opts['u'])) {
  352. $this->user = $opts['u'];
  353. } elseif (isset($this->config["user"])) {
  354. $this->user = $this->config["user"];
  355. }
  356. /**
  357. * If we want to daemonize, fork here and exit
  358. */
  359. if (isset($opts["d"])) {
  360. $pid = pcntl_fork();
  361. if ($pid>0) {
  362. $this->isparent = false;
  363. exit();
  364. }
  365. $this->pid = getmypid();
  366. posix_setsid();
  367. }
  368. if (!empty($this->config['pid_file'])) {
  369. $fp = @fopen($this->config['pid_file'], "w");
  370. if ($fp) {
  371. fwrite($fp, $this->pid);
  372. fclose($fp);
  373. } else {
  374. $this->show_help("Unable to write PID to {$this->config['pid_file']}");
  375. }
  376. $this->pid_file = $this->config['pid_file'];
  377. }
  378. if (!empty($this->config['log_file'])) {
  379. if ($this->config['log_file'] === 'syslog') {
  380. $this->log_syslog = true;
  381. } else {
  382. $this->log_file = $this->config['log_file'];
  383. $this->open_log_file();
  384. }
  385. }
  386. if (isset($opts["v"])) {
  387. switch ($opts["v"]) {
  388. case false:
  389. $this->verbose = GearmanManager::LOG_LEVEL_INFO;
  390. break;
  391. case "v":
  392. $this->verbose = GearmanManager::LOG_LEVEL_PROC_INFO;
  393. break;
  394. case "vv":
  395. $this->verbose = GearmanManager::LOG_LEVEL_WORKER_INFO;
  396. break;
  397. case "vvv":
  398. $this->verbose = GearmanManager::LOG_LEVEL_DEBUG;
  399. break;
  400. case "vvvv":
  401. default:
  402. $this->verbose = GearmanManager::LOG_LEVEL_CRAZY;
  403. break;
  404. }
  405. }
  406. if ($this->user) {
  407. $user = posix_getpwnam($this->user);
  408. if (!$user || !isset($user['uid'])) {
  409. $this->show_help("User ({$this->user}) not found.");
  410. }
  411. /**
  412. * Ensure new uid can read/write pid and log files
  413. */
  414. if (!empty($this->pid_file)) {
  415. if (!chown($this->pid_file, $user['uid'])) {
  416. $this->log("Unable to chown PID file to {$this->user}", GearmanManager::LOG_LEVEL_PROC_INFO);
  417. }
  418. }
  419. if (!empty($this->log_file_handle)) {
  420. if (!chown($this->log_file, $user['uid'])) {
  421. $this->log("Unable to chown log file to {$this->user}", GearmanManager::LOG_LEVEL_PROC_INFO);
  422. }
  423. }
  424. posix_setuid($user['uid']);
  425. if (posix_geteuid() != $user['uid']) {
  426. $this->show_help("Unable to change user to {$this->user} (UID: {$user['uid']}).");
  427. }
  428. $this->log("User set to {$this->user}", GearmanManager::LOG_LEVEL_PROC_INFO);
  429. }
  430. if (!empty($this->config['auto_update'])) {
  431. $this->check_code = true;
  432. }
  433. if (!empty($this->config['worker_dir'])) {
  434. $this->worker_dir = $this->config['worker_dir'];
  435. } else {
  436. $this->worker_dir = "./workers";
  437. }
  438. $dirs = is_array($this->worker_dir) ? $this->worker_dir : explode(",", $this->worker_dir);
  439. foreach ($dirs as &$dir) {
  440. $dir = trim($dir);
  441. if (!file_exists($dir)) {
  442. $this->show_help("Worker dir ".$dir." not found");
  443. }
  444. }
  445. unset($dir);
  446. if (isset($this->config['max_worker_lifetime']) && (int)$this->config['max_worker_lifetime'] > 0) {
  447. $this->max_run_time = (int)$this->config['max_worker_lifetime'];
  448. }
  449. if (isset($this->config['worker_restart_splay']) && (int)$this->config['worker_restart_splay'] > 0) {
  450. $this->worker_restart_splay = (int)$this->config['worker_restart_splay'];
  451. }
  452. if (isset($this->config['count']) && (int)$this->config['count'] > 0) {
  453. $this->do_all_count = (int)$this->config['count'];
  454. }
  455. if (!empty($this->config['host'])) {
  456. if (!is_array($this->config['host'])) {
  457. $this->servers = explode(",", $this->config['host']);
  458. } else {
  459. $this->servers = $this->config['host'];
  460. }
  461. } else {
  462. $this->servers = array("127.0.0.1");
  463. }
  464. if (!empty($this->config['include']) && $this->config['include'] != "*") {
  465. $this->config['include'] = explode(",", $this->config['include']);
  466. } else {
  467. $this->config['include'] = array();
  468. }
  469. if (!empty($this->config['exclude'])) {
  470. $this->config['exclude'] = explode(",", $this->config['exclude']);
  471. } else {
  472. $this->config['exclude'] = array();
  473. }
  474. /**
  475. * Debug option to dump the config and exit
  476. */
  477. if (isset($opts["Z"])) {
  478. print_r($this->config);
  479. exit();
  480. }
  481. }
  482. /**
  483. * Parses the config file
  484. *
  485. * @param string $file The config file. Just pass so we don't have
  486. * to keep it around in a var
  487. */
  488. protected function parse_config($file) {
  489. $this->log("Loading configuration from $file");
  490. if (substr($file, -4) == ".php") {
  491. require $file;
  492. } elseif (substr($file, -4) == ".ini") {
  493. $gearman_config = parse_ini_file($file, true);
  494. }
  495. if (empty($gearman_config)) {
  496. $this->show_help("No configuration found in $file");
  497. }
  498. if (isset($gearman_config[self::DEFAULT_CONFIG])) {
  499. $this->config = $gearman_config[self::DEFAULT_CONFIG];
  500. $this->config['functions'] = array();
  501. }
  502. foreach ($gearman_config as $function=>$data) {
  503. if (strcasecmp($function, self::DEFAULT_CONFIG) != 0) {
  504. $this->config['functions'][$function] = $data;
  505. }
  506. }
  507. }
  508. /**
  509. * Helper function to load and filter worker files
  510. *
  511. * return @void
  512. */
  513. protected function load_workers() {
  514. $this->functions = array();
  515. $dirs = is_array($this->worker_dir) ? $this->worker_dir : explode(",", $this->worker_dir);
  516. foreach ($dirs as $dir) {
  517. $this->log("Loading workers in ".$dir);
  518. $worker_files = glob($dir."/*.php");
  519. if (!empty($worker_files)) {
  520. foreach ($worker_files as $file) {
  521. $function = substr(basename($file), 0, -4);
  522. /**
  523. * include workers
  524. */
  525. if (!empty($this->config['include'])) {
  526. if (!in_array($function, $this->config['include'])) {
  527. continue;
  528. }
  529. }
  530. /**
  531. * exclude workers
  532. */
  533. if (in_array($function, $this->config['exclude'])) {
  534. continue;
  535. }
  536. if (!isset($this->functions[$function])) {
  537. $this->functions[$function] = array('name' => $function);
  538. }
  539. if (!empty($this->config['functions'][$function]['dedicated_only'])) {
  540. if (empty($this->config['functions'][$function]['dedicated_count'])) {
  541. $this->log("Invalid configuration for dedicated_count for function $function.", GearmanManager::LOG_LEVEL_PROC_INFO);
  542. exit();
  543. }
  544. $this->functions[$function]['dedicated_only'] = true;
  545. $this->functions[$function]["count"] = $this->config['functions'][$function]['dedicated_count'];
  546. } else {
  547. $min_count = max($this->do_all_count, 1);
  548. if (!empty($this->config['functions'][$function]['count'])) {
  549. $min_count = max($this->config['functions'][$function]['count'], $this->do_all_count);
  550. }
  551. if (!empty($this->config['functions'][$function]['dedicated_count'])) {
  552. $ded_count = $this->do_all_count + $this->config['functions'][$function]['dedicated_count'];
  553. } elseif (!empty($this->config["dedicated_count"])) {
  554. $ded_count = $this->do_all_count + $this->config["dedicated_count"];
  555. } else {
  556. $ded_count = $min_count;
  557. }
  558. $this->functions[$function]["count"] = max($min_count, $ded_count);
  559. }
  560. $this->functions[$function]['path'] = $file;
  561. /**
  562. * Note about priority. This exploits an undocumented feature
  563. * of the gearman daemon. This will only work as long as the
  564. * current behavior of the daemon remains the same. It is not
  565. * a defined part fo the protocol.
  566. */
  567. if (!empty($this->config['functions'][$function]['priority'])) {
  568. $priority = max(min(
  569. $this->config['functions'][$function]['priority'],
  570. self::MAX_PRIORITY), self::MIN_PRIORITY);
  571. } else {
  572. $priority = 0;
  573. }
  574. $this->functions[$function]['priority'] = $priority;
  575. }
  576. }
  577. }
  578. }
  579. /**
  580. * Forks the process and runs the given method. The parent then waits
  581. * for the child process to signal back that it can continue
  582. *
  583. * @param string $method Class method to run after forking
  584. *
  585. */
  586. protected function fork_me($method) {
  587. $this->wait_for_signal = true;
  588. $pid = pcntl_fork();
  589. switch ($pid) {
  590. case 0:
  591. $this->isparent = false;
  592. $this->parent_pid = $this->pid;
  593. $this->pid = getmypid();
  594. $this->$method();
  595. break;
  596. case -1:
  597. $this->log("Failed to fork");
  598. $this->stop_work = true;
  599. break;
  600. default:
  601. $this->log("Helper forked with pid $pid", GearmanManager::LOG_LEVEL_PROC_INFO);
  602. $this->helper_pid = $pid;
  603. while ($this->wait_for_signal && !$this->stop_work) {
  604. usleep(5000);
  605. pcntl_waitpid($pid, $status, WNOHANG);
  606. if (pcntl_wifexited($status) && $status) {
  607. $this->log("Helper child exited with non-zero exit code $status.");
  608. exit(1);
  609. }
  610. }
  611. break;
  612. }
  613. }
  614. /**
  615. * Forked method that validates the worker code and checks it if desired
  616. *
  617. */
  618. protected function validate_workers() {
  619. $this->load_workers();
  620. if (empty($this->functions)) {
  621. $this->log("No workers found");
  622. posix_kill($this->parent_pid, SIGUSR1);
  623. exit();
  624. }
  625. $this->validate_lib_workers();
  626. /**
  627. * Since we got here, all must be ok, send a CONTINUE
  628. */
  629. $this->log("Helper is running. Sending continue to $this->parent_pid.", GearmanManager::LOG_LEVEL_DEBUG);
  630. posix_kill($this->parent_pid, SIGCONT);
  631. if ($this->check_code) {
  632. $this->log("Running loop to check for new code", self::LOG_LEVEL_DEBUG);
  633. $last_check_time = 0;
  634. while (1) {
  635. $max_time = 0;
  636. foreach ($this->functions as $name => $func) {
  637. clearstatcache();
  638. $mtime = filemtime($func['path']);
  639. $max_time = max($max_time, $mtime);
  640. $this->log("{$func['path']} - $mtime $last_check_time", self::LOG_LEVEL_CRAZY);
  641. if ($last_check_time!=0 && $mtime > $last_check_time) {
  642. $this->log("New code found. Sending SIGHUP", self::LOG_LEVEL_PROC_INFO);
  643. posix_kill($this->parent_pid, SIGHUP);
  644. break;
  645. }
  646. }
  647. $last_check_time = $max_time;
  648. sleep(5);
  649. }
  650. } else {
  651. exit();
  652. }
  653. }
  654. /**
  655. * Bootstap a set of workers and any vars that need to be set
  656. *
  657. */
  658. protected function bootstrap() {
  659. $function_count = array();
  660. /**
  661. * If we have "do_all" workers, start them first
  662. * do_all workers register all functions
  663. */
  664. if (!empty($this->do_all_count) && is_int($this->do_all_count)) {
  665. for ($x=0;$x<$this->do_all_count;$x++) {
  666. $this->start_worker();
  667. /**
  668. * Don't start workers too fast. They can overwhelm the
  669. * gearmand server and lead to connection timeouts.
  670. */
  671. usleep(500000);
  672. }
  673. foreach ($this->functions as $worker => $settings) {
  674. if (empty($settings["dedicated_only"])) {
  675. $function_count[$worker] = $this->do_all_count;
  676. }
  677. }
  678. }
  679. /**
  680. * Next we loop the workers and ensure we have enough running
  681. * for each worker
  682. */
  683. foreach ($this->functions as $worker=>$config) {
  684. /**
  685. * If we don't have do_all workers, this won't be set, so we need
  686. * to init it here
  687. */
  688. if (empty($function_count[$worker])) {
  689. $function_count[$worker] = 0;
  690. }
  691. while ($function_count[$worker] < $config["count"]) {
  692. $this->start_worker($worker);
  693. $function_count[$worker]++;;
  694. /**
  695. * Don't start workers too fast. They can overwhelm the
  696. * gearmand server and lead to connection timeouts.
  697. */
  698. usleep(500000);
  699. }
  700. }
  701. /**
  702. * Set the last code check time to now since we just loaded all the code
  703. */
  704. $this->last_check_time = time();
  705. }
  706. protected function start_worker($worker="all") {
  707. static $all_workers;
  708. if (is_array($worker)) {
  709. $worker_list = $worker;
  710. } elseif ($worker == "all") {
  711. if (is_null($all_workers)) {
  712. $all_workers = array();
  713. foreach ($this->functions as $func=>$settings) {
  714. if (empty($settings["dedicated_only"])) {
  715. $all_workers[] = $func;
  716. }
  717. }
  718. }
  719. $worker_list = $all_workers;
  720. } else {
  721. $worker_list = array($worker);
  722. }
  723. $timeouts = array();
  724. $default_timeout = ((isset($this->config['timeout'])) ?
  725. (int) $this->config['timeout'] : null);
  726. // build up the list of worker timeouts
  727. foreach ($worker_list as $worker_name) {
  728. $timeouts[$worker_name] = ((isset($this->config['functions'][$worker_name]['timeout'])) ?
  729. (int) $this->config['functions'][$worker_name]['timeout'] : $default_timeout);
  730. }
  731. $pid = pcntl_fork();
  732. switch ($pid) {
  733. case 0:
  734. $this->isparent = false;
  735. $this->register_ticks(false);
  736. $this->pid = getmypid();
  737. if (count($worker_list) > 1) {
  738. // shuffle the list to avoid queue preference
  739. shuffle($worker_list);
  740. // sort the shuffled array by priority
  741. uasort($worker_list, array($this, "sort_priority"));
  742. }
  743. if ($this->worker_restart_splay > 0) {
  744. mt_srand($this->pid); // Since all child threads use the same seed, we need to reseed with the pid so that we get a new "random" number.
  745. $splay = mt_rand(0, $this->worker_restart_splay);
  746. $this->max_run_time += $splay;
  747. $this->log("Adjusted max run time to {$this->max_run_time} seconds", GearmanManager::LOG_LEVEL_DEBUG);
  748. }
  749. $this->start_lib_worker($worker_list, $timeouts);
  750. $this->log("Child exiting", GearmanManager::LOG_LEVEL_WORKER_INFO);
  751. exit();
  752. break;
  753. case -1:
  754. $this->log("Could not fork");
  755. $this->stop_work = true;
  756. $this->stop_children();
  757. break;
  758. default:
  759. // parent
  760. $this->log("Started child $pid (".implode(",", $worker_list).")", GearmanManager::LOG_LEVEL_PROC_INFO);
  761. $this->children[$pid] = array(
  762. "job" => $worker_list,
  763. "start_time" => time(),
  764. );
  765. }
  766. }
  767. /**
  768. * Sorts the function list by priority
  769. */
  770. private function sort_priority($a, $b) {
  771. $func_a = $this->functions[$a];
  772. $func_b = $this->functions[$b];
  773. if (!isset($func_a["priority"])) {
  774. $func_a["priority"] = 0;
  775. }
  776. if (!isset($func_b["priority"])) {
  777. $func_b["priority"] = 0;
  778. }
  779. if ($func_a["priority"] == $func_b["priority"]) {
  780. return 0;
  781. }
  782. return ($func_a["priority"] > $func_b["priority"]) ? -1 : 1;
  783. }
  784. /**
  785. * Stops all running children
  786. */
  787. protected function stop_children($signal=SIGTERM) {
  788. $this->log("Stopping children", GearmanManager::LOG_LEVEL_PROC_INFO);
  789. foreach ($this->children as $pid=>$child) {
  790. $this->log("Stopping child $pid (".implode(",", $child['job']).")", GearmanManager::LOG_LEVEL_PROC_INFO);
  791. posix_kill($pid, $signal);
  792. }
  793. }
  794. /**
  795. * Registers the process signal listeners
  796. */
  797. protected function register_ticks($parent=true) {
  798. if ($parent) {
  799. $this->log("Registering signals for parent", GearmanManager::LOG_LEVEL_DEBUG);
  800. pcntl_signal(SIGTERM, array($this, "signal"));
  801. pcntl_signal(SIGINT, array($this, "signal"));
  802. pcntl_signal(SIGUSR1, array($this, "signal"));
  803. pcntl_signal(SIGUSR2, array($this, "signal"));
  804. pcntl_signal(SIGCONT, array($this, "signal"));
  805. pcntl_signal(SIGHUP, array($this, "signal"));
  806. } else {
  807. $this->log("Registering signals for child", GearmanManager::LOG_LEVEL_DEBUG);
  808. $res = pcntl_signal(SIGTERM, array($this, "signal"));
  809. if (!$res) {
  810. exit();
  811. }
  812. }
  813. }
  814. /**
  815. * Handles signals
  816. */
  817. public function signal($signo) {
  818. static $term_count = 0;
  819. if (!$this->isparent) {
  820. $this->stop_work = true;
  821. } else {
  822. switch ($signo) {
  823. case SIGUSR1:
  824. $this->show_help("No worker files could be found");
  825. break;
  826. case SIGUSR2:
  827. $this->show_help("Error validating worker functions");
  828. break;
  829. case SIGCONT:
  830. $this->wait_for_signal = false;
  831. break;
  832. case SIGINT:
  833. case SIGTERM:
  834. $this->log("Shutting down...");
  835. $this->stop_work = true;
  836. $this->stop_time = time();
  837. $term_count++;
  838. if ($term_count < 5) {
  839. $this->stop_children();
  840. } else {
  841. $this->stop_children(SIGKILL);
  842. }
  843. break;
  844. case SIGHUP:
  845. $this->log("Restarting children", GearmanManager::LOG_LEVEL_PROC_INFO);
  846. if ($this->log_file) {
  847. $this->open_log_file();
  848. }
  849. $this->stop_children();
  850. break;
  851. default:
  852. // handle all other signals
  853. }
  854. }
  855. }
  856. /**
  857. * Opens the log file. If already open, closes it first.
  858. */
  859. protected function open_log_file() {
  860. if ($this->log_file) {
  861. if ($this->log_file_handle) {
  862. fclose($this->log_file_handle);
  863. }
  864. $this->log_file_handle = @fopen($this->config['log_file'], "a");
  865. if (!$this->log_file_handle) {
  866. $this->show_help("Could not open log file {$this->config['log_file']}");
  867. }
  868. }
  869. }
  870. /**
  871. * Logs data to disk or stdout
  872. */
  873. protected function log($message, $level=GearmanManager::LOG_LEVEL_INFO) {
  874. static $init = false;
  875. if ($level > $this->verbose) return;
  876. if ($this->log_syslog) {
  877. $this->syslog($message, $level);
  878. return;
  879. }
  880. if (!$init) {
  881. $init = true;
  882. if ($this->log_file_handle) {
  883. fwrite($this->log_file_handle, "Date PID Type Message\n");
  884. } else {
  885. echo "PID Type Message\n";
  886. }
  887. }
  888. $label = "";
  889. switch ($level) {
  890. case GearmanManager::LOG_LEVEL_INFO;
  891. $label = "INFO ";
  892. break;
  893. case GearmanManager::LOG_LEVEL_PROC_INFO:
  894. $label = "PROC ";
  895. break;
  896. case GearmanManager::LOG_LEVEL_WORKER_INFO:
  897. $label = "WORKER";
  898. break;
  899. case GearmanManager::LOG_LEVEL_DEBUG:
  900. $label = "DEBUG ";
  901. break;
  902. case GearmanManager::LOG_LEVEL_CRAZY:
  903. $label = "CRAZY ";
  904. break;
  905. }
  906. $log_pid = str_pad($this->pid, 5, " ", STR_PAD_LEFT);
  907. if ($this->log_file_handle) {
  908. list($ts, $ms) = explode(".", sprintf("%f", microtime(true)));
  909. $ds = date("Y-m-d H:i:s").".".str_pad($ms, 6, 0);
  910. $prefix = "[$ds] $log_pid $label";
  911. fwrite($this->log_file_handle, $prefix." ".str_replace("\n", "\n$prefix ", trim($message))."\n");
  912. } else {
  913. $prefix = "$log_pid $label";
  914. echo $prefix." ".str_replace("\n", "\n$prefix ", trim($message))."\n";
  915. }
  916. }
  917. /**
  918. * Logs data to syslog
  919. */
  920. protected function syslog($message, $level) {
  921. switch ($level) {
  922. case GearmanManager::LOG_LEVEL_INFO;
  923. case GearmanManager::LOG_LEVEL_PROC_INFO:
  924. case GearmanManager::LOG_LEVEL_WORKER_INFO:
  925. default:
  926. $priority = LOG_INFO;
  927. break;
  928. case GearmanManager::LOG_LEVEL_DEBUG:
  929. $priority = LOG_DEBUG;
  930. break;
  931. }
  932. if (!syslog($priority, $message)) {
  933. echo "Unable to write to syslog\n";
  934. }
  935. }
  936. /**
  937. * Function for logging the status of children. This simply logs the status
  938. * of the process. Wrapper classes can make use of this to do logging as
  939. * appropriate for individual environments.
  940. *
  941. * @param int $pid PID of the child process
  942. * @param array $jobs Array of jobs the child is/was running
  943. * @param string $status Status of the child process.
  944. * One of killed, exited or non-zero integer
  945. *
  946. * @return void
  947. */
  948. protected function child_status_monitor($pid, $jobs, $status) {
  949. switch ($status) {
  950. case "killed":
  951. $message = "Child $pid has been running too long. Forcibly killing process. (".implode(",", $jobs).")";
  952. break;
  953. case "exited":
  954. $message = "Child $pid exited cleanly. (".implode(",", $jobs).")";
  955. break;
  956. default:
  957. $message = "Child $pid died unexpectedly with exit code $status. (".implode(",", $jobs).")";
  958. break;
  959. }
  960. $this->log($message, GearmanManager::LOG_LEVEL_PROC_INFO);
  961. }
  962. /**
  963. * Shows the scripts help info with optional error message
  964. */
  965. protected function show_help($msg = "") {
  966. if ($msg) {
  967. echo "ERROR:\n";
  968. echo " ".wordwrap($msg, 72, "\n ")."\n\n";
  969. }
  970. echo "Gearman worker manager script\n\n";
  971. echo "USAGE:\n";
  972. echo " # ".basename(__FILE__)." -h | -c CONFIG [-v] [-l LOG_FILE] [-d] [-v] [-a] [-P PID_FILE]\n\n";
  973. echo "OPTIONS:\n";
  974. echo " -a Automatically check for new worker code\n";
  975. echo " -c CONFIG Worker configuration file\n";
  976. echo " -d Daemon, detach and run in the background\n";
  977. echo " -D NUMBER Start NUMBER workers that do all jobs\n";
  978. echo " -h HOST[:PORT] Connect to HOST and optional PORT\n";
  979. echo " -H Shows this help\n";
  980. echo " -l LOG_FILE Log output to LOG_FILE or use keyword 'syslog' for syslog support\n";
  981. echo " -p PREFIX Optional prefix for functions/classes of PECL workers. PEAR requires a constant be defined in code.\n";
  982. echo " -P PID_FILE File to write process ID out to\n";
  983. echo " -u USERNAME Run workers as USERNAME\n";
  984. echo " -v Increase verbosity level by one\n";
  985. echo " -w DIR Directory where workers are located, defaults to ./workers. If you are using PECL, you can provide multiple directories separated by a comma.\n";
  986. echo " -r NUMBER Maximum job iterations per worker\n";
  987. echo " -t SECONDS Maximum number of seconds gearmand server should wait for a worker to complete work before timing out and reissuing work to another worker.\n";
  988. echo " -x SECONDS Maximum seconds for a worker to live\n";
  989. echo " -Z Parse the command line and config file then dump it to the screen and exit.\n";
  990. echo "\n";
  991. exit();
  992. }
  993. }