/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
PHP | 611 lines | 487 code | 100 blank | 24 comment | 61 complexity | 1be5f88f442ec96a273e05b2f58ebacf MD5 | raw file
Possible License(s): JSON, MPL-2.0-no-copyleft-exception, Apache-2.0, BSD-3-Clause, LGPL-2.0, MIT, LGPL-2.1, LGPL-3.0
- <?php
- abstract class PhabricatorDaemonManagementWorkflow
- extends PhabricatorManagementWorkflow {
- private $runDaemonsAsUser = null;
- final protected function loadAvailableDaemonClasses() {
- return id(new PhutilSymbolLoader())
- ->setAncestorClass('PhutilDaemon')
- ->setConcreteOnly(true)
- ->selectSymbolsWithoutLoading();
- }
- final protected function getLogDirectory() {
- $path = PhabricatorEnv::getEnvConfig('phd.log-directory');
- return $this->getControlDirectory($path);
- }
- private function getControlDirectory($path) {
- if (!Filesystem::pathExists($path)) {
- list($err) = exec_manual('mkdir -p %s', $path);
- if ($err) {
- throw new Exception(
- pht(
- "%s requires the directory '%s' to exist, but it does not exist ".
- "and could not be created. Create this directory or update ".
- "'%s' in your configuration to point to an existing ".
- "directory.",
- 'phd',
- $path,
- 'phd.log-directory'));
- }
- }
- return $path;
- }
- private function findDaemonClass($substring) {
- $symbols = $this->loadAvailableDaemonClasses();
- $symbols = ipull($symbols, 'name');
- $match = array();
- foreach ($symbols as $symbol) {
- if (stripos($symbol, $substring) !== false) {
- if (strtolower($symbol) == strtolower($substring)) {
- $match = array($symbol);
- break;
- } else {
- $match[] = $symbol;
- }
- }
- }
- if (count($match) == 0) {
- throw new PhutilArgumentUsageException(
- pht(
- "No daemons match '%s'! Use '%s' for a list of available daemons.",
- $substring,
- 'phd list'));
- } else if (count($match) > 1) {
- throw new PhutilArgumentUsageException(
- pht(
- "Specify a daemon unambiguously. Multiple daemons match '%s': %s.",
- $substring,
- implode(', ', $match)));
- }
- return head($match);
- }
- final protected function launchDaemons(
- array $daemons,
- $debug,
- $run_as_current_user = false) {
- // Convert any shorthand classnames like "taskmaster" into proper class
- // names.
- foreach ($daemons as $key => $daemon) {
- $class = $this->findDaemonClass($daemon['class']);
- $daemons[$key]['class'] = $class;
- }
- $console = PhutilConsole::getConsole();
- if (!$run_as_current_user) {
- // Check if the script is started as the correct user
- $phd_user = PhabricatorEnv::getEnvConfig('phd.user');
- $current_user = posix_getpwuid(posix_geteuid());
- $current_user = $current_user['name'];
- if ($phd_user && $phd_user != $current_user) {
- if ($debug) {
- throw new PhutilArgumentUsageException(
- pht(
- "You are trying to run a daemon as a nonstandard user, ".
- "and `%s` was not able to `%s` to the correct user. \n".
- 'Phabricator is configured to run daemons as "%s", '.
- 'but the current user is "%s". '."\n".
- 'Use `%s` to run as a different user, pass `%s` to ignore this '.
- 'warning, or edit `%s` to change the configuration.',
- 'phd',
- 'sudo',
- $phd_user,
- $current_user,
- 'sudo',
- '--as-current-user',
- 'phd.user'));
- } else {
- $this->runDaemonsAsUser = $phd_user;
- $console->writeOut(pht('Starting daemons as %s', $phd_user)."\n");
- }
- }
- }
- $this->printLaunchingDaemons($daemons, $debug);
- $trace = PhutilArgumentParser::isTraceModeEnabled();
- $flags = array();
- if ($trace || PhabricatorEnv::getEnvConfig('phd.trace')) {
- $flags[] = '--trace';
- }
- if ($debug || PhabricatorEnv::getEnvConfig('phd.verbose')) {
- $flags[] = '--verbose';
- }
- $instance = $this->getInstance();
- if ($instance) {
- $flags[] = '-l';
- $flags[] = $instance;
- }
- $config = array();
- if (!$debug) {
- $config['daemonize'] = true;
- }
- if (!$debug) {
- $config['log'] = $this->getLogDirectory().'/daemons.log';
- }
- $config['daemons'] = $daemons;
- $command = csprintf('./phd-daemon %Ls', $flags);
- $phabricator_root = dirname(phutil_get_library_root('phabricator'));
- $daemon_script_dir = $phabricator_root.'/scripts/daemon/';
- if ($debug) {
- // Don't terminate when the user sends ^C; it will be sent to the
- // subprocess which will terminate normally.
- pcntl_signal(
- SIGINT,
- array(__CLASS__, 'ignoreSignal'));
- echo "\n phabricator/scripts/daemon/ \$ {$command}\n\n";
- $tempfile = new TempFile('daemon.config');
- Filesystem::writeFile($tempfile, json_encode($config));
- phutil_passthru(
- '(cd %s && exec %C < %s)',
- $daemon_script_dir,
- $command,
- $tempfile);
- } else {
- try {
- $this->executeDaemonLaunchCommand(
- $command,
- $daemon_script_dir,
- $config,
- $this->runDaemonsAsUser);
- } catch (Exception $ex) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Daemons are configured to run as user "%s" in configuration '.
- 'option `%s`, but the current user is "%s" and `phd` was unable '.
- 'to switch to the correct user with `sudo`. Command output:'.
- "\n\n".
- '%s',
- $phd_user,
- 'phd.user',
- $current_user,
- $ex->getMessage()));
- }
- }
- }
- private function executeDaemonLaunchCommand(
- $command,
- $daemon_script_dir,
- array $config,
- $run_as_user = null) {
- $is_sudo = false;
- if ($run_as_user) {
- // If anything else besides sudo should be
- // supported then insert it here (runuser, su, ...)
- $command = csprintf(
- 'sudo -En -u %s -- %C',
- $run_as_user,
- $command);
- $is_sudo = true;
- }
- $future = new ExecFuture('exec %C', $command);
- // Play games to keep 'ps' looking reasonable.
- $future->setCWD($daemon_script_dir);
- $future->write(json_encode($config));
- list($stdout, $stderr) = $future->resolvex();
- if ($is_sudo) {
- // On OSX, `sudo -n` exits 0 when the user does not have permission to
- // switch accounts without a password. This is not consistent with
- // sudo on Linux, and seems buggy/broken. Check for this by string
- // matching the output.
- if (preg_match('/sudo: a password is required/', $stderr)) {
- throw new Exception(
- pht(
- '%s exited with a zero exit code, but emitted output '.
- 'consistent with failure under OSX.',
- 'sudo'));
- }
- }
- }
- public static function ignoreSignal($signo) {
- return;
- }
- public static function requireExtensions() {
- self::mustHaveExtension('pcntl');
- self::mustHaveExtension('posix');
- }
- private static function mustHaveExtension($ext) {
- if (!extension_loaded($ext)) {
- echo pht(
- "ERROR: The PHP extension '%s' is not installed. You must ".
- "install it to run daemons on this machine.\n",
- $ext);
- exit(1);
- }
- $extension = new ReflectionExtension($ext);
- foreach ($extension->getFunctions() as $function) {
- $function = $function->name;
- if (!function_exists($function)) {
- echo pht(
- "ERROR: The PHP function %s is disabled. You must ".
- "enable it to run daemons on this machine.\n",
- $function.'()');
- exit(1);
- }
- }
- }
- /* -( Commands )----------------------------------------------------------- */
- final protected function executeStartCommand(array $options) {
- PhutilTypeSpec::checkMap(
- $options,
- array(
- 'keep-leases' => 'optional bool',
- 'force' => 'optional bool',
- 'reserve' => 'optional float',
- ));
- $console = PhutilConsole::getConsole();
- if (!idx($options, 'force')) {
- $process_refs = $this->getOverseerProcessRefs();
- if ($process_refs) {
- $this->logWarn(
- pht('RUNNING DAEMONS'),
- pht('Daemons are already running:'));
- fprintf(STDERR, '%s', "\n");
- foreach ($process_refs as $process_ref) {
- fprintf(
- STDERR,
- '%s',
- tsprintf(
- " %s %s\n",
- $process_ref->getPID(),
- $process_ref->getCommand()));
- }
- fprintf(STDERR, '%s', "\n");
- $this->logFail(
- pht('RUNNING DAEMONS'),
- pht(
- 'Use "phd stop" to stop daemons, "phd restart" to restart '.
- 'daemons, or "phd start --force" to ignore running processes.'));
- exit(1);
- }
- }
- if (idx($options, 'keep-leases')) {
- $console->writeErr("%s\n", pht('Not touching active task queue leases.'));
- } else {
- $console->writeErr("%s\n", pht('Freeing active task leases...'));
- $count = $this->freeActiveLeases();
- $console->writeErr(
- "%s\n",
- pht('Freed %s task lease(s).', new PhutilNumber($count)));
- }
- $daemons = array(
- array(
- 'class' => 'PhabricatorRepositoryPullLocalDaemon',
- 'label' => 'pull',
- ),
- array(
- 'class' => 'PhabricatorTriggerDaemon',
- 'label' => 'trigger',
- ),
- array(
- 'class' => 'PhabricatorFactDaemon',
- 'label' => 'fact',
- ),
- array(
- 'class' => 'PhabricatorTaskmasterDaemon',
- 'label' => 'task',
- 'pool' => PhabricatorEnv::getEnvConfig('phd.taskmasters'),
- 'reserve' => idx($options, 'reserve', 0),
- ),
- );
- $this->launchDaemons($daemons, $is_debug = false);
- $console->writeErr("%s\n", pht('Done.'));
- return 0;
- }
- final protected function executeStopCommand(array $options) {
- $grace_period = idx($options, 'graceful', 15);
- $force = idx($options, 'force');
- $query = id(new PhutilProcessQuery())
- ->withIsOverseer(true);
- $instance = $this->getInstance();
- if ($instance !== null && !$force) {
- $query->withInstances(array($instance));
- }
- try {
- $process_refs = $query->execute();
- } catch (Exception $ex) {
- // See T13321. If this fails for some reason, just continue for now so
- // that daemon management still works. In the long run, we don't expect
- // this to fail, but I don't want to break this workflow while we iron
- // bugs out.
- // See T12827. Particularly, this is likely to fail on Solaris.
- phlog($ex);
- $process_refs = array();
- }
- if (!$process_refs) {
- if ($instance !== null && !$force) {
- $this->logInfo(
- pht('NO DAEMONS'),
- pht(
- 'There are no running daemons for the current instance ("%s"). '.
- 'Use "--force" to stop daemons for all instances.',
- $instance));
- } else {
- $this->logInfo(
- pht('NO DAEMONS'),
- pht('There are no running daemons.'));
- }
- return 0;
- }
- $process_refs = mpull($process_refs, null, 'getPID');
- $stop_pids = array_keys($process_refs);
- $live_pids = $this->sendStopSignals($stop_pids, $grace_period);
- $stop_pids = array_fuse($stop_pids);
- $live_pids = array_fuse($live_pids);
- $dead_pids = array_diff_key($stop_pids, $live_pids);
- foreach ($dead_pids as $dead_pid) {
- $dead_ref = $process_refs[$dead_pid];
- $this->logOkay(
- pht('STOP'),
- pht(
- 'Stopped PID %d ("%s")',
- $dead_pid,
- $dead_ref->getCommand()));
- }
- foreach ($live_pids as $live_pid) {
- $live_ref = $process_refs[$live_pid];
- $this->logFail(
- pht('SURVIVED'),
- pht(
- 'Unable to stop PID %d ("%s").',
- $live_pid,
- $live_ref->getCommand()));
- }
- if ($live_pids) {
- $this->logWarn(
- pht('SURVIVORS'),
- pht(
- 'Unable to stop all daemon processes. You may need to run this '.
- 'command as root with "sudo".'));
- }
- return 0;
- }
- final protected function executeReloadCommand(array $pids) {
- $process_refs = $this->getOverseerProcessRefs();
- if (!$process_refs) {
- $this->logInfo(
- pht('NO DAEMONS'),
- pht('There are no running daemon processes to reload.'));
- return 0;
- }
- foreach ($process_refs as $process_ref) {
- $pid = $process_ref->getPID();
- $this->logInfo(
- pht('RELOAD'),
- pht('Reloading process %d...', $pid));
- posix_kill($pid, SIGHUP);
- }
- return 0;
- }
- private function sendStopSignals($pids, $grace_period) {
- // If we're doing a graceful shutdown, try SIGINT first.
- if ($grace_period) {
- $pids = $this->sendSignal($pids, SIGINT, $grace_period);
- }
- // If we still have daemons, SIGTERM them.
- if ($pids) {
- $pids = $this->sendSignal($pids, SIGTERM, 15);
- }
- // If the overseer is still alive, SIGKILL it.
- if ($pids) {
- $pids = $this->sendSignal($pids, SIGKILL, 0);
- }
- return $pids;
- }
- private function sendSignal(array $pids, $signo, $wait) {
- $console = PhutilConsole::getConsole();
- $pids = array_fuse($pids);
- foreach ($pids as $key => $pid) {
- if (!$pid) {
- // NOTE: We must have a PID to signal a daemon, since sending a signal
- // to PID 0 kills this process.
- unset($pids[$key]);
- continue;
- }
- switch ($signo) {
- case SIGINT:
- $message = pht('Interrupting process %d...', $pid);
- break;
- case SIGTERM:
- $message = pht('Terminating process %d...', $pid);
- break;
- case SIGKILL:
- $message = pht('Killing process %d...', $pid);
- break;
- }
- $console->writeOut("%s\n", $message);
- posix_kill($pid, $signo);
- }
- if ($wait) {
- $start = PhabricatorTime::getNow();
- do {
- foreach ($pids as $key => $pid) {
- if (!PhabricatorDaemonReference::isProcessRunning($pid)) {
- $console->writeOut(pht('Process %d exited.', $pid)."\n");
- unset($pids[$key]);
- }
- }
- if (empty($pids)) {
- break;
- }
- usleep(100000);
- } while (PhabricatorTime::getNow() < $start + $wait);
- }
- return $pids;
- }
- private function freeActiveLeases() {
- $task_table = id(new PhabricatorWorkerActiveTask());
- $conn_w = $task_table->establishConnection('w');
- queryfx(
- $conn_w,
- 'UPDATE %T SET leaseExpires = UNIX_TIMESTAMP()
- WHERE leaseExpires > UNIX_TIMESTAMP()',
- $task_table->getTableName());
- return $conn_w->getAffectedRows();
- }
- private function printLaunchingDaemons(array $daemons, $debug) {
- $console = PhutilConsole::getConsole();
- if ($debug) {
- $console->writeOut(pht('Launching daemons (in debug mode):'));
- } else {
- $console->writeOut(pht('Launching daemons:'));
- }
- $log_dir = $this->getLogDirectory().'/daemons.log';
- $console->writeOut(
- "\n%s\n\n",
- pht('(Logs will appear in "%s".)', $log_dir));
- foreach ($daemons as $daemon) {
- $pool_size = pht('(Pool: %s)', idx($daemon, 'pool', 1));
- $console->writeOut(
- " %s %s\n",
- $pool_size,
- $daemon['class'],
- implode(' ', idx($daemon, 'argv', array())));
- }
- $console->writeOut("\n");
- }
- protected function getAutoscaleReserveArgument() {
- return array(
- 'name' => 'autoscale-reserve',
- 'param' => 'ratio',
- 'help' => pht(
- 'Specify a proportion of machine memory which must be free '.
- 'before autoscale pools will grow. For example, a value of 0.25 '.
- 'means that pools will not grow unless the machine has at least '.
- '25%%%% of its RAM free.'),
- );
- }
- private function selectDaemonPIDs(array $daemons, array $pids) {
- $console = PhutilConsole::getConsole();
- $running_pids = array_fuse(mpull($daemons, 'getPID'));
- if (!$pids) {
- $select_pids = $running_pids;
- } else {
- // We were given a PID or set of PIDs to kill.
- $select_pids = array();
- foreach ($pids as $key => $pid) {
- if (!preg_match('/^\d+$/', $pid)) {
- $console->writeErr(pht("PID '%s' is not a valid PID.", $pid)."\n");
- continue;
- } else if (empty($running_pids[$pid])) {
- $console->writeErr(
- "%s\n",
- pht(
- 'PID "%d" is not a known Phabricator daemon PID.',
- $pid));
- continue;
- } else {
- $select_pids[$pid] = $pid;
- }
- }
- }
- return $select_pids;
- }
- protected function getOverseerProcessRefs() {
- $query = id(new PhutilProcessQuery())
- ->withIsOverseer(true);
- $instance = PhabricatorEnv::getEnvConfig('cluster.instance');
- if ($instance !== null) {
- $query->withInstances(array($instance));
- }
- return $query->execute();
- }
- protected function getInstance() {
- return PhabricatorEnv::getEnvConfig('cluster.instance');
- }
- }