PageRenderTime 41ms CodeModel.GetById 20ms RepoModel.GetById 0ms app.codeStats 0ms

/core/CliMulti.php

https://github.com/CodeYellowBV/piwik
PHP | 280 lines | 172 code | 52 blank | 56 comment | 17 complexity | 8955a43ea3c7989a62f4004e558c1fed MD5 | raw file
Possible License(s): LGPL-3.0, JSON, MIT, GPL-3.0, LGPL-2.1, GPL-2.0, AGPL-1.0, BSD-2-Clause, BSD-3-Clause
  1. <?php
  2. /**
  3. * Piwik - free/libre analytics platform
  4. *
  5. * @link http://piwik.org
  6. * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  7. */
  8. namespace Piwik;
  9. use Piwik\CliMulti\CliPhp;
  10. use Piwik\CliMulti\Output;
  11. use Piwik\CliMulti\Process;
  12. /**
  13. * Class CliMulti.
  14. */
  15. class CliMulti {
  16. /**
  17. * If set to true or false it will overwrite whether async is supported or not.
  18. *
  19. * @var null|bool
  20. */
  21. public $supportsAsync = null;
  22. /**
  23. * @var \Piwik\CliMulti\Process[]
  24. */
  25. private $processes = array();
  26. /**
  27. * If set it will issue at most concurrentProcessesLimit requests
  28. * @var int
  29. */
  30. private $concurrentProcessesLimit = null;
  31. /**
  32. * @var \Piwik\CliMulti\Output[]
  33. */
  34. private $outputs = array();
  35. private $acceptInvalidSSLCertificate = false;
  36. public function __construct()
  37. {
  38. $this->supportsAsync = $this->supportsAsync();
  39. }
  40. /**
  41. * It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
  42. * If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
  43. *
  44. * @param string[] $piwikUrls An array of urls, for instance:
  45. * array('http://www.example.com/piwik?module=API...')
  46. * @return array The response of each URL in the same order as the URLs. The array can contain null values in case
  47. * there was a problem with a request, for instance if the process died unexpected.
  48. */
  49. public function request(array $piwikUrls)
  50. {
  51. $chunks = array($piwikUrls);
  52. if($this->concurrentProcessesLimit) {
  53. $chunks = array_chunk( $piwikUrls, $this->concurrentProcessesLimit);
  54. }
  55. $results = array();
  56. foreach($chunks as $urlsChunk) {
  57. $results = array_merge($results, $this->requestUrls($urlsChunk));
  58. }
  59. return $results;
  60. }
  61. /**
  62. * Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
  63. * our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
  64. * @param $acceptInvalidSSLCertificate
  65. */
  66. public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
  67. {
  68. $this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
  69. }
  70. /**
  71. * @param $limit int Maximum count of requests to issue in parallel
  72. */
  73. public function setConcurrentProcessesLimit($limit)
  74. {
  75. $this->concurrentProcessesLimit = $limit;
  76. }
  77. private function start($piwikUrls)
  78. {
  79. foreach ($piwikUrls as $index => $url) {
  80. $cmdId = $this->generateCommandId($url) . $index;
  81. $output = new Output($cmdId);
  82. if ($this->supportsAsync) {
  83. $this->executeAsyncCli($url, $output, $cmdId);
  84. } else {
  85. $this->executeNotAsyncHttp($url, $output);
  86. }
  87. $this->outputs[] = $output;
  88. }
  89. }
  90. private function buildCommand($hostname, $query, $outputFile)
  91. {
  92. $bin = $this->findPhpBinary();
  93. return sprintf('%s -q %s/console climulti:request --piwik-domain=%s %s > %s 2>&1 &',
  94. $bin, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), escapeshellarg($query), $outputFile);
  95. }
  96. private function getResponse()
  97. {
  98. $response = array();
  99. foreach ($this->outputs as $output) {
  100. $response[] = $output->get();
  101. }
  102. return $response;
  103. }
  104. private function hasFinished()
  105. {
  106. foreach ($this->processes as $index => $process) {
  107. $hasStarted = $process->hasStarted();
  108. if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
  109. // if process was created more than 8 seconds ago but still not started there must be something wrong.
  110. // ==> declare the process as finished
  111. $process->finishProcess();
  112. continue;
  113. } elseif (!$hasStarted) {
  114. return false;
  115. }
  116. if ($process->isRunning()) {
  117. return false;
  118. }
  119. if ($process->hasFinished()) {
  120. // prevent from checking this process over and over again
  121. unset($this->processes[$index]);
  122. }
  123. }
  124. return true;
  125. }
  126. private function generateCommandId($command)
  127. {
  128. return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
  129. }
  130. /**
  131. * What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
  132. * and how to send a process into background in start()
  133. */
  134. public function supportsAsync()
  135. {
  136. return Process::isSupported() && $this->findPhpBinary();
  137. }
  138. private function findPhpBinary()
  139. {
  140. $cliPhp = new CliPhp();
  141. return $cliPhp->findPhpBinary();
  142. }
  143. private function cleanup()
  144. {
  145. foreach ($this->processes as $pid) {
  146. $pid->finishProcess();
  147. }
  148. foreach ($this->outputs as $output) {
  149. $output->destroy();
  150. }
  151. $this->processes = array();
  152. $this->outputs = array();
  153. }
  154. /**
  155. * Remove files older than one week. They should be cleaned up automatically after each request but for whatever
  156. * reason there can be always some files left.
  157. */
  158. public static function cleanupNotRemovedFiles()
  159. {
  160. $timeOneWeekAgo = strtotime('-1 week');
  161. $files = _glob(self::getTmpPath() . '/*');
  162. if(empty($files)) {
  163. return;
  164. }
  165. foreach ($files as $file) {
  166. $timeLastModified = filemtime($file);
  167. if ($timeOneWeekAgo > $timeLastModified) {
  168. unlink($file);
  169. }
  170. }
  171. }
  172. public static function getTmpPath()
  173. {
  174. $dir = PIWIK_INCLUDE_PATH . '/tmp/climulti';
  175. return SettingsPiwik::rewriteTmpPathWithInstanceId($dir);
  176. }
  177. private function executeAsyncCli($url, Output $output, $cmdId)
  178. {
  179. $this->processes[] = new Process($cmdId);
  180. $url = $this->appendTestmodeParamToUrlIfNeeded($url);
  181. $query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
  182. $hostname = UrlHelper::getHostFromUrl($url);
  183. $command = $this->buildCommand($hostname, $query, $output->getPathToFile());
  184. Log::debug($command);
  185. shell_exec($command);
  186. }
  187. private function executeNotAsyncHttp($url, Output $output)
  188. {
  189. try {
  190. Log::debug("Execute HTTP API request: " . $url);
  191. $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
  192. $output->write($response);
  193. } catch (\Exception $e) {
  194. $message = "Got invalid response from API request: $url. ";
  195. if (empty($response)) {
  196. $message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
  197. } else {
  198. $message .= "Response was '" . $e->getMessage() . "'";
  199. }
  200. $output->write($message);
  201. }
  202. }
  203. private function appendTestmodeParamToUrlIfNeeded($url)
  204. {
  205. $isTestMode = $url && false !== strpos($url, 'tests/PHPUnit/proxy');
  206. if ($isTestMode && false === strpos($url, '?')) {
  207. $url .= "?testmode=1";
  208. } elseif ($isTestMode) {
  209. $url .= "&testmode=1";
  210. }
  211. return $url;
  212. }
  213. /**
  214. * @param array $piwikUrls
  215. * @return array
  216. */
  217. private function requestUrls(array $piwikUrls)
  218. {
  219. $this->start($piwikUrls);
  220. do {
  221. usleep(100000); // 100 * 1000 = 100ms
  222. } while (!$this->hasFinished());
  223. $results = $this->getResponse($piwikUrls);
  224. $this->cleanup();
  225. self::cleanupNotRemovedFiles();
  226. return $results;
  227. }
  228. }