PageRenderTime 44ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/src/Packagist/WebBundle/Package/Dumper.php

https://github.com/hostnet/packagist
PHP | 453 lines | 340 code | 39 blank | 74 comment | 21 complexity | 038d1346fc6462fc9338c5838bb2385b MD5 | raw file
  1. <?php
  2. /*
  3. * This file is part of Packagist.
  4. *
  5. * (c) Jordi Boggiano <j.boggiano@seld.be>
  6. * Nils Adermann <naderman@naderman.de>
  7. *
  8. * For the full copyright and license information, please view the LICENSE
  9. * file that was distributed with this source code.
  10. */
  11. namespace Packagist\WebBundle\Package;
  12. use Symfony\Component\Filesystem\Filesystem;
  13. use Composer\Util\Filesystem as ComposerFilesystem;
  14. use Symfony\Bridge\Doctrine\RegistryInterface;
  15. use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
  16. use Symfony\Component\Finder\Finder;
  17. use Packagist\WebBundle\Entity\Version;
  18. /**
  19. * @author Jordi Boggiano <j.boggiano@seld.be>
  20. */
  21. class Dumper
  22. {
  23. /**
  24. * Doctrine
  25. * @var RegistryInterface
  26. */
  27. protected $doctrine;
  28. /**
  29. * @var Filesystem
  30. */
  31. protected $fs;
  32. /**
  33. * @var ComposerFilesystem
  34. */
  35. protected $cfs;
  36. /**
  37. * @var string
  38. */
  39. protected $webDir;
  40. /**
  41. * @var string
  42. */
  43. protected $buildDir;
  44. /**
  45. * @var UrlGeneratorInterface
  46. */
  47. protected $router;
  48. /**
  49. * Data cache
  50. * @var array
  51. */
  52. private $rootFile;
  53. /**
  54. * Data cache
  55. * @var array
  56. */
  57. private $listings = array();
  58. /**
  59. * Data cache
  60. * @var array
  61. */
  62. private $individualFiles = array();
  63. /**
  64. * Modified times of individual files
  65. * @var array
  66. */
  67. private $individualFilesMtime = array();
  68. /**
  69. * Constructor
  70. *
  71. * @param RegistryInterface $doctrine
  72. * @param Filesystem $filesystem
  73. * @param UrlGeneratorInterface $router
  74. * @param string $webDir web root
  75. * @param string $cacheDir cache dir
  76. */
  77. public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, $webDir, $cacheDir)
  78. {
  79. $this->doctrine = $doctrine;
  80. $this->fs = $filesystem;
  81. $this->cfs = new ComposerFilesystem;
  82. $this->router = $router;
  83. $this->webDir = realpath($webDir);
  84. $this->buildDir = $cacheDir . '/composer-packages-build';
  85. }
  86. /**
  87. * Dump a set of packages to the web root
  88. *
  89. * @param array $packageIds
  90. * @param Boolean $force
  91. * @param Boolean $verbose
  92. */
  93. public function dump(array $packageIds, $force = false, $verbose = false)
  94. {
  95. $cleanUpOldFiles = date('i') == 0;
  96. // prepare build dir
  97. $webDir = $this->webDir;
  98. $buildDir = $this->buildDir;
  99. $retries = 5;
  100. do {
  101. if (!$this->cfs->removeDirectory($buildDir)) {
  102. usleep(200);
  103. }
  104. clearstatcache();
  105. } while (is_dir($buildDir) && $retries--);
  106. if (is_dir($buildDir)) {
  107. echo 'Could not remove the build dir entirely, aborting';
  108. return false;
  109. }
  110. $this->fs->mkdir($buildDir);
  111. $this->fs->mkdir($webDir.'/p/');
  112. if (!$force) {
  113. if ($verbose) {
  114. echo 'Copying existing files'.PHP_EOL;
  115. }
  116. exec('cp -rpf '.escapeshellarg($webDir.'/p').' '.escapeshellarg($buildDir.'/p'), $output, $exit);
  117. if (0 !== $exit) {
  118. $this->fs->mirror($webDir.'/p/', $buildDir.'/p/', null, array('override' => true));
  119. }
  120. }
  121. $modifiedIndividualFiles = array();
  122. $total = count($packageIds);
  123. $current = 0;
  124. $step = 50;
  125. while ($packageIds) {
  126. $dumpTime = new \DateTime;
  127. $packages = $this->doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($packageIds, 0, $step));
  128. if ($verbose) {
  129. echo '['.sprintf('%'.strlen($total).'d', $current).'/'.$total.'] Processing '.$step.' packages'.PHP_EOL;
  130. }
  131. $current += $step;
  132. // prepare packages in memory
  133. foreach ($packages as $package) {
  134. $affectedFiles = array();
  135. $name = strtolower($package->getName());
  136. // clean up versions in individual files
  137. if (file_exists($buildDir.'/p/'.$name.'.files')) {
  138. $files = json_decode(file_get_contents($buildDir.'/p/'.$name.'.files'));
  139. foreach ($files as $file) {
  140. $key = $this->getIndividualFileKey($buildDir.'/'.$file);
  141. $this->loadIndividualFile($buildDir.'/'.$file, $key);
  142. if (isset($this->individualFiles[$key]['packages'][$name])) {
  143. unset($this->individualFiles[$key]['packages'][$name]);
  144. $modifiedIndividualFiles[$key] = true;
  145. }
  146. }
  147. }
  148. // (re)write versions in individual files
  149. foreach ($package->getVersions() as $version) {
  150. foreach (array_slice($version->getNames(), 0, 150) as $versionName) {
  151. if (!preg_match('{^[A-Za-z0-9_-][A-Za-z0-9_.-]*/[A-Za-z0-9_-][A-Za-z0-9_.-]*$}', $versionName) || strpos($versionName, '..')) {
  152. continue;
  153. }
  154. $file = $buildDir.'/p/'.$versionName.'.json';
  155. $key = $this->getIndividualFileKey($file);
  156. $this->dumpVersionToIndividualFile($version, $file, $key);
  157. $modifiedIndividualFiles[$key] = true;
  158. $affectedFiles[$key] = true;
  159. }
  160. }
  161. // store affected files to clean up properly in the next update
  162. $this->fs->mkdir(dirname($buildDir.'/p/'.$name));
  163. file_put_contents($buildDir.'/p/'.$name.'.files', json_encode(array_keys($affectedFiles)));
  164. $modifiedIndividualFiles['p/'.$name.'.files'] = true;
  165. $package->setDumpedAt($dumpTime);
  166. }
  167. // update dump dates
  168. $this->doctrine->getManager()->flush();
  169. $this->doctrine->getManager()->clear();
  170. unset($packages);
  171. if ($current % 250 === 0 || !$packageIds) {
  172. if ($verbose) {
  173. echo 'Dumping individual files'.PHP_EOL;
  174. }
  175. // dump individual files to build dir
  176. foreach ($this->individualFiles as $file => $dummy) {
  177. $this->dumpIndividualFile($buildDir.'/'.$file, $file);
  178. // write the hashed provider file
  179. $hash = hash_file('sha256', $buildDir.'/'.$file);
  180. $hashedFile = substr($buildDir.'/'.$file, 0, -5) . '$' . $hash . '.json';
  181. copy($buildDir.'/'.$file, $hashedFile);
  182. }
  183. $this->individualFiles = array();
  184. }
  185. }
  186. // prepare individual files listings
  187. if ($verbose) {
  188. echo 'Preparing individual files listings'.PHP_EOL;
  189. }
  190. $safeFiles = array();
  191. $individualHashedListings = array();
  192. $finder = Finder::create()->files()->ignoreVCS(true)->name('*.json')->in($buildDir.'/p/')->depth('1');
  193. foreach ($finder as $file) {
  194. // skipped hashed files
  195. if (strpos($file, '$')) {
  196. continue;
  197. }
  198. $key = $this->getIndividualFileKey(strtr($file, '\\', '/'));
  199. if ($force && !isset($modifiedIndividualFiles[$key])) {
  200. continue;
  201. }
  202. // add hashed provider to listing
  203. $listing = 'p/'.$this->getTargetListing($file);
  204. $key = substr($key, 2, -5);
  205. $hash = hash_file('sha256', $file);
  206. $safeFiles[] = 'p/'.$key.'$'.$hash.'.json';
  207. $this->listings[$listing]['providers'][$key] = array('sha256' => $hash);
  208. $individualHashedListings[$listing] = true;
  209. }
  210. // prepare root file
  211. $rootFile = $buildDir.'/p/packages.json';
  212. $this->rootFile = array('packages' => array());
  213. $url = $this->router->generate('track_download', array('name' => 'VND/PKG'));
  214. $this->rootFile['notify'] = str_replace('VND/PKG', '%package%', $url);
  215. $this->rootFile['notify-batch'] = $this->router->generate('track_download_batch');
  216. $this->rootFile['providers-url'] = $this->router->generate('home') . 'p/%package%$%hash%.json';
  217. $this->rootFile['search'] = $this->router->generate('search', array('_format' => 'json')) . '?q=%query%';
  218. if ($verbose) {
  219. echo 'Dumping individual listings'.PHP_EOL;
  220. }
  221. // dump listings to build dir
  222. foreach ($individualHashedListings as $listing => $dummy) {
  223. $this->dumpListing($buildDir.'/'.$listing);
  224. $hash = hash_file('sha256', $buildDir.'/'.$listing);
  225. $hashedListing = substr($listing, 0, -5) . '$' . $hash . '.json';
  226. rename($buildDir.'/'.$listing, $buildDir.'/'.$hashedListing);
  227. $this->rootFile['provider-includes'][str_replace($hash, '%hash%', $hashedListing)] = array('sha256' => $hash);
  228. $safeFiles[] = $hashedListing;
  229. }
  230. if ($verbose) {
  231. echo 'Dumping root'.PHP_EOL;
  232. }
  233. // sort & dump root file
  234. ksort($this->rootFile['packages']);
  235. ksort($this->rootFile['provider-includes']);
  236. $this->dumpRootFile($rootFile);
  237. if ($verbose) {
  238. echo 'Putting new files in production'.PHP_EOL;
  239. }
  240. // put the new files in production
  241. exec(sprintf('mv %s %s && mv %s %1$s', escapeshellarg($webDir.'/p'), escapeshellarg($webDir.'/p-old'), escapeshellarg($buildDir.'/p')), $out, $exit);
  242. if (0 !== $exit) {
  243. throw new \RuntimeException("Rename failed:\n\n".implode("\n", $out));
  244. }
  245. if (defined('PHP_WINDOWS_VERSION_BUILD')) {
  246. rename($webDir.'/p/packages.json', $webDir.'/packages.json');
  247. } else {
  248. $packagesJsonPath = $webDir.'/packages.json';
  249. if (!is_link($packagesJsonPath)) {
  250. if (file_exists($packagesJsonPath)) {
  251. unlink($packagesJsonPath);
  252. }
  253. symlink($webDir.'/p/packages.json', $webDir.'/packages.json');
  254. }
  255. }
  256. // clean up old dir
  257. $retries = 5;
  258. do {
  259. if (!$this->cfs->removeDirectory($webDir.'/p-old')) {
  260. usleep(200);
  261. }
  262. clearstatcache();
  263. } while (is_dir($webDir.'/p-old') && $retries--);
  264. // run only once an hour
  265. if ($cleanUpOldFiles) {
  266. if ($verbose) {
  267. echo 'Cleaning up old files'.PHP_EOL;
  268. }
  269. // clean up old files
  270. $finder = Finder::create()->directories()->ignoreVCS(true)->in($webDir.'/p/');
  271. foreach ($finder as $vendorDir) {
  272. $vendorFiles = Finder::create()->files()->ignoreVCS(true)
  273. ->name('/\$[a-f0-9]+\.json$/')
  274. ->date('until 10minutes ago')
  275. ->in((string) $vendorDir);
  276. $hashedFiles = iterator_to_array($vendorFiles->getIterator());
  277. foreach ($hashedFiles as $file) {
  278. $key = preg_replace('{(?:.*/|^)(p/[^/]+/[^/$]+\$[a-f0-9]+\.json)$}', '$1', strtr($file, '\\', '/'));
  279. if (!in_array($key, $safeFiles, true)) {
  280. unlink((string) $file);
  281. }
  282. }
  283. }
  284. // clean up old provider listings
  285. $finder = Finder::create()->depth(0)->files()->name('provider-*.json')->ignoreVCS(true)->in($webDir.'/p/')->date('until 10minutes ago');
  286. $providerFiles = array();
  287. foreach ($finder as $provider) {
  288. $key = preg_replace('{(?:.*/|^)(p/[^/$]+\$[a-f0-9]+\.json)$}', '$1', strtr($provider, '\\', '/'));
  289. if (!in_array($key, $safeFiles, true)) {
  290. unlink((string) $provider);
  291. }
  292. }
  293. }
  294. return true;
  295. }
  296. private function dumpRootFile($file)
  297. {
  298. // sort all versions and packages to make sha1 consistent
  299. ksort($this->rootFile['packages']);
  300. foreach ($this->rootFile['packages'] as $package => $versions) {
  301. ksort($this->rootFile['packages'][$package]);
  302. }
  303. file_put_contents($file, json_encode($this->rootFile));
  304. }
  305. private function dumpListing($listing)
  306. {
  307. $key = 'p/'.basename($listing);
  308. // sort files to make hash consistent
  309. ksort($this->listings[$key]['providers']);
  310. file_put_contents($listing, json_encode($this->listings[$key]));
  311. }
  312. private function loadIndividualFile($path, $key)
  313. {
  314. if (isset($this->individualFiles[$key])) {
  315. return;
  316. }
  317. if (file_exists($path)) {
  318. $this->individualFiles[$key] = json_decode(file_get_contents($path), true);
  319. $this->individualFilesMtime[$key] = filemtime($path);
  320. } else {
  321. $this->individualFiles[$key] = array();
  322. $this->individualFilesMtime[$key] = 0;
  323. }
  324. }
  325. private function dumpIndividualFile($path, $key)
  326. {
  327. // sort all versions and packages to make sha1 consistent
  328. ksort($this->individualFiles[$key]['packages']);
  329. foreach ($this->individualFiles[$key]['packages'] as $package => $versions) {
  330. ksort($this->individualFiles[$key]['packages'][$package]);
  331. }
  332. $this->fs->mkdir(dirname($path));
  333. file_put_contents($path, json_encode($this->individualFiles[$key]));
  334. touch($path, $this->individualFilesMtime[$key]);
  335. }
  336. private function dumpVersionToIndividualFile(Version $version, $file, $key)
  337. {
  338. $this->loadIndividualFile($file, $key);
  339. $data = $version->toArray();
  340. $data['uid'] = $version->getId();
  341. $this->individualFiles[$key]['packages'][strtolower($version->getName())][$version->getVersion()] = $data;
  342. if (!isset($this->individualFilesMtime[$key]) || $this->individualFilesMtime[$key] < $version->getReleasedAt()->getTimestamp()) {
  343. $this->individualFilesMtime[$key] = $version->getReleasedAt()->getTimestamp();
  344. }
  345. }
  346. private function getTargetFile(Version $version)
  347. {
  348. if ($version->isDevelopment()) {
  349. $distribution = 16;
  350. return 'packages-dev-' . chr(abs(crc32($version->getName())) % $distribution + 97) . '.json';
  351. }
  352. $date = $version->getReleasedAt();
  353. return 'packages-' . ($date->format('Y') === date('Y') ? $date->format('Y-m') : $date->format('Y')) . '.json';
  354. }
  355. private function getTargetListing($file)
  356. {
  357. static $firstOfTheMonth;
  358. if (!$firstOfTheMonth) {
  359. $date = new \DateTime;
  360. $date->setDate($date->format('Y'), $date->format('m'), 1);
  361. $date->setTime(0, 0, 0);
  362. $firstOfTheMonth = $date->format('U');
  363. }
  364. $mtime = filemtime($file);
  365. if ($mtime < $firstOfTheMonth - 86400 * 180) {
  366. return 'provider-archived.json';
  367. }
  368. if ($mtime < $firstOfTheMonth - 86400 * 60) {
  369. return 'provider-stale.json';
  370. }
  371. if ($mtime < $firstOfTheMonth - 86400 * 10) {
  372. return 'provider-active.json';
  373. }
  374. return 'provider-latest.json';
  375. }
  376. private function getIndividualFileKey($path)
  377. {
  378. return preg_replace('{^.*?[/\\\\](p[/\\\\].+?\.(json|files))$}', '$1', $path);
  379. }
  380. }