PageRenderTime 49ms CodeModel.GetById 19ms RepoModel.GetById 1ms app.codeStats 0ms

/src/Packagist/WebBundle/Package/Dumper.php

https://github.com/ajshort/packagist
PHP | 445 lines | 314 code | 58 blank | 73 comment | 28 complexity | d58f975531da6615230a984bb01b40b6 MD5 | raw file
  1. <?php
  2. /*
  3. * This file is part of Packagist.
  4. *
  5. * (c) Jordi Boggiano <j.boggiano@seld.be>
  6. * Nils Adermann <naderman@naderman.de>
  7. *
  8. * For the full copyright and license information, please view the LICENSE
  9. * file that was distributed with this source code.
  10. */
  11. namespace Packagist\WebBundle\Package;
  12. use Symfony\Component\Filesystem\Filesystem;
  13. use Symfony\Bridge\Doctrine\RegistryInterface;
  14. use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
  15. use Symfony\Component\Finder\Finder;
  16. use Packagist\WebBundle\Entity\Version;
  17. /**
  18. * @author Jordi Boggiano <j.boggiano@seld.be>
  19. */
  20. class Dumper
  21. {
  22. /**
  23. * Doctrine
  24. * @var RegistryInterface
  25. */
  26. protected $doctrine;
  27. /**
  28. * @var Filesystem
  29. */
  30. protected $fs;
  31. /**
  32. * @var string
  33. */
  34. protected $webDir;
  35. /**
  36. * @var string
  37. */
  38. protected $buildDir;
  39. /**
  40. * @var UrlGeneratorInterface
  41. */
  42. protected $router;
  43. /**
  44. * Data cache
  45. * @var array
  46. */
  47. private $files = array();
  48. /**
  49. * Data cache
  50. * @var array
  51. */
  52. private $listings = array();
  53. /**
  54. * Data cache
  55. * @var array
  56. */
  57. private $individualFiles = array();
  58. /**
  59. * Modified times of individual files
  60. * @var array
  61. */
  62. private $individualFilesMtime = array();
  63. /**
  64. * Constructor
  65. *
  66. * @param RegistryInterface $doctrine
  67. * @param Filesystem $filesystem
  68. * @param UrlGeneratorInterface $router
  69. * @param string $webDir web root
  70. * @param string $cacheDir cache dir
  71. */
  72. public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, $webDir, $cacheDir)
  73. {
  74. $this->doctrine = $doctrine;
  75. $this->fs = $filesystem;
  76. $this->router = $router;
  77. $this->webDir = realpath($webDir);
  78. $this->buildDir = $cacheDir . '/composer-packages-build';
  79. }
  80. /**
  81. * Dump a set of packages to the web root
  82. *
  83. * @param array $packageIds
  84. * @param Boolean $force
  85. * @param Boolean $verbose
  86. */
  87. public function dump(array $packageIds, $force = false, $verbose = false)
  88. {
  89. // prepare build dir
  90. $webDir = $this->webDir;
  91. $buildDir = $this->buildDir;
  92. $this->fs->remove($buildDir);
  93. $this->fs->mkdir($buildDir);
  94. $this->fs->mkdir($webDir.'/p/');
  95. if (!$force) {
  96. if ($verbose) {
  97. echo 'Copying existing files'.PHP_EOL;
  98. }
  99. exec('cp -rpf '.escapeshellarg($webDir.'/p').' '.escapeshellarg($buildDir.'/p'), $output, $exit);
  100. if (0 !== $exit) {
  101. $this->fs->mirror($webDir.'/p/', $buildDir.'/p/', null, array('override' => true));
  102. }
  103. }
  104. $modifiedFiles = array();
  105. $modifiedIndividualFiles = array();
  106. $total = count($packageIds);
  107. $current = 0;
  108. $step = 50;
  109. while ($packageIds) {
  110. $packages = $this->doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($packageIds, 0, $step));
  111. if ($verbose) {
  112. echo '['.sprintf('%'.strlen($total).'d', $current).'/'.$total.'] Processing '.$step.' packages'.PHP_EOL;
  113. }
  114. $current += $step;
  115. // prepare packages in memory
  116. foreach ($packages as $package) {
  117. $affectedFiles = array();
  118. $name = strtolower($package->getName());
  119. // clean up versions in individual files
  120. if (file_exists($buildDir.'/p/'.$name.'.files')) {
  121. $files = json_decode(file_get_contents($buildDir.'/p/'.$name.'.files'));
  122. foreach ($files as $file) {
  123. $key = $this->getIndividualFileKey($buildDir.'/'.$file);
  124. $this->loadIndividualFile($buildDir.'/'.$file, $key);
  125. if (isset($this->individualFiles[$key]['packages'][$name])) {
  126. unset($this->individualFiles[$key]['packages'][$name]);
  127. $modifiedIndividualFiles[$key] = true;
  128. }
  129. }
  130. }
  131. // (re)write versions in individual files
  132. foreach ($package->getVersions() as $version) {
  133. foreach (array_slice($version->getNames(), 0, 150) as $versionName) {
  134. if (!preg_match('{^[A-Za-z0-9_-][A-Za-z0-9_.-]+/[A-Za-z0-9_-][A-Za-z0-9_.-]+?$}', $versionName) || strpos($versionName, '..')) {
  135. continue;
  136. }
  137. $file = $buildDir.'/p/'.$versionName.'.json';
  138. $key = $this->getIndividualFileKey($file);
  139. $this->dumpVersionToIndividualFile($version, $file, $key);
  140. $modifiedIndividualFiles[$key] = true;
  141. $affectedFiles[$key] = true;
  142. }
  143. }
  144. // store affected files to clean up properly in the next update
  145. $this->fs->mkdir(dirname($buildDir.'/p/'.$name));
  146. file_put_contents($buildDir.'/p/'.$name.'.files', json_encode(array_keys($affectedFiles)));
  147. $modifiedIndividualFiles['p/'.$name.'.files'] = true;
  148. // clean up all versions of that package
  149. foreach (glob($buildDir.'/p/packages*.json') as $file) {
  150. $key = 'p/'.basename($file);
  151. $this->loadFile($file);
  152. if (isset($this->files[$key]['packages'][$name])) {
  153. unset($this->files[$key]['packages'][$name]);
  154. $modifiedFiles[$key] = true;
  155. }
  156. }
  157. // (re)write versions
  158. foreach ($package->getVersions() as $version) {
  159. $file = $buildDir.'/p/'.$this->getTargetFile($version);
  160. $modifiedFiles['p/'.basename($file)] = true;
  161. $this->dumpVersion($version, $file);
  162. }
  163. $package->setDumpedAt(new \DateTime);
  164. }
  165. // update dump dates
  166. $this->doctrine->getEntityManager()->flush();
  167. $this->doctrine->getEntityManager()->clear();
  168. unset($packages);
  169. if ($current % 250 === 0 || !$packageIds) {
  170. if ($verbose) {
  171. echo 'Dumping individual files'.PHP_EOL;
  172. }
  173. // dump individual files to build dir
  174. foreach ($this->individualFiles as $file => $dummy) {
  175. $this->dumpIndividualFile($buildDir.'/'.$file, $file);
  176. }
  177. $this->individualFiles = array();
  178. }
  179. }
  180. // prepare individual files listings
  181. if ($verbose) {
  182. echo 'Preparing individual files listings'.PHP_EOL;
  183. }
  184. $individualListings = array();
  185. $finder = Finder::create()->files()->ignoreVCS(true)->name('*.json')->in($buildDir.'/p/')->depth('1');
  186. foreach ($finder as $file) {
  187. $key = $this->getIndividualFileKey(strtr($file, '\\', '/'));
  188. if ($force && !isset($modifiedIndividualFiles[$key])) {
  189. continue;
  190. }
  191. $listing = 'p/'.$this->getTargetListing($file);
  192. $this->listings[$listing]['providers'][$key] = array('sha256' => hash_file('sha256', $file));
  193. $individualListings[$listing] = true;
  194. }
  195. // prepare root file
  196. $rootFile = $buildDir.'/p/packages.json';
  197. $this->loadFile($rootFile);
  198. if (!isset($this->files['p/packages.json']['packages'])) {
  199. $this->files['p/packages.json']['packages'] = array();
  200. }
  201. $url = $this->router->generate('track_download', array('name' => 'VND/PKG'));
  202. $this->files['p/packages.json']['notify'] = str_replace('VND/PKG', '%package%', $url);
  203. $this->files['p/packages.json']['notify_batch'] = $this->router->generate('track_download_batch');
  204. if ($verbose) {
  205. echo 'Dumping individual listings'.PHP_EOL;
  206. }
  207. // dump listings to build dir
  208. foreach ($individualListings as $listing => $dummy) {
  209. $this->dumpListing($buildDir.'/'.$listing);
  210. $this->files['p/packages.json']['providers-includes'][$listing] = array('sha256' => hash_file('sha256', $buildDir.'/'.$listing));
  211. }
  212. if ($verbose) {
  213. echo 'Dumping package metadata'.PHP_EOL;
  214. }
  215. // dump files to build dir
  216. foreach ($modifiedFiles as $file => $dummy) {
  217. $this->dumpFile($buildDir.'/'.$file);
  218. $this->files['p/packages.json']['includes'][$file] = array('sha1' => sha1_file($buildDir.'/'.$file));
  219. }
  220. if ($verbose) {
  221. echo 'Dumping root'.PHP_EOL;
  222. }
  223. // sort & dump root file
  224. ksort($this->files['p/packages.json']['packages']);
  225. ksort($this->files['p/packages.json']['providers-includes']);
  226. ksort($this->files['p/packages.json']['includes']);
  227. $this->dumpFile($rootFile);
  228. if ($verbose) {
  229. echo 'Putting new files in production'.PHP_EOL;
  230. }
  231. // put the new files in production
  232. exec(sprintf('mv %s %s && mv %s %1$s', escapeshellarg($webDir.'/p'), escapeshellarg($webDir.'/p-old'), escapeshellarg($buildDir.'/p')), $out, $exit);
  233. if (0 !== $exit) {
  234. throw new \RuntimeException("Rename failed:\n\n".implode("\n", $out));
  235. }
  236. if (defined('PHP_WINDOWS_VERSION_BUILD')) {
  237. rename($webDir.'/p/packages.json', $webDir.'/packages.json');
  238. } else {
  239. if (!is_link($webDir.'/packages.json')) {
  240. unlink($webDir.'/packages.json');
  241. symlink($webDir.'/p/packages.json', $webDir.'/packages.json');
  242. }
  243. }
  244. // clean up old dir
  245. $retries = 5;
  246. do {
  247. exec(sprintf('rm -rf %s', escapeshellarg($webDir.'/p-old')));
  248. usleep(200);
  249. clearstatcache();
  250. } while (is_dir($webDir.'/p-old') && $retries--);
  251. if ($force) {
  252. if ($verbose) {
  253. echo 'Cleaning up outdated files'.PHP_EOL;
  254. }
  255. // clear files that were not created in this build
  256. foreach (glob($webDir.'/p/packages-*.json') as $file) {
  257. if (!isset($modifiedFiles['p/'.basename($file)])) {
  258. unlink($file);
  259. }
  260. }
  261. foreach (glob($webDir.'/p/providers-*.json') as $file) {
  262. if (!isset($individualListings['p/'.basename($file)])) {
  263. unlink($file);
  264. }
  265. }
  266. $finder = Finder::create()->files()->depth('1')->ignoreVCS(true)->name('/\.(json|files)$/')->in($webDir.'/p/');
  267. foreach ($finder as $file) {
  268. $key = $this->getIndividualFileKey(strtr($file, '\\', '/'));
  269. if (!isset($modifiedIndividualFiles[$key])) {
  270. unlink($file);
  271. }
  272. }
  273. }
  274. }
  275. private function loadFile($file)
  276. {
  277. $key = 'p/'.basename($file);
  278. if (isset($this->files[$key])) {
  279. return;
  280. }
  281. if (file_exists($file)) {
  282. $this->files[$key] = json_decode(file_get_contents($file), true);
  283. } else {
  284. $this->files[$key] = array();
  285. }
  286. }
  287. private function dumpFile($file)
  288. {
  289. $key = 'p/'.basename($file);
  290. // sort all versions and packages to make sha1 consistent
  291. ksort($this->files[$key]['packages']);
  292. foreach ($this->files[$key]['packages'] as $package => $versions) {
  293. ksort($this->files[$key]['packages'][$package]);
  294. }
  295. file_put_contents($file, json_encode($this->files[$key]));
  296. }
  297. private function dumpListing($listing)
  298. {
  299. $key = 'p/'.basename($listing);
  300. // sort files to make hash consistent
  301. ksort($this->listings[$key]['providers']);
  302. file_put_contents($listing, json_encode($this->listings[$key]));
  303. }
  304. private function loadIndividualFile($path, $key)
  305. {
  306. if (isset($this->individualFiles[$key])) {
  307. return;
  308. }
  309. if (file_exists($path)) {
  310. $this->individualFiles[$key] = json_decode(file_get_contents($path), true);
  311. $this->individualFilesMtime[$key] = filemtime($path);
  312. } else {
  313. $this->individualFiles[$key] = array();
  314. $this->individualFilesMtime[$key] = 0;
  315. }
  316. }
  317. private function dumpIndividualFile($path, $key)
  318. {
  319. // sort all versions and packages to make sha1 consistent
  320. ksort($this->individualFiles[$key]['packages']);
  321. foreach ($this->individualFiles[$key]['packages'] as $package => $versions) {
  322. ksort($this->individualFiles[$key]['packages'][$package]);
  323. }
  324. $this->fs->mkdir(dirname($path));
  325. file_put_contents($path, json_encode($this->individualFiles[$key]));
  326. touch($path, $this->individualFilesMtime[$key]);
  327. }
  328. private function dumpVersion(Version $version, $file)
  329. {
  330. $this->loadFile($file);
  331. $this->files['p/'.basename($file)]['packages'][$version->getName()][$version->getVersion()] = $version->toArray();
  332. }
  333. private function dumpVersionToIndividualFile(Version $version, $file, $key)
  334. {
  335. $this->loadIndividualFile($file, $key);
  336. $data = $version->toArray();
  337. $data['uid'] = $version->getId();
  338. $this->individualFiles[$key]['packages'][strtolower($version->getName())][$version->getVersion()] = $data;
  339. if (!isset($this->individualFilesMtime[$key]) || $this->individualFilesMtime[$key] < $version->getReleasedAt()->getTimestamp()) {
  340. $this->individualFilesMtime[$key] = $version->getReleasedAt()->getTimestamp();
  341. }
  342. }
  343. private function getTargetFile(Version $version)
  344. {
  345. if ($version->isDevelopment()) {
  346. $distribution = 16;
  347. return 'packages-dev-' . chr(abs(crc32($version->getName())) % $distribution + 97) . '.json';
  348. }
  349. $date = $version->getReleasedAt();
  350. return 'packages-' . ($date->format('Y') === date('Y') ? $date->format('Y-m') : $date->format('Y')) . '.json';
  351. }
  352. private function getTargetListing($file)
  353. {
  354. $mtime = filemtime($file);
  355. $now = time();
  356. if ($mtime < $now - 86400 * 180) {
  357. return 'providers-archived.json';
  358. }
  359. if ($mtime < $now - 86400 * 60) {
  360. return 'providers-stale.json';
  361. }
  362. if ($mtime < $now - 86400 * 10) {
  363. return 'providers-active.json';
  364. }
  365. return 'providers-latest.json';
  366. }
  367. private function getIndividualFileKey($path)
  368. {
  369. return preg_replace('{^.*?[/\\\\](p[/\\\\].+?\.(json|files))$}', '$1', $path);
  370. }
  371. }