PageRenderTime 60ms CodeModel.GetById 23ms RepoModel.GetById 1ms app.codeStats 0ms

/includes/filebackend/SwiftFileBackend.php

https://bitbucket.org/ghostfreeman/freeside-wiki
PHP | 1544 lines | 977 code | 124 blank | 443 comment | 125 complexity | 2a7815267d5f03c177a715e484e000ad MD5 | raw file
Possible License(s): GPL-2.0, Apache-2.0, LGPL-3.0

Large files files are truncated, but you can click here to view the full file

  1. <?php
  2. /**
  3. * OpenStack Swift based file backend.
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License along
  16. * with this program; if not, write to the Free Software Foundation, Inc.,
  17. * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18. * http://www.gnu.org/copyleft/gpl.html
  19. *
  20. * @file
  21. * @ingroup FileBackend
  22. * @author Russ Nelson
  23. * @author Aaron Schulz
  24. */
  25. /**
  26. * @brief Class for an OpenStack Swift based file backend.
  27. *
  28. * This requires the SwiftCloudFiles MediaWiki extension, which includes
  29. * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  30. * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  31. *
  32. * Status messages should avoid mentioning the Swift account name.
  33. * Likewise, error suppression should be used to avoid path disclosure.
  34. *
  35. * @ingroup FileBackend
  36. * @since 1.19
  37. */
  38. class SwiftFileBackend extends FileBackendStore {
  39. /** @var CF_Authentication */
  40. protected $auth; // Swift authentication handler
  41. protected $authTTL; // integer seconds
  42. protected $swiftAnonUser; // string; username to handle unauthenticated requests
  43. protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  44. protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  45. protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  46. /** @var CF_Connection */
  47. protected $conn; // Swift connection handle
  48. protected $sessionStarted = 0; // integer UNIX timestamp
  49. /** @var CloudFilesException */
  50. protected $connException;
  51. protected $connErrorTime = 0; // UNIX timestamp
  52. /** @var BagOStuff */
  53. protected $srvCache;
  54. /** @var ProcessCacheLRU */
  55. protected $connContainerCache; // container object cache
  56. /**
  57. * @see FileBackendStore::__construct()
  58. * Additional $config params include:
  59. * - swiftAuthUrl : Swift authentication server URL
  60. * - swiftUser : Swift user used by MediaWiki (account:username)
  61. * - swiftKey : Swift authentication key for the above user
  62. * - swiftAuthTTL : Swift authentication TTL (seconds)
  63. * - swiftAnonUser : Swift user used for end-user requests (account:username).
  64. * If set, then views of public containers are assumed to go
  65. * through this user. If not set, then public containers are
  66. * accessible to unauthenticated requests via ".r:*" in the ACL.
  67. * - swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up
  68. * - swiftCDNExpiry : How long (in seconds) to store content in the CDN.
  69. * If files may likely change, this should probably not exceed
  70. * a few days. For example, deletions may take this long to apply.
  71. * If object purging is enabled, however, this is not an issue.
  72. * - swiftCDNPurgable : Whether object purge requests are allowed by the CDN.
  73. * - shardViaHashLevels : Map of container names to sharding config with:
  74. * - base : base of hash characters, 16 or 36
  75. * - levels : the number of hash levels (and digits)
  76. * - repeat : hash subdirectories are prefixed with all the
  77. * parent hash directory names (e.g. "a/ab/abc")
  78. * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
  79. * If those are not available, then the main cache will be used.
  80. * This is probably insecure in shared hosting environments.
  81. */
  82. public function __construct( array $config ) {
  83. parent::__construct( $config );
  84. if ( !MWInit::classExists( 'CF_Constants' ) ) {
  85. throw new MWException( 'SwiftCloudFiles extension not installed.' );
  86. }
  87. // Required settings
  88. $this->auth = new CF_Authentication(
  89. $config['swiftUser'],
  90. $config['swiftKey'],
  91. null, // account; unused
  92. $config['swiftAuthUrl']
  93. );
  94. // Optional settings
  95. $this->authTTL = isset( $config['swiftAuthTTL'] )
  96. ? $config['swiftAuthTTL']
  97. : 5 * 60; // some sane number
  98. $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  99. ? $config['swiftAnonUser']
  100. : '';
  101. $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  102. ? $config['shardViaHashLevels']
  103. : '';
  104. $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
  105. ? $config['swiftUseCDN']
  106. : false;
  107. $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
  108. ? $config['swiftCDNExpiry']
  109. : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
  110. $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
  111. ? $config['swiftCDNPurgable']
  112. : true;
  113. // Cache container information to mask latency
  114. $this->memCache = wfGetMainCache();
  115. // Process cache for container info
  116. $this->connContainerCache = new ProcessCacheLRU( 300 );
  117. // Cache auth token information to avoid RTTs
  118. if ( !empty( $config['cacheAuthInfo'] ) ) {
  119. if ( php_sapi_name() === 'cli' ) {
  120. $this->srvCache = wfGetMainCache(); // preferrably memcached
  121. } else {
  122. try { // look for APC, XCache, WinCache, ect...
  123. $this->srvCache = ObjectCache::newAccelerator( array() );
  124. } catch ( Exception $e ) {}
  125. }
  126. }
  127. $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
  128. }
  129. /**
  130. * @see FileBackendStore::resolveContainerPath()
  131. * @return null
  132. */
  133. protected function resolveContainerPath( $container, $relStoragePath ) {
  134. if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
  135. return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
  136. } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  137. return null; // too long for Swift
  138. }
  139. return $relStoragePath;
  140. }
  141. /**
  142. * @see FileBackendStore::isPathUsableInternal()
  143. * @return bool
  144. */
  145. public function isPathUsableInternal( $storagePath ) {
  146. list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
  147. if ( $rel === null ) {
  148. return false; // invalid
  149. }
  150. try {
  151. $this->getContainer( $container );
  152. return true; // container exists
  153. } catch ( NoSuchContainerException $e ) {
  154. } catch ( CloudFilesException $e ) { // some other exception?
  155. $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
  156. }
  157. return false;
  158. }
  159. /**
  160. * @param $disposition string Content-Disposition header value
  161. * @return string Truncated Content-Disposition header value to meet Swift limits
  162. */
  163. protected function truncDisp( $disposition ) {
  164. $res = '';
  165. foreach ( explode( ';', $disposition ) as $part ) {
  166. $part = trim( $part );
  167. $new = ( $res === '' ) ? $part : "{$res};{$part}";
  168. if ( strlen( $new ) <= 255 ) {
  169. $res = $new;
  170. } else {
  171. break; // too long; sigh
  172. }
  173. }
  174. return $res;
  175. }
  176. /**
  177. * @see FileBackendStore::doCreateInternal()
  178. * @return Status
  179. */
  180. protected function doCreateInternal( array $params ) {
  181. $status = Status::newGood();
  182. list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
  183. if ( $dstRel === null ) {
  184. $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
  185. return $status;
  186. }
  187. // (a) Check the destination container and object
  188. try {
  189. $dContObj = $this->getContainer( $dstCont );
  190. if ( empty( $params['overwrite'] ) &&
  191. $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
  192. {
  193. $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
  194. return $status;
  195. }
  196. } catch ( NoSuchContainerException $e ) {
  197. $status->fatal( 'backend-fail-create', $params['dst'] );
  198. return $status;
  199. } catch ( CloudFilesException $e ) { // some other exception?
  200. $this->handleException( $e, $status, __METHOD__, $params );
  201. return $status;
  202. }
  203. // (b) Get a SHA-1 hash of the object
  204. $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
  205. // (c) Actually create the object
  206. try {
  207. // Create a fresh CF_Object with no fields preloaded.
  208. // We don't want to preserve headers, metadata, and such.
  209. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
  210. // Note: metadata keys stored as [Upper case char][[Lower case char]...]
  211. $obj->metadata = array( 'Sha1base36' => $sha1Hash );
  212. // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
  213. // The MD5 here will be checked within Swift against its own MD5.
  214. $obj->set_etag( md5( $params['content'] ) );
  215. // Use the same content type as StreamFile for security
  216. $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
  217. if ( !strlen( $obj->content_type ) ) { // special case
  218. $obj->content_type = 'unknown/unknown';
  219. }
  220. // Set the Content-Disposition header if requested
  221. if ( isset( $params['disposition'] ) ) {
  222. $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
  223. }
  224. if ( !empty( $params['async'] ) ) { // deferred
  225. $op = $obj->write_async( $params['content'] );
  226. $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
  227. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  228. $status->value->affectedObjects[] = $obj;
  229. }
  230. } else { // actually write the object in Swift
  231. $obj->write( $params['content'] );
  232. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  233. $this->purgeCDNCache( array( $obj ) );
  234. }
  235. }
  236. } catch ( CDNNotEnabledException $e ) {
  237. // CDN not enabled; nothing to see here
  238. } catch ( BadContentTypeException $e ) {
  239. $status->fatal( 'backend-fail-contenttype', $params['dst'] );
  240. } catch ( CloudFilesException $e ) { // some other exception?
  241. $this->handleException( $e, $status, __METHOD__, $params );
  242. }
  243. return $status;
  244. }
  245. /**
  246. * @see SwiftFileBackend::doExecuteOpHandlesInternal()
  247. */
  248. protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
  249. try {
  250. $cfOp->getLastResponse();
  251. } catch ( BadContentTypeException $e ) {
  252. $status->fatal( 'backend-fail-contenttype', $params['dst'] );
  253. }
  254. }
  255. /**
  256. * @see FileBackendStore::doStoreInternal()
  257. * @return Status
  258. */
  259. protected function doStoreInternal( array $params ) {
  260. $status = Status::newGood();
  261. list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
  262. if ( $dstRel === null ) {
  263. $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
  264. return $status;
  265. }
  266. // (a) Check the destination container and object
  267. try {
  268. $dContObj = $this->getContainer( $dstCont );
  269. if ( empty( $params['overwrite'] ) &&
  270. $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
  271. {
  272. $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
  273. return $status;
  274. }
  275. } catch ( NoSuchContainerException $e ) {
  276. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  277. return $status;
  278. } catch ( CloudFilesException $e ) { // some other exception?
  279. $this->handleException( $e, $status, __METHOD__, $params );
  280. return $status;
  281. }
  282. // (b) Get a SHA-1 hash of the object
  283. $sha1Hash = sha1_file( $params['src'] );
  284. if ( $sha1Hash === false ) { // source doesn't exist?
  285. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  286. return $status;
  287. }
  288. $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
  289. // (c) Actually store the object
  290. try {
  291. // Create a fresh CF_Object with no fields preloaded.
  292. // We don't want to preserve headers, metadata, and such.
  293. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
  294. // Note: metadata keys stored as [Upper case char][[Lower case char]...]
  295. $obj->metadata = array( 'Sha1base36' => $sha1Hash );
  296. // The MD5 here will be checked within Swift against its own MD5.
  297. $obj->set_etag( md5_file( $params['src'] ) );
  298. // Use the same content type as StreamFile for security
  299. $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
  300. if ( !strlen( $obj->content_type ) ) { // special case
  301. $obj->content_type = 'unknown/unknown';
  302. }
  303. // Set the Content-Disposition header if requested
  304. if ( isset( $params['disposition'] ) ) {
  305. $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
  306. }
  307. if ( !empty( $params['async'] ) ) { // deferred
  308. wfSuppressWarnings();
  309. $fp = fopen( $params['src'], 'rb' );
  310. wfRestoreWarnings();
  311. if ( !$fp ) {
  312. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  313. } else {
  314. $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
  315. $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
  316. $status->value->resourcesToClose[] = $fp;
  317. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  318. $status->value->affectedObjects[] = $obj;
  319. }
  320. }
  321. } else { // actually write the object in Swift
  322. $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
  323. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  324. $this->purgeCDNCache( array( $obj ) );
  325. }
  326. }
  327. } catch ( CDNNotEnabledException $e ) {
  328. // CDN not enabled; nothing to see here
  329. } catch ( BadContentTypeException $e ) {
  330. $status->fatal( 'backend-fail-contenttype', $params['dst'] );
  331. } catch ( IOException $e ) {
  332. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  333. } catch ( CloudFilesException $e ) { // some other exception?
  334. $this->handleException( $e, $status, __METHOD__, $params );
  335. }
  336. return $status;
  337. }
  338. /**
  339. * @see SwiftFileBackend::doExecuteOpHandlesInternal()
  340. */
  341. protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
  342. try {
  343. $cfOp->getLastResponse();
  344. } catch ( BadContentTypeException $e ) {
  345. $status->fatal( 'backend-fail-contenttype', $params['dst'] );
  346. } catch ( IOException $e ) {
  347. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  348. }
  349. }
  350. /**
  351. * @see FileBackendStore::doCopyInternal()
  352. * @return Status
  353. */
  354. protected function doCopyInternal( array $params ) {
  355. $status = Status::newGood();
  356. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  357. if ( $srcRel === null ) {
  358. $status->fatal( 'backend-fail-invalidpath', $params['src'] );
  359. return $status;
  360. }
  361. list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
  362. if ( $dstRel === null ) {
  363. $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
  364. return $status;
  365. }
  366. // (a) Check the source/destination containers and destination object
  367. try {
  368. $sContObj = $this->getContainer( $srcCont );
  369. $dContObj = $this->getContainer( $dstCont );
  370. if ( empty( $params['overwrite'] ) &&
  371. $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
  372. {
  373. $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
  374. return $status;
  375. }
  376. } catch ( NoSuchContainerException $e ) {
  377. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  378. return $status;
  379. } catch ( CloudFilesException $e ) { // some other exception?
  380. $this->handleException( $e, $status, __METHOD__, $params );
  381. return $status;
  382. }
  383. // (b) Actually copy the file to the destination
  384. try {
  385. $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
  386. $hdrs = array(); // source file headers to override with new values
  387. if ( isset( $params['disposition'] ) ) {
  388. $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
  389. }
  390. if ( !empty( $params['async'] ) ) { // deferred
  391. $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
  392. $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
  393. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  394. $status->value->affectedObjects[] = $dstObj;
  395. }
  396. } else { // actually write the object in Swift
  397. $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
  398. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  399. $this->purgeCDNCache( array( $dstObj ) );
  400. }
  401. }
  402. } catch ( CDNNotEnabledException $e ) {
  403. // CDN not enabled; nothing to see here
  404. } catch ( NoSuchObjectException $e ) { // source object does not exist
  405. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  406. } catch ( CloudFilesException $e ) { // some other exception?
  407. $this->handleException( $e, $status, __METHOD__, $params );
  408. }
  409. return $status;
  410. }
  411. /**
  412. * @see SwiftFileBackend::doExecuteOpHandlesInternal()
  413. */
  414. protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
  415. try {
  416. $cfOp->getLastResponse();
  417. } catch ( NoSuchObjectException $e ) { // source object does not exist
  418. $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
  419. }
  420. }
  421. /**
  422. * @see FileBackendStore::doMoveInternal()
  423. * @return Status
  424. */
  425. protected function doMoveInternal( array $params ) {
  426. $status = Status::newGood();
  427. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  428. if ( $srcRel === null ) {
  429. $status->fatal( 'backend-fail-invalidpath', $params['src'] );
  430. return $status;
  431. }
  432. list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
  433. if ( $dstRel === null ) {
  434. $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
  435. return $status;
  436. }
  437. // (a) Check the source/destination containers and destination object
  438. try {
  439. $sContObj = $this->getContainer( $srcCont );
  440. $dContObj = $this->getContainer( $dstCont );
  441. if ( empty( $params['overwrite'] ) &&
  442. $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
  443. {
  444. $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
  445. return $status;
  446. }
  447. } catch ( NoSuchContainerException $e ) {
  448. $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
  449. return $status;
  450. } catch ( CloudFilesException $e ) { // some other exception?
  451. $this->handleException( $e, $status, __METHOD__, $params );
  452. return $status;
  453. }
  454. // (b) Actually move the file to the destination
  455. try {
  456. $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
  457. $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
  458. $hdrs = array(); // source file headers to override with new values
  459. if ( isset( $params['disposition'] ) ) {
  460. $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
  461. }
  462. if ( !empty( $params['async'] ) ) { // deferred
  463. $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
  464. $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
  465. $status->value->affectedObjects[] = $srcObj;
  466. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  467. $status->value->affectedObjects[] = $dstObj;
  468. }
  469. } else { // actually write the object in Swift
  470. $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
  471. $this->purgeCDNCache( array( $srcObj ) );
  472. if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
  473. $this->purgeCDNCache( array( $dstObj ) );
  474. }
  475. }
  476. } catch ( CDNNotEnabledException $e ) {
  477. // CDN not enabled; nothing to see here
  478. } catch ( NoSuchObjectException $e ) { // source object does not exist
  479. $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
  480. } catch ( CloudFilesException $e ) { // some other exception?
  481. $this->handleException( $e, $status, __METHOD__, $params );
  482. }
  483. return $status;
  484. }
  485. /**
  486. * @see SwiftFileBackend::doExecuteOpHandlesInternal()
  487. */
  488. protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
  489. try {
  490. $cfOp->getLastResponse();
  491. } catch ( NoSuchObjectException $e ) { // source object does not exist
  492. $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
  493. }
  494. }
  495. /**
  496. * @see FileBackendStore::doDeleteInternal()
  497. * @return Status
  498. */
  499. protected function doDeleteInternal( array $params ) {
  500. $status = Status::newGood();
  501. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  502. if ( $srcRel === null ) {
  503. $status->fatal( 'backend-fail-invalidpath', $params['src'] );
  504. return $status;
  505. }
  506. try {
  507. $sContObj = $this->getContainer( $srcCont );
  508. $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
  509. if ( !empty( $params['async'] ) ) { // deferred
  510. $op = $sContObj->delete_object_async( $srcRel );
  511. $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
  512. $status->value->affectedObjects[] = $srcObj;
  513. } else { // actually write the object in Swift
  514. $sContObj->delete_object( $srcRel );
  515. $this->purgeCDNCache( array( $srcObj ) );
  516. }
  517. } catch ( CDNNotEnabledException $e ) {
  518. // CDN not enabled; nothing to see here
  519. } catch ( NoSuchContainerException $e ) {
  520. $status->fatal( 'backend-fail-delete', $params['src'] );
  521. } catch ( NoSuchObjectException $e ) {
  522. if ( empty( $params['ignoreMissingSource'] ) ) {
  523. $status->fatal( 'backend-fail-delete', $params['src'] );
  524. }
  525. } catch ( CloudFilesException $e ) { // some other exception?
  526. $this->handleException( $e, $status, __METHOD__, $params );
  527. }
  528. return $status;
  529. }
  530. /**
  531. * @see SwiftFileBackend::doExecuteOpHandlesInternal()
  532. */
  533. protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
  534. try {
  535. $cfOp->getLastResponse();
  536. } catch ( NoSuchContainerException $e ) {
  537. $status->fatal( 'backend-fail-delete', $params['src'] );
  538. } catch ( NoSuchObjectException $e ) {
  539. if ( empty( $params['ignoreMissingSource'] ) ) {
  540. $status->fatal( 'backend-fail-delete', $params['src'] );
  541. }
  542. }
  543. }
  544. /**
  545. * @see FileBackendStore::doPrepareInternal()
  546. * @return Status
  547. */
  548. protected function doPrepareInternal( $fullCont, $dir, array $params ) {
  549. $status = Status::newGood();
  550. // (a) Check if container already exists
  551. try {
  552. $contObj = $this->getContainer( $fullCont );
  553. // NoSuchContainerException not thrown: container must exist
  554. return $status; // already exists
  555. } catch ( NoSuchContainerException $e ) {
  556. // NoSuchContainerException thrown: container does not exist
  557. } catch ( CloudFilesException $e ) { // some other exception?
  558. $this->handleException( $e, $status, __METHOD__, $params );
  559. return $status;
  560. }
  561. // (b) Create container as needed
  562. try {
  563. $contObj = $this->createContainer( $fullCont );
  564. if ( !empty( $params['noAccess'] ) ) {
  565. // Make container private to end-users...
  566. $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
  567. } else {
  568. // Make container public to end-users...
  569. $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
  570. }
  571. if ( $this->swiftUseCDN ) { // Rackspace style CDN
  572. $contObj->make_public( $this->swiftCDNExpiry );
  573. }
  574. } catch ( CDNNotEnabledException $e ) {
  575. // CDN not enabled; nothing to see here
  576. } catch ( CloudFilesException $e ) { // some other exception?
  577. $this->handleException( $e, $status, __METHOD__, $params );
  578. return $status;
  579. }
  580. return $status;
  581. }
  582. /**
  583. * @see FileBackendStore::doSecureInternal()
  584. * @return Status
  585. */
  586. protected function doSecureInternal( $fullCont, $dir, array $params ) {
  587. $status = Status::newGood();
  588. if ( empty( $params['noAccess'] ) ) {
  589. return $status; // nothing to do
  590. }
  591. // Restrict container from end-users...
  592. try {
  593. // doPrepareInternal() should have been called,
  594. // so the Swift container should already exist...
  595. $contObj = $this->getContainer( $fullCont ); // normally a cache hit
  596. // NoSuchContainerException not thrown: container must exist
  597. // Make container private to end-users...
  598. $status->merge( $this->setContainerAccess(
  599. $contObj,
  600. array( $this->auth->username ), // read
  601. array( $this->auth->username ) // write
  602. ) );
  603. if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
  604. $contObj->make_private();
  605. }
  606. } catch ( CDNNotEnabledException $e ) {
  607. // CDN not enabled; nothing to see here
  608. } catch ( CloudFilesException $e ) { // some other exception?
  609. $this->handleException( $e, $status, __METHOD__, $params );
  610. }
  611. return $status;
  612. }
  613. /**
  614. * @see FileBackendStore::doPublishInternal()
  615. * @return Status
  616. */
  617. protected function doPublishInternal( $fullCont, $dir, array $params ) {
  618. $status = Status::newGood();
  619. // Unrestrict container from end-users...
  620. try {
  621. // doPrepareInternal() should have been called,
  622. // so the Swift container should already exist...
  623. $contObj = $this->getContainer( $fullCont ); // normally a cache hit
  624. // NoSuchContainerException not thrown: container must exist
  625. // Make container public to end-users...
  626. if ( $this->swiftAnonUser != '' ) {
  627. $status->merge( $this->setContainerAccess(
  628. $contObj,
  629. array( $this->auth->username, $this->swiftAnonUser ), // read
  630. array( $this->auth->username, $this->swiftAnonUser ) // write
  631. ) );
  632. } else {
  633. $status->merge( $this->setContainerAccess(
  634. $contObj,
  635. array( $this->auth->username, '.r:*' ), // read
  636. array( $this->auth->username ) // write
  637. ) );
  638. }
  639. if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
  640. $contObj->make_public();
  641. }
  642. } catch ( CDNNotEnabledException $e ) {
  643. // CDN not enabled; nothing to see here
  644. } catch ( CloudFilesException $e ) { // some other exception?
  645. $this->handleException( $e, $status, __METHOD__, $params );
  646. }
  647. return $status;
  648. }
  649. /**
  650. * @see FileBackendStore::doCleanInternal()
  651. * @return Status
  652. */
  653. protected function doCleanInternal( $fullCont, $dir, array $params ) {
  654. $status = Status::newGood();
  655. // Only containers themselves can be removed, all else is virtual
  656. if ( $dir != '' ) {
  657. return $status; // nothing to do
  658. }
  659. // (a) Check the container
  660. try {
  661. $contObj = $this->getContainer( $fullCont, true );
  662. } catch ( NoSuchContainerException $e ) {
  663. return $status; // ok, nothing to do
  664. } catch ( CloudFilesException $e ) { // some other exception?
  665. $this->handleException( $e, $status, __METHOD__, $params );
  666. return $status;
  667. }
  668. // (b) Delete the container if empty
  669. if ( $contObj->object_count == 0 ) {
  670. try {
  671. $this->deleteContainer( $fullCont );
  672. } catch ( NoSuchContainerException $e ) {
  673. return $status; // race?
  674. } catch ( NonEmptyContainerException $e ) {
  675. return $status; // race? consistency delay?
  676. } catch ( CloudFilesException $e ) { // some other exception?
  677. $this->handleException( $e, $status, __METHOD__, $params );
  678. return $status;
  679. }
  680. }
  681. return $status;
  682. }
  683. /**
  684. * @see FileBackendStore::doFileExists()
  685. * @return array|bool|null
  686. */
  687. protected function doGetFileStat( array $params ) {
  688. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  689. if ( $srcRel === null ) {
  690. return false; // invalid storage path
  691. }
  692. $stat = false;
  693. try {
  694. $contObj = $this->getContainer( $srcCont );
  695. $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
  696. $this->addMissingMetadata( $srcObj, $params['src'] );
  697. $stat = array(
  698. // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
  699. 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
  700. 'size' => (int)$srcObj->content_length,
  701. 'sha1' => $srcObj->metadata['Sha1base36']
  702. );
  703. } catch ( NoSuchContainerException $e ) {
  704. } catch ( NoSuchObjectException $e ) {
  705. } catch ( CloudFilesException $e ) { // some other exception?
  706. $stat = null;
  707. $this->handleException( $e, null, __METHOD__, $params );
  708. }
  709. return $stat;
  710. }
  711. /**
  712. * Fill in any missing object metadata and save it to Swift
  713. *
  714. * @param $obj CF_Object
  715. * @param $path string Storage path to object
  716. * @return bool Success
  717. * @throws Exception cloudfiles exceptions
  718. */
  719. protected function addMissingMetadata( CF_Object $obj, $path ) {
  720. if ( isset( $obj->metadata['Sha1base36'] ) ) {
  721. return true; // nothing to do
  722. }
  723. wfProfileIn( __METHOD__ );
  724. $status = Status::newGood();
  725. $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
  726. if ( $status->isOK() ) {
  727. # Do not stat the file in getLocalCopy() to avoid infinite loops
  728. $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
  729. if ( $tmpFile ) {
  730. $hash = $tmpFile->getSha1Base36();
  731. if ( $hash !== false ) {
  732. $obj->metadata['Sha1base36'] = $hash;
  733. $obj->sync_metadata(); // save to Swift
  734. wfProfileOut( __METHOD__ );
  735. return true; // success
  736. }
  737. }
  738. }
  739. $obj->metadata['Sha1base36'] = false;
  740. wfProfileOut( __METHOD__ );
  741. return false; // failed
  742. }
  743. /**
  744. * @see FileBackend::getFileContents()
  745. * @return bool|null|string
  746. */
  747. public function getFileContents( array $params ) {
  748. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  749. if ( $srcRel === null ) {
  750. return false; // invalid storage path
  751. }
  752. if ( !$this->fileExists( $params ) ) {
  753. return null;
  754. }
  755. $data = false;
  756. try {
  757. $sContObj = $this->getContainer( $srcCont );
  758. $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
  759. $data = $obj->read( $this->headersFromParams( $params ) );
  760. } catch ( NoSuchContainerException $e ) {
  761. } catch ( CloudFilesException $e ) { // some other exception?
  762. $this->handleException( $e, null, __METHOD__, $params );
  763. }
  764. return $data;
  765. }
  766. /**
  767. * @see FileBackendStore::doDirectoryExists()
  768. * @return bool|null
  769. */
  770. protected function doDirectoryExists( $fullCont, $dir, array $params ) {
  771. try {
  772. $container = $this->getContainer( $fullCont );
  773. $prefix = ( $dir == '' ) ? null : "{$dir}/";
  774. return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
  775. } catch ( NoSuchContainerException $e ) {
  776. return false;
  777. } catch ( CloudFilesException $e ) { // some other exception?
  778. $this->handleException( $e, null, __METHOD__,
  779. array( 'cont' => $fullCont, 'dir' => $dir ) );
  780. }
  781. return null; // error
  782. }
  783. /**
  784. * @see FileBackendStore::getDirectoryListInternal()
  785. * @return SwiftFileBackendDirList
  786. */
  787. public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
  788. return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
  789. }
  790. /**
  791. * @see FileBackendStore::getFileListInternal()
  792. * @return SwiftFileBackendFileList
  793. */
  794. public function getFileListInternal( $fullCont, $dir, array $params ) {
  795. return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
  796. }
  797. /**
  798. * Do not call this function outside of SwiftFileBackendFileList
  799. *
  800. * @param $fullCont string Resolved container name
  801. * @param $dir string Resolved storage directory with no trailing slash
  802. * @param $after string|null Storage path of file to list items after
  803. * @param $limit integer Max number of items to list
  804. * @param $params Array Includes flag for 'topOnly'
  805. * @return Array List of relative paths of dirs directly under $dir
  806. */
  807. public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
  808. $dirs = array();
  809. if ( $after === INF ) {
  810. return $dirs; // nothing more
  811. }
  812. wfProfileIn( __METHOD__ . '-' . $this->name );
  813. try {
  814. $container = $this->getContainer( $fullCont );
  815. $prefix = ( $dir == '' ) ? null : "{$dir}/";
  816. // Non-recursive: only list dirs right under $dir
  817. if ( !empty( $params['topOnly'] ) ) {
  818. $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
  819. foreach ( $objects as $object ) { // files and dirs
  820. if ( substr( $object, -1 ) === '/' ) {
  821. $dirs[] = $object; // directories end in '/'
  822. }
  823. }
  824. // Recursive: list all dirs under $dir and its subdirs
  825. } else {
  826. // Get directory from last item of prior page
  827. $lastDir = $this->getParentDir( $after ); // must be first page
  828. $objects = $container->list_objects( $limit, $after, $prefix );
  829. foreach ( $objects as $object ) { // files
  830. $objectDir = $this->getParentDir( $object ); // directory of object
  831. if ( $objectDir !== false ) { // file has a parent dir
  832. // Swift stores paths in UTF-8, using binary sorting.
  833. // See function "create_container_table" in common/db.py.
  834. // If a directory is not "greater" than the last one,
  835. // then it was already listed by the calling iterator.
  836. if ( strcmp( $objectDir, $lastDir ) > 0 ) {
  837. $pDir = $objectDir;
  838. do { // add dir and all its parent dirs
  839. $dirs[] = "{$pDir}/";
  840. $pDir = $this->getParentDir( $pDir );
  841. } while ( $pDir !== false // sanity
  842. && strcmp( $pDir, $lastDir ) > 0 // not done already
  843. && strlen( $pDir ) > strlen( $dir ) // within $dir
  844. );
  845. }
  846. $lastDir = $objectDir;
  847. }
  848. }
  849. }
  850. if ( count( $objects ) < $limit ) {
  851. $after = INF; // avoid a second RTT
  852. } else {
  853. $after = end( $objects ); // update last item
  854. }
  855. } catch ( NoSuchContainerException $e ) {
  856. } catch ( CloudFilesException $e ) { // some other exception?
  857. $this->handleException( $e, null, __METHOD__,
  858. array( 'cont' => $fullCont, 'dir' => $dir ) );
  859. }
  860. wfProfileOut( __METHOD__ . '-' . $this->name );
  861. return $dirs;
  862. }
  863. protected function getParentDir( $path ) {
  864. return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
  865. }
  866. /**
  867. * Do not call this function outside of SwiftFileBackendFileList
  868. *
  869. * @param $fullCont string Resolved container name
  870. * @param $dir string Resolved storage directory with no trailing slash
  871. * @param $after string|null Storage path of file to list items after
  872. * @param $limit integer Max number of items to list
  873. * @param $params Array Includes flag for 'topOnly'
  874. * @return Array List of relative paths of files under $dir
  875. */
  876. public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
  877. $files = array();
  878. if ( $after === INF ) {
  879. return $files; // nothing more
  880. }
  881. wfProfileIn( __METHOD__ . '-' . $this->name );
  882. try {
  883. $container = $this->getContainer( $fullCont );
  884. $prefix = ( $dir == '' ) ? null : "{$dir}/";
  885. // Non-recursive: only list files right under $dir
  886. if ( !empty( $params['topOnly'] ) ) { // files and dirs
  887. $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
  888. foreach ( $objects as $object ) {
  889. if ( substr( $object, -1 ) !== '/' ) {
  890. $files[] = $object; // directories end in '/'
  891. }
  892. }
  893. // Recursive: list all files under $dir and its subdirs
  894. } else { // files
  895. $objects = $container->list_objects( $limit, $after, $prefix );
  896. $files = $objects;
  897. }
  898. if ( count( $objects ) < $limit ) {
  899. $after = INF; // avoid a second RTT
  900. } else {
  901. $after = end( $objects ); // update last item
  902. }
  903. } catch ( NoSuchContainerException $e ) {
  904. } catch ( CloudFilesException $e ) { // some other exception?
  905. $this->handleException( $e, null, __METHOD__,
  906. array( 'cont' => $fullCont, 'dir' => $dir ) );
  907. }
  908. wfProfileOut( __METHOD__ . '-' . $this->name );
  909. return $files;
  910. }
  911. /**
  912. * @see FileBackendStore::doGetFileSha1base36()
  913. * @return bool
  914. */
  915. protected function doGetFileSha1base36( array $params ) {
  916. $stat = $this->getFileStat( $params );
  917. if ( $stat ) {
  918. return $stat['sha1'];
  919. } else {
  920. return false;
  921. }
  922. }
  923. /**
  924. * @see FileBackendStore::doStreamFile()
  925. * @return Status
  926. */
  927. protected function doStreamFile( array $params ) {
  928. $status = Status::newGood();
  929. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  930. if ( $srcRel === null ) {
  931. $status->fatal( 'backend-fail-invalidpath', $params['src'] );
  932. }
  933. try {
  934. $cont = $this->getContainer( $srcCont );
  935. } catch ( NoSuchContainerException $e ) {
  936. $status->fatal( 'backend-fail-stream', $params['src'] );
  937. return $status;
  938. } catch ( CloudFilesException $e ) { // some other exception?
  939. $this->handleException( $e, $status, __METHOD__, $params );
  940. return $status;
  941. }
  942. try {
  943. $output = fopen( 'php://output', 'wb' );
  944. $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
  945. $obj->stream( $output, $this->headersFromParams( $params ) );
  946. } catch ( NoSuchObjectException $e ) {
  947. $status->fatal( 'backend-fail-stream', $params['src'] );
  948. } catch ( CloudFilesException $e ) { // some other exception?
  949. $this->handleException( $e, $status, __METHOD__, $params );
  950. }
  951. return $status;
  952. }
  953. /**
  954. * @see FileBackendStore::getLocalCopy()
  955. * @return null|TempFSFile
  956. */
  957. public function getLocalCopy( array $params ) {
  958. list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
  959. if ( $srcRel === null ) {
  960. return null;
  961. }
  962. // Blindly create a tmp file and stream to it, catching any exception if the file does
  963. // not exist. Also, doing a stat here will cause infinite loops when filling metadata.
  964. $tmpFile = null;
  965. try {
  966. $sContObj = $this->getContainer( $srcCont );
  967. $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
  968. // Get source file extension
  969. $ext = FileBackend::extensionFromPath( $srcRel );
  970. // Create a new temporary file...
  971. $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
  972. if ( $tmpFile ) {
  973. $handle = fopen( $tmpFile->getPath(), 'wb' );
  974. if ( $handle ) {
  975. $obj->stream( $handle, $this->headersFromParams( $params ) );
  976. fclose( $handle );
  977. } else {
  978. $tmpFile = null; // couldn't open temp file
  979. }
  980. }
  981. } catch ( NoSuchContainerException $e ) {
  982. $tmpFile = null;
  983. } catch ( NoSuchObjectException $e ) {
  984. $tmpFile = null;
  985. } catch ( CloudFilesException $e ) { // some other exception?
  986. $tmpFile = null;
  987. $this->handleException( $e, null, __METHOD__, $params );
  988. }
  989. return $tmpFile;
  990. }
  991. /**
  992. * @see FileBackendStore::directoriesAreVirtual()
  993. * @return bool
  994. */
  995. protected function directoriesAreVirtual() {
  996. return true;
  997. }
  998. /**
  999. * Get headers to send to Swift when reading a file based
  1000. * on a FileBackend params array, e.g. that of getLocalCopy().
  1001. * $params is currently only checked for a 'latest' flag.
  1002. *
  1003. * @param $params Array
  1004. * @return Array
  1005. */
  1006. protected function headersFromParams( array $params ) {
  1007. $hdrs = array();
  1008. if ( !empty( $params['latest'] ) ) {
  1009. $hdrs[] = 'X-Newest: true';
  1010. }
  1011. return $hdrs;
  1012. }
  1013. /**
  1014. * @see FileBackendStore::doExecuteOpHandlesInternal()
  1015. * @return Array List of corresponding Status objects
  1016. */
  1017. protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
  1018. $statuses = array();
  1019. $cfOps = array(); // list of CF_Async_Op objects
  1020. foreach ( $fileOpHandles as $index => $fileOpHandle ) {
  1021. $cfOps[$index] = $fileOpHandle->cfOp;
  1022. }
  1023. $batch = new CF_Async_Op_Batch( $cfOps );
  1024. $cfOps = $batch->execute();
  1025. foreach ( $cfOps as $index => $cfOp ) {
  1026. $status = Status::newGood();
  1027. try { // catch exceptions; update status
  1028. $function = '_getResponse' . $fileOpHandles[$index]->call;
  1029. $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
  1030. $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
  1031. } catch ( CloudFilesException $e ) { // some other exception?
  1032. $this->handleException( $e, $status,
  1033. __CLASS__ . ":$function", $fileOpHandles[$index]->params );
  1034. }
  1035. $statuses[$index] = $status;
  1036. }
  1037. return $statuses;
  1038. }
  1039. /**
  1040. * Set read/write permissions for a Swift container.
  1041. *
  1042. * $readGrps is a list of the possible criteria for a request to have
  1043. * access to read a container. Each item is one of the following formats:
  1044. * - account:user : Grants access if the request is by the given user
  1045. * - .r:<regex> : Grants access if the request is from a referrer host that
  1046. * matches the expression and the request is not for a listing.
  1047. * Setting this to '*' effectively makes a container public.
  1048. * - .rlistings:<regex> : Grants access if the request is from a referrer host that
  1049. * matches the expression and the request for a listing.
  1050. *
  1051. * $writeGrps is a list of the possible criteria for a request to have
  1052. * access to write to a container. Each item is of the following format:
  1053. * - account:user : Grants access if the request is by the given user
  1054. *
  1055. * @see http://swift.openstack.org/misc.html#acls
  1056. *
  1057. * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
  1058. * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
  1059. *
  1060. * @param $contObj CF_Container Swift container
  1061. * @param $readGrps Array List of read access routes
  1062. * @param $writeGrps Array List of write access routes
  1063. * @return Status
  1064. */
  1065. protected function setContainerAccess(
  1066. CF_Container $contObj, array $readGrps, array $writeGrps
  1067. ) {
  1068. $creds = $contObj->cfs_auth->export_credentials();
  1069. $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
  1070. // Note: 10 second timeout consistent with php-cloudfiles
  1071. $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
  1072. $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
  1073. $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
  1074. $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
  1075. return $req->execute(); // should return 204
  1076. }
  1077. /**
  1078. * Purge the CDN cache of affected objects if CDN caching is enabled.
  1079. * This is for Rackspace/Akamai CDNs.
  1080. *
  1081. * @param $objects Array List of CF_Object items
  1082. * @return void
  1083. */
  1084. public function purgeCDNCache( array $objects ) {
  1085. if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
  1086. foreach ( $objects as $object ) {
  1087. try {
  1088. $object->purge_from_cdn();
  1089. } catch ( CDNNotEnabledException $e ) {
  1090. // CDN not enabled; nothing to see here
  1091. } catch ( CloudFilesException $e ) {
  1092. $this->handleException( $e, null, __METHOD__,
  1093. array( 'cont' => $object->container->name, 'obj' => $object->name ) );
  1094. }
  1095. }
  1096. }
  1097. }
  1098. /**
  1099. * Get an authenticated connection handle to the Swift proxy
  1100. *
  1101. * @return CF_Connection|bool False on failure
  1102. * @throws CloudFilesException
  1103. */
  1104. protected function getConnection() {
  1105. if ( $this->connException instanceof CloudFilesException ) {
  1106. if ( ( time() - $this->connErrorTime ) < 60 ) {
  1107. throw $this->connException; // failed last attempt; don't bother
  1108. } else { // actually retry this time
  1109. $this->connException = null;
  1110. $this->connErrorTime = 0;
  1111. }
  1112. }
  1113. // Session keys expire after a while, so we renew them periodically
  1114. $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
  1115. // Authenticate with proxy and get a session key...
  1116. if ( !$this->conn || $reAuth ) {
  1117. $this->sessionStarted = 0;
  1118. $this->connContainerCache->clear();
  1119. $cacheKey = $this->getCredsCacheKey( $this->auth->username );
  1120. $creds = $this->srvCache->get( $cacheKey ); // credentials
  1121. if ( is_array( $creds ) ) { // cache hit
  1122. $this->auth->load_cached_credentials(
  1123. $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
  1124. $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
  1125. } else { // cache miss
  1126. try {
  1127. $this->auth->authenticate();
  1128. $creds = $this->auth->export_credentials();
  1129. $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
  1130. $this->sessionStarted = time();
  1131. } catch ( CloudFilesException $e ) {
  1132. $this->connException = $e; // don't keep re-trying
  1133. $this->connErrorTime = time();
  1134. throw $e; // throw it back
  1135. }
  1136. }
  1137. if ( $this->conn ) { // re-authorizing?
  1138. $this->conn->close(); // close active cURL handles in CF_Http object
  1139. }
  1140. $this->conn = new CF_Connection( $this->auth );
  1141. }
  1142. return $this->conn;
  1143. }
  1144. /**
  1145. * Close the connection to the Swift proxy
  1146. *
  1147. * @return void
  1148. */
  1149. protected function closeConnection() {
  1150. if ( $this->conn ) {
  1151. $this->conn->close(); // close active cURL handles in CF_Http object
  1152. $this->sessionStarted = 0;
  1153. $this->connContainerCache->clear();
  1154. }
  1155. }
  1156. /**
  1157. * Get the cache key for a container
  1158. *
  1159. * @param $username string
  1160. * @return string
  1161. */
  1162. private function getCredsCacheKey( $username ) {
  1163. return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
  1164. }
  1165. /**
  1166. * @see FileBackendStore::doClearCache()
  1167. */
  1168. protected function doClearCache( array $paths = null ) {
  1169. $this->connContainerCache->clear(); // clear container object cache
  1170. }
  1171. /**
  1172. * Get a Swift container object, possibly from process cache.
  1173. * Use $reCache if the file count or byte count is needed.
  1174. *
  1175. * @param $container string Container name
  1176. * @param $bypassCache bool Bypass all caches and load from Swift
  1177. * @return CF_Container
  1178. * @throws CloudFilesException
  1179. */
  1180. protected function getContainer( $container, $bypassCache = false ) {
  1181. $conn = $this->getConnection(); // Swift proxy connection
  1182. if ( $bypassCache ) { // purge cache
  1183. $this->connContainerCache->clear( $container );
  1184. } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
  1185. $this->primeContainerCache( array( $container ) ); // check persistent cache
  1186. }
  1187. if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
  1188. $contObj = $conn->get_container( $container );
  1189. // NoSuchContainerException not thrown: container must exist
  1190. $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
  1191. if ( !$bypassCache ) {
  1192. $this->setContainerCache( $container, // update persistent cache
  1193. array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
  1194. );
  1195. }
  1196. }
  1197. return $this->connContainerCache->get( $container, 'obj' );
  1198. }
  1199. /**
  1200. * Create a Swift container
  1201. *
  1202. * @param $container string Container name
  1203. * @return CF_Container
  1204. * @throws CloudFilesException
  1205. */
  1206. protected function createContainer( $container ) {
  1207. $conn = $this->getConnection(); // Swift proxy connection
  1208. $contObj = $conn->create_container( $container );
  1209. $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
  1210. return $contObj;
  1211. }
  1212. /**
  1213. * Delete a Swift container
  1214. *
  1215. * @param $container string Container name
  1216. * @return void
  1217. * @throws CloudFilesException
  1218. */
  1219. protected function deleteContainer( $container ) {
  1220. $conn = $this->getConnection(); // Swift proxy connection
  1221. $this->connContainerCache->clear( $container ); // purge
  1222. $conn->delete_container( $container );
  1223. }
  1224. /**
  1225. * @see FileBackendStore::doPrimeContainerCache()
  1226. * @return void
  1227. */
  1228. protected function doPrimeContainerCache( array $containerInfo ) {
  1229. try {
  1230. $conn = $this->getConnection(); // Swift proxy connection
  1231. foreach ( $containerInfo as $container => $info ) {
  1232. $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
  1233. $container, $info['count'], $info['bytes'] );
  1234. $this->connContainerCache->set( $container, 'obj', $contObj );
  1235. }
  1236. } catch ( CloudFilesException $e ) { // some other exception?
  1237. $this->handleException( $e, null, __METHOD__, array() );
  1238. }
  1239. }
  1240. /**
  1241. * Log an unexpected exception for this backend.
  1242. * This also sets the Status object to have a fatal error.
  1243. *
  1244. * @param $e Exception
  1245. * @param $status Status|null
  1246. * @param $func string
  1247. * @param $params Array
  1248. * @return void
  1249. */
  1250. protected function handleException( Exception $e, $status, $func, array $params ) {
  1251. if ( $status instanceof Status ) {
  1252. if ( $e instanceof AuthenticationException ) {
  1253. $status->fatal( 'backend-fail-connect', $this->name );
  1254. } else {
  1255. $status->fatal( 'backend-fail-internal', $this->name );
  1256. }
  1257. }
  1258. if ( $e->getMessage() ) {
  1259. trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
  1260. }
  1261. if ( $e instanceof InvalidResponseException ) { // possibly a stale token
  1262. $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
  1263. $this->closeConnection(); // force a re-connect and re-auth next time
  1264. }
  1265. wfDebugLog( 'SwiftBackend',
  1266. get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
  1267. ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
  1268. );
  1269. }
  1270. }
  1271. /**
  1272. * @see FileBackendStoreOpHandle
  1273. */
  1274. class SwiftFileOpHandle extends FileBackendStoreOpHandle {
  1275. /** @var CF_Async_Op */
  1276. public $cfOp;
  1277. /** @var Array */
  1278. public $affectedObjects = array();
  1279. public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
  1280. $this->backend = $backend;
  1281. $this->params = $params;
  1282. $this->call = $call;
  1283. $this->cfOp = $cfOp;
  1284. }
  1285. }
  1286. /**
  1287. * SwiftFileBackend helper class to page through listings.
  1288. * Swift also has a listing limit of 10,000 objects for sanity.
  1289. * Do not use this class from places outside SwiftFileBackend.
  1290. *
  1291. * @ingroup FileBackend
  1292. */
  1293. abstract class SwiftFileBackendList implements Iterator {
  1294. /** @var Array */
  1295. protected $bufferIter = array();
  1296. protected $bufferAfter = null; // string; list items *after* this path
  1297. protected $pos = 0; // integer
  1298. /** @var Array */
  1299. protected $params = array();
  1300. /** @var SwiftFileBackend */
  1301. protected $backend;
  1302. protected $container; // string; container name
  1303. protected $dir; // string; storage directory
  1304. protected $suffixStart; // integer
  1305. const PAGE_SIZE = 9000; // file listing buffer size
  1306. /**
  1307. * @param $backend SwiftFileBackend
  1308. * @param $fullCont string Resolved container name
  1309. * @param $dir string Resolved directory relative to container
  1310. * @param $params Array
  1311. */
  1312. public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
  1313. $this->backend = $backend;
  1314. $this->

Large files files are truncated, but you can click here to view the full file