PageRenderTime 58ms CodeModel.GetById 23ms RepoModel.GetById 0ms app.codeStats 1ms

/indra/llmessage/llhttpassetstorage.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 1459 lines | 1117 code | 188 blank | 154 comment | 176 complexity | 54a6752195081a8582b6310bc265e3c4 MD5 | raw file
Possible License(s): LGPL-2.1
  1. /**
  2. * @file llhttpassetstorage.cpp
  3. * @brief Subclass capable of loading asset data to/from an external
  4. * source. Currently, a web server accessed via curl
  5. *
  6. * $LicenseInfo:firstyear=2003&license=viewerlgpl$
  7. * Second Life Viewer Source Code
  8. * Copyright (C) 2010, Linden Research, Inc.
  9. *
  10. * This library is free software; you can redistribute it and/or
  11. * modify it under the terms of the GNU Lesser General Public
  12. * License as published by the Free Software Foundation;
  13. * version 2.1 of the License only.
  14. *
  15. * This library is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * Lesser General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU Lesser General Public
  21. * License along with this library; if not, write to the Free Software
  22. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23. *
  24. * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
  25. * $/LicenseInfo$
  26. */
  27. #include "linden_common.h"
  28. #include "llhttpassetstorage.h"
  29. #include <sys/stat.h>
  30. #include "indra_constants.h"
  31. #include "message.h"
  32. #include "llproxy.h"
  33. #include "llvfile.h"
  34. #include "llvfs.h"
  35. #ifdef LL_STANDALONE
  36. # include <zlib.h>
  37. #else
  38. # include "zlib/zlib.h"
  39. #endif
  40. const U32 MAX_RUNNING_REQUESTS = 1;
  41. const F32 MAX_PROCESSING_TIME = 0.005f;
  42. const S32 CURL_XFER_BUFFER_SIZE = 65536;
  43. // Try for 30 minutes for now.
  44. const F32 GET_URL_TO_FILE_TIMEOUT = 1800.0f;
  45. const S32 COMPRESSED_INPUT_BUFFER_SIZE = 4096;
  46. const S32 HTTP_OK = 200;
  47. const S32 HTTP_PUT_OK = 201;
  48. const S32 HTTP_NO_CONTENT = 204;
  49. const S32 HTTP_MISSING = 404;
  50. const S32 HTTP_SERVER_BAD_GATEWAY = 502;
  51. const S32 HTTP_SERVER_TEMP_UNAVAILABLE = 503;
  52. /////////////////////////////////////////////////////////////////////////////////
  53. // LLTempAssetData
  54. // An asset not stored on central asset store, but on a simulator node somewhere.
  55. /////////////////////////////////////////////////////////////////////////////////
  56. struct LLTempAssetData
  57. {
  58. LLUUID mAssetID;
  59. LLUUID mAgentID;
  60. std::string mHostName;
  61. };
  62. /////////////////////////////////////////////////////////////////////////////////
  63. // LLHTTPAssetRequest
  64. /////////////////////////////////////////////////////////////////////////////////
  65. class LLHTTPAssetRequest : public LLAssetRequest
  66. {
  67. public:
  68. LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid,
  69. LLAssetType::EType type, LLAssetStorage::ERequestType rt,
  70. const std::string& url, CURLM *curl_multi);
  71. virtual ~LLHTTPAssetRequest();
  72. void setupCurlHandle();
  73. void cleanupCurlHandle();
  74. void prepareCompressedUpload();
  75. void finishCompressedUpload();
  76. size_t readCompressedData(void* data, size_t size);
  77. static size_t curlCompressedUploadCallback(
  78. void *data, size_t size, size_t nmemb, void *user_data);
  79. virtual LLSD getTerseDetails() const;
  80. virtual LLSD getFullDetails() const;
  81. public:
  82. LLHTTPAssetStorage *mAssetStoragep;
  83. CURL *mCurlHandle;
  84. CURLM *mCurlMultiHandle;
  85. std::string mURLBuffer;
  86. struct curl_slist *mHTTPHeaders;
  87. LLVFile *mVFile;
  88. LLUUID mTmpUUID;
  89. LLAssetStorage::ERequestType mRequestType;
  90. bool mZInitialized;
  91. z_stream mZStream;
  92. char* mZInputBuffer;
  93. bool mZInputExhausted;
  94. FILE *mFP;
  95. };
  96. LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp,
  97. const LLUUID &uuid,
  98. LLAssetType::EType type,
  99. LLAssetStorage::ERequestType rt,
  100. const std::string& url,
  101. CURLM *curl_multi)
  102. : LLAssetRequest(uuid, type),
  103. mZInitialized(false)
  104. {
  105. memset(&mZStream, 0, sizeof(mZStream)); // we'll initialize this later, but for now zero the whole C-style struct to avoid debug/coverity noise
  106. mAssetStoragep = asp;
  107. mCurlHandle = NULL;
  108. mCurlMultiHandle = curl_multi;
  109. mVFile = NULL;
  110. mRequestType = rt;
  111. mHTTPHeaders = NULL;
  112. mFP = NULL;
  113. mZInputBuffer = NULL;
  114. mZInputExhausted = false;
  115. mURLBuffer = url;
  116. }
  117. LLHTTPAssetRequest::~LLHTTPAssetRequest()
  118. {
  119. // Cleanup/cancel the request
  120. if (mCurlHandle)
  121. {
  122. curl_multi_remove_handle(mCurlMultiHandle, mCurlHandle);
  123. cleanupCurlHandle();
  124. }
  125. if (mHTTPHeaders)
  126. {
  127. curl_slist_free_all(mHTTPHeaders);
  128. }
  129. delete mVFile;
  130. finishCompressedUpload();
  131. }
  132. // virtual
  133. LLSD LLHTTPAssetRequest::getTerseDetails() const
  134. {
  135. LLSD sd = LLAssetRequest::getTerseDetails();
  136. sd["url"] = mURLBuffer;
  137. return sd;
  138. }
  139. // virtual
  140. LLSD LLHTTPAssetRequest::getFullDetails() const
  141. {
  142. LLSD sd = LLAssetRequest::getFullDetails();
  143. if (mCurlHandle)
  144. {
  145. long curl_response = -1;
  146. long curl_connect = -1;
  147. double curl_total_time = -1.0f;
  148. double curl_size_upload = -1.0f;
  149. double curl_size_download = -1.0f;
  150. double curl_content_length_upload = -1.0f;
  151. double curl_content_length_download = -1.0f;
  152. long curl_request_size = -1;
  153. const char* curl_content_type = NULL;
  154. curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response);
  155. curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect);
  156. curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time);
  157. curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD, &curl_size_upload);
  158. curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download);
  159. curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD, &curl_content_length_upload);
  160. curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download);
  161. curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size);
  162. curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type);
  163. sd["curl_response_code"] = (int) curl_response;
  164. sd["curl_http_connect_code"] = (int) curl_connect;
  165. sd["curl_total_time"] = curl_total_time;
  166. sd["curl_size_upload"] = curl_size_upload;
  167. sd["curl_size_download"] = curl_size_download;
  168. sd["curl_content_length_upload"] = curl_content_length_upload;
  169. sd["curl_content_length_download"] = curl_content_length_download;
  170. sd["curl_request_size"] = (int) curl_request_size;
  171. if (curl_content_type)
  172. {
  173. sd["curl_content_type"] = curl_content_type;
  174. }
  175. else
  176. {
  177. sd["curl_content_type"] = "";
  178. }
  179. }
  180. sd["temp_id"] = mTmpUUID;
  181. sd["request_type"] = LLAssetStorage::getRequestName(mRequestType);
  182. sd["z_initialized"] = mZInitialized;
  183. sd["z_input_exhausted"] = mZInputExhausted;
  184. S32 file_size = -1;
  185. if (mFP)
  186. {
  187. struct stat file_stat;
  188. int file_desc = fileno(mFP);
  189. if ( fstat(file_desc, &file_stat) == 0)
  190. {
  191. file_size = file_stat.st_size;
  192. }
  193. }
  194. sd["file_size"] = file_size;
  195. return sd;
  196. }
  197. void LLHTTPAssetRequest::setupCurlHandle()
  198. {
  199. // *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC
  200. mCurlHandle = LLCurl::newEasyHandle();
  201. llassert_always(mCurlHandle != NULL) ;
  202. // Apply proxy settings if configured to do so
  203. LLProxy::getInstance()->applyProxySettings(mCurlHandle);
  204. curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
  205. curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
  206. curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer.c_str());
  207. curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
  208. if (LLAssetStorage::RT_DOWNLOAD == mRequestType)
  209. {
  210. curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
  211. // only do this on downloads, as uploads
  212. // to some apache configs (like our test grids)
  213. // mistakenly claim the response is gzip'd if the resource
  214. // name ends in .gz, even though in a PUT, the response is
  215. // just plain HTML saying "created"
  216. }
  217. /* Remove the Pragma: no-cache header that libcurl inserts by default;
  218. we want the cached version, if possible. */
  219. if (mZInitialized)
  220. {
  221. curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, "");
  222. // disable use of proxy, which can't handle chunked transfers
  223. }
  224. mHTTPHeaders = curl_slist_append(mHTTPHeaders, "Pragma:");
  225. // bug in curl causes DNS to be cached for too long a time, 0 sets it to never cache DNS results internally (to curl)
  226. curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
  227. // resist the temptation to explicitly add the Transfer-Encoding: chunked
  228. // header here - invokes a libCURL bug
  229. curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mHTTPHeaders);
  230. if (mAssetStoragep)
  231. {
  232. // Set the appropriate pending upload or download flag
  233. mAssetStoragep->addRunningRequest(mRequestType, this);
  234. }
  235. else
  236. {
  237. llerrs << "LLHTTPAssetRequest::setupCurlHandle - No asset storage associated with this request!" << llendl;
  238. }
  239. }
  240. void LLHTTPAssetRequest::cleanupCurlHandle()
  241. {
  242. LLCurl::deleteEasyHandle(mCurlHandle);
  243. if (mAssetStoragep)
  244. {
  245. // Terminating a request. Thus upload or download is no longer pending.
  246. mAssetStoragep->removeRunningRequest(mRequestType, this);
  247. }
  248. else
  249. {
  250. llerrs << "LLHTTPAssetRequest::~LLHTTPAssetRequest - No asset storage associated with this request!" << llendl;
  251. }
  252. mCurlHandle = NULL;
  253. }
  254. void LLHTTPAssetRequest::prepareCompressedUpload()
  255. {
  256. mZStream.next_in = Z_NULL;
  257. mZStream.avail_in = 0;
  258. mZStream.zalloc = Z_NULL;
  259. mZStream.zfree = Z_NULL;
  260. mZStream.opaque = Z_NULL;
  261. int r = deflateInit2(&mZStream,
  262. 1, // compression level
  263. Z_DEFLATED, // the only method defined
  264. 15 + 16, // the default windowBits + gzip header flag
  265. 8, // the default memLevel
  266. Z_DEFAULT_STRATEGY);
  267. if (r != Z_OK)
  268. {
  269. llerrs << "LLHTTPAssetRequest::prepareCompressedUpload defalateInit2() failed" << llendl;
  270. }
  271. mZInitialized = true;
  272. mZInputBuffer = new char[COMPRESSED_INPUT_BUFFER_SIZE];
  273. mZInputExhausted = false;
  274. mVFile = new LLVFile(gAssetStorage->mVFS,
  275. getUUID(), getType(), LLVFile::READ);
  276. }
  277. void LLHTTPAssetRequest::finishCompressedUpload()
  278. {
  279. if (mZInitialized)
  280. {
  281. llinfos << "LLHTTPAssetRequest::finishCompressedUpload: "
  282. << "read " << mZStream.total_in << " byte asset file, "
  283. << "uploaded " << mZStream.total_out << " byte compressed asset"
  284. << llendl;
  285. deflateEnd(&mZStream);
  286. delete[] mZInputBuffer;
  287. }
  288. }
  289. size_t LLHTTPAssetRequest::readCompressedData(void* data, size_t size)
  290. {
  291. llassert(mZInitialized);
  292. mZStream.next_out = (Bytef*)data;
  293. mZStream.avail_out = size;
  294. while (mZStream.avail_out > 0)
  295. {
  296. if (mZStream.avail_in == 0 && !mZInputExhausted)
  297. {
  298. S32 to_read = llmin(COMPRESSED_INPUT_BUFFER_SIZE,
  299. (S32)(mVFile->getSize() - mVFile->tell()));
  300. if ( to_read > 0 )
  301. {
  302. mVFile->read((U8*)mZInputBuffer, to_read); /*Flawfinder: ignore*/
  303. mZStream.next_in = (Bytef*)mZInputBuffer;
  304. mZStream.avail_in = mVFile->getLastBytesRead();
  305. }
  306. mZInputExhausted = mZStream.avail_in == 0;
  307. }
  308. int r = deflate(&mZStream,
  309. mZInputExhausted ? Z_FINISH : Z_NO_FLUSH);
  310. if (r == Z_STREAM_END || r < 0 || mZInputExhausted)
  311. {
  312. if (r < 0)
  313. {
  314. llwarns << "LLHTTPAssetRequest::readCompressedData: deflate returned error code "
  315. << (S32) r << llendl;
  316. }
  317. break;
  318. }
  319. }
  320. return size - mZStream.avail_out;
  321. }
  322. //static
  323. size_t LLHTTPAssetRequest::curlCompressedUploadCallback(
  324. void *data, size_t size, size_t nmemb, void *user_data)
  325. {
  326. size_t num_read = 0;
  327. if (gAssetStorage)
  328. {
  329. CURL *curl_handle = (CURL *)user_data;
  330. LLHTTPAssetRequest *req = NULL;
  331. curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
  332. if (req)
  333. {
  334. num_read = req->readCompressedData(data, size * nmemb);
  335. }
  336. }
  337. return num_read;
  338. }
  339. /////////////////////////////////////////////////////////////////////////////////
  340. // LLHTTPAssetStorage
  341. /////////////////////////////////////////////////////////////////////////////////
  342. LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
  343. LLVFS *vfs, LLVFS *static_vfs,
  344. const LLHost &upstream_host,
  345. const std::string& web_host,
  346. const std::string& local_web_host,
  347. const std::string& host_name)
  348. : LLAssetStorage(msg, xfer, vfs, static_vfs, upstream_host)
  349. {
  350. _init(web_host, local_web_host, host_name);
  351. }
  352. LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
  353. LLVFS *vfs,
  354. LLVFS *static_vfs,
  355. const std::string& web_host,
  356. const std::string& local_web_host,
  357. const std::string& host_name)
  358. : LLAssetStorage(msg, xfer, vfs, static_vfs)
  359. {
  360. _init(web_host, local_web_host, host_name);
  361. }
  362. void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& local_web_host, const std::string& host_name)
  363. {
  364. mBaseURL = web_host;
  365. mLocalBaseURL = local_web_host;
  366. mHostName = host_name;
  367. // curl_global_init moved to LLCurl::initClass()
  368. mCurlMultiHandle = LLCurl::newMultiHandle() ;
  369. llassert_always(mCurlMultiHandle != NULL) ;
  370. }
  371. LLHTTPAssetStorage::~LLHTTPAssetStorage()
  372. {
  373. LLCurl::deleteMultiHandle(mCurlMultiHandle);
  374. mCurlMultiHandle = NULL;
  375. // curl_global_cleanup moved to LLCurl::initClass()
  376. }
  377. // storing data is simpler than getting it, so we just overload the whole method
  378. void LLHTTPAssetStorage::storeAssetData(
  379. const LLUUID& uuid,
  380. LLAssetType::EType type,
  381. LLAssetStorage::LLStoreAssetCallback callback,
  382. void* user_data,
  383. bool temp_file,
  384. bool is_priority,
  385. bool store_local,
  386. const LLUUID& requesting_agent_id,
  387. bool user_waiting,
  388. F64 timeout)
  389. {
  390. if (mVFS->getExists(uuid, type)) // VFS treats nonexistant and zero-length identically
  391. {
  392. LLAssetRequest *req = new LLAssetRequest(uuid, type);
  393. req->mUpCallback = callback;
  394. req->mUserData = user_data;
  395. req->mRequestingAgentID = requesting_agent_id;
  396. req->mIsUserWaiting = user_waiting;
  397. req->mTimeout = timeout;
  398. // LLAssetStorage metric: Successful Request
  399. S32 size = mVFS->getSize(uuid, type);
  400. const char *message;
  401. if( store_local )
  402. {
  403. message = "Added to local upload queue";
  404. }
  405. else
  406. {
  407. message = "Added to upload queue";
  408. }
  409. reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, size, MR_OKAY, __FILE__, __LINE__, message );
  410. // this will get picked up and transmitted in checkForTimeouts
  411. if(store_local)
  412. {
  413. mPendingLocalUploads.push_back(req);
  414. }
  415. else if(is_priority)
  416. {
  417. mPendingUploads.push_front(req);
  418. }
  419. else
  420. {
  421. mPendingUploads.push_back(req);
  422. }
  423. }
  424. else
  425. {
  426. llwarns << "AssetStorage: attempt to upload non-existent vfile " << uuid << ":" << LLAssetType::lookup(type) << llendl;
  427. if (callback)
  428. {
  429. // LLAssetStorage metric: Zero size VFS
  430. reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file didn't exist or was zero length (VFS - can't tell which)" );
  431. callback(uuid, user_data, LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LL_EXSTAT_NONEXISTENT_FILE);
  432. }
  433. }
  434. }
  435. // virtual
  436. void LLHTTPAssetStorage::storeAssetData(
  437. const std::string& filename,
  438. const LLUUID& asset_id,
  439. LLAssetType::EType asset_type,
  440. LLStoreAssetCallback callback,
  441. void* user_data,
  442. bool temp_file,
  443. bool is_priority,
  444. bool user_waiting,
  445. F64 timeout)
  446. {
  447. llinfos << "LLAssetStorage::storeAssetData (legacy)" << asset_id << ":" << LLAssetType::lookup(asset_type) << llendl;
  448. LLLegacyAssetRequest *legacy = new LLLegacyAssetRequest;
  449. legacy->mUpCallback = callback;
  450. legacy->mUserData = user_data;
  451. FILE *fp = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/
  452. S32 size = 0;
  453. if (fp)
  454. {
  455. fseek(fp, 0, SEEK_END);
  456. size = ftell(fp);
  457. fseek(fp, 0, SEEK_SET);
  458. }
  459. if( size )
  460. {
  461. LLVFile file(mVFS, asset_id, asset_type, LLVFile::WRITE);
  462. file.setMaxSize(size);
  463. const S32 buf_size = 65536;
  464. U8 copy_buf[buf_size];
  465. while ((size = (S32)fread(copy_buf, 1, buf_size, fp)))
  466. {
  467. file.write(copy_buf, size);
  468. }
  469. fclose(fp);
  470. // if this upload fails, the caller needs to setup a new tempfile for us
  471. if (temp_file)
  472. {
  473. LLFile::remove(filename);
  474. }
  475. // LLAssetStorage metric: Success not needed; handled in the overloaded method here:
  476. storeAssetData(
  477. asset_id,
  478. asset_type,
  479. legacyStoreDataCallback,
  480. (void**)legacy,
  481. temp_file,
  482. is_priority,
  483. false,
  484. LLUUID::null,
  485. user_waiting,
  486. timeout);
  487. }
  488. else // !size
  489. {
  490. if( fp )
  491. {
  492. // LLAssetStorage metric: Zero size
  493. reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file was zero length" );
  494. fclose( fp );
  495. }
  496. else
  497. {
  498. // LLAssetStorage metric: Missing File
  499. reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_FILE_NONEXIST, __FILE__, __LINE__, "The file didn't exist" );
  500. }
  501. if (callback)
  502. {
  503. callback(LLUUID::null, user_data, LL_ERR_CANNOT_OPEN_FILE, LL_EXSTAT_BLOCKED_FILE);
  504. }
  505. delete legacy;
  506. }
  507. }
  508. // virtual
  509. LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt,
  510. LLAssetType::EType asset_type,
  511. const std::string& detail_prefix) const
  512. {
  513. LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix);
  514. const request_list_t* running = getRunningList(rt);
  515. if (running)
  516. {
  517. // Loop through the pending requests sd, and add extra info about its running status.
  518. S32 num_pending = sd["requests"].size();
  519. S32 i;
  520. for (i = 0; i < num_pending; ++i)
  521. {
  522. LLSD& pending = sd["requests"][i];
  523. // See if this pending request is running.
  524. const LLAssetRequest* req = findRequest(running,
  525. LLAssetType::lookup(pending["type"].asString()),
  526. pending["asset_id"]);
  527. if (req)
  528. {
  529. // Keep the detail_url so we don't have to rebuild it.
  530. LLURI detail_url = pending["detail"];
  531. pending = req->getTerseDetails();
  532. pending["detail"] = detail_url;
  533. pending["is_running"] = true;
  534. }
  535. else
  536. {
  537. pending["is_running"] = false;
  538. }
  539. }
  540. }
  541. return sd;
  542. }
  543. // virtual
  544. LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt,
  545. LLAssetType::EType asset_type,
  546. const LLUUID& asset_id) const
  547. {
  548. // Look for this asset in the running list first.
  549. const request_list_t* running = getRunningList(rt);
  550. if (running)
  551. {
  552. LLSD sd = LLAssetStorage::getPendingRequestImpl(running, asset_type, asset_id);
  553. if (sd)
  554. {
  555. sd["is_running"] = true;
  556. return sd;
  557. }
  558. }
  559. LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id);
  560. if (sd)
  561. {
  562. sd["is_running"] = false;
  563. }
  564. return sd;
  565. }
  566. // virtual
  567. bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt,
  568. LLAssetType::EType asset_type,
  569. const LLUUID& asset_id)
  570. {
  571. // Try removing this from the running list first.
  572. request_list_t* running = getRunningList(rt);
  573. if (running)
  574. {
  575. LLAssetRequest* req = findRequest(running, asset_type, asset_id);
  576. if (req)
  577. {
  578. // Remove this request from the running list to get it out of curl.
  579. running->remove(req);
  580. // Find this request in the pending list, so we can move it to the end of the line.
  581. request_list_t* pending = getRequestList(rt);
  582. if (pending)
  583. {
  584. request_list_t::iterator result = std::find_if(pending->begin(), pending->end(),
  585. std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req));
  586. if (pending->end() != result)
  587. {
  588. // This request was found in the pending list. Move it to the end!
  589. LLAssetRequest* pending_req = *result;
  590. pending->remove(pending_req);
  591. if (!pending_req->mIsUserWaiting) //A user is waiting on this request. Toss it.
  592. {
  593. pending->push_back(pending_req);
  594. }
  595. else
  596. {
  597. if (pending_req->mUpCallback) //Clean up here rather than _callUploadCallbacks because this request is already cleared the req.
  598. {
  599. pending_req->mUpCallback(pending_req->getUUID(), pending_req->mUserData, -1, LL_EXSTAT_REQUEST_DROPPED);
  600. }
  601. }
  602. llinfos << "Asset " << getRequestName(rt) << " request for "
  603. << asset_id << "." << LLAssetType::lookup(asset_type)
  604. << " removed from curl and placed at the end of the pending queue."
  605. << llendl;
  606. }
  607. else
  608. {
  609. llwarns << "Unable to find pending " << getRequestName(rt) << " request for "
  610. << asset_id << "." << LLAssetType::lookup(asset_type) << llendl;
  611. }
  612. }
  613. delete req;
  614. return true;
  615. }
  616. }
  617. return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id);
  618. }
  619. // internal requester, used by getAssetData in superclass
  620. void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type,
  621. void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat),
  622. void *user_data, BOOL duplicate,
  623. BOOL is_priority)
  624. {
  625. // stash the callback info so we can find it after we get the response message
  626. LLAssetRequest *req = new LLAssetRequest(uuid, type);
  627. req->mDownCallback = callback;
  628. req->mUserData = user_data;
  629. req->mIsPriority = is_priority;
  630. // this will get picked up and downloaded in checkForTimeouts
  631. //
  632. // HAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACK! Asset requests were taking too long and timing out.
  633. // Since texture requests are the LEAST sensitive (on the simulator) to being delayed, add
  634. // non-texture requests to the front, and add texture requests to the back. The theory is
  635. // that we always want them first, even if they're out of order.
  636. //
  637. if (req->getType() == LLAssetType::AT_TEXTURE)
  638. {
  639. mPendingDownloads.push_back(req);
  640. }
  641. else
  642. {
  643. mPendingDownloads.push_front(req);
  644. }
  645. }
  646. LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending,
  647. LLAssetStorage::request_list_t& running)
  648. {
  649. // Early exit if the running list is full, or we don't have more pending than running.
  650. if (running.size() >= MAX_RUNNING_REQUESTS
  651. || pending.size() <= running.size()) return NULL;
  652. // Look for the first pending request that is not already running.
  653. request_list_t::iterator running_begin = running.begin();
  654. request_list_t::iterator running_end = running.end();
  655. request_list_t::iterator pending_iter = pending.begin();
  656. request_list_t::iterator pending_end = pending.end();
  657. // Loop over all pending requests until we miss finding it in the running list.
  658. for (; pending_iter != pending.end(); ++pending_iter)
  659. {
  660. LLAssetRequest* req = *pending_iter;
  661. // Look for this pending request in the running list.
  662. if (running_end == std::find_if(running_begin, running_end,
  663. std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)))
  664. {
  665. // It isn't running! Return it.
  666. return req;
  667. }
  668. }
  669. return NULL;
  670. }
  671. // overloaded to additionally move data to/from the webserver
  672. void LLHTTPAssetStorage::checkForTimeouts()
  673. {
  674. CURLMcode mcode;
  675. LLAssetRequest *req;
  676. while ( (req = findNextRequest(mPendingDownloads, mRunningDownloads)) )
  677. {
  678. // Setup this curl download request
  679. // We need to generate a new request here
  680. // since the one in the list could go away
  681. std::string tmp_url;
  682. std::string uuid_str;
  683. req->getUUID().toString(uuid_str);
  684. std::string base_url = getBaseURL(req->getUUID(), req->getType());
  685. tmp_url = llformat("%s/%36s.%s", base_url.c_str() , uuid_str.c_str(), LLAssetType::lookup(req->getType()));
  686. LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(),
  687. req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle);
  688. new_req->mTmpUUID.generate();
  689. // Sets pending download flag internally
  690. new_req->setupCurlHandle();
  691. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
  692. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback);
  693. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle);
  694. mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
  695. if (mcode > CURLM_OK)
  696. {
  697. // Failure. Deleting the pending request will remove it from the running
  698. // queue, and push it to the end of the pending queue.
  699. new_req->cleanupCurlHandle();
  700. deletePendingRequest(RT_DOWNLOAD, new_req->getType(), new_req->getUUID());
  701. break;
  702. }
  703. else
  704. {
  705. llinfos << "Requesting " << new_req->mURLBuffer << llendl;
  706. }
  707. }
  708. while ( (req = findNextRequest(mPendingUploads, mRunningUploads)) )
  709. {
  710. // setup this curl upload request
  711. bool do_compress = req->getType() == LLAssetType::AT_OBJECT;
  712. std::string tmp_url;
  713. std::string uuid_str;
  714. req->getUUID().toString(uuid_str);
  715. tmp_url = mBaseURL + "/" + uuid_str + "." + LLAssetType::lookup(req->getType());
  716. if (do_compress) tmp_url += ".gz";
  717. LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(),
  718. req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle);
  719. if (req->mIsUserWaiting) //If a user is waiting on a realtime response, we want to perserve information across upload attempts.
  720. {
  721. new_req->mTime = req->mTime;
  722. new_req->mTimeout = req->mTimeout;
  723. new_req->mIsUserWaiting = req->mIsUserWaiting;
  724. }
  725. if (do_compress)
  726. {
  727. new_req->prepareCompressedUpload();
  728. }
  729. // Sets pending upload flag internally
  730. new_req->setupCurlHandle();
  731. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_UPLOAD, 1);
  732. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);
  733. if (do_compress)
  734. {
  735. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
  736. &LLHTTPAssetRequest::curlCompressedUploadCallback);
  737. }
  738. else
  739. {
  740. LLVFile file(mVFS, req->getUUID(), req->getType());
  741. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
  742. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
  743. &curlUpCallback);
  744. }
  745. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
  746. mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
  747. if (mcode > CURLM_OK)
  748. {
  749. // Failure. Deleting the pending request will remove it from the running
  750. // queue, and push it to the end of the pending queue.
  751. new_req->cleanupCurlHandle();
  752. deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());
  753. break;
  754. }
  755. else
  756. {
  757. // Get the uncompressed file size.
  758. LLVFile file(mVFS,new_req->getUUID(),new_req->getType());
  759. S32 size = file.getSize();
  760. llinfos << "Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << llendl;
  761. if (size == 0)
  762. {
  763. llwarns << "Rejecting zero size PUT request!" << llendl;
  764. new_req->cleanupCurlHandle();
  765. deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());
  766. }
  767. }
  768. // Pending upload will have been flagged by the request
  769. }
  770. while ( (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) )
  771. {
  772. // setup this curl upload request
  773. LLVFile file(mVFS, req->getUUID(), req->getType());
  774. std::string tmp_url;
  775. std::string uuid_str;
  776. req->getUUID().toString(uuid_str);
  777. // KLW - All temporary uploads are saved locally "http://localhost:12041/asset"
  778. tmp_url = llformat("%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str.c_str(), LLAssetType::lookup(req->getType()));
  779. LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(),
  780. req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle);
  781. new_req->mRequestingAgentID = req->mRequestingAgentID;
  782. // Sets pending upload flag internally
  783. new_req->setupCurlHandle();
  784. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_PUT, 1);
  785. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
  786. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);
  787. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback);
  788. curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
  789. mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
  790. if (mcode > CURLM_OK)
  791. {
  792. // Failure. Deleting the pending request will remove it from the running
  793. // queue, and push it to the end of the pending queue.
  794. new_req->cleanupCurlHandle();
  795. deletePendingRequest(RT_LOCALUPLOAD, new_req->getType(), new_req->getUUID());
  796. break;
  797. }
  798. else
  799. {
  800. // Get the uncompressed file size.
  801. S32 size = file.getSize();
  802. llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!"
  803. << " Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << llendl;
  804. if (size == 0)
  805. {
  806. llwarns << "Rejecting zero size PUT request!" << llendl;
  807. new_req->cleanupCurlHandle();
  808. deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());
  809. }
  810. }
  811. // Pending upload will have been flagged by the request
  812. }
  813. S32 count = 0;
  814. int queue_length;
  815. do
  816. {
  817. mcode = curl_multi_perform(mCurlMultiHandle, &queue_length);
  818. count++;
  819. } while (mcode == CURLM_CALL_MULTI_PERFORM && (count < 5));
  820. CURLMsg *curl_msg;
  821. do
  822. {
  823. curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);
  824. if (curl_msg && curl_msg->msg == CURLMSG_DONE)
  825. {
  826. long curl_result = 0;
  827. S32 xfer_result = LL_ERR_NOERR;
  828. LLHTTPAssetRequest *req = NULL;
  829. curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req);
  830. // TODO: Throw curl_result at all callbacks.
  831. curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
  832. if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType)
  833. {
  834. if (curl_msg->data.result == CURLE_OK &&
  835. ( curl_result == HTTP_OK
  836. || curl_result == HTTP_PUT_OK
  837. || curl_result == HTTP_NO_CONTENT))
  838. {
  839. llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl;
  840. if (RT_LOCALUPLOAD == req->mRequestType)
  841. {
  842. addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName);
  843. }
  844. }
  845. else if (curl_msg->data.result == CURLE_COULDNT_CONNECT ||
  846. curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
  847. curl_result == HTTP_SERVER_BAD_GATEWAY ||
  848. curl_result == HTTP_SERVER_TEMP_UNAVAILABLE)
  849. {
  850. llwarns << "Re-requesting upload for " << req->getUUID() << ". Received upload error to " << req->mURLBuffer <<
  851. " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
  852. ////HACK (probably) I am sick of this getting requeued and driving me mad.
  853. //if (req->mIsUserWaiting)
  854. //{
  855. // deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
  856. //}
  857. }
  858. else
  859. {
  860. llwarns << "Failure uploading " << req->getUUID() << " to " << req->mURLBuffer <<
  861. " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
  862. xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
  863. }
  864. if (!(curl_msg->data.result == CURLE_COULDNT_CONNECT ||
  865. curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
  866. curl_result == HTTP_SERVER_BAD_GATEWAY ||
  867. curl_result == HTTP_SERVER_TEMP_UNAVAILABLE))
  868. {
  869. // shared upload finished callback
  870. // in the base class, this is called from processUploadComplete
  871. _callUploadCallbacks(req->getUUID(), req->getType(), (xfer_result == 0), LL_EXSTAT_CURL_RESULT | curl_result);
  872. // Pending upload flag will get cleared when the request is deleted
  873. }
  874. }
  875. else if (RT_DOWNLOAD == req->mRequestType)
  876. {
  877. if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
  878. {
  879. if (req->mVFile && req->mVFile->getSize() > 0)
  880. {
  881. llinfos << "Success downloading " << req->mURLBuffer << ", size " << req->mVFile->getSize() << llendl;
  882. req->mVFile->rename(req->getUUID(), req->getType());
  883. }
  884. else
  885. {
  886. // *TODO: if this actually indicates a bad asset on the server
  887. // (not certain at this point), then delete it
  888. llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl;
  889. xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
  890. }
  891. }
  892. else
  893. {
  894. // KLW - TAT See if an avatar owns this texture, and if so request re-upload.
  895. llwarns << "Failure downloading " << req->mURLBuffer <<
  896. " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
  897. xfer_result = (curl_result == HTTP_MISSING) ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;
  898. if (req->mVFile)
  899. {
  900. req->mVFile->remove();
  901. }
  902. }
  903. // call the static callback for transfer completion
  904. // this will cleanup all requests for this asset, including ours
  905. downloadCompleteCallback(
  906. xfer_result,
  907. req->getUUID(),
  908. req->getType(),
  909. (void *)req,
  910. LL_EXSTAT_CURL_RESULT | curl_result);
  911. // Pending download flag will get cleared when the request is deleted
  912. }
  913. else
  914. {
  915. // nothing, just axe this request
  916. // currently this can only mean an asset delete
  917. }
  918. // Deleting clears the pending upload/download flag if it's set and the request is transferring
  919. delete req;
  920. req = NULL;
  921. }
  922. } while (curl_msg && queue_length > 0);
  923. // Cleanup
  924. // We want to bump to the back of the line any running uploads that have timed out.
  925. bumpTimedOutUploads();
  926. LLAssetStorage::checkForTimeouts();
  927. }
  928. void LLHTTPAssetStorage::bumpTimedOutUploads()
  929. {
  930. bool user_waiting=FALSE;
  931. F64 mt_secs = LLMessageSystem::getMessageTimeSeconds();
  932. if (mPendingUploads.size())
  933. {
  934. request_list_t::iterator it = mPendingUploads.begin();
  935. LLAssetRequest* req = *it;
  936. user_waiting=req->mIsUserWaiting;
  937. }
  938. // No point bumping currently running uploads if there are no others in line.
  939. if (!(mPendingUploads.size() > mRunningUploads.size()) && !user_waiting)
  940. {
  941. return;
  942. }
  943. // deletePendingRequest will modify the mRunningUploads list so we don't want to iterate over it.
  944. request_list_t temp_running = mRunningUploads;
  945. request_list_t::iterator it = temp_running.begin();
  946. request_list_t::iterator end = temp_running.end();
  947. for ( ; it != end; ++it)
  948. {
  949. //request_list_t::iterator curiter = iter++;
  950. LLAssetRequest* req = *it;
  951. if ( req->mTimeout < (mt_secs - req->mTime) )
  952. {
  953. llwarns << "Asset upload request timed out for "
  954. << req->getUUID() << "."
  955. << LLAssetType::lookup(req->getType())
  956. << ", bumping to the back of the line!" << llendl;
  957. deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
  958. }
  959. }
  960. }
  961. // static
  962. size_t LLHTTPAssetStorage::curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
  963. {
  964. if (!gAssetStorage)
  965. {
  966. llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
  967. return 0;
  968. }
  969. S32 bytes = (S32)(size * nmemb);
  970. CURL *curl_handle = (CURL *)user_data;
  971. LLHTTPAssetRequest *req = NULL;
  972. curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
  973. if (! req->mVFile)
  974. {
  975. req->mVFile = new LLVFile(gAssetStorage->mVFS, req->mTmpUUID, LLAssetType::AT_NONE, LLVFile::APPEND);
  976. }
  977. double content_length = 0.0;
  978. curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length);
  979. // sanitize content_length, reconcile w/ actual data
  980. S32 file_length = llmax(0, (S32)llmin(content_length, 20000000.0), bytes + req->mVFile->getSize());
  981. req->mVFile->setMaxSize(file_length);
  982. req->mVFile->write((U8*)data, bytes);
  983. return nmemb;
  984. }
  985. // static
  986. size_t LLHTTPAssetStorage::curlUpCallback(void *data, size_t size, size_t nmemb, void *user_data)
  987. {
  988. if (!gAssetStorage)
  989. {
  990. llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
  991. return 0;
  992. }
  993. CURL *curl_handle = (CURL *)user_data;
  994. LLHTTPAssetRequest *req = NULL;
  995. curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
  996. if (! req->mVFile)
  997. {
  998. req->mVFile = new LLVFile(gAssetStorage->mVFS, req->getUUID(), req->getType(), LLVFile::READ);
  999. }
  1000. S32 bytes = llmin((S32)(size * nmemb), (S32)(req->mVFile->getSize() - req->mVFile->tell()));
  1001. req->mVFile->read((U8*)data, bytes);/*Flawfinder: ignore*/
  1002. return req->mVFile->getLastBytesRead();
  1003. }
  1004. // static
  1005. size_t LLHTTPAssetStorage::nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data)
  1006. {
  1007. // do nothing, this is here to soak up script output so it doesn't end up on stdout
  1008. return nmemb;
  1009. }
  1010. // blocking asset fetch which bypasses the VFS
  1011. // this is a very limited function for use by the simstate loader and other one-offs
  1012. S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const std::string &url, const std::string& filename, progress_callback callback, void *userdata)
  1013. {
  1014. // *NOTE: There is no guarantee that the uuid and the asset_type match
  1015. // - not that it matters. - Doug
  1016. lldebugs << "LLHTTPAssetStorage::getURLToFile() - " << url << llendl;
  1017. FILE *fp = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/
  1018. if (! fp)
  1019. {
  1020. llwarns << "Failed to open " << filename << " for writing" << llendl;
  1021. return LL_ERR_ASSET_REQUEST_FAILED;
  1022. }
  1023. // make sure we use the normal curl setup, even though we don't really need a request object
  1024. LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url, mCurlMultiHandle);
  1025. req.mFP = fp;
  1026. req.setupCurlHandle();
  1027. curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
  1028. curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEFUNCTION, &curlFileDownCallback);
  1029. curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEDATA, req.mCurlHandle);
  1030. curl_multi_add_handle(mCurlMultiHandle, req.mCurlHandle);
  1031. llinfos << "Requesting as file " << req.mURLBuffer << llendl;
  1032. // braindead curl loop
  1033. int queue_length;
  1034. CURLMsg *curl_msg;
  1035. LLTimer timeout;
  1036. timeout.setTimerExpirySec(GET_URL_TO_FILE_TIMEOUT);
  1037. bool success = false;
  1038. S32 xfer_result = 0;
  1039. do
  1040. {
  1041. curl_multi_perform(mCurlMultiHandle, &queue_length);
  1042. curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);
  1043. if (callback)
  1044. {
  1045. callback(userdata);
  1046. }
  1047. if ( curl_msg && (CURLMSG_DONE == curl_msg->msg) )
  1048. {
  1049. success = true;
  1050. }
  1051. else if (timeout.hasExpired())
  1052. {
  1053. llwarns << "Request for " << url << " has timed out." << llendl;
  1054. success = false;
  1055. xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
  1056. break;
  1057. }
  1058. } while (!success);
  1059. if (success)
  1060. {
  1061. long curl_result = 0;
  1062. curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
  1063. if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
  1064. {
  1065. S32 size = ftell(req.mFP);
  1066. if (size > 0)
  1067. {
  1068. // everything seems to be in order
  1069. llinfos << "Success downloading " << req.mURLBuffer << " to file, size " << size << llendl;
  1070. }
  1071. else
  1072. {
  1073. llwarns << "Found " << req.mURLBuffer << " to be zero size" << llendl;
  1074. xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
  1075. }
  1076. }
  1077. else
  1078. {
  1079. xfer_result = curl_result == HTTP_MISSING ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;
  1080. llinfos << "Failure downloading " << req.mURLBuffer <<
  1081. " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
  1082. }
  1083. }
  1084. fclose(fp);
  1085. if (xfer_result)
  1086. {
  1087. LLFile::remove(filename);
  1088. }
  1089. return xfer_result;
  1090. }
  1091. // static
  1092. size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
  1093. {
  1094. CURL *curl_handle = (CURL *)user_data;
  1095. LLHTTPAssetRequest *req = NULL;
  1096. curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
  1097. if (! req->mFP)
  1098. {
  1099. llwarns << "Missing mFP, aborting curl file download callback!" << llendl;
  1100. return 0;
  1101. }
  1102. return fwrite(data, size, nmemb, req->mFP);
  1103. }
  1104. LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt)
  1105. {
  1106. switch (rt)
  1107. {
  1108. case RT_DOWNLOAD:
  1109. return &mRunningDownloads;
  1110. case RT_UPLOAD:
  1111. return &mRunningUploads;
  1112. case RT_LOCALUPLOAD:
  1113. return &mRunningLocalUploads;
  1114. default:
  1115. return NULL;
  1116. }
  1117. }
  1118. const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const
  1119. {
  1120. switch (rt)
  1121. {
  1122. case RT_DOWNLOAD:
  1123. return &mRunningDownloads;
  1124. case RT_UPLOAD:
  1125. return &mRunningUploads;
  1126. case RT_LOCALUPLOAD:
  1127. return &mRunningLocalUploads;
  1128. default:
  1129. return NULL;
  1130. }
  1131. }
  1132. void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
  1133. {
  1134. request_list_t* requests = getRunningList(rt);
  1135. if (requests)
  1136. {
  1137. requests->push_back(request);
  1138. }
  1139. else
  1140. {
  1141. llerrs << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << llendl;
  1142. }
  1143. }
  1144. void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
  1145. {
  1146. request_list_t* requests = getRunningList(rt);
  1147. if (requests)
  1148. {
  1149. requests->remove(request);
  1150. }
  1151. else
  1152. {
  1153. llerrs << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << llendl;
  1154. }
  1155. }
  1156. // virtual
  1157. void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name)
  1158. {
  1159. if (agent_id.isNull() || asset_id.isNull())
  1160. {
  1161. llwarns << "TAT: addTempAssetData bad id's asset_id: " << asset_id << " agent_id: " << agent_id << llendl;
  1162. return;
  1163. }
  1164. LLTempAssetData temp_asset_data;
  1165. temp_asset_data.mAssetID = asset_id;
  1166. temp_asset_data.mAgentID = agent_id;
  1167. temp_asset_data.mHostName = host_name;
  1168. mTempAssets[asset_id] = temp_asset_data;
  1169. }
  1170. // virtual
  1171. BOOL LLHTTPAssetStorage::hasTempAssetData(const LLUUID& texture_id) const
  1172. {
  1173. uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
  1174. BOOL found = (citer != mTempAssets.end());
  1175. return found;
  1176. }
  1177. // virtual
  1178. std::string LLHTTPAssetStorage::getTempAssetHostName(const LLUUID& texture_id) const
  1179. {
  1180. uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
  1181. if (citer != mTempAssets.end())
  1182. {
  1183. return citer->second.mHostName;
  1184. }
  1185. else
  1186. {
  1187. return std::string();
  1188. }
  1189. }
  1190. // virtual
  1191. LLUUID LLHTTPAssetStorage::getTempAssetAgentID(const LLUUID& texture_id) const
  1192. {
  1193. uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
  1194. if (citer != mTempAssets.end())
  1195. {
  1196. return citer->second.mAgentID;
  1197. }
  1198. else
  1199. {
  1200. return LLUUID::null;
  1201. }
  1202. }
  1203. // virtual
  1204. void LLHTTPAssetStorage::removeTempAssetData(const LLUUID& asset_id)
  1205. {
  1206. mTempAssets.erase(asset_id);
  1207. }
  1208. // virtual
  1209. void LLHTTPAssetStorage::removeTempAssetDataByAgentID(const LLUUID& agent_id)
  1210. {
  1211. uuid_tempdata_map::iterator it = mTempAssets.begin();
  1212. uuid_tempdata_map::iterator end = mTempAssets.end();
  1213. while (it != end)
  1214. {
  1215. const LLTempAssetData& asset_data = it->second;
  1216. if (asset_data.mAgentID == agent_id)
  1217. {
  1218. mTempAssets.erase(it++);
  1219. }
  1220. else
  1221. {
  1222. ++it;
  1223. }
  1224. }
  1225. }
  1226. std::string LLHTTPAssetStorage::getBaseURL(const LLUUID& asset_id, LLAssetType::EType asset_type)
  1227. {
  1228. if (LLAssetType::AT_TEXTURE == asset_type)
  1229. {
  1230. uuid_tempdata_map::const_iterator citer = mTempAssets.find(asset_id);
  1231. if (citer != mTempAssets.end())
  1232. {
  1233. const std::string& host_name = citer->second.mHostName;
  1234. std::string url = llformat(LOCAL_ASSET_URL_FORMAT, host_name.c_str());
  1235. return url;
  1236. }
  1237. }
  1238. return mBaseURL;
  1239. }
  1240. void LLHTTPAssetStorage::dumpTempAssetData(const LLUUID& avatar_id) const
  1241. {
  1242. uuid_tempdata_map::const_iterator it = mTempAssets.begin();
  1243. uuid_tempdata_map::const_iterator end = mTempAssets.end();
  1244. S32 count = 0;
  1245. for ( ; it != end; ++it)
  1246. {
  1247. const LLTempAssetData& temp_asset_data = it->second;
  1248. if (avatar_id.isNull()
  1249. || avatar_id == temp_asset_data.mAgentID)
  1250. {
  1251. llinfos << "TAT: dump agent " << temp_asset_data.mAgentID
  1252. << " texture " << temp_asset_data.mAssetID
  1253. << " host " << temp_asset_data.mHostName
  1254. << llendl;
  1255. count++;
  1256. }
  1257. }
  1258. if (avatar_id.isNull())
  1259. {
  1260. llinfos << "TAT: dumped " << count << " entries for all avatars" << llendl;
  1261. }
  1262. else
  1263. {
  1264. llinfos << "TAT: dumped " << count << " entries for avatar " << avatar_id << llendl;
  1265. }
  1266. }
  1267. void LLHTTPAssetStorage::clearTempAssetData()
  1268. {
  1269. llinfos << "TAT: Clearing temp asset data map" << llendl;
  1270. mTempAssets.clear();
  1271. }