PageRenderTime 144ms CodeModel.GetById 20ms app.highlight 113ms RepoModel.GetById 1ms app.codeStats 1ms

/indra/llmessage/llhttpassetstorage.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 1459 lines | 1117 code | 188 blank | 154 comment | 176 complexity | 54a6752195081a8582b6310bc265e3c4 MD5 | raw file
   1/** 
   2 * @file llhttpassetstorage.cpp
   3 * @brief Subclass capable of loading asset data to/from an external
   4 * source. Currently, a web server accessed via curl
   5 *
   6 * $LicenseInfo:firstyear=2003&license=viewerlgpl$
   7 * Second Life Viewer Source Code
   8 * Copyright (C) 2010, Linden Research, Inc.
   9 * 
  10 * This library is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU Lesser General Public
  12 * License as published by the Free Software Foundation;
  13 * version 2.1 of the License only.
  14 * 
  15 * This library is distributed in the hope that it will be useful,
  16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 * Lesser General Public License for more details.
  19 * 
  20 * You should have received a copy of the GNU Lesser General Public
  21 * License along with this library; if not, write to the Free Software
  22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  23 * 
  24 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  25 * $/LicenseInfo$
  26 */
  27
  28#include "linden_common.h"
  29
  30#include "llhttpassetstorage.h"
  31
  32#include <sys/stat.h>
  33
  34#include "indra_constants.h"
  35#include "message.h"
  36#include "llproxy.h"
  37#include "llvfile.h"
  38#include "llvfs.h"
  39
  40#ifdef LL_STANDALONE
  41# include <zlib.h>
  42#else
  43# include "zlib/zlib.h"
  44#endif
  45
  46const U32 MAX_RUNNING_REQUESTS = 1;
  47const F32 MAX_PROCESSING_TIME = 0.005f;
  48const S32 CURL_XFER_BUFFER_SIZE = 65536;
  49// Try for 30 minutes for now.
  50const F32 GET_URL_TO_FILE_TIMEOUT = 1800.0f;
  51
  52const S32 COMPRESSED_INPUT_BUFFER_SIZE = 4096;
  53
  54const S32 HTTP_OK = 200;
  55const S32 HTTP_PUT_OK = 201;
  56const S32 HTTP_NO_CONTENT = 204;
  57const S32 HTTP_MISSING = 404;
  58const S32 HTTP_SERVER_BAD_GATEWAY = 502;
  59const S32 HTTP_SERVER_TEMP_UNAVAILABLE = 503;
  60
  61/////////////////////////////////////////////////////////////////////////////////
  62// LLTempAssetData
  63// An asset not stored on central asset store, but on a simulator node somewhere.
  64/////////////////////////////////////////////////////////////////////////////////
  65struct LLTempAssetData
  66{
  67	LLUUID	mAssetID;
  68	LLUUID	mAgentID;
  69	std::string	mHostName;
  70};
  71
  72/////////////////////////////////////////////////////////////////////////////////
  73// LLHTTPAssetRequest
  74/////////////////////////////////////////////////////////////////////////////////
  75
  76class LLHTTPAssetRequest : public LLAssetRequest
  77{
  78public:
  79	LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, 
  80					   LLAssetType::EType type, LLAssetStorage::ERequestType rt,
  81					   const std::string& url, CURLM *curl_multi);
  82	virtual ~LLHTTPAssetRequest();
  83	
  84	void setupCurlHandle();
  85	void cleanupCurlHandle();
  86
  87	void   	prepareCompressedUpload();
  88	void	finishCompressedUpload();
  89	size_t	readCompressedData(void* data, size_t size);
  90
  91	static size_t curlCompressedUploadCallback(
  92					void *data, size_t size, size_t nmemb, void *user_data);
  93
  94	virtual LLSD getTerseDetails() const;
  95	virtual LLSD getFullDetails() const;
  96
  97public:
  98	LLHTTPAssetStorage *mAssetStoragep;
  99
 100	CURL  *mCurlHandle;
 101	CURLM *mCurlMultiHandle;
 102	std::string mURLBuffer;
 103	struct curl_slist *mHTTPHeaders;
 104	LLVFile *mVFile;
 105	LLUUID  mTmpUUID;
 106	LLAssetStorage::ERequestType mRequestType;
 107
 108	bool		mZInitialized;
 109	z_stream	mZStream;
 110	char*		mZInputBuffer;
 111	bool		mZInputExhausted;
 112
 113	FILE *mFP;
 114};
 115
 116
 117LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, 
 118						const LLUUID &uuid, 
 119						LLAssetType::EType type, 
 120						LLAssetStorage::ERequestType rt,
 121						const std::string& url, 
 122						CURLM *curl_multi)
 123	: LLAssetRequest(uuid, type),
 124	  mZInitialized(false)
 125{
 126	memset(&mZStream, 0, sizeof(mZStream)); // we'll initialize this later, but for now zero the whole C-style struct to avoid debug/coverity noise
 127	mAssetStoragep = asp;
 128	mCurlHandle = NULL;
 129	mCurlMultiHandle = curl_multi;
 130	mVFile = NULL;
 131	mRequestType = rt;
 132	mHTTPHeaders = NULL;
 133	mFP = NULL;
 134	mZInputBuffer = NULL;
 135	mZInputExhausted = false;
 136	
 137	mURLBuffer = url;
 138}
 139
 140LLHTTPAssetRequest::~LLHTTPAssetRequest()
 141{
 142	// Cleanup/cancel the request
 143	if (mCurlHandle)
 144	{
 145		curl_multi_remove_handle(mCurlMultiHandle, mCurlHandle);
 146		cleanupCurlHandle();
 147	}
 148	if (mHTTPHeaders)
 149	{
 150		curl_slist_free_all(mHTTPHeaders);
 151	}
 152	delete   mVFile;
 153	finishCompressedUpload();
 154}
 155
 156// virtual
 157LLSD LLHTTPAssetRequest::getTerseDetails() const
 158{
 159	LLSD sd = LLAssetRequest::getTerseDetails();
 160
 161	sd["url"] = mURLBuffer;
 162
 163	return sd;
 164}
 165
 166// virtual
 167LLSD LLHTTPAssetRequest::getFullDetails() const
 168{
 169	LLSD sd = LLAssetRequest::getFullDetails();
 170
 171	if (mCurlHandle)
 172	{
 173		long curl_response = -1;
 174		long curl_connect = -1;
 175		double curl_total_time = -1.0f;
 176		double curl_size_upload = -1.0f;
 177		double curl_size_download = -1.0f;
 178		double curl_content_length_upload = -1.0f;
 179		double curl_content_length_download = -1.0f;
 180		long curl_request_size = -1;
 181		const char* curl_content_type = NULL;
 182
 183		curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response);
 184		curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect);
 185		curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time);
 186		curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD,  &curl_size_upload);
 187		curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download);
 188		curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD,   &curl_content_length_upload);
 189		curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download);
 190		curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size);
 191		curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type);
 192
 193		sd["curl_response_code"] = (int) curl_response;
 194		sd["curl_http_connect_code"] = (int) curl_connect;
 195		sd["curl_total_time"] = curl_total_time;
 196		sd["curl_size_upload"]   = curl_size_upload;
 197		sd["curl_size_download"] = curl_size_download;
 198		sd["curl_content_length_upload"]   =  curl_content_length_upload;
 199		sd["curl_content_length_download"] =  curl_content_length_download;
 200		sd["curl_request_size"] = (int) curl_request_size;
 201		if (curl_content_type)
 202		{
 203			sd["curl_content_type"] = curl_content_type;
 204		}
 205		else
 206		{
 207			sd["curl_content_type"] = "";
 208		}
 209	}
 210
 211	sd["temp_id"] = mTmpUUID;
 212	sd["request_type"] = LLAssetStorage::getRequestName(mRequestType);
 213	sd["z_initialized"] = mZInitialized;
 214	sd["z_input_exhausted"] = mZInputExhausted;
 215
 216	S32 file_size = -1;
 217	if (mFP)
 218	{
 219		struct stat file_stat;
 220		int file_desc = fileno(mFP);
 221		if ( fstat(file_desc, &file_stat) == 0)
 222		{
 223			file_size = file_stat.st_size;
 224		}
 225	}
 226	sd["file_size"] = file_size;
 227
 228	return sd;
 229}
 230
 231
 232void LLHTTPAssetRequest::setupCurlHandle()
 233{
 234	// *NOTE: Similar code exists in mapserver/llcurlutil.cpp  JC
 235	mCurlHandle = LLCurl::newEasyHandle();
 236	llassert_always(mCurlHandle != NULL) ;
 237
 238	// Apply proxy settings if configured to do so
 239	LLProxy::getInstance()->applyProxySettings(mCurlHandle);
 240
 241	curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
 242	curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
 243	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer.c_str());
 244	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
 245	if (LLAssetStorage::RT_DOWNLOAD == mRequestType)
 246	{
 247		curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
 248		// only do this on downloads, as uploads 
 249		// to some apache configs (like our test grids)
 250		// mistakenly claim the response is gzip'd if the resource
 251		// name ends in .gz, even though in a PUT, the response is
 252		// just plain HTML saying "created"
 253	}
 254	/* Remove the Pragma: no-cache header that libcurl inserts by default;
 255	   we want the cached version, if possible. */
 256	if (mZInitialized)
 257	{
 258		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, "");
 259			// disable use of proxy, which can't handle chunked transfers
 260	}
 261	mHTTPHeaders = curl_slist_append(mHTTPHeaders, "Pragma:");
 262
 263	// bug in curl causes DNS to be cached for too long a time, 0 sets it to never cache DNS results internally (to curl)
 264	curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0);
 265	
 266	// resist the temptation to explicitly add the Transfer-Encoding: chunked
 267	// header here - invokes a libCURL bug
 268	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mHTTPHeaders);
 269	if (mAssetStoragep)
 270	{
 271		// Set the appropriate pending upload or download flag
 272		mAssetStoragep->addRunningRequest(mRequestType, this);
 273	}
 274	else
 275	{
 276		llerrs << "LLHTTPAssetRequest::setupCurlHandle - No asset storage associated with this request!" << llendl;
 277	}
 278}
 279
 280void LLHTTPAssetRequest::cleanupCurlHandle()
 281{
 282	LLCurl::deleteEasyHandle(mCurlHandle);
 283	if (mAssetStoragep)
 284	{
 285		// Terminating a request.  Thus upload or download is no longer pending.
 286		mAssetStoragep->removeRunningRequest(mRequestType, this);
 287	}
 288	else
 289	{
 290		llerrs << "LLHTTPAssetRequest::~LLHTTPAssetRequest - No asset storage associated with this request!" << llendl;
 291	}
 292	mCurlHandle = NULL;
 293}
 294
 295void LLHTTPAssetRequest::prepareCompressedUpload()
 296{
 297	mZStream.next_in = Z_NULL;
 298	mZStream.avail_in = 0;
 299	mZStream.zalloc = Z_NULL;
 300	mZStream.zfree = Z_NULL;
 301	mZStream.opaque = Z_NULL;
 302
 303	int r = deflateInit2(&mZStream,
 304			1,			// compression level
 305			Z_DEFLATED,	// the only method defined
 306			15 + 16,	// the default windowBits + gzip header flag
 307			8,			// the default memLevel
 308			Z_DEFAULT_STRATEGY);
 309
 310	if (r != Z_OK)
 311	{
 312		llerrs << "LLHTTPAssetRequest::prepareCompressedUpload defalateInit2() failed" << llendl;
 313	}
 314
 315	mZInitialized = true;
 316	mZInputBuffer = new char[COMPRESSED_INPUT_BUFFER_SIZE];
 317	mZInputExhausted = false;
 318
 319	mVFile = new LLVFile(gAssetStorage->mVFS,
 320					getUUID(), getType(), LLVFile::READ);
 321}
 322
 323void LLHTTPAssetRequest::finishCompressedUpload()
 324{
 325	if (mZInitialized)
 326	{
 327		llinfos << "LLHTTPAssetRequest::finishCompressedUpload: "
 328			<< "read " << mZStream.total_in << " byte asset file, "
 329			<< "uploaded " << mZStream.total_out << " byte compressed asset"
 330			<< llendl;
 331
 332		deflateEnd(&mZStream);
 333		delete[] mZInputBuffer;
 334	}
 335}
 336
 337size_t LLHTTPAssetRequest::readCompressedData(void* data, size_t size)
 338{
 339	llassert(mZInitialized);
 340
 341	mZStream.next_out = (Bytef*)data;
 342	mZStream.avail_out = size;
 343
 344	while (mZStream.avail_out > 0)
 345	{
 346		if (mZStream.avail_in == 0 && !mZInputExhausted)
 347		{
 348			S32 to_read = llmin(COMPRESSED_INPUT_BUFFER_SIZE,
 349							(S32)(mVFile->getSize() - mVFile->tell()));
 350			
 351			if ( to_read > 0 )
 352			{
 353				mVFile->read((U8*)mZInputBuffer, to_read); /*Flawfinder: ignore*/
 354				mZStream.next_in = (Bytef*)mZInputBuffer;
 355				mZStream.avail_in = mVFile->getLastBytesRead();
 356			}
 357
 358			mZInputExhausted = mZStream.avail_in == 0;
 359		}
 360
 361		int r = deflate(&mZStream,
 362					mZInputExhausted ? Z_FINISH : Z_NO_FLUSH);
 363
 364		if (r == Z_STREAM_END || r < 0 || mZInputExhausted)
 365		{
 366			if (r < 0)
 367			{
 368				llwarns << "LLHTTPAssetRequest::readCompressedData: deflate returned error code " 
 369						<< (S32) r << llendl;
 370			}
 371			break;
 372		}
 373	}
 374
 375	return size - mZStream.avail_out;
 376}
 377
 378//static
 379size_t LLHTTPAssetRequest::curlCompressedUploadCallback(
 380		void *data, size_t size, size_t nmemb, void *user_data)
 381{
 382	size_t num_read = 0;
 383
 384	if (gAssetStorage)
 385	{
 386		CURL *curl_handle = (CURL *)user_data;
 387		LLHTTPAssetRequest *req = NULL;
 388		curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
 389		if (req)
 390		{
 391			num_read = req->readCompressedData(data, size * nmemb);
 392		}
 393	}
 394
 395	return num_read;
 396}
 397
 398/////////////////////////////////////////////////////////////////////////////////
 399// LLHTTPAssetStorage
 400/////////////////////////////////////////////////////////////////////////////////
 401
 402
 403LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
 404									 LLVFS *vfs, LLVFS *static_vfs, 
 405									 const LLHost &upstream_host,
 406									 const std::string& web_host,
 407									 const std::string& local_web_host,
 408									 const std::string& host_name)
 409	: LLAssetStorage(msg, xfer, vfs, static_vfs, upstream_host)
 410{
 411	_init(web_host, local_web_host, host_name);
 412}
 413
 414LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
 415									   LLVFS *vfs,
 416									   LLVFS *static_vfs,
 417									   const std::string& web_host,
 418									   const std::string& local_web_host,
 419									   const std::string& host_name)
 420	: LLAssetStorage(msg, xfer, vfs, static_vfs)
 421{
 422	_init(web_host, local_web_host, host_name);
 423}
 424
 425void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& local_web_host, const std::string& host_name)
 426{
 427	mBaseURL = web_host;
 428	mLocalBaseURL = local_web_host;
 429	mHostName = host_name;
 430
 431	// curl_global_init moved to LLCurl::initClass()
 432	
 433	mCurlMultiHandle = LLCurl::newMultiHandle() ;
 434	llassert_always(mCurlMultiHandle != NULL) ;
 435}
 436
 437LLHTTPAssetStorage::~LLHTTPAssetStorage()
 438{
 439	LLCurl::deleteMultiHandle(mCurlMultiHandle);
 440	mCurlMultiHandle = NULL;
 441	
 442	// curl_global_cleanup moved to LLCurl::initClass()
 443}
 444
 445// storing data is simpler than getting it, so we just overload the whole method
 446void LLHTTPAssetStorage::storeAssetData(
 447	const LLUUID& uuid,
 448	LLAssetType::EType type,
 449	LLAssetStorage::LLStoreAssetCallback callback,
 450	void* user_data,
 451	bool temp_file,
 452	bool is_priority,
 453	bool store_local,
 454	const LLUUID& requesting_agent_id,
 455	bool user_waiting,
 456	F64 timeout)
 457{
 458	if (mVFS->getExists(uuid, type)) // VFS treats nonexistant and zero-length identically
 459	{
 460		LLAssetRequest *req = new LLAssetRequest(uuid, type);
 461		req->mUpCallback    = callback;
 462		req->mUserData      = user_data;
 463		req->mRequestingAgentID = requesting_agent_id;
 464		req->mIsUserWaiting = user_waiting;
 465		req->mTimeout       = timeout;
 466
 467		// LLAssetStorage metric: Successful Request
 468		S32 size = mVFS->getSize(uuid, type);
 469		const char *message;
 470		if( store_local )
 471		{
 472			message = "Added to local upload queue";
 473		}
 474		else
 475		{
 476			message = "Added to upload queue";
 477		}
 478		reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, size, MR_OKAY, __FILE__, __LINE__, message );
 479
 480		// this will get picked up and transmitted in checkForTimeouts
 481		if(store_local)
 482		{
 483			mPendingLocalUploads.push_back(req);
 484		}
 485		else if(is_priority)
 486		{
 487			mPendingUploads.push_front(req);
 488		}
 489		else
 490		{
 491			mPendingUploads.push_back(req);
 492		}
 493	}
 494	else
 495	{
 496		llwarns << "AssetStorage: attempt to upload non-existent vfile " << uuid << ":" << LLAssetType::lookup(type) << llendl;
 497		if (callback)
 498		{
 499			// LLAssetStorage metric: Zero size VFS
 500			reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file didn't exist or was zero length (VFS - can't tell which)" );
 501			callback(uuid, user_data,  LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LL_EXSTAT_NONEXISTENT_FILE);
 502		}
 503	}
 504}
 505
 506// virtual
 507void LLHTTPAssetStorage::storeAssetData(
 508	const std::string& filename,
 509	const LLUUID& asset_id,
 510	LLAssetType::EType asset_type,
 511	LLStoreAssetCallback callback,
 512	void* user_data,
 513	bool temp_file,
 514	bool is_priority,
 515	bool user_waiting,
 516	F64 timeout)
 517{
 518	llinfos << "LLAssetStorage::storeAssetData (legacy)" << asset_id << ":" << LLAssetType::lookup(asset_type) << llendl;
 519
 520	LLLegacyAssetRequest *legacy = new LLLegacyAssetRequest;
 521
 522	legacy->mUpCallback = callback;
 523	legacy->mUserData = user_data;
 524
 525	FILE *fp = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/
 526	S32 size = 0;
 527	if (fp)
 528	{
 529		fseek(fp, 0, SEEK_END);
 530		size = ftell(fp);
 531		fseek(fp, 0, SEEK_SET);
 532	}
 533
 534	if( size )
 535	{
 536		LLVFile file(mVFS, asset_id, asset_type, LLVFile::WRITE);
 537
 538		file.setMaxSize(size);
 539
 540		const S32 buf_size = 65536;
 541		U8 copy_buf[buf_size];
 542		while ((size = (S32)fread(copy_buf, 1, buf_size, fp)))
 543		{
 544			file.write(copy_buf, size);
 545		}
 546		fclose(fp);
 547
 548		// if this upload fails, the caller needs to setup a new tempfile for us
 549		if (temp_file)
 550		{
 551			LLFile::remove(filename);
 552		}
 553		
 554		// LLAssetStorage metric: Success not needed; handled in the overloaded method here:
 555		storeAssetData(
 556			asset_id,
 557			asset_type,
 558			legacyStoreDataCallback,
 559			(void**)legacy,
 560			temp_file,
 561			is_priority,
 562			false,
 563			LLUUID::null,
 564			user_waiting,
 565			timeout);
 566	}
 567	else // !size
 568	{
 569		if( fp )
 570		{
 571			// LLAssetStorage metric: Zero size
 572			reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file was zero length" );
 573			fclose( fp );
 574		}
 575		else
 576		{
 577			// LLAssetStorage metric: Missing File
 578			reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_FILE_NONEXIST, __FILE__, __LINE__, "The file didn't exist" );
 579		}
 580		if (callback)
 581		{
 582			callback(LLUUID::null, user_data, LL_ERR_CANNOT_OPEN_FILE, LL_EXSTAT_BLOCKED_FILE);
 583		}
 584		delete legacy;
 585	}
 586}
 587
 588// virtual
 589LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt,
 590										LLAssetType::EType asset_type,
 591										const std::string& detail_prefix) const
 592{
 593	LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix);
 594	const request_list_t* running = getRunningList(rt);
 595	if (running)
 596	{
 597		// Loop through the pending requests sd, and add extra info about its running status.
 598		S32 num_pending = sd["requests"].size();
 599		S32 i;
 600		for (i = 0; i < num_pending; ++i)
 601		{
 602			LLSD& pending = sd["requests"][i];
 603			// See if this pending request is running.
 604			const LLAssetRequest* req = findRequest(running, 
 605									LLAssetType::lookup(pending["type"].asString()),
 606									pending["asset_id"]);
 607			if (req)
 608			{
 609				// Keep the detail_url so we don't have to rebuild it.
 610				LLURI detail_url = pending["detail"];
 611				pending = req->getTerseDetails();
 612				pending["detail"] = detail_url;
 613				pending["is_running"] = true;
 614			}
 615			else
 616			{
 617				pending["is_running"] = false;
 618			}
 619		}
 620	}
 621	return sd;
 622}
 623
 624// virtual
 625LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt,
 626										LLAssetType::EType asset_type,
 627										const LLUUID& asset_id) const
 628{
 629	// Look for this asset in the running list first.
 630	const request_list_t* running = getRunningList(rt);
 631	if (running)
 632	{
 633		LLSD sd = LLAssetStorage::getPendingRequestImpl(running, asset_type, asset_id);
 634		if (sd)
 635		{
 636			sd["is_running"] = true;
 637			return sd;
 638		}
 639	}
 640	LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id);
 641	if (sd)
 642	{
 643		sd["is_running"] = false;
 644	}
 645	return sd;
 646}
 647
 648// virtual
 649bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt,
 650											LLAssetType::EType asset_type,
 651											const LLUUID& asset_id)
 652{
 653	// Try removing this from the running list first.
 654	request_list_t* running = getRunningList(rt);
 655	if (running)
 656	{
 657		LLAssetRequest* req = findRequest(running, asset_type, asset_id);
 658		if (req)
 659		{
 660			// Remove this request from the running list to get it out of curl.
 661			running->remove(req);
 662			
 663			// Find this request in the pending list, so we can move it to the end of the line.
 664			request_list_t* pending = getRequestList(rt);
 665			if (pending)
 666			{
 667				request_list_t::iterator result = std::find_if(pending->begin(), pending->end(),
 668										std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req));
 669				if (pending->end() != result)
 670				{
 671					// This request was found in the pending list.  Move it to the end!
 672					LLAssetRequest* pending_req = *result;
 673					pending->remove(pending_req);
 674
 675					if (!pending_req->mIsUserWaiting)				//A user is waiting on this request.  Toss it.
 676					{
 677						pending->push_back(pending_req);
 678					}
 679					else
 680					{
 681						if (pending_req->mUpCallback)	//Clean up here rather than _callUploadCallbacks because this request is already cleared the req.
 682						{
 683							pending_req->mUpCallback(pending_req->getUUID(), pending_req->mUserData, -1, LL_EXSTAT_REQUEST_DROPPED);
 684						}
 685
 686					}
 687
 688					llinfos << "Asset " << getRequestName(rt) << " request for "
 689							<< asset_id << "." << LLAssetType::lookup(asset_type)
 690							<< " removed from curl and placed at the end of the pending queue."
 691							<< llendl;
 692				}
 693				else
 694				{
 695					llwarns << "Unable to find pending " << getRequestName(rt) << " request for "
 696							<< asset_id << "." << LLAssetType::lookup(asset_type) << llendl;
 697				}
 698			}
 699			delete req;
 700
 701			return true;
 702		}
 703	}
 704	return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id);
 705}
 706
 707// internal requester, used by getAssetData in superclass
 708void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type,
 709										  void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat),
 710										  void *user_data, BOOL duplicate,
 711										   BOOL is_priority)
 712{
 713	// stash the callback info so we can find it after we get the response message
 714	LLAssetRequest *req = new LLAssetRequest(uuid, type);
 715	req->mDownCallback = callback;
 716	req->mUserData = user_data;
 717	req->mIsPriority = is_priority;
 718
 719	// this will get picked up and downloaded in checkForTimeouts
 720
 721	//
 722	// HAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACK!  Asset requests were taking too long and timing out.
 723	// Since texture requests are the LEAST sensitive (on the simulator) to being delayed, add
 724	// non-texture requests to the front, and add texture requests to the back.  The theory is
 725	// that we always want them first, even if they're out of order.
 726	//
 727	
 728	if (req->getType() == LLAssetType::AT_TEXTURE)
 729	{
 730		mPendingDownloads.push_back(req);
 731	}
 732	else
 733	{
 734		mPendingDownloads.push_front(req);
 735	}
 736}
 737
 738LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending, 
 739													LLAssetStorage::request_list_t& running)
 740{
 741	// Early exit if the running list is full, or we don't have more pending than running.
 742	if (running.size() >= MAX_RUNNING_REQUESTS
 743		|| pending.size() <= running.size()) return NULL;
 744
 745	// Look for the first pending request that is not already running.
 746	request_list_t::iterator running_begin = running.begin();
 747	request_list_t::iterator running_end   = running.end();
 748
 749	request_list_t::iterator pending_iter = pending.begin();
 750	request_list_t::iterator pending_end  = pending.end();
 751	// Loop over all pending requests until we miss finding it in the running list.
 752	for (; pending_iter != pending.end(); ++pending_iter)
 753	{
 754		LLAssetRequest* req = *pending_iter;
 755		// Look for this pending request in the running list.
 756		if (running_end == std::find_if(running_begin, running_end, 
 757						std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)))
 758		{
 759			// It isn't running!  Return it.
 760			return req;
 761		}
 762	}
 763	return NULL;
 764}
 765
 766// overloaded to additionally move data to/from the webserver
 767void LLHTTPAssetStorage::checkForTimeouts()
 768{
 769	CURLMcode mcode;
 770	LLAssetRequest *req;
 771	while ( (req = findNextRequest(mPendingDownloads, mRunningDownloads)) )
 772	{
 773		// Setup this curl download request
 774		// We need to generate a new request here
 775		// since the one in the list could go away
 776		std::string tmp_url;
 777		std::string uuid_str;
 778		req->getUUID().toString(uuid_str);
 779		std::string base_url = getBaseURL(req->getUUID(), req->getType());
 780		tmp_url = llformat("%s/%36s.%s", base_url.c_str() , uuid_str.c_str(), LLAssetType::lookup(req->getType()));
 781
 782		LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
 783										req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle);
 784		new_req->mTmpUUID.generate();
 785
 786		// Sets pending download flag internally
 787		new_req->setupCurlHandle();
 788		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
 789		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback);
 790		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle);
 791	
 792		mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
 793		if (mcode > CURLM_OK)
 794		{
 795			// Failure.  Deleting the pending request will remove it from the running
 796			// queue, and push it to the end of the pending queue.
 797			new_req->cleanupCurlHandle();
 798			deletePendingRequest(RT_DOWNLOAD, new_req->getType(), new_req->getUUID());
 799			break;
 800		}
 801		else
 802		{
 803			llinfos << "Requesting " << new_req->mURLBuffer << llendl;
 804		}
 805	}
 806
 807	while ( (req = findNextRequest(mPendingUploads, mRunningUploads)) )
 808	{
 809		// setup this curl upload request
 810
 811		bool do_compress = req->getType() == LLAssetType::AT_OBJECT;
 812
 813		std::string tmp_url;
 814		std::string uuid_str;
 815		req->getUUID().toString(uuid_str);
 816		tmp_url = mBaseURL + "/" + uuid_str + "." + LLAssetType::lookup(req->getType());
 817		if (do_compress) tmp_url += ".gz";
 818
 819		LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
 820									req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle);
 821
 822		if (req->mIsUserWaiting) //If a user is waiting on a realtime response, we want to perserve information across upload attempts.
 823		{
 824			new_req->mTime          = req->mTime;
 825			new_req->mTimeout       = req->mTimeout;
 826			new_req->mIsUserWaiting = req->mIsUserWaiting;
 827		}
 828
 829		if (do_compress)
 830		{
 831			new_req->prepareCompressedUpload();
 832		}
 833
 834		// Sets pending upload flag internally
 835		new_req->setupCurlHandle();
 836		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_UPLOAD, 1);
 837		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);
 838
 839		if (do_compress)
 840		{
 841			curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
 842					&LLHTTPAssetRequest::curlCompressedUploadCallback);
 843		}
 844		else
 845		{
 846			LLVFile file(mVFS, req->getUUID(), req->getType());
 847			curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
 848			curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
 849					&curlUpCallback);
 850		}
 851		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
 852	
 853		mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
 854		if (mcode > CURLM_OK)
 855		{
 856			// Failure.  Deleting the pending request will remove it from the running
 857			// queue, and push it to the end of the pending queue.
 858			new_req->cleanupCurlHandle();
 859			deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());
 860			break;
 861		}
 862		else
 863		{
 864			// Get the uncompressed file size.
 865			LLVFile file(mVFS,new_req->getUUID(),new_req->getType());
 866			S32 size = file.getSize();
 867			llinfos << "Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << llendl;
 868			if (size == 0)
 869			{
 870				llwarns << "Rejecting zero size PUT request!" << llendl;
 871				new_req->cleanupCurlHandle();
 872				deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());				
 873			}
 874		}
 875		// Pending upload will have been flagged by the request
 876	}
 877
 878	while ( (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) )
 879	{
 880		// setup this curl upload request
 881		LLVFile file(mVFS, req->getUUID(), req->getType());
 882
 883		std::string tmp_url;
 884		std::string uuid_str;
 885		req->getUUID().toString(uuid_str);
 886		
 887		// KLW - All temporary uploads are saved locally "http://localhost:12041/asset"
 888		tmp_url = llformat("%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str.c_str(), LLAssetType::lookup(req->getType()));
 889
 890		LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
 891										req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle);
 892		new_req->mRequestingAgentID = req->mRequestingAgentID;
 893
 894		// Sets pending upload flag internally
 895		new_req->setupCurlHandle();
 896		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_PUT, 1);
 897		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
 898		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);
 899		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback);
 900		curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
 901	
 902		mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
 903		if (mcode > CURLM_OK)
 904		{
 905			// Failure.  Deleting the pending request will remove it from the running
 906			// queue, and push it to the end of the pending queue.
 907			new_req->cleanupCurlHandle();
 908			deletePendingRequest(RT_LOCALUPLOAD, new_req->getType(), new_req->getUUID());
 909			break;
 910		}
 911		else
 912		{
 913			// Get the uncompressed file size.
 914			S32 size = file.getSize();
 915
 916			llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!"
 917				<< " Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << llendl;
 918			if (size == 0)
 919			{
 920				
 921				llwarns << "Rejecting zero size PUT request!" << llendl;
 922				new_req->cleanupCurlHandle();
 923				deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());				
 924			}
 925
 926		}
 927		// Pending upload will have been flagged by the request
 928	}
 929	S32 count = 0;
 930	int queue_length;
 931	do
 932	{
 933		mcode = curl_multi_perform(mCurlMultiHandle, &queue_length);
 934		count++;
 935	} while (mcode == CURLM_CALL_MULTI_PERFORM && (count < 5));
 936
 937	CURLMsg *curl_msg;
 938	do
 939	{
 940		curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);
 941		if (curl_msg && curl_msg->msg == CURLMSG_DONE)
 942		{
 943			long curl_result = 0;
 944			S32 xfer_result = LL_ERR_NOERR;
 945
 946			LLHTTPAssetRequest *req = NULL;
 947			curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req);
 948								
 949			// TODO: Throw curl_result at all callbacks.
 950			curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
 951			if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType)
 952			{
 953				if (curl_msg->data.result == CURLE_OK && 
 954					(   curl_result == HTTP_OK 
 955					 || curl_result == HTTP_PUT_OK 
 956					 || curl_result == HTTP_NO_CONTENT))
 957				{
 958					llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl;
 959					if (RT_LOCALUPLOAD == req->mRequestType)
 960					{
 961						addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName);
 962					}
 963				}
 964				else if (curl_msg->data.result == CURLE_COULDNT_CONNECT ||
 965						curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
 966						curl_result == HTTP_SERVER_BAD_GATEWAY ||
 967						curl_result == HTTP_SERVER_TEMP_UNAVAILABLE)
 968				{
 969					llwarns << "Re-requesting upload for " << req->getUUID() << ".  Received upload error to " << req->mURLBuffer <<
 970						" with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
 971
 972					////HACK (probably) I am sick of this getting requeued and driving me mad.
 973					//if (req->mIsUserWaiting)
 974					//{
 975					//	deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
 976					//}
 977				}
 978				else
 979				{
 980					llwarns << "Failure uploading " << req->getUUID() << " to " << req->mURLBuffer <<
 981						" with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
 982
 983					xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
 984				}
 985
 986				if (!(curl_msg->data.result == CURLE_COULDNT_CONNECT ||
 987						curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
 988						curl_result == HTTP_SERVER_BAD_GATEWAY ||
 989						curl_result == HTTP_SERVER_TEMP_UNAVAILABLE))
 990				{
 991					// shared upload finished callback
 992					// in the base class, this is called from processUploadComplete
 993					_callUploadCallbacks(req->getUUID(), req->getType(), (xfer_result == 0), LL_EXSTAT_CURL_RESULT | curl_result);
 994					// Pending upload flag will get cleared when the request is deleted
 995				}
 996			}
 997			else if (RT_DOWNLOAD == req->mRequestType)
 998			{
 999				if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
1000				{
1001					if (req->mVFile && req->mVFile->getSize() > 0)
1002					{					
1003						llinfos << "Success downloading " << req->mURLBuffer << ", size " << req->mVFile->getSize() << llendl;
1004
1005						req->mVFile->rename(req->getUUID(), req->getType());
1006					}
1007					else
1008					{
1009						// *TODO: if this actually indicates a bad asset on the server
1010						// (not certain at this point), then delete it
1011						llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl;
1012						xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
1013					}
1014				}
1015				else
1016				{
1017					// KLW - TAT See if an avatar owns this texture, and if so request re-upload.
1018					llwarns << "Failure downloading " << req->mURLBuffer << 
1019						" with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
1020
1021					xfer_result = (curl_result == HTTP_MISSING) ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;
1022
1023					if (req->mVFile)
1024					{
1025						req->mVFile->remove();
1026					}
1027				}
1028
1029				// call the static callback for transfer completion
1030				// this will cleanup all requests for this asset, including ours
1031				downloadCompleteCallback(
1032					xfer_result,
1033					req->getUUID(),
1034					req->getType(),
1035					(void *)req,
1036					LL_EXSTAT_CURL_RESULT | curl_result);
1037				// Pending download flag will get cleared when the request is deleted
1038			}
1039			else
1040			{
1041				// nothing, just axe this request
1042				// currently this can only mean an asset delete
1043			}
1044
1045			// Deleting clears the pending upload/download flag if it's set and the request is transferring
1046			delete req;
1047			req = NULL;
1048		}
1049
1050	} while (curl_msg && queue_length > 0);
1051	
1052
1053	// Cleanup 
1054	// We want to bump to the back of the line any running uploads that have timed out.
1055	bumpTimedOutUploads();
1056
1057	LLAssetStorage::checkForTimeouts();
1058}
1059
1060void LLHTTPAssetStorage::bumpTimedOutUploads()
1061{
1062	bool user_waiting=FALSE;
1063
1064	F64 mt_secs = LLMessageSystem::getMessageTimeSeconds();
1065
1066	if (mPendingUploads.size())
1067	{
1068		request_list_t::iterator it = mPendingUploads.begin();
1069		LLAssetRequest* req = *it;
1070		user_waiting=req->mIsUserWaiting;
1071	}
1072
1073	// No point bumping currently running uploads if there are no others in line.
1074	if (!(mPendingUploads.size() > mRunningUploads.size()) && !user_waiting) 
1075	{
1076		return;
1077	}
1078
1079	// deletePendingRequest will modify the mRunningUploads list so we don't want to iterate over it.
1080	request_list_t temp_running = mRunningUploads;
1081
1082	request_list_t::iterator it = temp_running.begin();
1083	request_list_t::iterator end = temp_running.end();
1084	for ( ; it != end; ++it)
1085	{
1086		//request_list_t::iterator curiter = iter++;
1087		LLAssetRequest* req = *it;
1088
1089		if ( req->mTimeout < (mt_secs - req->mTime) )
1090		{
1091			llwarns << "Asset upload request timed out for "
1092					<< req->getUUID() << "."
1093					<< LLAssetType::lookup(req->getType()) 
1094					<< ", bumping to the back of the line!" << llendl;
1095
1096			deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
1097		}
1098	}
1099}
1100
1101// static
1102size_t LLHTTPAssetStorage::curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
1103{
1104	if (!gAssetStorage)
1105	{
1106		llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
1107		return 0;
1108	}
1109	S32 bytes = (S32)(size * nmemb);
1110	CURL *curl_handle = (CURL *)user_data;
1111	LLHTTPAssetRequest *req = NULL;
1112	curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
1113
1114	if (! req->mVFile)
1115	{
1116		req->mVFile = new LLVFile(gAssetStorage->mVFS, req->mTmpUUID, LLAssetType::AT_NONE, LLVFile::APPEND);
1117	}
1118
1119	double content_length = 0.0;
1120	curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length);
1121
1122	// sanitize content_length, reconcile w/ actual data
1123	S32 file_length = llmax(0, (S32)llmin(content_length, 20000000.0), bytes + req->mVFile->getSize());
1124
1125	req->mVFile->setMaxSize(file_length);
1126	req->mVFile->write((U8*)data, bytes);
1127
1128	return nmemb;
1129}
1130
1131// static 
1132size_t LLHTTPAssetStorage::curlUpCallback(void *data, size_t size, size_t nmemb, void *user_data)
1133{
1134	if (!gAssetStorage)
1135	{
1136		llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
1137		return 0;
1138	}
1139	CURL *curl_handle = (CURL *)user_data;
1140	LLHTTPAssetRequest *req = NULL;
1141	curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
1142
1143	if (! req->mVFile)
1144	{
1145		req->mVFile = new LLVFile(gAssetStorage->mVFS, req->getUUID(), req->getType(), LLVFile::READ);
1146	}
1147
1148	S32 bytes = llmin((S32)(size * nmemb), (S32)(req->mVFile->getSize() - req->mVFile->tell()));
1149
1150	req->mVFile->read((U8*)data, bytes);/*Flawfinder: ignore*/
1151
1152	return req->mVFile->getLastBytesRead();
1153}
1154
1155// static
1156size_t LLHTTPAssetStorage::nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data)
1157{
1158	// do nothing, this is here to soak up script output so it doesn't end up on stdout
1159
1160	return nmemb;
1161}
1162
1163
1164
1165// blocking asset fetch which bypasses the VFS
1166// this is a very limited function for use by the simstate loader and other one-offs
1167S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const std::string &url, const std::string& filename, progress_callback callback, void *userdata)
1168{
1169	// *NOTE: There is no guarantee that the uuid and the asset_type match
1170	// - not that it matters. - Doug
1171	lldebugs << "LLHTTPAssetStorage::getURLToFile() - " << url << llendl;
1172
1173	FILE *fp = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/
1174	if (! fp)
1175	{
1176		llwarns << "Failed to open " << filename << " for writing" << llendl;
1177		return LL_ERR_ASSET_REQUEST_FAILED;
1178	}
1179
1180	// make sure we use the normal curl setup, even though we don't really need a request object
1181	LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url, mCurlMultiHandle);
1182	req.mFP = fp;
1183	
1184	req.setupCurlHandle();
1185	curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
1186	curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEFUNCTION, &curlFileDownCallback);
1187	curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEDATA, req.mCurlHandle);
1188
1189	curl_multi_add_handle(mCurlMultiHandle, req.mCurlHandle);
1190	llinfos << "Requesting as file " << req.mURLBuffer << llendl;
1191
1192	// braindead curl loop
1193	int queue_length;
1194	CURLMsg *curl_msg;
1195	LLTimer timeout;
1196	timeout.setTimerExpirySec(GET_URL_TO_FILE_TIMEOUT);
1197	bool success = false;
1198	S32 xfer_result = 0;
1199	do
1200	{
1201		curl_multi_perform(mCurlMultiHandle, &queue_length);
1202		curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);
1203
1204		if (callback)
1205		{
1206			callback(userdata);
1207		}
1208
1209		if ( curl_msg && (CURLMSG_DONE == curl_msg->msg) )
1210		{
1211			success = true;
1212		}
1213		else if (timeout.hasExpired())
1214		{
1215			llwarns << "Request for " << url << " has timed out." << llendl;
1216			success = false;
1217			xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
1218			break;
1219		}
1220	} while (!success);
1221
1222	if (success)
1223	{
1224		long curl_result = 0;
1225		curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
1226		
1227		if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
1228		{
1229			S32 size = ftell(req.mFP);
1230			if (size > 0)
1231			{
1232				// everything seems to be in order
1233				llinfos << "Success downloading " << req.mURLBuffer << " to file, size " << size << llendl;
1234			}
1235			else
1236			{
1237				llwarns << "Found " << req.mURLBuffer << " to be zero size" << llendl;
1238				xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
1239			}
1240		}
1241		else
1242		{
1243			xfer_result = curl_result == HTTP_MISSING ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;
1244			llinfos << "Failure downloading " << req.mURLBuffer << 
1245				" with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
1246		}
1247	}
1248
1249	fclose(fp);
1250	if (xfer_result)
1251	{
1252		LLFile::remove(filename);
1253	}
1254	return xfer_result;
1255}
1256
1257
1258// static
1259size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
1260{	
1261	CURL *curl_handle = (CURL *)user_data;
1262	LLHTTPAssetRequest *req = NULL;
1263	curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
1264
1265	if (! req->mFP)
1266	{
1267		llwarns << "Missing mFP, aborting curl file download callback!" << llendl;
1268		return 0;
1269	}
1270
1271	return fwrite(data, size, nmemb, req->mFP);
1272}
1273
1274LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt)
1275{
1276	switch (rt)
1277	{
1278	case RT_DOWNLOAD:
1279		return &mRunningDownloads;
1280	case RT_UPLOAD:
1281		return &mRunningUploads;
1282	case RT_LOCALUPLOAD:
1283		return &mRunningLocalUploads;
1284	default:
1285		return NULL;
1286	}
1287}
1288
1289const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const
1290{
1291	switch (rt)
1292	{
1293	case RT_DOWNLOAD:
1294		return &mRunningDownloads;
1295	case RT_UPLOAD:
1296		return &mRunningUploads;
1297	case RT_LOCALUPLOAD:
1298		return &mRunningLocalUploads;
1299	default:
1300		return NULL;
1301	}
1302}
1303
1304
1305void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
1306{
1307	request_list_t* requests = getRunningList(rt);
1308	if (requests)
1309	{
1310		requests->push_back(request);
1311	}
1312	else
1313	{
1314		llerrs << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << llendl;
1315	}
1316}
1317
1318void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
1319{
1320	request_list_t* requests = getRunningList(rt);
1321	if (requests)
1322	{
1323		requests->remove(request);
1324	}
1325	else
1326	{
1327		llerrs << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << llendl;
1328	}
1329}
1330
1331// virtual 
1332void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name)
1333{
1334	if (agent_id.isNull() || asset_id.isNull())
1335	{
1336		llwarns << "TAT: addTempAssetData bad id's asset_id: " << asset_id << "  agent_id: " << agent_id << llendl;
1337		return;
1338	}
1339
1340	LLTempAssetData temp_asset_data;
1341	temp_asset_data.mAssetID = asset_id;
1342	temp_asset_data.mAgentID = agent_id;
1343	temp_asset_data.mHostName = host_name;
1344
1345	mTempAssets[asset_id] = temp_asset_data;
1346}
1347
1348// virtual
1349BOOL LLHTTPAssetStorage::hasTempAssetData(const LLUUID& texture_id) const
1350{
1351	uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
1352	BOOL found = (citer != mTempAssets.end());
1353	return found;
1354}
1355
1356// virtual
1357std::string LLHTTPAssetStorage::getTempAssetHostName(const LLUUID& texture_id) const
1358{
1359	uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
1360	if (citer != mTempAssets.end())
1361	{
1362		return citer->second.mHostName;
1363	}
1364	else
1365	{
1366		return std::string();
1367	}
1368}
1369
1370// virtual 
1371LLUUID LLHTTPAssetStorage::getTempAssetAgentID(const LLUUID& texture_id) const
1372{
1373	uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
1374	if (citer != mTempAssets.end())
1375	{
1376		return citer->second.mAgentID;
1377	}
1378	else
1379	{
1380		return LLUUID::null;
1381	}
1382}
1383
1384// virtual 
1385void LLHTTPAssetStorage::removeTempAssetData(const LLUUID& asset_id)
1386{
1387	mTempAssets.erase(asset_id);
1388}
1389
1390// virtual 
1391void LLHTTPAssetStorage::removeTempAssetDataByAgentID(const LLUUID& agent_id)
1392{
1393	uuid_tempdata_map::iterator it = mTempAssets.begin();
1394	uuid_tempdata_map::iterator end = mTempAssets.end();
1395
1396	while (it != end)
1397	{
1398		const LLTempAssetData& asset_data = it->second;
1399		if (asset_data.mAgentID == agent_id)
1400		{
1401			mTempAssets.erase(it++);
1402		}
1403		else
1404		{
1405			++it;
1406		}
1407	}
1408}
1409
1410std::string LLHTTPAssetStorage::getBaseURL(const LLUUID& asset_id, LLAssetType::EType asset_type)
1411{
1412	if (LLAssetType::AT_TEXTURE == asset_type)
1413	{
1414		uuid_tempdata_map::const_iterator citer = mTempAssets.find(asset_id);
1415		if (citer != mTempAssets.end())
1416		{
1417			const std::string& host_name = citer->second.mHostName;
1418			std::string url = llformat(LOCAL_ASSET_URL_FORMAT, host_name.c_str());
1419			return url;
1420		}
1421	}
1422
1423	return mBaseURL;
1424}
1425
1426void LLHTTPAssetStorage::dumpTempAssetData(const LLUUID& avatar_id) const
1427{
1428	uuid_tempdata_map::const_iterator it = mTempAssets.begin();
1429	uuid_tempdata_map::const_iterator end = mTempAssets.end();
1430	S32 count = 0;
1431	for ( ; it != end; ++it)
1432	{
1433		const LLTempAssetData& temp_asset_data = it->second;
1434		if (avatar_id.isNull()
1435			|| avatar_id == temp_asset_data.mAgentID)
1436		{
1437			llinfos << "TAT: dump agent " << temp_asset_data.mAgentID
1438				<< " texture " << temp_asset_data.mAssetID
1439				<< " host " << temp_asset_data.mHostName
1440				<< llendl;
1441			count++;
1442		}
1443	}
1444
1445	if (avatar_id.isNull())
1446	{
1447		llinfos << "TAT: dumped " << count << " entries for all avatars" << llendl;
1448	}
1449	else
1450	{
1451		llinfos << "TAT: dumped " << count << " entries for avatar " << avatar_id << llendl;
1452	}
1453}
1454
1455void LLHTTPAssetStorage::clearTempAssetData()
1456{
1457	llinfos << "TAT: Clearing temp asset data map" << llendl;
1458	mTempAssets.clear();
1459}