PageRenderTime 983ms CodeModel.GetById 120ms app.highlight 659ms RepoModel.GetById 106ms app.codeStats 1ms

/indra/newview/lltexturefetch.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 2341 lines | 1767 code | 213 blank | 361 comment | 299 complexity | 36b1c05cd48903ad9e843aad5149a48e MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/** 
   2 * @file lltexturefetch.cpp
   3 * @brief Object which fetches textures from the cache and/or network
   4 *
   5 * $LicenseInfo:firstyear=2000&license=viewerlgpl$
   6 * Second Life Viewer Source Code
   7 * Copyright (C) 2010, Linden Research, Inc.
   8 * 
   9 * This library is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU Lesser General Public
  11 * License as published by the Free Software Foundation;
  12 * version 2.1 of the License only.
  13 * 
  14 * This library is distributed in the hope that it will be useful,
  15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 * Lesser General Public License for more details.
  18 * 
  19 * You should have received a copy of the GNU Lesser General Public
  20 * License along with this library; if not, write to the Free Software
  21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  22 * 
  23 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  24 * $/LicenseInfo$
  25 */
  26
  27#include "llviewerprecompiledheaders.h"
  28
  29#include <iostream>
  30#include <map>
  31
  32#include "llstl.h"
  33
  34#include "lltexturefetch.h"
  35
  36#include "llcurl.h"
  37#include "lldir.h"
  38#include "llhttpclient.h"
  39#include "llhttpstatuscodes.h"
  40#include "llimage.h"
  41#include "llimagej2c.h"
  42#include "llimageworker.h"
  43#include "llworkerthread.h"
  44#include "message.h"
  45
  46#include "llagent.h"
  47#include "lltexturecache.h"
  48#include "llviewercontrol.h"
  49#include "llviewertexturelist.h"
  50#include "llviewertexture.h"
  51#include "llviewerregion.h"
  52#include "llviewerstats.h"
  53#include "llviewerassetstats.h"
  54#include "llworld.h"
  55
  56//////////////////////////////////////////////////////////////////////////////
  57class LLTextureFetchWorker : public LLWorkerClass
  58{
  59	friend class LLTextureFetch;
  60	friend class HTTPGetResponder;
  61	
  62private:
  63	class CacheReadResponder : public LLTextureCache::ReadResponder
  64	{
  65	public:
  66		CacheReadResponder(LLTextureFetch* fetcher, const LLUUID& id, LLImageFormatted* image)
  67			: mFetcher(fetcher), mID(id)
  68		{
  69			setImage(image);
  70		}
  71		virtual void completed(bool success)
  72		{
  73			LLTextureFetchWorker* worker = mFetcher->getWorker(mID);
  74			if (worker)
  75			{
  76 				worker->callbackCacheRead(success, mFormattedImage, mImageSize, mImageLocal);
  77			}
  78		}
  79	private:
  80		LLTextureFetch* mFetcher;
  81		LLUUID mID;
  82	};
  83
  84	class CacheWriteResponder : public LLTextureCache::WriteResponder
  85	{
  86	public:
  87		CacheWriteResponder(LLTextureFetch* fetcher, const LLUUID& id)
  88			: mFetcher(fetcher), mID(id)
  89		{
  90		}
  91		virtual void completed(bool success)
  92		{
  93			LLTextureFetchWorker* worker = mFetcher->getWorker(mID);
  94			if (worker)
  95			{
  96				worker->callbackCacheWrite(success);
  97			}
  98		}
  99	private:
 100		LLTextureFetch* mFetcher;
 101		LLUUID mID;
 102	};
 103	
 104	class DecodeResponder : public LLImageDecodeThread::Responder
 105	{
 106	public:
 107		DecodeResponder(LLTextureFetch* fetcher, const LLUUID& id, LLTextureFetchWorker* worker)
 108			: mFetcher(fetcher), mID(id), mWorker(worker)
 109		{
 110		}
 111		virtual void completed(bool success, LLImageRaw* raw, LLImageRaw* aux)
 112		{
 113			LLTextureFetchWorker* worker = mFetcher->getWorker(mID);
 114			if (worker)
 115			{
 116 				worker->callbackDecoded(success, raw, aux);
 117			}
 118		}
 119	private:
 120		LLTextureFetch* mFetcher;
 121		LLUUID mID;
 122		LLTextureFetchWorker* mWorker; // debug only (may get deleted from under us, use mFetcher/mID)
 123	};
 124
 125	struct Compare
 126	{
 127		// lhs < rhs
 128		bool operator()(const LLTextureFetchWorker* lhs, const LLTextureFetchWorker* rhs) const
 129		{
 130			// greater priority is "less"
 131			const F32 lpriority = lhs->mImagePriority;
 132			const F32 rpriority = rhs->mImagePriority;
 133			if (lpriority > rpriority) // higher priority
 134				return true;
 135			else if (lpriority < rpriority)
 136				return false;
 137			else
 138				return lhs < rhs;
 139		}
 140	};
 141	
 142public:
 143	/*virtual*/ bool doWork(S32 param); // Called from LLWorkerThread::processRequest()
 144	/*virtual*/ void finishWork(S32 param, bool completed); // called from finishRequest() (WORK THREAD)
 145	/*virtual*/ bool deleteOK(); // called from update() (WORK THREAD)
 146
 147	~LLTextureFetchWorker();
 148	// void relese() { --mActiveCount; }
 149
 150	S32 callbackHttpGet(const LLChannelDescriptors& channels,
 151						 const LLIOPipe::buffer_ptr_t& buffer,
 152						 bool partial, bool success);
 153	void callbackCacheRead(bool success, LLImageFormatted* image,
 154						   S32 imagesize, BOOL islocal);
 155	void callbackCacheWrite(bool success);
 156	void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux);
 157	
 158	void setGetStatus(U32 status, const std::string& reason)
 159	{
 160		LLMutexLock lock(&mWorkMutex);
 161
 162		mGetStatus = status;
 163		mGetReason = reason;
 164	}
 165
 166	void setCanUseHTTP(bool can_use_http) { mCanUseHTTP = can_use_http; }
 167	bool getCanUseHTTP() const { return mCanUseHTTP; }
 168
 169	LLTextureFetch & getFetcher() { return *mFetcher; }
 170	
 171protected:
 172	LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host,
 173						 F32 priority, S32 discard, S32 size);
 174
 175private:
 176	/*virtual*/ void startWork(S32 param); // called from addWork() (MAIN THREAD)
 177	/*virtual*/ void endWork(S32 param, bool aborted); // called from doWork() (MAIN THREAD)
 178
 179	void resetFormattedData();
 180	
 181	void setImagePriority(F32 priority);
 182	void setDesiredDiscard(S32 discard, S32 size);
 183	bool insertPacket(S32 index, U8* data, S32 size);
 184	void clearPackets();
 185	void setupPacketData();
 186	U32 calcWorkPriority();
 187	void removeFromCache();
 188	bool processSimulatorPackets();
 189	bool writeToCacheComplete();
 190	
 191	void lockWorkMutex() { mWorkMutex.lock(); }
 192	void unlockWorkMutex() { mWorkMutex.unlock(); }
 193
 194private:
 195	enum e_state // mState
 196	{
 197		// NOTE: Affects LLTextureBar::draw in lltextureview.cpp (debug hack)
 198		INVALID = 0,
 199		INIT,
 200		LOAD_FROM_TEXTURE_CACHE,
 201		CACHE_POST,
 202		LOAD_FROM_NETWORK,
 203		LOAD_FROM_SIMULATOR,
 204		SEND_HTTP_REQ,
 205		WAIT_HTTP_REQ,
 206		DECODE_IMAGE,
 207		DECODE_IMAGE_UPDATE,
 208		WRITE_TO_CACHE,
 209		WAIT_ON_WRITE,
 210		DONE
 211	};
 212	enum e_request_state // mSentRequest
 213	{
 214		UNSENT = 0,
 215		QUEUED = 1,
 216		SENT_SIM = 2
 217	};
 218	enum e_write_to_cache_state //mWriteToCacheState
 219	{
 220		NOT_WRITE = 0,
 221		CAN_WRITE = 1,
 222		SHOULD_WRITE = 2
 223	};
 224	static const char* sStateDescs[];
 225	e_state mState;
 226	e_write_to_cache_state mWriteToCacheState;
 227	LLTextureFetch* mFetcher;
 228	LLPointer<LLImageFormatted> mFormattedImage;
 229	LLPointer<LLImageRaw> mRawImage;
 230	LLPointer<LLImageRaw> mAuxImage;
 231	LLUUID mID;
 232	LLHost mHost;
 233	std::string mUrl;
 234	U8 mType;
 235	F32 mImagePriority;
 236	U32 mWorkPriority;
 237	F32 mRequestedPriority;
 238	S32 mDesiredDiscard;
 239	S32 mSimRequestedDiscard;
 240	S32 mRequestedDiscard;
 241	S32 mLoadedDiscard;
 242	S32 mDecodedDiscard;
 243	LLFrameTimer mRequestedTimer;
 244	LLFrameTimer mFetchTimer;
 245	LLTextureCache::handle_t mCacheReadHandle;
 246	LLTextureCache::handle_t mCacheWriteHandle;
 247	U8* mBuffer;
 248	S32 mBufferSize;
 249	S32 mRequestedSize;
 250	S32 mDesiredSize;
 251	S32 mFileSize;
 252	S32 mCachedSize;	
 253	e_request_state mSentRequest;
 254	handle_t mDecodeHandle;
 255	BOOL mLoaded;
 256	BOOL mDecoded;
 257	BOOL mWritten;
 258	BOOL mNeedsAux;
 259	BOOL mHaveAllData;
 260	BOOL mInLocalCache;
 261	bool mCanUseHTTP ;
 262	bool mCanUseNET ; //can get from asset server.
 263	S32 mHTTPFailCount;
 264	S32 mRetryAttempt;
 265	S32 mActiveCount;
 266	U32 mGetStatus;
 267	std::string mGetReason;
 268	
 269	// Work Data
 270	LLMutex mWorkMutex;
 271	struct PacketData
 272	{
 273		PacketData(U8* data, S32 size) { mData = data; mSize = size; }
 274		~PacketData() { clearData(); }
 275		void clearData() { delete[] mData; mData = NULL; }
 276		U8* mData;
 277		U32 mSize;
 278	};
 279	std::vector<PacketData*> mPackets;
 280	S32 mFirstPacket;
 281	S32 mLastPacket;
 282	U16 mTotalPackets;
 283	U8 mImageCodec;
 284
 285	LLViewerAssetStats::duration_t mMetricsStartTime;
 286};
 287
 288//////////////////////////////////////////////////////////////////////////////
 289
 290class HTTPGetResponder : public LLCurl::Responder
 291{
 292	LOG_CLASS(HTTPGetResponder);
 293public:
 294	HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir)
 295		: mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir)
 296	{
 297	}
 298	~HTTPGetResponder()
 299	{
 300	}
 301
 302	virtual void completedRaw(U32 status, const std::string& reason,
 303							  const LLChannelDescriptors& channels,
 304							  const LLIOPipe::buffer_ptr_t& buffer)
 305	{
 306		static LLCachedControl<bool> log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog");
 307		static LLCachedControl<bool> log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator");
 308		static LLCachedControl<bool> log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ;
 309
 310		if (log_to_viewer_log || log_to_sim)
 311		{
 312			mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime);
 313			U64 timeNow = LLTimer::getTotalTime();
 314			mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP);
 315			mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize);
 316			mFetcher->mTextureInfo.setRequestOffset(mID, mOffset);
 317			mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow);
 318		}
 319
 320		lldebugs << "HTTP COMPLETE: " << mID << llendl;
 321		LLTextureFetchWorker* worker = mFetcher->getWorker(mID);
 322		if (worker)
 323		{
 324			bool success = false;
 325			bool partial = false;
 326			if (HTTP_OK <= status &&  status < HTTP_MULTIPLE_CHOICES)
 327			{
 328				success = true;
 329				if (HTTP_PARTIAL_CONTENT == status) // partial information
 330				{
 331					partial = true;
 332				}
 333			}
 334
 335			if (!success)
 336			{
 337				worker->setGetStatus(status, reason);
 338// 				llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl;
 339			}
 340			
 341			S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success);
 342			
 343			if(log_texture_traffic && data_size > 0)
 344			{
 345				LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ;
 346				if(tex)
 347				{
 348					gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ;
 349				}
 350			}
 351
 352			mFetcher->removeFromHTTPQueue(mID, data_size);
 353
 354			if (worker->mMetricsStartTime)
 355			{
 356				LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE,
 357															  true,
 358															  LLImageBase::TYPE_AVATAR_BAKE == worker->mType,
 359															  LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime);
 360				worker->mMetricsStartTime = 0;
 361			}
 362			LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE,
 363														 true,
 364														 LLImageBase::TYPE_AVATAR_BAKE == worker->mType);
 365		}
 366		else
 367		{
 368			mFetcher->removeFromHTTPQueue(mID);
 369 			llwarns << "Worker not found: " << mID << llendl;
 370		}
 371	}
 372
 373	virtual bool followRedir()
 374	{
 375		return mFollowRedir;
 376	}
 377	
 378private:
 379	LLTextureFetch* mFetcher;
 380	LLUUID mID;
 381	U64 mStartTime;
 382	S32 mRequestedSize;
 383	U32 mOffset;
 384	bool mFollowRedir;
 385};
 386
 387//////////////////////////////////////////////////////////////////////////////
 388
 389// Cross-thread messaging for asset metrics.
 390
 391/**
 392 * @brief Base class for cross-thread requests made of the fetcher
 393 *
 394 * I believe the intent of the LLQueuedThread class was to
 395 * have these operations derived from LLQueuedThread::QueuedRequest
 396 * but the texture fetcher has elected to manage the queue
 397 * in its own manner.  So these are free-standing objects which are
 398 * managed in simple FIFO order on the mCommands queue of the
 399 * LLTextureFetch object.
 400 *
 401 * What each represents is a simple command sent from an
 402 * outside thread into the TextureFetch thread to be processed
 403 * in order and in a timely fashion (though not an absolute
 404 * higher priority than other operations of the thread).
 405 * Each operation derives a new class from the base customizing
 406 * members, constructors and the doWork() method to effect
 407 * the command.
 408 *
 409 * The flow is one-directional.  There are two global instances
 410 * of the LLViewerAssetStats collector, one for the main program's
 411 * thread pointed to by gViewerAssetStatsMain and one for the
 412 * TextureFetch thread pointed to by gViewerAssetStatsThread1.
 413 * Common operations has each thread recording metrics events
 414 * into the respective collector unconcerned with locking and
 415 * the state of any other thread.  But when the agent moves into
 416 * a different region or the metrics timer expires and a report
 417 * needs to be sent back to the grid, messaging across threads
 418 * is required to distribute data and perform global actions.
 419 * In pseudo-UML, it looks like:
 420 *
 421 *                       Main                 Thread1
 422 *                        .                      .
 423 *                        .                      .
 424 *                     +-----+                   .
 425 *                     | AM  |                   .
 426 *                     +--+--+                   .
 427 *      +-------+         |                      .
 428 *      | Main  |      +--+--+                   .
 429 *      |       |      | SRE |---.               .
 430 *      | Stats |      +-----+    \              .
 431 *      |       |         |        \  (uuid)  +-----+
 432 *      | Coll. |      +--+--+      `-------->| SR  |
 433 *      +-------+      | MSC |                +--+--+
 434 *         | ^         +-----+                   |
 435 *         | |  (uuid)  / .                   +-----+ (uuid)
 436 *         |  `--------'  .                   | MSC |---------.
 437 *         |              .                   +-----+         |
 438 *         |           +-----+                   .            v
 439 *         |           | TE  |                   .        +-------+
 440 *         |           +--+--+                   .        | Thd1  |
 441 *         |              |                      .        |       |
 442 *         |           +-----+                   .        | Stats |
 443 *          `--------->| RSC |                   .        |       |
 444 *                     +--+--+                   .        | Coll. |
 445 *                        |                      .        +-------+
 446 *                     +--+--+                   .            |
 447 *                     | SME |---.               .            |
 448 *                     +-----+    \              .            |
 449 *                        .        \ (clone)  +-----+         |
 450 *                        .         `-------->| SM  |         |
 451 *                        .                   +--+--+         |
 452 *                        .                      |            |
 453 *                        .                   +-----+         |
 454 *                        .                   | RSC |<--------'
 455 *                        .                   +-----+
 456 *                        .                      |
 457 *                        .                   +-----+
 458 *                        .                   | CP  |--> HTTP POST
 459 *                        .                   +-----+
 460 *                        .                      .
 461 *                        .                      .
 462 *
 463 *
 464 * Key:
 465 *
 466 * SRE - Set Region Enqueued.  Enqueue a 'Set Region' command in
 467 *       the other thread providing the new UUID of the region.
 468 *       TFReqSetRegion carries the data.
 469 * SR  - Set Region.  New region UUID is sent to the thread-local
 470 *       collector.
 471 * SME - Send Metrics Enqueued.  Enqueue a 'Send Metrics' command
 472 *       including an ownership transfer of a cloned LLViewerAssetStats.
 473 *       TFReqSendMetrics carries the data.
 474 * SM  - Send Metrics.  Global metrics reporting operation.  Takes
 475 *       the cloned stats from the command, merges it with the
 476 *       thread's local stats, converts to LLSD and sends it on
 477 *       to the grid.
 478 * AM  - Agent Moved.  Agent has completed some sort of move to a
 479 *       new region.
 480 * TE  - Timer Expired.  Metrics timer has expired (on the order
 481 *       of 10 minutes).
 482 * CP  - CURL Post
 483 * MSC - Modify Stats Collector.  State change in the thread-local
 484 *       collector.  Typically a region change which affects the
 485 *       global pointers used to find the 'current stats'.
 486 * RSC - Read Stats Collector.  Extract collector data cloning it
 487 *       (i.e. deep copy) when necessary.
 488 *
 489 */
 490class LLTextureFetch::TFRequest // : public LLQueuedThread::QueuedRequest
 491{
 492public:
 493	// Default ctors and assignment operator are correct.
 494
 495	virtual ~TFRequest()
 496		{}
 497
 498	// Patterned after QueuedRequest's method but expected behavior
 499	// is different.  Always expected to complete on the first call
 500	// and work dispatcher will assume the same and delete the
 501	// request after invocation.
 502	virtual bool doWork(LLTextureFetch * fetcher) = 0;
 503};
 504
 505namespace 
 506{
 507
 508/**
 509 * @brief Implements a 'Set Region' cross-thread command.
 510 *
 511 * When an agent moves to a new region, subsequent metrics need
 512 * to be binned into a new or existing stats collection in 1:1
 513 * relationship with the region.  We communicate this region
 514 * change across the threads involved in the communication with
 515 * this message.
 516 *
 517 * Corresponds to LLTextureFetch::commandSetRegion()
 518 */
 519class TFReqSetRegion : public LLTextureFetch::TFRequest
 520{
 521public:
 522	TFReqSetRegion(U64 region_handle)
 523		: LLTextureFetch::TFRequest(),
 524		  mRegionHandle(region_handle)
 525		{}
 526	TFReqSetRegion & operator=(const TFReqSetRegion &);	// Not defined
 527
 528	virtual ~TFReqSetRegion()
 529		{}
 530
 531	virtual bool doWork(LLTextureFetch * fetcher);
 532		
 533public:
 534	const U64 mRegionHandle;
 535};
 536
 537
 538/**
 539 * @brief Implements a 'Send Metrics' cross-thread command.
 540 *
 541 * This is the big operation.  The main thread gathers metrics
 542 * for a period of minutes into LLViewerAssetStats and other
 543 * objects then makes a snapshot of the data by cloning the
 544 * collector.  This command transfers the clone, along with a few
 545 * additional arguments (UUIDs), handing ownership to the
 546 * TextureFetch thread.  It then merges its own data into the
 547 * cloned copy, converts to LLSD and kicks off an HTTP POST of
 548 * the resulting data to the currently active metrics collector.
 549 *
 550 * Corresponds to LLTextureFetch::commandSendMetrics()
 551 */
 552class TFReqSendMetrics : public LLTextureFetch::TFRequest
 553{
 554public:
 555    /**
 556	 * Construct the 'Send Metrics' command to have the TextureFetch
 557	 * thread add and log metrics data.
 558	 *
 559	 * @param	caps_url		URL of a "ViewerMetrics" Caps target
 560	 *							to receive the data.  Does not have to
 561	 *							be associated with a particular region.
 562	 *
 563	 * @param	session_id		UUID of the agent's session.
 564	 *
 565	 * @param	agent_id		UUID of the agent.  (Being pure here...)
 566	 *
 567	 * @param	main_stats		Pointer to a clone of the main thread's
 568	 *							LLViewerAssetStats data.  Thread1 takes
 569	 *							ownership of the copy and disposes of it
 570	 *							when done.
 571	 */
 572	TFReqSendMetrics(const std::string & caps_url,
 573					 const LLUUID & session_id,
 574					 const LLUUID & agent_id,
 575					 LLViewerAssetStats * main_stats)
 576		: LLTextureFetch::TFRequest(),
 577		  mCapsURL(caps_url),
 578		  mSessionID(session_id),
 579		  mAgentID(agent_id),
 580		  mMainStats(main_stats)
 581		{}
 582	TFReqSendMetrics & operator=(const TFReqSendMetrics &);	// Not defined
 583
 584	virtual ~TFReqSendMetrics();
 585
 586	virtual bool doWork(LLTextureFetch * fetcher);
 587		
 588public:
 589	const std::string mCapsURL;
 590	const LLUUID mSessionID;
 591	const LLUUID mAgentID;
 592	LLViewerAssetStats * mMainStats;
 593};
 594
 595/*
 596 * Examines the merged viewer metrics report and if found to be too long,
 597 * will attempt to truncate it in some reasonable fashion.
 598 *
 599 * @param		max_regions		Limit of regions allowed in report.
 600 *
 601 * @param		metrics			Full, merged viewer metrics report.
 602 *
 603 * @returns		If data was truncated, returns true.
 604 */
 605bool truncate_viewer_metrics(int max_regions, LLSD & metrics);
 606
 607} // end of anonymous namespace
 608
 609
 610//////////////////////////////////////////////////////////////////////////////
 611
 612//static
 613const char* LLTextureFetchWorker::sStateDescs[] = {
 614	"INVALID",
 615	"INIT",
 616	"LOAD_FROM_TEXTURE_CACHE",
 617	"CACHE_POST",
 618	"LOAD_FROM_NETWORK",
 619	"LOAD_FROM_SIMULATOR",
 620	"SEND_HTTP_REQ",
 621	"WAIT_HTTP_REQ",
 622	"DECODE_IMAGE",
 623	"DECODE_IMAGE_UPDATE",
 624	"WRITE_TO_CACHE",
 625	"WAIT_ON_WRITE",
 626	"DONE",
 627};
 628
 629// static
 630volatile bool LLTextureFetch::svMetricsDataBreak(true);	// Start with a data break
 631
 632// called from MAIN THREAD
 633
 634LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher,
 635										   const std::string& url, // Optional URL
 636										   const LLUUID& id,	// Image UUID
 637										   const LLHost& host,	// Simulator host
 638										   F32 priority,		// Priority
 639										   S32 discard,			// Desired discard
 640										   S32 size)			// Desired size
 641	: LLWorkerClass(fetcher, "TextureFetch"),
 642	  mState(INIT),
 643	  mWriteToCacheState(NOT_WRITE),
 644	  mFetcher(fetcher),
 645	  mID(id),
 646	  mHost(host),
 647	  mUrl(url),
 648	  mImagePriority(priority),
 649	  mWorkPriority(0),
 650	  mRequestedPriority(0.f),
 651	  mDesiredDiscard(-1),
 652	  mSimRequestedDiscard(-1),
 653	  mRequestedDiscard(-1),
 654	  mLoadedDiscard(-1),
 655	  mDecodedDiscard(-1),
 656	  mCacheReadHandle(LLTextureCache::nullHandle()),
 657	  mCacheWriteHandle(LLTextureCache::nullHandle()),
 658	  mBuffer(NULL),
 659	  mBufferSize(0),
 660	  mRequestedSize(0),
 661	  mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE),
 662	  mFileSize(0),
 663	  mCachedSize(0),
 664	  mLoaded(FALSE),
 665	  mSentRequest(UNSENT),
 666	  mDecodeHandle(0),
 667	  mDecoded(FALSE),
 668	  mWritten(FALSE),
 669	  mNeedsAux(FALSE),
 670	  mHaveAllData(FALSE),
 671	  mInLocalCache(FALSE),
 672	  mCanUseHTTP(true),
 673	  mHTTPFailCount(0),
 674	  mRetryAttempt(0),
 675	  mActiveCount(0),
 676	  mGetStatus(0),
 677	  mWorkMutex(NULL),
 678	  mFirstPacket(0),
 679	  mLastPacket(-1),
 680	  mTotalPackets(0),
 681	  mImageCodec(IMG_CODEC_INVALID),
 682	  mMetricsStartTime(0)
 683{
 684	mCanUseNET = mUrl.empty() ;
 685
 686	calcWorkPriority();
 687	mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL;
 688// 	llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << llendl;
 689	if (!mFetcher->mDebugPause)
 690	{
 691		U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH;
 692		addWork(0, work_priority );
 693	}
 694	setDesiredDiscard(discard, size);
 695}
 696
 697LLTextureFetchWorker::~LLTextureFetchWorker()
 698{
 699// 	llinfos << "Destroy: " << mID
 700// 			<< " Decoded=" << mDecodedDiscard
 701// 			<< " Requested=" << mRequestedDiscard
 702// 			<< " Desired=" << mDesiredDiscard << llendl;
 703	llassert_always(!haveWork());
 704	lockWorkMutex();
 705	if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache)
 706	{
 707		mFetcher->mTextureCache->readComplete(mCacheReadHandle, true);
 708	}
 709	if (mCacheWriteHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache)
 710	{
 711		mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true);
 712	}
 713	mFormattedImage = NULL;
 714	clearPackets();
 715	unlockWorkMutex();
 716	mFetcher->removeFromHTTPQueue(mID);
 717}
 718
 719void LLTextureFetchWorker::clearPackets()
 720{
 721	for_each(mPackets.begin(), mPackets.end(), DeletePointer());
 722	mPackets.clear();
 723	mTotalPackets = 0;
 724	mLastPacket = -1;
 725	mFirstPacket = 0;
 726}
 727
 728void LLTextureFetchWorker::setupPacketData()
 729{
 730	S32 data_size = 0;
 731	if (mFormattedImage.notNull())
 732	{
 733		data_size = mFormattedImage->getDataSize();
 734	}
 735	if (data_size > 0)
 736	{
 737		// Only used for simulator requests
 738		mFirstPacket = (data_size - FIRST_PACKET_SIZE) / MAX_IMG_PACKET_SIZE + 1;
 739		if (FIRST_PACKET_SIZE + (mFirstPacket-1) * MAX_IMG_PACKET_SIZE != data_size)
 740		{
 741			llwarns << "Bad CACHED TEXTURE size: " << data_size << " removing." << llendl;
 742			removeFromCache();
 743			resetFormattedData();
 744			clearPackets();
 745		}
 746		else if (mFileSize > 0)
 747		{
 748			mLastPacket = mFirstPacket-1;
 749			mTotalPackets = (mFileSize - FIRST_PACKET_SIZE + MAX_IMG_PACKET_SIZE-1) / MAX_IMG_PACKET_SIZE + 1;
 750		}
 751		else
 752		{
 753			// This file was cached using HTTP so we have to refetch the first packet
 754			resetFormattedData();
 755			clearPackets();
 756		}
 757	}
 758}
 759
 760U32 LLTextureFetchWorker::calcWorkPriority()
 761{
 762 	//llassert_always(mImagePriority >= 0 && mImagePriority <= LLViewerFetchedTexture::maxDecodePriority());
 763	static const F32 PRIORITY_SCALE = (F32)LLWorkerThread::PRIORITY_LOWBITS / LLViewerFetchedTexture::maxDecodePriority();
 764
 765	mWorkPriority = llmin((U32)LLWorkerThread::PRIORITY_LOWBITS, (U32)(mImagePriority * PRIORITY_SCALE));
 766	return mWorkPriority;
 767}
 768
 769// mWorkMutex is locked
 770void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size)
 771{
 772	bool prioritize = false;
 773	if (mDesiredDiscard != discard)
 774	{
 775		if (!haveWork())
 776		{
 777			calcWorkPriority();
 778			if (!mFetcher->mDebugPause)
 779			{
 780				U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH;
 781				addWork(0, work_priority);
 782			}
 783		}
 784		else if (mDesiredDiscard < discard)
 785		{
 786			prioritize = true;
 787		}
 788		mDesiredDiscard = discard;
 789		mDesiredSize = size;
 790	}
 791	else if (size > mDesiredSize)
 792	{
 793		mDesiredSize = size;
 794		prioritize = true;
 795	}
 796	mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE);
 797	if ((prioritize && mState == INIT) || mState == DONE)
 798	{
 799		mState = INIT;
 800		U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH;
 801		setPriority(work_priority);
 802	}
 803}
 804
 805void LLTextureFetchWorker::setImagePriority(F32 priority)
 806{
 807// 	llassert_always(priority >= 0 && priority <= LLViewerTexture::maxDecodePriority());
 808	F32 delta = fabs(priority - mImagePriority);
 809	if (delta > (mImagePriority * .05f) || mState == DONE)
 810	{
 811		mImagePriority = priority;
 812		calcWorkPriority();
 813		U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS);
 814		setPriority(work_priority);
 815	}
 816}
 817
 818void LLTextureFetchWorker::resetFormattedData()
 819{
 820	FREE_MEM(LLImageBase::getPrivatePool(), mBuffer);
 821	mBuffer = NULL;
 822	mBufferSize = 0;
 823	if (mFormattedImage.notNull())
 824	{
 825		mFormattedImage->deleteData();
 826	}
 827	mHaveAllData = FALSE;
 828}
 829
 830// Called from MAIN thread
 831void LLTextureFetchWorker::startWork(S32 param)
 832{
 833	llassert(mFormattedImage.isNull());
 834}
 835
 836#include "llviewertexturelist.h" // debug
 837
 838// Called from LLWorkerThread::processRequest()
 839bool LLTextureFetchWorker::doWork(S32 param)
 840{
 841	LLMutexLock lock(&mWorkMutex);
 842
 843	if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED)))
 844	{
 845		if (mState < DECODE_IMAGE)
 846		{
 847			return true; // abort
 848		}
 849	}
 850
 851	if(mImagePriority < F_ALMOST_ZERO)
 852	{
 853		if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR)
 854		{
 855			return true; // abort
 856		}
 857	}
 858	if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP)
 859	{
 860		//nowhere to get data, abort.
 861		return true ;
 862	}
 863
 864	if (mFetcher->mDebugPause)
 865	{
 866		return false; // debug: don't do any work
 867	}
 868	if (mID == mFetcher->mDebugID)
 869	{
 870		mFetcher->mDebugCount++; // for setting breakpoints
 871	}
 872
 873	if (mState != DONE)
 874	{
 875		mFetchTimer.reset();
 876	}
 877
 878	if (mState == INIT)
 879	{		
 880		mRawImage = NULL ;
 881		mRequestedDiscard = -1;
 882		mLoadedDiscard = -1;
 883		mDecodedDiscard = -1;
 884		mRequestedSize = 0;
 885		mFileSize = 0;
 886		mCachedSize = 0;
 887		mLoaded = FALSE;
 888		mSentRequest = UNSENT;
 889		mDecoded  = FALSE;
 890		mWritten  = FALSE;
 891		FREE_MEM(LLImageBase::getPrivatePool(), mBuffer);
 892		mBuffer = NULL;
 893		mBufferSize = 0;
 894		mHaveAllData = FALSE;
 895		clearPackets(); // TODO: Shouldn't be necessary
 896		mCacheReadHandle = LLTextureCache::nullHandle();
 897		mCacheWriteHandle = LLTextureCache::nullHandle();
 898		mState = LOAD_FROM_TEXTURE_CACHE;
 899		mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); // min desired size is TEXTURE_CACHE_ENTRY_SIZE
 900		LL_DEBUGS("Texture") << mID << ": Priority: " << llformat("%8.0f",mImagePriority)
 901							 << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL;
 902		// fall through
 903	}
 904
 905	if (mState == LOAD_FROM_TEXTURE_CACHE)
 906	{
 907		if (mCacheReadHandle == LLTextureCache::nullHandle())
 908		{
 909			U32 cache_priority = mWorkPriority;
 910			S32 offset = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0;
 911			S32 size = mDesiredSize - offset;
 912			if (size <= 0)
 913			{
 914				mState = CACHE_POST;
 915				return false;
 916			}
 917			mFileSize = 0;
 918			mLoaded = FALSE;			
 919			
 920			if (mUrl.compare(0, 7, "file://") == 0)
 921			{
 922				setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it
 923
 924				// read file from local disk
 925				std::string filename = mUrl.substr(7, std::string::npos);
 926				CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage);
 927				mCacheReadHandle = mFetcher->mTextureCache->readFromCache(filename, mID, cache_priority,
 928																		  offset, size, responder);
 929			}
 930			else if (mUrl.empty())
 931			{
 932				setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it
 933
 934				CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage);
 935				mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority,
 936																		  offset, size, responder);
 937			}
 938			else if(mCanUseHTTP)
 939			{
 940				if (!(mUrl.compare(0, 7, "http://") == 0))
 941				{
 942					// *TODO:?remove this warning
 943					llwarns << "Unknown URL Type: " << mUrl << llendl;
 944				}
 945				setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
 946				mState = SEND_HTTP_REQ;
 947			}
 948			else
 949			{
 950				setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
 951				mState = LOAD_FROM_NETWORK;
 952			}
 953		}
 954
 955		if (mLoaded)
 956		{
 957			// Make sure request is complete. *TODO: make this auto-complete
 958			if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, false))
 959			{
 960				mCacheReadHandle = LLTextureCache::nullHandle();
 961				mState = CACHE_POST;
 962				// fall through
 963			}
 964			else
 965			{
 966				return false;
 967			}
 968		}
 969		else
 970		{
 971			return false;
 972		}
 973	}
 974
 975	if (mState == CACHE_POST)
 976	{
 977		mCachedSize = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0;
 978		// Successfully loaded
 979		if ((mCachedSize >= mDesiredSize) || mHaveAllData)
 980		{
 981			// we have enough data, decode it
 982			llassert_always(mFormattedImage->getDataSize() > 0);
 983			mLoadedDiscard = mDesiredDiscard;
 984			mState = DECODE_IMAGE;
 985			mWriteToCacheState = NOT_WRITE ;
 986			LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize()
 987								 << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight())
 988								 << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL;
 989			// fall through
 990		}
 991		else
 992		{
 993			if (mUrl.compare(0, 7, "file://") == 0)
 994			{
 995				// failed to load local file, we're done.
 996				return true;
 997			}
 998			// need more data
 999			else
1000			{
1001				LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL;
1002				mState = LOAD_FROM_NETWORK;
1003			}
1004			// fall through
1005		}
1006	}
1007
1008	if (mState == LOAD_FROM_NETWORK)
1009	{
1010		static LLCachedControl<bool> use_http(gSavedSettings,"ImagePipelineUseHTTP");
1011
1012// 		if (mHost != LLHost::invalid) get_url = false;
1013		if ( use_http && mCanUseHTTP && mUrl.empty())//get http url.
1014		{
1015			LLViewerRegion* region = NULL;
1016			if (mHost == LLHost::invalid)
1017				region = gAgent.getRegion();
1018			else
1019				region = LLWorld::getInstance()->getRegion(mHost);
1020
1021			if (region)
1022			{
1023				std::string http_url = region->getHttpUrl() ;
1024				if (!http_url.empty())
1025				{
1026					mUrl = http_url + "/?texture_id=" + mID.asString().c_str();
1027					mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id.
1028				}
1029				else
1030				{
1031					mCanUseHTTP = false ;
1032				}
1033			}
1034			else
1035			{
1036				// This will happen if not logged in or if a region deoes not have HTTP Texture enabled
1037				//llwarns << "Region not found for host: " << mHost << llendl;
1038				mCanUseHTTP = false;
1039			}
1040		}
1041		if (mCanUseHTTP && !mUrl.empty())
1042		{
1043			mState = LLTextureFetchWorker::SEND_HTTP_REQ;
1044			setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1045			if(mWriteToCacheState != NOT_WRITE)
1046			{
1047				mWriteToCacheState = CAN_WRITE ;
1048			}
1049			// don't return, fall through to next state
1050		}
1051		else if (mSentRequest == UNSENT && mCanUseNET)
1052		{
1053			// Add this to the network queue and sit here.
1054			// LLTextureFetch::update() will send off a request which will change our state
1055			mWriteToCacheState = CAN_WRITE ;
1056			mRequestedSize = mDesiredSize;
1057			mRequestedDiscard = mDesiredDiscard;
1058			mSentRequest = QUEUED;
1059			mFetcher->addToNetworkQueue(this);
1060			if (! mMetricsStartTime)
1061			{
1062				mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp();
1063			}
1064			LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE,
1065														 false,
1066														 LLImageBase::TYPE_AVATAR_BAKE == mType);
1067			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1068			
1069			return false;
1070		}
1071		else
1072		{
1073			// Shouldn't need to do anything here
1074			//llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end());
1075			// Make certain this is in the network queue
1076			//mFetcher->addToNetworkQueue(this);
1077			//if (! mMetricsStartTime)
1078			//{
1079			//   mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp();
1080			//}
1081			//LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false,
1082			//                                             LLImageBase::TYPE_AVATAR_BAKE == mType);
1083			//setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1084			return false;
1085		}
1086	}
1087	
1088	if (mState == LOAD_FROM_SIMULATOR)
1089	{
1090		if (mFormattedImage.isNull())
1091		{
1092			mFormattedImage = new LLImageJ2C;
1093		}
1094		if (processSimulatorPackets())
1095		{
1096			LL_DEBUGS("Texture") << mID << ": Loaded from Sim. Bytes: " << mFormattedImage->getDataSize() << LL_ENDL;
1097			mFetcher->removeFromNetworkQueue(this, false);
1098			if (mFormattedImage.isNull() || !mFormattedImage->getDataSize())
1099			{
1100				// processSimulatorPackets() failed
1101// 				llwarns << "processSimulatorPackets() failed to load buffer" << llendl;
1102				return true; // failed
1103			}
1104			setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1105			mState = DECODE_IMAGE;
1106			mWriteToCacheState = SHOULD_WRITE;
1107
1108			if (mMetricsStartTime)
1109			{
1110				LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE,
1111															  false,
1112															  LLImageBase::TYPE_AVATAR_BAKE == mType,
1113															  LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime);
1114				mMetricsStartTime = 0;
1115			}
1116			LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE,
1117														 false,
1118														 LLImageBase::TYPE_AVATAR_BAKE == mType);
1119		}
1120		else
1121		{
1122			mFetcher->addToNetworkQueue(this); // failsafe
1123			if (! mMetricsStartTime)
1124			{
1125				mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp();
1126			}
1127			LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE,
1128														 false,
1129														 LLImageBase::TYPE_AVATAR_BAKE == mType);
1130			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1131		}
1132		return false;
1133	}
1134	
1135	if (mState == SEND_HTTP_REQ)
1136	{
1137		if(mCanUseHTTP)
1138		{
1139			//NOTE:
1140			//control the number of the http requests issued for:
1141			//1, not openning too many file descriptors at the same time;
1142			//2, control the traffic of http so udp gets bandwidth.
1143			//
1144			static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ;
1145			if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE)
1146			{
1147				return false ; //wait.
1148			}
1149
1150			mFetcher->removeFromNetworkQueue(this, false);
1151			
1152			S32 cur_size = 0;
1153			if (mFormattedImage.notNull())
1154			{
1155				cur_size = mFormattedImage->getDataSize(); // amount of data we already have
1156				if (mFormattedImage->getDiscardLevel() == 0)
1157				{
1158					if(cur_size > 0)
1159					{
1160						// We already have all the data, just decode it
1161						mLoadedDiscard = mFormattedImage->getDiscardLevel();
1162						mState = DECODE_IMAGE;
1163						return false;
1164					}
1165					else
1166					{
1167						return true ; //abort.
1168					}
1169				}
1170			}
1171			mRequestedSize = mDesiredSize;
1172			mRequestedDiscard = mDesiredDiscard;
1173			mRequestedSize -= cur_size;
1174			S32 offset = cur_size;
1175			mBufferSize = cur_size; // This will get modified by callbackHttpGet()
1176			
1177			bool res = false;
1178			if (!mUrl.empty())
1179			{
1180				mLoaded = FALSE;
1181				mGetStatus = 0;
1182				mGetReason.clear();
1183				LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset
1184									 << " Bytes: " << mRequestedSize
1185									 << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth
1186									 << LL_ENDL;
1187				setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1188				mState = WAIT_HTTP_REQ;	
1189
1190				mFetcher->addToHTTPQueue(mID);
1191				if (! mMetricsStartTime)
1192				{
1193					mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp();
1194				}
1195				LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE,
1196															 true,
1197															 LLImageBase::TYPE_AVATAR_BAKE == mType);
1198
1199				// Will call callbackHttpGet when curl request completes
1200				std::vector<std::string> headers;
1201				headers.push_back("Accept: image/x-j2c");
1202				res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize,
1203															  new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true));
1204			}
1205			if (!res)
1206			{
1207				llwarns << "HTTP GET request failed for " << mID << llendl;
1208				resetFormattedData();
1209				++mHTTPFailCount;
1210				return true; // failed
1211			}
1212			// fall through
1213		}
1214		else //can not use http fetch.
1215		{
1216			return true ; //abort
1217		}
1218	}
1219	
1220	if (mState == WAIT_HTTP_REQ)
1221	{
1222		if (mLoaded)
1223		{
1224			S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0;
1225			if (mRequestedSize < 0)
1226			{
1227				S32 max_attempts;
1228				if (mGetStatus == HTTP_NOT_FOUND)
1229				{
1230					mHTTPFailCount = max_attempts = 1; // Don't retry
1231					llwarns << "Texture missing from server (404): " << mUrl << llendl;
1232
1233					//roll back to try UDP
1234					if(mCanUseNET)
1235					{
1236						mState = INIT ;
1237						mCanUseHTTP = false ;
1238						setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1239						return false ;
1240					}
1241				}
1242				else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE)
1243				{
1244					// *TODO: Should probably introduce a timer here to delay future HTTP requsts
1245					// for a short time (~1s) to ease server load? Ideally the server would queue
1246					// requests instead of returning 503... we already limit the number pending.
1247					++mHTTPFailCount;
1248					max_attempts = mHTTPFailCount+1; // Keep retrying
1249					LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL;
1250				}
1251				else
1252				{
1253					const S32 HTTP_MAX_RETRY_COUNT = 3;
1254					max_attempts = HTTP_MAX_RETRY_COUNT + 1;
1255					++mHTTPFailCount;
1256					llinfos << "HTTP GET failed for: " << mUrl
1257							<< " Status: " << mGetStatus << " Reason: '" << mGetReason << "'"
1258							<< " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl;
1259				}
1260
1261				if (mHTTPFailCount >= max_attempts)
1262				{
1263					if (cur_size > 0)
1264					{
1265						// Use available data
1266						mLoadedDiscard = mFormattedImage->getDiscardLevel();
1267						mState = DECODE_IMAGE;
1268						return false; 
1269					}
1270					else
1271					{
1272						resetFormattedData();
1273						mState = DONE;
1274						return true; // failed
1275					}
1276				}
1277				else
1278				{
1279					mState = SEND_HTTP_REQ;
1280					return false; // retry
1281				}
1282			}
1283			
1284			llassert_always(mBufferSize == cur_size + mRequestedSize);
1285			if(!mBufferSize)//no data received.
1286			{
1287				FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); 
1288				mBuffer = NULL;
1289
1290				//abort.
1291				mState = DONE;
1292				return true;
1293			}
1294
1295			if (mFormattedImage.isNull())
1296			{
1297				// For now, create formatted image based on extension
1298				std::string extension = gDirUtilp->getExtension(mUrl);
1299				mFormattedImage = LLImageFormatted::createFromType(LLImageBase::getCodecFromExtension(extension));
1300				if (mFormattedImage.isNull())
1301				{
1302					mFormattedImage = new LLImageJ2C; // default
1303				}
1304			}
1305						
1306			if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded.
1307			{
1308				mFileSize = mBufferSize;
1309			}
1310			else //the file size is unknown.
1311			{
1312				mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded.
1313			}
1314			
1315			U8* buffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mBufferSize);
1316			if (cur_size > 0)
1317			{
1318				memcpy(buffer, mFormattedImage->getData(), cur_size);
1319			}
1320			memcpy(buffer + cur_size, mBuffer, mRequestedSize); // append
1321			// NOTE: setData releases current data and owns new data (buffer)
1322			mFormattedImage->setData(buffer, mBufferSize);
1323			// delete temp data
1324			FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); // Note: not 'buffer' (assigned in setData())
1325			mBuffer = NULL;
1326			mBufferSize = 0;
1327			mLoadedDiscard = mRequestedDiscard;
1328			mState = DECODE_IMAGE;
1329			if(mWriteToCacheState != NOT_WRITE)
1330			{
1331				mWriteToCacheState = SHOULD_WRITE ;
1332			}
1333			setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1334			return false;
1335		}
1336		else
1337		{
1338			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1339			return false;
1340		}
1341	}
1342	
1343	if (mState == DECODE_IMAGE)
1344	{
1345		static LLCachedControl<bool> textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled");
1346		if(textures_decode_disabled)
1347		{
1348			// for debug use, don't decode
1349			mState = DONE;
1350			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1351			return true;
1352		}
1353
1354		if (mDesiredDiscard < 0)
1355		{
1356			// We aborted, don't decode
1357			mState = DONE;
1358			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1359			return true;
1360		}
1361		
1362		if (mFormattedImage->getDataSize() <= 0)
1363		{
1364			//llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl;
1365			
1366			//abort, don't decode
1367			mState = DONE;
1368			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1369			return true;
1370		}
1371		if (mLoadedDiscard < 0)
1372		{
1373			//llerrs << "Decode entered with invalid mLoadedDiscard. ID = " << mID << llendl;
1374
1375			//abort, don't decode
1376			mState = DONE;
1377			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1378			return true;
1379		}
1380		setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it
1381		mRawImage = NULL;
1382		mAuxImage = NULL;
1383		llassert_always(mFormattedImage.notNull());
1384		S32 discard = mHaveAllData ? 0 : mLoadedDiscard;
1385		U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority;
1386		mDecoded  = FALSE;
1387		mState = DECODE_IMAGE_UPDATE;
1388		LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard
1389				<< " All Data: " << mHaveAllData << LL_ENDL;
1390		mDecodeHandle = mFetcher->mImageDecodeThread->decodeImage(mFormattedImage, image_priority, discard, mNeedsAux,
1391																  new DecodeResponder(mFetcher, mID, this));
1392		// fall though
1393	}
1394	
1395	if (mState == DECODE_IMAGE_UPDATE)
1396	{
1397		if (mDecoded)
1398		{
1399			if (mDecodedDiscard < 0)
1400			{
1401				LL_DEBUGS("Texture") << mID << ": Failed to Decode." << LL_ENDL;
1402				if (mCachedSize > 0 && !mInLocalCache && mRetryAttempt == 0)
1403				{
1404					// Cache file should be deleted, try again
1405// 					llwarns << mID << ": Decode of cached file failed (removed), retrying" << llendl;
1406					llassert_always(mDecodeHandle == 0);
1407					mFormattedImage = NULL;
1408					++mRetryAttempt;
1409					setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1410					mState = INIT;
1411					return false;
1412				}
1413				else
1414				{
1415// 					llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl;
1416					mState = DONE; // failed
1417				}
1418			}
1419			else
1420			{
1421				llassert_always(mRawImage.notNull());
1422				LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard
1423						<< " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL;
1424				setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1425				mState = WRITE_TO_CACHE;
1426			}
1427			// fall through
1428		}
1429		else
1430		{
1431			return false;
1432		}
1433	}
1434
1435	if (mState == WRITE_TO_CACHE)
1436	{
1437		if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull())
1438		{
1439			// If we're in a local cache or we didn't actually receive any new data,
1440			// or we failed to load anything, skip
1441			mState = DONE;
1442			return false;
1443		}
1444		S32 datasize = mFormattedImage->getDataSize();
1445		if(mFileSize < datasize)//This could happen when http fetching and sim fetching mixed.
1446		{
1447			if(mHaveAllData)
1448			{
1449				mFileSize = datasize ;
1450			}
1451			else
1452			{
1453				mFileSize = datasize + 1 ; //flag not fully loaded.
1454			}
1455		}
1456		llassert_always(datasize);
1457		setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it
1458		U32 cache_priority = mWorkPriority;
1459		mWritten = FALSE;
1460		mState = WAIT_ON_WRITE;
1461		CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID);
1462		mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority,
1463																  mFormattedImage->getData(), datasize,
1464																  mFileSize, responder);
1465		// fall through
1466	}
1467	
1468	if (mState == WAIT_ON_WRITE)
1469	{
1470		if (writeToCacheComplete())
1471		{
1472			mState = DONE;
1473			// fall through
1474		}
1475		else
1476		{
1477			if (mDesiredDiscard < mDecodedDiscard)
1478			{
1479				// We're waiting for this write to complete before we can receive more data
1480				// (we can't touch mFormattedImage until the write completes)
1481				// Prioritize the write
1482				mFetcher->mTextureCache->prioritizeWrite(mCacheWriteHandle);
1483			}
1484			return false;
1485		}
1486	}
1487
1488	if (mState == DONE)
1489	{
1490		if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard)
1491		{
1492			// More data was requested, return to INIT
1493			mState = INIT;
1494			setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1495			return false;
1496		}
1497		else
1498		{
1499			setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
1500			return true;
1501		}
1502	}
1503	
1504	return false;
1505}
1506
1507// Called from MAIN thread
1508void LLTextureFetchWorker::endWork(S32 param, bool aborted)
1509{
1510	if (mDecodeHandle != 0)
1511	{
1512		mFetcher->mImageDecodeThread->abortRequest(mDecodeHandle, false);
1513		mDecodeHandle = 0;
1514	}
1515	mFormattedImage = NULL;
1516}
1517
1518//////////////////////////////////////////////////////////////////////////////
1519
1520// virtual
1521void LLTextureFetchWorker::finishWork(S32 param, bool completed)
1522{
1523	// The following are required in case the work was aborted
1524	if (mCacheReadHandle != LLTextureCache::nullHandle())
1525	{
1526		mFetcher->mTextureCache->readComplete(mCacheReadHandle, true);
1527		mCacheReadHandle = LLTextureCache::nullHandle();
1528	}
1529	if (mCacheWriteHandle != LLTextureCache::nullHandle())
1530	{
1531		mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true);
1532		mCacheWriteHandle = LLTextureCache::nullHandle();
1533	}
1534}
1535
1536// virtual
1537bool LLTextureFetchWorker::deleteOK()
1538{
1539	bool delete_ok = true;
1540	// Allow any pending reads or writes to complete
1541	if (mCacheReadHandle != LLTextureCache::nullHandle())
1542	{
1543		if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, true))
1544		{
1545			mCacheReadHandle = LLTextureCache::nullHandle();
1546		}
1547		else
1548		{
1549			delete_ok = false;
1550		}
1551	}
1552	if (mCacheWriteHandle != LLTextureCache::nullHandle())
1553	{
1554		if (mFetcher->mTextureCache->writeComplete(mCacheWriteHandle))
1555		{
1556			mCacheWriteHandle = LLTextureCache::nullHandle();
1557		}
1558		else
1559		{
1560			delete_ok = false;
1561		}
1562	}
1563
1564	if ((haveWork() &&
1565		 // not ok to delete from these states
1566		 ((mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE))))
1567	{
1568		delete_ok = false;
1569	}
1570	
1571	return delete_ok;
1572}
1573
1574void LLTextureFetchWorker::removeFromCache()
1575{
1576	if (!mInLocalCache)
1577	{
1578		mFetcher->mTextureCache->removeFromCache(mID);
1579	}
1580}
1581
1582
1583//////////////////////////////////////////////////////////////////////////////
1584
1585bool LLTextureFetchWorker::processSimulatorPackets()
1586{
1587	if (mFormattedImage.isNull() || mRequestedSize < 0)
1588	{
1589		// not sure how we got here, but not a valid state, abort!
1590		llassert_always(mDecodeHandle == 0);
1591		mFormattedImage = NULL;
1592		return true;
1593	}
1594	
1595	if (mLastPacket >= mFirstPacket)
1596	{
1597		S32 buffer_size = mFormattedImage->getDataSize();
1598		for (S32 i = mFirstPacket; i<=mLastPacket; i++)
1599		{
1600			llassert_always(mPackets[i]);
1601			buffer_size += mPackets[i]->mSize;
1602		}
1603		bool have_all_data = mLastPacket >= mTotalPackets-1;
1604		if (mRequestedSize <= 0)
1605		{
1606			// We received a packed but haven't requested anything yet (edge case)
1607			// Return true (we're "done") since we didn't request anything
1608			return true;
1609		}
1610		if (buffer_size >= mRequestedSize || have_all_data)
1611		{
1612			/// We have enough (or all) data
1613			if (have_all_data)
1614			{
1615				mHaveAllData = TRUE;
1616			}
1617			S32 cur_size = mFormattedImage->getDataSize();
1618			if (buffer_size > cur_size)
1619			{
1620				/// We have new data
1621				U8* buffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), buffer_size);
1622				S32 offset = 0;
1623				if (cur_size > 0 && mFirstPacket > 0)
1624				{
1625					memcpy(buffer, mFormattedImage->getData(), cur_size);
1626					offset = cur_size;
1627				}
1628				for (S32 i=mFirstPacket; i<=mLastPacket; i++)
1629				{
1630					memcpy(buffer + offset, mPackets[i]->mData, mPackets[i]->mSize);
1631					offset += mPackets[i]->mSize;
1632				}
1633				// NOTE: setData releases current data
1634				mFormattedImage->setData(buffer, buffer_size);
1635			}
1636			mLoadedDiscard = mRequestedDiscard;
1637			return true;
1638		}
1639	}
1640	return false;
1641}
1642
1643//////////////////////////////////////////////////////////////////////////////
1644
1645S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels,
1646										   const LLIOPipe::buffer_ptr_t& buffer,
1647										   bool partial, bool success)
1648{
1649	S32 data_size = 0 ;
1650
1651	LLMutexLock lock(&mWorkMutex);
1652
1653	if (mState != WAIT_HTTP_REQ)
1654	{
1655		llwarns << "callbackHttpGet for unrequested fetch worker: " << mID
1656				<< " req=" << mSentRequest << " state= " << mState << llendl;
1657		return data_size;
1658	}
1659	if (mLoaded)
1660	{
1661		llwarns << "Duplicate callback for " << mID.asString() << llendl;
1662		return data_size ; // ignore duplicate callback
1663	}
1664	if (success)
1665	{
1666		// get length of stream:
1667		data_size = buffer->countAfter(channels.in(), NULL);		
1668	
1669		LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL;
1670		if (data_size > 0)
1671		{
1672			// *TODO: set the formatted image data here directly to avoid the copy
1673			mBuffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), data_size);
1674			buffer->readAfter(channels.in(), NULL, mBuffer, data_size);
1675			mBufferSize += data_size;
1676			if (data_size < mRequestedSize && mRequestedDiscard == 0)
1677			{
1678				mHaveAllData = TRUE;
1679			}
1680			else if (data_size > mRequestedSize)
1681			{
1682				// *TODO: This shouldn't be happening any more
1683				llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl;
1684				mHaveAllData = TRUE;
1685				llassert_always(mDecodeHandle == 0);
1686				mFormattedImage = NULL; // discard any previous data we had
1687				mBufferSize = data_size;
1688			}
1689		}
1690		else
1691		{
1692			// We requested data but received none (and no error),
1693			// so presumably we have all of it
1694			mHaveAllData = TRUE;
1695		}
1696		mRequestedSize = data_size;
1697	}
1698	else
1699	{
1700		mRequestedSize = -1; // error
1701	}
1702	mLoaded = TRUE;
1703	setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority);
1704
1705	return data_size ;
1706}
1707
1708//////////////////////////////////////////////////////////////////////////////
1709
1710void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* image,
1711											 S32 imagesize, BOOL islocal)
1712{
1713	LLMutexLock lock(&mWorkMutex);
1714	if (mState != LOAD_FROM_TEXTURE_CACHE)

Large files files are truncated, but you can click here to view the full file