PageRenderTime 427ms CodeModel.GetById 50ms app.highlight 289ms RepoModel.GetById 14ms app.codeStats 3ms

/indra/newview/lltexturecache.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 1944 lines | 1532 code | 209 blank | 203 comment | 255 complexity | 20571c17dd21ffc121ea67c5e7673f1b MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/** 
   2 * @file lltexturecache.cpp
   3 * @brief Object which handles local texture caching
   4 *
   5 * $LicenseInfo:firstyear=2000&license=viewerlgpl$
   6 * Second Life Viewer Source Code
   7 * Copyright (C) 2010, Linden Research, Inc.
   8 * 
   9 * This library is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU Lesser General Public
  11 * License as published by the Free Software Foundation;
  12 * version 2.1 of the License only.
  13 * 
  14 * This library is distributed in the hope that it will be useful,
  15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 * Lesser General Public License for more details.
  18 * 
  19 * You should have received a copy of the GNU Lesser General Public
  20 * License along with this library; if not, write to the Free Software
  21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  22 * 
  23 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  24 * $/LicenseInfo$
  25 */
  26
  27#include "llviewerprecompiledheaders.h"
  28
  29#include "lltexturecache.h"
  30
  31#include "llapr.h"
  32#include "lldir.h"
  33#include "llimage.h"
  34#include "lllfsthread.h"
  35#include "llviewercontrol.h"
  36
  37// Included to allow LLTextureCache::purgeTextures() to pause watchdog timeout
  38#include "llappviewer.h" 
  39#include "llmemory.h"
  40
  41// Cache organization:
  42// cache/texture.entries
  43//  Unordered array of Entry structs
  44// cache/texture.cache
  45//  First TEXTURE_CACHE_ENTRY_SIZE bytes of each texture in texture.entries in same order
  46// cache/textures/[0-F]/UUID.texture
  47//  Actual texture body files
  48
  49//note: there is no good to define 1024 for TEXTURE_CACHE_ENTRY_SIZE while FIRST_PACKET_SIZE is 600 on sim side.
  50const S32 TEXTURE_CACHE_ENTRY_SIZE = FIRST_PACKET_SIZE;//1024;
  51const F32 TEXTURE_CACHE_PURGE_AMOUNT = .20f; // % amount to reduce the cache by when it exceeds its limit
  52const F32 TEXTURE_CACHE_LRU_SIZE = .10f; // % amount for LRU list (low overhead to regenerate)
  53
  54class LLTextureCacheWorker : public LLWorkerClass
  55{
  56	friend class LLTextureCache;
  57
  58private:
  59	class ReadResponder : public LLLFSThread::Responder
  60	{
  61	public:
  62		ReadResponder(LLTextureCache* cache, handle_t handle) : mCache(cache), mHandle(handle) {}
  63		~ReadResponder() {}
  64		void completed(S32 bytes)
  65		{
  66			mCache->lockWorkers();
  67			LLTextureCacheWorker* reader = mCache->getReader(mHandle);
  68			if (reader) reader->ioComplete(bytes);
  69			mCache->unlockWorkers();
  70		}
  71		LLTextureCache* mCache;
  72		LLTextureCacheWorker::handle_t mHandle;
  73	};
  74
  75	class WriteResponder : public LLLFSThread::Responder
  76	{
  77	public:
  78		WriteResponder(LLTextureCache* cache, handle_t handle) : mCache(cache), mHandle(handle) {}
  79		~WriteResponder() {}
  80		void completed(S32 bytes)
  81		{
  82			mCache->lockWorkers();
  83			LLTextureCacheWorker* writer = mCache->getWriter(mHandle);
  84			if (writer) writer->ioComplete(bytes);
  85			mCache->unlockWorkers();
  86		}
  87		LLTextureCache* mCache;
  88		LLTextureCacheWorker::handle_t mHandle;
  89	};
  90	
  91public:
  92	LLTextureCacheWorker(LLTextureCache* cache, U32 priority, const LLUUID& id,
  93						 U8* data, S32 datasize, S32 offset,
  94						 S32 imagesize, // for writes
  95						 LLTextureCache::Responder* responder)
  96		: LLWorkerClass(cache, "LLTextureCacheWorker"),
  97		  mID(id),
  98		  mCache(cache),
  99		  mPriority(priority),
 100		  mReadData(NULL),
 101		  mWriteData(data),
 102		  mDataSize(datasize),
 103		  mOffset(offset),
 104		  mImageSize(imagesize),
 105		  mImageFormat(IMG_CODEC_J2C),
 106		  mImageLocal(FALSE),
 107		  mResponder(responder),
 108		  mFileHandle(LLLFSThread::nullHandle()),
 109		  mBytesToRead(0),
 110		  mBytesRead(0)
 111	{
 112		mPriority &= LLWorkerThread::PRIORITY_LOWBITS;
 113	}
 114	~LLTextureCacheWorker()
 115	{
 116		llassert_always(!haveWork());
 117		FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 118	}
 119
 120	// override this interface
 121	virtual bool doRead() = 0;
 122	virtual bool doWrite() = 0;
 123
 124	virtual bool doWork(S32 param); // Called from LLWorkerThread::processRequest()
 125
 126	handle_t read() { addWork(0, LLWorkerThread::PRIORITY_HIGH | mPriority); return mRequestHandle; }
 127	handle_t write() { addWork(1, LLWorkerThread::PRIORITY_HIGH | mPriority); return mRequestHandle; }
 128	bool complete() { return checkWork(); }
 129	void ioComplete(S32 bytes)
 130	{
 131		mBytesRead = bytes;
 132		setPriority(LLWorkerThread::PRIORITY_HIGH | mPriority);
 133	}
 134
 135private:
 136	virtual void startWork(S32 param); // called from addWork() (MAIN THREAD)
 137	virtual void finishWork(S32 param, bool completed); // called from finishRequest() (WORK THREAD)
 138	virtual void endWork(S32 param, bool aborted); // called from doWork() (MAIN THREAD)
 139
 140protected:
 141	LLTextureCache* mCache;
 142	U32 mPriority;
 143	LLUUID	mID;
 144	
 145	U8* mReadData;
 146	U8* mWriteData;
 147	S32 mDataSize;
 148	S32 mOffset;
 149	S32 mImageSize;
 150	EImageCodec mImageFormat;
 151	BOOL mImageLocal;
 152	LLPointer<LLTextureCache::Responder> mResponder;
 153	LLLFSThread::handle_t mFileHandle;
 154	S32 mBytesToRead;
 155	LLAtomicS32 mBytesRead;
 156};
 157
 158class LLTextureCacheLocalFileWorker : public LLTextureCacheWorker
 159{
 160public:
 161	LLTextureCacheLocalFileWorker(LLTextureCache* cache, U32 priority, const std::string& filename, const LLUUID& id,
 162						 U8* data, S32 datasize, S32 offset,
 163						 S32 imagesize, // for writes
 164						 LLTextureCache::Responder* responder) 
 165			: LLTextureCacheWorker(cache, priority, id, data, datasize, offset, imagesize, responder),
 166			mFileName(filename)
 167
 168	{
 169	}
 170
 171	virtual bool doRead();
 172	virtual bool doWrite();
 173	
 174private:
 175	std::string	mFileName;
 176};
 177
 178bool LLTextureCacheLocalFileWorker::doRead()
 179{
 180	S32 local_size = LLAPRFile::size(mFileName, mCache->getLocalAPRFilePool());
 181
 182	if (local_size > 0 && mFileName.size() > 4)
 183	{
 184		mDataSize = local_size; // Only a complete file is valid
 185
 186		std::string extension = mFileName.substr(mFileName.size() - 3, 3);
 187
 188		mImageFormat = LLImageBase::getCodecFromExtension(extension);
 189
 190		if (mImageFormat == IMG_CODEC_INVALID)
 191		{
 192// 			llwarns << "Unrecognized file extension " << extension << " for local texture " << mFileName << llendl;
 193			mDataSize = 0; // no data
 194			return true;
 195		}
 196	}
 197	else
 198	{
 199		// file doesn't exist
 200		mDataSize = 0; // no data
 201		return true;
 202	}
 203
 204#if USE_LFS_READ
 205	if (mFileHandle == LLLFSThread::nullHandle())
 206	{
 207		mImageLocal = TRUE;
 208		mImageSize = local_size;
 209		if (!mDataSize || mDataSize + mOffset > local_size)
 210		{
 211			mDataSize = local_size - mOffset;
 212		}
 213		if (mDataSize <= 0)
 214		{
 215			// no more data to read
 216			mDataSize = 0;
 217			return true;
 218		}
 219		mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize);
 220		mBytesRead = -1;
 221		mBytesToRead = mDataSize;
 222		setPriority(LLWorkerThread::PRIORITY_LOW | mPriority);
 223		mFileHandle = LLLFSThread::sLocal->read(local_filename, mReadData, mOffset, mDataSize,
 224												new ReadResponder(mCache, mRequestHandle));
 225		return false;
 226	}
 227	else
 228	{
 229		if (mBytesRead >= 0)
 230		{
 231			if (mBytesRead != mBytesToRead)
 232			{
 233// 				llwarns << "Error reading file from local cache: " << local_filename
 234// 						<< " Bytes: " << mDataSize << " Offset: " << mOffset
 235// 						<< " / " << mDataSize << llendl;
 236				mDataSize = 0; // failed
 237				FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 238				mReadData = NULL;
 239			}
 240			return true;
 241		}
 242		else
 243		{
 244			return false;
 245		}
 246	}
 247#else
 248	if (!mDataSize || mDataSize > local_size)
 249	{
 250		mDataSize = local_size;
 251	}
 252	mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize);
 253	
 254	S32 bytes_read = LLAPRFile::readEx(mFileName, mReadData, mOffset, mDataSize, mCache->getLocalAPRFilePool());	
 255
 256	if (bytes_read != mDataSize)
 257	{
 258// 		llwarns << "Error reading file from local cache: " << mFileName
 259// 				<< " Bytes: " << mDataSize << " Offset: " << mOffset
 260// 				<< " / " << mDataSize << llendl;
 261		mDataSize = 0;
 262		FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 263		mReadData = NULL;
 264	}
 265	else
 266	{
 267		mImageSize = local_size;
 268		mImageLocal = TRUE;
 269	}
 270	return true;
 271#endif
 272}
 273
 274bool LLTextureCacheLocalFileWorker::doWrite()
 275{
 276	// no writes for local files
 277	return false;
 278}
 279
 280class LLTextureCacheRemoteWorker : public LLTextureCacheWorker
 281{
 282public:
 283	LLTextureCacheRemoteWorker(LLTextureCache* cache, U32 priority, const LLUUID& id,
 284						 U8* data, S32 datasize, S32 offset,
 285						 S32 imagesize, // for writes
 286						 LLTextureCache::Responder* responder) 
 287			: LLTextureCacheWorker(cache, priority, id, data, datasize, offset, imagesize, responder),
 288			mState(INIT)
 289	{
 290	}
 291
 292	virtual bool doRead();
 293	virtual bool doWrite();
 294
 295private:
 296	enum e_state
 297	{
 298		INIT = 0,
 299		LOCAL = 1,
 300		CACHE = 2,
 301		HEADER = 3,
 302		BODY = 4
 303	};
 304
 305	e_state mState;
 306};
 307
 308
 309//virtual
 310void LLTextureCacheWorker::startWork(S32 param)
 311{
 312}
 313
 314// This is where a texture is read from the cache system (header and body)
 315// Current assumption are:
 316// - the whole data are in a raw form, will be stored at mReadData
 317// - the size of this raw data is mDataSize and can be smaller than TEXTURE_CACHE_ENTRY_SIZE (the size of a record in the header cache)
 318// - the code supports offset reading but this is actually never exercised in the viewer
 319bool LLTextureCacheRemoteWorker::doRead()
 320{
 321	bool done = false;
 322	S32 idx = -1;
 323
 324	S32 local_size = 0;
 325	std::string local_filename;
 326	
 327	// First state / stage : find out if the file is local
 328	if (mState == INIT)
 329	{
 330#if 0
 331		std::string filename = mCache->getLocalFileName(mID);	
 332		// Is it a JPEG2000 file? 
 333		{
 334			local_filename = filename + ".j2c";
 335			local_size = LLAPRFile::size(local_filename, mCache->getLocalAPRFilePool());
 336			if (local_size > 0)
 337			{
 338				mImageFormat = IMG_CODEC_J2C;
 339			}
 340		}
 341		// If not, is it a jpeg file?
 342		if (local_size == 0)
 343		{
 344			local_filename = filename + ".jpg";
 345			local_size = LLAPRFile::size(local_filename, mCache->getLocalAPRFilePool());
 346			if (local_size > 0)
 347			{
 348				mImageFormat = IMG_CODEC_JPEG;
 349				mDataSize = local_size; // Only a complete .jpg file is valid
 350			}
 351		}
 352		// Hmm... What about a targa file? (used for UI texture mostly)
 353		if (local_size == 0)
 354		{
 355			local_filename = filename + ".tga";
 356			local_size = LLAPRFile::size(local_filename, mCache->getLocalAPRFilePool());
 357			if (local_size > 0)
 358			{
 359				mImageFormat = IMG_CODEC_TGA;
 360				mDataSize = local_size; // Only a complete .tga file is valid
 361			}
 362		}
 363		// Determine the next stage: if we found a file, then LOCAL else CACHE
 364		mState = (local_size > 0 ? LOCAL : CACHE);
 365
 366		llassert_always(mState == CACHE) ;
 367#else
 368		mState = CACHE;
 369#endif
 370	}
 371
 372	// Second state / stage : if the file is local, load it and leave
 373	if (!done && (mState == LOCAL))
 374	{
 375		llassert(local_size != 0);	// we're assuming there is a non empty local file here...
 376		if (!mDataSize || mDataSize > local_size)
 377		{
 378			mDataSize = local_size;
 379		}
 380		// Allocate read buffer
 381		mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize);
 382		S32 bytes_read = LLAPRFile::readEx(local_filename, 
 383											 mReadData, mOffset, mDataSize, mCache->getLocalAPRFilePool());
 384		if (bytes_read != mDataSize)
 385		{
 386 			llwarns << "Error reading file from local cache: " << local_filename
 387 					<< " Bytes: " << mDataSize << " Offset: " << mOffset
 388 					<< " / " << mDataSize << llendl;
 389			mDataSize = 0;
 390			FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 391			mReadData = NULL;
 392		}
 393		else
 394		{
 395			//llinfos << "texture " << mID.asString() << " found in local_assets" << llendl;
 396			mImageSize = local_size;
 397			mImageLocal = TRUE;
 398		}
 399		// We're done...
 400		done = true;
 401	}
 402
 403	// Second state / stage : identify the cache or not...
 404	if (!done && (mState == CACHE))
 405	{
 406		LLTextureCache::Entry entry ;
 407		idx = mCache->getHeaderCacheEntry(mID, entry);
 408		if (idx < 0)
 409		{
 410			// The texture is *not* cached. We're done here...
 411			mDataSize = 0; // no data 
 412			done = true;
 413		}
 414		else
 415		{
 416			mImageSize = entry.mImageSize ;
 417			// If the read offset is bigger than the header cache, we read directly from the body
 418			// Note that currently, we *never* read with offset from the cache, so the result is *always* HEADER
 419			mState = mOffset < TEXTURE_CACHE_ENTRY_SIZE ? HEADER : BODY;
 420		}
 421	}
 422
 423	// Third state / stage : read data from the header cache (texture.entries) file
 424	if (!done && (mState == HEADER))
 425	{
 426		llassert_always(idx >= 0);	// we need an entry here or reading the header makes no sense
 427		llassert_always(mOffset < TEXTURE_CACHE_ENTRY_SIZE);
 428		S32 offset = idx * TEXTURE_CACHE_ENTRY_SIZE + mOffset;
 429		// Compute the size we need to read (in bytes)
 430		S32 size = TEXTURE_CACHE_ENTRY_SIZE - mOffset;
 431		size = llmin(size, mDataSize);
 432		// Allocate the read buffer
 433		mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), size);
 434		S32 bytes_read = LLAPRFile::readEx(mCache->mHeaderDataFileName, 
 435											 mReadData, offset, size, mCache->getLocalAPRFilePool());
 436		if (bytes_read != size)
 437		{
 438			llwarns << "LLTextureCacheWorker: "  << mID
 439					<< " incorrect number of bytes read from header: " << bytes_read
 440					<< " / " << size << llendl;
 441			FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 442			mReadData = NULL;
 443			mDataSize = -1; // failed
 444			done = true;
 445		}
 446		// If we already read all we expected, we're actually done
 447		if (mDataSize <= bytes_read)
 448		{
 449			done = true;
 450		}
 451		else
 452		{
 453			mState = BODY;
 454		}
 455	}
 456
 457	// Fourth state / stage : read the rest of the data from the UUID based cached file
 458	if (!done && (mState == BODY))
 459	{
 460		std::string filename = mCache->getTextureFileName(mID);
 461		S32 filesize = LLAPRFile::size(filename, mCache->getLocalAPRFilePool());
 462
 463		if (filesize && (filesize + TEXTURE_CACHE_ENTRY_SIZE) > mOffset)
 464		{
 465			S32 max_datasize = TEXTURE_CACHE_ENTRY_SIZE + filesize - mOffset;
 466			mDataSize = llmin(max_datasize, mDataSize);
 467
 468			S32 data_offset, file_size, file_offset;
 469			
 470			// Reserve the whole data buffer first
 471			U8* data = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize);
 472
 473			// Set the data file pointers taking the read offset into account. 2 cases:
 474			if (mOffset < TEXTURE_CACHE_ENTRY_SIZE)
 475			{
 476				// Offset within the header record. That means we read something from the header cache.
 477				// Note: most common case is (mOffset = 0), so this is the "normal" code path.
 478				data_offset = TEXTURE_CACHE_ENTRY_SIZE - mOffset;	// i.e. TEXTURE_CACHE_ENTRY_SIZE if mOffset nul (common case)
 479				file_offset = 0;
 480				file_size = mDataSize - data_offset;
 481				// Copy the raw data we've been holding from the header cache into the new sized buffer
 482				llassert_always(mReadData);
 483				memcpy(data, mReadData, data_offset);
 484				FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 485				mReadData = NULL;
 486			}
 487			else
 488			{
 489				// Offset bigger than the header record. That means we haven't read anything yet.
 490				data_offset = 0;
 491				file_offset = mOffset - TEXTURE_CACHE_ENTRY_SIZE;
 492				file_size = mDataSize;
 493				// No data from header cache to copy in that case, we skipped it all
 494			}
 495
 496			// Now use that buffer as the object read buffer
 497			llassert_always(mReadData == NULL);
 498			mReadData = data;
 499
 500			// Read the data at last
 501			S32 bytes_read = LLAPRFile::readEx(filename, 
 502											 mReadData + data_offset,
 503											 file_offset, file_size,
 504											 mCache->getLocalAPRFilePool());
 505			if (bytes_read != file_size)
 506			{
 507				llwarns << "LLTextureCacheWorker: "  << mID
 508						<< " incorrect number of bytes read from body: " << bytes_read
 509						<< " / " << file_size << llendl;
 510				FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 511				mReadData = NULL;
 512				mDataSize = -1; // failed
 513				done = true;
 514			}
 515		}
 516		else
 517		{
 518			// No body, we're done.
 519			mDataSize = llmax(TEXTURE_CACHE_ENTRY_SIZE - mOffset, 0);
 520			lldebugs << "No body file for: " << filename << llendl;
 521		}	
 522		// Nothing else to do at that point...
 523		done = true;
 524	}
 525
 526	// Clean up and exit
 527	return done;
 528}
 529
 530// This is where *everything* about a texture is written down in the cache system (entry map, header and body)
 531// Current assumption are:
 532// - the whole data are in a raw form, starting at mWriteData
 533// - the size of this raw data is mDataSize and can be smaller than TEXTURE_CACHE_ENTRY_SIZE (the size of a record in the header cache)
 534// - the code *does not* support offset writing so there are no difference between buffer addresses and start of data
 535bool LLTextureCacheRemoteWorker::doWrite()
 536{
 537	bool done = false;
 538	S32 idx = -1;	
 539
 540	// First state / stage : check that what we're trying to cache is in an OK shape
 541	if (mState == INIT)
 542	{
 543		llassert_always(mOffset == 0);	// We currently do not support write offsets
 544		llassert_always(mDataSize > 0); // Things will go badly wrong if mDataSize is nul or negative...
 545		llassert_always(mImageSize >= mDataSize);
 546		mState = CACHE;
 547	}
 548	
 549	// No LOCAL state for write(): because it doesn't make much sense to cache a local file...
 550
 551	// Second state / stage : set an entry in the headers entry (texture.entries) file
 552	if (!done && (mState == CACHE))
 553	{
 554		bool alreadyCached = false;
 555		LLTextureCache::Entry entry ;
 556
 557		// Checks if this image is already in the entry list
 558		idx = mCache->getHeaderCacheEntry(mID, entry);
 559		if(idx < 0)
 560		{
 561			idx = mCache->setHeaderCacheEntry(mID, entry, mImageSize, mDataSize); // create the new entry.
 562		}
 563		else
 564		{
 565			alreadyCached = mCache->updateEntry(idx, entry, mImageSize, mDataSize); // update the existing entry.
 566		}
 567
 568		if (idx < 0)
 569		{
 570			llwarns << "LLTextureCacheWorker: "  << mID
 571					<< " Unable to create header entry for writing!" << llendl;
 572			mDataSize = -1; // failed
 573			done = true;
 574		}
 575		else
 576		{
 577			if (alreadyCached && (mDataSize <= TEXTURE_CACHE_ENTRY_SIZE))
 578			{
 579				// Small texture already cached case: we're done with writing
 580				done = true;
 581			}
 582			else
 583			{
 584				// If the texture has already been cached, we don't resave the header and go directly to the body part
 585				mState = alreadyCached ? BODY : HEADER;
 586			}
 587		}
 588	}
 589
 590	// Third stage / state : write the header record in the header file (texture.cache)
 591	if (!done && (mState == HEADER))
 592	{
 593		llassert_always(idx >= 0);	// we need an entry here or storing the header makes no sense
 594		S32 offset = idx * TEXTURE_CACHE_ENTRY_SIZE;	// skip to the correct spot in the header file
 595		S32 size = TEXTURE_CACHE_ENTRY_SIZE;			// record size is fixed for the header
 596		S32 bytes_written;
 597
 598		if (mDataSize < TEXTURE_CACHE_ENTRY_SIZE)
 599		{
 600			// We need to write a full record in the header cache so, if the amount of data is smaller
 601			// than a record, we need to transfer the data to a buffer padded with 0 and write that
 602			U8* padBuffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), TEXTURE_CACHE_ENTRY_SIZE);
 603			memset(padBuffer, 0, TEXTURE_CACHE_ENTRY_SIZE);		// Init with zeros
 604			memcpy(padBuffer, mWriteData, mDataSize);			// Copy the write buffer
 605			bytes_written = LLAPRFile::writeEx(mCache->mHeaderDataFileName, padBuffer, offset, size, mCache->getLocalAPRFilePool());
 606			FREE_MEM(LLImageBase::getPrivatePool(), padBuffer);
 607		}
 608		else
 609		{
 610			// Write the header record (== first TEXTURE_CACHE_ENTRY_SIZE bytes of the raw file) in the header file
 611			bytes_written = LLAPRFile::writeEx(mCache->mHeaderDataFileName, mWriteData, offset, size, mCache->getLocalAPRFilePool());
 612		}
 613
 614		if (bytes_written <= 0)
 615		{
 616			llwarns << "LLTextureCacheWorker: "  << mID
 617					<< " Unable to write header entry!" << llendl;
 618			mDataSize = -1; // failed
 619			done = true;
 620		}
 621
 622		// If we wrote everything (may be more with padding) in the header cache, 
 623		// we're done so we don't have a body to store
 624		if (mDataSize <= bytes_written)
 625		{
 626			done = true;
 627		}
 628		else
 629		{
 630			mState = BODY;
 631		}
 632	}
 633	
 634	// Fourth stage / state : write the body file, i.e. the rest of the texture in a "UUID" file name
 635	if (!done && (mState == BODY))
 636	{
 637		llassert(mDataSize > TEXTURE_CACHE_ENTRY_SIZE);	// wouldn't make sense to be here otherwise...
 638		S32 file_size = mDataSize - TEXTURE_CACHE_ENTRY_SIZE;
 639		
 640		{
 641			// build the cache file name from the UUID
 642			std::string filename = mCache->getTextureFileName(mID);			
 643// 			llinfos << "Writing Body: " << filename << " Bytes: " << file_offset+file_size << llendl;
 644			S32 bytes_written = LLAPRFile::writeEx(	filename, 
 645													mWriteData + TEXTURE_CACHE_ENTRY_SIZE,
 646													0, file_size,
 647													mCache->getLocalAPRFilePool());
 648			if (bytes_written <= 0)
 649			{
 650				llwarns << "LLTextureCacheWorker: "  << mID
 651						<< " incorrect number of bytes written to body: " << bytes_written
 652						<< " / " << file_size << llendl;
 653				mDataSize = -1; // failed
 654				done = true;
 655			}
 656		}
 657		
 658		// Nothing else to do at that point...
 659		done = true;
 660	}
 661
 662	// Clean up and exit
 663	return done;
 664}
 665
 666//virtual
 667bool LLTextureCacheWorker::doWork(S32 param)
 668{
 669	bool res = false;
 670	if (param == 0) // read
 671	{
 672		res = doRead();
 673	}
 674	else if (param == 1) // write
 675	{
 676		res = doWrite();
 677	}
 678	else
 679	{
 680		llassert_always(0);
 681	}
 682	return res;
 683}
 684
 685//virtual (WORKER THREAD)
 686void LLTextureCacheWorker::finishWork(S32 param, bool completed)
 687{
 688	if (mResponder.notNull())
 689	{
 690		bool success = (completed && mDataSize > 0);
 691		if (param == 0)
 692		{
 693			// read
 694			if (success)
 695			{
 696				mResponder->setData(mReadData, mDataSize, mImageSize, mImageFormat, mImageLocal);
 697				mReadData = NULL; // responder owns data
 698				mDataSize = 0;
 699			}
 700			else
 701			{
 702				FREE_MEM(LLImageBase::getPrivatePool(), mReadData);
 703				mReadData = NULL;
 704			}
 705		}
 706		else
 707		{
 708			// write
 709			mWriteData = NULL; // we never owned data
 710			mDataSize = 0;
 711		}
 712		mCache->addCompleted(mResponder, success);
 713	}
 714}
 715
 716//virtual (MAIN THREAD)
 717void LLTextureCacheWorker::endWork(S32 param, bool aborted)
 718{
 719	if (aborted)
 720	{
 721		// Let the destructor handle any cleanup
 722		return;
 723	}
 724	switch(param)
 725	{
 726	  default:
 727	  case 0: // read
 728	  case 1: // write
 729	  {
 730		  if (mDataSize < 0)
 731		  {
 732			  // failed
 733			  mCache->removeFromCache(mID);
 734		  }
 735		  break;
 736	  }
 737	}
 738}
 739
 740//////////////////////////////////////////////////////////////////////////////
 741
 742LLTextureCache::LLTextureCache(bool threaded)
 743	: LLWorkerThread("TextureCache", threaded),
 744	  mWorkersMutex(NULL),
 745	  mHeaderMutex(NULL),
 746	  mListMutex(NULL),
 747	  mHeaderAPRFile(NULL),
 748	  mReadOnly(TRUE), //do not allow to change the texture cache until setReadOnly() is called.
 749	  mTexturesSizeTotal(0),
 750	  mDoPurge(FALSE)
 751{
 752}
 753
 754LLTextureCache::~LLTextureCache()
 755{
 756	clearDeleteList() ;
 757	writeUpdatedEntries() ;
 758}
 759
 760//////////////////////////////////////////////////////////////////////////////
 761
 762//virtual
 763S32 LLTextureCache::update(F32 max_time_ms)
 764{
 765	static LLFrameTimer timer ;
 766	static const F32 MAX_TIME_INTERVAL = 300.f ; //seconds.
 767
 768	S32 res;
 769	res = LLWorkerThread::update(max_time_ms);
 770
 771	mListMutex.lock();
 772	handle_list_t priorty_list = mPrioritizeWriteList; // copy list
 773	mPrioritizeWriteList.clear();
 774	responder_list_t completed_list = mCompletedList; // copy list
 775	mCompletedList.clear();
 776	mListMutex.unlock();
 777	
 778	lockWorkers();
 779	
 780	for (handle_list_t::iterator iter1 = priorty_list.begin();
 781		 iter1 != priorty_list.end(); ++iter1)
 782	{
 783		handle_t handle = *iter1;
 784		handle_map_t::iterator iter2 = mWriters.find(handle);
 785		if(iter2 != mWriters.end())
 786		{
 787			LLTextureCacheWorker* worker = iter2->second;
 788			worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mPriority);
 789		}
 790	}
 791
 792	unlockWorkers(); 
 793	
 794	// call 'completed' with workers list unlocked (may call readComplete() or writeComplete()
 795	for (responder_list_t::iterator iter1 = completed_list.begin();
 796		 iter1 != completed_list.end(); ++iter1)
 797	{
 798		Responder *responder = iter1->first;
 799		bool success = iter1->second;
 800		responder->completed(success);
 801	}
 802	
 803	if(!res && timer.getElapsedTimeF32() > MAX_TIME_INTERVAL)
 804	{
 805		timer.reset() ;
 806		writeUpdatedEntries() ;
 807	}
 808
 809	return res;
 810}
 811
 812//////////////////////////////////////////////////////////////////////////////
 813// search for local copy of UUID-based image file
 814std::string LLTextureCache::getLocalFileName(const LLUUID& id)
 815{
 816	// Does not include extension
 817	std::string idstr = id.asString();
 818	// TODO: should we be storing cached textures in skin directory?
 819	std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_LOCAL_ASSETS, idstr);
 820	return filename;
 821}
 822
 823std::string LLTextureCache::getTextureFileName(const LLUUID& id)
 824{
 825	std::string idstr = id.asString();
 826	std::string delem = gDirUtilp->getDirDelimiter();
 827	std::string filename = mTexturesDirName + delem + idstr[0] + delem + idstr + ".texture";
 828	return filename;
 829}
 830
 831//debug
 832BOOL LLTextureCache::isInCache(const LLUUID& id) 
 833{
 834	LLMutexLock lock(&mHeaderMutex);
 835	id_map_t::const_iterator iter = mHeaderIDMap.find(id);
 836	
 837	return (iter != mHeaderIDMap.end()) ;
 838}
 839
 840//debug
 841BOOL LLTextureCache::isInLocal(const LLUUID& id) 
 842{
 843	S32 local_size = 0;
 844	std::string local_filename;
 845	
 846	std::string filename = getLocalFileName(id);	
 847	// Is it a JPEG2000 file? 
 848	{
 849		local_filename = filename + ".j2c";
 850		local_size = LLAPRFile::size(local_filename, getLocalAPRFilePool());
 851		if (local_size > 0)
 852		{
 853			return TRUE ;
 854		}
 855	}
 856		
 857	// If not, is it a jpeg file?		
 858	{
 859		local_filename = filename + ".jpg";
 860		local_size = LLAPRFile::size(local_filename, getLocalAPRFilePool());
 861		if (local_size > 0)
 862		{
 863			return TRUE ;
 864		}
 865	}
 866		
 867	// Hmm... What about a targa file? (used for UI texture mostly)		
 868	{
 869		local_filename = filename + ".tga";
 870		local_size = LLAPRFile::size(local_filename, getLocalAPRFilePool());
 871		if (local_size > 0)
 872		{
 873			return TRUE ;
 874		}
 875	}
 876		
 877	return FALSE ;
 878}
 879//////////////////////////////////////////////////////////////////////////////
 880
 881//static
 882const S32 MAX_REASONABLE_FILE_SIZE = 512*1024*1024; // 512 MB
 883F32 LLTextureCache::sHeaderCacheVersion = 1.4f;
 884U32 LLTextureCache::sCacheMaxEntries = MAX_REASONABLE_FILE_SIZE / TEXTURE_CACHE_ENTRY_SIZE;
 885S64 LLTextureCache::sCacheMaxTexturesSize = 0; // no limit
 886const char* entries_filename = "texture.entries";
 887const char* cache_filename = "texture.cache";
 888const char* old_textures_dirname = "textures";
 889//change the location of the texture cache to prevent from being deleted by old version viewers.
 890const char* textures_dirname = "texturecache";
 891
 892void LLTextureCache::setDirNames(ELLPath location)
 893{
 894	std::string delem = gDirUtilp->getDirDelimiter();
 895
 896	mHeaderEntriesFileName = gDirUtilp->getExpandedFilename(location, textures_dirname, entries_filename);
 897	mHeaderDataFileName = gDirUtilp->getExpandedFilename(location, textures_dirname, cache_filename);
 898	mTexturesDirName = gDirUtilp->getExpandedFilename(location, textures_dirname);
 899}
 900
 901void LLTextureCache::purgeCache(ELLPath location)
 902{
 903	LLMutexLock lock(&mHeaderMutex);
 904
 905	if (!mReadOnly)
 906	{
 907		setDirNames(location);
 908		llassert_always(mHeaderAPRFile == NULL);
 909
 910		//remove the legacy cache if exists
 911		std::string texture_dir = mTexturesDirName ;
 912		mTexturesDirName = gDirUtilp->getExpandedFilename(location, old_textures_dirname);
 913		if(LLFile::isdir(mTexturesDirName))
 914		{
 915			std::string file_name = gDirUtilp->getExpandedFilename(location, entries_filename);
 916			LLAPRFile::remove(file_name, getLocalAPRFilePool());
 917
 918			file_name = gDirUtilp->getExpandedFilename(location, cache_filename);
 919			LLAPRFile::remove(file_name, getLocalAPRFilePool());
 920
 921			purgeAllTextures(true);
 922		}
 923		mTexturesDirName = texture_dir ;
 924	}
 925
 926	//remove the current texture cache.
 927	purgeAllTextures(true);
 928}
 929
 930//is called in the main thread before initCache(...) is called.
 931void LLTextureCache::setReadOnly(BOOL read_only)
 932{
 933	mReadOnly = read_only ;
 934}
 935
 936//called in the main thread.
 937S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL texture_cache_mismatch)
 938{
 939	llassert_always(getPending() == 0) ; //should not start accessing the texture cache before initialized.
 940
 941	S64 header_size = (max_size * 2) / 10;
 942	S64 max_entries = header_size / TEXTURE_CACHE_ENTRY_SIZE;
 943	sCacheMaxEntries = (S32)(llmin((S64)sCacheMaxEntries, max_entries));
 944	header_size = sCacheMaxEntries * TEXTURE_CACHE_ENTRY_SIZE;
 945	max_size -= header_size;
 946	if (sCacheMaxTexturesSize > 0)
 947		sCacheMaxTexturesSize = llmin(sCacheMaxTexturesSize, max_size);
 948	else
 949		sCacheMaxTexturesSize = max_size;
 950	max_size -= sCacheMaxTexturesSize;
 951	
 952	LL_INFOS("TextureCache") << "Headers: " << sCacheMaxEntries
 953			<< " Textures size: " << sCacheMaxTexturesSize / (1024 * 1024) << " MB" << LL_ENDL;
 954
 955	setDirNames(location);
 956	
 957	if(texture_cache_mismatch) 
 958	{
 959		//if readonly, disable the texture cache,
 960		//otherwise wipe out the texture cache.
 961		purgeAllTextures(true); 
 962
 963		if(mReadOnly)
 964		{
 965			return max_size ;
 966		}
 967	}
 968	
 969	if (!mReadOnly)
 970	{
 971		LLFile::mkdir(mTexturesDirName);
 972		
 973		const char* subdirs = "0123456789abcdef";
 974		for (S32 i=0; i<16; i++)
 975		{
 976			std::string dirname = mTexturesDirName + gDirUtilp->getDirDelimiter() + subdirs[i];
 977			LLFile::mkdir(dirname);
 978		}
 979	}
 980	readHeaderCache();
 981	purgeTextures(true); // calc mTexturesSize and make some room in the texture cache if we need it
 982
 983	llassert_always(getPending() == 0) ; //should not start accessing the texture cache before initialized.
 984
 985	return max_size; // unused cache space
 986}
 987
 988//----------------------------------------------------------------------------
 989// mHeaderMutex must be locked for the following functions!
 990
 991LLAPRFile* LLTextureCache::openHeaderEntriesFile(bool readonly, S32 offset)
 992{
 993	llassert_always(mHeaderAPRFile == NULL);
 994	apr_int32_t flags = readonly ? APR_READ|APR_BINARY : APR_READ|APR_WRITE|APR_BINARY;
 995	mHeaderAPRFile = new LLAPRFile(mHeaderEntriesFileName, flags, getLocalAPRFilePool());
 996	if(offset > 0)
 997	{
 998		mHeaderAPRFile->seek(APR_SET, offset);
 999	}
1000	return mHeaderAPRFile;
1001}
1002
1003void LLTextureCache::closeHeaderEntriesFile()
1004{
1005	if(!mHeaderAPRFile)
1006	{
1007		return ;
1008	}
1009
1010	delete mHeaderAPRFile;
1011	mHeaderAPRFile = NULL;
1012}
1013
1014void LLTextureCache::readEntriesHeader()
1015{
1016	// mHeaderEntriesInfo initializes to default values so safe not to read it
1017	llassert_always(mHeaderAPRFile == NULL);
1018	if (LLAPRFile::isExist(mHeaderEntriesFileName, getLocalAPRFilePool()))
1019	{
1020		LLAPRFile::readEx(mHeaderEntriesFileName, (U8*)&mHeaderEntriesInfo, 0, sizeof(EntriesInfo),
1021						  getLocalAPRFilePool());
1022	}
1023	else //create an empty entries header.
1024	{
1025		mHeaderEntriesInfo.mVersion = sHeaderCacheVersion ;
1026		mHeaderEntriesInfo.mEntries = 0 ;
1027		writeEntriesHeader() ;
1028	}
1029}
1030
1031void LLTextureCache::writeEntriesHeader()
1032{
1033	llassert_always(mHeaderAPRFile == NULL);
1034	if (!mReadOnly)
1035	{
1036		LLAPRFile::writeEx(mHeaderEntriesFileName, (U8*)&mHeaderEntriesInfo, 0, sizeof(EntriesInfo),
1037						   getLocalAPRFilePool());
1038	}
1039}
1040
1041//mHeaderMutex is locked before calling this.
1042S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create)
1043{
1044	S32 idx = -1;
1045	
1046	id_map_t::iterator iter1 = mHeaderIDMap.find(id);
1047	if (iter1 != mHeaderIDMap.end())
1048	{
1049		idx = iter1->second;
1050	}
1051
1052	if (idx < 0)
1053	{
1054		if (create && !mReadOnly)
1055		{
1056			if (mHeaderEntriesInfo.mEntries < sCacheMaxEntries)
1057			{
1058				// Add an entry to the end of the list
1059				idx = mHeaderEntriesInfo.mEntries++;
1060
1061			}
1062			else if (!mFreeList.empty())
1063			{
1064				idx = *(mFreeList.begin());
1065				mFreeList.erase(mFreeList.begin());
1066			}
1067			else
1068			{
1069				// Look for a still valid entry in the LRU
1070				for (std::set<LLUUID>::iterator iter2 = mLRU.begin(); iter2 != mLRU.end();)
1071				{
1072					std::set<LLUUID>::iterator curiter2 = iter2++;
1073					LLUUID oldid = *curiter2;
1074					// Erase entry from LRU regardless
1075					mLRU.erase(curiter2);
1076					// Look up entry and use it if it is valid
1077					id_map_t::iterator iter3 = mHeaderIDMap.find(oldid);
1078					if (iter3 != mHeaderIDMap.end() && iter3->second >= 0)
1079					{
1080						idx = iter3->second;
1081						removeCachedTexture(oldid) ;//remove the existing cached texture to release the entry index.
1082						break;
1083					}
1084				}
1085				// if (idx < 0) at this point, we will rebuild the LRU 
1086				//  and retry if called from setHeaderCacheEntry(),
1087				//  otherwise this shouldn't happen and will trigger an error
1088			}
1089			if (idx >= 0)
1090			{
1091				entry.mID = id ;
1092				entry.mImageSize = -1 ; //mark it is a brand-new entry.					
1093				entry.mBodySize = 0 ;
1094			}
1095		}
1096	}
1097	else
1098	{
1099		// Remove this entry from the LRU if it exists
1100		mLRU.erase(id);
1101		// Read the entry
1102		idx_entry_map_t::iterator iter = mUpdatedEntryMap.find(idx) ;
1103		if(iter != mUpdatedEntryMap.end())
1104		{
1105			entry = iter->second ;
1106		}
1107		else
1108		{
1109			readEntryFromHeaderImmediately(idx, entry) ;
1110		}
1111		if(entry.mImageSize <= entry.mBodySize)//it happens on 64-bit systems, do not know why
1112		{
1113			llwarns << "corrupted entry: " << id << " entry image size: " << entry.mImageSize << " entry body size: " << entry.mBodySize << llendl ;
1114
1115			//erase this entry and the cached texture from the cache.
1116			std::string tex_filename = getTextureFileName(id);
1117			removeEntry(idx, entry, tex_filename) ;
1118			mUpdatedEntryMap.erase(idx) ;
1119			idx = -1 ;
1120		}
1121	}
1122	return idx;
1123}
1124
1125//mHeaderMutex is locked before calling this.
1126void LLTextureCache::writeEntryToHeaderImmediately(S32& idx, Entry& entry, bool write_header)
1127{	
1128	LLAPRFile* aprfile ;
1129	S32 bytes_written ;
1130	S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry);
1131	if(write_header)
1132	{
1133		aprfile = openHeaderEntriesFile(false, 0);		
1134		bytes_written = aprfile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)) ;
1135		if(bytes_written != sizeof(EntriesInfo))
1136		{
1137			clearCorruptedCache() ; //clear the cache.
1138			idx = -1 ;//mark the idx invalid.
1139			return ;
1140		}
1141
1142		mHeaderAPRFile->seek(APR_SET, offset);
1143	}
1144	else
1145	{
1146		aprfile = openHeaderEntriesFile(false, offset);
1147	}
1148	bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry));
1149	if(bytes_written != sizeof(Entry))
1150	{
1151		clearCorruptedCache() ; //clear the cache.
1152		idx = -1 ;//mark the idx invalid.
1153
1154		return ;
1155	}
1156
1157	closeHeaderEntriesFile();
1158	mUpdatedEntryMap.erase(idx) ;
1159}
1160
1161//mHeaderMutex is locked before calling this.
1162void LLTextureCache::readEntryFromHeaderImmediately(S32& idx, Entry& entry)
1163{
1164	S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry);
1165	LLAPRFile* aprfile = openHeaderEntriesFile(true, offset);
1166	S32 bytes_read = aprfile->read((void*)&entry, (S32)sizeof(Entry));
1167	closeHeaderEntriesFile();
1168
1169	if(bytes_read != sizeof(Entry))
1170	{
1171		clearCorruptedCache() ; //clear the cache.
1172		idx = -1 ;//mark the idx invalid.
1173	}
1174}
1175
1176//mHeaderMutex is locked before calling this.
1177//update an existing entry time stamp, delay writing.
1178void LLTextureCache::updateEntryTimeStamp(S32 idx, Entry& entry)
1179{
1180	static const U32 MAX_ENTRIES_WITHOUT_TIME_STAMP = (U32)(LLTextureCache::sCacheMaxEntries * 0.75f) ;
1181
1182	if(mHeaderEntriesInfo.mEntries < MAX_ENTRIES_WITHOUT_TIME_STAMP)
1183	{
1184		return ; //there are enough empty entry index space, no need to stamp time.
1185	}
1186
1187	if (idx >= 0)
1188	{
1189		if (!mReadOnly)
1190		{
1191			entry.mTime = time(NULL);			
1192			mUpdatedEntryMap[idx] = entry ;
1193		}
1194	}
1195}
1196
1197//update an existing entry, write to header file immediately.
1198bool LLTextureCache::updateEntry(S32& idx, Entry& entry, S32 new_image_size, S32 new_data_size)
1199{
1200	S32 new_body_size = llmax(0, new_data_size - TEXTURE_CACHE_ENTRY_SIZE) ;
1201	
1202	if(new_image_size == entry.mImageSize && new_body_size == entry.mBodySize)
1203	{
1204		return true ; //nothing changed.
1205	}
1206	else 
1207	{
1208		bool purge = false ;
1209
1210		lockHeaders() ;
1211
1212		bool update_header = false ;
1213		if(entry.mImageSize < 0) //is a brand-new entry
1214		{
1215			mHeaderIDMap[entry.mID] = idx;
1216			mTexturesSizeMap[entry.mID] = new_body_size ;
1217			mTexturesSizeTotal += new_body_size ;
1218			
1219			// Update Header
1220			update_header = true ;
1221		}				
1222		else if (entry.mBodySize != new_body_size)
1223		{
1224			//already in mHeaderIDMap.
1225			mTexturesSizeMap[entry.mID] = new_body_size ;
1226			mTexturesSizeTotal -= entry.mBodySize ;
1227			mTexturesSizeTotal += new_body_size ;
1228		}
1229		entry.mTime = time(NULL);
1230		entry.mImageSize = new_image_size ; 
1231		entry.mBodySize = new_body_size ;
1232		
1233		writeEntryToHeaderImmediately(idx, entry, update_header) ;
1234	
1235		if (mTexturesSizeTotal > sCacheMaxTexturesSize)
1236		{
1237			purge = true;
1238		}
1239		
1240		unlockHeaders() ;
1241
1242		if (purge)
1243		{
1244			mDoPurge = TRUE;
1245		}
1246	}
1247
1248	return false ;
1249}
1250
1251U32 LLTextureCache::openAndReadEntries(std::vector<Entry>& entries)
1252{
1253	U32 num_entries = mHeaderEntriesInfo.mEntries;
1254
1255	mHeaderIDMap.clear();
1256	mTexturesSizeMap.clear();
1257	mFreeList.clear();
1258	mTexturesSizeTotal = 0;
1259
1260	LLAPRFile* aprfile = NULL; 
1261	if(mUpdatedEntryMap.empty())
1262	{
1263		aprfile = openHeaderEntriesFile(true, (S32)sizeof(EntriesInfo));
1264	}
1265	else //update the header file first.
1266	{
1267		aprfile = openHeaderEntriesFile(false, 0);
1268		updatedHeaderEntriesFile() ;
1269		if(!aprfile)
1270		{
1271			return 0;
1272		}
1273		aprfile->seek(APR_SET, (S32)sizeof(EntriesInfo));
1274	}
1275	for (U32 idx=0; idx<num_entries; idx++)
1276	{
1277		Entry entry;
1278		S32 bytes_read = aprfile->read((void*)(&entry), (S32)sizeof(Entry));
1279		if (bytes_read < sizeof(Entry))
1280		{
1281			llwarns << "Corrupted header entries, failed at " << idx << " / " << num_entries << llendl;
1282			closeHeaderEntriesFile();
1283			purgeAllTextures(false);
1284			return 0;
1285		}
1286		entries.push_back(entry);
1287// 		llinfos << "ENTRY: " << entry.mTime << " TEX: " << entry.mID << " IDX: " << idx << " Size: " << entry.mImageSize << llendl;
1288		if(entry.mImageSize > entry.mBodySize)
1289		{
1290			mHeaderIDMap[entry.mID] = idx;
1291			mTexturesSizeMap[entry.mID] = entry.mBodySize;
1292			mTexturesSizeTotal += entry.mBodySize;
1293		}
1294		else
1295		{
1296			mFreeList.insert(idx);
1297		}
1298	}
1299	closeHeaderEntriesFile();
1300	return num_entries;
1301}
1302
1303void LLTextureCache::writeEntriesAndClose(const std::vector<Entry>& entries)
1304{
1305	S32 num_entries = entries.size();
1306	llassert_always(num_entries == mHeaderEntriesInfo.mEntries);
1307	
1308	if (!mReadOnly)
1309	{
1310		LLAPRFile* aprfile = openHeaderEntriesFile(false, (S32)sizeof(EntriesInfo));
1311		for (S32 idx=0; idx<num_entries; idx++)
1312		{
1313			S32 bytes_written = aprfile->write((void*)(&entries[idx]), (S32)sizeof(Entry));
1314			if(bytes_written != sizeof(Entry))
1315			{
1316				clearCorruptedCache() ; //clear the cache.
1317				return ;
1318			}
1319		}
1320		closeHeaderEntriesFile();
1321	}
1322}
1323
1324void LLTextureCache::writeUpdatedEntries()
1325{
1326	lockHeaders() ;
1327	if (!mReadOnly && !mUpdatedEntryMap.empty())
1328	{
1329		openHeaderEntriesFile(false, 0);
1330		updatedHeaderEntriesFile() ;
1331		closeHeaderEntriesFile();
1332	}
1333	unlockHeaders() ;
1334}
1335
1336//mHeaderMutex is locked and mHeaderAPRFile is created before calling this.
1337void LLTextureCache::updatedHeaderEntriesFile()
1338{
1339	if (!mReadOnly && !mUpdatedEntryMap.empty() && mHeaderAPRFile)
1340	{
1341		//entriesInfo
1342		mHeaderAPRFile->seek(APR_SET, 0);
1343		S32 bytes_written = mHeaderAPRFile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)) ;
1344		if(bytes_written != sizeof(EntriesInfo))
1345		{
1346			clearCorruptedCache() ; //clear the cache.
1347			return ;
1348		}
1349		
1350		//write each updated entry
1351		S32 entry_size = (S32)sizeof(Entry) ;
1352		S32 prev_idx = -1 ;
1353		S32 delta_idx ;
1354		for (idx_entry_map_t::iterator iter = mUpdatedEntryMap.begin(); iter != mUpdatedEntryMap.end(); ++iter)
1355		{
1356			delta_idx = iter->first - prev_idx - 1;
1357			prev_idx = iter->first ;
1358			if(delta_idx)
1359			{
1360				mHeaderAPRFile->seek(APR_CUR, delta_idx * entry_size);
1361			}
1362			
1363			bytes_written = mHeaderAPRFile->write((void*)(&iter->second), entry_size);
1364			if(bytes_written != entry_size)
1365			{
1366				clearCorruptedCache() ; //clear the cache.
1367				return ;
1368			}
1369		}
1370		mUpdatedEntryMap.clear() ;
1371	}
1372}
1373//----------------------------------------------------------------------------
1374
1375// Called from either the main thread or the worker thread
1376void LLTextureCache::readHeaderCache()
1377{
1378	mHeaderMutex.lock();
1379
1380	mLRU.clear(); // always clear the LRU
1381
1382	readEntriesHeader();
1383	
1384	if (mHeaderEntriesInfo.mVersion != sHeaderCacheVersion)
1385	{
1386		if (!mReadOnly)
1387		{
1388			purgeAllTextures(false);
1389		}
1390	}
1391	else
1392	{
1393		std::vector<Entry> entries;
1394		U32 num_entries = openAndReadEntries(entries);
1395		if (num_entries)
1396		{
1397			U32 empty_entries = 0;
1398			typedef std::pair<U32, S32> lru_data_t;
1399			std::set<lru_data_t> lru;
1400			std::set<U32> purge_list;
1401			for (U32 i=0; i<num_entries; i++)
1402			{
1403				Entry& entry = entries[i];
1404				if (entry.mImageSize <= 0)
1405				{
1406					// This will be in the Free List, don't put it in the LRU
1407					++empty_entries;
1408				}
1409				else
1410				{
1411					lru.insert(std::make_pair(entry.mTime, i));
1412					if (entry.mBodySize > 0)
1413					{
1414						if (entry.mBodySize > entry.mImageSize)
1415						{
1416							// Shouldn't happen, failsafe only
1417							llwarns << "Bad entry: " << i << ": " << entry.mID << ": BodySize: " << entry.mBodySize << llendl;
1418							purge_list.insert(i);
1419						}
1420					}
1421				}
1422			}
1423			if (num_entries - empty_entries > sCacheMaxEntries)
1424			{
1425				// Special case: cache size was reduced, need to remove entries
1426				// Note: After we prune entries, we will call this again and create the LRU
1427				U32 entries_to_purge = (num_entries - empty_entries) - sCacheMaxEntries;
1428				llinfos << "Texture Cache Entries: " << num_entries << " Max: " << sCacheMaxEntries << " Empty: " << empty_entries << " Purging: " << entries_to_purge << llendl;
1429				// We can exit the following loop with the given condition, since if we'd reach the end of the lru set we'd have:
1430				// purge_list.size() = lru.size() = num_entries - empty_entries = entries_to_purge + sCacheMaxEntries >= entries_to_purge
1431				// So, it's certain that iter will never reach lru.end() first.
1432				std::set<lru_data_t>::iterator iter = lru.begin();
1433				while (purge_list.size() < entries_to_purge)
1434				{
1435					purge_list.insert(iter->second);
1436					++iter;
1437				}
1438			}
1439			else
1440			{
1441				S32 lru_entries = (S32)((F32)sCacheMaxEntries * TEXTURE_CACHE_LRU_SIZE);
1442				for (std::set<lru_data_t>::iterator iter = lru.begin(); iter != lru.end(); ++iter)
1443				{
1444					mLRU.insert(entries[iter->second].mID);
1445// 					llinfos << "LRU: " << iter->first << " : " << iter->second << llendl;
1446					if (--lru_entries <= 0)
1447						break;
1448				}
1449			}
1450			
1451			if (purge_list.size() > 0)
1452			{
1453				for (std::set<U32>::iterator iter = purge_list.begin(); iter != purge_list.end(); ++iter)
1454				{
1455					std::string tex_filename = getTextureFileName(entries[*iter].mID);
1456					removeEntry((S32)*iter, entries[*iter], tex_filename);
1457				}
1458				// If we removed any entries, we need to rebuild the entries list,
1459				// write the header, and call this again
1460				std::vector<Entry> new_entries;
1461				for (U32 i=0; i<num_entries; i++)
1462				{
1463					const Entry& entry = entries[i];
1464					if (entry.mImageSize > 0)
1465					{
1466						new_entries.push_back(entry);
1467					}
1468				}
1469				llassert_always(new_entries.size() <= sCacheMaxEntries);
1470				mHeaderEntriesInfo.mEntries = new_entries.size();
1471				writeEntriesHeader();
1472				writeEntriesAndClose(new_entries);
1473				mHeaderMutex.unlock(); // unlock the mutex before calling again
1474				readHeaderCache(); // repeat with new entries file
1475				mHeaderMutex.lock();
1476			}
1477			else
1478			{
1479				//entries are not changed, nothing here.
1480			}
1481		}
1482	}
1483	mHeaderMutex.unlock();
1484}
1485
1486//////////////////////////////////////////////////////////////////////////////
1487
1488//the header mutex is locked before calling this.
1489void LLTextureCache::clearCorruptedCache()
1490{
1491	llwarns << "the texture cache is corrupted, need to be cleared." << llendl ;
1492
1493	closeHeaderEntriesFile();//close possible file handler
1494	purgeAllTextures(false) ; //clear the cache.
1495	
1496	if (!mReadOnly) //regenerate the directory tree if not exists.
1497	{
1498		LLFile::mkdir(mTexturesDirName);
1499		
1500		const char* subdirs = "0123456789abcdef";
1501		for (S32 i=0; i<16; i++)
1502		{
1503			std::string dirname = mTexturesDirName + gDirUtilp->getDirDelimiter() + subdirs[i];
1504			LLFile::mkdir(dirname);
1505		}
1506	}
1507
1508	return ;
1509}
1510
1511void LLTextureCache::purgeAllTextures(bool purge_directories)
1512{
1513	if (!mReadOnly)
1514	{
1515		const char* subdirs = "0123456789abcdef";
1516		std::string delem = gDirUtilp->getDirDelimiter();
1517		std::string mask = "*";
1518		for (S32 i=0; i<16; i++)
1519		{
1520			std::string dirname = mTexturesDirName + delem + subdirs[i];
1521			llinfos << "Deleting files in directory: " << dirname << llendl;
1522			gDirUtilp->deleteFilesInDir(dirname, mask);
1523			if (purge_directories)
1524			{
1525				LLFile::rmdir(dirname);
1526			}
1527		}
1528		if (purge_directories)
1529		{
1530			gDirUtilp->deleteFilesInDir(mTexturesDirName, mask);
1531			LLFile::rmdir(mTexturesDirName);
1532		}		
1533	}
1534	mHeaderIDMap.clear();
1535	mTexturesSizeMap.clear();
1536	mTexturesSizeTotal = 0;
1537	mFreeList.clear();
1538	mTexturesSizeTotal = 0;
1539	mUpdatedEntryMap.clear();
1540
1541	// Info with 0 entries
1542	mHeaderEntriesInfo.mVersion = sHeaderCacheVersion;
1543	mHeaderEntriesInfo.mEntries = 0;
1544	writeEntriesHeader();
1545
1546	llinfos << "The entire texture cache is cleared." << llendl ;
1547}
1548
1549void LLTextureCache::purgeTextures(bool validate)
1550{
1551	if (mReadOnly)
1552	{
1553		return;
1554	}
1555
1556	if (!mThreaded)
1557	{
1558		// *FIX:Mani - watchdog off.
1559		LLAppViewer::instance()->pauseMainloopTimeout();
1560	}
1561	
1562	LLMutexLock lock(&mHeaderMutex);
1563
1564	llinfos << "TEXTURE CACHE: Purging." << llendl;
1565
1566	// Read the entries list
1567	std::vector<Entry> entries;
1568	U32 num_entries = openAndReadEntries(entries);
1569	if (!num_entries)
1570	{
1571		return; // nothing to purge
1572	}
1573	
1574	// Use mTexturesSizeMap to collect UUIDs of textures with bodies
1575	typedef std::set<std::pair<U32,S32> > time_idx_set_t;
1576	std::set<std::pair<U32,S32> > time_idx_set;
1577	for (size_map_t::iterator iter1 = mTexturesSizeMap.begin();
1578		 iter1 != mTexturesSizeMap.end(); ++iter1)
1579	{
1580		if (iter1->second > 0)
1581		{
1582			id_map_t::iterator iter2 = mHeaderIDMap.find(iter1->first);
1583			if (iter2 != mHeaderIDMap.end())
1584			{
1585				S32 idx = iter2->second;
1586				time_idx_set.insert(std::make_pair(entries[idx].mTime, idx));
1587// 				llinfos << "TIME: " << entries[idx].mTime << " TEX: " << entries[idx].mID << " IDX: " << idx << " Size: " << entries[idx].mImageSize << llendl;
1588			}
1589			else
1590			{
1591				llerrs << "mTexturesSizeMap / mHeaderIDMap corrupted." << llendl ;
1592			}
1593		}
1594	}
1595	
1596	// Validate 1/256th of the files on startup
1597	U32 validate_idx = 0;
1598	if (validate)
1599	{
1600		validate_idx = gSavedSettings.getU32("CacheValidateCounter");
1601		U32 next_idx = (validate_idx + 1) % 256;
1602		gSavedSettings.setU32("CacheValidateCounter", next_idx);
1603		LL_DEBUGS("TextureCache") << "TEXTURE CACHE: Validating: " << validate_idx << LL_ENDL;
1604	}
1605
1606	S64 cache_size = mTexturesSizeTotal;
1607	S64 purged_cache_size = (sCacheMaxTexturesSize * (S64)((1.f-TEXTURE_CACHE_PURGE_AMOUNT)*100)) / 100;
1608	S32 purge_count = 0;
1609	for (time_idx_set_t::iterator iter = time_idx_set.begin();
1610		 iter != time_idx_set.end(); ++iter)
1611	{
1612		S32 idx = iter->second;
1613		bool purge_entry = false;
1614		std::string filename = getTextureFileName(entries[idx].mID);
1615		if (cache_size >= purged_cache_size)
1616		{
1617			purge_entry = true;
1618		}
1619		else if (validate)
1620		{
1621			// make sure file exists and is the correct size
1622			U32 uuididx = entries[idx].mID.mData[0];
1623			if (uuididx == validate_idx)
1624			{
1625 				LL_DEBUGS("TextureCache") << "Validating: " << filename << "Size: " << entries[idx].mBodySize << LL_ENDL;
1626				S32 bodysize = LLAPRFile::size(filename, getLocalAPRFilePool());
1627				if (bodysize != entries[idx].mBodySize)
1628				{
1629					LL_WARNS("TextureCache") << "TEXTURE CACHE BODY HAS BAD SIZE: " << bodysize << " != " << entries[idx].mBodySize
1630							<< filename << LL_ENDL;
1631					purge_entry = true;
1632				}
1633			}
1634		}
1635		else
1636		{
1637			break;
1638		}
1639		
1640		if (purge_entry)
1641		{
1642			purge_count++;
1643	 		LL_DEBUGS("TextureCache") << "PURGING: " << filename << LL_ENDL;
1644			cache_size -= entries[idx].mBodySize;
1645			removeEntry(idx, entries[idx], filename) ;			
1646		}
1647	}
1648
1649	LL_DEBUGS("TextureCache") << "TEXTURE CACHE: Writing Entries: " << num_entries << LL_ENDL;
1650
1651	writeEntriesAndClose(entries);
1652	
1653	// *FIX:Mani - watchdog back on.
1654	LLAppViewer::instance()->resumeMainloopTimeout();
1655	
1656	LL_INFOS("TextureCache") << "TEXTURE CACHE:"
1657			<< " PURGED: " << purge_count
1658			<< " ENTRIES: " << num_entries
1659			<< " CACHE SIZE: " << mTexturesSizeTotal / (1024 * 1024) << " MB"
1660			<< llendl;
1661}
1662
1663//////////////////////////////////////////////////////////////////////////////
1664
1665// call lockWorkers() first!
1666LLTextureCacheWorker* LLTextureCache::getReader(handle_t handle)
1667{
1668	LLTextureCacheWorker* res = NULL;
1669	handle_map_t::iterator iter = mReaders.find(handle);
1670	if (iter != mReaders.end())
1671	{
1672		res = iter->second;
1673	}
1674	return res;
1675}
1676
1677LLTextureCacheWorker* LLTextureCache::getWriter(handle_t handle)
1678{
1679	LLTextureCacheWorker* res = NULL;
1680	handle_map_t::iterator iter = mWriters.find(handle);
1681	if (iter != mWriters.end())
1682	{
1683		res = iter->second;
1684	}
1685	return res;
1686}
1687
1688//////////////////////////////////////////////////////////////////////////////
1689// Called from work thread
1690
1691// Reads imagesize from the header, updates timestamp
1692S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, Entry& entry)
1693{
1694	LLMutexLock lock(&mHeaderMutex);	
1695	S32 idx = openAndReadEntry(id, entry, false);
1696	if (idx >= 0)
1697	{		
1698		updateEntryTimeStamp(idx, entry); // updates time
1699	}
1700	return idx;
1701}
1702
1703// Writes imagesize to the header, updates timestamp
1704S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize)
1705{
1706	mHeaderMutex.lock();
1707	S32 idx = openAndReadEntry(id, entry, true);
1708	mHeaderMutex.unlock();
1709
1710	if (idx >= 0)
1711	{
1712		updateEntry(idx, entry, imagesize, datasize);				
1713	}
1714
1715	if(idx < 0) // retry
1716	{
1717		readHeaderCache(); // We couldn't write an entry, so refresh the LRU
1718	
1719		mHeaderMutex.lock();
1720		llassert_always(!mLRU.empty() || mHeaderEntriesInfo.mEntries < sCacheMaxEntries);
1721		mHeaderMutex.unlock();
1722
1723		idx = setHeaderCacheEntry(id, entry, imagesize, datasize); // assert above ensures no inf. recursion
1724	}
1725	return idx;
1726}
1727
1728//////////////////////////////////////////////////////////////////////////////
1729
1730// Calls from texture pipeline thread (i.e. LLTextureFetch)
1731
1732LLTextureCache::handle_t LLTextureCache::readFromCache(const std::string& filename, const LLUUID& id, U32 priority,
1733													   S32 offset, S32 size, ReadResponder* responder)
1734{
1735	// Note: checking to see if an entry exists can cause a stall,
1736	//  so let the thread handle it
1737	LLMutexLock lock(&mWorkersMutex);
1738	LLTextureCacheWorker* worker = new LLTextureCacheLocalFileWorker(this, priority, filename, id,
1739																	 NULL, size, offset, 0,
1740																	 responder);
1741	handle_t handle = worker->read();
1742	mReaders[handle] = worker;
1743	return handle;
1744}
1745
1746LLTextureCache::handle_t LLTextureCache::readFromCache(const LLUUID& id, U32 priority,
1747													   S32 offset, S32 size, ReadResponder* responder)
1748{
1749	// Note: checking to see if an entry exists can cause a stall,
1750	//  so let the thread handle it
1751	LLMutexLock lock(&mWorkersMutex);
1752	LLTextureCacheWorker* worker = new LLTextureCacheRemoteWorker(this, priority, id,
1753																  NULL, size, offset,
1754																  0, responder);
1755	handle_t handle = worker->read();
1756	mReaders[handle] = worker;
1757	return handle;
1758}
1759
1760
1761bool LLTextureCache::readComplete(handle_t handle, bool abort)
1762{
1763	lockWorkers();
1764	handle_map_t::iterator iter = mReaders.find(handle);
1765	LLTextureCacheWorker* worker = NULL;
1766	bool complete = false;
1767	if (iter != mReaders.end())
1768	{
1769		worker = iter->second;
1770		complete = work

Large files files are truncated, but you can click here to view the full file