PageRenderTime 167ms CodeModel.GetById 7ms app.highlight 144ms RepoModel.GetById 1ms app.codeStats 1ms

/indra/llvfs/llvfs.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 2203 lines | 1641 code | 324 blank | 238 comment | 283 complexity | 0a0817c9115987fb04a19241b0351be4 MD5 | raw file
   1/** 
   2 * @file llvfs.cpp
   3 * @brief Implementation of virtual file system
   4 *
   5 * $LicenseInfo:firstyear=2002&license=viewerlgpl$
   6 * Second Life Viewer Source Code
   7 * Copyright (C) 2010, Linden Research, Inc.
   8 * 
   9 * This library is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU Lesser General Public
  11 * License as published by the Free Software Foundation;
  12 * version 2.1 of the License only.
  13 * 
  14 * This library is distributed in the hope that it will be useful,
  15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 * Lesser General Public License for more details.
  18 * 
  19 * You should have received a copy of the GNU Lesser General Public
  20 * License along with this library; if not, write to the Free Software
  21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  22 * 
  23 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  24 * $/LicenseInfo$
  25 */
  26
  27#include "linden_common.h"
  28
  29#include "llvfs.h"
  30
  31#include <sys/stat.h>
  32#include <set>
  33#include <map>
  34#if LL_WINDOWS
  35#include <share.h>
  36#elif LL_SOLARIS
  37#include <sys/types.h>
  38#include <unistd.h>
  39#include <fcntl.h>
  40#else
  41#include <sys/file.h>
  42#endif
  43    
  44#include "llstl.h"
  45#include "lltimer.h"
  46    
  47const S32 FILE_BLOCK_MASK = 0x000003FF;	 // 1024-byte blocks
  48const S32 VFS_CLEANUP_SIZE = 5242880;  // how much space we free up in a single stroke
  49const S32 BLOCK_LENGTH_INVALID = -1;	// mLength for invalid LLVFSFileBlocks
  50
  51LLVFS *gVFS = NULL;
  52
  53// internal class definitions
  54class LLVFSBlock
  55{
  56public:
  57	LLVFSBlock() 
  58	{
  59		mLocation = 0;
  60		mLength = 0;
  61	}
  62    
  63	LLVFSBlock(U32 loc, S32 size)
  64	{
  65		mLocation = loc;
  66		mLength = size;
  67	}
  68    
  69	static bool locationSortPredicate(
  70		const LLVFSBlock* lhs,
  71		const LLVFSBlock* rhs)
  72	{
  73		return lhs->mLocation < rhs->mLocation;
  74	}
  75
  76public:
  77	U32 mLocation;
  78	S32	mLength;		// allocated block size
  79};
  80    
  81LLVFSFileSpecifier::LLVFSFileSpecifier()
  82:	mFileID(),
  83	mFileType( LLAssetType::AT_NONE )
  84{
  85}
  86    
  87LLVFSFileSpecifier::LLVFSFileSpecifier(const LLUUID &file_id, const LLAssetType::EType file_type)
  88{
  89	mFileID = file_id;
  90	mFileType = file_type;
  91}
  92    
  93bool LLVFSFileSpecifier::operator<(const LLVFSFileSpecifier &rhs) const
  94{
  95	return (mFileID == rhs.mFileID)
  96		? mFileType < rhs.mFileType
  97		: mFileID < rhs.mFileID;
  98}
  99    
 100bool LLVFSFileSpecifier::operator==(const LLVFSFileSpecifier &rhs) const
 101{
 102	return (mFileID == rhs.mFileID && 
 103			mFileType == rhs.mFileType);
 104}
 105    
 106    
 107class LLVFSFileBlock : public LLVFSBlock, public LLVFSFileSpecifier
 108{
 109public:
 110	LLVFSFileBlock() : LLVFSBlock(), LLVFSFileSpecifier()
 111	{
 112		init();
 113	}
 114    
 115	LLVFSFileBlock(const LLUUID &file_id, LLAssetType::EType file_type, U32 loc = 0, S32 size = 0)
 116		: LLVFSBlock(loc, size), LLVFSFileSpecifier( file_id, file_type )
 117	{
 118		init();
 119	}
 120
 121	void init()
 122	{
 123		mSize = 0;
 124		mIndexLocation = -1;
 125		mAccessTime = (U32)time(NULL);
 126
 127		for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++)
 128		{
 129			mLocks[(EVFSLock)i] = 0;
 130		}
 131	}
 132
 133	#ifdef LL_LITTLE_ENDIAN
 134	inline void swizzleCopy(void *dst, void *src, int size) { memcpy(dst, src, size); /* Flawfinder: ignore */}
 135
 136	#else
 137	
 138	inline U32 swizzle32(U32 x)
 139	{
 140		return(((x >> 24) & 0x000000FF) | ((x >> 8)  & 0x0000FF00) | ((x << 8)  & 0x00FF0000) |((x << 24) & 0xFF000000));
 141	}
 142	
 143	inline U16 swizzle16(U16 x)
 144	{
 145		return(	((x >> 8)  & 0x000000FF) | ((x << 8)  & 0x0000FF00) );
 146	}
 147	
 148	inline void swizzleCopy(void *dst, void *src, int size) 
 149	{
 150		if(size == 4)
 151		{
 152			((U32*)dst)[0] = swizzle32(((U32*)src)[0]); 
 153		}
 154		else if(size == 2)
 155		{
 156			((U16*)dst)[0] = swizzle16(((U16*)src)[0]); 
 157		}
 158		else
 159		{
 160			// Perhaps this should assert...
 161			memcpy(dst, src, size);	/* Flawfinder: ignore */
 162		}
 163	}
 164	
 165	#endif
 166
 167	void serialize(U8 *buffer)
 168	{
 169		swizzleCopy(buffer, &mLocation, 4);
 170		buffer += 4;
 171		swizzleCopy(buffer, &mLength, 4);
 172		buffer +=4;
 173		swizzleCopy(buffer, &mAccessTime, 4);
 174		buffer +=4;
 175		memcpy(buffer, &mFileID.mData, 16); /* Flawfinder: ignore */	
 176		buffer += 16;
 177		S16 temp_type = mFileType;
 178		swizzleCopy(buffer, &temp_type, 2);
 179		buffer += 2;
 180		swizzleCopy(buffer, &mSize, 4);
 181	}
 182    
 183	void deserialize(U8 *buffer, const S32 index_loc)
 184	{
 185		mIndexLocation = index_loc;
 186    
 187		swizzleCopy(&mLocation, buffer, 4);
 188		buffer += 4;
 189		swizzleCopy(&mLength, buffer, 4);
 190		buffer += 4;
 191		swizzleCopy(&mAccessTime, buffer, 4);
 192		buffer += 4;
 193		memcpy(&mFileID.mData, buffer, 16);
 194		buffer += 16;
 195		S16 temp_type;
 196		swizzleCopy(&temp_type, buffer, 2);
 197		mFileType = (LLAssetType::EType)temp_type;
 198		buffer += 2;
 199		swizzleCopy(&mSize, buffer, 4);
 200	}
 201    
 202	static BOOL insertLRU(LLVFSFileBlock* const& first,
 203						  LLVFSFileBlock* const& second)
 204	{
 205		return (first->mAccessTime == second->mAccessTime)
 206			? *first < *second
 207			: first->mAccessTime < second->mAccessTime;
 208	}
 209    
 210public:
 211	S32  mSize;
 212	S32  mIndexLocation; // location of index entry
 213	U32  mAccessTime;
 214	BOOL mLocks[VFSLOCK_COUNT]; // number of outstanding locks of each type
 215    
 216	static const S32 SERIAL_SIZE;
 217};
 218
 219// Helper structure for doing lru w/ stl... is there a simpler way?
 220struct LLVFSFileBlock_less
 221{
 222	bool operator()(LLVFSFileBlock* const& lhs, LLVFSFileBlock* const& rhs) const
 223	{
 224		return (LLVFSFileBlock::insertLRU(lhs, rhs)) ? true : false;
 225	}
 226};
 227
 228
 229const S32 LLVFSFileBlock::SERIAL_SIZE = 34;
 230     
 231
 232LLVFS::LLVFS(const std::string& index_filename, const std::string& data_filename, const BOOL read_only, const U32 presize, const BOOL remove_after_crash)
 233:	mRemoveAfterCrash(remove_after_crash),
 234	mDataFP(NULL),
 235	mIndexFP(NULL)
 236{
 237	mDataMutex = new LLMutex(0);
 238
 239	S32 i;
 240	for (i = 0; i < VFSLOCK_COUNT; i++)
 241	{
 242		mLockCounts[i] = 0;
 243	}
 244	mValid = VFSVALID_OK;
 245	mReadOnly = read_only;
 246	mIndexFilename = index_filename;
 247	mDataFilename = data_filename;
 248    
 249	const char *file_mode = mReadOnly ? "rb" : "r+b";
 250    
 251	LL_INFOS("VFS") << "Attempting to open VFS index file " << mIndexFilename << LL_ENDL;
 252	LL_INFOS("VFS") << "Attempting to open VFS data file " << mDataFilename << LL_ENDL;
 253
 254	mDataFP = openAndLock(mDataFilename, file_mode, mReadOnly);
 255	if (!mDataFP)
 256	{
 257		if (mReadOnly)
 258		{
 259			LL_WARNS("VFS") << "Can't find " << mDataFilename << " to open read-only VFS" << LL_ENDL;
 260			mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY;
 261			return;
 262		}
 263
 264		mDataFP = openAndLock(mDataFilename, "w+b", FALSE);
 265		if (mDataFP)
 266		{
 267			// Since we're creating this data file, assume any index file is bogus
 268			// remove the index, since this vfs is now blank
 269			LLFile::remove(mIndexFilename);
 270		}
 271		else
 272		{
 273			LL_WARNS("VFS") << "Couldn't open vfs data file " 
 274				<< mDataFilename << LL_ENDL;
 275			mValid = VFSVALID_BAD_CANNOT_CREATE;
 276			return;
 277		}
 278
 279		if (presize)
 280		{
 281			presizeDataFile(presize);
 282		}
 283	}
 284
 285	// Did we leave this file open for writing last time?
 286	// If so, close it and start over.
 287	if (!mReadOnly && mRemoveAfterCrash)
 288	{
 289		llstat marker_info;
 290		std::string marker = mDataFilename + ".open";
 291		if (!LLFile::stat(marker, &marker_info))
 292		{
 293			// marker exists, kill the lock and the VFS files
 294			unlockAndClose(mDataFP);
 295			mDataFP = NULL;
 296
 297			LL_WARNS("VFS") << "VFS: File left open on last run, removing old VFS file " << mDataFilename << LL_ENDL;
 298			LLFile::remove(mIndexFilename);
 299			LLFile::remove(mDataFilename);
 300			LLFile::remove(marker);
 301
 302			mDataFP = openAndLock(mDataFilename, "w+b", FALSE);
 303			if (!mDataFP)
 304			{
 305				LL_WARNS("VFS") << "Can't open VFS data file in crash recovery" << LL_ENDL;
 306				mValid = VFSVALID_BAD_CANNOT_CREATE;
 307				return;
 308			}
 309
 310			if (presize)
 311			{
 312				presizeDataFile(presize);
 313			}
 314		}
 315	}
 316
 317	// determine the real file size
 318	fseek(mDataFP, 0, SEEK_END);
 319	U32 data_size = ftell(mDataFP);
 320
 321	// read the index file
 322	// make sure there's at least one file in it too
 323	// if not, we'll treat this as a new vfs
 324	llstat fbuf;
 325	if (! LLFile::stat(mIndexFilename, &fbuf) &&
 326		fbuf.st_size >= LLVFSFileBlock::SERIAL_SIZE &&
 327		(mIndexFP = openAndLock(mIndexFilename, file_mode, mReadOnly))	// Yes, this is an assignment and not '=='
 328		)
 329	{	
 330		std::vector<U8> buffer(fbuf.st_size);
 331    		size_t buf_offset = 0;
 332		size_t nread = fread(&buffer[0], 1, fbuf.st_size, mIndexFP);
 333 
 334		std::vector<LLVFSFileBlock*> files_by_loc;
 335		
 336		while (buf_offset < nread)
 337		{
 338			LLVFSFileBlock *block = new LLVFSFileBlock();
 339    
 340			block->deserialize(&buffer[buf_offset], (S32)buf_offset);
 341    
 342			// Do sanity check on this block.
 343			// Note that this skips zero size blocks, which helps VFS
 344			// to heal after some errors. JC
 345			if (block->mLength > 0 &&
 346				(U32)block->mLength <= data_size &&
 347				block->mLocation < data_size &&
 348				block->mSize > 0 &&
 349				block->mSize <= block->mLength &&
 350				block->mFileType >= LLAssetType::AT_NONE &&
 351				block->mFileType < LLAssetType::AT_COUNT)
 352			{
 353				mFileBlocks.insert(fileblock_map::value_type(*block, block));
 354				files_by_loc.push_back(block);
 355			}
 356			else
 357			if (block->mLength && block->mSize > 0)
 358			{
 359				// this is corrupt, not empty
 360				LL_WARNS("VFS") << "VFS corruption: " << block->mFileID << " (" << block->mFileType << ") at index " << block->mIndexLocation << " DS: " << data_size << LL_ENDL;
 361				LL_WARNS("VFS") << "Length: " << block->mLength << "\tLocation: " << block->mLocation << "\tSize: " << block->mSize << LL_ENDL;
 362				LL_WARNS("VFS") << "File has bad data - VFS removed" << LL_ENDL;
 363
 364				delete block;
 365
 366				unlockAndClose( mIndexFP );
 367				mIndexFP = NULL;
 368				LLFile::remove( mIndexFilename );
 369
 370				unlockAndClose( mDataFP );
 371				mDataFP = NULL;
 372				LLFile::remove( mDataFilename );
 373
 374				LL_WARNS("VFS") << "Deleted corrupt VFS files " 
 375					<< mDataFilename 
 376					<< " and "
 377					<< mIndexFilename
 378					<< LL_ENDL;
 379
 380				mValid = VFSVALID_BAD_CORRUPT;
 381				return;
 382			}
 383			else
 384			{
 385				// this is a null or bad entry, skip it
 386				mIndexHoles.push_back(buf_offset);
 387    
 388				delete block;
 389			}
 390    
 391			buf_offset += LLVFSFileBlock::SERIAL_SIZE;
 392		}
 393
 394		std::sort(
 395			files_by_loc.begin(),
 396			files_by_loc.end(),
 397			LLVFSFileBlock::locationSortPredicate);
 398
 399		// There are 3 cases that have to be considered.
 400		// 1. No blocks
 401		// 2. One block.
 402		// 3. Two or more blocks.
 403		if (!files_by_loc.empty())
 404		{
 405			// cur walks through the list.
 406			std::vector<LLVFSFileBlock*>::iterator cur = files_by_loc.begin();
 407			std::vector<LLVFSFileBlock*>::iterator end = files_by_loc.end();
 408			LLVFSFileBlock* last_file_block = *cur;
 409			
 410			// Check to see if there is an empty space before the first file.
 411			if (last_file_block->mLocation > 0)
 412			{
 413				// If so, create a free block.
 414				addFreeBlock(new LLVFSBlock(0, last_file_block->mLocation));
 415			}
 416
 417			// Walk through the 2nd+ block.  If there is a free space
 418			// between cur_file_block and last_file_block, add it to
 419			// the free space collection.  This block will not need to
 420			// run in the case there is only one entry in the VFS.
 421			++cur;
 422			while( cur != end )
 423			{
 424				LLVFSFileBlock* cur_file_block = *cur;
 425
 426				// Dupe check on the block
 427				if (cur_file_block->mLocation == last_file_block->mLocation
 428					&& cur_file_block->mLength == last_file_block->mLength)
 429				{
 430					LL_WARNS("VFS") << "VFS: removing duplicate entry"
 431						<< " at " << cur_file_block->mLocation 
 432						<< " length " << cur_file_block->mLength 
 433						<< " size " << cur_file_block->mSize
 434						<< " ID " << cur_file_block->mFileID 
 435						<< " type " << cur_file_block->mFileType 
 436						<< LL_ENDL;
 437
 438					// Duplicate entries.  Nuke them both for safety.
 439					mFileBlocks.erase(*cur_file_block);	// remove ID/type entry
 440					if (cur_file_block->mLength > 0)
 441					{
 442						// convert to hole
 443						addFreeBlock(
 444							new LLVFSBlock(
 445								cur_file_block->mLocation,
 446								cur_file_block->mLength));
 447					}
 448					lockData();						// needed for sync()
 449					sync(cur_file_block, TRUE);		// remove first on disk
 450					sync(last_file_block, TRUE);	// remove last on disk
 451					unlockData();					// needed for sync()
 452					last_file_block = cur_file_block;
 453					++cur;
 454					continue;
 455				}
 456
 457				// Figure out where the last block ended.
 458				S32 loc = last_file_block->mLocation+last_file_block->mLength;
 459
 460				// Figure out how much space there is between where
 461				// the last block ended and this block begins.
 462				S32 length = cur_file_block->mLocation - loc;
 463    
 464				// Check for more errors...  Seeing if the current
 465				// entry and the last entry make sense together.
 466				if (length < 0 || loc < 0 || (U32)loc > data_size)
 467				{
 468					// Invalid VFS
 469					unlockAndClose( mIndexFP );
 470					mIndexFP = NULL;
 471					LLFile::remove( mIndexFilename );
 472
 473					unlockAndClose( mDataFP );
 474					mDataFP = NULL;
 475					LLFile::remove( mDataFilename );
 476
 477					LL_WARNS("VFS") << "VFS: overlapping entries"
 478						<< " at " << cur_file_block->mLocation 
 479						<< " length " << cur_file_block->mLength 
 480						<< " ID " << cur_file_block->mFileID 
 481						<< " type " << cur_file_block->mFileType 
 482						<< LL_ENDL;
 483
 484					LL_WARNS("VFS") << "Deleted corrupt VFS files " 
 485						<< mDataFilename 
 486						<< " and "
 487						<< mIndexFilename
 488						<< LL_ENDL;
 489
 490					mValid = VFSVALID_BAD_CORRUPT;
 491					return;
 492				}
 493
 494				// we don't want to add empty blocks to the list...
 495				if (length > 0)
 496				{
 497					addFreeBlock(new LLVFSBlock(loc, length));
 498				}
 499				last_file_block = cur_file_block;
 500				++cur;
 501			}
 502    
 503			// also note any empty space at the end
 504			U32 loc = last_file_block->mLocation + last_file_block->mLength;
 505			if (loc < data_size)
 506			{
 507				addFreeBlock(new LLVFSBlock(loc, data_size - loc));
 508			}
 509		}
 510		else // There where no blocks in the file.
 511		{
 512			addFreeBlock(new LLVFSBlock(0, data_size));
 513		}
 514	}
 515	else	// Pre-existing index file wasn't opened
 516	{
 517		if (mReadOnly)
 518		{
 519			LL_WARNS("VFS") << "Can't find " << mIndexFilename << " to open read-only VFS" << LL_ENDL;
 520			mValid = VFSVALID_BAD_CANNOT_OPEN_READONLY;
 521			return;
 522		}
 523    
 524	
 525		mIndexFP = openAndLock(mIndexFilename, "w+b", FALSE);
 526		if (!mIndexFP)
 527		{
 528			LL_WARNS("VFS") << "Couldn't open an index file for the VFS, probably a sharing violation!" << LL_ENDL;
 529
 530			unlockAndClose( mDataFP );
 531			mDataFP = NULL;
 532			LLFile::remove( mDataFilename );
 533			
 534			mValid = VFSVALID_BAD_CANNOT_CREATE;
 535			return;
 536		}
 537	
 538		// no index file, start from scratch w/ 1GB allocation
 539		LLVFSBlock *first_block = new LLVFSBlock(0, data_size ? data_size : 0x40000000);
 540		addFreeBlock(first_block);
 541	}
 542
 543	// Open marker file to look for bad shutdowns
 544	if (!mReadOnly && mRemoveAfterCrash)
 545	{
 546		std::string marker = mDataFilename + ".open";
 547		LLFILE* marker_fp = LLFile::fopen(marker, "w");	/* Flawfinder: ignore */
 548		if (marker_fp)
 549		{
 550			fclose(marker_fp);
 551			marker_fp = NULL;
 552		}
 553	}
 554
 555	LL_INFOS("VFS") << "Using VFS index file " << mIndexFilename << LL_ENDL;
 556	LL_INFOS("VFS") << "Using VFS data file " << mDataFilename << LL_ENDL;
 557
 558	mValid = VFSVALID_OK;
 559}
 560    
 561LLVFS::~LLVFS()
 562{
 563	if (mDataMutex->isLocked())
 564	{
 565		LL_ERRS("VFS") << "LLVFS destroyed with mutex locked" << LL_ENDL;
 566	}
 567	
 568	unlockAndClose(mIndexFP);
 569	mIndexFP = NULL;
 570
 571	fileblock_map::const_iterator it;
 572	for (it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
 573	{
 574		delete (*it).second;
 575	}
 576	mFileBlocks.clear();
 577	
 578	mFreeBlocksByLength.clear();
 579
 580	for_each(mFreeBlocksByLocation.begin(), mFreeBlocksByLocation.end(), DeletePairedPointer());
 581    
 582	unlockAndClose(mDataFP);
 583	mDataFP = NULL;
 584    
 585	// Remove marker file
 586	if (!mReadOnly && mRemoveAfterCrash)
 587	{
 588		std::string marker = mDataFilename + ".open";
 589		LLFile::remove(marker);
 590	}
 591
 592	delete mDataMutex;
 593}
 594
 595
 596// Use this function normally to create LLVFS files.  
 597// Will append digits to the end of the filename with multiple re-trys
 598// static 
 599LLVFS * LLVFS::createLLVFS(const std::string& index_filename, 
 600		const std::string& data_filename, 
 601		const BOOL read_only, 
 602		const U32 presize, 
 603		const BOOL remove_after_crash)
 604{
 605	LLVFS * new_vfs = new LLVFS(index_filename, data_filename, read_only, presize, remove_after_crash);
 606
 607	if( !new_vfs->isValid() )
 608	{	// First name failed, retry with new names
 609		std::string retry_vfs_index_name;
 610		std::string retry_vfs_data_name;
 611		S32 count = 0;
 612		while (!new_vfs->isValid() &&
 613				count < 256)
 614		{	// Append '.<number>' to end of filenames
 615			retry_vfs_index_name = index_filename + llformat(".%u",count);
 616			retry_vfs_data_name = data_filename + llformat(".%u", count);
 617
 618			delete new_vfs;	// Delete bad VFS and try again
 619			new_vfs = new LLVFS(retry_vfs_index_name, retry_vfs_data_name, read_only, presize, remove_after_crash);
 620
 621			count++;
 622		}
 623	}
 624
 625	if( !new_vfs->isValid() )
 626	{
 627		delete new_vfs;		// Delete bad VFS
 628		new_vfs = NULL;		// Total failure
 629	}
 630
 631	return new_vfs;
 632}
 633
 634
 635
 636void LLVFS::presizeDataFile(const U32 size)
 637{
 638	if (!mDataFP)
 639	{
 640		llerrs << "LLVFS::presizeDataFile() with no data file open" << llendl;
 641		return;
 642	}
 643
 644	// we're creating this file for the first time, size it
 645	fseek(mDataFP, size-1, SEEK_SET);
 646	S32 tmp = 0;
 647	tmp = (S32)fwrite(&tmp, 1, 1, mDataFP);
 648	// fflush(mDataFP);
 649
 650	// also remove any index, since this vfs is now blank
 651	LLFile::remove(mIndexFilename);
 652
 653	if (tmp)
 654	{
 655		llinfos << "Pre-sized VFS data file to " << ftell(mDataFP) << " bytes" << llendl;
 656	}
 657	else
 658	{
 659		llwarns << "Failed to pre-size VFS data file" << llendl;
 660	}
 661}
 662
 663BOOL LLVFS::getExists(const LLUUID &file_id, const LLAssetType::EType file_type)
 664{
 665	LLVFSFileBlock *block = NULL;
 666		
 667	if (!isValid())
 668	{
 669		llerrs << "Attempting to use invalid VFS!" << llendl;
 670	}
 671
 672	lockData();
 673	
 674	LLVFSFileSpecifier spec(file_id, file_type);
 675	fileblock_map::iterator it = mFileBlocks.find(spec);
 676	if (it != mFileBlocks.end())
 677	{
 678		block = (*it).second;
 679		block->mAccessTime = (U32)time(NULL);
 680	}
 681
 682	BOOL res = (block && block->mLength > 0) ? TRUE : FALSE;
 683	
 684	unlockData();
 685	
 686	return res;
 687}
 688    
 689S32	 LLVFS::getSize(const LLUUID &file_id, const LLAssetType::EType file_type)
 690{
 691	S32 size = 0;
 692	
 693	if (!isValid())
 694	{
 695		llerrs << "Attempting to use invalid VFS!" << llendl;
 696
 697	}
 698
 699	lockData();
 700	
 701	LLVFSFileSpecifier spec(file_id, file_type);
 702	fileblock_map::iterator it = mFileBlocks.find(spec);
 703	if (it != mFileBlocks.end())
 704	{
 705		LLVFSFileBlock *block = (*it).second;
 706
 707		block->mAccessTime = (U32)time(NULL);
 708		size = block->mSize;
 709	}
 710
 711	unlockData();
 712	
 713	return size;
 714}
 715    
 716S32  LLVFS::getMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type)
 717{
 718	S32 size = 0;
 719	
 720	if (!isValid())
 721	{
 722		llerrs << "Attempting to use invalid VFS!" << llendl;
 723	}
 724
 725	lockData();
 726	
 727	LLVFSFileSpecifier spec(file_id, file_type);
 728	fileblock_map::iterator it = mFileBlocks.find(spec);
 729	if (it != mFileBlocks.end())
 730	{
 731		LLVFSFileBlock *block = (*it).second;
 732
 733		block->mAccessTime = (U32)time(NULL);
 734		size = block->mLength;
 735	}
 736
 737	unlockData();
 738
 739	return size;
 740}
 741
 742BOOL LLVFS::checkAvailable(S32 max_size)
 743{
 744	lockData();
 745	
 746	blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(max_size); // first entry >= size
 747	const BOOL res(iter == mFreeBlocksByLength.end() ? FALSE : TRUE);
 748
 749	unlockData();
 750	
 751	return res;
 752}
 753
 754BOOL LLVFS::setMaxSize(const LLUUID &file_id, const LLAssetType::EType file_type, S32 max_size)
 755{
 756	if (!isValid())
 757	{
 758		llerrs << "Attempting to use invalid VFS!" << llendl;
 759	}
 760	if (mReadOnly)
 761	{
 762		llerrs << "Attempt to write to read-only VFS" << llendl;
 763	}
 764	if (max_size <= 0)
 765	{
 766		llwarns << "VFS: Attempt to assign size " << max_size << " to vfile " << file_id << llendl;
 767		return FALSE;
 768	}
 769
 770	lockData();
 771	
 772	LLVFSFileSpecifier spec(file_id, file_type);
 773	LLVFSFileBlock *block = NULL;
 774	fileblock_map::iterator it = mFileBlocks.find(spec);
 775	if (it != mFileBlocks.end())
 776	{
 777		block = (*it).second;
 778	}
 779    
 780	// round all sizes upward to KB increments
 781	// SJB: Need to not round for the new texture-pipeline code so we know the correct
 782	//      max file size. Need to investigate the potential problems with this...
 783	if (file_type != LLAssetType::AT_TEXTURE)
 784	{
 785		if (max_size & FILE_BLOCK_MASK)
 786		{
 787			max_size += FILE_BLOCK_MASK;
 788			max_size &= ~FILE_BLOCK_MASK;
 789		}
 790    }
 791	
 792	if (block && block->mLength > 0)
 793	{    
 794		block->mAccessTime = (U32)time(NULL);
 795    
 796		if (max_size == block->mLength)
 797		{
 798			unlockData();
 799			return TRUE;
 800		}
 801		else if (max_size < block->mLength)
 802		{
 803			// this file is shrinking
 804			LLVFSBlock *free_block = new LLVFSBlock(block->mLocation + max_size, block->mLength - max_size);
 805
 806			addFreeBlock(free_block);
 807    
 808			block->mLength = max_size;
 809    
 810			if (block->mLength < block->mSize)
 811			{
 812				// JC: Was a warning, but Ian says it's bad.
 813				llerrs << "Truncating virtual file " << file_id << " to " << block->mLength << " bytes" << llendl;
 814				block->mSize = block->mLength;
 815			}
 816    
 817			sync(block);
 818			//mergeFreeBlocks();
 819
 820			unlockData();
 821			return TRUE;
 822		}
 823		else if (max_size > block->mLength)
 824		{
 825			// this file is growing
 826			// first check for an adjacent free block to grow into
 827			S32 size_increase = max_size - block->mLength;
 828
 829			// Find the first free block with and addres > block->mLocation
 830			LLVFSBlock *free_block;
 831			blocks_location_map_t::iterator iter = mFreeBlocksByLocation.upper_bound(block->mLocation);
 832			if (iter != mFreeBlocksByLocation.end())
 833			{
 834				free_block = iter->second;
 835			
 836				if (free_block->mLocation == block->mLocation + block->mLength &&
 837					free_block->mLength >= size_increase)
 838				{
 839					// this free block is at the end of the file and is large enough
 840
 841					// Must call useFreeSpace before sync(), as sync()
 842					// unlocks data structures.
 843					useFreeSpace(free_block, size_increase);
 844					block->mLength += size_increase;
 845					sync(block);
 846
 847					unlockData();
 848					return TRUE;
 849				}
 850			}
 851			
 852			// no adjecent free block, find one in the list
 853			free_block = findFreeBlock(max_size, block);
 854    
 855			if (free_block)
 856			{
 857				// Save location where data is going, useFreeSpace will move free_block->mLocation;
 858				U32 new_data_location = free_block->mLocation;
 859
 860				//mark the free block as used so it does not
 861				//interfere with other operations such as addFreeBlock
 862				useFreeSpace(free_block, max_size);		// useFreeSpace takes ownership (and may delete) free_block
 863
 864				if (block->mLength > 0)
 865				{
 866					// create a new free block where this file used to be
 867					LLVFSBlock *new_free_block = new LLVFSBlock(block->mLocation, block->mLength);
 868
 869					addFreeBlock(new_free_block);
 870					
 871					if (block->mSize > 0)
 872					{
 873						// move the file into the new block
 874						std::vector<U8> buffer(block->mSize);
 875						fseek(mDataFP, block->mLocation, SEEK_SET);
 876						if (fread(&buffer[0], block->mSize, 1, mDataFP) == 1)
 877						{
 878							fseek(mDataFP, new_data_location, SEEK_SET);
 879							if (fwrite(&buffer[0], block->mSize, 1, mDataFP) != 1)
 880							{
 881								llwarns << "Short write" << llendl;
 882							}
 883						} else {
 884							llwarns << "Short read" << llendl;
 885						}
 886					}
 887				}
 888    
 889				block->mLocation = new_data_location;
 890    
 891				block->mLength = max_size;
 892
 893
 894				sync(block);
 895
 896				unlockData();
 897				return TRUE;
 898			}
 899			else
 900			{
 901				llwarns << "VFS: No space (" << max_size << ") to resize existing vfile " << file_id << llendl;
 902				//dumpMap();
 903				unlockData();
 904				dumpStatistics();
 905				return FALSE;
 906			}
 907		}
 908	}
 909	else
 910	{
 911		// find a free block in the list
 912		LLVFSBlock *free_block = findFreeBlock(max_size);
 913    
 914		if (free_block)
 915		{        
 916			if (block)
 917			{
 918				block->mLocation = free_block->mLocation;
 919				block->mLength = max_size;
 920			}
 921			else
 922			{
 923				// this file doesn't exist, create it
 924				block = new LLVFSFileBlock(file_id, file_type, free_block->mLocation, max_size);
 925				mFileBlocks.insert(fileblock_map::value_type(spec, block));
 926			}
 927
 928			// Must call useFreeSpace before sync(), as sync()
 929			// unlocks data structures.
 930			useFreeSpace(free_block, max_size);
 931			block->mAccessTime = (U32)time(NULL);
 932
 933			sync(block);
 934		}
 935		else
 936		{
 937			llwarns << "VFS: No space (" << max_size << ") for new virtual file " << file_id << llendl;
 938			//dumpMap();
 939			unlockData();
 940			dumpStatistics();
 941			return FALSE;
 942		}
 943	}
 944	unlockData();
 945	return TRUE;
 946}
 947
 948
 949// WARNING: HERE BE DRAGONS!
 950// rename is the weirdest VFS op, because the file moves but the locks don't!
 951void LLVFS::renameFile(const LLUUID &file_id, const LLAssetType::EType file_type,
 952					   const LLUUID &new_id, const LLAssetType::EType &new_type)
 953{
 954	if (!isValid())
 955	{
 956		llerrs << "Attempting to use invalid VFS!" << llendl;
 957	}
 958	if (mReadOnly)
 959	{
 960		llerrs << "Attempt to write to read-only VFS" << llendl;
 961	}
 962
 963	lockData();
 964	
 965	LLVFSFileSpecifier new_spec(new_id, new_type);
 966	LLVFSFileSpecifier old_spec(file_id, file_type);
 967	
 968	fileblock_map::iterator it = mFileBlocks.find(old_spec);
 969	if (it != mFileBlocks.end())
 970	{
 971		LLVFSFileBlock *src_block = (*it).second;
 972
 973		// this will purge the data but leave the file block in place, w/ locks, if any
 974		// WAS: removeFile(new_id, new_type); NOW uses removeFileBlock() to avoid mutex lock recursion
 975		fileblock_map::iterator new_it = mFileBlocks.find(new_spec);
 976		if (new_it != mFileBlocks.end())
 977		{
 978			LLVFSFileBlock *new_block = (*new_it).second;
 979			removeFileBlock(new_block);
 980		}
 981		
 982		// if there's something in the target location, remove it but inherit its locks
 983		it = mFileBlocks.find(new_spec);
 984		if (it != mFileBlocks.end())
 985		{
 986			LLVFSFileBlock *dest_block = (*it).second;
 987
 988			for (S32 i = 0; i < (S32)VFSLOCK_COUNT; i++)
 989			{
 990				if(dest_block->mLocks[i])
 991				{
 992					llerrs << "Renaming VFS block to a locked file." << llendl;
 993				}
 994				dest_block->mLocks[i] = src_block->mLocks[i];
 995			}
 996			
 997			mFileBlocks.erase(new_spec);
 998			delete dest_block;
 999		}
1000
1001		src_block->mFileID = new_id;
1002		src_block->mFileType = new_type;
1003		src_block->mAccessTime = (U32)time(NULL);
1004   
1005		mFileBlocks.erase(old_spec);
1006		mFileBlocks.insert(fileblock_map::value_type(new_spec, src_block));
1007
1008		sync(src_block);
1009	}
1010	else
1011	{
1012		llwarns << "VFS: Attempt to rename nonexistent vfile " << file_id << ":" << file_type << llendl;
1013	}
1014	unlockData();
1015}
1016
1017// mDataMutex must be LOCKED before calling this
1018void LLVFS::removeFileBlock(LLVFSFileBlock *fileblock)
1019{
1020	// convert this into an unsaved, dummy fileblock to preserve locks
1021	// a more rubust solution would store the locks in a seperate data structure
1022	sync(fileblock, TRUE);
1023	
1024	if (fileblock->mLength > 0)
1025	{
1026		// turn this file into an empty block
1027		LLVFSBlock *free_block = new LLVFSBlock(fileblock->mLocation, fileblock->mLength);
1028		
1029		addFreeBlock(free_block);
1030	}
1031	
1032	fileblock->mLocation = 0;
1033	fileblock->mSize = 0;
1034	fileblock->mLength = BLOCK_LENGTH_INVALID;
1035	fileblock->mIndexLocation = -1;
1036
1037	//mergeFreeBlocks();
1038}
1039
1040void LLVFS::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type)
1041{
1042	if (!isValid())
1043	{
1044		llerrs << "Attempting to use invalid VFS!" << llendl;
1045	}
1046	if (mReadOnly)
1047	{
1048		llerrs << "Attempt to write to read-only VFS" << llendl;
1049	}
1050
1051    lockData();
1052	
1053	LLVFSFileSpecifier spec(file_id, file_type);
1054	fileblock_map::iterator it = mFileBlocks.find(spec);
1055	if (it != mFileBlocks.end())
1056	{
1057		LLVFSFileBlock *block = (*it).second;
1058		removeFileBlock(block);
1059	}
1060	else
1061	{
1062		llwarns << "VFS: attempting to remove nonexistent file " << file_id << " type " << file_type << llendl;
1063	}
1064
1065	unlockData();
1066}
1067    
1068    
1069S32 LLVFS::getData(const LLUUID &file_id, const LLAssetType::EType file_type, U8 *buffer, S32 location, S32 length)
1070{
1071	S32 bytesread = 0;
1072	
1073	if (!isValid())
1074	{
1075		llerrs << "Attempting to use invalid VFS!" << llendl;
1076	}
1077	llassert(location >= 0);
1078	llassert(length >= 0);
1079
1080	BOOL do_read = FALSE;
1081	
1082    lockData();
1083	
1084	LLVFSFileSpecifier spec(file_id, file_type);
1085	fileblock_map::iterator it = mFileBlocks.find(spec);
1086	if (it != mFileBlocks.end())
1087	{
1088		LLVFSFileBlock *block = (*it).second;
1089
1090		block->mAccessTime = (U32)time(NULL);
1091    
1092		if (location > block->mSize)
1093		{
1094			llwarns << "VFS: Attempt to read location " << location << " in file " << file_id << " of length " << block->mSize << llendl;
1095		}
1096		else
1097		{
1098			if (length > block->mSize - location)
1099			{
1100				length = block->mSize - location;
1101			}
1102			location += block->mLocation;
1103			do_read = TRUE;
1104		}
1105	}
1106
1107	if (do_read)
1108	{
1109		fseek(mDataFP, location, SEEK_SET);
1110		bytesread = (S32)fread(buffer, 1, length, mDataFP);
1111	}
1112	
1113	unlockData();
1114
1115	return bytesread;
1116}
1117    
1118S32 LLVFS::storeData(const LLUUID &file_id, const LLAssetType::EType file_type, const U8 *buffer, S32 location, S32 length)
1119{
1120	if (!isValid())
1121	{
1122		llerrs << "Attempting to use invalid VFS!" << llendl;
1123	}
1124	if (mReadOnly)
1125	{
1126		llerrs << "Attempt to write to read-only VFS" << llendl;
1127	}
1128    
1129	llassert(length > 0);
1130
1131    lockData();
1132    
1133	LLVFSFileSpecifier spec(file_id, file_type);
1134	fileblock_map::iterator it = mFileBlocks.find(spec);
1135	if (it != mFileBlocks.end())
1136	{
1137		LLVFSFileBlock *block = (*it).second;
1138
1139		S32 in_loc = location;
1140		if (location == -1)
1141		{
1142			location = block->mSize;
1143		}
1144		llassert(location >= 0);
1145		
1146		block->mAccessTime = (U32)time(NULL);
1147    
1148		if (block->mLength == BLOCK_LENGTH_INVALID)
1149		{
1150			// Block was removed, ignore write
1151			llwarns << "VFS: Attempt to write to invalid block"
1152					<< " in file " << file_id 
1153					<< " location: " << in_loc
1154					<< " bytes: " << length
1155					<< llendl;
1156			unlockData();
1157			return length;
1158		}
1159		else if (location > block->mLength)
1160		{
1161			llwarns << "VFS: Attempt to write to location " << location 
1162					<< " in file " << file_id 
1163					<< " type " << S32(file_type)
1164					<< " of size " << block->mSize
1165					<< " block length " << block->mLength
1166					<< llendl;
1167			unlockData();
1168			return length;
1169		}
1170		else
1171		{
1172			if (length > block->mLength - location )
1173			{
1174				llwarns << "VFS: Truncating write to virtual file " << file_id << " type " << S32(file_type) << llendl;
1175				length = block->mLength - location;
1176			}
1177			U32 file_location = location + block->mLocation;
1178			
1179			fseek(mDataFP, file_location, SEEK_SET);
1180			S32 write_len = (S32)fwrite(buffer, 1, length, mDataFP);
1181			if (write_len != length)
1182			{
1183				llwarns << llformat("VFS Write Error: %d != %d",write_len,length) << llendl;
1184			}
1185			// fflush(mDataFP);
1186			
1187			if (location + length > block->mSize)
1188			{
1189				block->mSize = location + write_len;
1190				sync(block);
1191			}
1192			unlockData();
1193			
1194			return write_len;
1195		}
1196	}
1197	else
1198	{
1199		unlockData();
1200		return 0;
1201	}
1202}
1203 
1204void LLVFS::incLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
1205{
1206	lockData();
1207
1208	LLVFSFileSpecifier spec(file_id, file_type);
1209	LLVFSFileBlock *block;
1210	
1211 	fileblock_map::iterator it = mFileBlocks.find(spec);
1212	if (it != mFileBlocks.end())
1213	{
1214		block = (*it).second;
1215	}
1216	else
1217	{
1218		// Create a dummy block which isn't saved
1219		block = new LLVFSFileBlock(file_id, file_type, 0, BLOCK_LENGTH_INVALID);
1220    	block->mAccessTime = (U32)time(NULL);
1221		mFileBlocks.insert(fileblock_map::value_type(spec, block));
1222	}
1223
1224	block->mLocks[lock]++;
1225	mLockCounts[lock]++;
1226	
1227	unlockData();
1228}
1229
1230void LLVFS::decLock(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
1231{
1232	lockData();
1233
1234	LLVFSFileSpecifier spec(file_id, file_type);
1235 	fileblock_map::iterator it = mFileBlocks.find(spec);
1236	if (it != mFileBlocks.end())
1237	{
1238		LLVFSFileBlock *block = (*it).second;
1239
1240		if (block->mLocks[lock] > 0)
1241		{
1242			block->mLocks[lock]--;
1243		}
1244		else
1245		{
1246			llwarns << "VFS: Decrementing zero-value lock " << lock << llendl;
1247		}
1248		mLockCounts[lock]--;
1249	}
1250
1251	unlockData();
1252}
1253
1254BOOL LLVFS::isLocked(const LLUUID &file_id, const LLAssetType::EType file_type, EVFSLock lock)
1255{
1256	lockData();
1257	
1258	BOOL res = FALSE;
1259	
1260	LLVFSFileSpecifier spec(file_id, file_type);
1261 	fileblock_map::iterator it = mFileBlocks.find(spec);
1262	if (it != mFileBlocks.end())
1263	{
1264		LLVFSFileBlock *block = (*it).second;
1265		res = (block->mLocks[lock] > 0);
1266	}
1267
1268	unlockData();
1269
1270	return res;
1271}
1272
1273//============================================================================
1274// protected
1275//============================================================================
1276
1277void LLVFS::eraseBlockLength(LLVFSBlock *block)
1278{
1279	// find the corresponding map entry in the length map and erase it
1280	S32 length = block->mLength;
1281	blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(length);
1282	blocks_length_map_t::iterator end = mFreeBlocksByLength.end();
1283	bool found_block = false;
1284	while(iter != end)
1285	{
1286		LLVFSBlock *tblock = iter->second;
1287		llassert(tblock->mLength == length); // there had -better- be an entry with our length!
1288		if (tblock == block)
1289		{
1290			mFreeBlocksByLength.erase(iter);
1291			found_block = true;
1292			break;
1293		}
1294		++iter;
1295	}
1296	if(!found_block)
1297	{
1298		llerrs << "eraseBlock could not find block" << llendl;
1299	}
1300}
1301
1302
1303// Remove block from both free lists (by location and by length).
1304void LLVFS::eraseBlock(LLVFSBlock *block)
1305{
1306	eraseBlockLength(block);
1307	// find the corresponding map entry in the location map and erase it	
1308	U32 location = block->mLocation;
1309	llverify(mFreeBlocksByLocation.erase(location) == 1); // we should only have one entry per location.
1310}
1311
1312
1313// Add the region specified by block location and length to the free lists.
1314// Also incrementally defragment by merging with previous and next free blocks.
1315void LLVFS::addFreeBlock(LLVFSBlock *block)
1316{
1317#if LL_DEBUG
1318	size_t dbgcount = mFreeBlocksByLocation.count(block->mLocation);
1319	if(dbgcount > 0)
1320	{
1321		llerrs << "addFreeBlock called with block already in list" << llendl;
1322	}
1323#endif
1324
1325	// Get a pointer to the next free block (by location).
1326	blocks_location_map_t::iterator next_free_it = mFreeBlocksByLocation.lower_bound(block->mLocation);
1327
1328	// We can merge with previous if it ends at our requested location.
1329	LLVFSBlock* prev_block = NULL;
1330	bool merge_prev = false;
1331	if (next_free_it != mFreeBlocksByLocation.begin())
1332	{
1333		blocks_location_map_t::iterator prev_free_it = next_free_it;
1334		--prev_free_it;
1335		prev_block = prev_free_it->second;
1336		merge_prev = (prev_block->mLocation + prev_block->mLength == block->mLocation);
1337	}
1338
1339	// We can merge with next if our block ends at the next block's location.
1340	LLVFSBlock* next_block = NULL;
1341	bool merge_next = false;
1342	if (next_free_it != mFreeBlocksByLocation.end())
1343	{
1344		next_block = next_free_it->second;
1345		merge_next = (block->mLocation + block->mLength == next_block->mLocation);
1346	}
1347
1348	if (merge_prev && merge_next)
1349	{
1350		// llinfos << "VFS merge BOTH" << llendl;
1351		// Previous block is changing length (a lot), so only need to update length map.
1352		// Next block is going away completely. JC
1353		eraseBlockLength(prev_block);
1354		eraseBlock(next_block);
1355		prev_block->mLength += block->mLength + next_block->mLength;
1356		mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block));
1357		delete block;
1358		block = NULL;
1359		delete next_block;
1360		next_block = NULL;
1361	}
1362	else if (merge_prev)
1363	{
1364		// llinfos << "VFS merge previous" << llendl;
1365		// Previous block is maintaining location, only changing length,
1366		// therefore only need to update the length map. JC
1367		eraseBlockLength(prev_block);
1368		prev_block->mLength += block->mLength;
1369		mFreeBlocksByLength.insert(blocks_length_map_t::value_type(prev_block->mLength, prev_block)); // multimap insert
1370		delete block;
1371		block = NULL;
1372	}
1373	else if (merge_next)
1374	{
1375		// llinfos << "VFS merge next" << llendl;
1376		// Next block is changing both location and length,
1377		// so both free lists must update. JC
1378		eraseBlock(next_block);
1379		next_block->mLocation = block->mLocation;
1380		next_block->mLength += block->mLength;
1381		// Don't hint here, next_free_it iterator may be invalid.
1382		mFreeBlocksByLocation.insert(blocks_location_map_t::value_type(next_block->mLocation, next_block)); // multimap insert
1383		mFreeBlocksByLength.insert(blocks_length_map_t::value_type(next_block->mLength, next_block)); // multimap insert			
1384		delete block;
1385		block = NULL;
1386	}
1387	else
1388	{
1389		// Can't merge with other free blocks.
1390		// Hint that insert should go near next_free_it.
1391 		mFreeBlocksByLocation.insert(next_free_it, blocks_location_map_t::value_type(block->mLocation, block)); // multimap insert
1392 		mFreeBlocksByLength.insert(blocks_length_map_t::value_type(block->mLength, block)); // multimap insert
1393	}
1394}
1395
1396// Superceeded by new addFreeBlock which does incremental free space merging.
1397// Incremental is faster overall.
1398//void LLVFS::mergeFreeBlocks()
1399//{
1400// 	if (!isValid())
1401// 	{
1402// 		llerrs << "Attempting to use invalid VFS!" << llendl;
1403// 	}
1404// 	// TODO: could we optimize this with hints from the calling code?
1405// 	blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin();	
1406// 	blocks_location_map_t::iterator end = mFreeBlocksByLocation.end();	
1407// 	LLVFSBlock *first_block = iter->second;
1408// 	while(iter != end)
1409// 	{
1410// 		blocks_location_map_t::iterator first_iter = iter; // save for if we do a merge
1411// 		if (++iter == end)
1412// 			break;
1413// 		LLVFSBlock *second_block = iter->second;
1414// 		if (first_block->mLocation + first_block->mLength == second_block->mLocation)
1415// 		{
1416// 			// remove the first block from the length map
1417// 			eraseBlockLength(first_block);
1418// 			// merge first_block with second_block, since they're adjacent
1419// 			first_block->mLength += second_block->mLength;
1420// 			// add the first block to the length map (with the new size)
1421// 			mFreeBlocksByLength.insert(blocks_length_map_t::value_type(first_block->mLength, first_block)); // multimap insert
1422//
1423// 			// erase and delete the second block
1424// 			eraseBlock(second_block);
1425// 			delete second_block;
1426//
1427// 			// reset iterator
1428// 			iter = first_iter; // haven't changed first_block, so corresponding iterator is still valid
1429// 			end = mFreeBlocksByLocation.end();
1430// 		}
1431// 		first_block = second_block;
1432// 	}
1433//}
1434	
1435// length bytes from free_block are going to be used (so they are no longer free)
1436void LLVFS::useFreeSpace(LLVFSBlock *free_block, S32 length)
1437{
1438	if (free_block->mLength == length)
1439	{
1440		eraseBlock(free_block);
1441		delete free_block;
1442	}
1443	else
1444	{
1445		eraseBlock(free_block);
1446  		
1447		free_block->mLocation += length;
1448		free_block->mLength -= length;
1449
1450		addFreeBlock(free_block);
1451	}
1452}
1453
1454// NOTE! mDataMutex must be LOCKED before calling this
1455// sync this index entry out to the index file
1456// we need to do this constantly to avoid corruption on viewer crash
1457void LLVFS::sync(LLVFSFileBlock *block, BOOL remove)
1458{
1459	if (!isValid())
1460	{
1461		llerrs << "Attempting to use invalid VFS!" << llendl;
1462	}
1463	if (mReadOnly)
1464	{
1465		llwarns << "Attempt to sync read-only VFS" << llendl;
1466		return;
1467	}
1468	if (block->mLength == BLOCK_LENGTH_INVALID)
1469	{
1470		// This is a dummy file, don't save
1471		return;
1472	}
1473	if (block->mLength == 0)
1474	{
1475		llerrs << "VFS syncing zero-length block" << llendl;
1476	}
1477
1478    BOOL set_index_to_end = FALSE;
1479	long seek_pos = block->mIndexLocation;
1480		
1481	if (-1 == seek_pos)
1482	{
1483		if (!mIndexHoles.empty())
1484		{
1485			seek_pos = mIndexHoles.front();
1486			mIndexHoles.pop_front();
1487		}
1488		else
1489		{
1490			set_index_to_end = TRUE;
1491		}
1492	}
1493
1494    if (set_index_to_end)
1495	{
1496		// Need fseek/ftell to update the seek_pos and hence data
1497		// structures, so can't unlockData() before this.
1498		fseek(mIndexFP, 0, SEEK_END);
1499		seek_pos = ftell(mIndexFP);
1500	}
1501	    
1502	block->mIndexLocation = seek_pos;
1503	if (remove)
1504	{
1505		mIndexHoles.push_back(seek_pos);
1506	}
1507
1508	U8 buffer[LLVFSFileBlock::SERIAL_SIZE];
1509	if (remove)
1510	{
1511		memset(buffer, 0, LLVFSFileBlock::SERIAL_SIZE);
1512	}
1513	else
1514	{
1515		block->serialize(buffer);
1516	}
1517
1518	// If set_index_to_end, file pointer is already at seek_pos
1519	// and we don't need to do anything.  Only seek if not at end.
1520	if (!set_index_to_end)
1521	{
1522		fseek(mIndexFP, seek_pos, SEEK_SET);
1523	}
1524
1525	if (fwrite(buffer, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1)
1526	{
1527		llwarns << "Short write" << llendl;
1528	}
1529
1530	// *NOTE:  Why was this commented out?
1531	// fflush(mIndexFP);
1532	
1533	return;
1534}
1535
1536// mDataMutex must be LOCKED before calling this
1537// Can initiate LRU-based file removal to make space.
1538// The immune file block will not be removed.
1539LLVFSBlock *LLVFS::findFreeBlock(S32 size, LLVFSFileBlock *immune)
1540{
1541	if (!isValid())
1542	{
1543		llerrs << "Attempting to use invalid VFS!" << llendl;
1544	}
1545
1546	LLVFSBlock *block = NULL;
1547	BOOL have_lru_list = FALSE;
1548	
1549	typedef std::set<LLVFSFileBlock*, LLVFSFileBlock_less> lru_set;
1550	lru_set lru_list;
1551    
1552	LLTimer timer;
1553
1554	while (! block)
1555	{
1556		// look for a suitable free block
1557		blocks_length_map_t::iterator iter = mFreeBlocksByLength.lower_bound(size); // first entry >= size
1558		if (iter != mFreeBlocksByLength.end())
1559			block = iter->second;
1560    	
1561		// no large enough free blocks, time to clean out some junk
1562		if (! block)
1563		{
1564			// create a list of files sorted by usage time
1565			// this is far faster than sorting a linked list
1566			if (! have_lru_list)
1567			{
1568				for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
1569				{
1570					LLVFSFileBlock *tmp = (*it).second;
1571
1572					if (tmp != immune &&
1573						tmp->mLength > 0 &&
1574						! tmp->mLocks[VFSLOCK_READ] &&
1575						! tmp->mLocks[VFSLOCK_APPEND] &&
1576						! tmp->mLocks[VFSLOCK_OPEN])
1577					{
1578						lru_list.insert(tmp);
1579					}
1580				}
1581				
1582				have_lru_list = TRUE;
1583			}
1584
1585			if (lru_list.size() == 0)
1586			{
1587				// No more files to delete, and still not enough room!
1588				llwarns << "VFS: Can't make " << size << " bytes of free space in VFS, giving up" << llendl;
1589				break;
1590			}
1591
1592			// is the oldest file big enough?  (Should be about half the time)
1593			lru_set::iterator it = lru_list.begin();
1594			LLVFSFileBlock *file_block = *it;
1595			if (file_block->mLength >= size && file_block != immune)
1596			{
1597				// ditch this file and look again for a free block - should find it
1598				// TODO: it'll be faster just to assign the free block and break
1599				llinfos << "LRU: Removing " << file_block->mFileID << ":" << file_block->mFileType << llendl;
1600				lru_list.erase(it);
1601				removeFileBlock(file_block);
1602				file_block = NULL;
1603				continue;
1604			}
1605
1606			
1607			llinfos << "VFS: LRU: Aggressive: " << (S32)lru_list.size() << " files remain" << llendl;
1608			dumpLockCounts();
1609			
1610			// Now it's time to aggressively make more space
1611			// Delete the oldest 5MB of the vfs or enough to hold the file, which ever is larger
1612			// This may yield too much free space, but we'll use it up soon enough
1613			U32 cleanup_target = (size > VFS_CLEANUP_SIZE) ? size : VFS_CLEANUP_SIZE;
1614			U32 cleaned_up = 0;
1615		   	for (it = lru_list.begin();
1616				 it != lru_list.end() && cleaned_up < cleanup_target;
1617				 )
1618			{
1619				file_block = *it;
1620				
1621				// TODO: it would be great to be able to batch all these sync() calls
1622				// llinfos << "LRU2: Removing " << file_block->mFileID << ":" << file_block->mFileType << " last accessed" << file_block->mAccessTime << llendl;
1623
1624				cleaned_up += file_block->mLength;
1625				lru_list.erase(it++);
1626				removeFileBlock(file_block);
1627				file_block = NULL;
1628			}
1629			//mergeFreeBlocks();
1630		}
1631	}
1632    
1633	F32 time = timer.getElapsedTimeF32();
1634	if (time > 0.5f)
1635	{
1636		llwarns << "VFS: Spent " << time << " seconds in findFreeBlock!" << llendl;
1637	}
1638
1639	return block;
1640}
1641
1642//============================================================================
1643// public
1644//============================================================================
1645
1646void LLVFS::pokeFiles()
1647{
1648	if (!isValid())
1649	{
1650		llerrs << "Attempting to use invalid VFS!" << llendl;
1651	}
1652	U32 word;
1653	
1654	// only write data if we actually read 4 bytes
1655	// otherwise we're writing garbage and screwing up the file
1656	fseek(mDataFP, 0, SEEK_SET);
1657	if (fread(&word, sizeof(word), 1, mDataFP) == 1)
1658	{
1659		fseek(mDataFP, 0, SEEK_SET);
1660		if (fwrite(&word, sizeof(word), 1, mDataFP) != 1)
1661		{
1662			llwarns << "Could not write to data file" << llendl;
1663		}
1664		fflush(mDataFP);
1665	}
1666
1667	fseek(mIndexFP, 0, SEEK_SET);
1668	if (fread(&word, sizeof(word), 1, mIndexFP) == 1)
1669	{
1670		fseek(mIndexFP, 0, SEEK_SET);
1671		if (fwrite(&word, sizeof(word), 1, mIndexFP) != 1)
1672		{
1673			llwarns << "Could not write to index file" << llendl;
1674		}
1675		fflush(mIndexFP);
1676	}
1677}
1678
1679    
1680void LLVFS::dumpMap()
1681{
1682	llinfos << "Files:" << llendl;
1683	for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
1684	{
1685		LLVFSFileBlock *file_block = (*it).second;
1686		llinfos << "Location: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << llendl;
1687	}
1688    
1689	llinfos << "Free Blocks:" << llendl;
1690	for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(),
1691			 end = mFreeBlocksByLocation.end();
1692		 iter != end; iter++)
1693	{
1694		LLVFSBlock *free_block = iter->second;
1695		llinfos << "Location: " << free_block->mLocation << "\tLength: " << free_block->mLength << llendl;
1696	}
1697}
1698    
1699// verify that the index file contents match the in-memory file structure
1700// Very slow, do not call routinely. JC
1701void LLVFS::audit()
1702{
1703	// Lock the mutex through this whole function.
1704	LLMutexLock lock_data(mDataMutex);
1705	
1706	fflush(mIndexFP);
1707
1708	fseek(mIndexFP, 0, SEEK_END);
1709	size_t index_size = ftell(mIndexFP);
1710	fseek(mIndexFP, 0, SEEK_SET);
1711    
1712	BOOL vfs_corrupt = FALSE;
1713	
1714	// since we take the address of element 0, we need to have at least one element.
1715	std::vector<U8> buffer(llmax<size_t>(index_size,1U));
1716
1717	if (fread(&buffer[0], 1, index_size, mIndexFP) != index_size)
1718	{
1719		llwarns << "Index truncated" << llendl;
1720		vfs_corrupt = TRUE;
1721	}
1722    
1723	size_t buf_offset = 0;
1724    
1725	std::map<LLVFSFileSpecifier, LLVFSFileBlock*>	found_files;
1726	U32 cur_time = (U32)time(NULL);
1727
1728	std::vector<LLVFSFileBlock*> audit_blocks;
1729	while (!vfs_corrupt && buf_offset < index_size)
1730	{
1731		LLVFSFileBlock *block = new LLVFSFileBlock();
1732		audit_blocks.push_back(block);
1733		
1734		block->deserialize(&buffer[buf_offset], (S32)buf_offset);
1735		buf_offset += block->SERIAL_SIZE;
1736    
1737		// do sanity check on this block
1738		if (block->mLength >= 0 &&
1739			block->mSize >= 0 &&
1740			block->mSize <= block->mLength &&
1741			block->mFileType >= LLAssetType::AT_NONE &&
1742			block->mFileType < LLAssetType::AT_COUNT &&
1743			block->mAccessTime <= cur_time &&
1744			block->mFileID != LLUUID::null)
1745		{
1746			if (mFileBlocks.find(*block) == mFileBlocks.end())
1747			{
1748				llwarns << "VFile " << block->mFileID << ":" << block->mFileType << " on disk, not in memory, loc " << block->mIndexLocation << llendl;
1749			}
1750			else if (found_files.find(*block) != found_files.end())
1751			{
1752				std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator it;
1753				it = found_files.find(*block);
1754				LLVFSFileBlock* dupe = it->second;
1755				// try to keep data from being lost
1756				unlockAndClose(mIndexFP);
1757				mIndexFP = NULL;
1758				unlockAndClose(mDataFP);
1759				mDataFP = NULL;
1760				llwarns << "VFS: Original block index " << block->mIndexLocation
1761					<< " location " << block->mLocation 
1762					<< " length " << block->mLength 
1763					<< " size " << block->mSize 
1764					<< " id " << block->mFileID
1765					<< " type " << block->mFileType
1766					<< llendl;
1767				llwarns << "VFS: Duplicate block index " << dupe->mIndexLocation
1768					<< " location " << dupe->mLocation 
1769					<< " length " << dupe->mLength 
1770					<< " size " << dupe->mSize 
1771					<< " id " << dupe->mFileID
1772					<< " type " << dupe->mFileType
1773					<< llendl;
1774				llwarns << "VFS: Index size " << index_size << llendl;
1775				llwarns << "VFS: INDEX CORRUPT" << llendl;
1776				vfs_corrupt = TRUE;
1777				break;
1778			}
1779			else
1780			{
1781				found_files[*block] = block;
1782			}
1783		}
1784		else
1785		{
1786			if (block->mLength)
1787			{
1788				llwarns << "VFile " << block->mFileID << ":" << block->mFileType << " corrupt on disk" << llendl;
1789			}
1790			// else this is just a hole
1791		}
1792	}
1793    
1794	if (!vfs_corrupt)
1795	{
1796		for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
1797		{
1798			LLVFSFileBlock* block = (*it).second;
1799
1800			if (block->mSize > 0)
1801			{
1802				if (! found_files.count(*block))
1803				{
1804					llwarns << "VFile " << block->mFileID << ":" << block->mFileType << " in memory, not on disk, loc " << block->mIndexLocation<< llendl;
1805					fseek(mIndexFP, block->mIndexLocation, SEEK_SET);
1806					U8 buf[LLVFSFileBlock::SERIAL_SIZE];
1807					if (fread(buf, LLVFSFileBlock::SERIAL_SIZE, 1, mIndexFP) != 1)
1808					{
1809						llwarns << "VFile " << block->mFileID
1810								<< " gave short read" << llendl;
1811					}
1812    			
1813					LLVFSFileBlock disk_block;
1814					disk_block.deserialize(buf, block->mIndexLocation);
1815				
1816					llwarns << "Instead found " << disk_block.mFileID << ":" << block->mFileType << llendl;
1817				}
1818				else
1819				{
1820					block = found_files.find(*block)->second;
1821					found_files.erase(*block);
1822				}
1823			}
1824		}
1825    
1826		for (std::map<LLVFSFileSpecifier, LLVFSFileBlock*>::iterator iter = found_files.begin();
1827			 iter != found_files.end(); iter++)
1828		{
1829			LLVFSFileBlock* block = iter->second;
1830			llwarns << "VFile " << block->mFileID << ":" << block->mFileType << " szie:" << block->mSize << " leftover" << llendl;
1831		}
1832    
1833		llinfos << "VFS: audit OK" << llendl;
1834		// mutex released by LLMutexLock() destructor.
1835	}
1836
1837	for_each(audit_blocks.begin(), audit_blocks.end(), DeletePointer());
1838}
1839    
1840    
1841// quick check for uninitialized blocks
1842// Slow, do not call in release.
1843void LLVFS::checkMem()
1844{
1845	lockData();
1846	
1847	for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
1848	{
1849		LLVFSFileBlock *block = (*it).second;
1850		llassert(block->mFileType >= LLAssetType::AT_NONE &&
1851				 block->mFileType < LLAssetType::AT_COUNT &&
1852				 block->mFileID != LLUUID::null);
1853    
1854		for (std::deque<S32>::iterator iter = mIndexHoles.begin();
1855			 iter != mIndexHoles.end(); ++iter)
1856		{
1857			S32 index_loc = *iter;
1858			if (index_loc == block->mIndexLocation)
1859			{
1860				llwarns << "VFile block " << block->mFileID << ":" << block->mFileType << " is marked as a hole" << llendl;
1861			}
1862		}
1863	}
1864    
1865	llinfos << "VFS: mem check OK" << llendl;
1866
1867	unlockData();
1868}
1869
1870void LLVFS::dumpLockCounts()
1871{
1872	S32 i;
1873	for (i = 0; i < VFSLOCK_COUNT; i++)
1874	{
1875		llinfos << "LockType: " << i << ": " << mLockCounts[i] << llendl;
1876	}
1877}
1878
1879void LLVFS::dumpStatistics()
1880{
1881	lockData();
1882	
1883	// Investigate file blocks.
1884	std::map<S32, S32> size_counts;
1885	std::map<U32, S32> location_counts;
1886	std::map<LLAssetType::EType, std::pair<S32,S32> > filetype_counts;
1887
1888	S32 max_file_size = 0;
1889	S32 total_file_size = 0;
1890	S32 invalid_file_count = 0;
1891	for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
1892	{
1893		LLVFSFileBlock *file_block = (*it).second;
1894		if (file_block->mLength == BLOCK_LENGTH_INVALID)
1895		{
1896			invalid_file_count++;
1897		}
1898		else if (file_block->mLength <= 0)
1899		{
1900			llinfos << "Bad file block at: " << file_block->mLocation << "\tLength: " << file_block->mLength << "\t" << file_block->mFileID << "\t" << file_block->mFileType << llendl;
1901			size_counts[file_block->mLength]++;
1902			location_counts[file_block->mLocation]++;
1903		}
1904		else
1905		{
1906			total_file_size += file_block->mLength;
1907		}
1908
1909		if (file_block->mLength > max_file_size)
1910		{
1911			max_file_size = file_block->mLength;
1912		}
1913
1914		filetype_counts[file_block->mFileType].first++;
1915		filetype_counts[file_block->mFileType].second += file_block->mLength;
1916	}
1917    
1918	for (std::map<S32,S32>::iterator it = size_counts.begin(); it != size_counts.end(); ++it)
1919	{
1920		S32 size = it->first;
1921		S32 size_count = it->second;
1922		llinfos << "Bad files size " << size << " count " << size_count << llendl;
1923	}
1924	for (std::map<U32,S32>::iterator it = location_counts.begin(); it != location_counts.end(); ++it)
1925	{
1926		U32 location = it->first;
1927		S32 location_count = it->second;
1928		llinfos << "Bad files location " << location << " count " << location_count << llendl;
1929	}
1930
1931	// Investigate free list.
1932	S32 max_free_size = 0;
1933	S32 total_free_size = 0;
1934	std::map<S32, S32> free_length_counts;
1935	for (blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin(),
1936			 end = mFreeBlocksByLocation.end();
1937		 iter != end; iter++)
1938	{
1939		LLVFSBlock *free_block = iter->second;
1940		if (free_block->mLength <= 0)
1941		{
1942			llinfos << "Bad free block at: " << free_block->mLocation << "\tLength: " << free_block->mLength << llendl;
1943		}
1944		else
1945		{
1946			llinfos << "Block: " << free_block->mLocation
1947					<< "\tLength: " << free_block->mLength
1948					<< "\tEnd: " << free_block->mLocation + free_block->mLength
1949					<< llendl;
1950			total_free_size += free_block->mLength;
1951		}
1952
1953		if (free_block->mLength > max_free_size)
1954		{
1955			max_free_size = free_block->mLength;
1956		}
1957
1958		free_length_counts[free_block->mLength]++;
1959	}
1960
1961	// Dump histogram of free block sizes
1962	for (std::map<S32,S32>::iterator it = free_length_counts.begin(); it != free_length_counts.end(); ++it)
1963	{
1964		llinfos << "Free length " << it->first << " count " << it->second << llendl;
1965	}
1966
1967	llinfos << "Invalid blocks: " << invalid_file_count << llendl;
1968	llinfos << "File blocks:    " << mFileBlocks.size() << llendl;
1969
1970	S32 length_list_count = (S32)mFreeBlocksByLength.size();
1971	S32 location_list_count = (S32)mFreeBlocksByLocation.size();
1972	if (length_list_count == location_list_count)
1973	{
1974		llinfos << "Free list lengths match, free blocks: " << location_list_count << llendl;
1975	}
1976	else
1977	{
1978		llwarns << "Free list lengths do not match!" << llendl;
1979		llwarns << "By length: " << length_list_count << llendl;
1980		llwarns << "By location: " << location_list_count << llendl;
1981	}
1982	llinfos << "Max file: " << max_file_size/1024 << "K" << llendl;
1983	llinfos << "Max free: " << max_free_size/1024 << "K" << llendl;
1984	llinfos << "Total file size: " << total_file_size/1024 << "K" << llendl;
1985	llinfos << "Total free size: " << total_free_size/1024 << "K" << llendl;
1986	llinfos << "Sum: " << (total_file_size + total_free_size) << " bytes" << llendl;
1987	llinfos << llformat("%.0f%% full",((F32)(total_file_size)/(F32)(total_file_size+total_free_size))*100.f) << llendl;
1988
1989	llinfos << " " << llendl;
1990	for (std::map<LLAssetType::EType, std::pair<S32,S32> >::iterator iter = filetype_counts.begin();
1991		 iter != filetype_counts.end(); ++iter)
1992	{
1993		llinfos << "Type: " << LLAssetType::getDesc(iter->first)
1994				<< " Count: " << iter->second.first
1995				<< " Bytes: " << (iter->second.second>>20) << " MB" << llendl;
1996	}
1997	
1998	// Look for potential merges 
1999	{
2000 		blocks_location_map_t::iterator iter = mFreeBlocksByLocation.begin();	
2001 		blocks_location_map_t::iterator end = mFreeBlocksByLocation.end();	
2002 		LLVFSBlock *first_block = iter->second;
2003 		while(iter != end)
2004 		{
2005 			if (++iter == end)
2006 				break;
2007 			LLVFSBlock *second_block = iter->second;
2008 			if (first_block->mLocation + first_block->mLength == second_block->mLocation)
2009 			{
2010				llinfos << "Potential merge at " << first_block->mLocation << llendl;
2011 			}
2012 			first_block = second_block;
2013 		}
2014	}
2015	unlockData();
2016}
2017
2018// Debug Only!
2019std::string get_extension(LLAssetType::EType type)
2020{
2021	std::string extension;
2022	switch(type)
2023	{
2024	case LLAssetType::AT_TEXTURE:
2025		extension = ".jp2";	// formerly ".j2c"
2026		break;
2027	case LLAssetType::AT_SOUND:
2028		extension = ".ogg";
2029		break;
2030	case LLAssetType::AT_SOUND_WAV:
2031		extension = ".wav";
2032		break;
2033	case LLAssetType::AT_TEXTURE_TGA:
2034		extension = ".tga";
2035		break;
2036	case LLAssetType::AT_ANIMATION:
2037		extension = ".lla";
2038		break;
2039	case LLAssetType::AT_MESH:
2040		extension = ".slm";
2041		break;
2042	default:
2043		// Just use the asset server filename extension in most cases
2044		extension += ".";
2045		extension += LLAssetType::lookup(type);
2046		break;
2047	}
2048	return extension;
2049}
2050
2051void LLVFS::listFiles()
2052{
2053	lockData();
2054	
2055	for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
2056	{
2057		LLVFSFileSpecifier file_spec = it->first;
2058		LLVFSFileBlock *file_block = it->second;
2059		S32 length = file_block->mLength;
2060		S32 size = file_block->mSize;
2061		if (length != BLOCK_LENGTH_INVALID && size > 0)
2062		{
2063			LLUUID id = file_spec.mFileID;
2064			std::string extension = get_extension(file_spec.mFileType);
2065			llinfos << " File: " << id
2066					<< " Type: " << LLAssetType::getDesc(file_spec.mFileType)
2067					<< " Size: " << size
2068					<< llendl;
2069		}
2070	}
2071	
2072	unlockData();
2073}
2074
2075#include "llapr.h"
2076void LLVFS::dumpFiles()
2077{
2078	lockData();
2079	
2080	S32 files_extracted = 0;
2081	for (fileblock_map::iterator it = mFileBlocks.begin(); it != mFileBlocks.end(); ++it)
2082	{
2083		LLVFSFileSpecifier file_spec = it->first;
2084		LLVFSFileBlock *file_block = it->second;
2085		S32 length = file_block->mLength;
2086		S32 size = file_block->mSize;
2087		if (length != BLOCK_LENGTH_INVALID && size > 0)
2088		{
2089			LLUUID id = file_spec.mFileID;
2090			LLAssetType::EType type = file_spec.mFileType;
2091			std::vector<U8> buffer(size);
2092
2093			unlockData();
2094			getData(id, type, &buffer[0], 0, size);
2095			lockData();
2096			
2097			std::string extension = get_extension(type);
2098			std::string filename = id.asString() + extension;
2099			llinfos << " Writing " << filename << llendl;
2100			
2101			LLAPRFile outfile;
2102			outfile.open(filename, LL_APR_WB);
2103			outfile.write(&buffer[0], size);
2104			outfile.close();
2105
2106			files_extracted++;
2107		}
2108	}
2109	
2110	unlockData();
2111
2112	llinfos << "Extracted " << files_extracted << " files out of " << mFileBlocks.size() << llendl;
2113}
2114
2115//============================================================================
2116// protected
2117//============================================================================
2118
2119// static
2120LLFILE *LLVFS::openAndLock(const std::string& filename, const char* mode, BOOL read_lock)
2121{
2122#if LL_WINDOWS
2123    	
2124	return LLFile::_fsopen(filename, mode, (read_lock ? _SH_DENYWR : _SH_DENYRW));
2125    	
2126#else
2127
2128	LLFILE *fp;
2129	int fd;
2130	
2131	// first test the lock in a non-destructive way
2132#if LL_SOLARIS
2133        struct flock fl;
2134        fl.l_whence = SEEK_SET;
2135        fl.l_start = 0;
2136        fl.l_len = 1;
2137#else // !LL_SOLARIS
2138	if (strchr(mode, 'w') != NULL)
2139	{
2140		fp = LLFile::fopen(filename, "rb");	/* Flawfinder: ignore */
2141		if (fp)
2142		{
2143			fd = fileno(fp);
2144			if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1)
2145			{
2146				fclose(fp);
2147				return NULL;
2148			}
2149		  
2150			fclose(fp);
2151		}
2152	}
2153#endif // !LL_SOLARIS
2154
2155	// now actually open the file for use
2156	fp = LLFile::fopen(filename, mode);	/* Flawfinder: ignore */
2157	if (fp)
2158	{
2159		fd = fileno(fp);
2160#if LL_SOLARIS
2161                fl.l_type = read_lock ? F_RDLCK : F_WRLCK;
2162                if (fcntl(fd, F_SETLK, &fl) == -1)
2163#else
2164		if (flock(fd, (read_lock ? LOCK_SH : LOCK_EX) | LOCK_NB) == -1)
2165#endif
2166		{
2167			fclose(fp);
2168			fp = NULL;
2169		}
2170	}
2171
2172	return fp;
2173    	
2174#endif
2175}
2176    
2177// static
2178void LLVFS::unlockAndClose(LLFILE *fp)
2179{
2180	if (fp)
2181	{
2182	// IW: we don't actually want to unlock on linux
2183	// this is because a forked process can kill the parent's lock
2184	// with an explicit unlock
2185	// however, fclose() will implicitly remove the lock
2186	// but only once both parent and child have closed the file
2187    /*	
2188	  #if !LL_WINDOWS
2189	  int fd = fileno(fp);
2190	  flock(fd, LOCK_UN);
2191	  #endif
2192    */
2193#if LL_SOLARIS
2194	        struct flock fl;
2195		fl.l_whence = SEEK_SET;
2196		fl.l_start = 0;
2197		fl.l_len = 1;
2198		fl.l_type = F_UNLCK;
2199		fcntl(fileno(fp), F_SETLK, &fl);
2200#endif
2201		fclose(fp);
2202	}
2203}