/indra/newview/llinventorymodelbackgroundfetch.cpp
C++ | 634 lines | 480 code | 83 blank | 71 comment | 90 complexity | 3646f5b463d824fb696defd97218bef4 MD5 | raw file
1/** 2 * @file llinventorymodel.cpp 3 * @brief Implementation of the inventory model used to track agent inventory. 4 * 5 * $LicenseInfo:firstyear=2002&license=viewerlgpl$ 6 * Second Life Viewer Source Code 7 * Copyright (C) 2010, Linden Research, Inc. 8 * 9 * This library is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; 12 * version 2.1 of the License only. 13 * 14 * This library is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this library; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 24 * $/LicenseInfo$ 25 */ 26 27#include "llviewerprecompiledheaders.h" 28#include "llinventorymodelbackgroundfetch.h" 29 30#include "llagent.h" 31#include "llappviewer.h" 32#include "llcallbacklist.h" 33#include "llinventorypanel.h" 34#include "llinventorymodel.h" 35#include "llviewercontrol.h" 36#include "llviewerinventory.h" 37#include "llviewermessage.h" 38#include "llviewerregion.h" 39#include "llviewerwindow.h" 40 41const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; 42const S32 MAX_FETCH_RETRIES = 10; 43 44LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : 45 mBackgroundFetchActive(FALSE), 46 mAllFoldersFetched(FALSE), 47 mRecursiveInventoryFetchStarted(FALSE), 48 mRecursiveLibraryFetchStarted(FALSE), 49 mNumFetchRetries(0), 50 mMinTimeBetweenFetches(0.3f), 51 mMaxTimeBetweenFetches(10.f), 52 mTimelyFetchPending(FALSE), 53 mBulkFetchCount(0) 54{ 55} 56 57LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() 58{ 59} 60 61bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const 62{ 63 return mFetchQueue.empty() && mBulkFetchCount<=0; 64} 65 66bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const 67{ 68 return mRecursiveLibraryFetchStarted; 69} 70 71bool LLInventoryModelBackgroundFetch::libraryFetchCompleted() const 72{ 73 return libraryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getLibraryRootFolderID()); 74} 75 76bool LLInventoryModelBackgroundFetch::libraryFetchInProgress() const 77{ 78 return libraryFetchStarted() && !libraryFetchCompleted(); 79} 80 81bool LLInventoryModelBackgroundFetch::inventoryFetchStarted() const 82{ 83 return mRecursiveInventoryFetchStarted; 84} 85 86bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const 87{ 88 return inventoryFetchStarted() && fetchQueueContainsNoDescendentsOf(gInventory.getRootFolderID()); 89} 90 91bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const 92{ 93 return inventoryFetchStarted() && !inventoryFetchCompleted(); 94} 95 96bool LLInventoryModelBackgroundFetch::isEverythingFetched() const 97{ 98 return mAllFoldersFetched; 99} 100 101BOOL LLInventoryModelBackgroundFetch::backgroundFetchActive() const 102{ 103 return mBackgroundFetchActive; 104} 105 106void LLInventoryModelBackgroundFetch::start(const LLUUID& cat_id, BOOL recursive) 107{ 108 if (!mAllFoldersFetched || cat_id.notNull()) 109 { 110 LL_DEBUGS("InventoryFetch") << "Start fetching category: " << cat_id << ", recursive: " << recursive << LL_ENDL; 111 112 mBackgroundFetchActive = TRUE; 113 if (cat_id.isNull()) 114 { 115 if (!mRecursiveInventoryFetchStarted) 116 { 117 mRecursiveInventoryFetchStarted |= recursive; 118 mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); 119 gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); 120 } 121 if (!mRecursiveLibraryFetchStarted) 122 { 123 mRecursiveLibraryFetchStarted |= recursive; 124 mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); 125 gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); 126 } 127 } 128 else 129 { 130 // Specific folder requests go to front of queue. 131 if (mFetchQueue.empty() || mFetchQueue.front().mCatUUID != cat_id) 132 { 133 mFetchQueue.push_front(FetchQueueInfo(cat_id, recursive)); 134 gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); 135 } 136 if (cat_id == gInventory.getLibraryRootFolderID()) 137 { 138 mRecursiveLibraryFetchStarted |= recursive; 139 } 140 if (cat_id == gInventory.getRootFolderID()) 141 { 142 mRecursiveInventoryFetchStarted |= recursive; 143 } 144 } 145 } 146} 147 148void LLInventoryModelBackgroundFetch::findLostItems() 149{ 150 mBackgroundFetchActive = TRUE; 151 mFetchQueue.push_back(FetchQueueInfo(LLUUID::null, TRUE)); 152 gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); 153} 154 155void LLInventoryModelBackgroundFetch::stopBackgroundFetch() 156{ 157 if (mBackgroundFetchActive) 158 { 159 mBackgroundFetchActive = FALSE; 160 gIdleCallbacks.deleteFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); 161 mBulkFetchCount=0; 162 mMinTimeBetweenFetches=0.0f; 163 } 164} 165 166void LLInventoryModelBackgroundFetch::setAllFoldersFetched() 167{ 168 if (mRecursiveInventoryFetchStarted && 169 mRecursiveLibraryFetchStarted) 170 { 171 mAllFoldersFetched = TRUE; 172 } 173 stopBackgroundFetch(); 174} 175 176void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) 177{ 178 LLInventoryModelBackgroundFetch::instance().backgroundFetch(); 179} 180 181void LLInventoryModelBackgroundFetch::backgroundFetch() 182{ 183 if (mBackgroundFetchActive && gAgent.getRegion()) 184 { 185 // If we'll be using the capability, we'll be sending batches and the background thing isn't as important. 186 std::string url = gAgent.getRegion()->getCapability("FetchInventoryDescendents2"); 187 if (gSavedSettings.getBOOL("UseHTTPInventory") && !url.empty()) 188 { 189 bulkFetch(url); 190 return; 191 } 192 193#if 1 194 //-------------------------------------------------------------------------------- 195 // DEPRECATED OLD CODE 196 // 197 198 // No more categories to fetch, stop fetch process. 199 if (mFetchQueue.empty()) 200 { 201 llinfos << "Inventory fetch completed" << llendl; 202 203 setAllFoldersFetched(); 204 return; 205 } 206 207 F32 fast_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.1f); 208 F32 slow_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.5f); 209 if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() > slow_fetch_time) 210 { 211 // Double timeouts on failure. 212 mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); 213 mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); 214 lldebugs << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl; 215 // fetch is no longer considered "timely" although we will wait for full time-out. 216 mTimelyFetchPending = FALSE; 217 } 218 219 while(1) 220 { 221 if (mFetchQueue.empty()) 222 { 223 break; 224 } 225 226 if(gDisconnected) 227 { 228 // Just bail if we are disconnected. 229 break; 230 } 231 232 const FetchQueueInfo info = mFetchQueue.front(); 233 LLViewerInventoryCategory* cat = gInventory.getCategory(info.mCatUUID); 234 235 // Category has been deleted, remove from queue. 236 if (!cat) 237 { 238 mFetchQueue.pop_front(); 239 continue; 240 } 241 242 if (mFetchTimer.getElapsedTimeF32() > mMinTimeBetweenFetches && 243 LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()) 244 { 245 // Category exists but has no children yet, fetch the descendants 246 // for now, just request every time and rely on retry timer to throttle. 247 if (cat->fetch()) 248 { 249 mFetchTimer.reset(); 250 mTimelyFetchPending = TRUE; 251 } 252 else 253 { 254 // The catagory also tracks if it has expired and here it says it hasn't 255 // yet. Get out of here because nothing is going to happen until we 256 // update the timers. 257 break; 258 } 259 } 260 // Do I have all my children? 261 else if (gInventory.isCategoryComplete(info.mCatUUID)) 262 { 263 // Finished with this category, remove from queue. 264 mFetchQueue.pop_front(); 265 266 // Add all children to queue. 267 LLInventoryModel::cat_array_t* categories; 268 LLInventoryModel::item_array_t* items; 269 gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); 270 for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); 271 it != categories->end(); 272 ++it) 273 { 274 mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(),info.mRecursive)); 275 } 276 277 // We received a response in less than the fast time. 278 if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() < fast_fetch_time) 279 { 280 // Shrink timeouts based on success. 281 mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); 282 mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); 283 lldebugs << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl; 284 } 285 286 mTimelyFetchPending = FALSE; 287 continue; 288 } 289 else if (mFetchTimer.getElapsedTimeF32() > mMaxTimeBetweenFetches) 290 { 291 // Received first packet, but our num descendants does not match db's num descendants 292 // so try again later. 293 mFetchQueue.pop_front(); 294 295 if (mNumFetchRetries++ < MAX_FETCH_RETRIES) 296 { 297 // push on back of queue 298 mFetchQueue.push_back(info); 299 } 300 mTimelyFetchPending = FALSE; 301 mFetchTimer.reset(); 302 break; 303 } 304 305 // Not enough time has elapsed to do a new fetch 306 break; 307 } 308 309 // 310 // DEPRECATED OLD CODE 311 //-------------------------------------------------------------------------------- 312#endif 313 } 314} 315 316void LLInventoryModelBackgroundFetch::incrBulkFetch(S16 fetching) 317{ 318 mBulkFetchCount += fetching; 319 if (mBulkFetchCount < 0) 320 { 321 mBulkFetchCount = 0; 322 } 323} 324 325 326class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder 327{ 328public: 329 LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : 330 mRequestSD(request_sd), 331 mRecursiveCatUUIDs(recursive_cats) 332 {}; 333 //LLInventoryModelFetchDescendentsResponder() {}; 334 void result(const LLSD& content); 335 void error(U32 status, const std::string& reason); 336protected: 337 BOOL getIsRecursive(const LLUUID& cat_id) const; 338private: 339 LLSD mRequestSD; 340 uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive 341}; 342 343// If we get back a normal response, handle it here. 344void LLInventoryModelFetchDescendentsResponder::result(const LLSD& content) 345{ 346 LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); 347 if (content.has("folders")) 348 { 349 350 for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); 351 folder_it != content["folders"].endArray(); 352 ++folder_it) 353 { 354 LLSD folder_sd = *folder_it; 355 356 357 //LLUUID agent_id = folder_sd["agent_id"]; 358 359 //if(agent_id != gAgent.getID()) //This should never happen. 360 //{ 361 // llwarns << "Got a UpdateInventoryItem for the wrong agent." 362 // << llendl; 363 // break; 364 //} 365 366 LLUUID parent_id = folder_sd["folder_id"]; 367 LLUUID owner_id = folder_sd["owner_id"]; 368 S32 version = (S32)folder_sd["version"].asInteger(); 369 S32 descendents = (S32)folder_sd["descendents"].asInteger(); 370 LLPointer<LLViewerInventoryCategory> tcategory = new LLViewerInventoryCategory(owner_id); 371 372 if (parent_id.isNull()) 373 { 374 LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; 375 for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); 376 item_it != folder_sd["items"].endArray(); 377 ++item_it) 378 { 379 const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); 380 if (lost_uuid.notNull()) 381 { 382 LLSD item = *item_it; 383 titem->unpackMessage(item); 384 385 LLInventoryModel::update_list_t update; 386 LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); 387 update.push_back(new_folder); 388 gInventory.accountForUpdate(update); 389 390 titem->setParent(lost_uuid); 391 titem->updateParentOnServer(FALSE); 392 gInventory.updateItem(titem); 393 gInventory.notifyObservers(); 394 395 } 396 } 397 } 398 399 LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); 400 if (!pcat) 401 { 402 continue; 403 } 404 405 for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); 406 category_it != folder_sd["categories"].endArray(); 407 ++category_it) 408 { 409 LLSD category = *category_it; 410 tcategory->fromLLSD(category); 411 412 const BOOL recursive = getIsRecursive(tcategory->getUUID()); 413 414 if (recursive) 415 { 416 fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); 417 } 418 else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) 419 { 420 gInventory.updateCategory(tcategory); 421 } 422 423 } 424 LLPointer<LLViewerInventoryItem> titem = new LLViewerInventoryItem; 425 for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); 426 item_it != folder_sd["items"].endArray(); 427 ++item_it) 428 { 429 LLSD item = *item_it; 430 titem->unpackMessage(item); 431 432 gInventory.updateItem(titem); 433 } 434 435 // Set version and descendentcount according to message. 436 LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); 437 if(cat) 438 { 439 cat->setVersion(version); 440 cat->setDescendentCount(descendents); 441 cat->determineFolderType(); 442 } 443 444 } 445 } 446 447 if (content.has("bad_folders")) 448 { 449 for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); 450 folder_it != content["bad_folders"].endArray(); 451 ++folder_it) 452 { 453 LLSD folder_sd = *folder_it; 454 455 // These folders failed on the dataserver. We probably don't want to retry them. 456 llinfos << "Folder " << folder_sd["folder_id"].asString() 457 << "Error: " << folder_sd["error"].asString() << llendl; 458 } 459 } 460 461 fetcher->incrBulkFetch(-1); 462 463 if (fetcher->isBulkFetchProcessingComplete()) 464 { 465 llinfos << "Inventory fetch completed" << llendl; 466 fetcher->setAllFoldersFetched(); 467 } 468 469 gInventory.notifyObservers(); 470} 471 472// If we get back an error (not found, etc...), handle it here. 473void LLInventoryModelFetchDescendentsResponder::error(U32 status, const std::string& reason) 474{ 475 LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); 476 477 llinfos << "LLInventoryModelFetchDescendentsResponder::error " 478 << status << ": " << reason << llendl; 479 480 fetcher->incrBulkFetch(-1); 481 482 if (status==499) // timed out 483 { 484 for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); 485 folder_it != mRequestSD["folders"].endArray(); 486 ++folder_it) 487 { 488 LLSD folder_sd = *folder_it; 489 LLUUID folder_id = folder_sd["folder_id"]; 490 const BOOL recursive = getIsRecursive(folder_id); 491 fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); 492 } 493 } 494 else 495 { 496 if (fetcher->isBulkFetchProcessingComplete()) 497 { 498 fetcher->setAllFoldersFetched(); 499 } 500 } 501 gInventory.notifyObservers(); 502} 503 504BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const 505{ 506 return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); 507} 508 509// Bundle up a bunch of requests to send all at once. 510// static 511void LLInventoryModelBackgroundFetch::bulkFetch(std::string url) 512{ 513 //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. 514 //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was 515 //sent. If it exceeds our retry time, go ahead and fire off another batch. 516 //Stopbackgroundfetch will be run from the Responder instead of here. 517 518 S16 max_concurrent_fetches=8; 519 F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. 520 if (mMinTimeBetweenFetches < new_min_time) 521 { 522 mMinTimeBetweenFetches=new_min_time; //HACK! See above. 523 } 524 525 if (gDisconnected || 526 (mBulkFetchCount > max_concurrent_fetches) || 527 (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) 528 { 529 return; // just bail if we are disconnected 530 } 531 532 U32 folder_count=0; 533 U32 max_batch_size=5; 534 535 U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; 536 537 uuid_vec_t recursive_cats; 538 539 LLSD body; 540 LLSD body_lib; 541 542 while (!(mFetchQueue.empty()) && (folder_count < max_batch_size)) 543 { 544 const FetchQueueInfo& fetch_info = mFetchQueue.front(); 545 const LLUUID &cat_id = fetch_info.mCatUUID; 546 if (cat_id.isNull()) //DEV-17797 547 { 548 LLSD folder_sd; 549 folder_sd["folder_id"] = LLUUID::null.asString(); 550 folder_sd["owner_id"] = gAgent.getID(); 551 folder_sd["sort_order"] = (LLSD::Integer)sort_order; 552 folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; 553 folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; 554 body["folders"].append(folder_sd); 555 folder_count++; 556 } 557 else 558 { 559 const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); 560 561 if (cat) 562 { 563 if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion()) 564 { 565 LLSD folder_sd; 566 folder_sd["folder_id"] = cat->getUUID(); 567 folder_sd["owner_id"] = cat->getOwnerID(); 568 folder_sd["sort_order"] = (LLSD::Integer)sort_order; 569 folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; 570 folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; 571 572 if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) 573 body_lib["folders"].append(folder_sd); 574 else 575 body["folders"].append(folder_sd); 576 folder_count++; 577 } 578 // May already have this folder, but append child folders to list. 579 if (fetch_info.mRecursive) 580 { 581 LLInventoryModel::cat_array_t* categories; 582 LLInventoryModel::item_array_t* items; 583 gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); 584 for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); 585 it != categories->end(); 586 ++it) 587 { 588 mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive)); 589 } 590 } 591 } 592 } 593 if (fetch_info.mRecursive) 594 recursive_cats.push_back(cat_id); 595 596 mFetchQueue.pop_front(); 597 } 598 599 if (folder_count > 0) 600 { 601 mBulkFetchCount++; 602 if (body["folders"].size()) 603 { 604 LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(body, recursive_cats); 605 LLHTTPClient::post(url, body, fetcher, 300.0); 606 } 607 if (body_lib["folders"].size()) 608 { 609 std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); 610 611 LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(body_lib, recursive_cats); 612 LLHTTPClient::post(url_lib, body_lib, fetcher, 300.0); 613 } 614 mFetchTimer.reset(); 615 } 616 else if (isBulkFetchProcessingComplete()) 617 { 618 setAllFoldersFetched(); 619 } 620} 621 622bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const 623{ 624 for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); 625 it != mFetchQueue.end(); ++it) 626 { 627 const LLUUID& fetch_id = (*it).mCatUUID; 628 if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) 629 return false; 630 } 631 return true; 632} 633 634