/src/libtomahawk/Pipeline.cpp
C++ | 717 lines | 513 code | 157 blank | 47 comment | 59 complexity | 7e0ea52bab968b8952fc4713a839b0e0 MD5 | raw file
1/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> === 2 * 3 * Copyright 2010-2015, Christian Muehlhaeuser <muesli@tomahawk-player.org> 4 * 5 * Tomahawk is free software: you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation, either version 3 of the License, or 8 * (at your option) any later version. 9 * 10 * Tomahawk is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with Tomahawk. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19#include "Pipeline_p.h" 20 21#include <QMutexLocker> 22 23#include "database/Database.h" 24#include "resolvers/ExternalResolver.h" 25#include "resolvers/ScriptResolver.h" 26#include "resolvers/JSResolver.h" 27#include "utils/ResultUrlChecker.h" 28#include "utils/Logger.h" 29 30#include "FuncTimeout.h" 31#include "Result.h" 32#include "Source.h" 33#include "SourceList.h" 34 35#define DEFAULT_CONCURRENT_QUERIES 4 36#define MAX_CONCURRENT_QUERIES 16 37#define CLEANUP_TIMEOUT 5 * 60 * 1000 38#define MINSCORE 0.5 39#define DEFAULT_RESOLVER_TIMEOUT 5000 // 5 seconds 40 41using namespace Tomahawk; 42 43Pipeline* PipelinePrivate::s_instance = 0; 44 45 46Pipeline* 47Pipeline::instance() 48{ 49 return PipelinePrivate::s_instance; 50} 51 52 53Pipeline::Pipeline( QObject* parent ) 54 : QObject( parent ) 55 , d_ptr( new PipelinePrivate( this ) ) 56{ 57 Q_D( Pipeline ); 58 PipelinePrivate::s_instance = this; 59 60 d->maxConcurrentQueries = 24; 61 tDebug() << Q_FUNC_INFO << "Using" << d->maxConcurrentQueries << "threads"; 62 63 d->temporaryQueryTimer.setInterval( CLEANUP_TIMEOUT ); 64 connect( &d->temporaryQueryTimer, SIGNAL( timeout() ), SLOT( onTemporaryQueryTimer() ) ); 65} 66 67 68Pipeline::~Pipeline() 69{ 70 Q_D( Pipeline ); 71 tDebug() << Q_FUNC_INFO; 72 d->running = false; 73 74 // stop script resolvers 75 foreach ( QPointer< ExternalResolver > r, d->scriptResolvers ) 76 if ( !r.isNull() ) 77 r.data()->deleteLater(); 78 79 d->scriptResolvers.clear(); 80} 81 82 83bool 84Pipeline::isRunning() const 85{ 86 Q_D( const Pipeline ); 87 return d->running; 88} 89 90 91unsigned int 92Pipeline::pendingQueryCount() const 93{ 94 Q_D( const Pipeline ); 95 return d->queries_pending.count(); 96} 97 98 99unsigned int 100Pipeline::activeQueryCount() const 101{ 102 Q_D( const Pipeline ); 103 return d->qidsState.uniqueKeys().count(); 104} 105 106 107void 108Pipeline::databaseReady() 109{ 110 connect( Database::instance(), SIGNAL( ready() ), this, SLOT( start() ), Qt::QueuedConnection ); 111 Database::instance()->loadIndex(); 112} 113 114 115void 116Pipeline::start() 117{ 118 Q_D( Pipeline ); 119 120 tDebug() << Q_FUNC_INFO << "Shunting" << d->queries_pending.size() << "queries!"; 121 d->running = true; 122 emit running(); 123 124 shuntNext(); 125} 126 127 128void 129Pipeline::stop() 130{ 131 Q_D( Pipeline ); 132 133 d->running = false; 134} 135 136 137void 138Pipeline::removeResolver( Resolver* r ) 139{ 140 Q_D( Pipeline ); 141 QMutexLocker lock( &d->mut ); 142 143 tDebug() << "Removed resolver:" << r->name(); 144 d->resolvers.removeAll( r ); 145 if ( d->running ) { 146 // Only notify if Pipeline is still active. 147 emit resolverRemoved( r ); 148 } 149} 150 151 152QList< Tomahawk::Resolver* > 153Pipeline::resolvers() const 154{ 155 Q_D( const Pipeline ); 156 157 return d->resolvers; 158} 159 160 161void 162Pipeline::addResolver( Resolver* r ) 163{ 164 Q_D( Pipeline ); 165 QMutexLocker lock( &d->mut ); 166 167 tDebug() << "Adding resolver" << r->name(); 168 d->resolvers.append( r ); 169 emit resolverAdded( r ); 170} 171 172 173void 174Pipeline::addExternalResolverFactory( ResolverFactoryFunc resolverFactory ) 175{ 176 Q_D( Pipeline ); 177 d->resolverFactories << resolverFactory; 178} 179 180 181Tomahawk::ExternalResolver* 182Pipeline::addScriptResolver( const QString& accountId, const QString& path, const QStringList& additionalScriptPaths ) 183{ 184 Q_D( Pipeline ); 185 ExternalResolver* res = 0; 186 187 foreach ( ResolverFactoryFunc factory, d->resolverFactories ) 188 { 189 res = factory( accountId, path, additionalScriptPaths ); 190 if ( !res ) 191 continue; 192 193 d->scriptResolvers << QPointer< ExternalResolver > ( res ); 194 195 break; 196 } 197 198 return res; 199} 200 201 202void 203Pipeline::stopScriptResolver( const QString& path ) 204{ 205 Q_D( Pipeline ); 206 foreach ( QPointer< ExternalResolver > res, d->scriptResolvers ) 207 { 208 if ( res.data()->filePath() == path ) 209 res.data()->stop(); 210 } 211} 212 213 214void 215Pipeline::removeScriptResolver( const QString& scriptPath ) 216{ 217 Q_D( Pipeline ); 218 QPointer< ExternalResolver > r; 219 foreach ( QPointer< ExternalResolver > res, d->scriptResolvers ) 220 { 221 if ( res.isNull() ) 222 continue; 223 224 if ( res.data()->filePath() == scriptPath ) 225 r = res; 226 } 227 d->scriptResolvers.removeAll( r ); 228 229 if ( !r.isNull() ) 230 { 231 r.data()->stop(); 232 r.data()->deleteLater(); 233 } 234} 235 236 237QList<QPointer<ExternalResolver> > 238Pipeline::scriptResolvers() const 239{ 240 Q_D( const Pipeline ); 241 242 return d->scriptResolvers; 243} 244 245 246ExternalResolver* 247Pipeline::resolverForPath( const QString& scriptPath ) 248{ 249 Q_D( Pipeline ); 250 251 foreach ( QPointer< ExternalResolver > res, d->scriptResolvers ) 252 { 253 if ( res.data()->filePath() == scriptPath ) 254 return res.data(); 255 } 256 return 0; 257} 258 259 260void 261Pipeline::resolve( const QList<query_ptr>& qlist, bool prioritized, bool temporaryQuery ) 262{ 263 Q_D( Pipeline ); 264 265 { 266 QMutexLocker lock( &d->mut ); 267 268 int i = 0; 269 foreach ( const query_ptr& q, qlist ) 270 { 271 if ( q->resolvingFinished() ) 272 continue; 273 if ( d->qidsState.contains( q->id() ) ) 274 continue; 275 if ( d->queries_pending.contains( q ) ) 276 { 277 if ( prioritized ) 278 { 279 d->queries_pending.insert( i++, d->queries_pending.takeAt( d->queries_pending.indexOf( q ) ) ); 280 } 281 continue; 282 } 283 284 if ( !d->qids.contains( q->id() ) ) 285 d->qids.insert( q->id(), q ); 286 287 if ( prioritized ) 288 d->queries_pending.insert( i++, q ); 289 else 290 d->queries_pending << q; 291 292 if ( temporaryQuery ) 293 { 294 d->queries_temporary << q; 295 296 if ( d->temporaryQueryTimer.isActive() ) 297 d->temporaryQueryTimer.stop(); 298 d->temporaryQueryTimer.start(); 299 } 300 } 301 } 302 303 shuntNext(); 304} 305 306 307bool 308Pipeline::isResolving( const query_ptr& q ) const 309{ 310 Q_D( const Pipeline ); 311 312 return d->qids.contains( q->id() ) && d->qidsState.contains( q->id() ); 313} 314 315 316void 317Pipeline::resolve( const query_ptr& q, bool prioritized, bool temporaryQuery ) 318{ 319 if ( q.isNull() ) 320 return; 321 322 QList< query_ptr > qlist; 323 qlist << q; 324 resolve( qlist, prioritized, temporaryQuery ); 325} 326 327 328void 329Pipeline::resolve( QID qid, bool prioritized, bool temporaryQuery ) 330{ 331 resolve( query( qid ), prioritized, temporaryQuery ); 332} 333 334 335void 336Pipeline::reportError( QID qid, Tomahawk::Resolver* r ) 337{ 338 reportResults( qid, r, QList< result_ptr>() ); 339} 340 341 342void 343Pipeline::reportResults( QID qid, Tomahawk::Resolver* r, const QList< result_ptr >& results ) 344{ 345 Q_D( Pipeline ); 346 if ( !d->running ) 347 return; 348 if ( !d->qids.contains( qid ) ) 349 { 350 if ( !results.isEmpty() ) 351 { 352 Resolver* resolvedBy = results[0]->resolvedBy(); 353 if ( resolvedBy ) 354 { 355 tDebug() << "Result arrived too late for:" << qid << "by" << resolvedBy->name(); 356 } 357 else 358 { 359 tDebug() << "Result arrived too late for:" << qid; 360 } 361 } 362 return; 363 } 364 const query_ptr& q = d->qids.value( qid ); 365 366 Q_ASSERT( !q.isNull() ); 367 if ( q.isNull() ) 368 return; 369 370 QList< result_ptr > cleanResults; 371 QList< result_ptr > httpResults; 372 foreach ( const result_ptr& r, results ) 373 { 374 if ( !r ) 375 continue; 376 377 if ( !r->checked() && ( r->url().startsWith( "http" ) && !r->url().startsWith( "http://localhost" ) ) ) 378 httpResults << r; 379 else 380 cleanResults << r; 381 } 382 383 addResultsToQuery( q, cleanResults ); 384 if ( !httpResults.isEmpty() ) 385 { 386 const ResultUrlChecker* checker = new ResultUrlChecker( q, r, httpResults ); 387 connect( checker, SIGNAL( done() ), SLOT( onResultUrlCheckerDone() ) ); 388 } 389 else 390 { 391 decQIDState( q, r ); 392 } 393 394/* if ( q->solved() && !q->isFullTextQuery() ) 395 { 396 checkQIDState( q, 0 ); 397 return; 398 }*/ 399} 400 401 402void 403Pipeline::addResultsToQuery( const query_ptr& query, const QList< result_ptr >& results ) 404{ 405 Q_D( Pipeline ); 406// tDebug( LOGVERBOSE ) << Q_FUNC_INFO << query->toString() << results.count(); 407 408 QList< result_ptr > cleanResults; 409 foreach ( const result_ptr& r, results ) 410 { 411 if ( !query->isFullTextQuery() && query->howSimilar( r ) < MINSCORE ) 412 continue; 413 414 cleanResults << r; 415 } 416 417 if ( !cleanResults.isEmpty() ) 418 { 419 query->addResults( cleanResults ); 420 421 if ( d->queries_temporary.contains( query ) ) 422 { 423 foreach ( const result_ptr& r, cleanResults ) 424 { 425 d->rids.insert( r->id(), r ); 426 } 427 } 428 } 429} 430 431 432void 433Pipeline::onResultUrlCheckerDone() 434{ 435 ResultUrlChecker* checker = qobject_cast< ResultUrlChecker* >( sender() ); 436 if ( !checker ) 437 return; 438 439 checker->deleteLater(); 440 441 const query_ptr q = checker->query(); 442 addResultsToQuery( q, checker->validResults() ); 443/* if ( q && !q->isFullTextQuery() ) 444 { 445 checkQIDState( q, 0 ); 446 return; 447 }*/ 448 449 decQIDState( q, reinterpret_cast<Tomahawk::Resolver*>( checker->userData() ) ); 450} 451 452 453void 454Pipeline::reportAlbums( QID qid, const QList< album_ptr >& albums ) 455{ 456 Q_D( Pipeline ); 457 if ( !d->running ) 458 return; 459 460 if ( !d->qids.contains( qid ) ) 461 { 462 tDebug() << "Albums arrived too late for:" << qid; 463 return; 464 } 465 const query_ptr& q = d->qids.value( qid ); 466 Q_ASSERT( q->isFullTextQuery() ); 467 468 QList< album_ptr > cleanAlbums; 469 foreach ( const album_ptr& r, albums ) 470 { 471// float score = q->howSimilar( r ); 472 473 cleanAlbums << r; 474 } 475 476 if ( !cleanAlbums.isEmpty() ) 477 { 478 q->addAlbums( cleanAlbums ); 479 } 480} 481 482 483void 484Pipeline::reportArtists( QID qid, const QList< artist_ptr >& artists ) 485{ 486 Q_D( Pipeline ); 487 if ( !d->running ) 488 return; 489 490 if ( !d->qids.contains( qid ) ) 491 { 492 tDebug() << "Artists arrived too late for:" << qid; 493 return; 494 } 495 const query_ptr& q = d->qids.value( qid ); 496 Q_ASSERT( q->isFullTextQuery() ); 497 498 QList< artist_ptr > cleanArtists; 499 foreach ( const artist_ptr& r, artists ) 500 { 501// float score = q->howSimilar( r ); 502 503 cleanArtists << r; 504 } 505 506 if ( !cleanArtists.isEmpty() ) 507 { 508 q->addArtists( cleanArtists ); 509 } 510} 511 512 513void 514Pipeline::shuntNext() 515{ 516 Q_D( Pipeline ); 517 if ( !d->running ) 518 return; 519 520 unsigned int rc; 521 query_ptr q; 522 { 523 QMutexLocker lock( &d->mut ); 524 525 rc = d->resolvers.count(); 526 if ( d->queries_pending.isEmpty() ) 527 { 528 if ( d->qidsState.isEmpty() ) 529 emit idle(); 530 return; 531 } 532 533 // Check if we are ready to dispatch more queries 534 if ( activeQueryCount() >= d->maxConcurrentQueries ) 535 return; 536 537 /* 538 Since resolvers are async, we now dispatch to the highest weighted ones 539 and after timeout, dispatch to next highest etc, aborting when solved 540 */ 541 q = d->queries_pending.takeFirst(); 542 q->setCurrentResolver( 0 ); 543 } 544 545 // Zero-patient, a stub so that query is not resolved until we go through 546 // all resolvers 547 // As query considered as 'finished trying to resolve' when there are no 548 // more qid entries in qidsState we'll put one as sort of 'keep this until 549 // we kick off all our resolvers' entry 550 // once we kick off all resolvers we'll remove this entry 551 incQIDState( q, nullptr ); 552 checkQIDState( q ); 553} 554 555 556void 557Pipeline::timeoutShunt( const query_ptr& q, Tomahawk::Resolver* r ) 558{ 559 Q_D( Pipeline ); 560 if ( !d->running ) 561 return; 562 563 decQIDState( q, r ); 564} 565 566 567void 568Pipeline::shunt( const query_ptr& q ) 569{ 570 Q_D( Pipeline ); 571 if ( !d->running ) 572 return; 573 574 Resolver* r = 0; 575 if ( !q->resolvingFinished() ) 576 r = nextResolver( q ); 577 578 if ( r ) 579 { 580 tLog( LOGVERBOSE ) << "Dispatching to resolver" << r->name() << r->timeout() << q->toString() << q->solved() << q->id(); 581 582 incQIDState( q, r ); 583 q->setCurrentResolver( r ); 584 r->resolve( q ); 585 emit resolving( q ); 586 587 auto timeout = r->timeout(); 588 if ( timeout == 0 ) 589 timeout = DEFAULT_RESOLVER_TIMEOUT; 590 591 new FuncTimeout( timeout, std::bind( &Pipeline::timeoutShunt, this, q, r ), this ); 592 } 593 else 594 { 595 // we get here if we disable a resolver while a query is resolving 596 // OR we are just out of resolvers while query is still resolving 597 598 // since we seem to at least tried to kick off all of the resolvers, 599 // remove the '.keep' entry 600 decQIDState( q, nullptr ); 601 return; 602 } 603 604 shuntNext(); 605} 606 607 608Tomahawk::Resolver* 609Pipeline::nextResolver( const Tomahawk::query_ptr& query ) const 610{ 611 Q_D( const Pipeline ); 612 Resolver* newResolver = 0; 613 614 foreach ( Resolver* r, d->resolvers ) 615 { 616 if ( query->resolvedBy().contains( r ) ) 617 continue; 618 619 if ( !newResolver ) 620 { 621 newResolver = r; 622 continue; 623 } 624 625 if ( r->weight() > newResolver->weight() ) 626 newResolver = r; 627 } 628 629 return newResolver; 630} 631 632 633void 634Pipeline::checkQIDState( const Tomahawk::query_ptr& query ) 635{ 636 Q_D( Pipeline ); 637 QMutexLocker lock( &d->mut ); 638 639 tDebug() << Q_FUNC_INFO << query->id() << d->qidsState.count( query->id() ); 640 641 if ( d->qidsState.contains( query->id() ) ) 642 { 643 new FuncTimeout( 0, std::bind( &Pipeline::shunt, this, query ), this ); 644 } 645 else 646 { 647 query->onResolvingFinished(); 648 649 if ( !d->queries_temporary.contains( query ) ) 650 d->qids.remove( query->id() ); 651 652 new FuncTimeout( 0, std::bind( &Pipeline::shuntNext, this ), this ); 653 } 654} 655 656 657void 658Pipeline::incQIDState( const Tomahawk::query_ptr& query, Tomahawk::Resolver* r ) 659{ 660 Q_D( Pipeline ); 661 QMutexLocker lock( &d->mut ); 662 663 d->qidsState.insert( query->id(), r ); 664} 665 666 667void 668Pipeline::decQIDState( const Tomahawk::query_ptr& query, Tomahawk::Resolver* r ) 669{ 670 Q_D( Pipeline ); 671 672 if ( d->qidsState.contains( query->id(), r ) ) 673 { 674 { 675 QMutexLocker lock( &d->mut ); 676 d->qidsState.remove( query->id(), r ); // Removes all matching pairs 677 } 678 679 checkQIDState( query ); 680 } 681} 682 683 684void 685Pipeline::onTemporaryQueryTimer() 686{ 687 Q_D( Pipeline ); 688 tDebug() << Q_FUNC_INFO; 689 690 QMutexLocker lock( &d->mut ); 691 d->temporaryQueryTimer.stop(); 692 693 for ( int i = d->queries_temporary.count() - 1; i >= 0; i-- ) 694 { 695 query_ptr q = d->queries_temporary.takeAt( i ); 696 697 d->qids.remove( q->id() ); 698 foreach ( const Tomahawk::result_ptr& r, q->results() ) 699 d->rids.remove( r->id() ); 700 } 701} 702 703 704query_ptr 705Pipeline::query( const QID& qid ) const 706{ 707 Q_D( const Pipeline ); 708 return d->qids.value( qid ); 709} 710 711 712result_ptr 713Pipeline::result( const RID& rid ) const 714{ 715 Q_D( const Pipeline ); 716 return d->rids.value( rid ); 717}