/src/libtomahawk/Pipeline.cpp

  1/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
  2 *
  3 *   Copyright 2010-2015, Christian Muehlhaeuser <muesli@tomahawk-player.org>
  4 *
  5 *   Tomahawk is free software: you can redistribute it and/or modify
  6 *   it under the terms of the GNU General Public License as published by
  7 *   the Free Software Foundation, either version 3 of the License, or
  8 *   (at your option) any later version.
  9 *
 10 *   Tomahawk is distributed in the hope that it will be useful,
 11 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 13 *   GNU General Public License for more details.
 14 *
 15 *   You should have received a copy of the GNU General Public License
 16 *   along with Tomahawk. If not, see <http://www.gnu.org/licenses/>.
 17 */
 18
 19#include "Pipeline_p.h"
 20
 21#include <QMutexLocker>
 22
 23#include "database/Database.h"
 24#include "resolvers/ExternalResolver.h"
 25#include "resolvers/ScriptResolver.h"
 26#include "resolvers/JSResolver.h"
 27#include "utils/ResultUrlChecker.h"
 28#include "utils/Logger.h"
 29
 30#include "FuncTimeout.h"
 31#include "Result.h"
 32#include "Source.h"
 33#include "SourceList.h"
 34
 35#define DEFAULT_CONCURRENT_QUERIES 4
 36#define MAX_CONCURRENT_QUERIES 16
 37#define CLEANUP_TIMEOUT 5 * 60 * 1000
 38#define MINSCORE 0.5
 39#define DEFAULT_RESOLVER_TIMEOUT 5000 // 5 seconds
 40
 41using namespace Tomahawk;
 42
 43Pipeline* PipelinePrivate::s_instance = 0;
 44
 45
 46Pipeline*
 47Pipeline::instance()
 48{
 49    return PipelinePrivate::s_instance;
 50}
 51
 52
 53Pipeline::Pipeline( QObject* parent )
 54    : QObject( parent )
 55    , d_ptr( new PipelinePrivate( this ) )
 56{
 57    Q_D( Pipeline );
 58    PipelinePrivate::s_instance = this;
 59
 60    d->maxConcurrentQueries = 24;
 61    tDebug() << Q_FUNC_INFO << "Using" << d->maxConcurrentQueries << "threads";
 62
 63    d->temporaryQueryTimer.setInterval( CLEANUP_TIMEOUT );
 64    connect( &d->temporaryQueryTimer, SIGNAL( timeout() ), SLOT( onTemporaryQueryTimer() ) );
 65}
 66
 67
 68Pipeline::~Pipeline()
 69{
 70    Q_D( Pipeline );
 71    tDebug() << Q_FUNC_INFO;
 72    d->running = false;
 73
 74    // stop script resolvers
 75    foreach ( QPointer< ExternalResolver > r, d->scriptResolvers )
 76        if ( !r.isNull() )
 77            r.data()->deleteLater();
 78
 79    d->scriptResolvers.clear();
 80}
 81
 82
 83bool
 84Pipeline::isRunning() const
 85{
 86    Q_D( const Pipeline );
 87    return d->running;
 88}
 89
 90
 91unsigned int
 92Pipeline::pendingQueryCount() const
 93{
 94    Q_D( const Pipeline );
 95    return d->queries_pending.count();
 96}
 97
 98
 99unsigned int
100Pipeline::activeQueryCount() const
101{
102    Q_D( const Pipeline );
103    return d->qidsState.uniqueKeys().count();
104}
105
106
107void
108Pipeline::databaseReady()
109{
110    connect( Database::instance(), SIGNAL( ready() ), this, SLOT( start() ), Qt::QueuedConnection );
111    Database::instance()->loadIndex();
112}
113
114
115void
116Pipeline::start()
117{
118    Q_D( Pipeline );
119
120    tDebug() << Q_FUNC_INFO << "Shunting" << d->queries_pending.size() << "queries!";
121    d->running = true;
122    emit running();
123
124    shuntNext();
125}
126
127
128void
129Pipeline::stop()
130{
131    Q_D( Pipeline );
132
133    d->running = false;
134}
135
136
137void
138Pipeline::removeResolver( Resolver* r )
139{
140    Q_D( Pipeline );
141    QMutexLocker lock( &d->mut );
142
143    tDebug() << "Removed resolver:" << r->name();
144    d->resolvers.removeAll( r );
145    if ( d->running ) {
146        // Only notify if Pipeline is still active.
147        emit resolverRemoved( r );
148    }
149}
150
151
152QList< Tomahawk::Resolver* >
153Pipeline::resolvers() const
154{
155    Q_D( const Pipeline );
156
157    return d->resolvers;
158}
159
160
161void
162Pipeline::addResolver( Resolver* r )
163{
164    Q_D( Pipeline );
165    QMutexLocker lock( &d->mut );
166
167    tDebug() << "Adding resolver" << r->name();
168    d->resolvers.append( r );
169    emit resolverAdded( r );
170}
171
172
173void
174Pipeline::addExternalResolverFactory( ResolverFactoryFunc resolverFactory )
175{
176    Q_D( Pipeline );
177    d->resolverFactories << resolverFactory;
178}
179
180
181Tomahawk::ExternalResolver*
182Pipeline::addScriptResolver( const QString& accountId, const QString& path, const QStringList& additionalScriptPaths )
183{
184    Q_D( Pipeline );
185    ExternalResolver* res = 0;
186
187    foreach ( ResolverFactoryFunc factory, d->resolverFactories )
188    {
189        res = factory( accountId, path, additionalScriptPaths );
190        if ( !res )
191            continue;
192
193        d->scriptResolvers << QPointer< ExternalResolver > ( res );
194
195        break;
196    }
197
198    return res;
199}
200
201
202void
203Pipeline::stopScriptResolver( const QString& path )
204{
205    Q_D( Pipeline );
206    foreach ( QPointer< ExternalResolver > res, d->scriptResolvers )
207    {
208        if ( res.data()->filePath() == path )
209            res.data()->stop();
210    }
211}
212
213
214void
215Pipeline::removeScriptResolver( const QString& scriptPath )
216{
217    Q_D( Pipeline );
218    QPointer< ExternalResolver > r;
219    foreach ( QPointer< ExternalResolver > res, d->scriptResolvers )
220    {
221        if ( res.isNull() )
222            continue;
223
224        if ( res.data()->filePath() == scriptPath )
225            r = res;
226    }
227    d->scriptResolvers.removeAll( r );
228
229    if ( !r.isNull() )
230    {
231        r.data()->stop();
232        r.data()->deleteLater();
233    }
234}
235
236
237QList<QPointer<ExternalResolver> >
238Pipeline::scriptResolvers() const
239{
240    Q_D( const Pipeline );
241
242    return d->scriptResolvers;
243}
244
245
246ExternalResolver*
247Pipeline::resolverForPath( const QString& scriptPath )
248{
249    Q_D( Pipeline );
250
251    foreach ( QPointer< ExternalResolver > res, d->scriptResolvers )
252    {
253        if ( res.data()->filePath() == scriptPath )
254            return res.data();
255    }
256    return 0;
257}
258
259
260void
261Pipeline::resolve( const QList<query_ptr>& qlist, bool prioritized, bool temporaryQuery )
262{
263    Q_D( Pipeline );
264
265    {
266        QMutexLocker lock( &d->mut );
267
268        int i = 0;
269        foreach ( const query_ptr& q, qlist )
270        {
271            if ( q->resolvingFinished() )
272                continue;
273            if ( d->qidsState.contains( q->id() ) )
274                continue;
275            if ( d->queries_pending.contains( q ) )
276            {
277                if ( prioritized )
278                {
279                    d->queries_pending.insert( i++, d->queries_pending.takeAt( d->queries_pending.indexOf( q ) ) );
280                }
281                continue;
282            }
283
284            if ( !d->qids.contains( q->id() ) )
285                d->qids.insert( q->id(), q );
286
287            if ( prioritized )
288                d->queries_pending.insert( i++, q );
289            else
290                d->queries_pending << q;
291
292            if ( temporaryQuery )
293            {
294                d->queries_temporary << q;
295
296                if ( d->temporaryQueryTimer.isActive() )
297                    d->temporaryQueryTimer.stop();
298                d->temporaryQueryTimer.start();
299            }
300        }
301    }
302
303    shuntNext();
304}
305
306
307bool
308Pipeline::isResolving( const query_ptr& q ) const
309{
310    Q_D( const Pipeline );
311
312    return d->qids.contains( q->id() ) && d->qidsState.contains( q->id() );
313}
314
315
316void
317Pipeline::resolve( const query_ptr& q, bool prioritized, bool temporaryQuery )
318{
319    if ( q.isNull() )
320        return;
321
322    QList< query_ptr > qlist;
323    qlist << q;
324    resolve( qlist, prioritized, temporaryQuery );
325}
326
327
328void
329Pipeline::resolve( QID qid, bool prioritized, bool temporaryQuery )
330{
331    resolve( query( qid ), prioritized, temporaryQuery );
332}
333
334
335void
336Pipeline::reportError( QID qid, Tomahawk::Resolver* r )
337{
338    reportResults( qid, r, QList< result_ptr>() );
339}
340
341
342void
343Pipeline::reportResults( QID qid, Tomahawk::Resolver* r, const QList< result_ptr >& results )
344{
345    Q_D( Pipeline );
346    if ( !d->running )
347        return;
348    if ( !d->qids.contains( qid ) )
349    {
350        if ( !results.isEmpty() )
351        {
352            Resolver* resolvedBy = results[0]->resolvedBy();
353            if ( resolvedBy )
354            {
355                tDebug() << "Result arrived too late for:" << qid << "by" << resolvedBy->name();
356            }
357            else
358            {
359                tDebug() << "Result arrived too late for:" << qid;
360            }
361        }
362        return;
363    }
364    const query_ptr& q = d->qids.value( qid );
365
366    Q_ASSERT( !q.isNull() );
367    if ( q.isNull() )
368        return;
369
370    QList< result_ptr > cleanResults;
371    QList< result_ptr > httpResults;
372    foreach ( const result_ptr& r, results )
373    {
374        if ( !r )
375            continue;
376
377        if ( !r->checked() && ( r->url().startsWith( "http" ) && !r->url().startsWith( "http://localhost" ) ) )
378            httpResults << r;
379        else
380            cleanResults << r;
381    }
382
383    addResultsToQuery( q, cleanResults );
384    if ( !httpResults.isEmpty() )
385    {
386        const ResultUrlChecker* checker = new ResultUrlChecker( q, r, httpResults );
387        connect( checker, SIGNAL( done() ), SLOT( onResultUrlCheckerDone() ) );
388    }
389    else
390    {
391        decQIDState( q, r );
392    }
393
394/*    if ( q->solved() && !q->isFullTextQuery() )
395    {
396        checkQIDState( q, 0 );
397        return;
398    }*/
399}
400
401
402void
403Pipeline::addResultsToQuery( const query_ptr& query, const QList< result_ptr >& results )
404{
405    Q_D( Pipeline );
406//    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << query->toString() << results.count();
407
408    QList< result_ptr > cleanResults;
409    foreach ( const result_ptr& r, results )
410    {
411        if ( !query->isFullTextQuery() && query->howSimilar( r ) < MINSCORE )
412            continue;
413
414        cleanResults << r;
415    }
416
417    if ( !cleanResults.isEmpty() )
418    {
419        query->addResults( cleanResults );
420
421        if ( d->queries_temporary.contains( query ) )
422        {
423            foreach ( const result_ptr& r, cleanResults )
424            {
425                d->rids.insert( r->id(), r );
426            }
427        }
428    }
429}
430
431
432void
433Pipeline::onResultUrlCheckerDone()
434{
435    ResultUrlChecker* checker = qobject_cast< ResultUrlChecker* >( sender() );
436    if ( !checker )
437        return;
438
439    checker->deleteLater();
440
441    const query_ptr q = checker->query();
442    addResultsToQuery( q, checker->validResults() );
443/*    if ( q && !q->isFullTextQuery() )
444    {
445        checkQIDState( q, 0 );
446        return;
447    }*/
448
449    decQIDState( q, reinterpret_cast<Tomahawk::Resolver*>( checker->userData() ) );
450}
451
452
453void
454Pipeline::reportAlbums( QID qid, const QList< album_ptr >& albums )
455{
456    Q_D( Pipeline );
457    if ( !d->running )
458        return;
459
460    if ( !d->qids.contains( qid ) )
461    {
462        tDebug() << "Albums arrived too late for:" << qid;
463        return;
464    }
465    const query_ptr& q = d->qids.value( qid );
466    Q_ASSERT( q->isFullTextQuery() );
467
468    QList< album_ptr > cleanAlbums;
469    foreach ( const album_ptr& r, albums )
470    {
471//        float score = q->howSimilar( r );
472
473        cleanAlbums << r;
474    }
475
476    if ( !cleanAlbums.isEmpty() )
477    {
478        q->addAlbums( cleanAlbums );
479    }
480}
481
482
483void
484Pipeline::reportArtists( QID qid, const QList< artist_ptr >& artists )
485{
486    Q_D( Pipeline );
487    if ( !d->running )
488        return;
489
490    if ( !d->qids.contains( qid ) )
491    {
492        tDebug() << "Artists arrived too late for:" << qid;
493        return;
494    }
495    const query_ptr& q = d->qids.value( qid );
496    Q_ASSERT( q->isFullTextQuery() );
497
498    QList< artist_ptr > cleanArtists;
499    foreach ( const artist_ptr& r, artists )
500    {
501//        float score = q->howSimilar( r );
502
503        cleanArtists << r;
504    }
505
506    if ( !cleanArtists.isEmpty() )
507    {
508        q->addArtists( cleanArtists );
509    }
510}
511
512
513void
514Pipeline::shuntNext()
515{
516    Q_D( Pipeline );
517    if ( !d->running )
518        return;
519
520    unsigned int rc;
521    query_ptr q;
522    {
523        QMutexLocker lock( &d->mut );
524
525        rc = d->resolvers.count();
526        if ( d->queries_pending.isEmpty() )
527        {
528            if ( d->qidsState.isEmpty() )
529                emit idle();
530            return;
531        }
532
533        // Check if we are ready to dispatch more queries
534        if ( activeQueryCount() >= d->maxConcurrentQueries )
535            return;
536
537        /*
538            Since resolvers are async, we now dispatch to the highest weighted ones
539            and after timeout, dispatch to next highest etc, aborting when solved
540        */
541        q = d->queries_pending.takeFirst();
542        q->setCurrentResolver( 0 );
543    }
544
545    // Zero-patient, a stub so that query is not resolved until we go through
546    // all resolvers
547    // As query considered as 'finished trying to resolve' when there are no
548    // more qid entries in qidsState we'll put one as sort of 'keep this until
549    // we kick off all our resolvers' entry
550    // once we kick off all resolvers we'll remove this entry
551    incQIDState( q, nullptr );
552    checkQIDState( q );
553}
554
555
556void
557Pipeline::timeoutShunt( const query_ptr& q, Tomahawk::Resolver* r )
558{
559    Q_D( Pipeline );
560    if ( !d->running )
561        return;
562
563    decQIDState( q, r );
564}
565
566
567void
568Pipeline::shunt( const query_ptr& q )
569{
570    Q_D( Pipeline );
571    if ( !d->running )
572        return;
573
574    Resolver* r = 0;
575    if ( !q->resolvingFinished() )
576        r = nextResolver( q );
577
578    if ( r )
579    {
580        tLog( LOGVERBOSE ) << "Dispatching to resolver" << r->name() << r->timeout() << q->toString() << q->solved() << q->id();
581
582        incQIDState( q, r );
583        q->setCurrentResolver( r );
584        r->resolve( q );
585        emit resolving( q );
586
587        auto timeout = r->timeout();
588        if ( timeout == 0 )
589            timeout = DEFAULT_RESOLVER_TIMEOUT;
590
591        new FuncTimeout( timeout, std::bind( &Pipeline::timeoutShunt, this, q, r ), this );
592    }
593    else
594    {
595        // we get here if we disable a resolver while a query is resolving
596        // OR we are just out of resolvers while query is still resolving
597
598        // since we seem to at least tried to kick off all of the resolvers,
599        // remove the '.keep' entry
600        decQIDState( q, nullptr );
601        return;
602    }
603
604    shuntNext();
605}
606
607
608Tomahawk::Resolver*
609Pipeline::nextResolver( const Tomahawk::query_ptr& query ) const
610{
611    Q_D( const Pipeline );
612    Resolver* newResolver = 0;
613
614    foreach ( Resolver* r, d->resolvers )
615    {
616        if ( query->resolvedBy().contains( r ) )
617            continue;
618
619        if ( !newResolver )
620        {
621            newResolver = r;
622            continue;
623        }
624
625        if ( r->weight() > newResolver->weight() )
626            newResolver = r;
627    }
628
629    return newResolver;
630}
631
632
633void
634Pipeline::checkQIDState( const Tomahawk::query_ptr& query )
635{
636    Q_D( Pipeline );
637    QMutexLocker lock( &d->mut );
638
639    tDebug() << Q_FUNC_INFO << query->id() << d->qidsState.count( query->id() );
640
641    if ( d->qidsState.contains( query->id() ) )
642    {
643        new FuncTimeout( 0, std::bind( &Pipeline::shunt, this, query ), this );
644    }
645    else
646    {
647        query->onResolvingFinished();
648
649        if ( !d->queries_temporary.contains( query ) )
650            d->qids.remove( query->id() );
651
652        new FuncTimeout( 0, std::bind( &Pipeline::shuntNext, this ), this );
653    }
654}
655
656
657void
658Pipeline::incQIDState( const Tomahawk::query_ptr& query, Tomahawk::Resolver* r )
659{
660    Q_D( Pipeline );
661    QMutexLocker lock( &d->mut );
662
663    d->qidsState.insert( query->id(), r );
664}
665
666
667void
668Pipeline::decQIDState( const Tomahawk::query_ptr& query, Tomahawk::Resolver* r )
669{
670    Q_D( Pipeline );
671
672    if ( d->qidsState.contains( query->id(), r ) )
673    {
674        {
675            QMutexLocker lock( &d->mut );
676            d->qidsState.remove( query->id(), r ); // Removes all matching pairs
677        }
678
679        checkQIDState( query );
680    }
681}
682
683
684void
685Pipeline::onTemporaryQueryTimer()
686{
687    Q_D( Pipeline );
688    tDebug() << Q_FUNC_INFO;
689
690    QMutexLocker lock( &d->mut );
691    d->temporaryQueryTimer.stop();
692
693    for ( int i = d->queries_temporary.count() - 1; i >= 0; i-- )
694    {
695        query_ptr q = d->queries_temporary.takeAt( i );
696
697        d->qids.remove( q->id() );
698        foreach ( const Tomahawk::result_ptr& r, q->results() )
699            d->rids.remove( r->id() );
700    }
701}
702
703
704query_ptr
705Pipeline::query( const QID& qid ) const
706{
707    Q_D( const Pipeline );
708    return d->qids.value( qid );
709}
710
711
712result_ptr
713Pipeline::result( const RID& rid ) const
714{
715    Q_D( const Pipeline );
716    return d->rids.value( rid );
717}