/src/libtomahawk/EchonestCatalogSynchronizer.cpp

http://github.com/tomahawk-player/tomahawk · C++ · 376 lines · 264 code · 62 blank · 50 comment · 25 complexity · ca982068e527dabca5cecf4a0d2e3f3f MD5 · raw file

  1. /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
  2. *
  3. * Copyright 2010-2011, Leo Franchi <lfranchi@kde.org>
  4. * Copyright 2015, Christian Muehlhaeuser <muesli@tomahawk-player.org>
  5. *
  6. * Tomahawk is free software: you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation, either version 3 of the License, or
  9. * (at your option) any later version.
  10. *
  11. * Tomahawk is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. * GNU General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with Tomahawk. If not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "EchonestCatalogSynchronizer.h"
  20. #include "collection/Collection.h"
  21. #include "database/Database.h"
  22. #include "database/DatabaseImpl.h"
  23. #include "database/DatabaseCommand_GenericSelect.h"
  24. #include "database/DatabaseCommand_SetCollectionAttributes.h"
  25. #include "database/DatabaseCommand_LoadFiles.h"
  26. #include "database/DatabaseCommand_SetTrackAttributes.h"
  27. #include "utils/Logger.h"
  28. #include "PlaylistEntry.h"
  29. #include "Query.h"
  30. #include "SourceList.h"
  31. #include "TomahawkSettings.h"
  32. #include "Track.h"
  33. #include <echonest5/CatalogUpdateEntry.h>
  34. #include <echonest5/Config.h>
  35. using namespace Tomahawk;
  36. EchonestCatalogSynchronizer* EchonestCatalogSynchronizer::s_instance = 0;
  37. EchonestCatalogSynchronizer::EchonestCatalogSynchronizer( QObject *parent )
  38. : QObject( parent )
  39. {
  40. m_syncing = TomahawkSettings::instance()->enableEchonestCatalogs();
  41. qRegisterMetaType<QList<QStringList> >("QList<QStringList>");
  42. connect( TomahawkSettings::instance(), SIGNAL( changed() ), this, SLOT( checkSettingsChanged() ) );
  43. connect( SourceList::instance()->getLocal()->dbCollection().data(), SIGNAL( tracksAdded( QList<unsigned int> ) ), this, SLOT( tracksAdded( QList<unsigned int> ) ), Qt::QueuedConnection );
  44. connect( SourceList::instance()->getLocal()->dbCollection().data(), SIGNAL( tracksRemoved( QList<unsigned int> ) ), this, SLOT( tracksRemoved( QList<unsigned int> ) ), Qt::QueuedConnection );
  45. const QByteArray artist = TomahawkSettings::instance()->value( "collection/artistCatalog" ).toByteArray();
  46. const QByteArray song = TomahawkSettings::instance()->value( "collection/songCatalog" ).toByteArray();
  47. if ( !artist.isEmpty() )
  48. m_artistCatalog.setId( artist );
  49. if ( !song.isEmpty() )
  50. m_songCatalog.setId( song );
  51. // Sanity check
  52. if ( !song.isEmpty() && !m_syncing )
  53. {
  54. // Not syncing but have a catalog id... lets fix this
  55. QNetworkReply* r = m_songCatalog.deleteCatalog();
  56. connect( r, SIGNAL( finished() ), this, SLOT( catalogDeleted() ) );
  57. r->setProperty( "type", "song" );
  58. }
  59. if ( !artist.isEmpty() && !m_syncing )
  60. {
  61. QNetworkReply* r = m_artistCatalog.deleteCatalog();
  62. connect( r, SIGNAL( finished() ), this, SLOT( catalogDeleted() ) );
  63. r->setProperty( "type", "artist" );
  64. }
  65. }
  66. void
  67. EchonestCatalogSynchronizer::checkSettingsChanged()
  68. {
  69. if ( TomahawkSettings::instance()->enableEchonestCatalogs() && !m_syncing )
  70. {
  71. // enable, and upload whole db
  72. m_syncing = true;
  73. tDebug() << "Echonest Catalog sync pref changed, uploading!!";
  74. uploadDb();
  75. } else if ( !TomahawkSettings::instance()->enableEchonestCatalogs() && m_syncing )
  76. {
  77. tDebug() << "Found echonest change, doing catalog deletes!";
  78. // delete all track nums and catalog ids from our peers
  79. {
  80. DatabaseCommand_SetTrackAttributes* cmd = new DatabaseCommand_SetTrackAttributes( DatabaseCommand_SetTrackAttributes::EchonestCatalogId );
  81. Database::instance()->enqueue( Tomahawk::dbcmd_ptr( cmd ) );
  82. }
  83. {
  84. DatabaseCommand_SetCollectionAttributes* cmd = new DatabaseCommand_SetCollectionAttributes( DatabaseCommand_SetCollectionAttributes::EchonestSongCatalog, true );
  85. Database::instance()->enqueue( Tomahawk::dbcmd_ptr( cmd ) );
  86. }
  87. if ( !m_songCatalog.id().isEmpty() )
  88. {
  89. QNetworkReply* r = m_songCatalog.deleteCatalog();
  90. connect( r, SIGNAL( finished() ), this, SLOT( catalogDeleted() ) );
  91. r->setProperty( "type", "song" );
  92. }
  93. if ( !m_artistCatalog.id().isEmpty() )
  94. {
  95. QNetworkReply* r = m_artistCatalog.deleteCatalog();
  96. connect( r, SIGNAL( finished() ), this, SLOT( catalogDeleted() ) );
  97. r->setProperty( "type", "artist" );
  98. }
  99. m_syncing = false;
  100. }
  101. }
  102. void
  103. EchonestCatalogSynchronizer::catalogDeleted()
  104. {
  105. QNetworkReply* r = qobject_cast< QNetworkReply* >( sender() );
  106. Q_ASSERT( r );
  107. r->deleteLater();
  108. QString toDel = QString( "collection/%1Catalog" ).arg( r->property( "type" ).toString() );
  109. try
  110. {
  111. // HACK libechonest bug, should be a static method but it's not. Doesn't actually use any instance vars though
  112. m_songCatalog.parseDelete( r );
  113. // If we didn't throw, no errors, so clear our config
  114. TomahawkSettings::instance()->setValue( toDel, QString() );
  115. } catch ( const Echonest::ParseError& e )
  116. {
  117. tLog() << "Error in libechonest parsing catalog delete:" << e.what();
  118. }
  119. }
  120. void
  121. EchonestCatalogSynchronizer::uploadDb()
  122. {
  123. // create two catalogs: uuid_song, and uuid_artist.
  124. QNetworkReply* r = Echonest::Catalog::create( QString( "%1_song" ).arg( Database::instance()->impl()->dbid() ), Echonest::CatalogTypes::Song );
  125. connect( r, SIGNAL( finished() ), this, SLOT( songCreateFinished() ) );
  126. // r = Echonest::Catalog::create( QString( "%1_artist" ).arg( Database::instance()->dbid() ), Echonest::CatalogTypes::Artist );
  127. // connect( r, SIGNAL( finished() ), this, SLOT( artistCreateFinished() ) );
  128. }
  129. void
  130. EchonestCatalogSynchronizer::songCreateFinished()
  131. {
  132. QNetworkReply* r = qobject_cast< QNetworkReply* >( sender() );
  133. Q_ASSERT( r );
  134. r->deleteLater();
  135. tDebug() << "Finished creating song catalog, updating data now!!";
  136. try
  137. {
  138. m_songCatalog = Echonest::Catalog::parseCreate( r );
  139. TomahawkSettings::instance()->setValue( "collection/songCatalog", m_songCatalog.id() );
  140. Tomahawk::dbcmd_ptr cmd( new DatabaseCommand_SetCollectionAttributes( DatabaseCommand_SetCollectionAttributes::EchonestSongCatalog,
  141. m_songCatalog.id() ) );
  142. Database::instance()->enqueue( cmd );
  143. } catch ( const Echonest::ParseError& e )
  144. {
  145. tLog() << "Echonest threw an exception parsing song catalog create:" << e.what();
  146. return;
  147. }
  148. QString sql( "SELECT file.id, track.name, artist.name, album.name "
  149. "FROM file, artist, track, file_join "
  150. "LEFT OUTER JOIN album "
  151. "ON file_join.album = album.id "
  152. "WHERE file.id = file_join.file "
  153. "AND file_join.artist = artist.id "
  154. "AND file_join.track = track.id "
  155. "AND file.source IS NULL");
  156. DatabaseCommand_GenericSelect* cmd = new DatabaseCommand_GenericSelect( sql, DatabaseCommand_GenericSelect::Track, true );
  157. connect( cmd, SIGNAL( rawData( QList< QStringList > ) ), this, SLOT( rawTracksAdd( QList< QStringList > ) ) );
  158. Database::instance()->enqueue( Tomahawk::dbcmd_ptr( cmd ) );
  159. }
  160. void
  161. EchonestCatalogSynchronizer::artistCreateFinished()
  162. {
  163. QNetworkReply* r = qobject_cast< QNetworkReply* >( sender() );
  164. Q_ASSERT( r );
  165. r->deleteLater();
  166. // We don't support artist catalogs at the moment
  167. return;
  168. /*
  169. try
  170. {
  171. m_artistCatalog = Echonest::Catalog::parseCreate( r );
  172. TomahawkSettings::instance()->setValue( "collection/artistCatalog", m_artistCatalog.id() );
  173. // Tomahawk::dbcmd_ptr cmd( new DatabaseCommand_SetCollectionAttributes( SourceList::instance()->getLocal(),
  174. // DatabaseCommand_SetCollectionAttributes::EchonestSongCatalog,
  175. // m_songCatalog.id() ) );
  176. // Database::instance()->enqueue( cmd );
  177. } catch ( const Echonest::ParseError& e )
  178. {
  179. tLog() << "Echonest threw an exception parsing artist catalog create:" << e.what();
  180. return;
  181. }*/
  182. }
  183. void
  184. EchonestCatalogSynchronizer::rawTracksAdd( const QList< QStringList >& tracks )
  185. {
  186. tDebug() << "Got raw tracks, num:" << tracks.size();
  187. // int limit = ( tracks.size() < 1000 ) ? tracks.size() : 1000;
  188. int cur = 0;
  189. while ( cur < tracks.size() )
  190. {
  191. int prev = cur;
  192. cur = ( cur + 2000 > tracks.size() ) ? tracks.size() : cur + 2000;
  193. tDebug() << "Enqueueing a batch of tracks to upload to echonest catalog:" << cur - prev;
  194. Echonest::CatalogUpdateEntries entries;
  195. for ( int i = prev; i < cur; i++ )
  196. {
  197. if ( tracks[i][1].isEmpty() || tracks[i][2].isEmpty() )
  198. continue;
  199. entries.append( entryFromTrack( tracks[i], Echonest::CatalogTypes::Update ) );
  200. }
  201. tDebug() << "Done queuing:" << entries.size() << "tracks";
  202. m_queuedUpdates.enqueue( entries );
  203. }
  204. doUploadJob();
  205. }
  206. void
  207. EchonestCatalogSynchronizer::doUploadJob()
  208. {
  209. if ( m_queuedUpdates.isEmpty() )
  210. return;
  211. Echonest::CatalogUpdateEntries entries = m_queuedUpdates.dequeue();
  212. tDebug() << "Updating number of entries:" << entries.count();
  213. QNetworkReply* updateJob = m_songCatalog.update( entries );
  214. connect( updateJob, SIGNAL( finished() ), this, SLOT( songUpdateFinished() ) );
  215. }
  216. Echonest::CatalogUpdateEntry
  217. EchonestCatalogSynchronizer::entryFromTrack( const QStringList& track, Echonest::CatalogTypes::Action action ) const
  218. {
  219. //qDebug() << "UPLOADING:" << track[0] << track[1] << track[2];
  220. Echonest::CatalogUpdateEntry entry;
  221. entry.setAction( action );
  222. entry.setItemId(track[ 0 ].toLatin1() ); // track dbid
  223. entry.setSongName( escape( track[ 1 ] ) );
  224. entry.setArtistName( escape( track[ 2 ] ) );
  225. entry.setRelease( escape( track[ 3 ] ) );
  226. return entry;
  227. }
  228. void
  229. EchonestCatalogSynchronizer::songUpdateFinished()
  230. {
  231. QNetworkReply* r = qobject_cast< QNetworkReply* >( sender() );
  232. Q_ASSERT( r );
  233. r->deleteLater();
  234. try
  235. {
  236. QByteArray ticket = m_songCatalog.parseTicket( r );
  237. QNetworkReply* tJob = m_songCatalog.status( ticket );
  238. connect( tJob, SIGNAL( finished() ), this, SLOT( checkTicket() ) );
  239. } catch ( const Echonest::ParseError& e )
  240. {
  241. tLog() << "Echonest threw an exception parsing catalog update finished:" << e.what();
  242. }
  243. doUploadJob();
  244. }
  245. void
  246. EchonestCatalogSynchronizer::checkTicket()
  247. {
  248. QNetworkReply* r = qobject_cast< QNetworkReply* >( sender() );
  249. Q_ASSERT( r );
  250. r->deleteLater();
  251. try
  252. {
  253. Echonest::CatalogStatus status = m_songCatalog.parseStatus( r );
  254. tLog() << "Catalog status update:" << status.status << status.details << status.items;
  255. } catch ( const Echonest::ParseError& e )
  256. {
  257. tLog() << "Echonest threw an exception parsing catalog create:" << e.what();
  258. return;
  259. }
  260. }
  261. void
  262. EchonestCatalogSynchronizer::tracksAdded( const QList< unsigned int >& tracks )
  263. {
  264. if ( !m_syncing || m_songCatalog.id().isEmpty() || tracks.isEmpty() )
  265. return;
  266. qDebug() << Q_FUNC_INFO << "Got tracks added from db, fetching metadata" << tracks;
  267. // Get the result_ptrs from the tracks
  268. DatabaseCommand_LoadFiles* cmd = new DatabaseCommand_LoadFiles( tracks );
  269. connect( cmd, SIGNAL( results( QList<Tomahawk::result_ptr> ) ), this, SLOT( loadedResults( QList<Tomahawk::result_ptr> ) ) );
  270. Database::instance()->enqueue( Tomahawk::dbcmd_ptr( cmd ) );
  271. }
  272. void
  273. EchonestCatalogSynchronizer::loadedResults( const QList<result_ptr>& results )
  274. {
  275. QList< QStringList > rawTracks;
  276. qDebug() << Q_FUNC_INFO << "Got track metadata..." << results.size();
  277. foreach( const result_ptr& result, results )
  278. {
  279. if ( result.isNull() )
  280. continue;
  281. qDebug() << "Metadata for item:" << result->fileId();
  282. rawTracks << ( QStringList() << QString::number( result->fileId() ) << result->track()->track() << result->track()->artist() << result->track()->album() );
  283. }
  284. rawTracksAdd( rawTracks );
  285. }
  286. void
  287. EchonestCatalogSynchronizer::tracksRemoved( const QList< unsigned int >& trackIds )
  288. {
  289. if ( !m_syncing || m_songCatalog.id().isEmpty() || trackIds.isEmpty() )
  290. return;
  291. Echonest::CatalogUpdateEntries entries;
  292. entries.reserve( trackIds.size() );
  293. foreach ( unsigned int id, trackIds )
  294. {
  295. Echonest::CatalogUpdateEntry e( Echonest::CatalogTypes::Delete );
  296. e.setItemId( QString::number( id ).toLatin1() );
  297. entries.append( e );
  298. }
  299. QNetworkReply* reply = m_songCatalog.update( entries );
  300. connect( reply, SIGNAL( finished() ), this, SLOT( songUpdateFinished() ) );
  301. }
  302. QByteArray
  303. EchonestCatalogSynchronizer::escape( const QString &in ) const
  304. {
  305. // TODO echonest chokes on some chars in the output. But if we percent-encode those chars it works
  306. // We can't percent-encode the whole string, because then any UTF-8 chars that have been url-encoded, fail.
  307. // God this sucks. It's going to break...
  308. QString clean = in;
  309. clean.replace( "&", "%25" );
  310. clean.replace( ";", "%3B" );
  311. return clean.toUtf8();
  312. //return QUrl::toPercentEncoding( in. );
  313. }