/services/sync/modules/engines.js

http://github.com/zpao/v8monkey · JavaScript · 1394 lines · 809 code · 178 blank · 407 comment · 133 complexity · 5c67c74d9c7815e9e3bc7c1a5f169220 MD5 · raw file

  1. /* ***** BEGIN LICENSE BLOCK *****
  2. * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  3. *
  4. * The contents of this file are subject to the Mozilla Public License Version
  5. * 1.1 (the "License"); you may not use this file except in compliance with
  6. * the License. You may obtain a copy of the License at
  7. * http://www.mozilla.org/MPL/
  8. *
  9. * Software distributed under the License is distributed on an "AS IS" basis,
  10. * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11. * for the specific language governing rights and limitations under the
  12. * License.
  13. *
  14. * The Original Code is Bookmarks Sync.
  15. *
  16. * The Initial Developer of the Original Code is Mozilla.
  17. * Portions created by the Initial Developer are Copyright (C) 2007
  18. * the Initial Developer. All Rights Reserved.
  19. *
  20. * Contributor(s):
  21. * Dan Mills <thunder@mozilla.com>
  22. * Myk Melez <myk@mozilla.org>
  23. * Anant Narayanan <anant@kix.in>
  24. * Philipp von Weitershausen <philipp@weitershausen.de>
  25. * Richard Newman <rnewman@mozilla.com>
  26. *
  27. * Alternatively, the contents of this file may be used under the terms of
  28. * either the GNU General Public License Version 2 or later (the "GPL"), or
  29. * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  30. * in which case the provisions of the GPL or the LGPL are applicable instead
  31. * of those above. If you wish to allow use of your version of this file only
  32. * under the terms of either the GPL or the LGPL, and not to allow others to
  33. * use your version of this file under the terms of the MPL, indicate your
  34. * decision by deleting the provisions above and replace them with the notice
  35. * and other provisions required by the GPL or the LGPL. If you do not delete
  36. * the provisions above, a recipient may use your version of this file under
  37. * the terms of any one of the MPL, the GPL or the LGPL.
  38. *
  39. * ***** END LICENSE BLOCK ***** */
  40. const EXPORTED_SYMBOLS = ['Engines', 'Engine', 'SyncEngine',
  41. 'Tracker', 'Store'];
  42. const Cc = Components.classes;
  43. const Ci = Components.interfaces;
  44. const Cr = Components.results;
  45. const Cu = Components.utils;
  46. Cu.import("resource://services-sync/async.js");
  47. Cu.import("resource://services-sync/record.js");
  48. Cu.import("resource://services-sync/constants.js");
  49. Cu.import("resource://services-sync/ext/Observers.js");
  50. Cu.import("resource://services-sync/identity.js");
  51. Cu.import("resource://services-sync/log4moz.js");
  52. Cu.import("resource://services-sync/resource.js");
  53. Cu.import("resource://services-sync/util.js");
  54. Cu.import("resource://services-sync/main.js"); // So we can get to Service for callbacks.
  55. /*
  56. * Trackers are associated with a single engine and deal with
  57. * listening for changes to their particular data type.
  58. *
  59. * There are two things they keep track of:
  60. * 1) A score, indicating how urgently the engine wants to sync
  61. * 2) A list of IDs for all the changed items that need to be synced
  62. * and updating their 'score', indicating how urgently they
  63. * want to sync.
  64. *
  65. */
  66. function Tracker(name) {
  67. name = name || "Unnamed";
  68. this.name = this.file = name.toLowerCase();
  69. this._log = Log4Moz.repository.getLogger("Sync.Tracker." + name);
  70. let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
  71. this._log.level = Log4Moz.Level[level];
  72. this._score = 0;
  73. this._ignored = [];
  74. this.ignoreAll = false;
  75. this.changedIDs = {};
  76. this.loadChangedIDs();
  77. }
  78. Tracker.prototype = {
  79. /*
  80. * Score can be called as often as desired to decide which engines to sync
  81. *
  82. * Valid values for score:
  83. * -1: Do not sync unless the user specifically requests it (almost disabled)
  84. * 0: Nothing has changed
  85. * 100: Please sync me ASAP!
  86. *
  87. * Setting it to other values should (but doesn't currently) throw an exception
  88. */
  89. get score() {
  90. return this._score;
  91. },
  92. set score(value) {
  93. this._score = value;
  94. Observers.notify("weave:engine:score:updated", this.name);
  95. },
  96. // Should be called by service everytime a sync has been done for an engine
  97. resetScore: function T_resetScore() {
  98. this._score = 0;
  99. },
  100. saveChangedIDs: function T_saveChangedIDs() {
  101. Utils.namedTimer(function() {
  102. Utils.jsonSave("changes/" + this.file, this, this.changedIDs);
  103. }, 1000, this, "_lazySave");
  104. },
  105. loadChangedIDs: function T_loadChangedIDs() {
  106. Utils.jsonLoad("changes/" + this.file, this, function(json) {
  107. if (json) {
  108. this.changedIDs = json;
  109. }
  110. });
  111. },
  112. // ignore/unignore specific IDs. Useful for ignoring items that are
  113. // being processed, or that shouldn't be synced.
  114. // But note: not persisted to disk
  115. ignoreID: function T_ignoreID(id) {
  116. this.unignoreID(id);
  117. this._ignored.push(id);
  118. },
  119. unignoreID: function T_unignoreID(id) {
  120. let index = this._ignored.indexOf(id);
  121. if (index != -1)
  122. this._ignored.splice(index, 1);
  123. },
  124. addChangedID: function addChangedID(id, when) {
  125. if (!id) {
  126. this._log.warn("Attempted to add undefined ID to tracker");
  127. return false;
  128. }
  129. if (this.ignoreAll || (id in this._ignored))
  130. return false;
  131. // Default to the current time in seconds if no time is provided
  132. if (when == null)
  133. when = Math.floor(Date.now() / 1000);
  134. // Add/update the entry if we have a newer time
  135. if ((this.changedIDs[id] || -Infinity) < when) {
  136. this._log.trace("Adding changed ID: " + [id, when]);
  137. this.changedIDs[id] = when;
  138. this.saveChangedIDs();
  139. }
  140. return true;
  141. },
  142. removeChangedID: function T_removeChangedID(id) {
  143. if (!id) {
  144. this._log.warn("Attempted to remove undefined ID to tracker");
  145. return false;
  146. }
  147. if (this.ignoreAll || (id in this._ignored))
  148. return false;
  149. if (this.changedIDs[id] != null) {
  150. this._log.trace("Removing changed ID " + id);
  151. delete this.changedIDs[id];
  152. this.saveChangedIDs();
  153. }
  154. return true;
  155. },
  156. clearChangedIDs: function T_clearChangedIDs() {
  157. this._log.trace("Clearing changed ID list");
  158. this.changedIDs = {};
  159. this.saveChangedIDs();
  160. }
  161. };
  162. /**
  163. * The Store serves as the interface between Sync and stored data.
  164. *
  165. * The name "store" is slightly a misnomer because it doesn't actually "store"
  166. * anything. Instead, it serves as a gateway to something that actually does
  167. * the "storing."
  168. *
  169. * The store is responsible for record management inside an engine. It tells
  170. * Sync what items are available for Sync, converts items to and from Sync's
  171. * record format, and applies records from Sync into changes on the underlying
  172. * store.
  173. *
  174. * Store implementations require a number of functions to be implemented. These
  175. * are all documented below.
  176. *
  177. * For stores that deal with many records or which have expensive store access
  178. * routines, it is highly recommended to implement a custom applyIncomingBatch
  179. * and/or applyIncoming function on top of the basic APIs.
  180. */
  181. function Store(name) {
  182. name = name || "Unnamed";
  183. this.name = name.toLowerCase();
  184. this._log = Log4Moz.repository.getLogger("Sync.Store." + name);
  185. let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
  186. this._log.level = Log4Moz.Level[level];
  187. XPCOMUtils.defineLazyGetter(this, "_timer", function() {
  188. return Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
  189. });
  190. }
  191. Store.prototype = {
  192. _sleep: function _sleep(delay) {
  193. let cb = Async.makeSyncCallback();
  194. this._timer.initWithCallback(cb, delay, Ci.nsITimer.TYPE_ONE_SHOT);
  195. Async.waitForSyncCallback(cb);
  196. },
  197. /**
  198. * Apply multiple incoming records against the store.
  199. *
  200. * This is called with a set of incoming records to process. The function
  201. * should look at each record, reconcile with the current local state, and
  202. * make the local changes required to bring its state in alignment with the
  203. * record.
  204. *
  205. * The default implementation simply iterates over all records and calls
  206. * applyIncoming(). Store implementations may overwrite this function
  207. * if desired.
  208. *
  209. * @param records Array of records to apply
  210. * @return Array of record IDs which did not apply cleanly
  211. */
  212. applyIncomingBatch: function applyIncomingBatch(records) {
  213. let failed = [];
  214. for each (let record in records) {
  215. try {
  216. this.applyIncoming(record);
  217. } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
  218. // This kind of exception should have a 'cause' attribute, which is an
  219. // originating exception.
  220. // ex.cause will carry its stack with it when rethrown.
  221. throw ex.cause;
  222. } catch (ex) {
  223. this._log.warn("Failed to apply incoming record " + record.id);
  224. this._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
  225. failed.push(record.id);
  226. }
  227. };
  228. return failed;
  229. },
  230. /**
  231. * Apply a single record against the store.
  232. *
  233. * This takes a single record and makes the local changes required so the
  234. * local state matches what's in the record.
  235. *
  236. * The default implementation calls one of remove(), create(), or update()
  237. * depending on the state obtained from the store itself. Store
  238. * implementations may overwrite this function if desired.
  239. *
  240. * @param record
  241. * Record to apply
  242. */
  243. applyIncoming: function Store_applyIncoming(record) {
  244. if (record.deleted)
  245. this.remove(record);
  246. else if (!this.itemExists(record.id))
  247. this.create(record);
  248. else
  249. this.update(record);
  250. },
  251. // override these in derived objects
  252. /**
  253. * Create an item in the store from a record.
  254. *
  255. * This is called by the default implementation of applyIncoming(). If using
  256. * applyIncomingBatch(), this won't be called unless your store calls it.
  257. *
  258. * @param record
  259. * The store record to create an item from
  260. */
  261. create: function Store_create(record) {
  262. throw "override create in a subclass";
  263. },
  264. /**
  265. * Remove an item in the store from a record.
  266. *
  267. * This is called by the default implementation of applyIncoming(). If using
  268. * applyIncomingBatch(), this won't be called unless your store calls it.
  269. *
  270. * @param record
  271. * The store record to delete an item from
  272. */
  273. remove: function Store_remove(record) {
  274. throw "override remove in a subclass";
  275. },
  276. /**
  277. * Update an item from a record.
  278. *
  279. * This is called by the default implementation of applyIncoming(). If using
  280. * applyIncomingBatch(), this won't be called unless your store calls it.
  281. *
  282. * @param record
  283. * The record to use to update an item from
  284. */
  285. update: function Store_update(record) {
  286. throw "override update in a subclass";
  287. },
  288. /**
  289. * Determine whether a record with the specified ID exists.
  290. *
  291. * Takes a string record ID and returns a booleans saying whether the record
  292. * exists.
  293. *
  294. * @param id
  295. * string record ID
  296. * @return boolean indicating whether record exists locally
  297. */
  298. itemExists: function Store_itemExists(id) {
  299. throw "override itemExists in a subclass";
  300. },
  301. /**
  302. * Create a record from the specified ID.
  303. *
  304. * If the ID is known, the record should be populated with metadata from
  305. * the store. If the ID is not known, the record should be created with the
  306. * delete field set to true.
  307. *
  308. * @param id
  309. * string record ID
  310. * @param collection
  311. * Collection to add record to. This is typically passed into the
  312. * constructor for the newly-created record.
  313. * @return record type for this engine
  314. */
  315. createRecord: function Store_createRecord(id, collection) {
  316. throw "override createRecord in a subclass";
  317. },
  318. /**
  319. * Change the ID of a record.
  320. *
  321. * @param oldID
  322. * string old/current record ID
  323. * @param newID
  324. * string new record ID
  325. */
  326. changeItemID: function Store_changeItemID(oldID, newID) {
  327. throw "override changeItemID in a subclass";
  328. },
  329. /**
  330. * Obtain the set of all known record IDs.
  331. *
  332. * @return Object with ID strings as keys and values of true. The values
  333. * are ignored.
  334. */
  335. getAllIDs: function Store_getAllIDs() {
  336. throw "override getAllIDs in a subclass";
  337. },
  338. /**
  339. * Wipe all data in the store.
  340. *
  341. * This function is called during remote wipes or when replacing local data
  342. * with remote data.
  343. *
  344. * This function should delete all local data that the store is managing. It
  345. * can be thought of as clearing out all state and restoring the "new
  346. * browser" state.
  347. */
  348. wipe: function Store_wipe() {
  349. throw "override wipe in a subclass";
  350. }
  351. };
  352. // Singleton service, holds registered engines
  353. XPCOMUtils.defineLazyGetter(this, "Engines", function() {
  354. return new EngineManagerSvc();
  355. });
  356. function EngineManagerSvc() {
  357. this._engines = {};
  358. this._log = Log4Moz.repository.getLogger("Sync.EngineManager");
  359. this._log.level = Log4Moz.Level[Svc.Prefs.get(
  360. "log.logger.service.engines", "Debug")];
  361. }
  362. EngineManagerSvc.prototype = {
  363. get: function EngMgr_get(name) {
  364. // Return an array of engines if we have an array of names
  365. if (Array.isArray(name)) {
  366. let engines = [];
  367. name.forEach(function(name) {
  368. let engine = this.get(name);
  369. if (engine)
  370. engines.push(engine);
  371. }, this);
  372. return engines;
  373. }
  374. let engine = this._engines[name];
  375. if (!engine) {
  376. this._log.debug("Could not get engine: " + name);
  377. if (Object.keys)
  378. this._log.debug("Engines are: " + JSON.stringify(Object.keys(this._engines)));
  379. }
  380. return engine;
  381. },
  382. getAll: function EngMgr_getAll() {
  383. return [engine for ([name, engine] in Iterator(Engines._engines))];
  384. },
  385. getEnabled: function EngMgr_getEnabled() {
  386. return this.getAll().filter(function(engine) engine.enabled);
  387. },
  388. /**
  389. * Register an Engine to the service. Alternatively, give an array of engine
  390. * objects to register.
  391. *
  392. * @param engineObject
  393. * Engine object used to get an instance of the engine
  394. * @return The engine object if anything failed
  395. */
  396. register: function EngMgr_register(engineObject) {
  397. if (Array.isArray(engineObject))
  398. return engineObject.map(this.register, this);
  399. try {
  400. let engine = new engineObject();
  401. let name = engine.name;
  402. if (name in this._engines)
  403. this._log.error("Engine '" + name + "' is already registered!");
  404. else
  405. this._engines[name] = engine;
  406. }
  407. catch(ex) {
  408. let mesg = ex.message ? ex.message : ex;
  409. let name = engineObject || "";
  410. name = name.prototype || "";
  411. name = name.name || "";
  412. let out = "Could not initialize engine '" + name + "': " + mesg;
  413. this._log.error(out);
  414. return engineObject;
  415. }
  416. },
  417. unregister: function EngMgr_unregister(val) {
  418. let name = val;
  419. if (val instanceof Engine)
  420. name = val.name;
  421. delete this._engines[name];
  422. }
  423. };
  424. function Engine(name) {
  425. this.Name = name || "Unnamed";
  426. this.name = name.toLowerCase();
  427. this._notify = Utils.notify("weave:engine:");
  428. this._log = Log4Moz.repository.getLogger("Sync.Engine." + this.Name);
  429. let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
  430. this._log.level = Log4Moz.Level[level];
  431. this._tracker; // initialize tracker to load previously changed IDs
  432. this._log.debug("Engine initialized");
  433. }
  434. Engine.prototype = {
  435. // _storeObj, and _trackerObj should to be overridden in subclasses
  436. _storeObj: Store,
  437. _trackerObj: Tracker,
  438. // Local 'constant'.
  439. // Signal to the engine that processing further records is pointless.
  440. eEngineAbortApplyIncoming: "error.engine.abort.applyincoming",
  441. get prefName() this.name,
  442. get enabled() Svc.Prefs.get("engine." + this.prefName, false),
  443. set enabled(val) Svc.Prefs.set("engine." + this.prefName, !!val),
  444. get score() this._tracker.score,
  445. get _store() {
  446. let store = new this._storeObj(this.Name);
  447. this.__defineGetter__("_store", function() store);
  448. return store;
  449. },
  450. get _tracker() {
  451. let tracker = new this._trackerObj(this.Name);
  452. this.__defineGetter__("_tracker", function() tracker);
  453. return tracker;
  454. },
  455. sync: function Engine_sync() {
  456. if (!this.enabled)
  457. return;
  458. if (!this._sync)
  459. throw "engine does not implement _sync method";
  460. this._notify("sync", this.name, this._sync)();
  461. },
  462. /**
  463. * Get rid of any local meta-data
  464. */
  465. resetClient: function Engine_resetClient() {
  466. if (!this._resetClient)
  467. throw "engine does not implement _resetClient method";
  468. this._notify("reset-client", this.name, this._resetClient)();
  469. },
  470. _wipeClient: function Engine__wipeClient() {
  471. this.resetClient();
  472. this._log.debug("Deleting all local data");
  473. this._tracker.ignoreAll = true;
  474. this._store.wipe();
  475. this._tracker.ignoreAll = false;
  476. this._tracker.clearChangedIDs();
  477. },
  478. wipeClient: function Engine_wipeClient() {
  479. this._notify("wipe-client", this.name, this._wipeClient)();
  480. }
  481. };
  482. function SyncEngine(name) {
  483. Engine.call(this, name || "SyncEngine");
  484. this.loadToFetch();
  485. this.loadPreviousFailed();
  486. }
  487. // Enumeration to define approaches to handling bad records.
  488. // Attached to the constructor to allow use as a kind of static enumeration.
  489. SyncEngine.kRecoveryStrategy = {
  490. ignore: "ignore",
  491. retry: "retry",
  492. error: "error"
  493. };
  494. SyncEngine.prototype = {
  495. __proto__: Engine.prototype,
  496. _recordObj: CryptoWrapper,
  497. version: 1,
  498. // How many records to pull in a single sync. This is primarily to avoid very
  499. // long first syncs against profiles with many history records.
  500. downloadLimit: null,
  501. // How many records to pull at one time when specifying IDs. This is to avoid
  502. // URI length limitations.
  503. guidFetchBatchSize: DEFAULT_GUID_FETCH_BATCH_SIZE,
  504. mobileGUIDFetchBatchSize: DEFAULT_MOBILE_GUID_FETCH_BATCH_SIZE,
  505. // How many records to process in a single batch.
  506. applyIncomingBatchSize: DEFAULT_STORE_BATCH_SIZE,
  507. get storageURL() Svc.Prefs.get("clusterURL") + SYNC_API_VERSION +
  508. "/" + ID.get("WeaveID").username + "/storage/",
  509. get engineURL() this.storageURL + this.name,
  510. get cryptoKeysURL() this.storageURL + "crypto/keys",
  511. get metaURL() this.storageURL + "meta/global",
  512. get syncID() {
  513. // Generate a random syncID if we don't have one
  514. let syncID = Svc.Prefs.get(this.name + ".syncID", "");
  515. return syncID == "" ? this.syncID = Utils.makeGUID() : syncID;
  516. },
  517. set syncID(value) {
  518. Svc.Prefs.set(this.name + ".syncID", value);
  519. },
  520. /*
  521. * lastSync is a timestamp in server time.
  522. */
  523. get lastSync() {
  524. return parseFloat(Svc.Prefs.get(this.name + ".lastSync", "0"));
  525. },
  526. set lastSync(value) {
  527. // Reset the pref in-case it's a number instead of a string
  528. Svc.Prefs.reset(this.name + ".lastSync");
  529. // Store the value as a string to keep floating point precision
  530. Svc.Prefs.set(this.name + ".lastSync", value.toString());
  531. },
  532. resetLastSync: function SyncEngine_resetLastSync() {
  533. this._log.debug("Resetting " + this.name + " last sync time");
  534. Svc.Prefs.reset(this.name + ".lastSync");
  535. Svc.Prefs.set(this.name + ".lastSync", "0");
  536. this.lastSyncLocal = 0;
  537. },
  538. get toFetch() this._toFetch,
  539. set toFetch(val) {
  540. // Coerce the array to a string for more efficient comparison.
  541. if (val + "" == this._toFetch) {
  542. return;
  543. }
  544. this._toFetch = val;
  545. Utils.namedTimer(function () {
  546. Utils.jsonSave("toFetch/" + this.name, this, val);
  547. }, 0, this, "_toFetchDelay");
  548. },
  549. loadToFetch: function loadToFetch() {
  550. // Initialize to empty if there's no file
  551. this._toFetch = [];
  552. Utils.jsonLoad("toFetch/" + this.name, this, function(toFetch) {
  553. if (toFetch) {
  554. this._toFetch = toFetch;
  555. }
  556. });
  557. },
  558. get previousFailed() this._previousFailed,
  559. set previousFailed(val) {
  560. // Coerce the array to a string for more efficient comparison.
  561. if (val + "" == this._previousFailed) {
  562. return;
  563. }
  564. this._previousFailed = val;
  565. Utils.namedTimer(function () {
  566. Utils.jsonSave("failed/" + this.name, this, val);
  567. }, 0, this, "_previousFailedDelay");
  568. },
  569. loadPreviousFailed: function loadPreviousFailed() {
  570. // Initialize to empty if there's no file
  571. this._previousFailed = [];
  572. Utils.jsonLoad("failed/" + this.name, this, function(previousFailed) {
  573. if (previousFailed) {
  574. this._previousFailed = previousFailed;
  575. }
  576. });
  577. },
  578. /*
  579. * lastSyncLocal is a timestamp in local time.
  580. */
  581. get lastSyncLocal() {
  582. return parseInt(Svc.Prefs.get(this.name + ".lastSyncLocal", "0"), 10);
  583. },
  584. set lastSyncLocal(value) {
  585. // Store as a string because pref can only store C longs as numbers.
  586. Svc.Prefs.set(this.name + ".lastSyncLocal", value.toString());
  587. },
  588. /*
  589. * Returns a mapping of IDs -> changed timestamp. Engine implementations
  590. * can override this method to bypass the tracker for certain or all
  591. * changed items.
  592. */
  593. getChangedIDs: function getChangedIDs() {
  594. return this._tracker.changedIDs;
  595. },
  596. // Create a new record using the store and add in crypto fields
  597. _createRecord: function SyncEngine__createRecord(id) {
  598. let record = this._store.createRecord(id, this.name);
  599. record.id = id;
  600. record.collection = this.name;
  601. return record;
  602. },
  603. // Any setup that needs to happen at the beginning of each sync.
  604. _syncStartup: function SyncEngine__syncStartup() {
  605. // Determine if we need to wipe on outdated versions
  606. let metaGlobal = Records.get(this.metaURL);
  607. let engines = metaGlobal.payload.engines || {};
  608. let engineData = engines[this.name] || {};
  609. let needsWipe = false;
  610. // Assume missing versions are 0 and wipe the server
  611. if ((engineData.version || 0) < this.version) {
  612. this._log.debug("Old engine data: " + [engineData.version, this.version]);
  613. // Prepare to clear the server and upload everything
  614. needsWipe = true;
  615. this.syncID = "";
  616. // Set the newer version and newly generated syncID
  617. engineData.version = this.version;
  618. engineData.syncID = this.syncID;
  619. // Put the new data back into meta/global and mark for upload
  620. engines[this.name] = engineData;
  621. metaGlobal.payload.engines = engines;
  622. metaGlobal.changed = true;
  623. }
  624. // Don't sync this engine if the server has newer data
  625. else if (engineData.version > this.version) {
  626. let error = new String("New data: " + [engineData.version, this.version]);
  627. error.failureCode = VERSION_OUT_OF_DATE;
  628. throw error;
  629. }
  630. // Changes to syncID mean we'll need to upload everything
  631. else if (engineData.syncID != this.syncID) {
  632. this._log.debug("Engine syncIDs: " + [engineData.syncID, this.syncID]);
  633. this.syncID = engineData.syncID;
  634. this._resetClient();
  635. };
  636. // Delete any existing data and reupload on bad version or missing meta.
  637. // No crypto component here...? We could regenerate per-collection keys...
  638. if (needsWipe) {
  639. this.wipeServer();
  640. }
  641. // Save objects that need to be uploaded in this._modified. We also save
  642. // the timestamp of this fetch in this.lastSyncLocal. As we successfully
  643. // upload objects we remove them from this._modified. If an error occurs
  644. // or any objects fail to upload, they will remain in this._modified. At
  645. // the end of a sync, or after an error, we add all objects remaining in
  646. // this._modified to the tracker.
  647. this.lastSyncLocal = Date.now();
  648. if (this.lastSync) {
  649. this._modified = this.getChangedIDs();
  650. } else {
  651. // Mark all items to be uploaded, but treat them as changed from long ago
  652. this._log.debug("First sync, uploading all items");
  653. this._modified = {};
  654. for (let id in this._store.getAllIDs()) {
  655. this._modified[id] = 0;
  656. }
  657. }
  658. // Clear the tracker now. If the sync fails we'll add the ones we failed
  659. // to upload back.
  660. this._tracker.clearChangedIDs();
  661. this._log.info(Object.keys(this._modified).length +
  662. " outgoing items pre-reconciliation");
  663. // Keep track of what to delete at the end of sync
  664. this._delete = {};
  665. },
  666. // Process incoming records
  667. _processIncoming: function SyncEngine__processIncoming() {
  668. this._log.trace("Downloading & applying server changes");
  669. // Figure out how many total items to fetch this sync; do less on mobile.
  670. let batchSize = Infinity;
  671. let newitems = new Collection(this.engineURL, this._recordObj);
  672. let isMobile = (Svc.Prefs.get("client.type") == "mobile");
  673. if (isMobile) {
  674. batchSize = MOBILE_BATCH_SIZE;
  675. }
  676. newitems.newer = this.lastSync;
  677. newitems.full = true;
  678. newitems.limit = batchSize;
  679. // applied => number of items that should be applied.
  680. // failed => number of items that failed in this sync.
  681. // newFailed => number of items that failed for the first time in this sync.
  682. // reconciled => number of items that were reconciled.
  683. let count = {applied: 0, failed: 0, newFailed: 0, reconciled: 0};
  684. let handled = [];
  685. let applyBatch = [];
  686. let failed = [];
  687. let failedInPreviousSync = this.previousFailed;
  688. let fetchBatch = Utils.arrayUnion(this.toFetch, failedInPreviousSync);
  689. // Reset previousFailed for each sync since previously failed items may not fail again.
  690. this.previousFailed = [];
  691. // Used (via exceptions) to allow the record handler/reconciliation/etc.
  692. // methods to signal that they would like processing of incoming records to
  693. // cease.
  694. let aborting = undefined;
  695. function doApplyBatch() {
  696. this._tracker.ignoreAll = true;
  697. try {
  698. failed = failed.concat(this._store.applyIncomingBatch(applyBatch));
  699. } catch (ex) {
  700. // Catch any error that escapes from applyIncomingBatch. At present
  701. // those will all be abort events.
  702. this._log.warn("Got exception " + Utils.exceptionStr(ex) +
  703. ", aborting processIncoming.");
  704. aborting = ex;
  705. }
  706. this._tracker.ignoreAll = false;
  707. applyBatch = [];
  708. }
  709. function doApplyBatchAndPersistFailed() {
  710. // Apply remaining batch.
  711. if (applyBatch.length) {
  712. doApplyBatch.call(this);
  713. }
  714. // Persist failed items so we refetch them.
  715. if (failed.length) {
  716. this.previousFailed = Utils.arrayUnion(failed, this.previousFailed);
  717. count.failed += failed.length;
  718. this._log.debug("Records that failed to apply: " + failed);
  719. failed = [];
  720. }
  721. }
  722. // Not binding this method to 'this' for performance reasons. It gets
  723. // called for every incoming record.
  724. let self = this;
  725. newitems.recordHandler = function(item) {
  726. if (aborting) {
  727. return;
  728. }
  729. // Grab a later last modified if possible
  730. if (self.lastModified == null || item.modified > self.lastModified)
  731. self.lastModified = item.modified;
  732. // Track the collection for the WBO.
  733. item.collection = self.name;
  734. // Remember which records were processed
  735. handled.push(item.id);
  736. try {
  737. try {
  738. item.decrypt();
  739. } catch (ex if Utils.isHMACMismatch(ex)) {
  740. let strategy = self.handleHMACMismatch(item, true);
  741. if (strategy == SyncEngine.kRecoveryStrategy.retry) {
  742. // You only get one retry.
  743. try {
  744. // Try decrypting again, typically because we've got new keys.
  745. self._log.info("Trying decrypt again...");
  746. item.decrypt();
  747. strategy = null;
  748. } catch (ex if Utils.isHMACMismatch(ex)) {
  749. strategy = self.handleHMACMismatch(item, false);
  750. }
  751. }
  752. switch (strategy) {
  753. case null:
  754. // Retry succeeded! No further handling.
  755. break;
  756. case SyncEngine.kRecoveryStrategy.retry:
  757. self._log.debug("Ignoring second retry suggestion.");
  758. // Fall through to error case.
  759. case SyncEngine.kRecoveryStrategy.error:
  760. self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
  761. failed.push(item.id);
  762. return;
  763. case SyncEngine.kRecoveryStrategy.ignore:
  764. self._log.debug("Ignoring record " + item.id +
  765. " with bad HMAC: already handled.");
  766. return;
  767. }
  768. }
  769. } catch (ex) {
  770. self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
  771. failed.push(item.id);
  772. return;
  773. }
  774. let shouldApply;
  775. try {
  776. shouldApply = self._reconcile(item);
  777. } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
  778. self._log.warn("Reconciliation failed: aborting incoming processing.");
  779. failed.push(item.id);
  780. aborting = ex.cause;
  781. } catch (ex) {
  782. self._log.warn("Failed to reconcile incoming record " + item.id);
  783. self._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
  784. failed.push(item.id);
  785. return;
  786. }
  787. if (shouldApply) {
  788. count.applied++;
  789. applyBatch.push(item);
  790. } else {
  791. count.reconciled++;
  792. self._log.trace("Skipping reconciled incoming item " + item.id);
  793. }
  794. if (applyBatch.length == self.applyIncomingBatchSize) {
  795. doApplyBatch.call(self);
  796. }
  797. self._store._sleep(0);
  798. };
  799. // Only bother getting data from the server if there's new things
  800. if (this.lastModified == null || this.lastModified > this.lastSync) {
  801. let resp = newitems.get();
  802. doApplyBatchAndPersistFailed.call(this);
  803. if (!resp.success) {
  804. resp.failureCode = ENGINE_DOWNLOAD_FAIL;
  805. throw resp;
  806. }
  807. if (aborting) {
  808. throw aborting;
  809. }
  810. }
  811. // Mobile: check if we got the maximum that we requested; get the rest if so.
  812. if (handled.length == newitems.limit) {
  813. let guidColl = new Collection(this.engineURL);
  814. // Sort and limit so that on mobile we only get the last X records.
  815. guidColl.limit = this.downloadLimit;
  816. guidColl.newer = this.lastSync;
  817. // index: Orders by the sortindex descending (highest weight first).
  818. guidColl.sort = "index";
  819. let guids = guidColl.get();
  820. if (!guids.success)
  821. throw guids;
  822. // Figure out which guids weren't just fetched then remove any guids that
  823. // were already waiting and prepend the new ones
  824. let extra = Utils.arraySub(guids.obj, handled);
  825. if (extra.length > 0) {
  826. fetchBatch = Utils.arrayUnion(extra, fetchBatch);
  827. this.toFetch = Utils.arrayUnion(extra, this.toFetch);
  828. }
  829. }
  830. // Fast-foward the lastSync timestamp since we have stored the
  831. // remaining items in toFetch.
  832. if (this.lastSync < this.lastModified) {
  833. this.lastSync = this.lastModified;
  834. }
  835. // Process any backlog of GUIDs.
  836. // At this point we impose an upper limit on the number of items to fetch
  837. // in a single request, even for desktop, to avoid hitting URI limits.
  838. batchSize = isMobile ? this.mobileGUIDFetchBatchSize :
  839. this.guidFetchBatchSize;
  840. while (fetchBatch.length && !aborting) {
  841. // Reuse the original query, but get rid of the restricting params
  842. // and batch remaining records.
  843. newitems.limit = 0;
  844. newitems.newer = 0;
  845. newitems.ids = fetchBatch.slice(0, batchSize);
  846. // Reuse the existing record handler set earlier
  847. let resp = newitems.get();
  848. if (!resp.success) {
  849. resp.failureCode = ENGINE_DOWNLOAD_FAIL;
  850. throw resp;
  851. }
  852. // This batch was successfully applied. Not using
  853. // doApplyBatchAndPersistFailed() here to avoid writing toFetch twice.
  854. fetchBatch = fetchBatch.slice(batchSize);
  855. this.toFetch = Utils.arraySub(this.toFetch, newitems.ids);
  856. this.previousFailed = Utils.arrayUnion(this.previousFailed, failed);
  857. if (failed.length) {
  858. count.failed += failed.length;
  859. this._log.debug("Records that failed to apply: " + failed);
  860. }
  861. failed = [];
  862. if (aborting) {
  863. throw aborting;
  864. }
  865. if (this.lastSync < this.lastModified) {
  866. this.lastSync = this.lastModified;
  867. }
  868. }
  869. // Apply remaining items.
  870. doApplyBatchAndPersistFailed.call(this);
  871. count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length;
  872. this._log.info(["Records:",
  873. count.applied, "applied,",
  874. count.failed, "failed to apply,",
  875. count.newFailed, "newly failed to apply,",
  876. count.reconciled, "reconciled."].join(" "));
  877. Observers.notify("weave:engine:sync:applied", count, this.name);
  878. },
  879. /**
  880. * Find a GUID of an item that is a duplicate of the incoming item but happens
  881. * to have a different GUID
  882. *
  883. * @return GUID of the similar item; falsy otherwise
  884. */
  885. _findDupe: function _findDupe(item) {
  886. // By default, assume there's no dupe items for the engine
  887. },
  888. _deleteId: function _deleteId(id) {
  889. this._tracker.removeChangedID(id);
  890. // Remember this id to delete at the end of sync
  891. if (this._delete.ids == null)
  892. this._delete.ids = [id];
  893. else
  894. this._delete.ids.push(id);
  895. },
  896. /**
  897. * Reconcile incoming record with local state.
  898. *
  899. * This function essentially determines whether to apply an incoming record.
  900. *
  901. * @param item
  902. * Record from server to be tested for application.
  903. * @return boolean
  904. * Truthy if incoming record should be applied. False if not.
  905. */
  906. _reconcile: function _reconcile(item) {
  907. if (this._log.level <= Log4Moz.Level.Trace) {
  908. this._log.trace("Incoming: " + item);
  909. }
  910. // We start reconciling by collecting a bunch of state. We do this here
  911. // because some state may change during the course of this function and we
  912. // need to operate on the original values.
  913. let existsLocally = this._store.itemExists(item.id);
  914. let locallyModified = item.id in this._modified;
  915. // TODO Handle clock drift better. Tracked in bug 721181.
  916. let remoteAge = AsyncResource.serverTime - item.modified;
  917. let localAge = locallyModified ?
  918. (Date.now() / 1000 - this._modified[item.id]) : null;
  919. let remoteIsNewer = remoteAge < localAge;
  920. this._log.trace("Reconciling " + item.id + ". exists=" +
  921. existsLocally + "; modified=" + locallyModified +
  922. "; local age=" + localAge + "; incoming age=" +
  923. remoteAge);
  924. // We handle deletions first so subsequent logic doesn't have to check
  925. // deleted flags.
  926. if (item.deleted) {
  927. // If the item doesn't exist locally, there is nothing for us to do. We
  928. // can't check for duplicates because the incoming record has no data
  929. // which can be used for duplicate detection.
  930. if (!existsLocally) {
  931. this._log.trace("Ignoring incoming item because it was deleted and " +
  932. "the item does not exist locally.");
  933. return false;
  934. }
  935. // We decide whether to process the deletion by comparing the record
  936. // ages. If the item is not modified locally, the remote side wins and
  937. // the deletion is processed. If it is modified locally, we take the
  938. // newer record.
  939. if (!locallyModified) {
  940. this._log.trace("Applying incoming delete because the local item " +
  941. "exists and isn't modified.");
  942. return true;
  943. }
  944. // TODO As part of bug 720592, determine whether we should do more here.
  945. // In the case where the local changes are newer, it is quite possible
  946. // that the local client will restore data a remote client had tried to
  947. // delete. There might be a good reason for that delete and it might be
  948. // enexpected for this client to restore that data.
  949. this._log.trace("Incoming record is deleted but we had local changes. " +
  950. "Applying the youngest record.");
  951. return remoteIsNewer;
  952. }
  953. // At this point the incoming record is not for a deletion and must have
  954. // data. If the incoming record does not exist locally, we check for a local
  955. // duplicate existing under a different ID. The default implementation of
  956. // _findDupe() is empty, so engines have to opt in to this functionality.
  957. //
  958. // If we find a duplicate, we change the local ID to the incoming ID and we
  959. // refresh the metadata collected above. See bug 710448 for the history
  960. // of this logic.
  961. if (!existsLocally) {
  962. let dupeID = this._findDupe(item);
  963. if (dupeID) {
  964. this._log.trace("Local item " + dupeID + " is a duplicate for " +
  965. "incoming item " + item.id);
  966. // The local, duplicate ID is always deleted on the server.
  967. this._deleteId(dupeID);
  968. // The current API contract does not mandate that the ID returned by
  969. // _findDupe() actually exists. Therefore, we have to perform this
  970. // check.
  971. existsLocally = this._store.itemExists(dupeID);
  972. // We unconditionally change the item's ID in case the engine knows of
  973. // an item but doesn't expose it through itemExists. If the API
  974. // contract were stronger, this could be changed.
  975. this._log.debug("Switching local ID to incoming: " + dupeID + " -> " +
  976. item.id);
  977. this._store.changeItemID(dupeID, item.id);
  978. // If the local item was modified, we carry its metadata forward so
  979. // appropriate reconciling can be performed.
  980. if (dupeID in this._modified) {
  981. locallyModified = true;
  982. localAge = Date.now() / 1000 - this._modified[dupeID];
  983. remoteIsNewer = remoteAge < localAge;
  984. this._modified[item.id] = this._modified[dupeID];
  985. delete this._modified[dupeID];
  986. } else {
  987. locallyModified = false;
  988. localAge = null;
  989. }
  990. this._log.debug("Local item after duplication: age=" + localAge +
  991. "; modified=" + locallyModified + "; exists=" +
  992. existsLocally);
  993. } else {
  994. this._log.trace("No duplicate found for incoming item: " + item.id);
  995. }
  996. }
  997. // At this point we've performed duplicate detection. But, nothing here
  998. // should depend on duplicate detection as the above should have updated
  999. // state seamlessly.
  1000. if (!existsLocally) {
  1001. // If the item doesn't exist locally and we have no local modifications
  1002. // to the item (implying that it was not deleted), always apply the remote
  1003. // item.
  1004. if (!locallyModified) {
  1005. this._log.trace("Applying incoming because local item does not exist " +
  1006. "and was not deleted.");
  1007. return true;
  1008. }
  1009. // If the item was modified locally but isn't present, it must have
  1010. // been deleted. If the incoming record is younger, we restore from
  1011. // that record.
  1012. if (remoteIsNewer) {
  1013. this._log.trace("Applying incoming because local item was deleted " +
  1014. "before the incoming item was changed.");
  1015. delete this._modified[item.id];
  1016. return true;
  1017. }
  1018. this._log.trace("Ignoring incoming item because the local item's " +
  1019. "deletion is newer.");
  1020. return false;
  1021. }
  1022. // If the remote and local records are the same, there is nothing to be
  1023. // done, so we don't do anything. In the ideal world, this logic wouldn't
  1024. // be here and the engine would take a record and apply it. The reason we
  1025. // want to defer this logic is because it would avoid a redundant and
  1026. // possibly expensive dip into the storage layer to query item state.
  1027. // This should get addressed in the async rewrite, so we ignore it for now.
  1028. let localRecord = this._createRecord(item.id);
  1029. let recordsEqual = Utils.deepEquals(item.cleartext,
  1030. localRecord.cleartext);
  1031. // If the records are the same, we don't need to do anything. This does
  1032. // potentially throw away a local modification time. But, if the records
  1033. // are the same, does it matter?
  1034. if (recordsEqual) {
  1035. this._log.trace("Ignoring incoming item because the local item is " +
  1036. "identical.");
  1037. delete this._modified[item.id];
  1038. return false;
  1039. }
  1040. // At this point the records are different.
  1041. // If we have no local modifications, always take the server record.
  1042. if (!locallyModified) {
  1043. this._log.trace("Applying incoming record because no local conflicts.");
  1044. return true;
  1045. }
  1046. // At this point, records are different and the local record is modified.
  1047. // We resolve conflicts by record age, where the newest one wins. This does
  1048. // result in data loss and should be handled by giving the engine an
  1049. // opportunity to merge the records. Bug 720592 tracks this feature.
  1050. this._log.warn("DATA LOSS: Both local and remote changes to record: " +
  1051. item.id);
  1052. return remoteIsNewer;
  1053. },
  1054. // Upload outgoing records
  1055. _uploadOutgoing: function SyncEngine__uploadOutgoing() {
  1056. this._log.trace("Uploading local changes to server.");
  1057. let modifiedIDs = Object.keys(this._modified);
  1058. if (modifiedIDs.length) {
  1059. this._log.trace("Preparing " + modifiedIDs.length +
  1060. " outgoing records");
  1061. // collection we'll upload
  1062. let up = new Collection(this.engineURL);
  1063. let count = 0;
  1064. // Upload what we've got so far in the collection
  1065. let doUpload = Utils.bind2(this, function(desc) {
  1066. this._log.info("Uploading " + desc + " of " + modifiedIDs.length +
  1067. " records");
  1068. let resp = up.post();
  1069. if (!resp.success) {
  1070. this._log.debug("Uploading records failed: " + resp);
  1071. resp.failureCode = ENGINE_UPLOAD_FAIL;
  1072. throw resp;
  1073. }
  1074. // Update server timestamp from the upload.
  1075. let modified = resp.headers["x-weave-timestamp"];
  1076. if (modified > this.lastSync)
  1077. this.lastSync = modified;
  1078. let failed_ids = Object.keys(resp.obj.failed);
  1079. if (failed_ids.length)
  1080. this._log.debug("Records that will be uploaded again because "
  1081. + "the server couldn't store them: "
  1082. + failed_ids.join(", "));
  1083. // Clear successfully uploaded objects.
  1084. for each (let id in resp.obj.success) {
  1085. delete this._modified[id];
  1086. }
  1087. up.clearRecords();
  1088. });
  1089. for each (let id in modifiedIDs) {
  1090. try {
  1091. let out = this._createRecord(id);
  1092. if (this._log.level <= Log4Moz.Level.Trace)
  1093. this._log.trace("Outgoing: " + out);
  1094. out.encrypt();
  1095. up.pushData(out);
  1096. }
  1097. catch(ex) {
  1098. this._log.warn("Error creating record: " + Utils.exceptionStr(ex));
  1099. }
  1100. // Partial upload
  1101. if ((++count % MAX_UPLOAD_RECORDS) == 0)
  1102. doUpload((count - MAX_UPLOAD_RECORDS) + " - " + count + " out");
  1103. this._store._sleep(0);
  1104. }
  1105. // Final upload
  1106. if (count % MAX_UPLOAD_RECORDS > 0)
  1107. doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all");
  1108. }
  1109. },
  1110. // Any cleanup necessary.
  1111. // Save the current snapshot so as to calculate changes at next sync
  1112. _syncFinish: function SyncEngine__syncFinish() {
  1113. this._log.trace("Finishing up sync");
  1114. this._tracker.resetScore();
  1115. let doDelete = Utils.bind2(this, function(key, val) {
  1116. let coll = new Collection(this.engineURL, this._recordObj);
  1117. coll[key] = val;
  1118. coll.delete();
  1119. });
  1120. for (let [key, val] in Iterator(this._delete)) {
  1121. // Remove the key for future uses
  1122. delete this._delete[key];
  1123. // Send a simple delete for the property
  1124. if (key != "ids" || val.length <= 100)
  1125. doDelete(key, val);
  1126. else {
  1127. // For many ids, split into chunks of at most 100
  1128. while (val.length > 0) {
  1129. doDelete(key, val.slice(0, 100));
  1130. val = val.slice(100);
  1131. }
  1132. }
  1133. }
  1134. },
  1135. _syncCleanup: function _syncCleanup() {
  1136. if (!this._modified)
  1137. return;
  1138. // Mark failed WBOs as changed again so they are reuploaded next time.
  1139. for (let [id, when] in Iterator(this._modified)) {
  1140. this._tracker.addChangedID(id, when);
  1141. }
  1142. this._modified = {};
  1143. },
  1144. _sync: function SyncEngine__sync() {
  1145. try {
  1146. this._syncStartup();
  1147. Observers.notify("weave:engine:sync:status", "process-incoming");
  1148. this._processIncoming();
  1149. Observers.notify("weave:engine:sync:status", "upload-outgoing");
  1150. this._uploadOutgoing();
  1151. this._syncFinish();
  1152. } finally {
  1153. this._syncCleanup();
  1154. }
  1155. },
  1156. canDecrypt: function canDecrypt() {
  1157. // Report failure even if there's nothing to decrypt
  1158. let canDecrypt = false;
  1159. // Fetch the most recently uploaded record and try to decrypt it
  1160. let test = new Collection(this.engineURL, this._recordObj);
  1161. test.limit = 1;
  1162. test.sort = "newest";
  1163. test.full = true;
  1164. test.recordHandler = function(record) {
  1165. record.decrypt();
  1166. canDecrypt = true;
  1167. };
  1168. // Any failure fetching/decrypting will just result in false
  1169. try {
  1170. this._log.trace("Trying to decrypt a record from the server..");
  1171. test.get();
  1172. }
  1173. catch(ex) {
  1174. this._log.debug("Failed test decrypt: " + Utils.exceptionStr(ex));
  1175. }
  1176. return canDecrypt;
  1177. },
  1178. _resetClient: function SyncEngine__resetClient() {
  1179. this.resetLastSync();
  1180. this.previousFailed = [];
  1181. this.toFetch = [];
  1182. },
  1183. wipeServer: function wipeServer() {
  1184. let response = new Resource(this.engineURL).delete();
  1185. if (response.status != 200 && response.status != 404) {
  1186. throw response;
  1187. }
  1188. this._resetClient();
  1189. },
  1190. removeClientData: function removeClientData() {
  1191. // Implement this method in engines that store client specific data
  1192. // on the server.
  1193. },
  1194. /*
  1195. * Decide on (and partially effect) an error-handling strategy.
  1196. *
  1197. * Asks the Service to respond to an HMAC error, which might result in keys
  1198. * being downloaded. That call returns true if an action which might allow a
  1199. * retry to occur.
  1200. *
  1201. * If `mayRetry` is truthy, and the Service suggests a retry,
  1202. * handleHMACMismatch returns kRecoveryStrategy.retry. Otherwise, it returns
  1203. * kRecoveryStrategy.error.
  1204. *
  1205. * Subclasses of SyncEngine can override this method to allow for different
  1206. * behavior -- e.g., to delete and ignore erroneous entries.
  1207. *
  1208. * All return values will be part of the kRecoveryStrategy enumeration.
  1209. */
  1210. handleHMACMismatch: function handleHMACMismatch(item, mayRetry) {
  1211. // By default we either try again, or bail out noisily.
  1212. return (Weave.Service.handleHMACEvent() && mayRetry) ?
  1213. SyncEngine.kRecoveryStrategy.retry :
  1214. SyncEngine.kRecoveryStrategy.error;
  1215. }
  1216. };