/services/sync/modules/engines.js

   1/* ***** BEGIN LICENSE BLOCK *****
   2 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 *
   4 * The contents of this file are subject to the Mozilla Public License Version
   5 * 1.1 (the "License"); you may not use this file except in compliance with
   6 * the License. You may obtain a copy of the License at
   7 * http://www.mozilla.org/MPL/
   8 *
   9 * Software distributed under the License is distributed on an "AS IS" basis,
  10 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 * for the specific language governing rights and limitations under the
  12 * License.
  13 *
  14 * The Original Code is Bookmarks Sync.
  15 *
  16 * The Initial Developer of the Original Code is Mozilla.
  17 * Portions created by the Initial Developer are Copyright (C) 2007
  18 * the Initial Developer. All Rights Reserved.
  19 *
  20 * Contributor(s):
  21 *  Dan Mills <thunder@mozilla.com>
  22 *  Myk Melez <myk@mozilla.org>
  23 *  Anant Narayanan <anant@kix.in>
  24 *  Philipp von Weitershausen <philipp@weitershausen.de>
  25 *  Richard Newman <rnewman@mozilla.com>
  26 *
  27 * Alternatively, the contents of this file may be used under the terms of
  28 * either the GNU General Public License Version 2 or later (the "GPL"), or
  29 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  30 * in which case the provisions of the GPL or the LGPL are applicable instead
  31 * of those above. If you wish to allow use of your version of this file only
  32 * under the terms of either the GPL or the LGPL, and not to allow others to
  33 * use your version of this file under the terms of the MPL, indicate your
  34 * decision by deleting the provisions above and replace them with the notice
  35 * and other provisions required by the GPL or the LGPL. If you do not delete
  36 * the provisions above, a recipient may use your version of this file under
  37 * the terms of any one of the MPL, the GPL or the LGPL.
  38 *
  39 * ***** END LICENSE BLOCK ***** */
  40
  41const EXPORTED_SYMBOLS = ['Engines', 'Engine', 'SyncEngine',
  42                          'Tracker', 'Store'];
  43
  44const Cc = Components.classes;
  45const Ci = Components.interfaces;
  46const Cr = Components.results;
  47const Cu = Components.utils;
  48
  49Cu.import("resource://services-sync/async.js");
  50Cu.import("resource://services-sync/record.js");
  51Cu.import("resource://services-sync/constants.js");
  52Cu.import("resource://services-sync/ext/Observers.js");
  53Cu.import("resource://services-sync/identity.js");
  54Cu.import("resource://services-sync/log4moz.js");
  55Cu.import("resource://services-sync/resource.js");
  56Cu.import("resource://services-sync/util.js");
  57
  58Cu.import("resource://services-sync/main.js");    // So we can get to Service for callbacks.
  59
  60/*
  61 * Trackers are associated with a single engine and deal with
  62 * listening for changes to their particular data type.
  63 *
  64 * There are two things they keep track of:
  65 * 1) A score, indicating how urgently the engine wants to sync
  66 * 2) A list of IDs for all the changed items that need to be synced
  67 * and updating their 'score', indicating how urgently they
  68 * want to sync.
  69 *
  70 */
  71function Tracker(name) {
  72  name = name || "Unnamed";
  73  this.name = this.file = name.toLowerCase();
  74
  75  this._log = Log4Moz.repository.getLogger("Sync.Tracker." + name);
  76  let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
  77  this._log.level = Log4Moz.Level[level];
  78
  79  this._score = 0;
  80  this._ignored = [];
  81  this.ignoreAll = false;
  82  this.changedIDs = {};
  83  this.loadChangedIDs();
  84}
  85Tracker.prototype = {
  86  /*
  87   * Score can be called as often as desired to decide which engines to sync
  88   *
  89   * Valid values for score:
  90   * -1: Do not sync unless the user specifically requests it (almost disabled)
  91   * 0: Nothing has changed
  92   * 100: Please sync me ASAP!
  93   *
  94   * Setting it to other values should (but doesn't currently) throw an exception
  95   */
  96  get score() {
  97    return this._score;
  98  },
  99
 100  set score(value) {
 101    this._score = value;
 102    Observers.notify("weave:engine:score:updated", this.name);
 103  },
 104
 105  // Should be called by service everytime a sync has been done for an engine
 106  resetScore: function T_resetScore() {
 107    this._score = 0;
 108  },
 109
 110  saveChangedIDs: function T_saveChangedIDs() {
 111    Utils.namedTimer(function() {
 112      Utils.jsonSave("changes/" + this.file, this, this.changedIDs);
 113    }, 1000, this, "_lazySave");
 114  },
 115
 116  loadChangedIDs: function T_loadChangedIDs() {
 117    Utils.jsonLoad("changes/" + this.file, this, function(json) {
 118      if (json) {
 119        this.changedIDs = json;
 120      }
 121    });
 122  },
 123
 124  // ignore/unignore specific IDs.  Useful for ignoring items that are
 125  // being processed, or that shouldn't be synced.
 126  // But note: not persisted to disk
 127
 128  ignoreID: function T_ignoreID(id) {
 129    this.unignoreID(id);
 130    this._ignored.push(id);
 131  },
 132
 133  unignoreID: function T_unignoreID(id) {
 134    let index = this._ignored.indexOf(id);
 135    if (index != -1)
 136      this._ignored.splice(index, 1);
 137  },
 138
 139  addChangedID: function addChangedID(id, when) {
 140    if (!id) {
 141      this._log.warn("Attempted to add undefined ID to tracker");
 142      return false;
 143    }
 144    if (this.ignoreAll || (id in this._ignored))
 145      return false;
 146
 147    // Default to the current time in seconds if no time is provided
 148    if (when == null)
 149      when = Math.floor(Date.now() / 1000);
 150
 151    // Add/update the entry if we have a newer time
 152    if ((this.changedIDs[id] || -Infinity) < when) {
 153      this._log.trace("Adding changed ID: " + [id, when]);
 154      this.changedIDs[id] = when;
 155      this.saveChangedIDs();
 156    }
 157    return true;
 158  },
 159
 160  removeChangedID: function T_removeChangedID(id) {
 161    if (!id) {
 162      this._log.warn("Attempted to remove undefined ID to tracker");
 163      return false;
 164    }
 165    if (this.ignoreAll || (id in this._ignored))
 166      return false;
 167    if (this.changedIDs[id] != null) {
 168      this._log.trace("Removing changed ID " + id);
 169      delete this.changedIDs[id];
 170      this.saveChangedIDs();
 171    }
 172    return true;
 173  },
 174
 175  clearChangedIDs: function T_clearChangedIDs() {
 176    this._log.trace("Clearing changed ID list");
 177    this.changedIDs = {};
 178    this.saveChangedIDs();
 179  }
 180};
 181
 182
 183
 184/**
 185 * The Store serves as the interface between Sync and stored data.
 186 *
 187 * The name "store" is slightly a misnomer because it doesn't actually "store"
 188 * anything. Instead, it serves as a gateway to something that actually does
 189 * the "storing."
 190 *
 191 * The store is responsible for record management inside an engine. It tells
 192 * Sync what items are available for Sync, converts items to and from Sync's
 193 * record format, and applies records from Sync into changes on the underlying
 194 * store.
 195 *
 196 * Store implementations require a number of functions to be implemented. These
 197 * are all documented below.
 198 *
 199 * For stores that deal with many records or which have expensive store access
 200 * routines, it is highly recommended to implement a custom applyIncomingBatch
 201 * and/or applyIncoming function on top of the basic APIs.
 202 */
 203
 204function Store(name) {
 205  name = name || "Unnamed";
 206  this.name = name.toLowerCase();
 207
 208  this._log = Log4Moz.repository.getLogger("Sync.Store." + name);
 209  let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
 210  this._log.level = Log4Moz.Level[level];
 211
 212  XPCOMUtils.defineLazyGetter(this, "_timer", function() {
 213    return Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
 214  });
 215}
 216Store.prototype = {
 217
 218  _sleep: function _sleep(delay) {
 219    let cb = Async.makeSyncCallback();
 220    this._timer.initWithCallback(cb, delay, Ci.nsITimer.TYPE_ONE_SHOT);
 221    Async.waitForSyncCallback(cb);
 222  },
 223
 224  /**
 225   * Apply multiple incoming records against the store.
 226   *
 227   * This is called with a set of incoming records to process. The function
 228   * should look at each record, reconcile with the current local state, and
 229   * make the local changes required to bring its state in alignment with the
 230   * record.
 231   *
 232   * The default implementation simply iterates over all records and calls
 233   * applyIncoming(). Store implementations may overwrite this function
 234   * if desired.
 235   *
 236   * @param  records Array of records to apply
 237   * @return Array of record IDs which did not apply cleanly
 238   */
 239  applyIncomingBatch: function applyIncomingBatch(records) {
 240    let failed = [];
 241    for each (let record in records) {
 242      try {
 243        this.applyIncoming(record);
 244      } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
 245        // This kind of exception should have a 'cause' attribute, which is an
 246        // originating exception.
 247        // ex.cause will carry its stack with it when rethrown.
 248        throw ex.cause;
 249      } catch (ex) {
 250        this._log.warn("Failed to apply incoming record " + record.id);
 251        this._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
 252        failed.push(record.id);
 253      }
 254    };
 255    return failed;
 256  },
 257
 258  /**
 259   * Apply a single record against the store.
 260   *
 261   * This takes a single record and makes the local changes required so the
 262   * local state matches what's in the record.
 263   *
 264   * The default implementation calls one of remove(), create(), or update()
 265   * depending on the state obtained from the store itself. Store
 266   * implementations may overwrite this function if desired.
 267   *
 268   * @param record
 269   *        Record to apply
 270   */
 271  applyIncoming: function Store_applyIncoming(record) {
 272    if (record.deleted)
 273      this.remove(record);
 274    else if (!this.itemExists(record.id))
 275      this.create(record);
 276    else
 277      this.update(record);
 278  },
 279
 280  // override these in derived objects
 281
 282  /**
 283   * Create an item in the store from a record.
 284   *
 285   * This is called by the default implementation of applyIncoming(). If using
 286   * applyIncomingBatch(), this won't be called unless your store calls it.
 287   *
 288   * @param record
 289   *        The store record to create an item from
 290   */
 291  create: function Store_create(record) {
 292    throw "override create in a subclass";
 293  },
 294
 295  /**
 296   * Remove an item in the store from a record.
 297   *
 298   * This is called by the default implementation of applyIncoming(). If using
 299   * applyIncomingBatch(), this won't be called unless your store calls it.
 300   *
 301   * @param record
 302   *        The store record to delete an item from
 303   */
 304  remove: function Store_remove(record) {
 305    throw "override remove in a subclass";
 306  },
 307
 308  /**
 309   * Update an item from a record.
 310   *
 311   * This is called by the default implementation of applyIncoming(). If using
 312   * applyIncomingBatch(), this won't be called unless your store calls it.
 313   *
 314   * @param record
 315   *        The record to use to update an item from
 316   */
 317  update: function Store_update(record) {
 318    throw "override update in a subclass";
 319  },
 320
 321  /**
 322   * Determine whether a record with the specified ID exists.
 323   *
 324   * Takes a string record ID and returns a booleans saying whether the record
 325   * exists.
 326   *
 327   * @param  id
 328   *         string record ID
 329   * @return boolean indicating whether record exists locally
 330   */
 331  itemExists: function Store_itemExists(id) {
 332    throw "override itemExists in a subclass";
 333  },
 334
 335  /**
 336   * Create a record from the specified ID.
 337   *
 338   * If the ID is known, the record should be populated with metadata from
 339   * the store. If the ID is not known, the record should be created with the
 340   * delete field set to true.
 341   *
 342   * @param  id
 343   *         string record ID
 344   * @param  collection
 345   *         Collection to add record to. This is typically passed into the
 346   *         constructor for the newly-created record.
 347   * @return record type for this engine
 348   */
 349  createRecord: function Store_createRecord(id, collection) {
 350    throw "override createRecord in a subclass";
 351  },
 352
 353  /**
 354   * Change the ID of a record.
 355   *
 356   * @param  oldID
 357   *         string old/current record ID
 358   * @param  newID
 359   *         string new record ID
 360   */
 361  changeItemID: function Store_changeItemID(oldID, newID) {
 362    throw "override changeItemID in a subclass";
 363  },
 364
 365  /**
 366   * Obtain the set of all known record IDs.
 367   *
 368   * @return Object with ID strings as keys and values of true. The values
 369   *         are ignored.
 370   */
 371  getAllIDs: function Store_getAllIDs() {
 372    throw "override getAllIDs in a subclass";
 373  },
 374
 375  /**
 376   * Wipe all data in the store.
 377   *
 378   * This function is called during remote wipes or when replacing local data
 379   * with remote data.
 380   *
 381   * This function should delete all local data that the store is managing. It
 382   * can be thought of as clearing out all state and restoring the "new
 383   * browser" state.
 384   */
 385  wipe: function Store_wipe() {
 386    throw "override wipe in a subclass";
 387  }
 388};
 389
 390
 391// Singleton service, holds registered engines
 392
 393XPCOMUtils.defineLazyGetter(this, "Engines", function() {
 394  return new EngineManagerSvc();
 395});
 396
 397function EngineManagerSvc() {
 398  this._engines = {};
 399  this._log = Log4Moz.repository.getLogger("Sync.EngineManager");
 400  this._log.level = Log4Moz.Level[Svc.Prefs.get(
 401    "log.logger.service.engines", "Debug")];
 402}
 403EngineManagerSvc.prototype = {
 404  get: function EngMgr_get(name) {
 405    // Return an array of engines if we have an array of names
 406    if (Array.isArray(name)) {
 407      let engines = [];
 408      name.forEach(function(name) {
 409        let engine = this.get(name);
 410        if (engine)
 411          engines.push(engine);
 412      }, this);
 413      return engines;
 414    }
 415
 416    let engine = this._engines[name];
 417    if (!engine) {
 418      this._log.debug("Could not get engine: " + name);
 419      if (Object.keys)
 420        this._log.debug("Engines are: " + JSON.stringify(Object.keys(this._engines)));
 421    }
 422    return engine;
 423  },
 424  getAll: function EngMgr_getAll() {
 425    return [engine for ([name, engine] in Iterator(Engines._engines))];
 426  },
 427  getEnabled: function EngMgr_getEnabled() {
 428    return this.getAll().filter(function(engine) engine.enabled);
 429  },
 430  
 431  /**
 432   * Register an Engine to the service. Alternatively, give an array of engine
 433   * objects to register.
 434   *
 435   * @param engineObject
 436   *        Engine object used to get an instance of the engine
 437   * @return The engine object if anything failed
 438   */
 439  register: function EngMgr_register(engineObject) {
 440    if (Array.isArray(engineObject))
 441      return engineObject.map(this.register, this);
 442
 443    try {
 444      let engine = new engineObject();
 445      let name = engine.name;
 446      if (name in this._engines)
 447        this._log.error("Engine '" + name + "' is already registered!");
 448      else
 449        this._engines[name] = engine;
 450    }
 451    catch(ex) {
 452      let mesg = ex.message ? ex.message : ex;
 453      let name = engineObject || "";
 454      name = name.prototype || "";
 455      name = name.name || "";
 456
 457      let out = "Could not initialize engine '" + name + "': " + mesg;
 458      this._log.error(out);
 459
 460      return engineObject;
 461    }
 462  },
 463  unregister: function EngMgr_unregister(val) {
 464    let name = val;
 465    if (val instanceof Engine)
 466      name = val.name;
 467    delete this._engines[name];
 468  }
 469};
 470
 471function Engine(name) {
 472  this.Name = name || "Unnamed";
 473  this.name = name.toLowerCase();
 474
 475  this._notify = Utils.notify("weave:engine:");
 476  this._log = Log4Moz.repository.getLogger("Sync.Engine." + this.Name);
 477  let level = Svc.Prefs.get("log.logger.engine." + this.name, "Debug");
 478  this._log.level = Log4Moz.Level[level];
 479
 480  this._tracker; // initialize tracker to load previously changed IDs
 481  this._log.debug("Engine initialized");
 482}
 483Engine.prototype = {
 484  // _storeObj, and _trackerObj should to be overridden in subclasses
 485  _storeObj: Store,
 486  _trackerObj: Tracker,
 487
 488  // Local 'constant'.
 489  // Signal to the engine that processing further records is pointless.
 490  eEngineAbortApplyIncoming: "error.engine.abort.applyincoming",
 491
 492  get prefName() this.name,
 493  get enabled() Svc.Prefs.get("engine." + this.prefName, false),
 494  set enabled(val) Svc.Prefs.set("engine." + this.prefName, !!val),
 495
 496  get score() this._tracker.score,
 497
 498  get _store() {
 499    let store = new this._storeObj(this.Name);
 500    this.__defineGetter__("_store", function() store);
 501    return store;
 502  },
 503
 504  get _tracker() {
 505    let tracker = new this._trackerObj(this.Name);
 506    this.__defineGetter__("_tracker", function() tracker);
 507    return tracker;
 508  },
 509
 510  sync: function Engine_sync() {
 511    if (!this.enabled)
 512      return;
 513
 514    if (!this._sync)
 515      throw "engine does not implement _sync method";
 516
 517    this._notify("sync", this.name, this._sync)();
 518  },
 519
 520  /**
 521   * Get rid of any local meta-data
 522   */
 523  resetClient: function Engine_resetClient() {
 524    if (!this._resetClient)
 525      throw "engine does not implement _resetClient method";
 526
 527    this._notify("reset-client", this.name, this._resetClient)();
 528  },
 529
 530  _wipeClient: function Engine__wipeClient() {
 531    this.resetClient();
 532    this._log.debug("Deleting all local data");
 533    this._tracker.ignoreAll = true;
 534    this._store.wipe();
 535    this._tracker.ignoreAll = false;
 536    this._tracker.clearChangedIDs();
 537  },
 538
 539  wipeClient: function Engine_wipeClient() {
 540    this._notify("wipe-client", this.name, this._wipeClient)();
 541  }
 542};
 543
 544function SyncEngine(name) {
 545  Engine.call(this, name || "SyncEngine");
 546  this.loadToFetch();
 547  this.loadPreviousFailed();
 548}
 549
 550// Enumeration to define approaches to handling bad records.
 551// Attached to the constructor to allow use as a kind of static enumeration.
 552SyncEngine.kRecoveryStrategy = {
 553  ignore: "ignore",
 554  retry:  "retry",
 555  error:  "error"
 556};
 557
 558SyncEngine.prototype = {
 559  __proto__: Engine.prototype,
 560  _recordObj: CryptoWrapper,
 561  version: 1,
 562  
 563  // How many records to pull in a single sync. This is primarily to avoid very
 564  // long first syncs against profiles with many history records.
 565  downloadLimit: null,
 566  
 567  // How many records to pull at one time when specifying IDs. This is to avoid
 568  // URI length limitations.
 569  guidFetchBatchSize: DEFAULT_GUID_FETCH_BATCH_SIZE,
 570  mobileGUIDFetchBatchSize: DEFAULT_MOBILE_GUID_FETCH_BATCH_SIZE,
 571  
 572  // How many records to process in a single batch.
 573  applyIncomingBatchSize: DEFAULT_STORE_BATCH_SIZE,
 574
 575  get storageURL() Svc.Prefs.get("clusterURL") + SYNC_API_VERSION +
 576    "/" + ID.get("WeaveID").username + "/storage/",
 577
 578  get engineURL() this.storageURL + this.name,
 579
 580  get cryptoKeysURL() this.storageURL + "crypto/keys",
 581
 582  get metaURL() this.storageURL + "meta/global",
 583
 584  get syncID() {
 585    // Generate a random syncID if we don't have one
 586    let syncID = Svc.Prefs.get(this.name + ".syncID", "");
 587    return syncID == "" ? this.syncID = Utils.makeGUID() : syncID;
 588  },
 589  set syncID(value) {
 590    Svc.Prefs.set(this.name + ".syncID", value);
 591  },
 592
 593  /*
 594   * lastSync is a timestamp in server time.
 595   */
 596  get lastSync() {
 597    return parseFloat(Svc.Prefs.get(this.name + ".lastSync", "0"));
 598  },
 599  set lastSync(value) {
 600    // Reset the pref in-case it's a number instead of a string
 601    Svc.Prefs.reset(this.name + ".lastSync");
 602    // Store the value as a string to keep floating point precision
 603    Svc.Prefs.set(this.name + ".lastSync", value.toString());
 604  },
 605  resetLastSync: function SyncEngine_resetLastSync() {
 606    this._log.debug("Resetting " + this.name + " last sync time");
 607    Svc.Prefs.reset(this.name + ".lastSync");
 608    Svc.Prefs.set(this.name + ".lastSync", "0");
 609    this.lastSyncLocal = 0;
 610  },
 611
 612  get toFetch() this._toFetch,
 613  set toFetch(val) {
 614    // Coerce the array to a string for more efficient comparison.
 615    if (val + "" == this._toFetch) {
 616      return;
 617    }
 618    this._toFetch = val;
 619    Utils.namedTimer(function () {
 620      Utils.jsonSave("toFetch/" + this.name, this, val);
 621    }, 0, this, "_toFetchDelay");
 622  },
 623
 624  loadToFetch: function loadToFetch() {
 625    // Initialize to empty if there's no file
 626    this._toFetch = [];
 627    Utils.jsonLoad("toFetch/" + this.name, this, function(toFetch) {
 628      if (toFetch) {
 629        this._toFetch = toFetch;
 630      }
 631    });
 632  },
 633
 634  get previousFailed() this._previousFailed,
 635  set previousFailed(val) {
 636    // Coerce the array to a string for more efficient comparison.
 637    if (val + "" == this._previousFailed) {
 638      return;
 639    }
 640    this._previousFailed = val;
 641    Utils.namedTimer(function () {
 642      Utils.jsonSave("failed/" + this.name, this, val);
 643    }, 0, this, "_previousFailedDelay");
 644  },
 645
 646  loadPreviousFailed: function loadPreviousFailed() {
 647    // Initialize to empty if there's no file
 648    this._previousFailed = [];
 649    Utils.jsonLoad("failed/" + this.name, this, function(previousFailed) {
 650      if (previousFailed) {
 651        this._previousFailed = previousFailed;
 652      }
 653    });
 654  },
 655
 656  /*
 657   * lastSyncLocal is a timestamp in local time.
 658   */
 659  get lastSyncLocal() {
 660    return parseInt(Svc.Prefs.get(this.name + ".lastSyncLocal", "0"), 10);
 661  },
 662  set lastSyncLocal(value) {
 663    // Store as a string because pref can only store C longs as numbers.
 664    Svc.Prefs.set(this.name + ".lastSyncLocal", value.toString());
 665  },
 666
 667  /*
 668   * Returns a mapping of IDs -> changed timestamp. Engine implementations
 669   * can override this method to bypass the tracker for certain or all
 670   * changed items.
 671   */
 672  getChangedIDs: function getChangedIDs() {
 673    return this._tracker.changedIDs;
 674  },
 675
 676  // Create a new record using the store and add in crypto fields
 677  _createRecord: function SyncEngine__createRecord(id) {
 678    let record = this._store.createRecord(id, this.name);
 679    record.id = id;
 680    record.collection = this.name;
 681    return record;
 682  },
 683
 684  // Any setup that needs to happen at the beginning of each sync.
 685  _syncStartup: function SyncEngine__syncStartup() {
 686
 687    // Determine if we need to wipe on outdated versions
 688    let metaGlobal = Records.get(this.metaURL);
 689    let engines = metaGlobal.payload.engines || {};
 690    let engineData = engines[this.name] || {};
 691
 692    let needsWipe = false;
 693
 694    // Assume missing versions are 0 and wipe the server
 695    if ((engineData.version || 0) < this.version) {
 696      this._log.debug("Old engine data: " + [engineData.version, this.version]);
 697
 698      // Prepare to clear the server and upload everything
 699      needsWipe = true;
 700      this.syncID = "";
 701
 702      // Set the newer version and newly generated syncID
 703      engineData.version = this.version;
 704      engineData.syncID = this.syncID;
 705
 706      // Put the new data back into meta/global and mark for upload
 707      engines[this.name] = engineData;
 708      metaGlobal.payload.engines = engines;
 709      metaGlobal.changed = true;
 710    }
 711    // Don't sync this engine if the server has newer data
 712    else if (engineData.version > this.version) {
 713      let error = new String("New data: " + [engineData.version, this.version]);
 714      error.failureCode = VERSION_OUT_OF_DATE;
 715      throw error;
 716    }
 717    // Changes to syncID mean we'll need to upload everything
 718    else if (engineData.syncID != this.syncID) {
 719      this._log.debug("Engine syncIDs: " + [engineData.syncID, this.syncID]);
 720      this.syncID = engineData.syncID;
 721      this._resetClient();
 722    };
 723
 724    // Delete any existing data and reupload on bad version or missing meta.
 725    // No crypto component here...? We could regenerate per-collection keys...
 726    if (needsWipe) {
 727      this.wipeServer();
 728    }
 729
 730    // Save objects that need to be uploaded in this._modified. We also save
 731    // the timestamp of this fetch in this.lastSyncLocal. As we successfully
 732    // upload objects we remove them from this._modified. If an error occurs
 733    // or any objects fail to upload, they will remain in this._modified. At
 734    // the end of a sync, or after an error, we add all objects remaining in
 735    // this._modified to the tracker.
 736    this.lastSyncLocal = Date.now();
 737    if (this.lastSync) {
 738      this._modified = this.getChangedIDs();
 739    } else {
 740      // Mark all items to be uploaded, but treat them as changed from long ago
 741      this._log.debug("First sync, uploading all items");
 742      this._modified = {};
 743      for (let id in this._store.getAllIDs()) {
 744        this._modified[id] = 0;
 745      }
 746    }
 747    // Clear the tracker now. If the sync fails we'll add the ones we failed
 748    // to upload back.
 749    this._tracker.clearChangedIDs();
 750
 751    this._log.info(Object.keys(this._modified).length +
 752                   " outgoing items pre-reconciliation");
 753
 754    // Keep track of what to delete at the end of sync
 755    this._delete = {};
 756  },
 757
 758  // Process incoming records
 759  _processIncoming: function SyncEngine__processIncoming() {
 760    this._log.trace("Downloading & applying server changes");
 761
 762    // Figure out how many total items to fetch this sync; do less on mobile.
 763    let batchSize = Infinity;
 764    let newitems = new Collection(this.engineURL, this._recordObj);
 765    let isMobile = (Svc.Prefs.get("client.type") == "mobile");
 766
 767    if (isMobile) {
 768      batchSize = MOBILE_BATCH_SIZE;
 769    }
 770    newitems.newer = this.lastSync;
 771    newitems.full  = true;
 772    newitems.limit = batchSize;
 773    
 774    // applied    => number of items that should be applied.
 775    // failed     => number of items that failed in this sync.
 776    // newFailed  => number of items that failed for the first time in this sync.
 777    // reconciled => number of items that were reconciled.
 778    let count = {applied: 0, failed: 0, newFailed: 0, reconciled: 0};
 779    let handled = [];
 780    let applyBatch = [];
 781    let failed = [];
 782    let failedInPreviousSync = this.previousFailed;
 783    let fetchBatch = Utils.arrayUnion(this.toFetch, failedInPreviousSync);
 784    // Reset previousFailed for each sync since previously failed items may not fail again.
 785    this.previousFailed = [];
 786
 787    // Used (via exceptions) to allow the record handler/reconciliation/etc.
 788    // methods to signal that they would like processing of incoming records to
 789    // cease.
 790    let aborting = undefined;
 791
 792    function doApplyBatch() {
 793      this._tracker.ignoreAll = true;
 794      try {
 795        failed = failed.concat(this._store.applyIncomingBatch(applyBatch));
 796      } catch (ex) {
 797        // Catch any error that escapes from applyIncomingBatch. At present
 798        // those will all be abort events.
 799        this._log.warn("Got exception " + Utils.exceptionStr(ex) +
 800                       ", aborting processIncoming.");
 801        aborting = ex;
 802      }
 803      this._tracker.ignoreAll = false;
 804      applyBatch = [];
 805    }
 806
 807    function doApplyBatchAndPersistFailed() {
 808      // Apply remaining batch.
 809      if (applyBatch.length) {
 810        doApplyBatch.call(this);
 811      }
 812      // Persist failed items so we refetch them.
 813      if (failed.length) {
 814        this.previousFailed = Utils.arrayUnion(failed, this.previousFailed);
 815        count.failed += failed.length;
 816        this._log.debug("Records that failed to apply: " + failed);
 817        failed = [];
 818      }
 819    }
 820
 821    // Not binding this method to 'this' for performance reasons. It gets
 822    // called for every incoming record.
 823    let self = this;
 824    newitems.recordHandler = function(item) {
 825      if (aborting) {
 826        return;
 827      }
 828
 829      // Grab a later last modified if possible
 830      if (self.lastModified == null || item.modified > self.lastModified)
 831        self.lastModified = item.modified;
 832
 833      // Track the collection for the WBO.
 834      item.collection = self.name;
 835      
 836      // Remember which records were processed
 837      handled.push(item.id);
 838
 839      try {
 840        try {
 841          item.decrypt();
 842        } catch (ex if Utils.isHMACMismatch(ex)) {
 843          let strategy = self.handleHMACMismatch(item, true);
 844          if (strategy == SyncEngine.kRecoveryStrategy.retry) {
 845            // You only get one retry.
 846            try {
 847              // Try decrypting again, typically because we've got new keys.
 848              self._log.info("Trying decrypt again...");
 849              item.decrypt();
 850              strategy = null;
 851            } catch (ex if Utils.isHMACMismatch(ex)) {
 852              strategy = self.handleHMACMismatch(item, false);
 853            }
 854          }
 855          
 856          switch (strategy) {
 857            case null:
 858              // Retry succeeded! No further handling.
 859              break;
 860            case SyncEngine.kRecoveryStrategy.retry:
 861              self._log.debug("Ignoring second retry suggestion.");
 862              // Fall through to error case.
 863            case SyncEngine.kRecoveryStrategy.error:
 864              self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
 865              failed.push(item.id);
 866              return;
 867            case SyncEngine.kRecoveryStrategy.ignore:
 868              self._log.debug("Ignoring record " + item.id +
 869                              " with bad HMAC: already handled.");
 870              return;
 871          }
 872        }
 873      } catch (ex) {
 874        self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
 875        failed.push(item.id);
 876        return;
 877      }
 878
 879      let shouldApply;
 880      try {
 881        shouldApply = self._reconcile(item);
 882      } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
 883        self._log.warn("Reconciliation failed: aborting incoming processing.");
 884        failed.push(item.id);
 885        aborting = ex.cause;
 886      } catch (ex) {
 887        self._log.warn("Failed to reconcile incoming record " + item.id);
 888        self._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
 889        failed.push(item.id);
 890        return;
 891      }
 892
 893      if (shouldApply) {
 894        count.applied++;
 895        applyBatch.push(item);
 896      } else {
 897        count.reconciled++;
 898        self._log.trace("Skipping reconciled incoming item " + item.id);
 899      }
 900
 901      if (applyBatch.length == self.applyIncomingBatchSize) {
 902        doApplyBatch.call(self);
 903      }
 904      self._store._sleep(0);
 905    };
 906
 907    // Only bother getting data from the server if there's new things
 908    if (this.lastModified == null || this.lastModified > this.lastSync) {
 909      let resp = newitems.get();
 910      doApplyBatchAndPersistFailed.call(this);
 911      if (!resp.success) {
 912        resp.failureCode = ENGINE_DOWNLOAD_FAIL;
 913        throw resp;
 914      }
 915
 916      if (aborting) {
 917        throw aborting;
 918      }
 919    }
 920
 921    // Mobile: check if we got the maximum that we requested; get the rest if so.
 922    if (handled.length == newitems.limit) {
 923      let guidColl = new Collection(this.engineURL);
 924      
 925      // Sort and limit so that on mobile we only get the last X records.
 926      guidColl.limit = this.downloadLimit;
 927      guidColl.newer = this.lastSync;
 928
 929      // index: Orders by the sortindex descending (highest weight first).
 930      guidColl.sort  = "index";
 931
 932      let guids = guidColl.get();
 933      if (!guids.success)
 934        throw guids;
 935
 936      // Figure out which guids weren't just fetched then remove any guids that
 937      // were already waiting and prepend the new ones
 938      let extra = Utils.arraySub(guids.obj, handled);
 939      if (extra.length > 0) {
 940        fetchBatch = Utils.arrayUnion(extra, fetchBatch);
 941        this.toFetch = Utils.arrayUnion(extra, this.toFetch);
 942      }
 943    }
 944
 945    // Fast-foward the lastSync timestamp since we have stored the
 946    // remaining items in toFetch.
 947    if (this.lastSync < this.lastModified) {
 948      this.lastSync = this.lastModified;
 949    }
 950
 951    // Process any backlog of GUIDs.
 952    // At this point we impose an upper limit on the number of items to fetch
 953    // in a single request, even for desktop, to avoid hitting URI limits.
 954    batchSize = isMobile ? this.mobileGUIDFetchBatchSize :
 955                           this.guidFetchBatchSize;
 956
 957    while (fetchBatch.length && !aborting) {
 958      // Reuse the original query, but get rid of the restricting params
 959      // and batch remaining records.
 960      newitems.limit = 0;
 961      newitems.newer = 0;
 962      newitems.ids = fetchBatch.slice(0, batchSize);
 963
 964      // Reuse the existing record handler set earlier
 965      let resp = newitems.get();
 966      if (!resp.success) {
 967        resp.failureCode = ENGINE_DOWNLOAD_FAIL;
 968        throw resp;
 969      }
 970
 971      // This batch was successfully applied. Not using
 972      // doApplyBatchAndPersistFailed() here to avoid writing toFetch twice.
 973      fetchBatch = fetchBatch.slice(batchSize);
 974      this.toFetch = Utils.arraySub(this.toFetch, newitems.ids);
 975      this.previousFailed = Utils.arrayUnion(this.previousFailed, failed);
 976      if (failed.length) {
 977        count.failed += failed.length;
 978        this._log.debug("Records that failed to apply: " + failed);
 979      }
 980      failed = [];
 981
 982      if (aborting) {
 983        throw aborting;
 984      }
 985
 986      if (this.lastSync < this.lastModified) {
 987        this.lastSync = this.lastModified;
 988      }
 989    }
 990
 991    // Apply remaining items.
 992    doApplyBatchAndPersistFailed.call(this);
 993
 994    count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length;
 995    this._log.info(["Records:",
 996                    count.applied, "applied,",
 997                    count.failed, "failed to apply,",
 998                    count.newFailed, "newly failed to apply,",
 999                    count.reconciled, "reconciled."].join(" "));
1000    Observers.notify("weave:engine:sync:applied", count, this.name);
1001  },
1002
1003  /**
1004   * Find a GUID of an item that is a duplicate of the incoming item but happens
1005   * to have a different GUID
1006   *
1007   * @return GUID of the similar item; falsy otherwise
1008   */
1009  _findDupe: function _findDupe(item) {
1010    // By default, assume there's no dupe items for the engine
1011  },
1012
1013  _deleteId: function _deleteId(id) {
1014    this._tracker.removeChangedID(id);
1015
1016    // Remember this id to delete at the end of sync
1017    if (this._delete.ids == null)
1018      this._delete.ids = [id];
1019    else
1020      this._delete.ids.push(id);
1021  },
1022
1023  /**
1024   * Reconcile incoming record with local state.
1025   *
1026   * This function essentially determines whether to apply an incoming record.
1027   *
1028   * @param  item
1029   *         Record from server to be tested for application.
1030   * @return boolean
1031   *         Truthy if incoming record should be applied. False if not.
1032   */
1033  _reconcile: function _reconcile(item) {
1034    if (this._log.level <= Log4Moz.Level.Trace) {
1035      this._log.trace("Incoming: " + item);
1036    }
1037
1038    // We start reconciling by collecting a bunch of state. We do this here
1039    // because some state may change during the course of this function and we
1040    // need to operate on the original values.
1041    let existsLocally   = this._store.itemExists(item.id);
1042    let locallyModified = item.id in this._modified;
1043
1044    // TODO Handle clock drift better. Tracked in bug 721181.
1045    let remoteAge = AsyncResource.serverTime - item.modified;
1046    let localAge  = locallyModified ?
1047      (Date.now() / 1000 - this._modified[item.id]) : null;
1048    let remoteIsNewer = remoteAge < localAge;
1049
1050    this._log.trace("Reconciling " + item.id + ". exists=" +
1051                    existsLocally + "; modified=" + locallyModified +
1052                    "; local age=" + localAge + "; incoming age=" +
1053                    remoteAge);
1054
1055    // We handle deletions first so subsequent logic doesn't have to check
1056    // deleted flags.
1057    if (item.deleted) {
1058      // If the item doesn't exist locally, there is nothing for us to do. We
1059      // can't check for duplicates because the incoming record has no data
1060      // which can be used for duplicate detection.
1061      if (!existsLocally) {
1062        this._log.trace("Ignoring incoming item because it was deleted and " +
1063                        "the item does not exist locally.");
1064        return false;
1065      }
1066
1067      // We decide whether to process the deletion by comparing the record
1068      // ages. If the item is not modified locally, the remote side wins and
1069      // the deletion is processed. If it is modified locally, we take the
1070      // newer record.
1071      if (!locallyModified) {
1072        this._log.trace("Applying incoming delete because the local item " +
1073                        "exists and isn't modified.");
1074        return true;
1075      }
1076
1077      // TODO As part of bug 720592, determine whether we should do more here.
1078      // In the case where the local changes are newer, it is quite possible
1079      // that the local client will restore data a remote client had tried to
1080      // delete. There might be a good reason for that delete and it might be
1081      // enexpected for this client to restore that data.
1082      this._log.trace("Incoming record is deleted but we had local changes. " +
1083                      "Applying the youngest record.");
1084      return remoteIsNewer;
1085    }
1086
1087    // At this point the incoming record is not for a deletion and must have
1088    // data. If the incoming record does not exist locally, we check for a local
1089    // duplicate existing under a different ID. The default implementation of
1090    // _findDupe() is empty, so engines have to opt in to this functionality.
1091    //
1092    // If we find a duplicate, we change the local ID to the incoming ID and we
1093    // refresh the metadata collected above. See bug 710448 for the history
1094    // of this logic.
1095    if (!existsLocally) {
1096      let dupeID = this._findDupe(item);
1097      if (dupeID) {
1098        this._log.trace("Local item " + dupeID + " is a duplicate for " +
1099                        "incoming item " + item.id);
1100
1101        // The local, duplicate ID is always deleted on the server.
1102        this._deleteId(dupeID);
1103
1104        // The current API contract does not mandate that the ID returned by
1105        // _findDupe() actually exists. Therefore, we have to perform this
1106        // check.
1107        existsLocally = this._store.itemExists(dupeID);
1108
1109        // We unconditionally change the item's ID in case the engine knows of
1110        // an item but doesn't expose it through itemExists. If the API
1111        // contract were stronger, this could be changed.
1112        this._log.debug("Switching local ID to incoming: " + dupeID + " -> " +
1113                        item.id);
1114        this._store.changeItemID(dupeID, item.id);
1115
1116        // If the local item was modified, we carry its metadata forward so
1117        // appropriate reconciling can be performed.
1118        if (dupeID in this._modified) {
1119          locallyModified = true;
1120          localAge = Date.now() / 1000 - this._modified[dupeID];
1121          remoteIsNewer = remoteAge < localAge;
1122
1123          this._modified[item.id] = this._modified[dupeID];
1124          delete this._modified[dupeID];
1125        } else {
1126          locallyModified = false;
1127          localAge = null;
1128        }
1129
1130        this._log.debug("Local item after duplication: age=" + localAge +
1131                        "; modified=" + locallyModified + "; exists=" +
1132                        existsLocally);
1133      } else {
1134        this._log.trace("No duplicate found for incoming item: " + item.id);
1135      }
1136    }
1137
1138    // At this point we've performed duplicate detection. But, nothing here
1139    // should depend on duplicate detection as the above should have updated
1140    // state seamlessly.
1141
1142    if (!existsLocally) {
1143      // If the item doesn't exist locally and we have no local modifications
1144      // to the item (implying that it was not deleted), always apply the remote
1145      // item.
1146      if (!locallyModified) {
1147        this._log.trace("Applying incoming because local item does not exist " +
1148                        "and was not deleted.");
1149        return true;
1150      }
1151
1152      // If the item was modified locally but isn't present, it must have
1153      // been deleted. If the incoming record is younger, we restore from
1154      // that record.
1155      if (remoteIsNewer) {
1156        this._log.trace("Applying incoming because local item was deleted " +
1157                        "before the incoming item was changed.");
1158        delete this._modified[item.id];
1159        return true;
1160      }
1161
1162      this._log.trace("Ignoring incoming item because the local item's " +
1163                      "deletion is newer.");
1164      return false;
1165    }
1166
1167    // If the remote and local records are the same, there is nothing to be
1168    // done, so we don't do anything. In the ideal world, this logic wouldn't
1169    // be here and the engine would take a record and apply it. The reason we
1170    // want to defer this logic is because it would avoid a redundant and
1171    // possibly expensive dip into the storage layer to query item state.
1172    // This should get addressed in the async rewrite, so we ignore it for now.
1173    let localRecord = this._createRecord(item.id);
1174    let recordsEqual = Utils.deepEquals(item.cleartext,
1175                                        localRecord.cleartext);
1176
1177    // If the records are the same, we don't need to do anything. This does
1178    // potentially throw away a local modification time. But, if the records
1179    // are the same, does it matter?
1180    if (recordsEqual) {
1181      this._log.trace("Ignoring incoming item because the local item is " +
1182                      "identical.");
1183
1184      delete this._modified[item.id];
1185      return false;
1186    }
1187
1188    // At this point the records are different.
1189
1190    // If we have no local modifications, always take the server record.
1191    if (!locallyModified) {
1192      this._log.trace("Applying incoming record because no local conflicts.");
1193      return true;
1194    }
1195
1196    // At this point, records are different and the local record is modified.
1197    // We resolve conflicts by record age, where the newest one wins. This does
1198    // result in data loss and should be handled by giving the engine an
1199    // opportunity to merge the records. Bug 720592 tracks this feature.
1200    this._log.warn("DATA LOSS: Both local and remote changes to record: " +
1201                   item.id);
1202    return remoteIsNewer;
1203  },
1204
1205  // Upload outgoing records
1206  _uploadOutgoing: function SyncEngine__uploadOutgoing() {
1207    this._log.trace("Uploading local changes to server.");
1208
1209    let modifiedIDs = Object.keys(this._modified);
1210    if (modifiedIDs.length) {
1211      this._log.trace("Preparing " + modifiedIDs.length +
1212                      " outgoing records");
1213
1214      // collection we'll upload
1215      let up = new Collection(this.engineURL);
1216      let count = 0;
1217
1218      // Upload what we've got so far in the collection
1219      let doUpload = Utils.bind2(this, function(desc) {
1220        this._log.info("Uploading " + desc + " of " + modifiedIDs.length +
1221                       " records");
1222        let resp = up.post();
1223        if (!resp.success) {
1224          this._log.debug("Uploading records failed: " + resp);
1225          resp.failureCode = ENGINE_UPLOAD_FAIL;
1226          throw resp;
1227        }
1228
1229        // Update server timestamp from the upload.
1230        let modified = resp.headers["x-weave-timestamp"];
1231        if (modified > this.lastSync)
1232          this.lastSync = modified;
1233
1234        let failed_ids = Object.keys(resp.obj.failed);
1235        if (failed_ids.length)
1236          this._log.debug("Records that will be uploaded again because "
1237                          + "the server couldn't store them: "
1238                          + failed_ids.join(", "));
1239
1240        // Clear successfully uploaded objects.
1241        for each (let id in resp.obj.success) {
1242          delete this._modified[id];
1243        }
1244
1245        up.clearRecords();
1246      });
1247
1248      for each (let id in modifiedIDs) {
1249        try {
1250          let out = this._createRecord(id);
1251          if (this._log.level <= Log4Moz.Level.Trace)
1252            this._log.trace("Outgoing: " + out);
1253
1254          out.encrypt();
1255          up.pushData(out);
1256        }
1257        catch(ex) {
1258          this._log.warn("Error creating record: " + Utils.exceptionStr(ex));
1259        }
1260
1261        // Partial upload
1262        if ((++count % MAX_UPLOAD_RECORDS) == 0)
1263          doUpload((count - MAX_UPLOAD_RECORDS) + " - " + count + " out");
1264
1265        this._store._sleep(0);
1266      }
1267
1268      // Final upload
1269      if (count % MAX_UPLOAD_RECORDS > 0)
1270        doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all");
1271    }
1272  },
1273
1274  // Any cleanup necessary.
1275  // Save the current snapshot so as to calculate changes at next sync
1276  _syncFinish: function SyncEngine__syncFinish() {
1277    this._log.trace("Finishing up sync");
1278    this._tracker.resetScore();
1279
1280    let doDelete = Utils.bind2(this, function(key, val) {
1281      let coll = new Collection(this.engineURL, this._recordObj);
1282      coll[key] = val;
1283      coll.delete();
1284    });
1285
1286    for (let [key, val] in Iterator(this._delete)) {
1287      // Remove the key for future uses
1288      delete this._delete[key];
1289
1290      // Send a simple delete for the property
1291      if (key != "ids" || val.length <= 100)
1292        doDelete(key, val);
1293      else {
1294        // For many ids, split into chunks of at most 100
1295        while (val.length > 0) {
1296          doDelete(key, val.slice(0, 100));
1297          val = val.slice(100);
1298        }
1299      }
1300    }
1301  },
1302
1303  _syncCleanup: function _syncCleanup() {
1304    if (!this._modified)
1305      return;
1306
1307    // Mark failed WBOs as changed again so they are reuploaded next time.
1308    for (let [id, when] in Iterator(this._modified)) {
1309      this._tracker.addChangedID(id, when);
1310    }
1311    this._modified = {};
1312  },
1313
1314  _sync: function SyncEngine__sync() {
1315    try {
1316      this._syncStartup();
1317      Observers.notify("weave:engine:sync:status", "process-incoming");
1318      this._processIncoming();
1319      Observers.notify("weave:engine:sync:status", "upload-outgoing");
1320      this._uploadOutgoing();
1321      this._syncFinish();
1322    } finally {
1323      this._syncCleanup();
1324    }
1325  },
1326
1327  canDecrypt: function canDecrypt() {
1328    // Report failure even if there's nothing to decrypt
1329    let canDecrypt = false;
1330
1331    // Fetch the most recently uploaded record and try to decrypt it
1332    let test = new Collection(this.engineURL, this._recordObj);
1333    test.limit = 1;
1334    test.sort = "newest";
1335    test.full = true;
1336    test.recordHandler = function(record) {
1337      record.decrypt();
1338      canDecrypt = true;
1339    };
1340
1341    // Any failure fetching/decrypting will just result in false
1342    try {
1343      this._log.trace("Trying to decrypt a record from the server..");
1344      test.get();
1345    }
1346    catch(ex) {
1347      this._log.debug("Failed test decrypt: " + Utils.exceptionStr(ex));
1348    }
1349
1350    return canDecrypt;
1351  },
1352
1353  _resetClient: function SyncEngine__resetClient() {
1354    this.resetLastSync();
1355    this.previousFailed = [];
1356    this.toFetch = [];
1357  },
1358
1359  wipeServer: function wipeServer() {
1360    let response = new Resource(this.engineURL).delete();
1361    if (response.status != 200 && response.status != 404) {
1362      throw response;
1363    }
1364    this._resetClient();
1365  },
1366
1367  removeClientData: function removeClientData() {
1368    // Implement this method in engines that store client specific data
1369    // on the server.
1370  },
1371
1372  /*
1373   * Decide on (and partially effect) an error-handling strategy.
1374   *
1375   * Asks the Service to respond to an HMAC error, which might result in keys
1376   * being downloaded. That call returns true if an action which might allow a
1377   * retry to occur.
1378   *
1379   * If `mayRetry` is truthy, and the Service suggests a retry,
1380   * handleHMACMismatch returns kRecoveryStrategy.retry. Otherwise, it returns
1381   * kRecoveryStrategy.error.
1382   *
1383   * Subclasses of SyncEngine can override this method to allow for different
1384   * behavior -- e.g., to delete and ignore erroneous entries.
1385   *
1386   * All return values will be part of the kRecoveryStrategy enumeration.
1387   */
1388  handleHMACMismatch: function handleHMACMismatch(item, mayRetry) {
1389    // By default we either try again, or bail out noisily.
1390    return (Weave.Service.handleHMACEvent() && mayRetry) ?
1391           SyncEngine.kRecoveryStrategy.retry :
1392           SyncEngine.kRecoveryStrategy.error;
1393  }
1394};