/services/sync/tests/unit/test_errorhandler.js

Large files files are truncated, but you can click here to view the full file

   1/* Any copyright is dedicated to the Public Domain.
   2   http://creativecommons.org/publicdomain/zero/1.0/ */
   3
   4Cu.import("resource://services-sync/engines/clients.js");
   5Cu.import("resource://services-sync/constants.js");
   6Cu.import("resource://services-sync/policies.js");
   7Cu.import("resource://services-sync/status.js");
   8
   9Svc.DefaultPrefs.set("registerEngines", "");
  10Cu.import("resource://services-sync/service.js");
  11
  12const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
  13const LOG_PREFIX_SUCCESS = "success-";
  14const LOG_PREFIX_ERROR   = "error-";
  15
  16const PROLONGED_ERROR_DURATION =
  17  (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') * 2) * 1000;
  18
  19const NON_PROLONGED_ERROR_DURATION =
  20  (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') / 2) * 1000;
  21
  22function setLastSync(lastSyncValue) {
  23  Svc.Prefs.set("lastSync", (new Date(Date.now() - lastSyncValue)).toString());
  24}
  25
  26function CatapultEngine() {
  27  SyncEngine.call(this, "Catapult");
  28}
  29CatapultEngine.prototype = {
  30  __proto__: SyncEngine.prototype,
  31  exception: null, // tests fill this in
  32  _sync: function _sync() {
  33    if (this.exception) {
  34      throw this.exception;
  35    }
  36  }
  37};
  38
  39Engines.register(CatapultEngine);
  40
  41function run_test() {
  42  initTestLogging("Trace");
  43
  44  Log4Moz.repository.getLogger("Sync.Service").level = Log4Moz.Level.Trace;
  45  Log4Moz.repository.getLogger("Sync.SyncScheduler").level = Log4Moz.Level.Trace;
  46  Log4Moz.repository.getLogger("Sync.ErrorHandler").level = Log4Moz.Level.Trace;
  47
  48  run_next_test();
  49}
  50
  51function generateCredentialsChangedFailure() {
  52  // Make sync fail due to changed credentials. We simply re-encrypt
  53  // the keys with a different Sync Key, without changing the local one.
  54  let newSyncKeyBundle = new SyncKeyBundle(PWDMGR_PASSPHRASE_REALM, Service.username);
  55  newSyncKeyBundle.keyStr = "23456234562345623456234562";
  56  let keys = CollectionKeys.asWBO();
  57  keys.encrypt(newSyncKeyBundle);
  58  keys.upload(Service.cryptoKeysURL);
  59}
  60
  61function service_unavailable(request, response) {
  62  let body = "Service Unavailable";
  63  response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
  64  response.setHeader("Retry-After", "42");
  65  response.bodyOutputStream.write(body, body.length);
  66}
  67
  68function sync_httpd_setup() {
  69  let global = new ServerWBO("global", {
  70    syncID: Service.syncID,
  71    storageVersion: STORAGE_VERSION,
  72    engines: {clients: {version: Clients.version,
  73                        syncID: Clients.syncID},
  74              catapult: {version: Engines.get("catapult").version,
  75                         syncID: Engines.get("catapult").syncID}}
  76  });
  77  let clientsColl = new ServerCollection({}, true);
  78
  79  // Tracking info/collections.
  80  let collectionsHelper = track_collections_helper();
  81  let upd = collectionsHelper.with_updated_collection;
  82
  83  let handler_401 = httpd_handler(401, "Unauthorized");
  84  return httpd_setup({
  85    // Normal server behaviour.
  86    "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
  87    "/1.1/johndoe/info/collections": collectionsHelper.handler,
  88    "/1.1/johndoe/storage/crypto/keys":
  89      upd("crypto", (new ServerWBO("keys")).handler()),
  90    "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
  91
  92    // Credentials are wrong or node reallocated.
  93    "/1.1/janedoe/storage/meta/global": handler_401,
  94    "/1.1/janedoe/info/collections": handler_401,
  95
  96    // Maintenance or overloaded (503 + Retry-After) at info/collections.
  97    "/maintenance/1.1/broken.info/info/collections": service_unavailable,
  98
  99    // Maintenance or overloaded (503 + Retry-After) at meta/global.
 100    "/maintenance/1.1/broken.meta/storage/meta/global": service_unavailable,
 101    "/maintenance/1.1/broken.meta/info/collections": collectionsHelper.handler,
 102
 103    // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
 104    "/maintenance/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
 105    "/maintenance/1.1/broken.keys/info/collections": collectionsHelper.handler,
 106    "/maintenance/1.1/broken.keys/storage/crypto/keys": service_unavailable,
 107
 108    // Maintenance or overloaded (503 + Retry-After) at wiping collection.
 109    "/maintenance/1.1/broken.wipe/info/collections": collectionsHelper.handler,
 110    "/maintenance/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
 111    "/maintenance/1.1/broken.wipe/storage/crypto/keys":
 112      upd("crypto", (new ServerWBO("keys")).handler()),
 113    "/maintenance/1.1/broken.wipe/storage": service_unavailable,
 114    "/maintenance/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()),
 115    "/maintenance/1.1/broken.wipe/storage/catapult": service_unavailable
 116  });
 117}
 118
 119function setUp() {
 120  Service.username = "johndoe";
 121  Service.password = "ilovejane";
 122  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
 123  Service.serverURL  = "http://localhost:8080/";
 124  Service.clusterURL = "http://localhost:8080/";
 125  return generateAndUploadKeys();
 126}
 127
 128function generateAndUploadKeys() {
 129  generateNewKeys();
 130  let serverKeys = CollectionKeys.asWBO("crypto", "keys");
 131  serverKeys.encrypt(Service.syncKeyBundle);
 132  return serverKeys.upload(Service.cryptoKeysURL).success;
 133}
 134
 135function clean() {
 136  Service.startOver();
 137  Status.resetSync();
 138  Status.resetBackoff();
 139}
 140
 141add_test(function test_401_logout() {
 142  let server = sync_httpd_setup();
 143  setUp();
 144
 145  // By calling sync, we ensure we're logged in.
 146  Service.sync();
 147  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 148  do_check_true(Service.isLoggedIn);
 149
 150  Svc.Obs.add("weave:service:sync:error", onSyncError);
 151  function onSyncError() {
 152    _("Got weave:service:sync:error in first sync.");
 153    Svc.Obs.remove("weave:service:sync:error", onSyncError);
 154
 155    // Wait for the automatic next sync.
 156    function onLoginError() {
 157      _("Got weave:service:login:error in second sync.");
 158      Svc.Obs.remove("weave:service:login:error", onLoginError);
 159
 160      do_check_eq(Status.login, LOGIN_FAILED_LOGIN_REJECTED);
 161      do_check_false(Service.isLoggedIn);
 162
 163      // Clean up.
 164      Utils.nextTick(function () {
 165        Service.startOver();
 166        server.stop(run_next_test);
 167      });
 168    }
 169    Svc.Obs.add("weave:service:login:error", onLoginError);
 170  }
 171
 172  // Make sync fail due to login rejected.
 173  Service.username = "janedoe";
 174
 175  _("Starting first sync.");
 176  Service.sync();
 177  _("First sync done.");
 178});
 179
 180add_test(function test_credentials_changed_logout() {
 181  let server = sync_httpd_setup();
 182  setUp();
 183
 184  // By calling sync, we ensure we're logged in.
 185  Service.sync();
 186  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 187  do_check_true(Service.isLoggedIn);
 188
 189  generateCredentialsChangedFailure();
 190  Service.sync();
 191
 192  do_check_eq(Status.sync, CREDENTIALS_CHANGED);
 193  do_check_false(Service.isLoggedIn);
 194
 195  // Clean up.
 196  Service.startOver();
 197  server.stop(run_next_test);
 198});
 199
 200add_test(function test_no_lastSync_pref() {
 201  // Test reported error.
 202  Status.resetSync();
 203  ErrorHandler.dontIgnoreErrors = true;
 204  Status.sync = CREDENTIALS_CHANGED;
 205  do_check_true(ErrorHandler.shouldReportError());
 206
 207  // Test unreported error.
 208  Status.resetSync();
 209  ErrorHandler.dontIgnoreErrors = true;
 210  Status.login = LOGIN_FAILED_NETWORK_ERROR;
 211  do_check_true(ErrorHandler.shouldReportError());
 212
 213  run_next_test();
 214});
 215
 216add_test(function test_shouldReportError() {
 217  Status.login = MASTER_PASSWORD_LOCKED;
 218  do_check_false(ErrorHandler.shouldReportError());
 219
 220  // Give ourselves a clusterURL so that the temporary 401 no-error situation
 221  // doesn't come into play.
 222  Service.clusterURL = "http://localhost:8080/";
 223
 224  // Test dontIgnoreErrors, non-network, non-prolonged, login error reported
 225  Status.resetSync();
 226  setLastSync(NON_PROLONGED_ERROR_DURATION);
 227  ErrorHandler.dontIgnoreErrors = true;
 228  Status.login = LOGIN_FAILED_NO_PASSWORD;
 229  do_check_true(ErrorHandler.shouldReportError());
 230
 231  // Test dontIgnoreErrors, non-network, non-prolonged, sync error reported
 232  Status.resetSync();
 233  setLastSync(NON_PROLONGED_ERROR_DURATION);
 234  ErrorHandler.dontIgnoreErrors = true;
 235  Status.sync = CREDENTIALS_CHANGED;
 236  do_check_true(ErrorHandler.shouldReportError());
 237
 238  // Test dontIgnoreErrors, non-network, prolonged, login error reported
 239  Status.resetSync();
 240  setLastSync(PROLONGED_ERROR_DURATION);
 241  ErrorHandler.dontIgnoreErrors = true;
 242  Status.login = LOGIN_FAILED_NO_PASSWORD;
 243  do_check_true(ErrorHandler.shouldReportError());
 244
 245  // Test dontIgnoreErrors, non-network, prolonged, sync error reported
 246  Status.resetSync();
 247  setLastSync(PROLONGED_ERROR_DURATION);
 248  ErrorHandler.dontIgnoreErrors = true;
 249  Status.sync = CREDENTIALS_CHANGED;
 250  do_check_true(ErrorHandler.shouldReportError());
 251
 252  // Test dontIgnoreErrors, network, non-prolonged, login error reported
 253  Status.resetSync();
 254  setLastSync(NON_PROLONGED_ERROR_DURATION);
 255  ErrorHandler.dontIgnoreErrors = true;
 256  Status.login = LOGIN_FAILED_NETWORK_ERROR;
 257  do_check_true(ErrorHandler.shouldReportError());
 258
 259  // Test dontIgnoreErrors, network, non-prolonged, sync error reported
 260  Status.resetSync();
 261  setLastSync(NON_PROLONGED_ERROR_DURATION);
 262  ErrorHandler.dontIgnoreErrors = true;
 263  Status.sync = LOGIN_FAILED_NETWORK_ERROR;
 264  do_check_true(ErrorHandler.shouldReportError());
 265
 266  // Test dontIgnoreErrors, network, prolonged, login error reported
 267  Status.resetSync();
 268  setLastSync(PROLONGED_ERROR_DURATION);
 269  ErrorHandler.dontIgnoreErrors = true;
 270  Status.login = LOGIN_FAILED_NETWORK_ERROR;
 271  do_check_true(ErrorHandler.shouldReportError());
 272
 273  // Test dontIgnoreErrors, network, prolonged, sync error reported
 274  Status.resetSync();
 275  setLastSync(PROLONGED_ERROR_DURATION);
 276  ErrorHandler.dontIgnoreErrors = true;
 277  Status.sync = LOGIN_FAILED_NETWORK_ERROR;
 278  do_check_true(ErrorHandler.shouldReportError());
 279
 280  // Test non-network, prolonged, login error reported
 281  Status.resetSync();
 282  setLastSync(PROLONGED_ERROR_DURATION);
 283  ErrorHandler.dontIgnoreErrors = false;
 284  Status.login = LOGIN_FAILED_NO_PASSWORD;
 285  do_check_true(ErrorHandler.shouldReportError());
 286
 287  // Test non-network, prolonged, sync error reported
 288  Status.resetSync();
 289  setLastSync(PROLONGED_ERROR_DURATION);
 290  ErrorHandler.dontIgnoreErrors = false;
 291  Status.sync = CREDENTIALS_CHANGED;
 292  do_check_true(ErrorHandler.shouldReportError());
 293
 294  // Test network, prolonged, login error reported
 295  Status.resetSync();
 296  setLastSync(PROLONGED_ERROR_DURATION);
 297  ErrorHandler.dontIgnoreErrors = false;
 298  Status.login = LOGIN_FAILED_NETWORK_ERROR;
 299  do_check_true(ErrorHandler.shouldReportError());
 300
 301  // Test network, prolonged, sync error reported
 302  Status.resetSync();
 303  setLastSync(PROLONGED_ERROR_DURATION);
 304  ErrorHandler.dontIgnoreErrors = false;
 305  Status.sync = LOGIN_FAILED_NETWORK_ERROR;
 306  do_check_true(ErrorHandler.shouldReportError());
 307
 308  // Test non-network, non-prolonged, login error reported
 309  Status.resetSync();
 310  setLastSync(NON_PROLONGED_ERROR_DURATION);
 311  ErrorHandler.dontIgnoreErrors = false;
 312  Status.login = LOGIN_FAILED_NO_PASSWORD;
 313  do_check_true(ErrorHandler.shouldReportError());
 314
 315  // Test non-network, non-prolonged, sync error reported
 316  Status.resetSync();
 317  setLastSync(NON_PROLONGED_ERROR_DURATION);
 318  ErrorHandler.dontIgnoreErrors = false;
 319  Status.sync = CREDENTIALS_CHANGED;
 320  do_check_true(ErrorHandler.shouldReportError());
 321
 322  // Test network, non-prolonged, login error reported
 323  Status.resetSync();
 324  setLastSync(NON_PROLONGED_ERROR_DURATION);
 325  ErrorHandler.dontIgnoreErrors = false;
 326  Status.login = LOGIN_FAILED_NETWORK_ERROR;
 327  do_check_false(ErrorHandler.shouldReportError());
 328
 329  // Test network, non-prolonged, sync error reported
 330  Status.resetSync();
 331  setLastSync(NON_PROLONGED_ERROR_DURATION);
 332  ErrorHandler.dontIgnoreErrors = false;
 333  Status.sync = LOGIN_FAILED_NETWORK_ERROR;
 334  do_check_false(ErrorHandler.shouldReportError());
 335
 336  // Test server maintenance, sync errors are not reported
 337  Status.resetSync();
 338  setLastSync(NON_PROLONGED_ERROR_DURATION);
 339  ErrorHandler.dontIgnoreErrors = false;
 340  Status.sync = SERVER_MAINTENANCE;
 341  do_check_false(ErrorHandler.shouldReportError());
 342
 343  // Test server maintenance, login errors are not reported
 344  Status.resetSync();
 345  setLastSync(NON_PROLONGED_ERROR_DURATION);
 346  ErrorHandler.dontIgnoreErrors = false;
 347  Status.login = SERVER_MAINTENANCE;
 348  do_check_false(ErrorHandler.shouldReportError());
 349
 350  // Test prolonged, server maintenance, sync errors are reported
 351  Status.resetSync();
 352  setLastSync(PROLONGED_ERROR_DURATION);
 353  ErrorHandler.dontIgnoreErrors = false;
 354  Status.sync = SERVER_MAINTENANCE;
 355  do_check_true(ErrorHandler.shouldReportError());
 356
 357  // Test prolonged, server maintenance, login errors are reported
 358  Status.resetSync();
 359  setLastSync(PROLONGED_ERROR_DURATION);
 360  ErrorHandler.dontIgnoreErrors = false;
 361  Status.login = SERVER_MAINTENANCE;
 362  do_check_true(ErrorHandler.shouldReportError());
 363
 364  // Test dontIgnoreErrors, server maintenance, sync errors are reported
 365  Status.resetSync();
 366  setLastSync(NON_PROLONGED_ERROR_DURATION);
 367  ErrorHandler.dontIgnoreErrors = true;
 368  Status.sync = SERVER_MAINTENANCE;
 369  do_check_true(ErrorHandler.shouldReportError());
 370
 371  // Test dontIgnoreErrors, server maintenance, login errors are reported
 372  Status.resetSync();
 373  setLastSync(NON_PROLONGED_ERROR_DURATION);
 374  ErrorHandler.dontIgnoreErrors = true;
 375  Status.login = SERVER_MAINTENANCE;
 376  do_check_true(ErrorHandler.shouldReportError());
 377
 378  // Test dontIgnoreErrors, prolonged, server maintenance,
 379  // sync errors are reported
 380  Status.resetSync();
 381  setLastSync(PROLONGED_ERROR_DURATION);
 382  ErrorHandler.dontIgnoreErrors = true;
 383  Status.sync = SERVER_MAINTENANCE;
 384  do_check_true(ErrorHandler.shouldReportError());
 385
 386  // Test dontIgnoreErrors, prolonged, server maintenance,
 387  // login errors are reported
 388  Status.resetSync();
 389  setLastSync(PROLONGED_ERROR_DURATION);
 390  ErrorHandler.dontIgnoreErrors = true;
 391  Status.login = SERVER_MAINTENANCE;
 392  do_check_true(ErrorHandler.shouldReportError());
 393
 394  run_next_test();
 395});
 396
 397add_test(function test_shouldReportError_master_password() {
 398  _("Test error ignored due to locked master password");
 399  let server = sync_httpd_setup();
 400  setUp();
 401
 402  // Monkey patch Service.verifyLogin to imitate
 403  // master password being locked.
 404  Service._verifyLogin = Service.verifyLogin;
 405  Service.verifyLogin = function () {
 406    Status.login = MASTER_PASSWORD_LOCKED;
 407    return false;
 408  };
 409
 410  setLastSync(NON_PROLONGED_ERROR_DURATION);
 411  Service.sync();
 412  do_check_false(ErrorHandler.shouldReportError());
 413
 414  // Clean up.
 415  Service.verifyLogin = Service._verifyLogin;
 416  clean();
 417  server.stop(run_next_test);
 418});
 419
 420add_test(function test_login_syncAndReportErrors_non_network_error() {
 421  // Test non-network errors are reported
 422  // when calling syncAndReportErrors
 423  let server = sync_httpd_setup();
 424  setUp();
 425  Service.password = "";
 426
 427  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 428    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 429    do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
 430
 431    clean();
 432    server.stop(run_next_test);
 433  });
 434
 435  setLastSync(NON_PROLONGED_ERROR_DURATION);
 436  ErrorHandler.syncAndReportErrors();
 437});
 438
 439add_test(function test_sync_syncAndReportErrors_non_network_error() {
 440  // Test non-network errors are reported
 441  // when calling syncAndReportErrors
 442  let server = sync_httpd_setup();
 443  setUp();
 444
 445  // By calling sync, we ensure we're logged in.
 446  Service.sync();
 447  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 448  do_check_true(Service.isLoggedIn);
 449
 450  generateCredentialsChangedFailure();
 451
 452  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 453    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 454    do_check_eq(Status.sync, CREDENTIALS_CHANGED);
 455
 456    clean();
 457    server.stop(run_next_test);
 458  });
 459
 460  setLastSync(NON_PROLONGED_ERROR_DURATION);
 461  ErrorHandler.syncAndReportErrors();
 462});
 463
 464add_test(function test_login_syncAndReportErrors_prolonged_non_network_error() {
 465  // Test prolonged, non-network errors are
 466  // reported when calling syncAndReportErrors.
 467  let server = sync_httpd_setup();
 468  setUp();
 469  Service.password = "";
 470
 471  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 472    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 473    do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
 474
 475    clean();
 476    server.stop(run_next_test);
 477  });
 478
 479  setLastSync(PROLONGED_ERROR_DURATION);
 480  ErrorHandler.syncAndReportErrors();
 481});
 482
 483add_test(function test_sync_syncAndReportErrors_prolonged_non_network_error() {
 484  // Test prolonged, non-network errors are
 485  // reported when calling syncAndReportErrors.
 486  let server = sync_httpd_setup();
 487  setUp();
 488
 489  // By calling sync, we ensure we're logged in.
 490  Service.sync();
 491  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 492  do_check_true(Service.isLoggedIn);
 493
 494  generateCredentialsChangedFailure();
 495
 496  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 497    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 498    do_check_eq(Status.sync, CREDENTIALS_CHANGED);
 499
 500    clean();
 501    server.stop(run_next_test);
 502  });
 503
 504  setLastSync(PROLONGED_ERROR_DURATION);
 505  ErrorHandler.syncAndReportErrors();
 506});
 507
 508add_test(function test_login_syncAndReportErrors_network_error() {
 509  // Test network errors are reported when calling syncAndReportErrors.
 510  Service.username = "johndoe";
 511  Service.password = "ilovejane";
 512  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
 513  Service.clusterURL = "http://localhost:8080/";
 514
 515  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 516    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 517    do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
 518
 519    clean();
 520    run_next_test();
 521  });
 522
 523  setLastSync(NON_PROLONGED_ERROR_DURATION);
 524  ErrorHandler.syncAndReportErrors();
 525});
 526
 527
 528add_test(function test_sync_syncAndReportErrors_network_error() {
 529  // Test network errors are reported when calling syncAndReportErrors.
 530  Services.io.offline = true;
 531
 532  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 533    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 534    do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
 535
 536    Services.io.offline = false;
 537    clean();
 538    run_next_test();
 539  });
 540
 541  setLastSync(NON_PROLONGED_ERROR_DURATION);
 542  ErrorHandler.syncAndReportErrors();
 543});
 544
 545add_test(function test_login_syncAndReportErrors_prolonged_network_error() {
 546  // Test prolonged, network errors are reported
 547  // when calling syncAndReportErrors.
 548  Service.username = "johndoe";
 549  Service.password = "ilovejane";
 550  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
 551  Service.clusterURL = "http://localhost:8080/";
 552
 553  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 554    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 555    do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
 556
 557    clean();
 558    run_next_test();
 559  });
 560
 561  setLastSync(PROLONGED_ERROR_DURATION);
 562  ErrorHandler.syncAndReportErrors();
 563});
 564
 565add_test(function test_sync_syncAndReportErrors_prolonged_network_error() {
 566  // Test prolonged, network errors are reported
 567  // when calling syncAndReportErrors.
 568  Services.io.offline = true;
 569
 570  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 571    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 572    do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
 573
 574    Services.io.offline = false;
 575    clean();
 576    run_next_test();
 577  });
 578
 579  setLastSync(PROLONGED_ERROR_DURATION);
 580  ErrorHandler.syncAndReportErrors();
 581});
 582
 583add_test(function test_login_prolonged_non_network_error() {
 584  // Test prolonged, non-network errors are reported
 585  let server = sync_httpd_setup();
 586  setUp();
 587  Service.password = "";
 588
 589  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 590    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 591    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 592
 593    clean();
 594    server.stop(run_next_test);
 595  });
 596
 597  setLastSync(PROLONGED_ERROR_DURATION);
 598  Service.sync();
 599});
 600
 601add_test(function test_sync_prolonged_non_network_error() {
 602  // Test prolonged, non-network errors are reported
 603  let server = sync_httpd_setup();
 604  setUp();
 605
 606  // By calling sync, we ensure we're logged in.
 607  Service.sync();
 608  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 609  do_check_true(Service.isLoggedIn);
 610
 611  generateCredentialsChangedFailure();
 612
 613  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 614    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 615    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 616
 617    clean();
 618    server.stop(run_next_test);
 619  });
 620
 621  setLastSync(PROLONGED_ERROR_DURATION);
 622  Service.sync();
 623});
 624
 625add_test(function test_login_prolonged_network_error() {
 626  // Test prolonged, network errors are reported
 627  Service.username = "johndoe";
 628  Service.password = "ilovejane";
 629  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
 630  Service.clusterURL = "http://localhost:8080/";
 631
 632  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 633    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 634    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 635
 636    clean();
 637    run_next_test();
 638  });
 639
 640  setLastSync(PROLONGED_ERROR_DURATION);
 641  Service.sync();
 642});
 643
 644add_test(function test_sync_prolonged_network_error() {
 645  // Test prolonged, network errors are reported
 646  Services.io.offline = true;
 647
 648  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 649    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 650    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 651
 652    Services.io.offline = false;
 653    clean();
 654    run_next_test();
 655  });
 656
 657  setLastSync(PROLONGED_ERROR_DURATION);
 658  Service.sync();
 659});
 660
 661add_test(function test_login_non_network_error() {
 662  // Test non-network errors are reported
 663  let server = sync_httpd_setup();
 664  setUp();
 665  Service.password = "";
 666
 667  Svc.Obs.add("weave:ui:login:error", function onSyncError() {
 668    Svc.Obs.remove("weave:ui:login:error", onSyncError);
 669    do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
 670
 671    clean();
 672    server.stop(run_next_test);
 673  });
 674
 675  setLastSync(NON_PROLONGED_ERROR_DURATION);
 676  Service.sync();
 677});
 678
 679add_test(function test_sync_non_network_error() {
 680  // Test non-network errors are reported
 681  let server = sync_httpd_setup();
 682  setUp();
 683
 684  // By calling sync, we ensure we're logged in.
 685  Service.sync();
 686  do_check_eq(Status.sync, SYNC_SUCCEEDED);
 687  do_check_true(Service.isLoggedIn);
 688
 689  generateCredentialsChangedFailure();
 690
 691  Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
 692    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 693    do_check_eq(Status.sync, CREDENTIALS_CHANGED);
 694
 695    clean();
 696    server.stop(run_next_test);
 697  });
 698
 699  setLastSync(NON_PROLONGED_ERROR_DURATION);
 700  Service.sync();
 701});
 702
 703add_test(function test_login_network_error() {
 704  Service.username = "johndoe";
 705  Service.password = "ilovejane";
 706  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
 707  Service.clusterURL = "http://localhost:8080/";
 708
 709  // Test network errors are not reported.
 710  Svc.Obs.add("weave:ui:clear-error", function onClearError() {
 711    Svc.Obs.remove("weave:ui:clear-error", onClearError);
 712
 713    do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
 714
 715    Services.io.offline = false;
 716    clean();
 717    run_next_test();
 718  });
 719
 720  setLastSync(NON_PROLONGED_ERROR_DURATION);
 721  Service.sync();
 722});
 723
 724add_test(function test_sync_network_error() {
 725  // Test network errors are not reported.
 726  Services.io.offline = true;
 727
 728  Svc.Obs.add("weave:ui:sync:finish", function onUIUpdate() {
 729    Svc.Obs.remove("weave:ui:sync:finish", onUIUpdate);
 730    do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
 731
 732    Services.io.offline = false;
 733    clean();
 734    run_next_test();
 735  });
 736
 737  setLastSync(NON_PROLONGED_ERROR_DURATION);
 738  Service.sync();
 739});
 740
 741add_test(function test_sync_server_maintenance_error() {
 742  // Test server maintenance errors are not reported.
 743  let server = sync_httpd_setup();
 744  setUp();
 745
 746  const BACKOFF = 42;
 747  let engine = Engines.get("catapult");
 748  engine.enabled = true;
 749  engine.exception = {status: 503,
 750                      headers: {"retry-after": BACKOFF}};
 751
 752  function onSyncError() {
 753    do_throw("Shouldn't get here!");
 754  }
 755  Svc.Obs.add("weave:ui:sync:error", onSyncError);
 756
 757  do_check_eq(Status.service, STATUS_OK);
 758
 759  Svc.Obs.add("weave:ui:sync:finish", function onSyncFinish() {
 760    Svc.Obs.remove("weave:ui:sync:finish", onSyncFinish);
 761
 762    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
 763    do_check_eq(Status.sync, SERVER_MAINTENANCE);
 764
 765    Svc.Obs.remove("weave:ui:sync:error", onSyncError);
 766    clean();
 767    server.stop(run_next_test);
 768  });
 769
 770  setLastSync(NON_PROLONGED_ERROR_DURATION);
 771  Service.sync();
 772});
 773
 774add_test(function test_info_collections_login_server_maintenance_error() {
 775  // Test info/collections server maintenance errors are not reported.
 776  let server = sync_httpd_setup();
 777  setUp();
 778
 779  Service.username = "broken.info";
 780  Service.clusterURL = "http://localhost:8080/maintenance/";
 781
 782  let backoffInterval;
 783  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 784    Svc.Obs.remove("weave:service:backoff:interval", observe);
 785    backoffInterval = subject;
 786  });
 787
 788  function onUIUpdate() {
 789    do_throw("Shouldn't get here!");
 790  }
 791  Svc.Obs.add("weave:ui:login:error", onUIUpdate);
 792
 793  do_check_false(Status.enforceBackoff);
 794  do_check_eq(Status.service, STATUS_OK);
 795
 796  Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
 797    Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
 798
 799    do_check_true(Status.enforceBackoff);
 800    do_check_eq(backoffInterval, 42);
 801    do_check_eq(Status.service, LOGIN_FAILED);
 802    do_check_eq(Status.login, SERVER_MAINTENANCE);
 803
 804    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
 805    clean();
 806    server.stop(run_next_test);
 807  });
 808
 809  setLastSync(NON_PROLONGED_ERROR_DURATION);
 810  Service.sync();
 811});
 812
 813add_test(function test_meta_global_login_server_maintenance_error() {
 814  // Test meta/global server maintenance errors are not reported.
 815  let server = sync_httpd_setup();
 816  setUp();
 817
 818  Service.username = "broken.meta";
 819  Service.clusterURL = "http://localhost:8080/maintenance/";
 820
 821  let backoffInterval;
 822  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 823    Svc.Obs.remove("weave:service:backoff:interval", observe);
 824    backoffInterval = subject;
 825  });
 826
 827  function onUIUpdate() {
 828    do_throw("Shouldn't get here!");
 829  }
 830  Svc.Obs.add("weave:ui:login:error", onUIUpdate);
 831
 832  do_check_false(Status.enforceBackoff);
 833  do_check_eq(Status.service, STATUS_OK);
 834
 835  Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
 836    Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
 837
 838    do_check_true(Status.enforceBackoff);
 839    do_check_eq(backoffInterval, 42);
 840    do_check_eq(Status.service, LOGIN_FAILED);
 841    do_check_eq(Status.login, SERVER_MAINTENANCE);
 842
 843    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
 844    clean();
 845    server.stop(run_next_test);
 846  });
 847
 848  setLastSync(NON_PROLONGED_ERROR_DURATION);
 849  Service.sync();
 850});
 851
 852add_test(function test_crypto_keys_login_server_maintenance_error() {
 853  // Test crypto/keys server maintenance errors are not reported.
 854  let server = sync_httpd_setup();
 855  setUp();
 856
 857  Service.username = "broken.keys";
 858  Service.clusterURL = "http://localhost:8080/maintenance/";
 859  // Force re-download of keys
 860  CollectionKeys.clear();
 861
 862  let backoffInterval;
 863  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 864    Svc.Obs.remove("weave:service:backoff:interval", observe);
 865    backoffInterval = subject;
 866  });
 867
 868  function onUIUpdate() {
 869    do_throw("Shouldn't get here!");
 870  }
 871  Svc.Obs.add("weave:ui:login:error", onUIUpdate);
 872
 873  do_check_false(Status.enforceBackoff);
 874  do_check_eq(Status.service, STATUS_OK);
 875
 876  Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
 877    Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
 878
 879    do_check_true(Status.enforceBackoff);
 880    do_check_eq(backoffInterval, 42);
 881    do_check_eq(Status.service, LOGIN_FAILED);
 882    do_check_eq(Status.login, SERVER_MAINTENANCE);
 883
 884    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
 885    clean();
 886    server.stop(run_next_test);
 887  });
 888
 889  setLastSync(NON_PROLONGED_ERROR_DURATION);
 890  Service.sync();
 891});
 892
 893add_test(function test_sync_prolonged_server_maintenance_error() {
 894  // Test prolonged server maintenance errors are reported.
 895  let server = sync_httpd_setup();
 896  setUp();
 897
 898  const BACKOFF = 42;
 899  let engine = Engines.get("catapult");
 900  engine.enabled = true;
 901  engine.exception = {status: 503,
 902                      headers: {"retry-after": BACKOFF}};
 903
 904  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
 905    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
 906    do_check_eq(Status.service, SYNC_FAILED);
 907    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 908
 909    clean();
 910    server.stop(run_next_test);
 911  });
 912
 913  do_check_eq(Status.service, STATUS_OK);
 914
 915  setLastSync(PROLONGED_ERROR_DURATION);
 916  Service.sync();
 917});
 918
 919add_test(function test_info_collections_login_prolonged_server_maintenance_error(){
 920  // Test info/collections prolonged server maintenance errors are reported.
 921  let server = sync_httpd_setup();
 922  setUp();
 923
 924  Service.username = "broken.info";
 925  Service.clusterURL = "http://localhost:8080/maintenance/";
 926
 927  let backoffInterval;
 928  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 929    Svc.Obs.remove("weave:service:backoff:interval", observe);
 930    backoffInterval = subject;
 931  });
 932
 933  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
 934    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
 935    do_check_true(Status.enforceBackoff);
 936    do_check_eq(backoffInterval, 42);
 937    do_check_eq(Status.service, SYNC_FAILED);
 938    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 939
 940    clean();
 941    server.stop(run_next_test);
 942  });
 943
 944  do_check_false(Status.enforceBackoff);
 945  do_check_eq(Status.service, STATUS_OK);
 946
 947  setLastSync(PROLONGED_ERROR_DURATION);
 948  Service.sync();
 949});
 950
 951add_test(function test_meta_global_login_prolonged_server_maintenance_error(){
 952  // Test meta/global prolonged server maintenance errors are reported.
 953  let server = sync_httpd_setup();
 954  setUp();
 955
 956  Service.username = "broken.meta";
 957  Service.clusterURL = "http://localhost:8080/maintenance/";
 958
 959  let backoffInterval;
 960  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 961    Svc.Obs.remove("weave:service:backoff:interval", observe);
 962    backoffInterval = subject;
 963  });
 964
 965  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
 966    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
 967    do_check_true(Status.enforceBackoff);
 968    do_check_eq(backoffInterval, 42);
 969    do_check_eq(Status.service, SYNC_FAILED);
 970    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
 971
 972    clean();
 973    server.stop(run_next_test);
 974  });
 975
 976  do_check_false(Status.enforceBackoff);
 977  do_check_eq(Status.service, STATUS_OK);
 978
 979  setLastSync(PROLONGED_ERROR_DURATION);
 980  Service.sync();
 981});
 982
 983add_test(function test_download_crypto_keys_login_prolonged_server_maintenance_error(){
 984  // Test crypto/keys prolonged server maintenance errors are reported.
 985  let server = sync_httpd_setup();
 986  setUp();
 987
 988  Service.username = "broken.keys";
 989  Service.clusterURL = "http://localhost:8080/maintenance/";
 990  // Force re-download of keys
 991  CollectionKeys.clear();
 992
 993  let backoffInterval;
 994  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
 995    Svc.Obs.remove("weave:service:backoff:interval", observe);
 996    backoffInterval = subject;
 997  });
 998
 999  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1000    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1001    do_check_true(Status.enforceBackoff);
1002    do_check_eq(backoffInterval, 42);
1003    do_check_eq(Status.service, SYNC_FAILED);
1004    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
1005
1006    clean();
1007    server.stop(run_next_test);
1008  });
1009
1010  do_check_false(Status.enforceBackoff);
1011  do_check_eq(Status.service, STATUS_OK);
1012
1013  setLastSync(PROLONGED_ERROR_DURATION);
1014  Service.sync();
1015});
1016
1017add_test(function test_upload_crypto_keys_login_prolonged_server_maintenance_error(){
1018  // Test crypto/keys prolonged server maintenance errors are reported.
1019  let server = sync_httpd_setup();
1020
1021  // Start off with an empty account, do not upload a key.
1022  Service.username = "broken.keys";
1023  Service.password = "ilovejane";
1024  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1025  Service.clusterURL = "http://localhost:8080/maintenance/";
1026
1027  let backoffInterval;
1028  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1029    Svc.Obs.remove("weave:service:backoff:interval", observe);
1030    backoffInterval = subject;
1031  });
1032
1033  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1034    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1035    do_check_true(Status.enforceBackoff);
1036    do_check_eq(backoffInterval, 42);
1037    do_check_eq(Status.service, SYNC_FAILED);
1038    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
1039
1040    clean();
1041    server.stop(run_next_test);
1042  });
1043
1044  do_check_false(Status.enforceBackoff);
1045  do_check_eq(Status.service, STATUS_OK);
1046
1047  setLastSync(PROLONGED_ERROR_DURATION);
1048  Service.sync();
1049});
1050
1051add_test(function test_wipeServer_login_prolonged_server_maintenance_error(){
1052  // Test that we report prolonged server maintenance errors that occur whilst
1053  // wiping the server.
1054  let server = sync_httpd_setup();
1055
1056  // Start off with an empty account, do not upload a key.
1057  Service.username = "broken.wipe";
1058  Service.password = "ilovejane";
1059  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1060  Service.clusterURL = "http://localhost:8080/maintenance/";
1061
1062  let backoffInterval;
1063  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1064    Svc.Obs.remove("weave:service:backoff:interval", observe);
1065    backoffInterval = subject;
1066  });
1067
1068  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1069    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1070    do_check_true(Status.enforceBackoff);
1071    do_check_eq(backoffInterval, 42);
1072    do_check_eq(Status.service, SYNC_FAILED);
1073    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
1074
1075    clean();
1076    server.stop(run_next_test);
1077  });
1078
1079  do_check_false(Status.enforceBackoff);
1080  do_check_eq(Status.service, STATUS_OK);
1081
1082  setLastSync(PROLONGED_ERROR_DURATION);
1083  Service.sync();
1084});
1085
1086add_test(function test_wipeRemote_prolonged_server_maintenance_error(){
1087  // Test that we report prolonged server maintenance errors that occur whilst
1088  // wiping all remote devices.
1089  let server = sync_httpd_setup();
1090
1091  Service.username = "broken.wipe";
1092  Service.password = "ilovejane";
1093  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1094  Service.clusterURL = "http://localhost:8080/maintenance/";
1095  generateAndUploadKeys();
1096
1097  let engine = Engines.get("catapult");
1098  engine.exception = null;
1099  engine.enabled = true;
1100
1101  let backoffInterval;
1102  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1103    Svc.Obs.remove("weave:service:backoff:interval", observe);
1104    backoffInterval = subject;
1105  });
1106
1107  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
1108    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
1109    do_check_true(Status.enforceBackoff);
1110    do_check_eq(backoffInterval, 42);
1111    do_check_eq(Status.service, SYNC_FAILED);
1112    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
1113    do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
1114
1115    clean();
1116    server.stop(run_next_test);
1117  });
1118
1119  do_check_false(Status.enforceBackoff);
1120  do_check_eq(Status.service, STATUS_OK);
1121
1122  Svc.Prefs.set("firstSync", "wipeRemote");
1123  setLastSync(PROLONGED_ERROR_DURATION);
1124  Service.sync();
1125});
1126
1127add_test(function test_sync_syncAndReportErrors_server_maintenance_error() {
1128  // Test server maintenance errors are reported
1129  // when calling syncAndReportErrors.
1130  let server = sync_httpd_setup();
1131  setUp();
1132
1133  const BACKOFF = 42;
1134  let engine = Engines.get("catapult");
1135  engine.enabled = true;
1136  engine.exception = {status: 503,
1137                      headers: {"retry-after": BACKOFF}};
1138
1139  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
1140    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
1141    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
1142    do_check_eq(Status.sync, SERVER_MAINTENANCE);
1143
1144    clean();
1145    server.stop(run_next_test);
1146  });
1147
1148  do_check_eq(Status.service, STATUS_OK);
1149
1150  setLastSync(NON_PROLONGED_ERROR_DURATION);
1151  ErrorHandler.syncAndReportErrors();
1152});
1153
1154add_test(function test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
1155  // Test info/collections server maintenance errors are reported
1156  // when calling syncAndReportErrors.
1157  let server = sync_httpd_setup();
1158  setUp();
1159
1160  Service.username = "broken.info";
1161  Service.clusterURL = "http://localhost:8080/maintenance/";
1162
1163  let backoffInterval;
1164  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1165    Svc.Obs.remove("weave:service:backoff:interval", observe);
1166    backoffInterval = subject;
1167  });
1168
1169  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1170    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1171    do_check_true(Status.enforceBackoff);
1172    do_check_eq(backoffInterval, 42);
1173    do_check_eq(Status.service, LOGIN_FAILED);
1174    do_check_eq(Status.login, SERVER_MAINTENANCE);
1175
1176    clean();
1177    server.stop(run_next_test);
1178  });
1179
1180  do_check_false(Status.enforceBackoff);
1181  do_check_eq(Status.service, STATUS_OK);
1182
1183  setLastSync(NON_PROLONGED_ERROR_DURATION);
1184  ErrorHandler.syncAndReportErrors();
1185});
1186
1187add_test(function test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
1188  // Test meta/global server maintenance errors are reported
1189  // when calling syncAndReportErrors.
1190  let server = sync_httpd_setup();
1191  setUp();
1192
1193  Service.username = "broken.meta";
1194  Service.clusterURL = "http://localhost:8080/maintenance/";
1195
1196  let backoffInterval;
1197  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1198    Svc.Obs.remove("weave:service:backoff:interval", observe);
1199    backoffInterval = subject;
1200  });
1201
1202  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1203    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1204    do_check_true(Status.enforceBackoff);
1205    do_check_eq(backoffInterval, 42);
1206    do_check_eq(Status.service, LOGIN_FAILED);
1207    do_check_eq(Status.login, SERVER_MAINTENANCE);
1208
1209    clean();
1210    server.stop(run_next_test);
1211  });
1212
1213  do_check_false(Status.enforceBackoff);
1214  do_check_eq(Status.service, STATUS_OK);
1215
1216  setLastSync(NON_PROLONGED_ERROR_DURATION);
1217  ErrorHandler.syncAndReportErrors();
1218});
1219
1220add_test(function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
1221  // Test crypto/keys server maintenance errors are reported
1222  // when calling syncAndReportErrors.
1223  let server = sync_httpd_setup();
1224  setUp();
1225
1226  Service.username = "broken.keys";
1227  Service.clusterURL = "http://localhost:8080/maintenance/";
1228  // Force re-download of keys
1229  CollectionKeys.clear();
1230
1231  let backoffInterval;
1232  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1233    Svc.Obs.remove("weave:service:backoff:interval", observe);
1234    backoffInterval = subject;
1235  });
1236
1237  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1238    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1239    do_check_true(Status.enforceBackoff);
1240    do_check_eq(backoffInterval, 42);
1241    do_check_eq(Status.service, LOGIN_FAILED);
1242    do_check_eq(Status.login, SERVER_MAINTENANCE);
1243
1244    clean();
1245    server.stop(run_next_test);
1246  });
1247
1248  do_check_false(Status.enforceBackoff);
1249  do_check_eq(Status.service, STATUS_OK);
1250
1251  setLastSync(NON_PROLONGED_ERROR_DURATION);
1252  ErrorHandler.syncAndReportErrors();
1253});
1254
1255add_test(function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
1256  // Test crypto/keys server maintenance errors are reported
1257  // when calling syncAndReportErrors.
1258  let server = sync_httpd_setup();
1259
1260  // Start off with an empty account, do not upload a key.
1261  Service.username = "broken.keys";
1262  Service.password = "ilovejane";
1263  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1264  Service.clusterURL = "http://localhost:8080/maintenance/";
1265
1266  let backoffInterval;
1267  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1268    Svc.Obs.remove("weave:service:backoff:interval", observe);
1269    backoffInterval = subject;
1270  });
1271
1272  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1273    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1274    do_check_true(Status.enforceBackoff);
1275    do_check_eq(backoffInterval, 42);
1276    do_check_eq(Status.service, LOGIN_FAILED);
1277    do_check_eq(Status.login, SERVER_MAINTENANCE);
1278
1279    clean();
1280    server.stop(run_next_test);
1281  });
1282
1283  do_check_false(Status.enforceBackoff);
1284  do_check_eq(Status.service, STATUS_OK);
1285
1286  setLastSync(NON_PROLONGED_ERROR_DURATION);
1287  ErrorHandler.syncAndReportErrors();
1288});
1289
1290add_test(function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
1291  // Test crypto/keys server maintenance errors are reported
1292  // when calling syncAndReportErrors.
1293  let server = sync_httpd_setup();
1294
1295  // Start off with an empty account, do not upload a key.
1296  Service.username = "broken.wipe";
1297  Service.password = "ilovejane";
1298  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1299  Service.clusterURL = "http://localhost:8080/maintenance/";
1300
1301  let backoffInterval;
1302  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1303    Svc.Obs.remove("weave:service:backoff:interval", observe);
1304    backoffInterval = subject;
1305  });
1306
1307  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1308    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1309    do_check_true(Status.enforceBackoff);
1310    do_check_eq(backoffInterval, 42);
1311    do_check_eq(Status.service, LOGIN_FAILED);
1312    do_check_eq(Status.login, SERVER_MAINTENANCE);
1313
1314    clean();
1315    server.stop(run_next_test);
1316  });
1317
1318  do_check_false(Status.enforceBackoff);
1319  do_check_eq(Status.service, STATUS_OK);
1320
1321  setLastSync(NON_PROLONGED_ERROR_DURATION);
1322  ErrorHandler.syncAndReportErrors();
1323});
1324
1325add_test(function test_wipeRemote_syncAndReportErrors_server_maintenance_error(){
1326  // Test that we report prolonged server maintenance errors that occur whilst
1327  // wiping all remote devices.
1328  let server = sync_httpd_setup();
1329
1330  Service.username = "broken.wipe";
1331  Service.password = "ilovejane";
1332  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1333  Service.clusterURL = "http://localhost:8080/maintenance/";
1334  generateAndUploadKeys();
1335
1336  let engine = Engines.get("catapult");
1337  engine.exception = null;
1338  engine.enabled = true;
1339
1340  let backoffInterval;
1341  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1342    Svc.Obs.remove("weave:service:backoff:interval", observe);
1343    backoffInterval = subject;
1344  });
1345
1346  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
1347    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
1348    do_check_true(Status.enforceBackoff);
1349    do_check_eq(backoffInterval, 42);
1350    do_check_eq(Status.service, SYNC_FAILED);
1351    do_check_eq(Status.sync, SERVER_MAINTENANCE);
1352    do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
1353
1354    clean();
1355    server.stop(run_next_test);
1356  });
1357
1358  do_check_false(Status.enforceBackoff);
1359  do_check_eq(Status.service, STATUS_OK);
1360
1361  Svc.Prefs.set("firstSync", "wipeRemote");
1362  setLastSync(NON_PROLONGED_ERROR_DURATION);
1363  ErrorHandler.syncAndReportErrors();
1364});
1365
1366add_test(function test_sync_syncAndReportErrors_prolonged_server_maintenance_error() {
1367  // Test prolonged server maintenance errors are
1368  // reported when calling syncAndReportErrors.
1369  let server = sync_httpd_setup();
1370  setUp();
1371
1372  const BACKOFF = 42;
1373  let engine = Engines.get("catapult");
1374  engine.enabled = true;
1375  engine.exception = {status: 503,
1376                      headers: {"retry-after": BACKOFF}};
1377
1378  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
1379    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
1380    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
1381    do_check_eq(Status.sync, SERVER_MAINTENANCE);
1382
1383    clean();
1384    server.stop(run_next_test);
1385  });
1386
1387  do_check_eq(Status.service, STATUS_OK);
1388
1389  setLastSync(PROLONGED_ERROR_DURATION);
1390  ErrorHandler.syncAndReportErrors();
1391});
1392
1393add_test(function test_info_collections_login_syncAndReportErrors_prolonged_server_maintenance_error() {
1394  // Test info/collections server maintenance errors are reported
1395  // when calling syncAndReportErrors.
1396  let server = sync_httpd_setup();
1397  setUp();
1398
1399  Service.username = "broken.info";
1400  Service.clusterURL = "http://localhost:8080/maintenance/";
1401
1402  let backoffInterval;
1403  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1404    Svc.Obs.remove("weave:service:backoff:interval", observe);
1405    backoffInterval = subject;
1406  });
1407
1408  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1409    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1410    do_check_true(Status.enforceBackoff);
1411    do_check_eq(backoffInterval, 42);
1412    do_check_eq(Status.service, LOGIN_FAILED);
1413    do_check_eq(Status.login, SERVER_MAINTENANCE);
1414
1415    clean();
1416    server.stop(run_next_test);
1417  });
1418
1419  do_check_false(Status.enforceBackoff);
1420  do_check_eq(Status.service, STATUS_OK);
1421
1422  setLastSync(PROLONGED_ERROR_DURATION);
1423  ErrorHandler.syncAndReportErrors();
1424});
1425
1426add_test(function test_meta_global_login_syncAndReportErrors_prolonged_server_maintenance_error() {
1427  // Test meta/global server maintenance errors are reported
1428  // when calling syncAndReportErrors.
1429  let server = sync_httpd_setup();
1430  setUp();
1431
1432  Service.username = "broken.meta";
1433  Service.clusterURL = "http://localhost:8080/maintenance/";
1434
1435  let backoffInterval;
1436  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1437    Svc.Obs.remove("weave:service:backoff:interval", observe);
1438    backoffInterval = subject;
1439  });
1440
1441  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1442    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1443    do_check_true(Status.enforceBackoff);
1444    do_check_eq(backoffInterval, 42);
1445    do_check_eq(Status.service, LOGIN_FAILED);
1446    do_check_eq(Status.login, SERVER_MAINTENANCE);
1447
1448    clean();
1449    server.stop(run_next_test);
1450  });
1451
1452  do_check_false(Status.enforceBackoff);
1453  do_check_eq(Status.service, STATUS_OK);
1454
1455  setLastSync(PROLONGED_ERROR_DURATION);
1456  ErrorHandler.syncAndReportErrors();
1457});
1458
1459add_test(function test_download_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
1460  // Test crypto/keys server maintenance errors are reported
1461  // when calling syncAndReportErrors.
1462  let server = sync_httpd_setup();
1463  setUp();
1464
1465  Service.username = "broken.keys";
1466  Service.clusterURL = "http://localhost:8080/maintenance/";
1467  // Force re-download of keys
1468  CollectionKeys.clear();
1469
1470  let backoffInterval;
1471  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1472    Svc.Obs.remove("weave:service:backoff:interval", observe);
1473    backoffInterval = subject;
1474  });
1475
1476  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1477    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1478    do_check_true(Status.enforceBackoff);
1479    do_check_eq(backoffInterval, 42);
1480    do_check_eq(Status.service, LOGIN_FAILED);
1481    do_check_eq(Status.login, SERVER_MAINTENANCE);
1482
1483    clean();
1484    server.stop(run_next_test);
1485  });
1486
1487  do_check_false(Status.enforceBackoff);
1488  do_check_eq(Status.service, STATUS_OK);
1489
1490  setLastSync(PROLONGED_ERROR_DURATION);
1491  ErrorHandler.syncAndReportErrors();
1492});
1493
1494add_test(function test_upload_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
1495  // Test crypto/keys server maintenance errors are reported
1496  // when calling syncAndReportErrors.
1497  let server = sync_httpd_setup();
1498
1499  // Start off with an empty account, do not upload a key.
1500  Service.username = "broken.keys";
1501  Service.password = "ilovejane";
1502  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1503  Service.clusterURL = "http://localhost:8080/maintenance/";
1504
1505  let backoffInterval;
1506  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1507    Svc.Obs.remove("weave:service:backoff:interval", observe);
1508    backoffInterval = subject;
1509  });
1510
1511  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1512    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1513    do_check_true(Status.enforceBackoff);
1514    do_check_eq(backoffInterval, 42);
1515    do_check_eq(Status.service, LOGIN_FAILED);
1516    do_check_eq(Status.login, SERVER_MAINTENANCE);
1517
1518    clean();
1519    server.stop(run_next_test);
1520  });
1521
1522  do_check_false(Status.enforceBackoff);
1523  do_check_eq(Status.service, STATUS_OK);
1524
1525  setLastSync(PROLONGED_ERROR_DURATION);
1526  ErrorHandler.syncAndReportErrors();
1527});
1528
1529add_test(function test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
1530  // Test crypto/keys server maintenance errors are reported
1531  // when calling syncAndReportErrors.
1532  let server = sync_httpd_setup();
1533
1534  // Start off with an empty account, do not upload a key.
1535  Service.username = "broken.wipe";
1536  Service.password = "ilovejane";
1537  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
1538  Service.clusterURL = "http://localhost:8080/maintenance/";
1539
1540  let backoffInterval;
1541  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
1542    Svc.Obs.remove("weave:service:backoff:interval", observe);
1543    backoffInterval = subject;
1544  });
1545
1546  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
1547    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
1548    do_check_true(Status.enforceBackoff);
1549    do_check_eq(backoffInterval, 42);
1550    do_check_eq(Status.service, LOGIN_FAILED);
1551    do_check_eq(Status.login, SERVER_MAINTENANCE);
1552
1553    clean();
1554    server.stop(run_next_test);
1555  });
1556
1557  do_check_false(Status.enforceBackoff);
1558  do_check_eq(Status.service, STATUS_OK);
1559
1560  setLastSync(PROLONGED_ERROR_DURATION);
1561  ErrorHandler.syncAndReportErrors();
1562});
1563
1564add_test(function test_sync_engine_generic_fail() {
1565  let server = sync_httpd_setup();
1566
1567  let engine = Engines.get("catapult");
1568  engine.enabled = true;
1569  engine.sync = function sync() {
1570    Svc.Obs.notify("…