improve network errors processing, closes #66

.travis.yml

@@ -34,7 +34,7 @@ env:
   - MOZ_HEADLESS=1
 cache: false
 install:
-- yarn install
+- yarn --pure-lockfile install
 - yarn generate-npm-lockfile
 - npm audit
 before_script:

appveyor.yml

@@ -18,7 +18,7 @@ install:
 - node --version
 - npm --version
 - yarn --version
-- yarn install --mutex file
+- yarn install --pure-lockfile --mutex file
 - yarn generate-npm-lockfile
 - npm audit
 build_script:

package.json

@@ -77,14 +77,14 @@
   "dependencies": {
     "about-window": "1.12.1",
     "base64-arraybuffer": "0.1.5",
-    "class-transformer": "0.1.10",
+    "class-transformer": "0.2.0",
     "class-transformer-validator": "0.6.0",
     "class-validator": "0.9.1",
     "compare-versions": "3.4.0",
     "electron-log": "2.2.17",
     "electron-rpc-api": "3.2.0",
     "electron-unhandled": "1.1.0",
-    "electron-updater": "3.1.5",
+    "electron-updater": "3.2.2",
     "fs-json-store": "2.0.4",
     "fs-json-store-encryption-adapter": "1.3.2",
     "fs-no-eperm-anymore": "2.0.2",
@@ -105,38 +105,38 @@
     "valid-url": "1.0.9"
   },
   "devDependencies": {
-    "@angular-devkit/build-optimizer": "0.10.3",
-    "@angular/animations": "7.0.1",
-    "@angular/common": "7.0.1",
-    "@angular/compiler": "7.0.1",
-    "@angular/compiler-cli": "7.0.1",
-    "@angular/core": "7.0.1",
-    "@angular/forms": "7.0.1",
-    "@angular/http": "7.0.1",
-    "@angular/language-service": "7.0.1",
-    "@angular/platform-browser": "7.0.1",
-    "@angular/platform-browser-dynamic": "7.0.1",
-    "@angular/router": "7.0.1",
+    "@angular-devkit/build-optimizer": "0.10.4",
+    "@angular/animations": "7.0.3",
+    "@angular/common": "7.0.3",
+    "@angular/compiler": "7.0.3",
+    "@angular/compiler-cli": "7.0.3",
+    "@angular/core": "7.0.3",
+    "@angular/forms": "7.0.3",
+    "@angular/http": "7.0.3",
+    "@angular/language-service": "7.0.3",
+    "@angular/platform-browser": "7.0.3",
+    "@angular/platform-browser-dynamic": "7.0.3",
+    "@angular/router": "7.0.3",
     "@angularclass/hmr": "2.1.3",
     "@email-securely-app/import-sort-style": "0.1.0",
-    "@ng-select/ng-select": "2.11.2",
-    "@ngrx/effects": "6.1.0",
-    "@ngrx/router-store": "6.1.0",
-    "@ngrx/store": "6.1.0",
-    "@ngtools/webpack": "7.0.3",
+    "@ng-select/ng-select": "2.12.0",
+    "@ngrx/effects": "6.1.2",
+    "@ngrx/router-store": "6.1.2",
+    "@ngrx/store": "6.1.2",
+    "@ngtools/webpack": "7.0.4",
     "@types/angular": "1.6.51",
     "@types/base64-arraybuffer": "0.1.0",
     "@types/html-to-text": "1.4.31",
     "@types/html-webpack-plugin": "3.2.0",
-    "@types/jasmine": "2.8.9",
+    "@types/jasmine": "2.8.11",
     "@types/karma": "3.0.0",
     "@types/keytar": "4.0.1",
     "@types/mini-css-extract-plugin": "0.2.0",
     "@types/mkdirp": "0.5.2",
-    "@types/node": "10.12.0",
+    "@types/node": "10.12.3",
     "@types/node-fetch": "2.1.2",
     "@types/p-queue": "2.3.1",
-    "@types/ramda": "0.25.40",
+    "@types/ramda": "0.26.0",
     "@types/randomstring": "1.1.6",
     "@types/sanitize-html": "1.18.2",
     "@types/semver": "5.5.0",
@@ -154,10 +154,10 @@
     "@types/webpack-merge": "4.1.3",
     "@types/webpack-node-externals": "1.6.3",
     "@vladimiry/unionize": "2.1.2-add-tagprefix-option",
-    "ava": "1.0.0-beta.8",
+    "ava": "1.0.0-rc.1",
     "awesome-typescript-loader": "5.2.1",
     "bootstrap": "4.1.3",
-    "cache-loader": "1.2.2",
+    "cache-loader": "1.2.5",
     "circular-dependency-plugin": "5.0.2",
     "codelyzer": "4.5.0",
     "core-js": "2.5.7",
@@ -167,14 +167,14 @@
     "css-loader": "1.0.1",
     "cssnano": "4.1.7",
     "devtron": "1.4.0",
-    "electron": "3.0.6",
-    "electron-builder": "20.29.0",
+    "electron": "3.0.7",
+    "electron-builder": "20.33.2",
     "exports-loader": "0.7.0",
     "file-loader": "2.0.0",
     "font-awesome": "4.7.0",
     "html-loader": "0.5.5",
-    "html-webpack-plugin": "4.0.0-alpha",
-    "husky": "1.1.2",
+    "html-webpack-plugin": "4.0.0-beta.2",
+    "husky": "1.1.3",
     "immer": "1.7.4",
     "import-sort-cli": "5.2.0",
     "import-sort-parser-typescript": "5.0.0",
@@ -188,29 +188,29 @@
     "karma-webpack": "4.0.0-rc.2",
     "keysim": "2.1.0",
     "less-loader": "4.1.0",
-    "lint-staged": "8.0.2",
+    "lint-staged": "8.0.4",
     "mini-css-extract-plugin": "0.4.4",
     "mkdirp": "0.5.1",
-    "ng2-dragula": "2.1.0",
+    "ng2-dragula": "2.1.1",
     "ngx-bootstrap": "3.1.1",
-    "node-fetch": "2.2.0",
-    "node-sass": "4.9.4",
+    "node-fetch": "2.2.1",
+    "node-sass": "4.10.0",
     "npm-run-all": "4.1.3",
     "null-loader": "0.1.1",
     "otplib": "10.0.1",
-    "postcss-custom-properties": "8.0.8",
+    "postcss-custom-properties": "8.0.9",
     "postcss-loader": "3.0.0",
     "postcss-url": "8.0.0",
     "promise-parallel-throttle": "3.2.0",
     "randomstring": "1.1.5",
     "raw-loader": "0.5.1",
     "resolve-url-loader": "3.0.0",
-    "rewiremock": "3.10.0",
+    "rewiremock": "3.11.1",
     "rxjs-compat": "6.3.3",
     "sass-lint": "1.12.1",
     "sass-loader": "7.1.0",
     "script-loader": "0.7.2",
-    "sinon": "7.1.0",
+    "sinon": "7.1.1",
     "source-map": "0.7.3",
     "source-map-loader": "0.2.4",
     "source-map-support": "0.5.9",
@@ -224,21 +224,20 @@
     "tsconfig-paths": "3.6.0",
     "tsconfig-paths-webpack-plugin": "3.2.0",
     "tslint": "5.11.0",
-    "tslint-consistent-codestyle": "1.13.3",
+    "tslint-consistent-codestyle": "1.14.1",
     "tslint-eslint-rules": "5.4.0",
     "tslint-rules-bunch": "0.0.5",
-    "typescript": "3.1.4",
+    "typescript": "3.1.6",
     "uglifyjs-webpack-plugin": "2.0.1",
     "url-loader": "1.1.2",
-    "webpack": "4.23.1",
+    "webpack": "4.25.1",
     "webpack-cli": "3.1.2",
     "webpack-dev-server": "3.1.10",
     "webpack-merge": "4.1.4",
     "webpack-node-externals": "1.7.2",
     "zone.js": "0.8.26"
   },
   "resolutions": {
-    "ng2-dragula/**/@types/dragula": "2.1.33",
     "spectron/**/electron-chromedriver": "^3.0.0"
   }
 }

src/electron-preload/webview/protonmail/index.ts

@@ -1,7 +1,7 @@
 import * as Throttle from "promise-parallel-throttle";
-import {EMPTY, Observable, from, interval, merge, of, throwError} from "rxjs";
+import {EMPTY, Observable, defer, from, interval, merge, of} from "rxjs";
 import {authenticator} from "otplib";
-import {buffer, concatMap, debounceTime, delay, distinctUntilChanged, filter, map, mergeMap, retryWhen, take, tap} from "rxjs/operators";
+import {buffer, catchError, concatMap, debounceTime, distinctUntilChanged, filter, map, mergeMap, tap} from "rxjs/operators";
 import {omit} from "ramda";
 
 import * as Database from "./lib/database";
@@ -15,12 +15,20 @@ import {
 } from "src/electron-preload/webview/constants";
 import {ONE_SECOND_MS} from "src/shared/constants";
 import {PROTONMAIL_IPC_WEBVIEW_API, ProtonmailApi, ProtonmailNotificationOutput} from "src/shared/api/webview/protonmail";
+import {StatusCodeError} from "src/shared/model/error";
 import {Unpacked} from "src/shared/types";
+import {angularJsHttpResponseTypeGuard, isUpsertOperationType, preprocessError} from "./lib/uilt";
 import {buildContact, buildFolder, buildMail} from "./lib/database";
+import {
+    buildDbPatchRetryPipeline,
+    buildEmptyDbPatch,
+    fillInputValue,
+    getLocationHref,
+    submitTotpToken,
+    waitElements,
+} from "src/electron-preload/webview/util";
 import {buildLoggerBundle} from "src/electron-preload/util";
 import {curryFunctionMembers, isEntityUpdatesPatchNotEmpty} from "src/shared/util";
-import {fillInputValue, getLocationHref, submitTotpToken, waitElements} from "src/electron-preload/webview/util";
-import {isAngularJsHttpResponse, isUpsertOperationType} from "./lib/uilt";
 
 const _logger = curryFunctionMembers(WEBVIEW_LOGGERS.protonmail, "[index]");
 const WINDOW = window as any; // TODO remove "as any" casting on https://github.com/Microsoft/TypeScript/issues/14701 resolving
@@ -76,7 +84,7 @@ delete WINDOW.Notification;
 const endpoints: ProtonmailApi = {
     ping: () => of(null),
 
-    buildDbPatch: (input) => from((async (logger = curryFunctionMembers(_logger, "api:buildDbPatch()", input.zoneName)) => {
+    buildDbPatch: (input) => defer(() => (async (logger = curryFunctionMembers(_logger, "api:buildDbPatch()", input.zoneName)) => {
         logger.info();
 
         if (!isLoggedIn()) {
@@ -87,45 +95,31 @@ const endpoints: ProtonmailApi = {
             return await bootstrapDbPatch();
         }
 
-        const {missedEvents, latestEventId, hasMoreEvents} = await (async (
+        const {missedEvents, latestEventId} = await (async (
             {events, $http}: Api,
             id: Rest.Model.Event["EventID"],
         ) => {
-            const bufferSize = 50;
             const fetchedEvents: Rest.Model.Event[] = [];
-            const state: { iteration: number; hasMoreEvents?: boolean; } = {iteration: 0};
-
             do {
-                state.iteration++;
-
                 const response = await events.get(id, {params: {[ajaxSendNotificationSkipParam]: ""}});
+                const hasMoreEvents = response.More === 1;
                 const sameNextId = id === response.EventID;
-
                 fetchedEvents.push(response);
                 id = response.EventID;
-
-                if (response.More !== 1) {
+                if (!hasMoreEvents) {
                     break;
                 }
-                if (sameNextId) {
-                    throw new Error(
-                        `Events API indicates that there is next event in the queue, but responses with the same "next event id"`,
-                    );
-                }
-                if (state.iteration < bufferSize) {
+                if (!sameNextId) {
                     continue;
                 }
-
-                state.hasMoreEvents = true;
-                logger.verbose(`breaking after ${state.iteration} iterations`);
+                throw new Error(
+                    `Events API indicates that there is a next event in the queue, but responses with the same "next event id"`,
+                );
             } while (true);
-
             logger.info(`fetched ${fetchedEvents.length} missed events`);
-
             return {
                 latestEventId: id,
                 missedEvents: fetchedEvents,
-                hasMoreEvents: state.hasMoreEvents,
             };
         })(await resolveApi(), input.metadata.latestEventId);
 
@@ -135,20 +129,18 @@ const endpoints: ProtonmailApi = {
         return {
             patch,
             metadata,
-            hasMoreEvents,
         };
     })()).pipe(
-        retryWhen((errors) => errors.pipe(
-            mergeMap((error) => {
-                if (isAngularJsHttpResponse(error) && error.status === -1) {
-                    _logger.verbose(`retry "buildDbPatch" on network -1 error`);
-                    return of(error);
-                }
-                return throwError(error);
-            }),
-            delay(ONE_SECOND_MS * 3),
-            take(3),
-        )),
+        buildDbPatchRetryPipeline<Unpacked<ReturnType<ProtonmailApi["buildDbPatch"]>>>(preprocessError, _logger),
+        catchError((error) => {
+            if (StatusCodeError.hasStatusCodeValue(error, "SkipDbPatch")) {
+                return of({
+                    patch: buildEmptyDbPatch(),
+                    metadata: {},
+                });
+            }
+            throw error;
+        }),
     ),
 
     fillLogin: ({login, zoneName}) => from((async (logger = curryFunctionMembers(_logger, "api:fillLogin()", zoneName)) => {
@@ -361,7 +353,7 @@ PROTONMAIL_IPC_WEBVIEW_API.registerApi(
         logger: {
             error: (args: any[]) => {
                 _logger.error(...args.map((arg) => {
-                    if (isAngularJsHttpResponse(arg)) {
+                    if (angularJsHttpResponseTypeGuard(arg)) {
                         return {
                             // omitting possibly sensitive properties
                             ...omit(["config", "headers", "data"], arg),

src/electron-preload/webview/protonmail/lib/uilt.ts

@@ -1,5 +1,8 @@
+import {pick} from "ramda";
+
 import * as Rest from "./rest";
-import {Unpacked} from "src/shared/types";
+import {Arguments, Unpacked} from "src/shared/types";
+import {buildDbPatchRetryPipeline} from "src/electron-preload/webview/util";
 
 export const isUpsertOperationType = (<V = Unpacked<typeof Rest.Model.EVENT_ACTION._.values>>(
     types: Set<V>,
@@ -12,18 +15,40 @@ export const isUpsertOperationType = (<V = Unpacked<typeof Rest.Model.EVENT_ACTI
     Rest.Model.EVENT_ACTION.UPDATE_FLAGS,
 ]));
 
-export const isAngularJsHttpResponse: (data: ng.IHttpResponse<any> | any) => data is ng.IHttpResponse<any> = ((
-    signatureKeys: Array<keyof ng.IHttpResponse<any>> = ["data", "status", "headers", "config", "statusText", "xhrStatus"],
+export const angularJsHttpResponseTypeGuard: (data: ng.IHttpResponse<any> | any) => data is ng.IHttpResponse<any> = ((
+    signatureKeys = Object.freeze<keyof ng.IHttpResponse<any>>(["data", "status", "config", "statusText", "xhrStatus"]),
 ) => {
     return ((data: ng.IHttpResponse<any> | any) => {
         if (typeof data !== "object") {
             return false;
         }
-        try {
-            data = JSON.parse(data);
-        } catch {
-            return false;
-        }
         return signatureKeys.reduce((count, prop) => count + Number(prop in data), 0) === signatureKeys.length;
-    }) as typeof isAngularJsHttpResponse;
+    }) as typeof angularJsHttpResponseTypeGuard;
 })();
+
+export const preprocessError: Arguments<typeof buildDbPatchRetryPipeline>[0] = (rawError: any) => {
+    const error = angularJsHttpResponseTypeGuard(rawError)
+        ? { // TODO add tests to validate that "angularJsHttpResponseTypeGuard" call on this error still return "true"
+            // whitelistening properties if error is "angular http response" object
+            // so information like http headers and params is filtered out
+            data: "<wiped out>",
+            config: pick(["method", "url"], rawError.config),
+            ...pick(["status", "statusText", "xhrStatus"], rawError),
+            message: rawError.statusText || `HTTP request error`,
+        }
+        : rawError;
+    const retriable = !navigator.onLine || (error !== rawError && (
+        // network connection error, connection abort, etc
+        error.status === -1
+        ||
+        // requests to Protonmail's API end up with "503 service unavailable" error quite often during the day
+        // so we retry/skip such errors in addition to the network errors with -1 status
+        (error.status === 503 && error.statusText === "Service Unavailable")
+    ));
+
+    return {
+        error,
+        retriable,
+        skippable: retriable,
+    };
+};