diff --git a/scripts/test-app.sh b/scripts/test-app.sh index 4482e5e7..6ebe96d5 100755 --- a/scripts/test-app.sh +++ b/scripts/test-app.sh @@ -10,5 +10,5 @@ fi printf 'Running e2e tests\n' pushd $ROOT -./node_modules/.bin/mocha ./test/e2e-tooling/test.js "$@" +./node_modules/.bin/mocha --reporter ./test/e2e-tooling/reporter.js ./test/e2e-tooling/test.js "$@" popd diff --git a/test/e2e-app-template/network_security_config.xml b/test/e2e-app-template/network_security_config.xml index d908a810..9aacfa56 100644 --- a/test/e2e-app-template/network_security_config.xml +++ b/test/e2e-app-template/network_security_config.xml @@ -3,5 +3,7 @@ localhost httpbin.org + httpbingo.org + www.columbia.edu diff --git a/test/e2e-tooling/reporter.js b/test/e2e-tooling/reporter.js new file mode 100644 index 00000000..e58faba7 --- /dev/null +++ b/test/e2e-tooling/reporter.js @@ -0,0 +1,223 @@ +'use strict'; + +const Mocha = require('mocha'); +const milliseconds = require('ms'); +const Base = Mocha.reporters.Base; +const color = Base.color; + +const { + isString, + stringify, + inherits +} = Mocha.utils; + +const { + EVENT_RUN_BEGIN, + EVENT_RUN_END, + EVENT_TEST_FAIL, + EVENT_TEST_PASS, + EVENT_TEST_PENDING, + EVENT_SUITE_BEGIN, + EVENT_SUITE_END +} = Mocha.Runner.constants; + +function Spec(runner, options) { + Base.call(this, runner, options); + + var self = this; + var indents = 0; + var n = 0; + + function indent() { + return Array(indents).join(' '); + } + + runner.on(EVENT_RUN_BEGIN, function() { + Base.consoleLog(); + }); + + runner.on(EVENT_SUITE_BEGIN, function(suite) { + ++indents; + Base.consoleLog(color('suite', '%s%s'), indent(), suite.title); + }); + + runner.on(EVENT_SUITE_END, function() { + --indents; + if (indents === 1) { + Base.consoleLog(); + } + }); + + runner.on(EVENT_TEST_PENDING, function(test) { + var fmt = indent() + color('pending', ' - %s'); + Base.consoleLog(fmt, test.title); + }); + + runner.on(EVENT_TEST_PASS, function(test) { + var fmt; + if (test.speed === 'fast') { + fmt = + indent() + + color('checkmark', ' ' + Base.symbols.ok) + + color('pass', ' %s'); + Base.consoleLog(fmt, test.title); + } else { + fmt = + indent() + + color('checkmark', ' ' + Base.symbols.ok) + + color('pass', ' %s') + + color(test.speed, ' (%dms)'); + Base.consoleLog(fmt, test.title, test.duration); + } + }); + + runner.on(EVENT_TEST_FAIL, function(test) { + Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title); + }); + + runner.once(EVENT_RUN_END, self.epilogue.bind(self)); +} + +/** + * Inherit from `Base.prototype`. + */ +inherits(Spec, Base); + +Spec.description = 'custom reporter for HTTP plugin testing'; + +Spec.prototype.epilogue = function() { + var stats = this.stats; + var fmt; + + Base.consoleLog(); + + // passes + fmt = + color('bright pass', ' ') + + color('green', ' %d passing') + + color('light', ' (%s)'); + + Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration)); + + // pending + if (stats.pending) { + fmt = color('pending', ' ') + color('pending', ' %d pending'); + + Base.consoleLog(fmt, stats.pending); + } + + // failures + if (stats.failures) { + fmt = color('fail', ' %d failing'); + + Base.consoleLog(fmt, stats.failures); + + this.showList(this.failures); + Base.consoleLog(); + } + + Base.consoleLog(); +}; + +Spec.prototype.showList = function(failures) { + var multipleErr, multipleTest; + var self = this; + + Base.consoleLog(); + failures.forEach(function(test, i) { + // format + var fmt = + color('error title', ' %s) %s:\n') + + color('error message', ' %s') + + color('error stack', '\n%s\n'); + + // msg + var msg; + var err; + if (test.err && test.err.multiple) { + if (multipleTest !== test) { + multipleTest = test; + multipleErr = [test.err].concat(test.err.multiple); + } + err = multipleErr.shift(); + } else { + err = test.err; + } + var message; + if (err.message && typeof err.message.toString === 'function') { + message = err.message + ''; + } else if (typeof err.inspect === 'function') { + message = err.inspect() + ''; + } else { + message = ''; + } + var stack = err.stack || message; + var index = message ? stack.indexOf(message) : -1; + + if (index === -1) { + msg = message; + } else { + index += message.length; + msg = stack.slice(0, index); + // remove msg from stack + stack = stack.slice(index + 1); + } + + // uncaught + if (err.uncaught) { + msg = 'Uncaught ' + msg; + } + // explicitly show diff + if (Base.showDiff(err)) { + self.stringifyDiffObjs(err); + fmt = + color('error title', ' %s) %s:\n%s') + color('error stack', '\n%s\n'); + var match = message.match(/^([^:]+): expected/); + msg = '\n ' + color('error message', match ? match[1] : msg); + + msg += Base.generateDiff(err.actual, err.expected); + } + + // indent stack trace + stack = stack.replace(/^/gm, ' '); + + // indented test title + var testTitle = ''; + test.titlePath().forEach(function(str, index) { + if (index !== 0) { + testTitle += '\n '; + } + for (var i = 0; i < index; i++) { + testTitle += ' '; + } + testTitle += str; + }); + + Base.consoleLog(fmt, i + 1, testTitle, msg, stack); + self.showDetails(err); + }); +}; + +Spec.prototype.stringifyDiffObjs = function(err) { + if (!isString(err.actual) || !isString(err.expected)) { + err.actual = stringify(err.actual); + err.expected = stringify(err.expected); + } +} + +Spec.prototype.showDetails = function(err) { + if (!err.details) { + return; + } + + const details = JSON + .stringify(err.details, null, 2) + .replace(/^/gm, ' '); + + Base.consoleLog( + color('error stack', '\n Details:\n%s'), + details + ); +} + +exports = module.exports = Spec; diff --git a/test/e2e-tooling/test.js b/test/e2e-tooling/test.js index 30a7f140..11bfe278 100644 --- a/test/e2e-tooling/test.js +++ b/test/e2e-tooling/test.js @@ -7,6 +7,9 @@ const testDefinitions = require('../e2e-specs'); global.should = chai.should(); +let driver; +let allPassed = true; + describe('Advanced HTTP e2e test suite', function () { const isSauceLabs = !!process.env.SAUCE_USERNAME; const isBrowserStack = !!process.env.BROWSERSTACK_USERNAME; @@ -17,9 +20,6 @@ describe('Advanced HTTP e2e test suite', function () { const targetInfo = { isSauceLabs, isBrowserStack, isDevice, isAndroid }; const environment = isSauceLabs ? 'saucelabs' : isBrowserStack ? 'browserstack' : 'local'; - let driver; - let allPassed = true; - this.timeout(15000); this.slow(4000); @@ -121,7 +121,15 @@ async function waitToBeFinished(driver, timeout) { async function validateResult(driver, validationFunc, targetInfo) { const result = await driver.safeExecute('app.lastResult'); - validationFunc(driver, result, targetInfo); + + try { + validationFunc(driver, result, targetInfo); + } catch (error) { + allPassed = false; + error.details = result; + + throw error; + } } async function checkSkipped(driver) {