- chore: implement new test reporter to see more details on failed tests

- fix: fix broken e2e tests on Android 8+

scripts/test-app.sh

@@ -10,5 +10,5 @@ fi
 
 printf 'Running e2e tests\n'
 pushd $ROOT
-./node_modules/.bin/mocha ./test/e2e-tooling/test.js "$@"
+./node_modules/.bin/mocha --reporter ./test/e2e-tooling/reporter.js ./test/e2e-tooling/test.js "$@"
 popd

test/e2e-app-template/network_security_config.xml

@@ -3,5 +3,7 @@
     <domain-config cleartextTrafficPermitted="true">
         <domain includeSubdomains="true">localhost</domain>
         <domain includeSubdomains="true">httpbin.org</domain>
+        <domain includeSubdomains="true">httpbingo.org</domain>
+        <domain includeSubdomains="true">www.columbia.edu</domain>
     </domain-config>
 </network-security-config>

test/e2e-tooling/reporter.js

@@ -0,0 +1,223 @@
+'use strict';
+
+const Mocha = require('mocha');
+const milliseconds = require('ms');
+const Base = Mocha.reporters.Base;
+const color = Base.color;
+
+const {
+  isString,
+  stringify,
+  inherits
+} = Mocha.utils;
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END
+} = Mocha.Runner.constants;
+
+function Spec(runner, options) {
+  Base.call(this, runner, options);
+
+  var self = this;
+  var indents = 0;
+  var n = 0;
+
+  function indent() {
+    return Array(indents).join('  ');
+  }
+
+  runner.on(EVENT_RUN_BEGIN, function() {
+    Base.consoleLog();
+  });
+
+  runner.on(EVENT_SUITE_BEGIN, function(suite) {
+    ++indents;
+    Base.consoleLog(color('suite', '%s%s'), indent(), suite.title);
+  });
+
+  runner.on(EVENT_SUITE_END, function() {
+    --indents;
+    if (indents === 1) {
+      Base.consoleLog();
+    }
+  });
+
+  runner.on(EVENT_TEST_PENDING, function(test) {
+    var fmt = indent() + color('pending', '  - %s');
+    Base.consoleLog(fmt, test.title);
+  });
+
+  runner.on(EVENT_TEST_PASS, function(test) {
+    var fmt;
+    if (test.speed === 'fast') {
+      fmt =
+        indent() +
+        color('checkmark', '  ' + Base.symbols.ok) +
+        color('pass', ' %s');
+      Base.consoleLog(fmt, test.title);
+    } else {
+      fmt =
+        indent() +
+        color('checkmark', '  ' + Base.symbols.ok) +
+        color('pass', ' %s') +
+        color(test.speed, ' (%dms)');
+      Base.consoleLog(fmt, test.title, test.duration);
+    }
+  });
+
+  runner.on(EVENT_TEST_FAIL, function(test) {
+    Base.consoleLog(indent() + color('fail', '  %d) %s'), ++n, test.title);
+  });
+
+  runner.once(EVENT_RUN_END, self.epilogue.bind(self));
+}
+
+/**
+ * Inherit from `Base.prototype`.
+ */
+inherits(Spec, Base);
+
+Spec.description = 'custom reporter for HTTP plugin testing';
+
+Spec.prototype.epilogue = function() {
+  var stats = this.stats;
+  var fmt;
+
+  Base.consoleLog();
+
+  // passes
+  fmt =
+    color('bright pass', ' ') +
+    color('green', ' %d passing') +
+    color('light', ' (%s)');
+
+  Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration));
+
+  // pending
+  if (stats.pending) {
+    fmt = color('pending', ' ') + color('pending', ' %d pending');
+
+    Base.consoleLog(fmt, stats.pending);
+  }
+
+  // failures
+  if (stats.failures) {
+    fmt = color('fail', '  %d failing');
+
+    Base.consoleLog(fmt, stats.failures);
+
+    this.showList(this.failures);
+    Base.consoleLog();
+  }
+
+  Base.consoleLog();
+};
+
+Spec.prototype.showList = function(failures) {
+  var multipleErr, multipleTest;
+  var self = this;
+
+  Base.consoleLog();
+  failures.forEach(function(test, i) {
+    // format
+    var fmt =
+      color('error title', '  %s) %s:\n') +
+      color('error message', '     %s') +
+      color('error stack', '\n%s\n');
+
+    // msg
+    var msg;
+    var err;
+    if (test.err && test.err.multiple) {
+      if (multipleTest !== test) {
+        multipleTest = test;
+        multipleErr = [test.err].concat(test.err.multiple);
+      }
+      err = multipleErr.shift();
+    } else {
+      err = test.err;
+    }
+    var message;
+    if (err.message && typeof err.message.toString === 'function') {
+      message = err.message + '';
+    } else if (typeof err.inspect === 'function') {
+      message = err.inspect() + '';
+    } else {
+      message = '';
+    }
+    var stack = err.stack || message;
+    var index = message ? stack.indexOf(message) : -1;
+
+    if (index === -1) {
+      msg = message;
+    } else {
+      index += message.length;
+      msg = stack.slice(0, index);
+      // remove msg from stack
+      stack = stack.slice(index + 1);
+    }
+
+    // uncaught
+    if (err.uncaught) {
+      msg = 'Uncaught ' + msg;
+    }
+    // explicitly show diff
+    if (Base.showDiff(err)) {
+      self.stringifyDiffObjs(err);
+      fmt =
+        color('error title', '  %s) %s:\n%s') + color('error stack', '\n%s\n');
+      var match = message.match(/^([^:]+): expected/);
+      msg = '\n      ' + color('error message', match ? match[1] : msg);
+
+      msg += Base.generateDiff(err.actual, err.expected);
+    }
+
+    // indent stack trace
+    stack = stack.replace(/^/gm, '  ');
+
+    // indented test title
+    var testTitle = '';
+    test.titlePath().forEach(function(str, index) {
+      if (index !== 0) {
+        testTitle += '\n     ';
+      }
+      for (var i = 0; i < index; i++) {
+        testTitle += '  ';
+      }
+      testTitle += str;
+    });
+
+    Base.consoleLog(fmt, i + 1, testTitle, msg, stack);
+    self.showDetails(err);
+  });
+};
+
+Spec.prototype.stringifyDiffObjs = function(err) {
+  if (!isString(err.actual) || !isString(err.expected)) {
+    err.actual = stringify(err.actual);
+    err.expected = stringify(err.expected);
+  }
+}
+
+Spec.prototype.showDetails = function(err) {
+  if (!err.details) {
+    return;
+  }
+
+  const details = JSON
+    .stringify(err.details, null, 2)
+    .replace(/^/gm, '        ');
+
+  Base.consoleLog(
+    color('error stack', '\n      Details:\n%s'),
+    details
+  );
+}
+
+exports = module.exports = Spec;

test/e2e-tooling/test.js

@@ -7,6 +7,9 @@ const testDefinitions = require('../e2e-specs');
 
 global.should = chai.should();
 
+let driver;
+let allPassed = true;
+
 describe('Advanced HTTP e2e test suite', function () {
   const isSauceLabs = !!process.env.SAUCE_USERNAME;
   const isBrowserStack = !!process.env.BROWSERSTACK_USERNAME;
@@ -17,9 +20,6 @@ describe('Advanced HTTP e2e test suite', function () {
   const targetInfo = { isSauceLabs, isBrowserStack, isDevice, isAndroid };
   const environment = isSauceLabs ? 'saucelabs' : isBrowserStack ? 'browserstack' : 'local';
 
-  let driver;
-  let allPassed = true;
-
   this.timeout(15000);
   this.slow(4000);
 
@@ -121,7 +121,15 @@ async function waitToBeFinished(driver, timeout) {
 
 async function validateResult(driver, validationFunc, targetInfo) {
   const result = await driver.safeExecute('app.lastResult');
-  validationFunc(driver, result, targetInfo);
+
+  try {
+    validationFunc(driver, result, targetInfo);
+  } catch (error) {
+    allPassed = false;
+    error.details = result;
+
+    throw error;
+  }
 }
 
 async function checkSkipped(driver) {