From c1948fc0d40c30c78ff0a3c22d117f9305d5ea3e Mon Sep 17 00:00:00 2001 From: Alexander Sorokin Date: Mon, 22 Feb 2016 18:37:41 +0300 Subject: [PATCH] CB-10639 Appium tests: Added some timeouts, Taking a screenshot on failure, Retry taking a picture up to 3 times, Try to restart the Appium session if it's lost --- appium-tests/android/android.spec.js | 174 ++++++++++++++--------- appium-tests/helpers/cameraHelper.js | 2 +- appium-tests/helpers/screenshotHelper.js | 37 +++++ appium-tests/helpers/wdHelper.js | 7 +- appium-tests/ios/ios.spec.js | 38 +++-- 5 files changed, 176 insertions(+), 82 deletions(-) create mode 100644 appium-tests/helpers/screenshotHelper.js diff --git a/appium-tests/android/android.spec.js b/appium-tests/android/android.spec.js index 2690ca6..ad5ffa3 100644 --- a/appium-tests/android/android.spec.js +++ b/appium-tests/android/android.spec.js @@ -1,36 +1,43 @@ -/*jslint node: true, plusplus: true */ -/*global beforeEach, afterEach */ -/*global describe, it, xit, expect, jasmine */ -'use strict'; +/*jshint node: true, jasmine: true */ // these tests are meant to be executed by Cordova Medic Appium runner // you can find it here: https://github.com/apache/cordova-medic/ // it is not necessary to do a full CI setup to run these tests // just run "node cordova-medic/medic/medic.js appium --platform android --plugins cordova-plugin-camera" +'use strict'; + var wdHelper = require('../helpers/wdHelper'); var wd = wdHelper.getWD(); var cameraConstants = require('../../www/CameraConstants'); var cameraHelper = require('../helpers/cameraHelper'); +var screenshotHelper = require('../helpers/screenshotHelper'); + +var STARTING_MESSAGE = 'Ready for action!'; +var RETRY_COUNT = 3; // how many times to retry taking a picture before failing +var MINUTE = 60 * 1000; +var DEFAULT_SCREEN_WIDTH = 360; +var DEFAULT_SCREEN_HEIGHT = 567; +var DEFAULT_WEBVIEW_CONTEXT = 'WEBVIEW'; describe('Camera tests Android.', function () { - var driver, - startingMessage = 'Ready for action!', - // the name of webview context, it will be changed to match needed context if there are named ones: - webviewContext = 'WEBVIEW', - // this indicates if device library has test picture: - isTestPictureSaved = false, - // this indecates that there was critical error and tests cannot continue: - stopFlag = false, - // we need to know the screen width and height to properly click on the first image in the gallery - screenWidth = 360, - screenHeight = 567; + var driver; + // the name of webview context, it will be changed to match needed context if there are named ones: + var webviewContext = DEFAULT_WEBVIEW_CONTEXT; + // this indicates that the device library has the test picture: + var isTestPictureSaved = false; + // this indicates that there was a critical error and tests cannot continue: + var stopFlag = false; + // we need to know the screen width and height to properly click on an image in the gallery + var screenWidth = DEFAULT_SCREEN_WIDTH; + var screenHeight = DEFAULT_SCREEN_HEIGHT; function win() { expect(true).toBe(true); } function fail(error) { + screenshotHelper.saveScreenshot(driver); if (error && error.message) { console.log('An error occured: ' + error.message); expect(true).toFailWithMessage(error.message); @@ -42,11 +49,12 @@ describe('Camera tests Android.', function () { throw error; } // no message provided :( - console.log('An error without description occured'); expect(true).toBe(false); throw 'An error without description occured'; } + // generates test specs by combining all the specified options + // you can add more options to test more scenarios function generateSpecs() { var sourceTypes = [ cameraConstants.PictureSourceType.CAMERA, @@ -65,17 +73,22 @@ describe('Camera tests Android.', function () { return cameraHelper.generateSpecs(sourceTypes, destinationTypes, encodingTypes, allowEditOptions); } - function getPicture(options, skipUiInteractions) { + function getPicture(options, skipUiInteractions, retry) { if (!options) { options = {}; } + if (typeof retry === 'undefined') { + retry = 1; + } + var command = "navigator.camera.getPicture(function (result) { document.getElementById('info').innerHTML = result.slice(0, 100); }, " + "function (err) { document.getElementById('info').innerHTML = 'ERROR: ' + err; }," + JSON.stringify(options) + ");"; return driver .context(webviewContext) .execute(command) - .sleep(5000) + .sleep(7000) .context('NATIVE_APP') + .sleep(5000) .then(function () { if (skipUiInteractions) { return; @@ -105,8 +118,10 @@ describe('Camera tests Android.', function () { return driver .elementByXPath('//android.widget.ImageView[contains(@resource-id,\'shutter\')]') .click() + .sleep(3000) .elementByXPath('//android.widget.ImageView[contains(@resource-id,\'done\')]') - .click(); + .click() + .sleep(10000); }) .then(function () { if (skipUiInteractions) { @@ -122,25 +137,24 @@ describe('Camera tests Android.', function () { if (!skipUiInteractions) { return driver.sleep(10000); } + }) + .fail(function (error) { + if (retry < RETRY_COUNT) { + console.log('Failed to get a picture. Let\'s try it again... '); + return getPicture(options, skipUiInteractions, ++retry); + } else { + console.log('Tried ' + RETRY_COUNT + ' times but couldn\'t get the picture. Failing...'); + fail(error); + } }); } function enterTest() { - if (stopFlag) { - return driver - .context(webviewContext) - .then(function () { - throw 'stopFlag is on!'; - }); - } return driver // trying to determine where we are .context(webviewContext) - .then(function (result) { - console.log(result); - }) .fail(function (error) { - expect(true).toFailWithMessage(error); + fail(error); }) .elementById('info') .then(function () { @@ -175,16 +189,16 @@ describe('Camera tests Android.', function () { .elementById('info') .getAttribute('innerHTML') .then(function (html) { - if (html.indexOf(startingMessage) >= 0) { + if (html.indexOf(STARTING_MESSAGE) >= 0) { expect(true).toFailWithMessage('No callback was fired'); } else if (shouldLoad) { expect(html.length).toBeGreaterThan(0); if (html.indexOf('ERROR') >= 0) { - expect(true).toFailWithMessage(html); + fail(html); } } else { if (html.indexOf('ERROR') === -1) { - expect(true).toFailWithMessage('Unexpected success callback with result: ' + html); + fail('Unexpected success callback with result: ' + html); } expect(html.indexOf('ERROR')).toBe(0); } @@ -219,6 +233,18 @@ describe('Camera tests Android.', function () { }); } + function getDriver() { + driver = wdHelper.getDriver('Android'); + return driver; + } + + function checkStopFlag() { + if (stopFlag) { + fail('Something went wrong: the stopFlag is on. Please see the log for more details.'); + } + return stopFlag; + } + beforeEach(function () { jasmine.addMatchers({ toFailWithMessage : function () { @@ -229,7 +255,12 @@ describe('Camera tests Android.', function () { pass: false, message: msg }; - if (msg.indexOf('Error response status: 6') >= 0) { + // status 6 means that we've lost the session + // status 7 means that Appium couldn't find an element + // both these statuses mean that the test has failed but + // we should try to recreate the session for the following tests + if (msg.indexOf('Error response status: 6') >= 0 || + msg.indexOf('Error response status: 7') >= 0) { stopFlag = true; } return result; @@ -240,12 +271,8 @@ describe('Camera tests Android.', function () { }); it('camera.ui.util configuring driver and starting a session', function (done) { - driver = wdHelper.getDriver('Android', function () { - return driver - .sleep(10000) - .finally(done); - }); - }, 320000); + getDriver().then(done); + }, 5 * MINUTE); it('camera.ui.util determine webview context name', function (done) { var i = 0; @@ -261,7 +288,7 @@ describe('Camera tests Android.', function () { } done(); }); - }, 30000); + }, MINUTE); it('camera.ui.util determine screen dimensions', function (done) { return enterTest() @@ -270,22 +297,22 @@ describe('Camera tests Android.', function () { .elementById('info') .getAttribute('innerHTML') .then(function (html) { - if (html !== startingMessage) { + if (html !== STARTING_MESSAGE) { screenWidth = Number(html); } }) - .execute('document.getElementById(\'info\').innerHTML = \'' + startingMessage + '\';') + .execute('document.getElementById(\'info\').innerHTML = \'' + STARTING_MESSAGE + '\';') .execute('document.getElementById(\'info\').innerHTML = window.innerHeight;') .sleep(5000) .elementById('info') .getAttribute('innerHTML') .then(function (html) { - if (html !== startingMessage) { + if (html !== STARTING_MESSAGE) { screenHeight = Number(html); } done(); }); - }, 60000); + }, MINUTE); describe('Specs.', function () { beforeEach(function (done) { @@ -295,13 +322,36 @@ describe('Camera tests Android.', function () { .then(function () { return driver; // no-op }, function (error) { - expect(true).toFailWithMessage(error); + if (error.message.indexOf('Error response status: 6') >= 0) { + // the session has expired but we can fix this! + console.log('The session has expired. Trying to start a new one...'); + return getDriver(); + } else { + expect(true).toFailWithMessage(error); + } }) - .execute('document.getElementById("info").innerHTML = "' + startingMessage + '";') + .execute('document.getElementById("info").innerHTML = "' + STARTING_MESSAGE + '";') .finally(done); } done(); - }, 600000); + }, 3 * MINUTE); + + afterEach(function (done) { + if (!stopFlag) { + done(); + return; + } + // recreate the session if there was a critical error in the spec + return driver + .quit() + .then(function () { + return getDriver() + .then(function () { + stopFlag = false; + done(); + }); + }); + }, 3 * MINUTE); // getPicture() with saveToPhotoLibrary = true it('camera.ui.spec.1 Saving the picture to photo library', function (done) { @@ -322,12 +372,11 @@ describe('Camera tests Android.', function () { }) .then(win, fail) .finally(done); - }, 300000); + }, 3 * MINUTE); // getPicture() with mediaType: VIDEO, sourceType: PHOTOLIBRARY it('camera.ui.spec.2 Selecting only videos', function (done) { - if (stopFlag) { - expect(true).toFailWithMessage('Couldn\'t start tests execution.'); + if (checkStopFlag()) { done(); return; } @@ -347,6 +396,7 @@ describe('Camera tests Android.', function () { } }) .context('NATIVE_APP') + .sleep(5000) .then(function () { // try to find "Gallery" menu item // if there's none, the gallery should be already opened @@ -383,13 +433,12 @@ describe('Camera tests Android.', function () { return driver; }) .finally(done); - }, 300000); + }, 3 * MINUTE); // getPicture(), then dismiss // wait for the error callback to bee called it('camera.ui.spec.3 Dismissing the camera', function (done) { - if (stopFlag) { - expect(true).toFailWithMessage('Couldn\'t start tests execution.'); + if (checkStopFlag()) { done(); return; } @@ -418,13 +467,12 @@ describe('Camera tests Android.', function () { }) .then(win, fail) .finally(done); - }, 300000); + }, 3 * MINUTE); // getPicture(), then take picture but dismiss the edit // wait for the error cllback to be called it('camera.ui.spec.4 Dismissing the edit', function (done) { - if (stopFlag) { - expect(true).toFailWithMessage('Couldn\'t start tests execution.'); + if (checkStopFlag()) { done(); return; } @@ -458,24 +506,22 @@ describe('Camera tests Android.', function () { }) .then(win, fail) .finally(done); - }, 300000); + }, 3 * MINUTE); // combine various options for getPicture() generateSpecs().forEach(function (spec) { it('camera.ui.spec.5.' + spec.id + ' Combining options', function (done) { - if (stopFlag) { - expect(true).toFailWithMessage('Couldn\'t start tests execution.'); + if (checkStopFlag()) { done(); return; } runCombinedSpec(spec).then(done); - }, 3 * 60 * 1000); + }, 3 * MINUTE); }); it('camera.ui.util Delete test image from device library', function (done) { - if (stopFlag) { - expect(true).toFailWithMessage('Couldn\'t start tests executeion.'); + if (checkStopFlag()) { done(); return; } @@ -503,7 +549,7 @@ describe('Camera tests Android.', function () { } // couldn't save test picture earlier, so nothing to delete here done(); - }, 300000); + }, 3 * MINUTE); }); diff --git a/appium-tests/helpers/cameraHelper.js b/appium-tests/helpers/cameraHelper.js index b2a631a..5edd143 100644 --- a/appium-tests/helpers/cameraHelper.js +++ b/appium-tests/helpers/cameraHelper.js @@ -1,4 +1,4 @@ -/*jslint node: true, plusplus: true */ +/*jshint node: true */ 'use strict'; var cameraConstants = require('../../www/CameraConstants'); diff --git a/appium-tests/helpers/screenshotHelper.js b/appium-tests/helpers/screenshotHelper.js new file mode 100644 index 0000000..2bbbe4a --- /dev/null +++ b/appium-tests/helpers/screenshotHelper.js @@ -0,0 +1,37 @@ +/* jshint node: true */ +'use strict'; + +var path = require('path'); +var screenshotPath = global.SCREENSHOT_PATH || path.join(__dirname, '../../appium_screenshots/'); + +function generateScreenshotName() { + var date = new Date(); + + var month = date.getMonth() + 1; + var day = date.getDate(); + var hour = date.getHours(); + var min = date.getMinutes(); + var sec = date.getSeconds(); + + month = (month < 10 ? "0" : "") + month; + day = (day < 10 ? "0" : "") + day; + hour = (hour < 10 ? "0" : "") + hour; + min = (min < 10 ? "0" : "") + min; + sec = (sec < 10 ? "0" : "") + sec; + + return date.getFullYear() + '-' + month + '-' + day + '_' + hour + '.' + min + '.' + sec + '.png'; +} + +module.exports.saveScreenshot = function (driver) { + var oldContext; + return driver + .currentContext() + .then(function (cc) { + oldContext = cc; + }) + .context('NATIVE_APP') + .saveScreenshot(screenshotPath + generateScreenshotName()) + .then(function () { + return driver.context(oldContext); + }); +}; diff --git a/appium-tests/helpers/wdHelper.js b/appium-tests/helpers/wdHelper.js index c97708e..8689d40 100644 --- a/appium-tests/helpers/wdHelper.js +++ b/appium-tests/helpers/wdHelper.js @@ -1,4 +1,4 @@ -/*jslint node: true, plusplus: true */ +/* jshint node: true */ 'use strict'; var wd = global.WD || require('wd'); @@ -24,11 +24,10 @@ module.exports.getDriver = function (platform, callback) { } driver = wd.promiseChainRemote(serverConfig); module.exports.configureLogging(driver); - driver.init(driverConfig).setImplicitWaitTimeout(10000) + + return driver.init(driverConfig).setImplicitWaitTimeout(10000) .sleep(20000) // wait for the app to load .then(callback); - - return driver; }; module.exports.getWD = function () { diff --git a/appium-tests/ios/ios.spec.js b/appium-tests/ios/ios.spec.js index c880413..4da18e8 100644 --- a/appium-tests/ios/ios.spec.js +++ b/appium-tests/ios/ios.spec.js @@ -1,6 +1,10 @@ -/*jslint node: true, plusplus: true */ -/*global beforeEach, afterEach */ -/*global describe, it, xit, expect, jasmine, pending */ +/*jshint node: true, jasmine: true */ + +// these tests are meant to be executed by Cordova Medic Appium runner +// you can find it here: https://github.com/apache/cordova-medic/ +// it is not necessary to do a full CI setup to run these tests +// just run "node cordova-medic/medic/medic.js appium --platform android --plugins cordova-plugin-camera" + 'use strict'; var wdHelper = require('../helpers/wdHelper'); @@ -8,32 +12,40 @@ var wd = wdHelper.getWD(); var isDevice = global.DEVICE; var cameraConstants = require('../../www/CameraConstants'); var cameraHelper = require('../helpers/cameraHelper'); +var screenshotHelper = require('../helpers/screenshotHelper'); + +var MINUTE = 60 * 1000; +var DEFAULT_WEBVIEW_CONTEXT = 'WEBVIEW_1'; describe('Camera tests iOS.', function () { - var driver, - webviewContext = 'WEBVIEW_1', - startingMessage = 'Ready for action!'; + + var driver; + var webviewContext = DEFAULT_WEBVIEW_CONTEXT; + var startingMessage = 'Ready for action!'; function win() { expect(true).toBe(true); } function fail(error) { + screenshotHelper.saveScreenshot(driver); if (error && error.message) { console.log('An error occured: ' + error.message); expect(true).toFailWithMessage(error.message); - return; + throw error.message; } if (error) { console.log('Failed expectation: ' + error); expect(true).toFailWithMessage(error); - return; + throw error; } // no message provided :( - console.log('An error without description occured'); expect(true).toBe(false); + throw 'An error without description occured'; } + // generates test specs by combining all the specified options + // you can add more options to test more scenarios function generateSpecs() { var sourceTypes = [ cameraConstants.PictureSourceType.CAMERA, @@ -198,7 +210,7 @@ describe('Camera tests iOS.', function () { it('camera.ui.util Configuring driver and starting a session', function (done) { driver = wdHelper.getDriver('iOS', done); - }, 240000); + }, 3 * MINUTE); describe('Specs.', function () { // getPicture() with mediaType: VIDEO, sourceType: PHOTOLIBRARY @@ -213,7 +225,7 @@ describe('Camera tests iOS.', function () { .elementByName('Cancel') .click() .finally(done); - }, 300000); + }, 3 * MINUTE); // getPicture(), then dismiss // wait for the error callback to bee called @@ -235,7 +247,7 @@ describe('Camera tests iOS.', function () { return checkPicture(false); }, fail) .finally(done); - }, 300000); + }, 3 * MINUTE); // combine various options for getPicture() generateSpecs().forEach(function (spec) { @@ -245,7 +257,7 @@ describe('Camera tests iOS.', function () { pending(); } runCombinedSpec(spec).then(done); - }, 3 * 60 * 1000); + }, 3 * MINUTE); }); });