Skip to content

Commit

Permalink
add integration tests
Browse files Browse the repository at this point in the history
(c) Copyright IBM Corp. 2016
  • Loading branch information
peller committed Aug 5, 2016
1 parent d494ac5 commit 5a85062
Show file tree
Hide file tree
Showing 4 changed files with 242 additions and 4 deletions.
66 changes: 63 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

.PHONY: help init clean test system-test

ROOT_REPO:=jupyter/all-spark-notebook:258e25c03cba
CONTAINER_NAME:=declarativewidgets-explorer
REPO:=jupyter/declarativewidgets-explorer:258e25c03cba
Expand Down Expand Up @@ -40,10 +45,11 @@ node_modules: package.json
bower_components: node_modules bower.json
@npm run bower -- install $(BOWER_OPTS)

run: SERVER_NAME?=urth_explorer_server
run: PORT_MAP?=-p 8888:8888
run: CMD?=jupyter notebook --no-browser --port 8888 --ip="*"
run:
@docker $(DOCKER_OPTS) run --user root $(OPTIONS) \
@docker $(DOCKER_OPTS) run --user root $(OPTIONS) --name $(SERVER_NAME) \
$(PORT_MAP) \
-e USE_HTTP=1 \
-v `pwd`:/srv \
Expand All @@ -53,9 +59,63 @@ run:
$(REPO) bash -c '$(CMD)'

test: bower_components
@bower install ../declarativewidgets/elements/urth-core-behaviors/
@bower install ../declarativewidgets/elements/urth-viz-behaviors/
@bower install ../widgets/elements/urth-core-behaviors/
@bower install ../widgets/elements/urth-viz-behaviors/
@npm test

clean:
@-rm -rf bower_components node_modules

### System integration tests
BASEURL?=http://192.168.99.100:8888
BROWSER_LIST?=chrome
TEST_TYPE?=local
SPECS?=system-test/urth-viz-explorer-specs.js

remove-server:
-@docker $(DOCKER_OPTS) rm -f $(SERVER_NAME)

sdist:

run-test: SERVER_NAME?=urth_widgets_integration_test_server
run-test: sdist remove-server
@echo $(TEST_MSG)
@OPTIONS=-d SERVER_NAME=$(SERVER_NAME) $(MAKE) run
@echo 'Waiting for server to start...'
@LIMIT=60; while [ $$LIMIT -gt 0 ] && ! docker logs $(SERVER_NAME) 2>&1 | grep 'Notebook is running'; do echo waiting $$LIMIT...; sleep 1; LIMIT=$$(expr $$LIMIT - 1); done
@$(foreach browser, $(BROWSER_LIST), echo 'Running system integration tests on $(browser)...'; npm run system-test -- $(SPECS) --baseurl $(BASEURL) --test-type $(TEST_TYPE) --browser $(browser) || exit)
@SERVER_NAME=$(SERVER_NAME) $(MAKE) remove-server

system-test-python3: TEST_MSG="Starting system tests for Python 3"
system-test-python3:
TEST_MSG=$(TEST_MSG) TEST_TYPE=$(TEST_TYPE) BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) run-test

system-test-all: system-test-python3

start-selenium: node_modules stop-selenium
@echo "Installing and starting Selenium Server..."
@node_modules/selenium-standalone/bin/selenium-standalone install >/dev/null
@node_modules/selenium-standalone/bin/selenium-standalone start 2>/dev/null & echo $$! > SELENIUM_PID

stop-selenium:
-@kill `cat SELENIUM_PID`
-@rm SELENIUM_PID

system-test-all-local: TEST_TYPE:="local"
system-test-all-local: start-selenium system-test-all stop-selenium

system-test-all-remote: TEST_TYPE:="remote"
system-test-all-remote: system-test-all

system-test:
ifdef SAUCE_USER_NAME
@echo 'Running system tests on Sauce Labs...'
@BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-remote
else ifdef TRAVIS
@echo 'Starting system integration tests locally on Travis...'
@BROWSER_LIST="firefox" ALT_BROWSER_LIST="firefox" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-local
else
@echo 'Starting system integration tests locally...'
@BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-local
endif
@echo 'System integration tests complete.'
9 changes: 8 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,21 @@
"devDependencies": {
"bower": "^1.7.1",
"selenium-standalone": "latest",
"mocha" :"^2.0.0",
"wd": "latest",
"chai": "latest",
"chai-as-promised": "latest",
"colors": "latest",
"minimist": "latest",
"web-component-tester": "^4",
"wct-local": "2.0.3"
},
"config": {},
"scripts": {
"bower": "bower",
"test": "wct test/*",
"test-sauce": "wct --skip-plugin local --plugin sauce test/*"
"test-sauce": "wct --skip-plugin local --plugin sauce test/*",
"system-test": "node_modules/mocha/bin/mocha --timeout 500000 --reporter spec"
},
"repository": {
"type": "git",
Expand Down
49 changes: 49 additions & 0 deletions system-test/urth-viz-explorer-specs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright (c) Jupyter Development Team.
// Distributed under the terms of the Modified BSD License.

var wd = require('wd');
var chai = require('chai');
var Boilerplate = require('./utils/boilerplate');
var boilerplate = new Boilerplate();

describe('Urth Viz Explorer Test', function() {

var tagChaiAssertionError = function(err) {
// throw error and tag as retriable to poll again
err.retriable = err instanceof chai.AssertionError;
throw err;
};

wd.PromiseChainWebdriver.prototype.waitForWidgetElement = function(selector, browserSupportsShadowDOM, timeout, pollFreq) {
return this.waitForElementByCssSelector(
browserSupportsShadowDOM ? 'urth-viz-explorer::shadow urth-viz-vega::shadow svg' : 'urth-viz-explorer urth-viz-vega svg',
wd.asserters.isDisplayed,
timeout)
.catch(tagChaiAssertionError);
};

boilerplate.setup(this.title, '/notebooks/examples/urth-viz-explorer.ipynb');

it('should run all cells and find an explorer in the 5th output area', function(done) {
boilerplate.browser
.waitForElementsByCssSelector('div.output_area').nth(5)
.waitForWidgetElement("urth-viz-explorer", boilerplate.browserSupportsShadowDOM, 10000)
.nodeify(done);
});

it('should have 20 items plotted, by default, then 10 after setting the limit accordingly', function(done) {
boilerplate.browser
.waitForElementsByCssSelector('div.output_area').nth(7)
.moveTo()
.waitForElementsByCssSelector('urth-viz-explorer#v1::shadow urth-viz-vega::shadow svg .marks *', wd.asserters.isDisplayed, 10000)
.should.be.eventually.length(20)
.waitForElementByCssSelector('urth-viz-explorer#v1::shadow #viz-explorer-controls .viz-explorer-controls-section paper-input[label=Limit]::shadow paper-input-container #input', wd.asserters.isDisplayed, 10000)
.click()
.doubleclick()
.type('10')
.sleep(1000)
.elementsByCssSelector('urth-viz-explorer#v1::shadow urth-viz-vega::shadow svg .marks *')
.should.be.eventually.length(10)
.nodeify(done);
});
});
122 changes: 122 additions & 0 deletions system-test/utils/boilerplate.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright (c) Jupyter Development Team.
// Distributed under the terms of the Modified BSD License.

var wd = require('wd');
require('colors');
var chai = require('chai');
var chaiAsPromised = require('chai-as-promised');
var parseArgs = require('minimist');

// Parse the args
var args = parseArgs(process.argv);
args.local = args['test-type'] === 'local';
var remote = !args.local;
args.server = args.server || (remote ? 'ondemand.saucelabs.com' : 'localhost:4444');

// Setup chai
chai.use(chaiAsPromised);
chai.should();
chaiAsPromised.transferPromiseness = wd.transferPromiseness;

// Configure webdriver
wd.configureHttp({
timeout: 60000,
retryDelay: 15000,
retries: 5,
baseUrl: args.baseurl
});

// The browser capabilities we would like setup by selenium
var desired = {browserName: 'chrome', platform: 'OS X 10.10'};
if (args.browser && args.browser === 'InternetExplorer') {
args.browser = 'internet explorer';
}
desired.browserName = args.browser || desired.browserName;
desired.platform = args.platform ||
(desired.browserName === 'internet explorer' || desired.browserName === 'MicrosoftEdge') ?
'Windows 10': desired.platform;
desired.tags = ['widgets', 'system-test', desired.browserName];
// If there is a build number, include it in the desired attributes for sauce labs
if (process.env.TRAVIS_JOB_NUMBER) {
desired.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER;
desired.build = 'widgets-build-' + process.env.TRAVIS_JOB_NUMBER;
}

// The selenium test server, local or remote, we will be using to test against
var testServer = 'http://' + args.server + '/wd/hub';
if (remote) {
testServer = 'http://' + process.env.SAUCE_USERNAME + ':' + process.env.SAUCE_ACCESS_KEY + '@' + args.server + '/wd/hub';
}

console.log('Test server is ', args.server);
console.log('Travis job number is: ', process.env.TRAVIS_JOB_NUMBER);
console.log('Sauce user name is defined? ', !!process.env.SAUCE_USERNAME);
console.log('Sauce access key is defined? ', !!process.env.SAUCE_ACCESS_KEY);

/**
* A helper class to setup webdriver, create a browser, and widget objects for
* use within the system tests.
*/
var Boilerplate = function(){
this.browser = wd.promiseChainRemote(testServer);
this.allPassed = true;
this.SPECIAL_KEYS = wd.SPECIAL_KEYS;
};

/**
* Setups the before and after calls for each of your tests. The boilerplate
* will start each test on startingURL, which is a relative path to the resource to load.
*/
Boilerplate.prototype.setup = function(testName, startingURL){
var that = this;

before(function(done){
if (args.verbose) {
// optional logging
this.browser.on('status', function(info) {
console.log(info.cyan);
});
this.browser.on('command', function(meth, path, data) {
console.log(' > ' + meth.yellow, path.grey, data || '');
});
}

desired.name = testName ? 'Urth Widgets System Test - ' + testName
: 'Urth Widgets System Test';

this.browser.init(desired)
.get(startingURL || '/')
.waitForElementByCssSelector('#kernel_indicator_icon.kernel_idle_icon', wd.asserters.isDisplayed, 10000)
.waitForElementByLinkText('Cell', wd.asserters.isDisplayed, 10000)
.safeExecute('localStorage.clear()')
.elementByLinkText('Cell')
.click()
.waitForElementByLinkText('Run All', wd.asserters.isDisplayed, 10000)
.elementByLinkText('Run All')
.click()
.eval('!!document.body.createShadowRoot', function(err, value) {
this.browserSupportsShadowDOM = value;
}.bind(this))
.waitForElementByCssSelector('div.output_area', wd.asserters.isDisplayed, 10000)
.setAsyncScriptTimeout(15000)
.waitForConditionInBrowser('window.Urth && Urth.kernel && Urth.kernel.is_connected()', 10000)
.waitForElementByCssSelector('#kernel_indicator_icon.kernel_idle_icon', wd.asserters.isDisplayed, 20000)
.waitForConditionInBrowser('typeof Urth.whenReady === "function"', 10000)
.nodeify(done);
}.bind(this));

after(function(done){
var result = this.browser.quit();
if (remote) {
result = result.sauceJobStatus(this.allPassed);
}
result.nodeify(done);
}.bind(this));

afterEach(function(done) {
that.allPassed = that.allPassed && (this.currentTest.state === 'passed');
done();
});
};

module.exports = Boilerplate;

0 comments on commit 5a85062

Please sign in to comment.