[WIP] add integration tests

Makefile

@@ -1,3 +1,8 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+.PHONY: help init clean test system-test
+
 ROOT_REPO:=jupyter/all-spark-notebook:258e25c03cba
 CONTAINER_NAME:=declarativewidgets-explorer
 REPO:=jupyter/declarativewidgets-explorer:258e25c03cba
@@ -40,10 +45,11 @@ node_modules: package.json
 bower_components: node_modules bower.json
 	@npm run bower -- install $(BOWER_OPTS)
 
+run: SERVER_NAME?=urth_explorer_server
 run: PORT_MAP?=-p 8888:8888
 run: CMD?=jupyter notebook --no-browser --port 8888 --ip="*"
 run:
-	@docker $(DOCKER_OPTS) run --user root $(OPTIONS) \
+	@docker $(DOCKER_OPTS) run --user root $(OPTIONS) --name $(SERVER_NAME) \
 		$(PORT_MAP) \
 		-e USE_HTTP=1 \
 		-v `pwd`:/srv \
@@ -53,9 +59,63 @@ run:
 		$(REPO) bash -c '$(CMD)'
 
 test: bower_components
-	@bower install ../declarativewidgets/elements/urth-core-behaviors/
-	@bower install ../declarativewidgets/elements/urth-viz-behaviors/
+	@bower install ../widgets/elements/urth-core-behaviors/
+	@bower install ../widgets/elements/urth-viz-behaviors/
 	@npm test
 
 clean:
 	@-rm -rf bower_components node_modules
+
+### System integration tests
+BASEURL?=http://192.168.99.100:8888
+BROWSER_LIST?=chrome
+TEST_TYPE?=local
+SPECS?=system-test/urth-viz-explorer-specs.js
+
+remove-server:
+	-@docker $(DOCKER_OPTS) rm -f $(SERVER_NAME)
+
+sdist:
+
+run-test: SERVER_NAME?=urth_widgets_integration_test_server
+run-test: sdist remove-server
+	@echo $(TEST_MSG)
+	@OPTIONS=-d SERVER_NAME=$(SERVER_NAME) $(MAKE) run
+	@echo 'Waiting for server to start...'
+	@LIMIT=60; while [ $$LIMIT -gt 0 ] && ! docker logs $(SERVER_NAME) 2>&1 | grep 'Notebook is running'; do echo waiting $$LIMIT...; sleep 1; LIMIT=$$(expr $$LIMIT - 1); done
+	@$(foreach browser, $(BROWSER_LIST), echo 'Running system integration tests on $(browser)...'; npm run system-test -- $(SPECS) --baseurl $(BASEURL) --test-type $(TEST_TYPE) --browser $(browser) || exit)
+	@SERVER_NAME=$(SERVER_NAME) $(MAKE) remove-server
+
+system-test-python3: TEST_MSG="Starting system tests for Python 3"
+system-test-python3:
+	TEST_MSG=$(TEST_MSG) TEST_TYPE=$(TEST_TYPE) BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) run-test
+
+system-test-all: system-test-python3
+
+start-selenium: node_modules stop-selenium
+	@echo "Installing and starting Selenium Server..."
+	@node_modules/selenium-standalone/bin/selenium-standalone install >/dev/null
+	@node_modules/selenium-standalone/bin/selenium-standalone start 2>/dev/null & echo $$! > SELENIUM_PID
+
+stop-selenium:
+	-@kill `cat SELENIUM_PID`
+	-@rm SELENIUM_PID
+
+system-test-all-local: TEST_TYPE:="local"
+system-test-all-local: start-selenium system-test-all stop-selenium
+
+system-test-all-remote: TEST_TYPE:="remote"
+system-test-all-remote: system-test-all
+
+system-test:
+ifdef SAUCE_USER_NAME
+	@echo 'Running system tests on Sauce Labs...'
+	@BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-remote
+else ifdef TRAVIS
+	@echo 'Starting system integration tests locally on Travis...'
+	@BROWSER_LIST="firefox" ALT_BROWSER_LIST="firefox" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-local
+else
+	@echo 'Starting system integration tests locally...'
+	@BROWSER_LIST="$(BROWSER_LIST)" JUPYTER=$(JUPYTER) SPECS="$(SPECS)" BASEURL=$(BASEURL) $(MAKE) system-test-all-local
+endif
+	@echo 'System integration tests complete.'

package.json

@@ -5,14 +5,21 @@
   "devDependencies": {
     "bower": "^1.7.1",
     "selenium-standalone": "latest",
+    "mocha" :"^2.0.0",
+    "wd": "latest",
+    "chai": "latest",
+    "chai-as-promised": "latest",
+    "colors": "latest",
+    "minimist": "latest",
     "web-component-tester": "^4",
     "wct-local": "2.0.3"
   },
   "config": {},
   "scripts": {
     "bower": "bower",
     "test": "wct test/*",
-    "test-sauce": "wct --skip-plugin local --plugin sauce test/*"
+    "test-sauce": "wct --skip-plugin local --plugin sauce test/*",
+    "system-test": "node_modules/mocha/bin/mocha --timeout 500000 --reporter spec"
   },
   "repository": {
     "type": "git",

system-test/urth-viz-explorer-specs.js

@@ -0,0 +1,49 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+var wd = require('wd');
+var chai = require('chai');
+var Boilerplate = require('./utils/boilerplate');
+var boilerplate = new Boilerplate();
+
+describe('Urth Viz Explorer Test', function() {
+
+    var tagChaiAssertionError = function(err) {
+        // throw error and tag as retriable to poll again
+        err.retriable = err instanceof chai.AssertionError;
+        throw err;
+    };
+
+    wd.PromiseChainWebdriver.prototype.waitForWidgetElement = function(selector, browserSupportsShadowDOM, timeout, pollFreq) {
+        return this.waitForElementByCssSelector(
+            browserSupportsShadowDOM ? 'urth-viz-explorer::shadow urth-viz-vega::shadow svg' : 'urth-viz-explorer urth-viz-vega svg',
+            wd.asserters.isDisplayed,
+            timeout)
+        .catch(tagChaiAssertionError);
+    };
+
+    boilerplate.setup(this.title, '/notebooks/examples/urth-viz-explorer.ipynb');
+
+    it('should run all cells and find an explorer in the 5th output area', function(done) {
+        boilerplate.browser
+            .waitForElementsByCssSelector('div.output_area').nth(5)
+            .waitForWidgetElement("urth-viz-explorer", boilerplate.browserSupportsShadowDOM, 10000)
+            .nodeify(done);
+    });
+
+    it('should have 20 items plotted, by default, then 10 after setting the limit accordingly', function(done) {
+        boilerplate.browser
+            .waitForElementsByCssSelector('div.output_area').nth(7)
+            .moveTo()
+            .waitForElementsByCssSelector('urth-viz-explorer#v1::shadow urth-viz-vega::shadow svg .marks *', wd.asserters.isDisplayed, 10000)
+            .should.be.eventually.length(20)
+            .waitForElementByCssSelector('urth-viz-explorer#v1::shadow #viz-explorer-controls .viz-explorer-controls-section paper-input[label=Limit]::shadow paper-input-container #input', wd.asserters.isDisplayed, 10000)
+            .click()
+            .doubleclick()
+            .type('10')
+            .sleep(1000)
+            .elementsByCssSelector('urth-viz-explorer#v1::shadow urth-viz-vega::shadow svg .marks *')
+            .should.be.eventually.length(10)
+            .nodeify(done);
+    });
+});

system-test/utils/boilerplate.js

@@ -0,0 +1,122 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+var wd = require('wd');
+require('colors');
+var chai = require('chai');
+var chaiAsPromised = require('chai-as-promised');
+var parseArgs = require('minimist');
+
+//  Parse the args
+var args = parseArgs(process.argv);
+args.local = args['test-type'] === 'local';
+var remote = !args.local;
+args.server = args.server || (remote ? 'ondemand.saucelabs.com' : 'localhost:4444');
+
+//  Setup chai
+chai.use(chaiAsPromised);
+chai.should();
+chaiAsPromised.transferPromiseness = wd.transferPromiseness;
+
+// Configure webdriver
+wd.configureHttp({
+    timeout: 60000,
+    retryDelay: 15000,
+    retries: 5,
+    baseUrl: args.baseurl
+});
+
+//  The browser capabilities we would like setup by selenium
+var desired = {browserName: 'chrome', platform: 'OS X 10.10'};
+if (args.browser && args.browser === 'InternetExplorer') {
+    args.browser = 'internet explorer';
+}
+desired.browserName = args.browser || desired.browserName;
+desired.platform = args.platform ||
+    (desired.browserName === 'internet explorer' || desired.browserName === 'MicrosoftEdge') ?
+        'Windows 10': desired.platform;
+desired.tags = ['widgets', 'system-test', desired.browserName];
+// If there is a build number, include it in the desired attributes for sauce labs
+if (process.env.TRAVIS_JOB_NUMBER) {
+  desired.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER;
+  desired.build = 'widgets-build-' + process.env.TRAVIS_JOB_NUMBER;
+}
+
+//  The selenium test server, local or remote, we will be using to test against
+var testServer = 'http://' + args.server + '/wd/hub';
+if (remote) {
+  testServer = 'http://' + process.env.SAUCE_USERNAME + ':' + process.env.SAUCE_ACCESS_KEY + '@' + args.server + '/wd/hub';
+}
+
+console.log('Test server is ', args.server);
+console.log('Travis job number is: ', process.env.TRAVIS_JOB_NUMBER);
+console.log('Sauce user name is defined? ', !!process.env.SAUCE_USERNAME);
+console.log('Sauce access key is defined? ', !!process.env.SAUCE_ACCESS_KEY);
+
+/**
+  * A helper class to setup webdriver, create a browser, and widget objects for
+  * use within the system tests.
+  */
+var Boilerplate = function(){
+  this.browser = wd.promiseChainRemote(testServer);
+  this.allPassed = true;
+  this.SPECIAL_KEYS = wd.SPECIAL_KEYS;
+};
+
+/**
+  * Setups the before and after calls for each of your tests. The boilerplate
+  * will start each test on startingURL, which is a relative path to the resource to load.
+  */
+Boilerplate.prototype.setup = function(testName, startingURL){
+  var that = this;
+
+  before(function(done){
+    if (args.verbose) {
+        // optional logging
+        this.browser.on('status', function(info) {
+            console.log(info.cyan);
+        });
+        this.browser.on('command', function(meth, path, data) {
+            console.log(' > ' + meth.yellow, path.grey, data || '');
+        });
+    }
+
+    desired.name = testName ? 'Urth Widgets System Test - ' + testName
+        : 'Urth Widgets System Test';
+
+    this.browser.init(desired)
+        .get(startingURL || '/')
+        .waitForElementByCssSelector('#kernel_indicator_icon.kernel_idle_icon', wd.asserters.isDisplayed, 10000)
+        .waitForElementByLinkText('Cell', wd.asserters.isDisplayed, 10000)
+        .safeExecute('localStorage.clear()')
+        .elementByLinkText('Cell')
+        .click()
+        .waitForElementByLinkText('Run All', wd.asserters.isDisplayed, 10000)
+        .elementByLinkText('Run All')
+        .click()
+        .eval('!!document.body.createShadowRoot', function(err, value) {
+            this.browserSupportsShadowDOM = value;
+        }.bind(this))
+        .waitForElementByCssSelector('div.output_area', wd.asserters.isDisplayed, 10000)
+        .setAsyncScriptTimeout(15000)
+        .waitForConditionInBrowser('window.Urth && Urth.kernel && Urth.kernel.is_connected()', 10000)
+        .waitForElementByCssSelector('#kernel_indicator_icon.kernel_idle_icon', wd.asserters.isDisplayed, 20000)
+        .waitForConditionInBrowser('typeof Urth.whenReady === "function"', 10000)
+        .nodeify(done);
+  }.bind(this));
+
+  after(function(done){
+    var result = this.browser.quit();
+    if (remote) {
+      result = result.sauceJobStatus(this.allPassed);
+    }
+    result.nodeify(done);
+  }.bind(this));
+
+  afterEach(function(done) {
+    that.allPassed = that.allPassed && (this.currentTest.state === 'passed');
+    done();
+  });
+};
+
+module.exports = Boilerplate;