Testing with Selenium & Sauce labs (#3321)

* Initial selenium test

* Try configuring Travis to run selenium tests on Sauce

* Encryption key needs to be for my account, not jupyter

* Install selenium on Travis

* Get more data from server info file

* Set cwd when launching notebook server

Will this help on Travis?

* Use JUPYTER_TEST_BROWSER=chrome to test with Chrome

* Debugging test

* Separate fixtures into conftest.py

* Try with --Cls.a=b option syntax

* Try using sauce labs directly, not through Travis proxy

* Back to using proxy, with http instead of https

Idea from https://stackoverflow.com/questions/48236104/ssl-errors-using-
sauce-labs-in-travis-ci-with-selenium-webriver-tests-django-pr

* Specify browserName in desired_capabilities for Sauce

* Try connecting to Sauce for only some jobs in matrix

* Exclude selenium tests from regular test run

* Remove redundant JS test for dashboard navigation (converted to Selenium)

* Re-enable other tests

* Exclude selenium tests on Appveyor

* Later browser versions are available on Windows

* Try running tests with Firefox 57 instead of 58

* Try running with local Firefox on Travis

* Install geckodriver for Selenium tests

* Untar the right version of geckodriver

* Try stepping back one version of Firefox again

.travis.yml

@@ -11,6 +11,7 @@ python:
 
 sudo: required
 
+
 env:
   global:
     - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH
@@ -19,8 +20,6 @@ env:
     - GROUP=python
     - GROUP=js/base
     - GROUP=js/services
-    - GROUP=js/tree
-    - GROUP=docs
 
 before_install:
     - pip install --upgrade pip
@@ -40,6 +39,15 @@ before_install:
       if [[ $GROUP == docs ]]; then
         pip install -r docs/doc-requirements.txt
       fi
+    - |
+      if [[ $GROUP == selenium ]]; then
+        pip install selenium
+        # Install Webdriver backend for Firefox:
+        wget https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz
+        mkdir geckodriver
+        tar -xzf geckodriver-v0.19.1-linux64.tar.gz -C geckodriver
+        export PATH=$PATH:$PWD/geckodriver
+      fi
 
 install:
     - pip install -f travis-wheels/wheelhouse file://$PWD#egg=notebook[test]
@@ -59,7 +67,8 @@ script:
           true
       fi
     - 'if [[ $GROUP == js* ]]; then travis_retry python -m notebook.jstest ${GROUP:3}; fi'
-    - 'if [[ $GROUP == python ]]; then nosetests -v --with-coverage --cover-package=notebook notebook; fi'
+    - 'if [[ $GROUP == python ]]; then nosetests -v --exclude-dir notebook/tests/selenium --with-coverage --cover-package=notebook notebook; fi'
+    - 'if [[ $GROUP == selenium ]]; then py.test -sv notebook/tests/selenium; fi'
     - |
       if [[ $GROUP == docs ]]; then
         EXIT_STATUS=0
@@ -72,12 +81,19 @@ script:
 
 matrix:
     include:
+        - python: 3.6
+          env:
+          - GROUP=selenium
+          - JUPYTER_TEST_BROWSER=firefox
+          - MOZ_HEADLESS=1
+          addons:
+            firefox: 57.0
+
         - python: 3.4
           env: GROUP=python
         - python: 3.5
           env: GROUP=python
-    exclude:
-        - python: 2.7
+        - python: 3.6
           env: GROUP=docs
 
 after_success:

appveyor.yml

@@ -23,4 +23,4 @@ install:
   - cmd: pip install .[test]
 
 test_script:
-  - nosetests  -v notebook
+  - nosetests  -v notebook --exclude-dir notebook\tests\selenium

notebook/tests/selenium/conftest.py

@@ -0,0 +1,96 @@
+import json
+import os
+import pytest
+import requests
+from subprocess import Popen
+import sys
+from testpath.tempdir import TemporaryDirectory
+import time
+from urllib.parse import urljoin
+
+from selenium.webdriver import Firefox, Remote, Chrome
+
+pjoin = os.path.join
+
+def _wait_for_server(proc, info_file_path):
+    """Wait 30 seconds for the notebook server to start"""
+    for i in range(300):
+        if proc.poll() is not None:
+            raise RuntimeError("Notebook server failed to start")
+        if os.path.exists(info_file_path):
+            try:
+                with open(info_file_path) as f:
+                    return json.load(f)
+            except ValueError:
+                # If the server is halfway through writing the file, we may
+                # get invalid JSON; it should be ready next iteration.
+                pass
+        time.sleep(0.1)
+    raise RuntimeError("Didn't find %s in 30 seconds", info_file_path)
+
+@pytest.fixture(scope='session')
+def notebook_server():
+    info = {}
+    with TemporaryDirectory() as td:
+        nbdir = info['nbdir'] = pjoin(td, 'notebooks')
+        os.makedirs(pjoin(nbdir, u'sub ∂ir1', u'sub ∂ir 1a'))
+        os.makedirs(pjoin(nbdir, u'sub ∂ir2', u'sub ∂ir 1b'))
+
+        info['extra_env'] = {
+            'JUPYTER_CONFIG_DIR': pjoin(td, 'jupyter_config'),
+            'JUPYTER_RUNTIME_DIR': pjoin(td, 'jupyter_runtime'),
+            'IPYTHONDIR': pjoin(td, 'ipython'),
+        }
+        env = os.environ.copy()
+        env.update(info['extra_env'])
+
+        command = [sys.executable, '-m', 'notebook',
+                   '--no-browser',
+                   '--notebook-dir', nbdir,
+                   # run with a base URL that would be escaped,
+                   # to test that we don't double-escape URLs
+                   '--NotebookApp.base_url=/a@b/',
+                  ]
+        print("command=", command)
+        proc = info['popen'] = Popen(command, cwd=nbdir, env=env)
+        info_file_path = pjoin(td, 'jupyter_runtime', 'nbserver-%i.json' % proc.pid)
+        info.update(_wait_for_server(proc, info_file_path))
+
+        print("Notebook server info:", info)
+        yield info
+
+    # Shut the server down
+    requests.post(urljoin(info['url'], 'api/shutdown'),
+                  headers={'Authorization': 'token '+info['token']})
+
+
+def _get_selenium_driver():
+    if os.environ.get('SAUCE_USERNAME'):
+        username = os.environ["SAUCE_USERNAME"]
+        access_key = os.environ["SAUCE_ACCESS_KEY"]
+        capabilities = {
+            "tunnel-identifier": os.environ["TRAVIS_JOB_NUMBER"],
+            "build": os.environ["TRAVIS_BUILD_NUMBER"],
+            "tags": [os.environ['TRAVIS_PYTHON_VERSION'], 'CI'],
+            "platform": "Windows 10",
+            "browserName": os.environ['JUPYTER_TEST_BROWSER'],
+            "version": "latest",
+        }
+        if capabilities['browserName'] == 'firefox':
+            # Attempt to work around issue where browser loses authentication
+            capabilities['version'] = '57.0'
+        hub_url = "%s:%s@localhost:4445" % (username, access_key)
+        print("Connecting remote driver on Sauce Labs")
+        return Remote(desired_capabilities=capabilities,
+                      command_executor="http://%s/wd/hub" % hub_url)
+    elif os.environ.get('JUPYTER_TEST_BROWSER') == 'chrome':
+        return Chrome()
+    else:
+        return Firefox()
+
+@pytest.fixture
+def browser(notebook_server):
+    b = _get_selenium_driver()
+    b.get("{url}?token={token}".format(**notebook_server))
+    yield b
+    b.quit()

notebook/tests/selenium/test_dashboard_nav.py

@@ -0,0 +1,45 @@
+import os
+
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+pjoin = os.path.join
+
+def get_list_items(browser):
+    return [{
+        'link': a.get_attribute('href'),
+        'label': a.find_element_by_class_name('item_name').text,
+    } for a in browser.find_elements_by_class_name('item_link')]
+
+
+def wait_for_selector(browser, selector, timeout=10):
+    wait = WebDriverWait(browser, timeout)
+    return wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, selector)))
+
+
+
+def test_items(browser, visited=None):
+    tree_root_url = browser.current_url
+    if visited is None:
+        visited = set()
+
+    wait_for_selector(browser, '.item_link')
+    items = get_list_items(browser)
+    print(browser.current_url, len(items))
+    for item in items:
+        print(item)
+        url = item['link']
+        if url.startswith(tree_root_url):
+            print("Going to", url)
+            if url in visited:
+                continue
+            visited.add(url)
+            browser.get(url)
+            wait_for_selector(browser, '.item_link')
+            assert browser.current_url == url
+
+            test_items(browser, visited)
+            #browser.back()
+
+    print()

notebook/tests/tree/dashboard_nav.js

@@ -1,47 +0,0 @@
-
-
-casper.get_list_items = function () {
-    return this.evaluate(function () {
-        return $.makeArray($('.item_link').map(function () {
-            return {
-                link: $(this).attr('href'),
-                label: $(this).find('.item_name').text()
-            };
-        }));
-    });
-};
-
-casper.test_items = function (origin, prefix, visited) {
-    visited = visited || {};
-    casper.then(function () {
-        var items = casper.get_list_items();
-        var tree_link = RegExp('^' + (prefix + 'tree/').replace(/\//g, '\\/'));
-        casper.each(items, function (self, item) {
-            if (item.link.match(tree_link)) {
-                var followed_url = item.link;
-                if (!visited[followed_url]) {
-                    visited[followed_url] = true;
-                    casper.thenOpen(origin + followed_url, function () {
-                        this.waitFor(this.page_loaded);
-                        casper.wait_for_dashboard();
-                        // getCurrentUrl is with host, and url-decoded,
-                        // but item.link is without host, and url-encoded
-                        var expected = origin + decodeURIComponent(item.link);
-                        this.test.assertEquals(this.getCurrentUrl(), expected, 'Testing dashboard link: ' + expected);
-                        casper.test_items(origin, prefix, visited);
-                        this.back();
-                    });
-                }
-            }
-        });
-    });
-};
-
-casper.dashboard_test(function () {
-    var baseUrl = this.get_notebook_server();
-    m = /(https?:\/\/[^\/]+)(.*)/.exec(baseUrl);
-    origin = m[1];
-    prefix = m[2];
-    casper.test_items(origin, prefix);
-});
-

setup.py

@@ -91,7 +91,8 @@ for more information.
     ],
     extras_require = {
         'test:python_version == "2.7"': ['mock'],
-        'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', 'nbval'],
+        'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters',
+                 'nbval', 'nose-exclude'],
         'test:sys_platform == "win32"': ['nose-exclude'],
     },
     entry_points = {