End-to-end UI tests with Selenium (#72)

* selenium test file

* selenium test

* added save_to as interface attribute, 4 demo tests with selenium

* change name to test_demos

* fixed wait until, removed path insertion

* added selenium to circleci config

* trying chrome driver image

* removing cache from circlci

* using chromedriver_installer

* adding deps for chromedriver

* sudo

* removed chromedriver

* added chromium-chromedriver

* using chromedriver-py

* using service instead of exec path

* using latest selenium

* using browsers image

* added cwd to tmp and test paths

* saving artifacts

* added tmp.txt

* changed driver size

* elem.text sleep

* driver size fix

* saving artifacts

* saving artifacts correctly

* saving artifacts correctly

* saving artifactS

* print statement

* print statement

* print statement

* correct dir

* debugging

* debugging

* debugging

* fixing wonderful

* running comparison

* current dir fix

* changing longest_word

* fixed longest_word

* clean up

* fixing new output label name

* time limit on while loops

* refactoring common code

* removed setUp

* removed server_port declaration from demos

Co-authored-by: Ali Abid <aliabid94@gmail.com>

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7.2
+      - image: circleci/python:3.7.2-browsers
     steps:
       - checkout
       - run: mkdir test-reports
@@ -14,16 +14,20 @@ jobs:
             python3 -m venv venv
             . venv/bin/activate
             pip install -r gradio.egg-info/requires.txt
+            pip install selenium==4.0.0a6.post2
       - save_cache:
           key: deps1-{{ .Branch }}-{{ checksum "gradio.egg-info/requires.txt" }}
           paths:
             - "venv"
+      - run:
+          command: |
+            mkdir screenshots
       - run:
           command: |
             . venv/bin/activate
             python3 -m unittest
       - store_artifacts:
-          path: test-reports/
-          destination: tr1
+          path: /home/circleci/project/test/tmp
+          destination: screenshots
       - store_test_results:
           path: test-reports/

build/lib/gradio/interface.py

@@ -122,6 +122,8 @@ class Interface:
         self.flagging_dir = flagging_dir
         Interface.instances.add(self)
         self.analytics_enabled=analytics_enabled
+        self.launch_port = None
+        self.save_to = None
 
         data = {'fn': fn,
                 'inputs': inputs,
@@ -341,7 +343,6 @@ class Interface:
         server_port, app, thread = networking.start_server(
             self, self.server_port)
         path_to_local_server = "http://{}:{}/".format(self.server_name, server_port)
-        self.server_port = server_port
         self.status = "RUNNING"
         self.server = app
 
@@ -365,7 +366,7 @@ class Interface:
             print("This share link will expire in 6 hours. If you need a "
                   "permanent link, email support@gradio.app")
             try:
-                share_url = networking.setup_tunnel(server_port)
+                share_url = networking.setup_tunnel(self.launch_port)
                 print("Running on External URL:", share_url)
             except RuntimeError:
                 data = {'error': 'RuntimeError in launch method'}

build/lib/gradio/networking.py

@@ -173,6 +173,8 @@ def start_server(interface, server_port=None):
     app.cwd = os.getcwd()
     log = logging.getLogger('werkzeug')
     log.setLevel(logging.ERROR)
+    if interface.save_to is not None:
+        interface.save_to["port"] = port
     process = threading.Thread(target=app.run, kwargs={"port": port})
     process.start()
     return port, app, process

build/lib/gradio/static/js/vendor/fabric.js

@@ -930,7 +930,7 @@ fabric.Collection = {
 
     /**
      * Returns true if context has transparent pixel
-     * at specified location (taking tolerance into account)
+     * at specified location (taking TOLERANCE into account)
      * @param {CanvasRenderingContext2D} ctx context
      * @param {Number} x x coordinate
      * @param {Number} y y coordinate
@@ -938,7 +938,7 @@ fabric.Collection = {
      */
     isTransparent: function(ctx, x, y, tolerance) {
 
-      // If tolerance is > 0 adjust start coords to take into account.
+      // If TOLERANCE is > 0 adjust start coords to take into account.
       // If moves off Canvas fix to 0
       if (tolerance > 0) {
         if (x > tolerance) {
@@ -959,7 +959,7 @@ fabric.Collection = {
           imageData = ctx.getImageData(x, y, (tolerance * 2) || 1, (tolerance * 2) || 1),
           l = imageData.data.length;
 
-      // Split image data - for tolerance > 1, pixelDataSize = 4;
+      // Split image data - for TOLERANCE > 1, pixelDataSize = 4;
       for (i = 3; i < l; i += 4) {
         temp = imageData.data[i];
         _isTransparent = temp <= 0;

demo/diff_texts.py

@@ -21,8 +21,9 @@ io = gr.Interface(
         "+": "lightgreen",
         "-": "pink",
         " ": "none",
-    })
-)
+    }))
 
 io.test_launch()
-io.launch()
+
+if __name__ == "__main__":
+    io.launch()

demo/image_mod.py

@@ -6,7 +6,7 @@ import gradio as gr
 def image_mod(image):
     return image.rotate(45)
 
-  
+
 io = gr.Interface(image_mod, 
              gr.inputs.Image(type="pil"), 
              "image", 
@@ -15,8 +15,9 @@ io = gr.Interface(image_mod,
                  ["images/cheetah2.jpg"],
                  ["images/lion.jpg"],
              ],
-             live=True,
-             )
+             live=True)
 
 io.test_launch()
-io.launch()
+
+if __name__ == "__main__":
+    io.launch()

demo/longest_word.py

@@ -9,7 +9,10 @@ def longest_word(text):
 
 ex = "The quick brown fox jumped over the lazy dog."
 
-io = gr.Interface(longest_word, "textbox", "label", interpretation="default", examples=[[ex]])
+io = gr.Interface(longest_word, "textbox", "label",
+                  interpretation="default", examples=[[ex]])
 
 io.test_launch()
-io.launch()
+
+if __name__ == "__main__":
+    io.launch()

demo/matrix_transpose.py

@@ -14,4 +14,6 @@ io = gr.Interface(
 )
 
 io.test_launch()
-io.launch()
+
+if __name__ == "__main__":
+    io.launch()

demo/sentence_builder.py

@@ -24,4 +24,6 @@ io = gr.Interface(
     ])
 
 io.test_launch()
-io.launch()
+
+if __name__ == "__main__":
+    io.launch()

gradio.egg-info/SOURCES.txt

@@ -120,6 +120,7 @@ gradio/static/js/vendor/wavesurfer.min.js
 gradio/static/js/vendor/webcam.min.js
 gradio/static/js/vendor/white-theme.js
 gradio/templates/index.html
+test/test_diff_texts.py
 test/test_inputs.py
 test/test_interfaces.py
 test/test_interpretation.py

gradio/interface.py

@@ -119,6 +119,8 @@ class Interface:
         self.flagging_dir = flagging_dir
         Interface.instances.add(self)
         self.analytics_enabled=analytics_enabled
+        self.launch_port = None
+        self.save_to = None
 
         data = {'fn': fn,
                 'inputs': inputs,
@@ -338,7 +340,6 @@ class Interface:
         server_port, app, thread = networking.start_server(
             self, self.server_name, self.server_port)
         path_to_local_server = "http://{}:{}/".format(self.server_name, server_port)
-        self.server_port = server_port
         self.status = "RUNNING"
         self.server = app
 
@@ -362,7 +363,7 @@ class Interface:
             print("This share link will expire in 6 hours. If you need a "
                   "permanent link, email support@gradio.app")
             try:
-                share_url = networking.setup_tunnel(server_port)
+                share_url = networking.setup_tunnel(self.launch_port)
                 print("Running on External URL:", share_url)
             except RuntimeError:
                 data = {'error': 'RuntimeError in launch method'}

gradio/networking.py

@@ -181,11 +181,14 @@ def start_server(interface, server_name, server_port=None):
     app.cwd = os.getcwd()
     log = logging.getLogger('werkzeug')
     log.setLevel(logging.ERROR)
-    thread = threading.Thread(target=app.run, kwargs={"port": port, "host": server_name}, daemon=True)
+    if interface.save_to is not None:
+        interface.save_to["port"] = port
+    thread = threading.Thread(target=app.run,
+                              kwargs={"port": port, "host": server_name},
+                              daemon=True)
     thread.start()
     return port, app, thread
 
-
 def close_server(process):
     process.terminate()
     process.join()

gradio/static/js/vendor/fabric.js

@@ -930,7 +930,7 @@ fabric.Collection = {
 
     /**
      * Returns true if context has transparent pixel
-     * at specified location (taking tolerance into account)
+     * at specified location (taking TOLERANCE into account)
      * @param {CanvasRenderingContext2D} ctx context
      * @param {Number} x x coordinate
      * @param {Number} y y coordinate
@@ -938,7 +938,7 @@ fabric.Collection = {
      */
     isTransparent: function(ctx, x, y, tolerance) {
 
-      // If tolerance is > 0 adjust start coords to take into account.
+      // If TOLERANCE is > 0 adjust start coords to take into account.
       // If moves off Canvas fix to 0
       if (tolerance > 0) {
         if (x > tolerance) {
@@ -959,7 +959,7 @@ fabric.Collection = {
           imageData = ctx.getImageData(x, y, (tolerance * 2) || 1, (tolerance * 2) || 1),
           l = imageData.data.length;
 
-      // Split image data - for tolerance > 1, pixelDataSize = 4;
+      // Split image data - for TOLERANCE > 1, pixelDataSize = 4;
       for (i = 3; i < l; i += 4) {
         temp = imageData.data[i];
         _isTransparent = temp <= 0;

test/test_demos.py

@@ -0,0 +1,223 @@
+import unittest
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import multiprocessing
+import time
+import requests
+from matplotlib.testing.compare import compare_images
+import random
+import os
+
+current_dir = os.getcwd()
+
+LOCAL_HOST = "http://localhost:{}"
+GOLDEN_PATH = "test/golden/{}/{}.png"
+TOLERANCE = 0.1
+TIMEOUT = 10
+
+
+def wait_for_url(url):
+    for i in range(TIMEOUT):
+        try:
+            requests.get(url)
+            print("Interface connected.")
+            break
+        except:
+            time.sleep(0.2)
+    else:
+        raise ConnectionError("Could not connect to interface.")
+
+
+def hide_latency(driver):
+    js = "document.getElementsByClassName('loading_time')[" \
+         "0].style.visibility = " \
+         "'hidden';"
+    driver.execute_script(js)
+
+
+def diff_texts_thread(return_dict):
+    from demo.diff_texts import io
+    io.save_to = return_dict
+    io.launch()
+
+
+def image_mod_thread(return_dict):
+    from demo.image_mod import io
+    io.examples = None
+    io.save_to = return_dict
+    io.launch()
+
+
+def longest_word_thread(return_dict):
+    from demo.longest_word import io
+    io.save_to = return_dict
+    io.launch()
+
+
+def sentence_builder_thread(return_dict):
+    from demo.sentence_builder import io
+    io.save_to = return_dict
+    io.launch()
+
+
+class TestDemo(unittest.TestCase):
+    def start_test(self, target):
+        manager = multiprocessing.Manager()
+        return_dict = manager.dict()
+        self.i_thread = multiprocessing.Process(target=target,
+                                                args=(return_dict,))
+        self.i_thread.start()
+        total_sleep = 0
+        while not return_dict and total_sleep < TIMEOUT:
+            time.sleep(0.2)
+            total_sleep += 0.2
+        URL = LOCAL_HOST.format(return_dict["port"])
+        wait_for_url(URL)
+
+        driver = webdriver.Chrome()
+        driver.set_window_size(1200, 800)
+        driver.get(URL)
+        
+        return driver
+
+    def test_diff_texts(self):
+        driver = self.start_test(target=diff_texts_thread)
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".input_interface[interface_id='0'] .input_text"))
+        )
+        elem.clear()
+        elem.send_keys("Want to see a magic trick?")
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".input_interface[interface_id='1'] .input_text"))
+        )
+        elem.clear()
+        elem.send_keys("Let's go see a magic trick!")
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".submit"))
+        )
+        elem.click()
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".output_interface[interface_id='2'] .output_text"))
+        )
+
+        total_sleep = 0
+        while elem.text == "" and total_sleep < TIMEOUT:
+            time.sleep(0.2)
+            total_sleep += 0.2
+
+        self.assertEqual(elem.text, "LeWant's tgo see a magic trick?!")
+        golden_img = os.path.join(current_dir, GOLDEN_PATH.format(
+            "diff_texts", "magic_trick"))
+        tmp = os.path.join(current_dir, "test/tmp/{}.png".format(
+            random.getrandbits(32)))
+        hide_latency(driver)
+        driver.save_screenshot(tmp)
+        driver.close()
+        self.assertIsNone(compare_images(tmp, golden_img, TOLERANCE))
+        os.remove(tmp)
+
+    def test_image_mod(self):
+        driver = self.start_test(target=image_mod_thread)
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".input_interface["
+                                            "interface_id='0'] "
+                                            ".hidden_upload"))
+        )
+        hide_latency(driver)
+        cwd = os.getcwd()
+        rel = "demo/images/cheetah1.jpg"
+        elem.send_keys(os.path.join(cwd, rel))
+        golden_img = os.path.join(current_dir, GOLDEN_PATH.format(
+            "image_mod", "cheetah1"))
+        tmp = os.path.join(current_dir, "test/tmp/{}.png".format(
+            random.getrandbits(32)))
+        WebDriverWait(driver, TIMEOUT).until(
+            EC.visibility_of_element_located((By.CSS_SELECTOR,
+                                            ".output_interface["
+                                            "interface_id='1'] "
+                                            ".output_image"))
+        )
+
+        hide_latency(driver)
+        driver.save_screenshot(tmp)
+        self.assertIsNone(compare_images(tmp, golden_img, TOLERANCE))
+        os.remove(tmp)
+        driver.close()
+
+    def test_longest_word(self):
+        driver = self.start_test(target=longest_word_thread)
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".input_interface[interface_id='0'] .input_text"))
+        )
+        elem.send_keys("This is the most wonderful machine learning "
+                       "library.")
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".submit"))
+        )
+        elem.click()
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".output_interface["
+                                            "interface_id='1'] .output_class"))
+        )
+
+        total_sleep = 0
+        while elem.text == "" and total_sleep < TIMEOUT:
+            time.sleep(0.2)
+            total_sleep += 0.2
+
+        golden_img = os.path.join(current_dir, GOLDEN_PATH.format(
+            "longest_word", "wonderful"))
+        tmp = os.path.join(current_dir, "test/tmp/{}.png".format(
+            random.getrandbits(32)))
+        hide_latency(driver)
+        driver.save_screenshot(tmp)
+        driver.close()
+        self.assertIsNone(compare_images(tmp, golden_img, TOLERANCE))
+        os.remove(tmp)
+
+    def test_sentence_builder(self):
+        driver = self.start_test(target=sentence_builder_thread)
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".submit"))
+        )
+        elem.click()
+        elem = WebDriverWait(driver, TIMEOUT).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR,
+                                            ".output_interface["
+                                            "interface_id='5'] .output_text"))
+        )
+
+        total_sleep = 0
+        while elem.text == "" and total_sleep < TIMEOUT:
+            time.sleep(0.2)
+            total_sleep += 0.2
+
+        self.assertEqual(elem.text, "The 2 cats went to the park where they  until the night")
+        golden_img = os.path.join(current_dir, GOLDEN_PATH.format(
+            "sentence_builder", "two_cats"))
+        tmp = os.path.join(current_dir, "test/tmp/{}.png".format(
+            random.getrandbits(32)))
+        hide_latency(driver)
+        driver.save_screenshot(tmp)
+        self.assertIsNone(compare_images(tmp, golden_img, TOLERANCE))
+        os.remove(tmp)
+        driver.close()
+
+    def tearDown(self):
+        self.i_thread.terminate()
+        self.i_thread.join()
+
+
+if __name__ == '__main__':
+    unittest.main()

test/tmp/tmp.txt

@@ -0,0 +1 @@
+# tmp files saved here