saliency heatmap

gradio/static/css/interfaces/input/image_upload.css

@@ -4,13 +4,34 @@
 .image_display {
   height: 100%;
 }
-.image_preview_holder {
+.view_holders {
   flex-grow: 1;
   height: calc(100% - 36px);
+  background-color: #CCCCCC;
+  position: relative;
+}
+.image_preview_holder, .saliency_holder {
+  width: 100%;
+  height: 100%;
   display: flex;
   justify-content: center;
   align-items: center;
-  background-color: #CCCCCC;
+}
+.saliency_holder {
+  position: absolute;
+  top: 0;
+}
+.saliency {
+  display: flex;
+  flex-direction: column;
+}
+.saliency > div {
+  display: flex;
+  flex-grow: 1;
+}
+.saliency > div > div {
+  flex-grow: 1;
+  background-color: #EEA45D;
 }
 .image_preview {
   max-width: 100%;

gradio/static/js/all_io.js

@@ -10,7 +10,7 @@ var io_master = {
         data: JSON.stringify(post_data),
         success: function(output){
             if (output['action'] == 'output') {
-              io_master.output(output['data']);
+              io_master.output(output);
             }
         },
         error: function(XMLHttpRequest, textStatus, errorThrown) {
@@ -21,8 +21,11 @@ var io_master = {
     });
   },
   output: function(data) {
-    this.last_output = data;
+    this.last_output = data["data"];
-    this.output_interface.output(data);
+    this.output_interface.output(data["data"]);
+    if (this.input_interface.output && data["saliency"]) {
+      this.input_interface.output(data["saliency"]);
+    }
   },
   flag: function(message) {
     var post_data = {

gradio/static/js/interfaces/input/image_upload.js

@@ -7,8 +7,13 @@ const image_input = {
       <div class="edit_holder">
         <button class="edit_image interface_button primary">Edit</button>
       </div>
-      <div class="image_preview_holder">
+      <div class="view_holders">
-        <img class="image_preview" />
+        <div class="image_preview_holder">
+          <img class="image_preview" />
+        </div>
+        <div class="saliency_holder hide">
+          <div class="saliency"></div>
+        </div>
       </div>
     </div>
     <input class="hidden_upload" type="file" accept="image/x-png,image/gif,image/jpeg" />`
@@ -46,6 +51,7 @@ const image_input = {
     this.target.find('.edit_image').click(function (e) {
       let io = get_interface(e.target);
       io.overlay_target.removeClass("hide");
+      io.target.find(".saliency_holder").addClass("hide");
     })
     this.tui_editor = new tui.ImageEditor(this.overlay_target.
         find(".image_editor")[0], {
@@ -89,6 +95,24 @@ const image_input = {
     this.target.find(".hidden_upload").prop("value", "")
     this.state = "NO_IMAGE";
     this.image_data = null;
+    this.target.find(".saliency_holder").addClass("hide");
+  },
+  output: function(data) {
+    if (this.target.find(".image_preview").attr("src")) {
+      var image = this.target.find(".image_preview");
+      this.target.find(".saliency_holder").removeClass("hide");
+      html = '';
+      data.forEach(function(row) {
+        html += "<div>"
+        row.forEach(function(cell) {
+          opacity = cell > 0.3 ? cell : 0;
+          html += `<div style="opacity: ${opacity}"> </div>`
+        })
+        html += "</div>"
+      })
+      this.target.find(".saliency").width(
+          image.width()).height(image.height()).html(html)
+    }
   },
   state: "NO_IMAGE",
   image_data: null,

gradio/templates/base_template.html

@@ -73,6 +73,7 @@
     <script src="../static/js/utils.js"></script>
     <script src="../static/js/all_io.js"></script>
     <script src="../static/js/interfaces/input/csv.js"></script>
+    <script src="../static/js/vendor/heatmap.min.js"></script>
     <script src="../static/js/interfaces/input/image_upload.js"></script>
     <script src="../static/js/vendor/sketchpad.js"></script>
     <script src="../static/js/interfaces/input/sketchpad.js"></script>