Display images (#7569)

* Add methods to convert between images and bytes

* Use new Images utility class where applicable

* Make Image(...) MIME type support RenderedImage

* Add an example of displaying a BufferedImage

* Add a unit test for Images utility class

* Display image outputs as images by default

doc/groovy/Mime.ipynb

@@ -190,6 +190,25 @@
     "Image(bytes)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import java.awt.image.BufferedImage\n",
+    "bi = new BufferedImage(256, 80, BufferedImage.TYPE_INT_ARGB)\n",
+    "g = bi.getGraphics()\n",
+    "g.setColor(java.awt.Color.BLUE.darker())\n",
+    "g.fillRoundRect(10, 10, 246, 70, 10, 10)\n",
+    "g.setColor(java.awt.Color.YELLOW)\n",
+    "g.setFont(new java.awt.Font(\"monospaced\", java.awt.Font.BOLD, 30))\n",
+    "g.drawString(\"Hello world!\", 30, 55)\n",
+    "g.drawLine(30, 60, 246, 60)\n",
+    "g.dispose()\n",
+    "Image(bi)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

kernel/base/src/main/java/com/twosigma/beakerx/BeakerxDefaultDisplayers.java

@@ -16,9 +16,15 @@
 package com.twosigma.beakerx;
 
 import com.twosigma.beakerx.mimetype.MIMEContainer;
+import com.twosigma.beakerx.util.Images;
 import jupyter.Displayer;
 import jupyter.Displayers;
 
+import java.awt.image.RenderedImage;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -27,6 +33,7 @@ public class BeakerxDefaultDisplayers {
 
   public static void registerDefaults() {
     registerCodeCellDisplayer();
+    registerImageDisplayer();
   }
 
   private static void registerCodeCellDisplayer() {
@@ -43,4 +50,24 @@ public Map<String, String> display(CodeCell value) {
       }
     });
   }
+
+  private static void registerImageDisplayer() {
+    Displayers.register(RenderedImage.class, new Displayer<RenderedImage>() {
+      @Override
+      public Map<String, String> display(RenderedImage value) {
+        return new HashMap<String, String>() {{
+          try {
+            byte[] data = Images.encode(value);
+            String base64 = Base64.getEncoder().encodeToString(data);
+            put(MIMEContainer.MIME.IMAGE_PNG, base64);
+          }
+          catch (IOException exc) {
+            StringWriter sw = new StringWriter();
+            exc.printStackTrace(new PrintWriter(sw));
+            put(MIMEContainer.MIME.TEXT_HTML, "<div><pre>" + sw.toString() + "</pre></div>");
+          }
+        }};
+      }
+    });
+  }
 }

kernel/base/src/main/java/com/twosigma/beakerx/jvm/serialization/BufferedImageDeserializer.java

@@ -15,13 +15,10 @@
  */
 package com.twosigma.beakerx.jvm.serialization;
 
-import java.io.ByteArrayInputStream;
-
-import javax.imageio.ImageIO;
-
 import com.fasterxml.jackson.databind.JsonNode;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.twosigma.beakerx.util.Images;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,8 +40,7 @@ public Object deserialize(JsonNode n, ObjectMapper mapper) {
     try {
       if (n.has("imageData")) {
         byte [] data = n.get("imageData").binaryValue();
-        ByteArrayInputStream bais = new ByteArrayInputStream(data);
-        o = ImageIO.read(bais);
+        o = Images.decode(data);
       }      
     } catch (Exception e) {
       logger.error("exception deserializing ImageIcon ", e);

kernel/base/src/main/java/com/twosigma/beakerx/jvm/serialization/BufferedImageSerializer.java

@@ -16,25 +16,21 @@
 package com.twosigma.beakerx.jvm.serialization;
 
 import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
-import javax.imageio.ImageIO;
-
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
+import com.twosigma.beakerx.util.Images;
 
 public class BufferedImageSerializer extends JsonSerializer<BufferedImage> {
 
   @Override
   public void serialize(BufferedImage v, JsonGenerator jgen, SerializerProvider provider)
       throws IOException, JsonProcessingException {
     synchronized(v) {
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      ImageIO.write(v, "png", baos);
-      byte [] data = baos.toByteArray();
+      byte [] data = Images.encode(v);
       jgen.writeStartObject();
       jgen.writeStringField("type",  "ImageIcon");
       jgen.writeObjectField("imageData", data);

kernel/base/src/main/java/com/twosigma/beakerx/jvm/serialization/ImageIconSerializer.java

@@ -17,16 +17,15 @@
 
 import java.awt.Graphics;
 import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
-import javax.imageio.ImageIO;
 import javax.swing.ImageIcon;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
+import com.twosigma.beakerx.util.Images;
 
 public class ImageIconSerializer extends JsonSerializer<ImageIcon> {
 
@@ -44,9 +43,7 @@ public void serialize(ImageIcon vi, JsonGenerator jgen, SerializerProvider provi
       vi.paintIcon(null, g, 0, 0);
       g.dispose();
 
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      ImageIO.write(v, "png", baos);
-      byte [] data = baos.toByteArray();
+      byte [] data = Images.encode(v);
       jgen.writeStartObject();
       jgen.writeStringField("type",  "ImageIcon");
       jgen.writeObjectField("imageData", data);

kernel/base/src/main/java/com/twosigma/beakerx/mimetype/ImageContainer.java

@@ -16,8 +16,11 @@
 package com.twosigma.beakerx.mimetype;
 
 
+import java.awt.image.RenderedImage;
 import java.util.Base64;
 
+import com.twosigma.beakerx.util.Images;
+
 public class ImageContainer extends MIMEContainer {
 
   public ImageContainer(String mime, String code) {
@@ -28,6 +31,8 @@ public static MIMEContainer Image(Object data) throws Exception {
     byte[] image;
     if (data instanceof String) {
       image = getBytes(data);
+    } else if (data instanceof RenderedImage) {
+      image = Images.encode((RenderedImage) data);
     } else {
       image = (byte[]) data;
     }

kernel/base/src/main/java/com/twosigma/beakerx/util/Images.java

@@ -0,0 +1,78 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.util;
+
+import java.awt.image.BufferedImage;
+import java.awt.image.RenderedImage;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import javax.imageio.ImageIO;
+
+/**
+ * Utility methods for working with Java image objects.
+ *
+ * @see ImageIO
+ * @see RenderedImage
+ */
+public final class Images {
+
+  private Images() {
+    // Prevent instantiation of utility class.
+  }
+
+  /**
+   * Converts the given {@link RenderedImage} into a stream of PNG bytes.
+   *
+   * @param image The image to convert to a byte stream.
+   * @return A stream of bytes in PNG format.
+   */
+  public static byte[] encode(RenderedImage image) throws IOException {
+    return encode(image, "png");
+  }
+
+  /**
+   * Converts the given {@link RenderedImage} into a stream of bytes.
+   *
+   * @param image The image to convert to a byte stream.
+   * @param format The informal name of the format for the returned bytes; e.g.
+   *        "png" or "jpg". See {@link ImageIO#getImageWritersByFormatName(String)}.
+   * @return A stream of bytes in the requested format, or null if the
+   *         image cannot be converted to the specified format.
+   */
+  public static byte[] encode(RenderedImage image, String format)
+    throws IOException
+  {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    boolean success = ImageIO.write(image, format, baos);
+    return success ? baos.toByteArray() : null;
+  }
+
+  /**
+   * Converts the given byte array into a {@link BufferedImage}.
+   *
+   * @param data The bytes to convert to an image.
+   * @return A {@link BufferedImage} of the given bytes, or null if an
+   *         image cannot be decoded converted to the specified format.
+   */
+  public static BufferedImage decode(byte[] data)
+    throws IOException
+  {
+    return ImageIO.read(new ByteArrayInputStream(data));
+  }
+
+}

kernel/base/src/test/java/com/twosigma/beakerx/util/ImagesTest.java

@@ -0,0 +1,54 @@
+/*
+ *  Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.twosigma.beakerx.util;
+
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests {@link Images}.
+ */
+public class ImagesTest {
+
+  @Test
+  public void testRoundTrip() throws IOException {
+    byte[] happyFace = {
+      -0x77, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+      0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07,
+      0x08, 0x02, 0x00, 0x00, 0x00, 0x4b, 0x30, -0x40, -0x7c, 0x00, 0x00, 0x00,
+      0x3d, 0x49, 0x44, 0x41, 0x54, 0x78, -0x26, 0x63, 0x60, 0x60, 0x60, 0x50,
+      0x51, 0x51, -0x0f, -0x0c, -0x0c, -0x34, -0x33, -0x33, -0x63, 0x34, 0x69,
+      -0x2e, -0x4a, 0x6d, -0x25, 0x6e, -0x21, -0x42, -0x33, 0x00, 0x11, 0x65,
+      -0x68, -0x50, 0x13, 0x28, 0x0a, 0x24, -0x77, 0x10, -0x3b, 0x62, 0x42,
+      0x2c, 0x33, 0x03, 0x5c, 0x14, -0x38, -0x3a, 0x2b, 0x0a, 0x34, 0x01, -0x38,
+      -0x7f, 0x20, -0x78, 0x09, 0x00, 0x46, 0x2a, 0x38, 0x70, -0x7d, 0x64, 0x48,
+      0x7a, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, -0x52, 0x42, 0x60,
+      -0x7e
+    };
+    BufferedImage bi = Images.decode(happyFace);
+    assertEquals(7, bi.getWidth());
+    assertEquals(7, bi.getHeight());
+    assertEquals(0xFF0090b9, bi.getRGB(2, 1));
+    assertEquals(0xFF5d0300, bi.getRGB(5, 5));
+    byte[] reencoded = Images.encode(bi);
+    assertArrayEquals(happyFace, reencoded);
+  }
+}