From 198c90a5c6895c79198a2d01191b57de59d0927f Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:06:52 +0900 Subject: [PATCH 01/52] basic support all image type pattern of unsigned 8bit when its blob except for JPG and animated image - https://mimesniff.spec.whatwg.org/#matching-an-image-type-pattern --- Cargo.lock | 24 ++++++++++ ext/canvas/01_image.js | 17 +++----- ext/canvas/Cargo.toml | 2 +- ext/canvas/lib.rs | 68 +++++++++++++++++++---------- tests/testdata/image/1x1-red8.bmp | Bin 0 -> 126 bytes tests/testdata/image/1x1-red8.gif | Bin 0 -> 49 bytes tests/testdata/image/1x1-red8.ico | Bin 0 -> 95 bytes tests/testdata/image/1x1-red8.png | Bin 0 -> 73 bytes tests/testdata/image/1x1-red8.webp | Bin 0 -> 34 bytes tests/testdata/image/1x1-white.png | Bin 109 -> 0 bytes tests/unit/image_bitmap_test.ts | 44 +++++++++++++++---- 11 files changed, 112 insertions(+), 43 deletions(-) create mode 100644 tests/testdata/image/1x1-red8.bmp create mode 100644 tests/testdata/image/1x1-red8.gif create mode 100644 tests/testdata/image/1x1-red8.ico create mode 100644 tests/testdata/image/1x1-red8.png create mode 100644 tests/testdata/image/1x1-red8.webp delete mode 100644 tests/testdata/image/1x1-white.png diff --git a/Cargo.lock b/Cargo.lock index d97b04d4aa40d1..0aa9c78c82ff42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3252,6 +3252,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.29.0" @@ -3779,6 +3789,8 @@ dependencies = [ "bytemuck", "byteorder", "color_quant", + "gif", + "jpeg-decoder", "num-traits", "png", ] @@ -3961,6 +3973,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.69" @@ -8139,6 +8157,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "wgpu-core" version = "0.21.1" diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 6fb1ee62fc1226..d69c853e41e88f 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { internals, primordials } from "ext:core/mod.js"; -import { op_image_decode_png, op_image_process } from "ext:core/ops"; +import { op_image_decode, op_image_process } from "ext:core/ops"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import { DOMException } from "ext:deno_web/01_dom_exception.js"; import { createFilteredInspectProxy } from "ext:deno_console/01_console.js"; @@ -204,7 +204,7 @@ function createImageBitmap( if (options.resizeHeight === 0) { return PromiseReject( new DOMException( - "options.resizeWidth has to be greater than 0", + "options.resizeHeight has to be greater than 0", "InvalidStateError", ), ); @@ -231,15 +231,12 @@ function createImageBitmap( if (ObjectPrototypeIsPrototypeOf(BlobPrototype, image)) { return (async () => { const data = await image.arrayBuffer(); - const mimetype = sniffImage(image.type); - if (mimetype !== "image/png") { - throw new DOMException( - `Unsupported type '${image.type}'`, - "InvalidStateError", - ); - } - const { data: imageData, width, height } = op_image_decode_png( + const mimeType = sniffImage(image.type); + const { data: imageData, width, height } = op_image_decode( new Uint8Array(data), + { + mimeType, + }, ); const processedImage = processImage( imageData, diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 6f4e6139573e87..7353dcc2bf6367 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -16,5 +16,5 @@ path = "lib.rs" [dependencies] deno_core.workspace = true deno_webgpu.workspace = true -image = { version = "0.24.7", default-features = false, features = ["png"] } +image = { version = "0.24.7", default-features = false, features = ["png","jpeg","bmp","ico","webp","gif"] } serde = { workspace = true, features = ["derive"] } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 72173f133158ba..25bf5fd225c7c0 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -1,12 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use image::imageops::FilterType; -use image::ColorType; -use image::ImageDecoder; +use image::GenericImageView; use image::Pixel; use image::RgbaImage; use serde::Deserialize; @@ -109,34 +107,56 @@ fn op_image_process( } #[derive(Debug, Serialize)] -struct DecodedPng { +struct DecodedImage { data: ToJsBuffer, width: u32, height: u32, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImageDecodeOptions { + mime_type: String, +} + #[op2] #[serde] -fn op_image_decode_png(#[buffer] buf: &[u8]) -> Result { - let png = image::codecs::png::PngDecoder::new(buf)?; - - let (width, height) = png.dimensions(); - - // TODO(@crowlKats): maybe use DynamicImage https://docs.rs/image/0.24.7/image/enum.DynamicImage.html ? - if png.color_type() != ColorType::Rgba8 { - return Err(type_error(format!( - "Color type '{:?}' not supported", - png.color_type() - ))); - } - - // read_image will assert that the buffer is the correct size, so we need to fill it with zeros - let mut png_data = vec![0_u8; png.total_bytes() as usize]; - - png.read_image(&mut png_data)?; +fn op_image_decode( + #[buffer] buf: &[u8], + #[serde] options: ImageDecodeOptions, +) -> Result { + let reader = std::io::BufReader::new(std::io::Cursor::new(buf)); + let image = match &*options.mime_type { + "image/png" => { + let decoder = image::codecs::png::PngDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + "image/jpeg" => { + let decoder = image::codecs::jpeg::JpegDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + "image/gif" => { + let decoder = image::codecs::gif::GifDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + "image/bmp" => { + let decoder = image::codecs::bmp::BmpDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + "image/x-icon" => { + let decoder = image::codecs::ico::IcoDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + "image/webp" => { + let decoder = image::codecs::webp::WebPDecoder::new(reader)?; + image::DynamicImage::from_decoder(decoder)? + } + _ => unreachable!(), + }; + let (width, height) = image.dimensions(); - Ok(DecodedPng { - data: png_data.into(), + Ok(DecodedImage { + data: image.into_bytes().into(), width, height, }) @@ -145,7 +165,7 @@ fn op_image_decode_png(#[buffer] buf: &[u8]) -> Result { deno_core::extension!( deno_canvas, deps = [deno_webidl, deno_web, deno_webgpu], - ops = [op_image_process, op_image_decode_png], + ops = [op_image_process, op_image_decode], lazy_loaded_esm = ["01_image.js"], ); diff --git a/tests/testdata/image/1x1-red8.bmp b/tests/testdata/image/1x1-red8.bmp new file mode 100644 index 0000000000000000000000000000000000000000..c28d7968f81957dcb715fed9d9de82aea62ddb58 GIT binary patch literal 126 vcmZ?rtz&?IDj<~u#EfvPz`zV-vj8zB_|L!qK_IFBPVPa)m}&@#{QnOCMGy&Q literal 0 HcmV?d00001 diff --git a/tests/testdata/image/1x1-red8.gif b/tests/testdata/image/1x1-red8.gif new file mode 100644 index 0000000000000000000000000000000000000000..0e5a2d361d355d72f7b16e61351c522b672382a3 GIT binary patch literal 49 ocmZ?wbhEHbWMp7uXkdT>#h)x3Af^t80L%Obu^5<`To@Uw0h$5@f&c&j literal 0 HcmV?d00001 diff --git a/tests/testdata/image/1x1-red8.ico b/tests/testdata/image/1x1-red8.ico new file mode 100644 index 0000000000000000000000000000000000000000..4cdfe144bd9f7e3bd92f5955e0c16e0caa9e0a85 GIT binary patch literal 95 zcmZQzU<5%%1|U#i@C1@#K-?MN=g!L|#RX*YdV0770cns5Mh-S0DKC1Y6G#box;TbZ jFfy_-{QA%EpMl{&D+A;IUojmQfpQF&ZX<|Nmdl!1 Date: Wed, 21 Aug 2024 05:47:41 +0900 Subject: [PATCH 02/52] support 8bit JPG --- Cargo.lock | 55 ++++++++++++++---- ext/canvas/01_image.js | 38 +++++++++--- ext/canvas/Cargo.toml | 2 +- ext/canvas/lib.rs | 90 +++++++++++++++++------------ tests/testdata/image/1x1-red8.jpeg | Bin 0 -> 631 bytes tests/unit/image_bitmap_test.ts | 7 +++ 6 files changed, 137 insertions(+), 55 deletions(-) create mode 100644 tests/testdata/image/1x1-red8.jpeg diff --git a/Cargo.lock b/Cargo.lock index 0aa9c78c82ff42..31ccf7c41b4ab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,6 +598,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.6.0" @@ -3782,17 +3788,29 @@ dependencies = [ [[package]] name = "image" -version = "0.24.9" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" dependencies = [ "bytemuck", - "byteorder", + "byteorder-lite", "color_quant", "gif", - "jpeg-decoder", + "image-webp", "num-traits", "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", ] [[package]] @@ -3973,12 +3991,6 @@ dependencies = [ "libc", ] -[[package]] -name = "jpeg-decoder" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" - [[package]] name = "js-sys" version = "0.3.69" @@ -5504,6 +5516,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-junit" version = "0.3.6" @@ -5826,7 +5844,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -8766,3 +8784,18 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768" +dependencies = [ + "zune-core", +] diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index d69c853e41e88f..2c74584d435961 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -18,6 +18,8 @@ const { PromiseResolve, PromiseReject, RangeError, + ObjectAssign, + ArrayPrototypeJoin, } = primordials; import { _data, @@ -164,6 +166,11 @@ function createImageBitmap( options = undefined, ) { const prefix = "Failed to execute 'createImageBitmap'"; + // Add the value when implementing to add support for ImageBitmapSource + const imageBitmapSources = [ + "Blob", + "ImageData", + ]; // Overload: createImageBitmap(image [, options ]) if (arguments.length < 3) { @@ -212,6 +219,16 @@ function createImageBitmap( const imageBitmap = webidl.createBranded(ImageBitmap); + // 6. Switch on image: + let imageBitmapSource; + if (ObjectPrototypeIsPrototypeOf(BlobPrototype, image)) { + imageBitmapSource = imageBitmapSources[0]; + } + if (ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image)) { + imageBitmapSource = imageBitmapSources[1]; + } + const _options = ObjectAssign(options, { imageBitmapSource }); + if (ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image)) { const processedImage = processImage( image[_data], @@ -221,7 +238,7 @@ function createImageBitmap( sy, sw, sh, - options, + _options, ); imageBitmap[_bitmapData] = processedImage.data; imageBitmap[_width] = processedImage.outputWidth; @@ -230,23 +247,23 @@ function createImageBitmap( } if (ObjectPrototypeIsPrototypeOf(BlobPrototype, image)) { return (async () => { - const data = await image.arrayBuffer(); + const data = new Uint8Array(await image.arrayBuffer()); const mimeType = sniffImage(image.type); - const { data: imageData, width, height } = op_image_decode( - new Uint8Array(data), + const { width, height } = op_image_decode( + data, { mimeType, }, ); const processedImage = processImage( - imageData, + data, width, height, sxOrOptions, sy, sw, sh, - options, + _options, ); imageBitmap[_bitmapData] = processedImage.data; imageBitmap[_width] = processedImage.outputWidth; @@ -254,7 +271,13 @@ function createImageBitmap( return imageBitmap; })(); } else { - return PromiseReject(new TypeError("Invalid or unsupported image value")); + return PromiseReject( + new TypeError( + `${prefix}: The provided value is not of type '(${ + ArrayPrototypeJoin(imageBitmapSources, " or ") + })'.`, + ), + ); } } @@ -332,6 +355,7 @@ function processImage(input, width, height, sx, sy, sw, sh, options) { premultiply: options.premultiplyAlpha === "default" ? null : (options.premultiplyAlpha === "premultiply"), + imageBitmapSource: options.imageBitmapSource, }, ); diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 7353dcc2bf6367..ff9662b4185d16 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -16,5 +16,5 @@ path = "lib.rs" [dependencies] deno_core.workspace = true deno_webgpu.workspace = true -image = { version = "0.24.7", default-features = false, features = ["png","jpeg","bmp","ico","webp","gif"] } +image = { version = "0.25.2", default-features = false, features = ["png","jpeg","bmp","ico","webp","gif"] } serde = { workspace = true, features = ["derive"] } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 25bf5fd225c7c0..87796340961c24 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -6,9 +6,9 @@ use deno_core::ToJsBuffer; use image::imageops::FilterType; use image::GenericImageView; use image::Pixel; -use image::RgbaImage; use serde::Deserialize; use serde::Serialize; +use std::io::Cursor; use std::path::PathBuf; #[derive(Debug, Deserialize)] @@ -34,6 +34,14 @@ struct ImageProcessArgs { resize_quality: ImageResizeQuality, flip_y: bool, premultiply: Option, + image_bitmap_source: ImageBitmapSource, +} + +#[derive(Debug, Deserialize)] +// Follow the cases defined in the spec +enum ImageBitmapSource { + Blob, + ImageData, } #[op2] @@ -42,16 +50,31 @@ fn op_image_process( #[buffer] buf: &[u8], #[serde] args: ImageProcessArgs, ) -> Result { - let view = - RgbaImage::from_vec(args.width, args.height, buf.to_vec()).unwrap(); + let view = match args.image_bitmap_source { + ImageBitmapSource::Blob => image::ImageReader::new(Cursor::new(buf)) + .with_guessed_format()? + .decode()?, + ImageBitmapSource::ImageData => { + // > 4.12.5.1.15 Pixel manipulation + // > imagedata.data + // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. + // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation + let image: image::DynamicImage = + image::RgbaImage::from_raw(args.width, args.height, buf.into()) + .expect("Invalid ImageData.") + .into(); + image + } + }; + let color = view.color(); let surface = if !(args.width == args.surface_width && args.height == args.surface_height && args.input_x == 0 && args.input_y == 0) { - let mut surface = RgbaImage::new(args.surface_width, args.surface_height); - + let mut surface = + image::DynamicImage::new(args.surface_width, args.surface_height, color); image::imageops::overlay(&mut surface, &view, args.input_x, args.input_y); surface @@ -66,12 +89,10 @@ fn op_image_process( ImageResizeQuality::High => FilterType::Lanczos3, }; - let mut image_out = image::imageops::resize( - &surface, - args.output_width, - args.output_height, - filter_type, - ); + // should use resize_exact + // https://github.com/image-rs/image/issues/1220#issuecomment-632060015 + let mut image_out = + surface.resize_exact(args.output_width, args.output_height, filter_type); if args.flip_y { image::imageops::flip_vertical_in_place(&mut image_out); @@ -79,36 +100,37 @@ fn op_image_process( // ignore 9. - if let Some(premultiply) = args.premultiply { - let is_not_premultiplied = image_out.pixels().any(|pixel| { - (pixel.0[0].max(pixel.0[1]).max(pixel.0[2])) > (255 * pixel.0[3]) - }); - - if premultiply { - if is_not_premultiplied { - for pixel in image_out.pixels_mut() { - let alpha = pixel.0[3]; + if color.has_alpha() { + if let Some(premultiply) = args.premultiply { + let is_not_premultiplied = image_out.pixels().any(|(_, _, pixel)| { + (pixel[0].max(pixel[1]).max(pixel[2])) > (255 * pixel[3]) + }); + + if premultiply { + if is_not_premultiplied { + for (_, _, mut pixel) in &mut image_out.pixels() { + let alpha = pixel[3]; + pixel.apply_without_alpha(|channel| { + (channel as f32 * (alpha as f32 / 255.0)) as u8 + }) + } + } + } else if !is_not_premultiplied { + for (_, _, mut pixel) in &mut image_out.pixels() { + let alpha = pixel[3]; pixel.apply_without_alpha(|channel| { - (channel as f32 * (alpha as f32 / 255.0)) as u8 + (channel as f32 / (alpha as f32 / 255.0)) as u8 }) } } - } else if !is_not_premultiplied { - for pixel in image_out.pixels_mut() { - let alpha = pixel.0[3]; - pixel.apply_without_alpha(|channel| { - (channel as f32 / (alpha as f32 / 255.0)) as u8 - }) - } } } - Ok(image_out.to_vec().into()) + Ok(image_out.into_bytes().into()) } #[derive(Debug, Serialize)] struct DecodedImage { - data: ToJsBuffer, width: u32, height: u32, } @@ -125,7 +147,7 @@ fn op_image_decode( #[buffer] buf: &[u8], #[serde] options: ImageDecodeOptions, ) -> Result { - let reader = std::io::BufReader::new(std::io::Cursor::new(buf)); + let reader = std::io::BufReader::new(Cursor::new(buf)); let image = match &*options.mime_type { "image/png" => { let decoder = image::codecs::png::PngDecoder::new(reader)?; @@ -155,11 +177,7 @@ fn op_image_decode( }; let (width, height) = image.dimensions(); - Ok(DecodedImage { - data: image.into_bytes().into(), - width, - height, - }) + Ok(DecodedImage { width, height }) } deno_core::extension!( diff --git a/tests/testdata/image/1x1-red8.jpeg b/tests/testdata/image/1x1-red8.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..3d042f466c62d3384ca70ecc08a3aaed255df6b9 GIT binary patch literal 631 zcmex=ECr+Nabot8F zYu9hwy!G(W<0ns_J%91?)yGetzkL1n{m0K=Ab&A3FhjfrBq1I{^A|8W7@1gDm|56C z{$gY*2V!PH7FI<=HX+AA_QXPAC8I_T5vPd@Hy-3vHV*nAnpAX=OH9S&q3TDF*T6m_ k&SOnv`3&wcguiYv@Gvt1lM%BZgFVBe*U$Lx`2W8N07nYW<^TWy literal 0 HcmV?d00001 diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index f1cf82de24ff98..27f867236ecfee 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -100,6 +100,13 @@ Deno.test(async function imageBitmapFromBlob() { // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } + { + const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.jpeg`)], { type: "image/jpeg" }); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([254, 0, 0])); + } { const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.bmp`)], { type: "image/bmp" }); const imageBitmap = await createImageBitmap(imageData); From 77703e8ba2caec1d58d652ea30f3db92826c3f3c Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:29:48 +0900 Subject: [PATCH 03/52] handling unsuportted situation - adding `DOMExceptionInvalidStateError` in global due to handling not supported image format - animation image is not supported currently --- ext/canvas/error.rs | 30 ++++++++++ ext/canvas/lib.rs | 56 +++++++++++++++--- runtime/errors.rs | 1 + runtime/js/99_main.js | 6 ++ tests/testdata/image/1x1-animation-rgba8.webp | Bin 0 -> 188 bytes tests/testdata/image/1x1-red32f.exr | Bin 0 -> 452 bytes tests/unit/image_bitmap_test.ts | 16 ++++- 7 files changed, 99 insertions(+), 10 deletions(-) create mode 100644 ext/canvas/error.rs create mode 100644 tests/testdata/image/1x1-animation-rgba8.webp create mode 100644 tests/testdata/image/1x1-red32f.exr diff --git a/ext/canvas/error.rs b/ext/canvas/error.rs new file mode 100644 index 00000000000000..bd08f9e8d64910 --- /dev/null +++ b/ext/canvas/error.rs @@ -0,0 +1,30 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_core::error::AnyError; +use std::fmt; + +#[derive(Debug)] +pub struct DOMExceptionInvalidStateError { + pub msg: String, +} + +impl DOMExceptionInvalidStateError { + pub fn new(msg: &str) -> Self { + DOMExceptionInvalidStateError { + msg: msg.to_string(), + } + } +} + +impl fmt::Display for DOMExceptionInvalidStateError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.pad(&self.msg) + } +} + +impl std::error::Error for DOMExceptionInvalidStateError {} + +pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { + e.downcast_ref::() + .map(|_| "DOMExceptionInvalidStateError") +} diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 87796340961c24..ef5516bef5abcc 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -1,16 +1,22 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use image::imageops::FilterType; +use image::AnimationDecoder; use image::GenericImageView; use image::Pixel; use serde::Deserialize; use serde::Serialize; +use std::io::BufReader; use std::io::Cursor; use std::path::PathBuf; +pub mod error; +use error::DOMExceptionInvalidStateError; + #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case")] enum ImageResizeQuality { @@ -20,6 +26,13 @@ enum ImageResizeQuality { High, } +#[derive(Debug, Deserialize)] +// Follow the cases defined in the spec +enum ImageBitmapSource { + Blob, + ImageData, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct ImageProcessArgs { @@ -37,13 +50,6 @@ struct ImageProcessArgs { image_bitmap_source: ImageBitmapSource, } -#[derive(Debug, Deserialize)] -// Follow the cases defined in the spec -enum ImageBitmapSource { - Blob, - ImageData, -} - #[op2] #[serde] fn op_image_process( @@ -147,10 +153,25 @@ fn op_image_decode( #[buffer] buf: &[u8], #[serde] options: ImageDecodeOptions, ) -> Result { - let reader = std::io::BufReader::new(Cursor::new(buf)); + let reader = BufReader::new(Cursor::new(buf)); + // + // TODO: support animated images + // It's a little hard to implement animated images along spec because of the complexity. + // + // > If this is an animated image, imageBitmap's bitmap data must only be taken from + // > the default image of the animation (the one that the format defines is to be used when animation is + // > not supported or is disabled), or, if there is no such image, the first frame of the animation. + // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html + // + // see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) + // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 + // let image = match &*options.mime_type { "image/png" => { let decoder = image::codecs::png::PngDecoder::new(reader)?; + if decoder.is_apng()? { + return Err(type_error("Animation image is not supported.")); + } image::DynamicImage::from_decoder(decoder)? } "image/jpeg" => { @@ -158,6 +179,11 @@ fn op_image_decode( image::DynamicImage::from_decoder(decoder)? } "image/gif" => { + let decoder = image::codecs::gif::GifDecoder::new(reader)?; + if decoder.into_frames().count() > 1 { + return Err(type_error("Animation image is not supported.")); + } + let reader = BufReader::new(Cursor::new(buf)); let decoder = image::codecs::gif::GifDecoder::new(reader)?; image::DynamicImage::from_decoder(decoder)? } @@ -171,9 +197,21 @@ fn op_image_decode( } "image/webp" => { let decoder = image::codecs::webp::WebPDecoder::new(reader)?; + if decoder.has_animation() { + return Err(type_error("Animation image is not supported.")); + } image::DynamicImage::from_decoder(decoder)? } - _ => unreachable!(), + // return an error if the mime type is not supported in the variable list of ImageTypePatternTable below + // ext/web/01_mimesniff.js + _ => { + return Err( + DOMExceptionInvalidStateError::new( + "The source image is not a supported format.", + ) + .into(), + ) + } }; let (width, height) = image.dimensions(); diff --git a/runtime/errors.rs b/runtime/errors.rs index 694402773e5f99..bd92b5cef409eb 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -159,6 +159,7 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { .or_else(|| deno_web::get_error_class_name(e)) .or_else(|| deno_webstorage::get_not_supported_error_class_name(e)) .or_else(|| deno_websocket::get_network_error_class_name(e)) + .or_else(|| deno_canvas::error::get_error_class_name(e)) .or_else(|| { e.downcast_ref::() .map(get_dlopen_error_class) diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 8b0d579ab539cb..8250ce59b26001 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -453,6 +453,12 @@ core.registerErrorBuilder( return new DOMException(msg, "DataError"); }, ); +core.registerErrorBuilder( + "DOMExceptionInvalidStateError", + function DOMExceptionInvalidStateError(msg) { + return new DOMException(msg, "InvalidStateError"); + }, +); function runtimeStart( denoVersion, diff --git a/tests/testdata/image/1x1-animation-rgba8.webp b/tests/testdata/image/1x1-animation-rgba8.webp new file mode 100644 index 0000000000000000000000000000000000000000..3d237b7ecf74e693bd77917822f62f5597eda51d GIT binary patch literal 188 zcmWIYbaUInz`zjh>J$(bU=hIuWD5Z?1UUM6`mzC;|AByk!O_pxO#>zcrC%^JFaq`Y j@B{h!KrFy6@VEaTtJ0$Xj6|xx@V|p3^#T|FL)8NSf{Y@{ literal 0 HcmV?d00001 diff --git a/tests/testdata/image/1x1-red32f.exr b/tests/testdata/image/1x1-red32f.exr new file mode 100644 index 0000000000000000000000000000000000000000..23ab61731ed8d471a3feb33199be3d3c973d6305 GIT binary patch literal 452 zcmah_OHM;E44gi(09M=t(F;K70Zo5i847+;M1bZq{>Nt@!mGF&{GbS@xyAN3C`Sg%0B>R6I@~d literal 0 HcmV?d00001 diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 27f867236ecfee..4d6cee31ec54a1 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals } from "./test_util.ts"; +import { assertEquals, assertRejects } from "./test_util.ts"; function generateNumberedData(n: number): Uint8ClampedArray { return new Uint8ClampedArray( @@ -128,6 +128,14 @@ Deno.test(async function imageBitmapFromBlob() { // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } + { + // the chunk of animation webp is below (3 frames, 1x1, 8-bit, RGBA) + // [ 255, 0, 0, 127, + // 0, 255, 0, 127, + // 0, 0, 255, 127 ] + const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`)], { type: "image/webp" }); + await assertRejects(() => createImageBitmap(imageData), TypeError); + } { const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.ico`)], { type: "image/x-icon" }); const imageBitmap = await createImageBitmap(imageData); @@ -135,4 +143,10 @@ Deno.test(async function imageBitmapFromBlob() { // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } + { + // image/x-exr is a known mimetype for OpenEXR + // https://www.digipres.org/formats/sources/fdd/formats/#fdd000583 + const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red32f.exr`)], { type: "image/x-exr" }); + await assertRejects(() => createImageBitmap(imageData), DOMException); + } }); From 22cf2f9bd8540dd862767ef775980b0448dcfd3c Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:30:20 +0900 Subject: [PATCH 04/52] fmt --- ext/canvas/Cargo.toml | 2 +- tests/unit/image_bitmap_test.ts | 38 ++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index ff9662b4185d16..0ac5a5d4cdea96 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -16,5 +16,5 @@ path = "lib.rs" [dependencies] deno_core.workspace = true deno_webgpu.workspace = true -image = { version = "0.25.2", default-features = false, features = ["png","jpeg","bmp","ico","webp","gif"] } +image = { version = "0.25.2", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } serde = { workspace = true, features = ["derive"] } diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 4d6cee31ec54a1..38f54dbedb3003 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -94,35 +94,50 @@ Deno.test(async function imageBitmapFlipY() { Deno.test(async function imageBitmapFromBlob() { const prefix = "tests/testdata/image"; { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.png`)], { type: "image/png" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.png`)], + { type: "image/png" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.jpeg`)], { type: "image/jpeg" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.jpeg`)], + { type: "image/jpeg" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([254, 0, 0])); } { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.bmp`)], { type: "image/bmp" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.bmp`)], + { type: "image/bmp" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.gif`)], { type: "image/gif" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.gif`)], + { type: "image/gif" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.webp`)], { type: "image/webp" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.webp`)], + { type: "image/webp" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore @@ -133,11 +148,16 @@ Deno.test(async function imageBitmapFromBlob() { // [ 255, 0, 0, 127, // 0, 255, 0, 127, // 0, 0, 255, 127 ] - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`)], { type: "image/webp" }); + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`), + ], { type: "image/webp" }); await assertRejects(() => createImageBitmap(imageData), TypeError); } { - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red8.ico`)], { type: "image/x-icon" }); + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red8.ico`)], + { type: "image/x-icon" }, + ); const imageBitmap = await createImageBitmap(imageData); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore @@ -146,7 +166,9 @@ Deno.test(async function imageBitmapFromBlob() { { // image/x-exr is a known mimetype for OpenEXR // https://www.digipres.org/formats/sources/fdd/formats/#fdd000583 - const imageData = new Blob([await Deno.readFile(`${prefix}/1x1-red32f.exr`)], { type: "image/x-exr" }); + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-red32f.exr`), + ], { type: "image/x-exr" }); await assertRejects(() => createImageBitmap(imageData), DOMException); } }); From 32d71f5bdc98933bd8514ae35218b9f9d6649e41 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 21 Aug 2024 22:12:41 +0900 Subject: [PATCH 05/52] cleanup `premultiplyAlpha` and adding test --- ext/canvas/01_image.js | 4 +- ext/canvas/lib.rs | 74 ++++++++++++++++------ tests/testdata/image/2x2-transparent8.png | Bin 0 -> 86 bytes tests/unit/image_bitmap_test.ts | 61 +++++++++++++++++- 4 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 tests/testdata/image/2x2-transparent8.png diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 2c74584d435961..b87d30ebf0a081 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -352,9 +352,7 @@ function processImage(input, width, height, sx, sy, sw, sh, options) { outputHeight, resizeQuality: options.resizeQuality, flipY: options.imageOrientation === "flipY", - premultiply: options.premultiplyAlpha === "default" - ? null - : (options.premultiplyAlpha === "premultiply"), + premultiplyAlpha: options.premultiplyAlpha, imageBitmapSource: options.imageBitmapSource, }, ); diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index ef5516bef5abcc..97adfdbb93ca5e 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -6,6 +6,7 @@ use deno_core::op2; use deno_core::ToJsBuffer; use image::imageops::FilterType; use image::AnimationDecoder; +use image::GenericImage; use image::GenericImageView; use image::Pixel; use serde::Deserialize; @@ -26,13 +27,21 @@ enum ImageResizeQuality { High, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq)] // Follow the cases defined in the spec enum ImageBitmapSource { Blob, ImageData, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum PremultiplyAlpha { + Default, + Premultiply, + None, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct ImageProcessArgs { @@ -46,7 +55,7 @@ struct ImageProcessArgs { output_height: u32, resize_quality: ImageResizeQuality, flip_y: bool, - premultiply: Option, + premultiply_alpha: PremultiplyAlpha, image_bitmap_source: ImageBitmapSource, } @@ -106,27 +115,56 @@ fn op_image_process( // ignore 9. + // 10. if color.has_alpha() { - if let Some(premultiply) = args.premultiply { - let is_not_premultiplied = image_out.pixels().any(|(_, _, pixel)| { - (pixel[0].max(pixel[1]).max(pixel[2])) > (255 * pixel[3]) - }); + match args.premultiply_alpha { + // 1. + PremultiplyAlpha::Default => { /* noop */ } + + // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied - if premultiply { + // 2. + PremultiplyAlpha::Premultiply => { + for (x, y, mut pixel) in image_out.clone().pixels() { + let alpha = pixel[3]; + let normalized_alpha = alpha as f64 / u8::MAX as f64; + pixel.apply_without_alpha(|rgb| { + (rgb as f64 * normalized_alpha).round() as u8 + }); + // FIXME: Looking at the API, put_pixel doesn't seem to be necessary, + // but apply_without_alpha with DynamicImage doesn't seem to work as expected. + image_out.put_pixel(x, y, pixel); + } + } + // 3. + PremultiplyAlpha::None => { + // NOTE: It's not clear how to handle the case of ImageData. + // https://issues.chromium.org/issues/339759426 + // https://github.com/whatwg/html/issues/5365 + if args.image_bitmap_source == ImageBitmapSource::ImageData { + return Ok(image_out.into_bytes().into()); + } + + // To determine if the image is premultiplied alpha, + // checking premultiplied RGBA value is one where any of the R/G/B channel values exceeds the alpha channel value. + // https://www.w3.org/TR/webgpu/#color-spaces + let is_not_premultiplied = image_out.pixels().any(|(_, _, pixel)| { + let [r, g, b] = [pixel[0], pixel[1], pixel[2]]; + let alpha = pixel[3]; + (r.max(g).max(b)) > u8::MAX.saturating_mul(alpha) + }); if is_not_premultiplied { - for (_, _, mut pixel) in &mut image_out.pixels() { - let alpha = pixel[3]; - pixel.apply_without_alpha(|channel| { - (channel as f32 * (alpha as f32 / 255.0)) as u8 - }) - } + return Ok(image_out.into_bytes().into()); } - } else if !is_not_premultiplied { - for (_, _, mut pixel) in &mut image_out.pixels() { + + for (x, y, mut pixel) in image_out.clone().pixels() { let alpha = pixel[3]; - pixel.apply_without_alpha(|channel| { - (channel as f32 / (alpha as f32 / 255.0)) as u8 - }) + pixel.apply_without_alpha(|rgb| { + (rgb as f64 / (alpha as f64 / u8::MAX as f64)).round() as u8 + }); + // FIXME: Looking at the API, put_pixel doesn't seem to be necessary, + // but apply_without_alpha with DynamicImage doesn't seem to work as expected. + image_out.put_pixel(x, y, pixel); } } } diff --git a/tests/testdata/image/2x2-transparent8.png b/tests/testdata/image/2x2-transparent8.png new file mode 100644 index 0000000000000000000000000000000000000000..153838d3e15d9a426017dbde3bb758f50aaae84d GIT binary patch literal 86 zcmeAS@N?(olHy`uVBq!ia0vp^Od!m`1|*BN@u~nRSx*5tw55 d|G%Drq29@t?Pq56TV9|-22WQ%mvv4FO#tF66*d3> literal 0 HcmV?d00001 diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 38f54dbedb3003..faceb198fb7684 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -2,6 +2,8 @@ import { assertEquals, assertRejects } from "./test_util.ts"; +const prefix = "tests/testdata/image"; + function generateNumberedData(n: number): Uint8ClampedArray { return new Uint8ClampedArray( Array.from({ length: n }, (_, i) => [i + 1, 0, 0, 1]).flat(), @@ -91,8 +93,65 @@ Deno.test(async function imageBitmapFlipY() { ])); }); +Deno.test(async function imageBitmapPremultiplyAlpha() { + const imageData = new ImageData( + new Uint8ClampedArray([ + 255, + 255, + 0, + 153, + ]), + 1, + 1, + ); + { + const imageBitmap = await createImageBitmap(imageData, { + premultiplyAlpha: "default", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ + 255, 255, 0, 153, + ])); + } + { + const imageBitmap = await createImageBitmap(imageData, { + premultiplyAlpha: "premultiply", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ + 153, 153, 0, 153 + ])); + } + { + const imageBitmap = await createImageBitmap(imageData, { + premultiplyAlpha: "none", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ + 255, 255, 0, 153, + ])); + } + { + const imageData = new Blob( + [await Deno.readFile(`${prefix}/2x2-transparent8.png`)], + { type: "image/png" }, + ); + const imageBitmap = await createImageBitmap(imageData, { + premultiplyAlpha: "none", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ + 255, 0, 0, 255, 0, 255, 0, 255, + 0, 0, 255, 255, 255, 0, 0, 127 + ])); + } +}); + Deno.test(async function imageBitmapFromBlob() { - const prefix = "tests/testdata/image"; { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.png`)], From 827e7b1cfc99ceb32246d4f51792f28c46e2742e Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:45:05 +0900 Subject: [PATCH 06/52] handling unsupported bit depth --- ext/canvas/lib.rs | 13 +++++++++++++ tests/testdata/image/1x1-red16.png | Bin 0 -> 77 bytes tests/unit/image_bitmap_test.ts | 7 +++++++ 3 files changed, 20 insertions(+) create mode 100644 tests/testdata/image/1x1-red16.png diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 97adfdbb93ca5e..a8bae555314773 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -8,6 +8,7 @@ use image::imageops::FilterType; use image::AnimationDecoder; use image::GenericImage; use image::GenericImageView; +use image::ImageDecoder; use image::Pixel; use serde::Deserialize; use serde::Serialize; @@ -210,10 +211,16 @@ fn op_image_decode( if decoder.is_apng()? { return Err(type_error("Animation image is not supported.")); } + if decoder.color_type() != image::ColorType::Rgba8 { + return Err(type_error("Supports 8-bit RGBA only.")); + } image::DynamicImage::from_decoder(decoder)? } "image/jpeg" => { let decoder = image::codecs::jpeg::JpegDecoder::new(reader)?; + if decoder.color_type() != image::ColorType::Rgb8 { + return Err(type_error("Supports 8-bit RGB only.")); + } image::DynamicImage::from_decoder(decoder)? } "image/gif" => { @@ -227,10 +234,16 @@ fn op_image_decode( } "image/bmp" => { let decoder = image::codecs::bmp::BmpDecoder::new(reader)?; + if decoder.color_type() != image::ColorType::Rgba8 { + return Err(type_error("Supports 8-bit RGBA only.")); + } image::DynamicImage::from_decoder(decoder)? } "image/x-icon" => { let decoder = image::codecs::ico::IcoDecoder::new(reader)?; + if decoder.color_type() != image::ColorType::Rgba8 { + return Err(type_error("Supports 8-bit RGBA only.")); + } image::DynamicImage::from_decoder(decoder)? } "image/webp" => { diff --git a/tests/testdata/image/1x1-red16.png b/tests/testdata/image/1x1-red16.png new file mode 100644 index 0000000000000000000000000000000000000000..ee9e279c144f696e96059a0d74e20ec92dba030d GIT binary patch literal 77 zcmeAS@N?(olHy`uVBq!ia0vp^j36w)1| createImageBitmap(imageData), TypeError); + } { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.jpeg`)], From 0de8a9be8febf0702b834d961438f40680865205 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:58:55 +0900 Subject: [PATCH 07/52] port to rust --- Cargo.lock | 6 +- ext/canvas/01_image.js | 176 ++----- ext/canvas/Cargo.toml | 2 + ext/canvas/lib.rs | 801 +++++++++++++++++++++++++------- tests/unit/image_bitmap_test.ts | 16 +- 5 files changed, 696 insertions(+), 305 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31ccf7c41b4ab3..8955a075ec6b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1373,8 +1373,10 @@ name = "deno_canvas" version = "0.35.0" dependencies = [ "deno_core", + "deno_terminal 0.2.0", "deno_webgpu", "image", + "num-traits", "serde", ] @@ -4756,9 +4758,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index b87d30ebf0a081..41b5b948b915b0 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { internals, primordials } from "ext:core/mod.js"; -import { op_image_decode, op_image_process } from "ext:core/ops"; +import { op_image_process } from "ext:core/ops"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import { DOMException } from "ext:deno_web/01_dom_exception.js"; import { createFilteredInspectProxy } from "ext:deno_console/01_console.js"; @@ -14,11 +14,8 @@ const { TypeError, TypedArrayPrototypeGetBuffer, Uint8Array, - MathCeil, - PromiseResolve, PromiseReject, RangeError, - ObjectAssign, ArrayPrototypeJoin, } = primordials; import { @@ -219,55 +216,57 @@ function createImageBitmap( const imageBitmap = webidl.createBranded(ImageBitmap); - // 6. Switch on image: - let imageBitmapSource; - if (ObjectPrototypeIsPrototypeOf(BlobPrototype, image)) { - imageBitmapSource = imageBitmapSources[0]; - } - if (ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image)) { - imageBitmapSource = imageBitmapSources[1]; - } - const _options = ObjectAssign(options, { imageBitmapSource }); - - if (ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image)) { - const processedImage = processImage( - image[_data], - image[_width], - image[_height], - sxOrOptions, - sy, - sw, - sh, - _options, - ); - imageBitmap[_bitmapData] = processedImage.data; - imageBitmap[_width] = processedImage.outputWidth; - imageBitmap[_height] = processedImage.outputHeight; - return PromiseResolve(imageBitmap); - } - if (ObjectPrototypeIsPrototypeOf(BlobPrototype, image)) { + // 6. Switch on image + const isBlob = ObjectPrototypeIsPrototypeOf(BlobPrototype, image); + const isImageData = ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image); + if ( + isImageData || + isBlob + ) { return (async () => { - const data = new Uint8Array(await image.arrayBuffer()); - const mimeType = sniffImage(image.type); - const { width, height } = op_image_decode( - data, + let width = 0; + let height = 0; + let mimeType = ""; + let imageBitmapSource, buf; + if (isBlob) { + imageBitmapSource = imageBitmapSources[0]; + buf = new Uint8Array(await image.arrayBuffer()); + mimeType = sniffImage(image.type); + } + if (isImageData) { + width = image[_width]; + height = image[_height]; + imageBitmapSource = imageBitmapSources[1]; + buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); + } + + let sx; + if (typeof sxOrOptions === "number") { + sx = sxOrOptions; + } + + const processedImage = op_image_process( + buf, { + width, + height, + sx, + sy, + sw, + sh, + imageOrientation: options.imageOrientation ?? "from-image", + premultiplyAlpha: options.premultiplyAlpha ?? "default", + colorSpaceConversion: options.colorSpaceConversion ?? "default", + resizeWidth: options.resizeWidth, + resizeHeight: options.resizeHeight, + resizeQuality: options.resizeQuality ?? "low", + imageBitmapSource, mimeType, }, ); - const processedImage = processImage( - data, - width, - height, - sxOrOptions, - sy, - sw, - sh, - _options, - ); imageBitmap[_bitmapData] = processedImage.data; - imageBitmap[_width] = processedImage.outputWidth; - imageBitmap[_height] = processedImage.outputHeight; + imageBitmap[_width] = processedImage.width; + imageBitmap[_height] = processedImage.height; return imageBitmap; })(); } else { @@ -281,89 +280,6 @@ function createImageBitmap( } } -function processImage(input, width, height, sx, sy, sw, sh, options) { - let sourceRectangle; - - if ( - sx !== undefined && sy !== undefined && sw !== undefined && sh !== undefined - ) { - sourceRectangle = [ - [sx, sy], - [sx + sw, sy], - [sx + sw, sy + sh], - [sx, sy + sh], - ]; - } else { - sourceRectangle = [ - [0, 0], - [width, 0], - [width, height], - [0, height], - ]; - } - const widthOfSourceRect = sourceRectangle[1][0] - sourceRectangle[0][0]; - const heightOfSourceRect = sourceRectangle[3][1] - sourceRectangle[0][1]; - - let outputWidth; - if (options.resizeWidth !== undefined) { - outputWidth = options.resizeWidth; - } else if (options.resizeHeight !== undefined) { - outputWidth = MathCeil( - (widthOfSourceRect * options.resizeHeight) / heightOfSourceRect, - ); - } else { - outputWidth = widthOfSourceRect; - } - - let outputHeight; - if (options.resizeHeight !== undefined) { - outputHeight = options.resizeHeight; - } else if (options.resizeWidth !== undefined) { - outputHeight = MathCeil( - (heightOfSourceRect * options.resizeWidth) / widthOfSourceRect, - ); - } else { - outputHeight = heightOfSourceRect; - } - - if (options.colorSpaceConversion === "none") { - throw new TypeError("options.colorSpaceConversion 'none' is not supported"); - } - - /* - * The cropping works differently than the spec specifies: - * The spec states to create an infinite surface and place the top-left corner - * of the image a 0,0 and crop based on sourceRectangle. - * - * We instead create a surface the size of sourceRectangle, and position - * the image at the correct location, which is the inverse of the x & y of - * sourceRectangle's top-left corner. - */ - const data = op_image_process( - new Uint8Array(TypedArrayPrototypeGetBuffer(input)), - { - width, - height, - surfaceWidth: widthOfSourceRect, - surfaceHeight: heightOfSourceRect, - inputX: sourceRectangle[0][0] * -1, // input_x - inputY: sourceRectangle[0][1] * -1, // input_y - outputWidth, - outputHeight, - resizeQuality: options.resizeQuality, - flipY: options.imageOrientation === "flipY", - premultiplyAlpha: options.premultiplyAlpha, - imageBitmapSource: options.imageBitmapSource, - }, - ); - - return { - data, - outputWidth, - outputHeight, - }; -} - function getBitmapData(imageBitmap) { return imageBitmap[_bitmapData]; } diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 0ac5a5d4cdea96..3ae6b9d7cde7ea 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -15,6 +15,8 @@ path = "lib.rs" [dependencies] deno_core.workspace = true +deno_terminal.workspace = true deno_webgpu.workspace = true image = { version = "0.25.2", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } +num-traits = { version = "0.2.19" } serde = { workspace = true, features = ["derive"] } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index a8bae555314773..c55b2a5dac215f 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -3,24 +3,277 @@ use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::op2; +use deno_core::JsBuffer; use deno_core::ToJsBuffer; +use deno_terminal::colors::cyan; +use image::codecs::bmp::BmpDecoder; +use image::codecs::gif::GifDecoder; +use image::codecs::ico::IcoDecoder; +use image::codecs::jpeg::JpegDecoder; +use image::codecs::png::PngDecoder; +use image::codecs::webp::WebPDecoder; +use image::imageops::overlay; use image::imageops::FilterType; use image::AnimationDecoder; -use image::GenericImage; +use image::ColorType; +use image::DynamicImage; use image::GenericImageView; +use image::ImageBuffer; use image::ImageDecoder; +use image::LumaA; use image::Pixel; +use image::Primitive; +use image::Rgba; +use image::RgbaImage; +use num_traits::NumCast; +use num_traits::SaturatingMul; use serde::Deserialize; use serde::Serialize; +use std::borrow::Cow; +use std::io::BufRead; use std::io::BufReader; use std::io::Cursor; +use std::io::Seek; use std::path::PathBuf; pub mod error; use error::DOMExceptionInvalidStateError; -#[derive(Debug, Deserialize)] -#[serde(rename_all = "snake_case")] +fn to_js_buffer(image: &DynamicImage) -> ToJsBuffer { + image.as_bytes().to_vec().into() +} + +fn image_error_message<'a, T: Into>>( + opreation: T, + reason: T, +) -> String { + format!( + "An error has occurred while {}. +reason: {}", + opreation.into(), + reason.into(), + ) +} + +// reference +// https://github.com/image-rs/image/blob/6d19ffa72756c1b00e7979a90f8794a0ef847b88/src/color.rs#L739 +trait ProcessPremultiplyAlpha { + fn premultiply_alpha(&self) -> Self; +} + +impl ProcessPremultiplyAlpha for LumaA { + fn premultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); + + if normalized_alpha == 0.0 { + return LumaA::([pixel[0], pixel[alpha_index]]); + } + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) + .unwrap() + } + + LumaA::([pixel[0], pixel[alpha_index]]) + } +} + +impl ProcessPremultiplyAlpha for Rgba { + fn premultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); + + if normalized_alpha == 0.0 { + return Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]); + } + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) + .unwrap() + } + + Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + } +} + +fn process_premultiply_alpha(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + ProcessPremultiplyAlpha + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + for (x, y, pixel) in image.pixels() { + let pixel = pixel.premultiply_alpha(); + + out.put_pixel(x, y, pixel); + } + + out +} + +fn apply_premultiply_alpha( + image: &DynamicImage, +) -> Result { + match image.color() { + ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( + &image.to_luma_alpha8(), + ))), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_premultiply_alpha(&image.to_rgba8()), + )), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16( + process_premultiply_alpha(&image.to_luma_alpha16()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_premultiply_alpha(&image.to_rgba16()), + )), + _ => Err(type_error(image_error_message( + "apply premultiplyAlpha: premultiply", + "The color type is not supported.", + ))), + } +} + +trait ProcessUnpremultiplyAlpha { + /// To determine if the image is premultiplied alpha, + /// checking premultiplied RGBA value is one where any of the R/G/B channel values exceeds the alpha channel value.\ + /// https://www.w3.org/TR/webgpu/#color-spaces + fn is_premultiplied_alpha(&self) -> bool; + fn unpremultiply_alpha(&self) -> Self; +} + +impl ProcessUnpremultiplyAlpha for Rgba { + fn is_premultiplied_alpha(&self) -> bool { + let max_t = T::DEFAULT_MAX_VALUE; + + let pixel = [self.0[0], self.0[1], self.0[2]]; + let alpha_index = self.0.len() - 1; + let alpha = self.0[alpha_index]; + + match pixel.iter().max() { + Some(rgb_max) => rgb_max < &max_t.saturating_mul(&alpha), + // usually doesn't reach here + None => false, + } + } + + fn unpremultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from( + (rgb.to_f32().unwrap() + / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) + .round(), + ) + .unwrap(); + } + + Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + } +} + +impl ProcessUnpremultiplyAlpha + for LumaA +{ + fn is_premultiplied_alpha(&self) -> bool { + let max_t = T::DEFAULT_MAX_VALUE; + + let pixel = [self.0[0]]; + let alpha_index = self.0.len() - 1; + let alpha = self.0[alpha_index]; + + pixel[0] < max_t.saturating_mul(&alpha) + } + + fn unpremultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from( + (rgb.to_f32().unwrap() + / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) + .round(), + ) + .unwrap(); + } + + LumaA::([pixel[0], pixel[alpha_index]]) + } +} + +fn process_unpremultiply_alpha(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + ProcessUnpremultiplyAlpha + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + let is_premultiplied_alpha = image + .pixels() + .any(|(_, _, pixel)| pixel.is_premultiplied_alpha()); + + for (x, y, pixel) in image.pixels() { + let pixel = if is_premultiplied_alpha { + pixel.unpremultiply_alpha() + } else { + // return the original + pixel + }; + + out.put_pixel(x, y, pixel); + } + + out +} + +fn apply_unpremultiply_alpha( + image: &DynamicImage, +) -> Result { + match image.color() { + ColorType::La8 => Ok(DynamicImage::ImageLumaA8( + process_unpremultiply_alpha(&image.to_luma_alpha8()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_unpremultiply_alpha(&image.to_rgba8()), + )), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16( + process_unpremultiply_alpha(&image.to_luma_alpha16()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_unpremultiply_alpha(&image.to_rgba16()), + )), + _ => Err(type_error(image_error_message( + "apply premultiplyAlpha: none", + "The color type is not supported.", + ))), + } +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] enum ImageResizeQuality { Pixelated, Low, @@ -35,70 +288,370 @@ enum ImageBitmapSource { ImageData, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "snake_case")] +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] enum PremultiplyAlpha { Default, Premultiply, None, } +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum ColorSpaceConversion { + Default, + None, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum ImageOrientation { + FlipY, + #[serde(rename = "from-image")] + FromImage, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct ImageProcessArgs { width: u32, height: u32, - surface_width: u32, - surface_height: u32, - input_x: i64, - input_y: i64, - output_width: u32, - output_height: u32, - resize_quality: ImageResizeQuality, - flip_y: bool, + sx: Option, + sy: Option, + sw: Option, + sh: Option, + image_orientation: ImageOrientation, premultiply_alpha: PremultiplyAlpha, + color_space_conversion: ColorSpaceConversion, + resize_width: Option, + resize_height: Option, + resize_quality: ImageResizeQuality, image_bitmap_source: ImageBitmapSource, + mime_type: String, } -#[op2] -#[serde] -fn op_image_process( - #[buffer] buf: &[u8], - #[serde] args: ImageProcessArgs, -) -> Result { - let view = match args.image_bitmap_source { - ImageBitmapSource::Blob => image::ImageReader::new(Cursor::new(buf)) - .with_guessed_format()? - .decode()?, +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ImageProcessResult { + data: ToJsBuffer, + width: u32, + height: u32, +} + +trait ImageDecoderFromReader<'a, R: BufRead + Seek> { + fn to_decoder(reader: R) -> Result + where + Self: Sized; +} + +type ImageDecoderFromReaderType<'a> = BufReader>; + +macro_rules! impl_image_decoder_converter { + ($decoder:ty, $reader:ty) => { + impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { + fn to_decoder(reader: R) -> Result + where + Self: Sized, + { + match <$decoder>::new(reader) { + Ok(decoder) => Ok(decoder), + Err(err) => { + return Err( + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &err.to_string(), + )) + .into(), + ) + } + } + } + } + }; +} + +impl_image_decoder_converter!(PngDecoder, ImageDecoderFromReaderType); +impl_image_decoder_converter!(JpegDecoder, ImageDecoderFromReaderType); +impl_image_decoder_converter!(GifDecoder, ImageDecoderFromReaderType); +impl_image_decoder_converter!(BmpDecoder, ImageDecoderFromReaderType); +impl_image_decoder_converter!(IcoDecoder, ImageDecoderFromReaderType); +impl_image_decoder_converter!(WebPDecoder, ImageDecoderFromReaderType); + +fn decoder_to_intermediate_image( + decoder: impl ImageDecoder, +) -> Result { + match DynamicImage::from_decoder(decoder) { + Ok(image) => Ok(image), + Err(err) => Err( + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &err.to_string(), + )) + .into(), + ), + } +} + +fn decode_bitmap_data( + buf: &[u8], + width: u32, + height: u32, + image_bitmap_source: &ImageBitmapSource, + mime_type: String, +) -> Result<(DynamicImage, u32, u32), AnyError> { + let (view, width, height) = match image_bitmap_source { + ImageBitmapSource::Blob => { + let image = match &*mime_type { + // + // TODO: support animated images + // It's a little hard to implement animated images along spec because of the complexity. + // + // > If this is an animated image, imageBitmap's bitmap data must only be taken from + // > the default image of the animation (the one that the format defines is to be used when animation is + // > not supported or is disabled), or, if there is no such image, the first frame of the animation. + // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html + // + // see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) + // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 + // + "image/png" => { + let decoder: PngDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + if decoder.is_apng()? { + return Err( + DOMExceptionInvalidStateError::new( + "Animation image is not supported.", + ) + .into(), + ); + } + decoder_to_intermediate_image(decoder)? + } + "image/jpeg" => { + let decoder: JpegDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + decoder_to_intermediate_image(decoder)? + } + "image/gif" => { + let decoder: GifDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + if decoder.into_frames().count() > 1 { + return Err( + DOMExceptionInvalidStateError::new( + "Animation image is not supported.", + ) + .into(), + ); + } + let decoder: GifDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + decoder_to_intermediate_image(decoder)? + } + "image/bmp" => { + let decoder: BmpDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + decoder_to_intermediate_image(decoder)? + } + "image/x-icon" => { + let decoder: IcoDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + decoder_to_intermediate_image(decoder)? + } + "image/webp" => { + let decoder: WebPDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + if decoder.has_animation() { + return Err( + DOMExceptionInvalidStateError::new( + "Animation image is not supported.", + ) + .into(), + ); + } + decoder_to_intermediate_image(decoder)? + } + "" => { + return Err( + DOMExceptionInvalidStateError::new( + &format!("The MIME type of source image is not specified. +INFO: The behavior of the Blob constructor in browsers is different from the spec. +It needs to specify the MIME type like {} that works well between Deno and browsers. +See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", + cyan("new Blob([blobParts], { type: 'image/png' })") + )).into(), + ) + } + // return an error if the MIME type is not supported in the variable list of ImageTypePatternTable below + // ext/web/01_mimesniff.js + // + // NOTE: Chromium supports AVIF + // https://source.chromium.org/chromium/chromium/src/+/ef3f4e4ed97079dc57861d1195fb2389483bc195:third_party/blink/renderer/platform/image-decoders/image_decoder.cc;l=311 + x => { + return Err( + DOMExceptionInvalidStateError::new( + &format!("The the MIME type {} of source image is not a supported format. +INFO: The following MIME types are supported: +See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", + x + )).into() + ) + } + }; + + let width = image.width(); + let height = image.height(); + + (image, width, height) + } ImageBitmapSource::ImageData => { // > 4.12.5.1.15 Pixel manipulation // > imagedata.data // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation - let image: image::DynamicImage = - image::RgbaImage::from_raw(args.width, args.height, buf.into()) - .expect("Invalid ImageData.") - .into(); - image + let image = match RgbaImage::from_raw(width, height, buf.into()) { + Some(image) => image.into(), + None => { + return Err(type_error(image_error_message( + "decoding", + "The Chunk Data is not big enough with the specified width and height.", + ))) + } + }; + + (image, width, height) } }; + + Ok((view, width, height)) +} + +#[op2] +#[serde] +fn op_image_process( + #[buffer] zero_copy: JsBuffer, + #[serde] args: ImageProcessArgs, +) -> Result { + let buf = &*zero_copy; + let ImageProcessArgs { + width, + height, + sh, + sw, + sx, + sy, + image_orientation, + premultiply_alpha, + color_space_conversion, + resize_width, + resize_height, + resize_quality, + image_bitmap_source, + mime_type, + } = ImageProcessArgs { + width: args.width, + height: args.height, + sx: args.sx, + sy: args.sy, + sw: args.sw, + sh: args.sh, + image_orientation: args.image_orientation, + premultiply_alpha: args.premultiply_alpha, + color_space_conversion: args.color_space_conversion, + resize_width: args.resize_width, + resize_height: args.resize_height, + resize_quality: args.resize_quality, + image_bitmap_source: args.image_bitmap_source, + mime_type: args.mime_type, + }; + + let (view, width, height) = + decode_bitmap_data(buf, width, height, &image_bitmap_source, mime_type)?; + + #[rustfmt::skip] + let source_rectangle: [[i32; 2]; 4] = + if let (Some(sx), Some(sy), Some(sw), Some(sh)) = (sx, sy, sw, sh) { + [ + [sx, sy], + [sx + sw, sy], + [sx + sw, sy + sh], + [sx, sy + sh] + ] + } else { + [ + [0, 0], + [width as i32, 0], + [width as i32, height as i32], + [0, height as i32], + ] + }; + + /* + * The cropping works differently than the spec specifies: + * The spec states to create an infinite surface and place the top-left corner + * of the image a 0,0 and crop based on sourceRectangle. + * + * We instead create a surface the size of sourceRectangle, and position + * the image at the correct location, which is the inverse of the x & y of + * sourceRectangle's top-left corner. + */ + let input_x = -(source_rectangle[0][0] as i64); + let input_y = -(source_rectangle[0][1] as i64); + + let surface_width = (source_rectangle[1][0] - source_rectangle[0][0]) as u32; + let surface_height = (source_rectangle[3][1] - source_rectangle[0][1]) as u32; + + let output_width = if let Some(resize_width) = resize_width { + resize_width + } else if let Some(resize_height) = resize_height { + (surface_width * resize_height).div_ceil(surface_height) + } else { + surface_width + }; + + let output_height = if let Some(resize_height) = resize_height { + resize_height + } else if let Some(resize_width) = resize_width { + (surface_height * resize_width).div_ceil(surface_width) + } else { + surface_height + }; + + if color_space_conversion == ColorSpaceConversion::None { + return Err(type_error( + "options.colorSpaceConversion 'none' is not supported", + )); + } + let color = view.color(); - let surface = if !(args.width == args.surface_width - && args.height == args.surface_height - && args.input_x == 0 - && args.input_y == 0) + let surface = if !(width == surface_width + && height == surface_height + && input_x == 0 + && input_y == 0) { - let mut surface = - image::DynamicImage::new(args.surface_width, args.surface_height, color); - image::imageops::overlay(&mut surface, &view, args.input_x, args.input_y); + let mut surface = DynamicImage::new(surface_width, surface_height, color); + overlay(&mut surface, &view, input_x, input_y); surface } else { view }; - let filter_type = match args.resize_quality { + let filter_type = match resize_quality { ImageResizeQuality::Pixelated => FilterType::Nearest, ImageResizeQuality::Low => FilterType::Triangle, ImageResizeQuality::Medium => FilterType::CatmullRom, @@ -107,18 +660,28 @@ fn op_image_process( // should use resize_exact // https://github.com/image-rs/image/issues/1220#issuecomment-632060015 - let mut image_out = - surface.resize_exact(args.output_width, args.output_height, filter_type); + let image_out = + surface.resize_exact(output_width, output_height, filter_type); - if args.flip_y { - image::imageops::flip_vertical_in_place(&mut image_out); - } + // + // FIXME: It also need to fix about orientation when the spec is updated. + // + // > Multiple browser vendors discussed this a while back and (99% sure, from recollection) + // > agreed to change createImageBitmap's behavior. + // > The HTML spec should be updated to say: + // > first EXIF orientation is applied, and then if imageOrientation is flipY, the image is flipped vertically + // https://github.com/whatwg/html/issues/8085#issuecomment-2204696312 + let image_out = if image_orientation == ImageOrientation::FlipY { + image_out.flipv() + } else { + image_out + }; // ignore 9. // 10. if color.has_alpha() { - match args.premultiply_alpha { + match premultiply_alpha { // 1. PremultiplyAlpha::Default => { /* noop */ } @@ -126,153 +689,49 @@ fn op_image_process( // 2. PremultiplyAlpha::Premultiply => { - for (x, y, mut pixel) in image_out.clone().pixels() { - let alpha = pixel[3]; - let normalized_alpha = alpha as f64 / u8::MAX as f64; - pixel.apply_without_alpha(|rgb| { - (rgb as f64 * normalized_alpha).round() as u8 - }); - // FIXME: Looking at the API, put_pixel doesn't seem to be necessary, - // but apply_without_alpha with DynamicImage doesn't seem to work as expected. - image_out.put_pixel(x, y, pixel); - } + let result = apply_premultiply_alpha(&image_out)?; + let data = to_js_buffer(&result); + return Ok(ImageProcessResult { + data, + width: output_width, + height: output_height, + }); } // 3. PremultiplyAlpha::None => { // NOTE: It's not clear how to handle the case of ImageData. // https://issues.chromium.org/issues/339759426 // https://github.com/whatwg/html/issues/5365 - if args.image_bitmap_source == ImageBitmapSource::ImageData { - return Ok(image_out.into_bytes().into()); + if image_bitmap_source == ImageBitmapSource::ImageData { + return Ok(ImageProcessResult { + data: image_out.clone().into_bytes().into(), + width: output_width, + height: output_height, + }); } - // To determine if the image is premultiplied alpha, - // checking premultiplied RGBA value is one where any of the R/G/B channel values exceeds the alpha channel value. - // https://www.w3.org/TR/webgpu/#color-spaces - let is_not_premultiplied = image_out.pixels().any(|(_, _, pixel)| { - let [r, g, b] = [pixel[0], pixel[1], pixel[2]]; - let alpha = pixel[3]; - (r.max(g).max(b)) > u8::MAX.saturating_mul(alpha) + let result = apply_unpremultiply_alpha(&image_out)?; + let data = to_js_buffer(&result); + return Ok(ImageProcessResult { + data, + width: output_width, + height: output_height, }); - if is_not_premultiplied { - return Ok(image_out.into_bytes().into()); - } - - for (x, y, mut pixel) in image_out.clone().pixels() { - let alpha = pixel[3]; - pixel.apply_without_alpha(|rgb| { - (rgb as f64 / (alpha as f64 / u8::MAX as f64)).round() as u8 - }); - // FIXME: Looking at the API, put_pixel doesn't seem to be necessary, - // but apply_without_alpha with DynamicImage doesn't seem to work as expected. - image_out.put_pixel(x, y, pixel); - } } } } - Ok(image_out.into_bytes().into()) -} - -#[derive(Debug, Serialize)] -struct DecodedImage { - width: u32, - height: u32, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ImageDecodeOptions { - mime_type: String, -} - -#[op2] -#[serde] -fn op_image_decode( - #[buffer] buf: &[u8], - #[serde] options: ImageDecodeOptions, -) -> Result { - let reader = BufReader::new(Cursor::new(buf)); - // - // TODO: support animated images - // It's a little hard to implement animated images along spec because of the complexity. - // - // > If this is an animated image, imageBitmap's bitmap data must only be taken from - // > the default image of the animation (the one that the format defines is to be used when animation is - // > not supported or is disabled), or, if there is no such image, the first frame of the animation. - // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html - // - // see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) - // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 - // - let image = match &*options.mime_type { - "image/png" => { - let decoder = image::codecs::png::PngDecoder::new(reader)?; - if decoder.is_apng()? { - return Err(type_error("Animation image is not supported.")); - } - if decoder.color_type() != image::ColorType::Rgba8 { - return Err(type_error("Supports 8-bit RGBA only.")); - } - image::DynamicImage::from_decoder(decoder)? - } - "image/jpeg" => { - let decoder = image::codecs::jpeg::JpegDecoder::new(reader)?; - if decoder.color_type() != image::ColorType::Rgb8 { - return Err(type_error("Supports 8-bit RGB only.")); - } - image::DynamicImage::from_decoder(decoder)? - } - "image/gif" => { - let decoder = image::codecs::gif::GifDecoder::new(reader)?; - if decoder.into_frames().count() > 1 { - return Err(type_error("Animation image is not supported.")); - } - let reader = BufReader::new(Cursor::new(buf)); - let decoder = image::codecs::gif::GifDecoder::new(reader)?; - image::DynamicImage::from_decoder(decoder)? - } - "image/bmp" => { - let decoder = image::codecs::bmp::BmpDecoder::new(reader)?; - if decoder.color_type() != image::ColorType::Rgba8 { - return Err(type_error("Supports 8-bit RGBA only.")); - } - image::DynamicImage::from_decoder(decoder)? - } - "image/x-icon" => { - let decoder = image::codecs::ico::IcoDecoder::new(reader)?; - if decoder.color_type() != image::ColorType::Rgba8 { - return Err(type_error("Supports 8-bit RGBA only.")); - } - image::DynamicImage::from_decoder(decoder)? - } - "image/webp" => { - let decoder = image::codecs::webp::WebPDecoder::new(reader)?; - if decoder.has_animation() { - return Err(type_error("Animation image is not supported.")); - } - image::DynamicImage::from_decoder(decoder)? - } - // return an error if the mime type is not supported in the variable list of ImageTypePatternTable below - // ext/web/01_mimesniff.js - _ => { - return Err( - DOMExceptionInvalidStateError::new( - "The source image is not a supported format.", - ) - .into(), - ) - } - }; - let (width, height) = image.dimensions(); - - Ok(DecodedImage { width, height }) + Ok(ImageProcessResult { + data: image_out.clone().into_bytes().into(), + width: output_width, + height: output_height, + }) } deno_core::extension!( deno_canvas, deps = [deno_webidl, deno_web, deno_webgpu], - ops = [op_image_process, op_image_decode], + ops = [op_image_process], lazy_loaded_esm = ["01_image.js"], ); diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index a988145fa27522..a57a1babc49523 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -167,7 +167,19 @@ Deno.test(async function imageBitmapFromBlob() { [await Deno.readFile(`${prefix}/1x1-red16.png`)], { type: "image/png" }, ); - await assertRejects(() => createImageBitmap(imageData), TypeError); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), + new Uint8Array( + [ + 255, 255, // R + 0, 0, // G + 0, 0, // B + 255, 255 // A + ] + ) + ); } { const imageData = new Blob( @@ -217,7 +229,7 @@ Deno.test(async function imageBitmapFromBlob() { const imageData = new Blob([ await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`), ], { type: "image/webp" }); - await assertRejects(() => createImageBitmap(imageData), TypeError); + await assertRejects(() => createImageBitmap(imageData), DOMException); } { const imageData = new Blob( From d0da79eb73ea4bb6f9eb8fbe52a61c21a4c32c78 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 29 Aug 2024 22:18:44 +0900 Subject: [PATCH 08/52] support animated PNG --- ext/canvas/lib.rs | 101 ++++++++---------- .../image/1x1-2f-animated-has-def.png | Bin 0 -> 233 bytes .../testdata/image/1x1-3f-animated-no-def.png | Bin 0 -> 271 bytes tests/unit/image_bitmap_test.ts | 48 +++++++-- 4 files changed, 81 insertions(+), 68 deletions(-) create mode 100644 tests/testdata/image/1x1-2f-animated-has-def.png create mode 100644 tests/testdata/image/1x1-3f-animated-no-def.png diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index c55b2a5dac215f..023900e6fa31b4 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -19,7 +19,7 @@ use image::ColorType; use image::DynamicImage; use image::GenericImageView; use image::ImageBuffer; -use image::ImageDecoder; +use image::ImageError; use image::LumaA; use image::Pixel; use image::Primitive; @@ -338,15 +338,35 @@ struct ImageProcessResult { height: u32, } +// +// About the animated image +// > Blob .4 +// > ... If this is an animated image, imageBitmap's bitmap data must only be taken from +// > the default image of the animation (the one that the format defines is to be used when animation is +// > not supported or is disabled), or, if there is no such image, the first frame of the animation. +// https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html +// +// see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) +// https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 +// + trait ImageDecoderFromReader<'a, R: BufRead + Seek> { fn to_decoder(reader: R) -> Result where Self: Sized; + fn to_intermediate_image(self) -> Result; } type ImageDecoderFromReaderType<'a> = BufReader>; -macro_rules! impl_image_decoder_converter { +fn image_decoding_error(error: ImageError) -> DOMExceptionInvalidStateError { + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &error.to_string(), + )) +} + +macro_rules! impl_image_decoder_from_reader { ($decoder:ty, $reader:ty) => { impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { fn to_decoder(reader: R) -> Result @@ -355,42 +375,26 @@ macro_rules! impl_image_decoder_converter { { match <$decoder>::new(reader) { Ok(decoder) => Ok(decoder), - Err(err) => { - return Err( - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &err.to_string(), - )) - .into(), - ) - } + Err(err) => return Err(image_decoding_error(err).into()), + } + } + fn to_intermediate_image(self) -> Result { + match DynamicImage::from_decoder(self) { + Ok(image) => Ok(image), + Err(err) => Err(image_decoding_error(err).into()), } } } }; } -impl_image_decoder_converter!(PngDecoder, ImageDecoderFromReaderType); -impl_image_decoder_converter!(JpegDecoder, ImageDecoderFromReaderType); -impl_image_decoder_converter!(GifDecoder, ImageDecoderFromReaderType); -impl_image_decoder_converter!(BmpDecoder, ImageDecoderFromReaderType); -impl_image_decoder_converter!(IcoDecoder, ImageDecoderFromReaderType); -impl_image_decoder_converter!(WebPDecoder, ImageDecoderFromReaderType); - -fn decoder_to_intermediate_image( - decoder: impl ImageDecoder, -) -> Result { - match DynamicImage::from_decoder(decoder) { - Ok(image) => Ok(image), - Err(err) => Err( - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &err.to_string(), - )) - .into(), - ), - } -} +// If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. +impl_image_decoder_from_reader!(PngDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); fn decode_bitmap_data( buf: &[u8], @@ -402,39 +406,20 @@ fn decode_bitmap_data( let (view, width, height) = match image_bitmap_source { ImageBitmapSource::Blob => { let image = match &*mime_type { - // - // TODO: support animated images - // It's a little hard to implement animated images along spec because of the complexity. - // - // > If this is an animated image, imageBitmap's bitmap data must only be taken from - // > the default image of the animation (the one that the format defines is to be used when animation is - // > not supported or is disabled), or, if there is no such image, the first frame of the animation. - // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html - // - // see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) - // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 - // + // Should we support the "image/apng" MIME type here? "image/png" => { let decoder: PngDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - if decoder.is_apng()? { - return Err( - DOMExceptionInvalidStateError::new( - "Animation image is not supported.", - ) - .into(), - ); - } - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "image/jpeg" => { let decoder: JpegDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "image/gif" => { let decoder: GifDecoder = @@ -453,21 +438,21 @@ fn decode_bitmap_data( ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "image/bmp" => { let decoder: BmpDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "image/x-icon" => { let decoder: IcoDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "image/webp" => { let decoder: WebPDecoder = @@ -482,7 +467,7 @@ fn decode_bitmap_data( .into(), ); } - decoder_to_intermediate_image(decoder)? + decoder.to_intermediate_image()? } "" => { return Err( diff --git a/tests/testdata/image/1x1-2f-animated-has-def.png b/tests/testdata/image/1x1-2f-animated-has-def.png new file mode 100644 index 0000000000000000000000000000000000000000..d460137ce83541cf6b400dc85dcccf3ef7fa89eb GIT binary patch literal 233 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1|;Q0k8}blj>O~;A0W*P#0=-&?m7UZ1Uy|F zLn;^O~;A0W*P#0=-&?m7UZq|zW_ zAR({@5CaSt8RQrk90ROtfII createImageBitmap(imageData), DOMException); - } { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.ico`)], @@ -250,3 +240,41 @@ Deno.test(async function imageBitmapFromBlob() { await assertRejects(() => createImageBitmap(imageData), DOMException); } }); + +Deno.test(async function imageBitmapFromBlobAnimatedImage() { + { + // the chunk of animated apng is below (2 frames, 1x1, 8-bit, RGBA), has default [255, 0, 0, 255] image + // [ 0, 255, 0, 255, + // 0, 0, 255, 255 ] + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-2f-animated-has-def.png`), + ], { type: "image/png" }); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); + } + { + // the chunk of animated apng is below (3 frames, 1x1, 8-bit, RGBA), no default image + // [ 255, 0, 0, 255, + // 0, 255, 0, 255, + // 0, 0, 255, 255 ] + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-3f-animated-no-def.png`), + ], { type: "image/png" }); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); + } + { + // the chunk of animation webp is below (3 frames, 1x1, 8-bit, RGBA) + // [ 255, 0, 0, 127, + // 0, 255, 0, 127, + // 0, 0, 255, 127 ] + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`), + ], { type: "image/webp" }); + await assertRejects(() => createImageBitmap(imageData), DOMException); + } +}); From d255518f971b3ffa295c916937151bc870bea749 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:10:17 +0900 Subject: [PATCH 09/52] support animated WebP --- ext/canvas/lib.rs | 9 +-- ...3f-lossless-animated-semi-transparent.webp | Bin 0 -> 188 bytes tests/testdata/image/1x1-animation-rgba8.webp | Bin 188 -> 0 bytes tests/unit/image_bitmap_test.ts | 75 +++++++++++++++++- 4 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp delete mode 100644 tests/testdata/image/1x1-animation-rgba8.webp diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 023900e6fa31b4..66a6c06e94ac86 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -394,6 +394,7 @@ impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); +// The WebPDecoder decodes the first frame. impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); fn decode_bitmap_data( @@ -459,14 +460,6 @@ fn decode_bitmap_data( ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - if decoder.has_animation() { - return Err( - DOMExceptionInvalidStateError::new( - "Animation image is not supported.", - ) - .into(), - ); - } decoder.to_intermediate_image()? } "" => { diff --git a/tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp b/tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp new file mode 100644 index 0000000000000000000000000000000000000000..15d584d109e573eb3768494b6b795c8e804cb4fb GIT binary patch literal 188 zcmWIYbaUInz`zjh>J$(bU=hIuWD5Z?1UUM6`mzC;|AByk!O_pxO#>zcrBfIfn1K3x i_J$(bU=hIuWD5Z?1UUM6`mzC;|AByk!O_pxO#>zcrC%^JFaq`Y j@B{h!KrFy6@VEaTtJ0$Xj6|xx@V|p3^#T|FL)8NSf{Y@{ diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 33f7d5a05843b7..118ad775adf688 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -268,13 +268,82 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } { - // the chunk of animation webp is below (3 frames, 1x1, 8-bit, RGBA) + // the chunk of animated webp is below (3 frames, 1x1, 8-bit, RGBA) + // // [ 255, 0, 0, 127, // 0, 255, 0, 127, // 0, 0, 255, 127 ] + + // the command to generate the webp file + // % img2webp -loop 0 0.png 1.png 2.png -o out.webp -o out.webp + // https://developers.google.com/speed/webp/docs/img2webp + + // deno % webpinfo tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp + // File: tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp + // RIFF HEADER: + // File size: 188 + // Chunk VP8X at offset 12, length 18 + // ICCP: 0 + // Alpha: 1 + // EXIF: 0 + // XMP: 0 + // Animation: 1 + // Canvas size 1 x 1 + // Chunk ANIM at offset 30, length 14 + // Background color:(ARGB) ff ff ff ff + // Loop count : 0 + // Chunk ANMF at offset 44, length 48 + // Offset_X: 0 + // Offset_Y: 0 + // Width: 1 + // Height: 1 + // Duration: 100 + // Dispose: 0 + // Blend: 1 + // Chunk VP8L at offset 68, length 24 + // Width: 1 + // Height: 1 + // Alpha: 1 + // Animation: 0 + // Format: Lossless (2) + // Chunk ANMF at offset 92, length 48 + // Offset_X: 0 + // Offset_Y: 0 + // Width: 1 + // Height: 1 + // Duration: 100 + // Dispose: 0 + // Blend: 1 + // Chunk VP8L at offset 116, length 24 + // Width: 1 + // Height: 1 + // Alpha: 1 + // Animation: 0 + // Format: Lossless (2) + // Chunk ANMF at offset 140, length 48 + // Offset_X: 0 + // Offset_Y: 0 + // Width: 1 + // Height: 1 + // Duration: 100 + // Dispose: 0 + // Blend: 1 + // Chunk VP8L at offset 164, length 24 + // Width: 1 + // Height: 1 + // Alpha: 1 + // Animation: 0 + // Format: Lossless (2) + // No error detected. + const imageData = new Blob([ - await Deno.readFile(`${prefix}/1x1-animation-rgba8.webp`), + await Deno.readFile( + `${prefix}/1x1-3f-lossless-animated-semi-transparent.webp`, + ), ], { type: "image/webp" }); - await assertRejects(() => createImageBitmap(imageData), DOMException); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 127])); } }); From 396c09c24d3ff066c5e819c3b270ff6ede6ea301 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:00:05 +0900 Subject: [PATCH 10/52] support animated GIF --- ext/canvas/lib.rs | 14 +------------- tests/testdata/image/1x1-3f-animated.gif | Bin 0 -> 126 bytes tests/unit/image_bitmap_test.ts | 13 +++++++++++++ 3 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 tests/testdata/image/1x1-3f-animated.gif diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 66a6c06e94ac86..0c42c7efbfaf98 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -14,7 +14,6 @@ use image::codecs::png::PngDecoder; use image::codecs::webp::WebPDecoder; use image::imageops::overlay; use image::imageops::FilterType; -use image::AnimationDecoder; use image::ColorType; use image::DynamicImage; use image::GenericImageView; @@ -391,6 +390,7 @@ macro_rules! impl_image_decoder_from_reader { // If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. impl_image_decoder_from_reader!(PngDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); +// The GifDecoder decodes the first frame. impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); @@ -423,18 +423,6 @@ fn decode_bitmap_data( decoder.to_intermediate_image()? } "image/gif" => { - let decoder: GifDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - if decoder.into_frames().count() > 1 { - return Err( - DOMExceptionInvalidStateError::new( - "Animation image is not supported.", - ) - .into(), - ); - } let decoder: GifDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, diff --git a/tests/testdata/image/1x1-3f-animated.gif b/tests/testdata/image/1x1-3f-animated.gif new file mode 100644 index 0000000000000000000000000000000000000000..08d3cbc4008a209cce25d1e76f6f49faecb7c883 GIT binary patch literal 126 zcmZ?wbhEHbWMp7uXkdT>#sA!Xt|7tBjsdPldIrplKw-t7EG$MqS_h;AtoT2O#lXbm S!iYtZ0W6D45-barv<3ipl@L|{ literal 0 HcmV?d00001 diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 118ad775adf688..fb3ecf9595c154 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -346,4 +346,17 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 127])); } + { + // the chunk of animated gif is below (3 frames, 1x1, 8-bit, RGBA) + // [ 255, 0, 0, 255, + // 0, 255, 0, 255, + // 0, 0, 255, 255 ] + const imageData = new Blob([ + await Deno.readFile(`${prefix}/1x1-3f-animated.gif`), + ], { type: "image/gif" }); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); + } }); From 7d3e39d3af8aa741f9d9c4335959e3a663ee72c2 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:34:30 +0900 Subject: [PATCH 11/52] support `colorspaceConversion` for `ImageData` --- ext/canvas/01_image.js | 5 +- ext/canvas/lib.rs | 213 ++++++++++++++++++++++++++++++-- ext/web/16_image_data.js | 11 +- tests/unit/image_bitmap_test.ts | 23 ++++ 4 files changed, 239 insertions(+), 13 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 41b5b948b915b0..cbc4a6f889fd4d 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -19,6 +19,7 @@ const { ArrayPrototypeJoin, } = primordials; import { + _colorSpace, _data, _height, _width, @@ -227,7 +228,7 @@ function createImageBitmap( let width = 0; let height = 0; let mimeType = ""; - let imageBitmapSource, buf; + let imageBitmapSource, buf, predefinedColorSpace; if (isBlob) { imageBitmapSource = imageBitmapSources[0]; buf = new Uint8Array(await image.arrayBuffer()); @@ -238,6 +239,7 @@ function createImageBitmap( height = image[_height]; imageBitmapSource = imageBitmapSources[1]; buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); + predefinedColorSpace = image[_colorSpace]; } let sx; @@ -256,6 +258,7 @@ function createImageBitmap( sh, imageOrientation: options.imageOrientation ?? "from-image", premultiplyAlpha: options.premultiplyAlpha ?? "default", + predefinedColorSpace: predefinedColorSpace ?? "srgb", colorSpaceConversion: options.colorSpaceConversion ?? "default", resizeWidth: options.resizeWidth, resizeHeight: options.resizeHeight, diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 0c42c7efbfaf98..f05f8cbdef953d 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -22,6 +22,7 @@ use image::ImageError; use image::LumaA; use image::Pixel; use image::Primitive; +use image::Rgb; use image::Rgba; use image::RgbaImage; use num_traits::NumCast; @@ -271,6 +272,174 @@ fn apply_unpremultiply_alpha( } } +// reference +// https://www.w3.org/TR/css-color-4/#color-conversion-code +fn srgb_to_linear(value: T) -> f32 { + if value.to_f32().unwrap() <= 0.04045 { + value.to_f32().unwrap() / 12.92 + } else { + ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) + } +} + +// reference +// https://www.w3.org/TR/css-color-4/#color-conversion-code +fn linear_to_display_p3(value: T) -> f32 { + if value.to_f32().unwrap() <= 0.0031308 { + value.to_f32().unwrap() * 12.92 + } else { + 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 + } +} + +fn normalize_value_to_0_1(value: T) -> f32 { + value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() +} + +fn unnormalize_value_from_0_1(value: f32) -> T { + NumCast::from( + (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), + ) + .unwrap() +} + +fn srgb_to_display_p3(r: T, g: T, b: T) -> (T, T, T) { + // normalize the value to 0.0 - 1.0 + let (r, g, b) = ( + normalize_value_to_0_1(r), + normalize_value_to_0_1(g), + normalize_value_to_0_1(b), + ); + + // sRGB -> Linear RGB + let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); + + // Display-P3 (RGB) -> Display-P3 (XYZ) + // + // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] + // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html + // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html + + // [ sRGB (D65) to XYZ ] + #[rustfmt::skip] + let (m1x, m1y, m1z) = ( + [0.4124564, 0.3575761, 0.1804375], + [0.2126729, 0.7151522, 0.0721750], + [0.0193339, 0.1191920, 0.9503041], + ); + + let (r, g, b) = ( + r * m1x[0] + g * m1x[1] + b * m1x[2], + r * m1y[0] + g * m1y[1] + b * m1y[2], + r * m1z[0] + g * m1z[1] + b * m1z[2], + ); + + // inv[ P3-D65 (D65) to XYZ ] + #[rustfmt::skip] + let (m2x, m2y, m2z) = ( + [ 2.493496911941425, -0.9313836179191239, -0.40271078445071684 ], + [ -0.8294889695615747, 1.7626640603183463, 0.023624685841943577 ], + [ 0.03584583024378447,-0.07617238926804182, 0.9568845240076872 ], + ); + + let (r, g, b) = ( + r * m2x[0] + g * m2x[1] + b * m2x[2], + r * m2y[0] + g * m2y[1] + b * m2y[2], + r * m2z[0] + g * m2z[1] + b * m2z[2], + ); + + // This calculation is similar as above that it is a little faster, but less accurate. + // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; + // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; + // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; + + // Display-P3 (Linear) -> Display-P3 + let (r, g, b) = ( + linear_to_display_p3(r), + linear_to_display_p3(g), + linear_to_display_p3(b), + ); + + // unnormalize the value from 0.0 - 1.0 + ( + unnormalize_value_from_0_1(r), + unnormalize_value_from_0_1(g), + unnormalize_value_from_0_1(b), + ) +} + +trait ProcessColorSpaceConversion { + /// Display P3 Color Encoding (v 1.0) + /// https://www.color.org/chardata/rgb/DisplayP3.xalter + fn process_srgb_to_display_p3(&self) -> Self; +} + +impl ProcessColorSpaceConversion for Rgb { + fn process_srgb_to_display_p3(&self) -> Self { + let (r, g, b) = (self.0[0], self.0[1], self.0[2]); + + let (r, g, b) = srgb_to_display_p3(r, g, b); + + Rgb::([r, g, b]) + } +} + +impl ProcessColorSpaceConversion for Rgba { + fn process_srgb_to_display_p3(&self) -> Self { + let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); + + let (r, g, b) = srgb_to_display_p3(r, g, b); + + Rgba::([r, g, b, a]) + } +} + +fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + ProcessColorSpaceConversion + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + for (x, y, pixel) in image.pixels() { + let pixel = pixel.process_srgb_to_display_p3(); + + out.put_pixel(x, y, pixel); + } + + out +} + +fn apply_srgb_to_display_p3( + image: &DynamicImage, +) -> Result { + match image.color() { + // The conversion of the lumincance color types to the display-p3 color space is meaningless. + ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), + ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), + ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), + ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( + &image.to_rgb8(), + ))), + ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( + process_srgb_to_display_p3(&image.to_rgb16()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_srgb_to_display_p3(&image.to_rgba8()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_srgb_to_display_p3(&image.to_rgba16()), + )), + _ => Err(type_error(image_error_message( + "apply colorspace: display-p3", + "The color type is not supported.", + ))), + } +} + #[derive(Debug, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] enum ImageResizeQuality { @@ -295,6 +464,15 @@ enum PremultiplyAlpha { None, } +// https://github.com/gfx-rs/wgpu/blob/04618b36a89721c23dc46f5844c71c0e10fc7844/wgpu-types/src/lib.rs#L6948C10-L6948C30 +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum PredefinedColorSpace { + Srgb, + #[serde(rename = "display-p3")] + DisplayP3, +} + #[derive(Debug, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] enum ColorSpaceConversion { @@ -321,6 +499,7 @@ struct ImageProcessArgs { sh: Option, image_orientation: ImageOrientation, premultiply_alpha: PremultiplyAlpha, + predefined_color_space: PredefinedColorSpace, color_space_conversion: ColorSpaceConversion, resize_width: Option, resize_height: Option, @@ -521,6 +700,7 @@ fn op_image_process( sy, image_orientation, premultiply_alpha, + predefined_color_space, color_space_conversion, resize_width, resize_height, @@ -536,6 +716,7 @@ fn op_image_process( sh: args.sh, image_orientation: args.image_orientation, premultiply_alpha: args.premultiply_alpha, + predefined_color_space: args.predefined_color_space, color_space_conversion: args.color_space_conversion, resize_width: args.resize_width, resize_height: args.resize_height, @@ -596,12 +777,6 @@ fn op_image_process( surface_height }; - if color_space_conversion == ColorSpaceConversion::None { - return Err(type_error( - "options.colorSpaceConversion 'none' is not supported", - )); - } - let color = view.color(); let surface = if !(width == surface_width @@ -643,7 +818,31 @@ fn op_image_process( image_out }; - // ignore 9. + // 9. TODO: Implement color space conversion. + // Currently, the behavior of the color space conversion is always 'none' due to + // the decoder always returning the sRGB bitmap data. + // We need to apply ICC color profiles within the image from Blob, + // or the parameter of 'settings.colorSpace' from ImageData. + // https://github.com/whatwg/html/issues/10578 + // https://github.com/whatwg/html/issues/10577 + let image_out = match color_space_conversion { + ColorSpaceConversion::Default => { + match image_bitmap_source { + ImageBitmapSource::Blob => { + // If there is no color profile information, it will use sRGB. + image_out + } + ImageBitmapSource::ImageData => match predefined_color_space { + // If the color space is sRGB, return the image as is. + PredefinedColorSpace::Srgb => image_out, + PredefinedColorSpace::DisplayP3 => { + apply_srgb_to_display_p3(&image_out)? + } + }, + } + } + ColorSpaceConversion::None => image_out, + }; // 10. if color.has_alpha() { diff --git a/ext/web/16_image_data.js b/ext/web/16_image_data.js index 2048f002d5923e..1926aeffdc3eb9 100644 --- a/ext/web/16_image_data.js +++ b/ext/web/16_image_data.js @@ -31,6 +31,7 @@ webidl.converters["ImageDataSettings"] = webidl.createDictionaryConverter( const _data = Symbol("[[data]]"); const _width = Symbol("[[width]]"); const _height = Symbol("[[height]]"); +const _colorSpace = Symbol("[[colorSpace]]"); class ImageData { /** @type {number} */ [_width]; @@ -39,7 +40,7 @@ class ImageData { /** @type {Uint8Array} */ [_data]; /** @type {'srgb' | 'display-p3'} */ - #colorSpace; + [_colorSpace]; constructor(arg0, arg1, arg2 = undefined, arg3 = undefined) { webidl.requiredArguments( @@ -133,7 +134,7 @@ class ImageData { this[_height] = sourceHeight; } - this.#colorSpace = settings.colorSpace ?? "srgb"; + this[_colorSpace] = settings.colorSpace ?? "srgb"; this[_width] = sourceWidth; this[_data] = data; return; @@ -171,7 +172,7 @@ class ImageData { ); } - this.#colorSpace = settings.colorSpace ?? "srgb"; + this[_colorSpace] = settings.colorSpace ?? "srgb"; this[_width] = sourceWidth; this[_height] = sourceHeight; this[_data] = new Uint8ClampedArray(sourceWidth * sourceHeight * 4); @@ -194,7 +195,7 @@ class ImageData { get colorSpace() { webidl.assertBranded(this, ImageDataPrototype); - return this.#colorSpace; + return this[_colorSpace]; } [SymbolFor("Deno.privateCustomInspect")](inspect, inspectOptions) { @@ -216,4 +217,4 @@ class ImageData { const ImageDataPrototype = ImageData.prototype; -export { _data, _height, _width, ImageData, ImageDataPrototype }; +export { _colorSpace, _data, _height, _width, ImageData, ImageDataPrototype }; diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index fb3ecf9595c154..bea5cff1e036f9 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -171,6 +171,7 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), + // deno-fmt-ignore new Uint8Array( [ 255, 255, // R @@ -360,3 +361,25 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); } }); + +Deno.test(async function imageBitmapImageDataColorspaceConversion() { + { + const imageData = new ImageData( + new Uint8ClampedArray([ + 255, + 0, + 0, + 255, + ]), + 1, + 1, + { + colorSpace: "display-p3", + }, + ); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([234, 51, 35, 255])); + } +}); From a6604c1719e0c99abdf1e5f6f628b7713b0e5c3b Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 5 Sep 2024 18:08:39 +0900 Subject: [PATCH 12/52] support `colorspaceConversion` for `Blob` --- Cargo.lock | 35 +- ext/canvas/Cargo.toml | 5 + ext/canvas/lib.rs | 390 ++++++++++++++++---- tests/testdata/image/wide-gamut-pattern.png | Bin 0 -> 5320 bytes tests/unit/image_bitmap_test.ts | 41 ++ 5 files changed, 392 insertions(+), 79 deletions(-) create mode 100644 tests/testdata/image/wide-gamut-pattern.png diff --git a/Cargo.lock b/Cargo.lock index 8955a075ec6b4b..14405f3862e63b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,9 +588,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2" [[package]] name = "byteorder" @@ -1372,10 +1372,12 @@ dependencies = [ name = "deno_canvas" version = "0.35.0" dependencies = [ + "bytemuck", "deno_core", "deno_terminal 0.2.0", "deno_webgpu", "image", + "lcms2", "num-traits", "serde", ] @@ -2595,6 +2597,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.17" @@ -4119,6 +4127,29 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lcms2" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680ec3fa42c36e0af9ca02f20a3742a82229c7f1ee0e6754294de46a80be6f74" +dependencies = [ + "bytemuck", + "foreign-types", + "lcms2-sys", +] + +[[package]] +name = "lcms2-sys" +version = "4.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "593265f9a3172180024fb62580ee31348f31be924b19416da174ebb7fb623d2e" +dependencies = [ + "cc", + "dunce", + "libc", + "pkg-config", +] + [[package]] name = "lexical-core" version = "0.8.5" diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 3ae6b9d7cde7ea..1bf7323d7b3f4e 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -14,9 +14,14 @@ description = "OffscreenCanvas implementation for Deno" path = "lib.rs" [dependencies] +bytemuck = "1.17.1" deno_core.workspace = true deno_terminal.workspace = true deno_webgpu.workspace = true image = { version = "0.25.2", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } +# NOTE: The qcms is a color space conversion crate which parses ICC profiles that used in Gecko, +# however it supports only 8-bit color depth currently. +# https://searchfox.org/mozilla-central/rev/f09e3f9603a08b5b51bf504846091579bc2ff531/gfx/qcms/src/transform.rs#130-137 +lcms2 = "6.1.0" num-traits = { version = "0.2.19" } serde = { workspace = true, features = ["derive"] } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index f05f8cbdef953d..80898d4788fdbf 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use bytemuck::cast_slice; use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::op2; @@ -18,13 +19,19 @@ use image::ColorType; use image::DynamicImage; use image::GenericImageView; use image::ImageBuffer; +use image::ImageDecoder; use image::ImageError; +use image::Luma; use image::LumaA; use image::Pixel; use image::Primitive; use image::Rgb; use image::Rgba; use image::RgbaImage; +use lcms2::PixelFormat; +use lcms2::Pod; +use lcms2::Profile; +use lcms2::Transform; use num_traits::NumCast; use num_traits::SaturatingMul; use serde::Deserialize; @@ -322,11 +329,11 @@ fn srgb_to_display_p3(r: T, g: T, b: T) -> (T, T, T) { // [ sRGB (D65) to XYZ ] #[rustfmt::skip] - let (m1x, m1y, m1z) = ( - [0.4124564, 0.3575761, 0.1804375], - [0.2126729, 0.7151522, 0.0721750], - [0.0193339, 0.1191920, 0.9503041], - ); + let (m1x, m1y, m1z) = ( + [0.4124564, 0.3575761, 0.1804375], + [0.2126729, 0.7151522, 0.0721750], + [0.0193339, 0.1191920, 0.9503041], + ); let (r, g, b) = ( r * m1x[0] + g * m1x[1] + b * m1x[2], @@ -336,11 +343,11 @@ fn srgb_to_display_p3(r: T, g: T, b: T) -> (T, T, T) { // inv[ P3-D65 (D65) to XYZ ] #[rustfmt::skip] - let (m2x, m2y, m2z) = ( - [ 2.493496911941425, -0.9313836179191239, -0.40271078445071684 ], - [ -0.8294889695615747, 1.7626640603183463, 0.023624685841943577 ], - [ 0.03584583024378447,-0.07617238926804182, 0.9568845240076872 ], - ); + let (m2x, m2y, m2z) = ( + [ 2.493496911941425, -0.9313836179191239, -0.40271078445071684 ], + [ -0.8294889695615747, 1.7626640603183463, 0.023624685841943577 ], + [ 0.03584583024378447,-0.07617238926804182, 0.9568845240076872 ], + ); let (r, g, b) = ( r * m2x[0] + g * m2x[1] + b * m2x[2], @@ -412,31 +419,262 @@ where out } -fn apply_srgb_to_display_p3( +trait SliceToPixel { + fn slice_to_pixel(pixel: &[u8]) -> Self; +} + +impl SliceToPixel for Luma { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0]]; + + Luma::(pixel) + } +} + +impl SliceToPixel for LumaA { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1]]; + + LumaA::(pixel) + } +} + +impl SliceToPixel for Rgb { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1], pixel[2]]; + + Rgb::(pixel) + } +} + +impl SliceToPixel for Rgba { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1], pixel[2], pixel[3]]; + + Rgba::(pixel) + } +} + +/// Convert the pixel slice to an array to avoid the copy to Vec. +/// I implemented this trait because of I couldn't find a way to effectively combine +/// the `Transform` struct of `lcms2` and `Pixel` trait of `image`. +/// If there is an implementation that is safer and can withstand changes, I would like to adopt it. +trait SliceToArray { + fn slice_to_array(pixel: &[u8]) -> [u8; N]; +} + +macro_rules! impl_slice_to_array { + ($type:ty, $n:expr) => { + impl SliceToArray<$n> for $type { + fn slice_to_array(pixel: &[u8]) -> [u8; $n] { + let mut dst = [0_u8; $n]; + dst.copy_from_slice(&pixel[..$n]); + + dst + } + } + }; +} + +impl_slice_to_array!(Luma, 1); +impl_slice_to_array!(Luma, 2); +impl_slice_to_array!(LumaA, 2); +impl_slice_to_array!(LumaA, 4); +impl_slice_to_array!(Rgb, 3); +impl_slice_to_array!(Rgb, 6); +impl_slice_to_array!(Rgba, 4); +impl_slice_to_array!(Rgba, 8); + +fn process_color_space_from_icc_profile_to_srgb( image: &DynamicImage, + icc_profile: Profile, +) -> ImageBuffer> +where + P: Pixel + SliceToPixel + SliceToArray + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + let chunk_size = image.color().bytes_per_pixel() as usize; + let pixel_iter = image + .as_bytes() + .chunks_exact(chunk_size) + .zip(image.pixels()); + let pixel_format = match image.color() { + ColorType::L8 => PixelFormat::GRAY_8, + ColorType::L16 => PixelFormat::GRAY_16, + ColorType::La8 => PixelFormat::GRAYA_8, + ColorType::La16 => PixelFormat::GRAYA_16, + ColorType::Rgb8 => PixelFormat::RGB_8, + ColorType::Rgb16 => PixelFormat::RGB_16, + ColorType::Rgba8 => PixelFormat::RGBA_8, + ColorType::Rgba16 => PixelFormat::RGBA_16, + // This arm usually doesn't reach, but it should be handled with returning the original image. + _ => { + return { + for (pixel, (x, y, _)) in pixel_iter { + out.put_pixel(x, y, P::slice_to_pixel(&pixel)); + } + out + } + } + }; + let srgb_icc_profile = Profile::new_srgb(); + let transformer = Transform::new( + &icc_profile, + pixel_format, + &srgb_icc_profile, + pixel_format, + srgb_icc_profile.header_rendering_intent(), + ); + + for (pixel, (x, y, _)) in pixel_iter { + let pixel = match transformer { + Ok(ref transformer) => { + let mut dst = P::slice_to_array(pixel); + transformer.transform_in_place(&mut dst); + + dst + } + // This arm will reach when the ffi call fails. + Err(_) => P::slice_to_array(pixel), + }; + + out.put_pixel(x, y, P::slice_to_pixel(&pixel)); + } + + out +} + +/// According to the spec, it's not clear how to handle the color space conversion. +/// +/// Therefore, if you interpret the specification description from the implementation and wpt results, it will be as follows. +/// +/// Let val be the value of the colorSpaceConversion member of options, and then run these substeps: +/// 1. If val is "default", to convert to the sRGB color space. +/// 2. If val is "none", to use the decoded image data as is. +/// +/// related issue in whatwg +/// https://github.com/whatwg/html/issues/10578 +/// +/// reference in wpt +/// https://github.com/web-platform-tests/wpt/blob/d575dc75ede770df322fbc5da3112dcf81f192ec/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html#L18 +/// https://wpt.live/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html +fn apply_color_space_conversion( + image: DynamicImage, + icc_profile: Option>, + image_bitmap_source: &ImageBitmapSource, + color_space_conversion: &ColorSpaceConversion, + predefined_color_space: &PredefinedColorSpace, ) -> Result { - match image.color() { - // The conversion of the lumincance color types to the display-p3 color space is meaningless. - ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), - ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), - ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), - ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( - &image.to_rgb8(), - ))), - ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( - process_srgb_to_display_p3(&image.to_rgb16()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_srgb_to_display_p3(&image.to_rgba8()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_srgb_to_display_p3(&image.to_rgba16()), - )), - _ => Err(type_error(image_error_message( - "apply colorspace: display-p3", - "The color type is not supported.", - ))), + match color_space_conversion { + // return the decoded image as is. + ColorSpaceConversion::None => Ok(image), + ColorSpaceConversion::Default => { + match image_bitmap_source { + ImageBitmapSource::Blob => match icc_profile { + // If there is no color profile information, return the image as is. + None => Ok(image), + Some(icc_profile) => match Profile::new_icc(&icc_profile) { + // If the color profile information is invalid, return the image as is. + Err(_) => Ok(image), + Ok(icc_profile) => match image.color() { + ColorType::L8 => Ok(DynamicImage::ImageLuma8( + process_color_space_from_icc_profile_to_srgb::<_, _, 1>( + &image, + icc_profile, + ), + )), + ColorType::L16 => Ok(DynamicImage::ImageLuma16( + process_color_space_from_icc_profile_to_srgb::<_, _, 2>( + &image, + icc_profile, + ), + )), + ColorType::La8 => Ok(DynamicImage::ImageLumaA8( + process_color_space_from_icc_profile_to_srgb::<_, _, 2>( + &image, + icc_profile, + ), + )), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16( + process_color_space_from_icc_profile_to_srgb::<_, _, 4>( + &image, + icc_profile, + ), + )), + ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8( + process_color_space_from_icc_profile_to_srgb::<_, _, 3>( + &image, + icc_profile, + ), + )), + ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( + process_color_space_from_icc_profile_to_srgb::<_, _, 6>( + &image, + icc_profile, + ), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_color_space_from_icc_profile_to_srgb::<_, _, 4>( + &image, + icc_profile, + ), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_color_space_from_icc_profile_to_srgb::<_, _, 8>( + &image, + icc_profile, + ), + )), + _ => Err(type_error(image_error_message( + "apply colorspaceConversion: default", + "The color type is not supported.", + ))), + }, + }, + }, + ImageBitmapSource::ImageData => match predefined_color_space { + // If the color space is sRGB, return the image as is. + PredefinedColorSpace::Srgb => Ok(image), + PredefinedColorSpace::DisplayP3 => { + match image.color() { + // The conversion of the lumincance color types to the display-p3 color space is meaningless. + ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), + ColorType::L16 => { + Ok(DynamicImage::ImageLuma16(image.to_luma16())) + } + ColorType::La8 => { + Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())) + } + ColorType::La16 => { + Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())) + } + ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8( + process_srgb_to_display_p3(&image.to_rgb8()), + )), + ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( + process_srgb_to_display_p3(&image.to_rgb16()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_srgb_to_display_p3(&image.to_rgba8()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_srgb_to_display_p3(&image.to_rgba16()), + )), + _ => Err(type_error(image_error_message( + "apply colorspace: display-p3", + "The color type is not supported.", + ))), + } + } + }, + } + } } } @@ -533,6 +771,7 @@ trait ImageDecoderFromReader<'a, R: BufRead + Seek> { where Self: Sized; fn to_intermediate_image(self) -> Result; + fn get_icc_profile(&mut self) -> Option>; } type ImageDecoderFromReaderType<'a> = BufReader>; @@ -562,6 +801,12 @@ macro_rules! impl_image_decoder_from_reader { Err(err) => Err(image_decoding_error(err).into()), } } + fn get_icc_profile(&mut self) -> Option> { + match self.icc_profile() { + Ok(profile) => profile, + Err(_) => None, + } + } } }; } @@ -576,58 +821,66 @@ impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); // The WebPDecoder decodes the first frame. impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); +type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); + fn decode_bitmap_data( buf: &[u8], width: u32, height: u32, image_bitmap_source: &ImageBitmapSource, mime_type: String, -) -> Result<(DynamicImage, u32, u32), AnyError> { - let (view, width, height) = match image_bitmap_source { +) -> Result { + let (view, width, height, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { - let image = match &*mime_type { + let (image, icc_profile) = match &*mime_type { // Should we support the "image/apng" MIME type here? "image/png" => { - let decoder: PngDecoder = + let mut decoder: PngDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "image/jpeg" => { - let decoder: JpegDecoder = + let mut decoder: JpegDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "image/gif" => { - let decoder: GifDecoder = + let mut decoder: GifDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "image/bmp" => { - let decoder: BmpDecoder = + let mut decoder: BmpDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "image/x-icon" => { - let decoder: IcoDecoder = + let mut decoder: IcoDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "image/webp" => { - let decoder: WebPDecoder = + let mut decoder: WebPDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, )))?; - decoder.to_intermediate_image()? + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) } "" => { return Err( @@ -660,7 +913,7 @@ See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n" let width = image.width(); let height = image.height(); - (image, width, height) + (image, width, height, icc_profile) } ImageBitmapSource::ImageData => { // > 4.12.5.1.15 Pixel manipulation @@ -677,11 +930,11 @@ See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n" } }; - (image, width, height) + (image, width, height, None) } }; - Ok((view, width, height)) + Ok((view, width, height, icc_profile)) } #[op2] @@ -725,7 +978,7 @@ fn op_image_process( mime_type: args.mime_type, }; - let (view, width, height) = + let (view, width, height, icc_profile) = decode_bitmap_data(buf, width, height, &image_bitmap_source, mime_type)?; #[rustfmt::skip] @@ -818,31 +1071,14 @@ fn op_image_process( image_out }; - // 9. TODO: Implement color space conversion. - // Currently, the behavior of the color space conversion is always 'none' due to - // the decoder always returning the sRGB bitmap data. - // We need to apply ICC color profiles within the image from Blob, - // or the parameter of 'settings.colorSpace' from ImageData. - // https://github.com/whatwg/html/issues/10578 - // https://github.com/whatwg/html/issues/10577 - let image_out = match color_space_conversion { - ColorSpaceConversion::Default => { - match image_bitmap_source { - ImageBitmapSource::Blob => { - // If there is no color profile information, it will use sRGB. - image_out - } - ImageBitmapSource::ImageData => match predefined_color_space { - // If the color space is sRGB, return the image as is. - PredefinedColorSpace::Srgb => image_out, - PredefinedColorSpace::DisplayP3 => { - apply_srgb_to_display_p3(&image_out)? - } - }, - } - } - ColorSpaceConversion::None => image_out, - }; + // 9. + let image_out = apply_color_space_conversion( + image_out, + icc_profile, + &image_bitmap_source, + &color_space_conversion, + &predefined_color_space, + )?; // 10. if color.has_alpha() { diff --git a/tests/testdata/image/wide-gamut-pattern.png b/tests/testdata/image/wide-gamut-pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..f35cd4a2e1628bf0b6cd0bb58ea1c3b0d51f0e45 GIT binary patch literal 5320 zcma)=S2P?9w1sCFb=0UqFh&_=)KN$8ov2}SB1RV_I+2LpTM#WIzYr~IbkVyQAvz%# zz4vl+U+>d>*xz1jKb^JC%Q-tnUssKkn1L7o0FY{^D;xZC>i-o5{y#20?$QMS9?hba zl=L->j18=Xg#-ny^`Ba+8f$9<08X#>vVDz==g<4;qW1Uq$*`&a=$?|EnM^?hd@3(5 zufZ*poJ^qrDoM%Aq)17Z?SNmjFF&~uU%E*UU+OscDzjm=Tt98$J??wXycXQ|9zZCU zVrrnTY76L`=jt4nH-6nUvJe5h;It~Ew$M=b6Ym&>K zhIcJ*8RkddBbf6tA6^_Byjxx0Js>>eP4-1zU*pL`3Y6x!Bs%FP1sT-SB${WD3g&=G zY2a9*dvZerBGI(45m~YrdNTm#k>9EI1i-kdL4P4Sd3b0hk1F~8j+b~6W3qkk^sR{= zlj`H{chWQg@e4tU;pEi>OB3->gjjeTdEEvH+I~Qfh#p9#o~yW{Yg!w-f`1q>LOxk$ zio{juG-iLbLLOWs8U*9ST?x07R1&&kg%g4*(Jc@Q zl+VX2T6aU(=86{oAY4wb*0WGu$=b zm({D+T>xESJ>+tND03QdX>%|^*dW4?Ky`$Iwx+9kaDjg~C`~zrJ$4;F$jmMNg$m7# zg+1oXW-_BNqnX8#y|5jg2>J;R=2*|vDL`nG^F;%(d^ zBq5@A?6|_XEN>*0_>@Sy*`dNvfnsiZggwog1oT5p+F&yGhcD^qWbE7Q7_&6V9Fh~iM=Q={pJ^>cGk{fED)GsZMVHs;HZ!|%n9ssvRsR1#H!-A&x#`+$AvecgQ> zW<_QqW@J7n-#nivA8ch}1-A-Yl3vnXB9+CF^^!%k`?ou{d$oIqI);+o=iW-*?%ghf z4nfMGuXukzWgu0MZl#cwVT<3I`FZ`T852A&P$USBXAj~9MS+qcw<3++wnj1&))T_P z_Fyn^E?5nm1tuqccvM97l=g^r@DZBQjJB7ShFX*^ffi2pazwKCa(@z_VKZj`#aqv- z!(G9;2g!phvbP04xoMv89p3SpDX?Z~359g?6ho=l4PhAQiD9OJiPf}{m{Nn8p9-0& zuJV{Fze1(1yCyT|INXDs1!B!32Mhb!Sz1^`QJ7pcQBhs2{-v~V>&vD5QQn{CM@6aW zsc8mf2E~&fg&V#_e@l4z@fiHqj0QmiqycA0WH4lC7{BL9D*NJG=bW+K#Gt=sAYu?_ zP_EBt*kZJ4ByNyoRHe^i80F%LrbjEIjhy3MaCF>ts7C2Wokum`4)9PoxB_W@W?nl- zKc_g?FgG|?RiEW^=)>-7;Uny0?knb_;Y;Ha;^Xbx;(Kwrb%r>LILqHA#aeE8Z|83H zZ_jLxGuSXhGe|I$FxD}+GNdx{JYGWZA$G;3h3v$Ig=59aL~}&?glWb7gvJp2f=fb0 zqCXKtLfj&c#Z`o85cMKDqHdy#!o|XdB49CIQAQCpF*-z}xSqI&$b+qdOVM*m$ERo^ zw-_|HJJ#`P;9LK37ENwOjB)EPmvT`TVHcVm)*XW#B1RuZAx6_&H8B=9T~|prxNDjl z%C&LNY0qcZeg4-xe3ojSca~r_YgSLvMp7N=kEE6&L9!uZC7Pu+B+ezRB#w}?$mf#2 z$UF&eDA~x{_63emn_rQ2x^gw5~V&lo`;!4}T(x&}-+B#;#+yBkY zySs!aas1$v5RQ_tb@{rpF<*M;nPOxdrvsnbrkz-#SV<3c?~E>;Nj^R`Pc8 zL7iTO@`Wgyrz=Jjw#lmc+WOuT3W6ThTAsFt>+l~ahDJM&`Xwo1O2Q*TZlY{32l+2@ z9kM$L{2%PyI6s`b#eSImaO?K&?uQbxA)tStQf&Ney--{>7ii23YiU(BPS|INqj;vkG;agjIf}{UgT2Mmg&AbsE53vEh3t5)IPt{s zSQZ^t%?UkqJ$mgPttky{4JP#;I+L2sSyN$j7nSFW=`96mN2$JPn6#*U$|aGeEME`l zNWNY^4zA~%8DhW0>S4Z;38JH-6~cQ#Qen;3*X5^cHwG8bgI5lPj-}5RhUrJx#}K0< z!xjT}gD;26#^yp;uUU5$g2bE~RQ&?aFO{~V{K&OfLc`cEx2B65n5LLUKg56dV)4#m zW+7*xTN>Ni)Y{(PlLTkUl3p638zLV*;OXQR#x3K2^Z%21FdsF2@>E7&3AJX4F^@yJSfUKURhWt(gGSS; zX9dO`=Zd>3lX2PjOYi1Mp#o;sOM$Z&vHQP9O)aaqt%BO!gOqOj&UCb(3dP>`7}RZ} zuE(<2s;H!>s3;Yp1<_bXQQy>9SD#uR*cj@scEZ16Y0c=3dN%Kb)C;n>v50a~Gw%=l zPw|D-=4#O51&!r^rI=*`OSIf&@YbFqwtZ1%9y*;idva1d-5+Z;Wfis5pjTI*D5rBC zbN>8n?`-F6hHQn@pZqjzqFHxTZapJ=fT7hWQHLlqa%&zrn+3o>U}yPfeWZJ z0vukg999(o_^#Irrd29q=`4%^os;(_7nkCj4=3gW4`}s=psq@OhXEqILL5XmnIM3( z6Gs9-w}PuEPci_&E5!E+C#uHL?F55f!;BN6P`qN?HNrFk4Df_ z<47dZr5%IuQ&s}q@%LfOMEOKZ(ALMnQo#hb2DdsmzzF#WM@4$&5bWDD<#^?NWr66i z7-QCrT!<~#5i~$8R>xWOI^KunV;kaIz)P!R&*RX4Pl7WtoVx z<+j~6lz}uyF$dS&cMfMRM9Q~Wu^Ac&zkYOWh#&HfSe0GXyDyMao0H(PqvIn-j}7L3 z)9PmmnS@TWPt%A!ZeVIq@{>8|A@d|5yD7b+zx$I4^MaFyKN z(jLX$&%GSI-o1JpeC#>wU)Ys6UX+xS9G8rhRFt49F&0TpvRvMlpXE0UrcD}j5*79v z7CyGN2G`(dcPXs4vDMmr_!FD@ZBpd9(Q~@L?lgWGpbU>P=d!=f^v>?iED#`>7dU^9}h~G(VY$yZq+t0muEKq zwhqsy9JJBF!~Q>%d)P4!<3{6k(!UW>wok3Kt&Jf@!Nehn9$U+$ zr;%6Y*J2Aj-?2Wmy-7Xe!&SotbIZ#e0^!`O>^2g~P(dC~wrDPxn3L2qRtbpze&$3) zOCznKvSO+dtx~VzK_q`z@9azT-W4&3X-UO@-H`7nzs9LE)lMi($Z~v=!IA#;tpVoubJP=a0|q?EV-% zGBt2AShn15v3i9{^-8TsD^D9gKraRb9|Xl34K1c)jmD->Uu4h@=rJ@Yw%Vud^v9{& z*0MPEK8s|U2X5%zN2_t)>(81cl@`3p+(#=;PtGMBAF@@zL#@`;bSD z+h=rx>zG@(ndzDEw&XD3a(KQAN!An7)`jff4JGj*?L9 z^-JMFajZnP$W;JwfM)PeP??jn9Yc`0pOj5=aG?7;Pt^cUhtGkIUYR~7PyU1lMuT9i zw9z>(?@_t`eVEBf%JFz_eWQ6~eB!b`b6v#?b^vdKZNZ{oaGY|SW?WYsO`M2Ula{bn zyH>B3{c!29-LU)c_u-J?P2OPM23{=h6W*Zmk#f~?uktFBndyOPqnU`Ap&8B&TY1px zJ$w^&(3s{s=6iD1bJlluMtV&8E1@u9GT}%kU1wFNe>8qHakKz+ijqQ6qKr|PWucC^ zkXKjvXLolOi*DC$dwYj_LwkcPPI7O&O7e3DsiyKP%)D-QV_l6O~q_wDXy$R+UNHCAC)VYKTMGFP-x zbY?rbI@}!z-G%;yMir+Q_Z9!O_U~X1mAuou`E}}Y_I~Br@$V~%y(X;x8R$XzPY;&; ztmMc6OTB4yI<5td_SAa%*1r`1mTUs`#S%iBt>;pGn3R`#UFuJ2;;@ zks7T787jg*Q*tKplNp-ycXmenvK7RGH(5j|F>O%#pe@KcImj@}J)ND(AE~w~gQs4n z<6lvbE2s(Ikq_rNIIB&o=2Dd8;$9d+2w_4vPIA6Vg23&Zfb=F}4FSQJ8m6Yv&g4UC zSSfK2tvt!P$cIilVDqhLVNRO-SG=AL{j&Siyqq4fcwk&^Cl1EgIHG^YU^RXOQxk0&)jh z|B{Q(?L$)Lr)~sgQeR7EUEE79{VW};(U{HtMElS}%a-u|z|rQCMIm)jnfqPT$^)Vh zk62#D!I4(g10TOn@#8ML2Ia!*_)qd+6G6dST=UZ^7N%| Date: Thu, 5 Sep 2024 18:47:37 +0900 Subject: [PATCH 13/52] clean up unit test --- tests/unit/image_bitmap_test.ts | 189 +++++++++++--------------------- 1 file changed, 63 insertions(+), 126 deletions(-) diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 16ff8b3af10400..d244431b6e471b 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -93,7 +93,7 @@ Deno.test(async function imageBitmapFlipY() { ])); }); -Deno.test(async function imageBitmapPremultiplyAlpha() { +Deno.test("imageBitmapPremultiplyAlpha", async (t) => { const imageData = new ImageData( new Uint8ClampedArray([ 255, @@ -104,7 +104,7 @@ Deno.test(async function imageBitmapPremultiplyAlpha() { 1, 1, ); - { + await t.step('"ImageData" premultiplyAlpha: "default"', async () => { const imageBitmap = await createImageBitmap(imageData, { premultiplyAlpha: "default", }); @@ -113,8 +113,8 @@ Deno.test(async function imageBitmapPremultiplyAlpha() { assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ 255, 255, 0, 153, ])); - } - { + }); + await t.step('"ImageData" premultiplyAlpha: "premultiply"', async () => { const imageBitmap = await createImageBitmap(imageData, { premultiplyAlpha: "premultiply", }); @@ -123,8 +123,8 @@ Deno.test(async function imageBitmapPremultiplyAlpha() { assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ 153, 153, 0, 153 ])); - } - { + }); + await t.step('"ImageData" premultiplyAlpha: "none"', async () => { const imageBitmap = await createImageBitmap(imageData, { premultiplyAlpha: "none", }); @@ -133,8 +133,8 @@ Deno.test(async function imageBitmapPremultiplyAlpha() { assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ 255, 255, 0, 153, ])); - } - { + }); + await t.step('"Blob" premultiplyAlpha: "none"', async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/2x2-transparent8.png`)], { type: "image/png" }, @@ -148,11 +148,11 @@ Deno.test(async function imageBitmapPremultiplyAlpha() { 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 0, 0, 127 ])); - } + }); }); -Deno.test(async function imageBitmapFromBlob() { - { +Deno.test("imageBitmapFromBlob", async (t) => { + await t.step("8-bit png", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.png`)], { type: "image/png" }, @@ -161,8 +161,8 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("16-bit png", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red16.png`)], { type: "image/png" }, @@ -181,8 +181,8 @@ Deno.test(async function imageBitmapFromBlob() { ] ) ); - } - { + }); + await t.step("8-bit jpeg", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.jpeg`)], { type: "image/jpeg" }, @@ -191,8 +191,8 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([254, 0, 0])); - } - { + }); + await t.step("8-bit bmp", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.bmp`)], { type: "image/bmp" }, @@ -201,8 +201,8 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("8-bit gif", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.gif`)], { type: "image/gif" }, @@ -211,8 +211,8 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("8-bit webp", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.webp`)], { type: "image/webp" }, @@ -221,8 +221,8 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("8-bit ico", async () => { const imageData = new Blob( [await Deno.readFile(`${prefix}/1x1-red8.ico`)], { type: "image/x-icon" }, @@ -231,20 +231,20 @@ Deno.test(async function imageBitmapFromBlob() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("flotat-32-bit exr", async () => { // image/x-exr is a known mimetype for OpenEXR // https://www.digipres.org/formats/sources/fdd/formats/#fdd000583 const imageData = new Blob([ await Deno.readFile(`${prefix}/1x1-red32f.exr`), ], { type: "image/x-exr" }); await assertRejects(() => createImageBitmap(imageData), DOMException); - } + }); }); -Deno.test(async function imageBitmapFromBlobAnimatedImage() { - { - // the chunk of animated apng is below (2 frames, 1x1, 8-bit, RGBA), has default [255, 0, 0, 255] image +Deno.test("imageBitmapFromBlobAnimatedImage", async (t) => { + await t.step("animated png has a default image", async () => { + // the chunk of animated apng is below (2 frames, 1x1, 8-bit, RGBA), default [255, 0, 0, 255] image // [ 0, 255, 0, 255, // 0, 0, 255, 255 ] const imageData = new Blob([ @@ -254,9 +254,9 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { - // the chunk of animated apng is below (3 frames, 1x1, 8-bit, RGBA), no default image + }); + await t.step("animated png does not have any default image", async () => { + // the chunk of animated apng is below (3 frames, 1x1, 8-bit, RGBA) // [ 255, 0, 0, 255, // 0, 255, 0, 255, // 0, 0, 255, 255 ] @@ -267,76 +267,13 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } - { + }); + await t.step("animated webp", async () => { // the chunk of animated webp is below (3 frames, 1x1, 8-bit, RGBA) // // [ 255, 0, 0, 127, // 0, 255, 0, 127, // 0, 0, 255, 127 ] - - // the command to generate the webp file - // % img2webp -loop 0 0.png 1.png 2.png -o out.webp -o out.webp - // https://developers.google.com/speed/webp/docs/img2webp - - // deno % webpinfo tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp - // File: tests/testdata/image/1x1-3f-lossless-animated-semi-transparent.webp - // RIFF HEADER: - // File size: 188 - // Chunk VP8X at offset 12, length 18 - // ICCP: 0 - // Alpha: 1 - // EXIF: 0 - // XMP: 0 - // Animation: 1 - // Canvas size 1 x 1 - // Chunk ANIM at offset 30, length 14 - // Background color:(ARGB) ff ff ff ff - // Loop count : 0 - // Chunk ANMF at offset 44, length 48 - // Offset_X: 0 - // Offset_Y: 0 - // Width: 1 - // Height: 1 - // Duration: 100 - // Dispose: 0 - // Blend: 1 - // Chunk VP8L at offset 68, length 24 - // Width: 1 - // Height: 1 - // Alpha: 1 - // Animation: 0 - // Format: Lossless (2) - // Chunk ANMF at offset 92, length 48 - // Offset_X: 0 - // Offset_Y: 0 - // Width: 1 - // Height: 1 - // Duration: 100 - // Dispose: 0 - // Blend: 1 - // Chunk VP8L at offset 116, length 24 - // Width: 1 - // Height: 1 - // Alpha: 1 - // Animation: 0 - // Format: Lossless (2) - // Chunk ANMF at offset 140, length 48 - // Offset_X: 0 - // Offset_Y: 0 - // Width: 1 - // Height: 1 - // Duration: 100 - // Dispose: 0 - // Blend: 1 - // Chunk VP8L at offset 164, length 24 - // Width: 1 - // Height: 1 - // Alpha: 1 - // Animation: 0 - // Format: Lossless (2) - // No error detected. - const imageData = new Blob([ await Deno.readFile( `${prefix}/1x1-3f-lossless-animated-semi-transparent.webp`, @@ -346,8 +283,8 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 127])); - } - { + }); + await t.step("animated gif", async () => { // the chunk of animated gif is below (3 frames, 1x1, 8-bit, RGBA) // [ 255, 0, 0, 255, // 0, 255, 0, 255, @@ -359,29 +296,27 @@ Deno.test(async function imageBitmapFromBlobAnimatedImage() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([255, 0, 0, 255])); - } + }); }); Deno.test(async function imageBitmapImageDataColorspaceConversion() { - { - const imageData = new ImageData( - new Uint8ClampedArray([ - 255, - 0, - 0, - 255, - ]), - 1, - 1, - { - colorSpace: "display-p3", - }, - ); - const imageBitmap = await createImageBitmap(imageData); - // @ts-ignore: Deno[Deno.internal].core allowed - // deno-fmt-ignore - assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([234, 51, 35, 255])); - } + const imageData = new ImageData( + new Uint8ClampedArray([ + 255, + 0, + 0, + 255, + ]), + 1, + 1, + { + colorSpace: "display-p3", + }, + ); + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([234, 51, 35, 255])); }); /** @@ -395,11 +330,11 @@ function extractHighBytes(array: Uint8Array): Uint8Array { return highBytes; } -Deno.test(async function imageBitmapFromBlobColorspaceConversion() { +Deno.test("imageBitmapFromBlobColorspaceConversion", async (t) => { // reference: // https://github.com/web-platform-tests/wpt/blob/d575dc75ede770df322fbc5da3112dcf81f192ec/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html#L18 // https://wpt.fyi/results/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html?label=experimental&label=master&aligned - { + await t.step('"Blob" colorSpaceConversion: "none"', async () => { const imageData = new Blob([ await Deno.readFile(`${prefix}/wide-gamut-pattern.png`), ], { type: "image/png" }); @@ -408,10 +343,11 @@ Deno.test(async function imageBitmapFromBlobColorspaceConversion() { }); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore - const firstPixel = extractHighBytes(Deno[Deno.internal].getBitmapData(imageBitmap),).slice(0, 4); + const firstPixel = extractHighBytes(Deno[Deno.internal].getBitmapData(imageBitmap)).slice(0, 4); + // picking the high bytes of the first pixel assertEquals(firstPixel, new Uint8Array([123, 0, 27, 255])); - } - { + }); + await t.step('"Blob" colorSpaceConversion: "default"', async () => { const imageData = new Blob([ await Deno.readFile(`${prefix}/wide-gamut-pattern.png`), ], { type: "image/png" }); @@ -420,7 +356,8 @@ Deno.test(async function imageBitmapFromBlobColorspaceConversion() { }); // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore - const firstPixel = extractHighBytes(Deno[Deno.internal].getBitmapData(imageBitmap),).slice(0, 4); + const firstPixel = extractHighBytes(Deno[Deno.internal].getBitmapData(imageBitmap)).slice(0, 4); + // picking the high bytes of the first pixel assertEquals(firstPixel, new Uint8Array([255, 0, 0, 255])); - } + }); }); From 1f7524a47c96defd13962563bf2420554e8639d1 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sat, 7 Sep 2024 21:57:32 +0900 Subject: [PATCH 14/52] clean up --- ext/canvas/01_image.js | 116 +-- ext/canvas/Cargo.toml | 2 +- ext/canvas/error.rs | 14 + ext/canvas/idl.rs | 11 + ext/canvas/image_decoder.rs | 89 ++ ext/canvas/image_ops.rs | 597 ++++++++++++++ ext/canvas/lib.rs | 1132 +------------------------- ext/canvas/op_create_image_bitmap.rs | 453 +++++++++++ 8 files changed, 1229 insertions(+), 1185 deletions(-) create mode 100644 ext/canvas/idl.rs create mode 100644 ext/canvas/image_decoder.rs create mode 100644 ext/canvas/image_ops.rs create mode 100644 ext/canvas/op_create_image_bitmap.rs diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index cbc4a6f889fd4d..22ad44687dad84 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { internals, primordials } from "ext:core/mod.js"; -import { op_image_process } from "ext:core/ops"; +import { op_create_image_bitmap } from "ext:core/ops"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import { DOMException } from "ext:deno_web/01_dom_exception.js"; import { createFilteredInspectProxy } from "ext:deno_console/01_console.js"; @@ -11,7 +11,6 @@ const { ObjectPrototypeIsPrototypeOf, Symbol, SymbolFor, - TypeError, TypedArrayPrototypeGetBuffer, Uint8Array, PromiseReject, @@ -189,6 +188,7 @@ function createImageBitmap( "Argument 6", ); + // 1. if (sw === 0) { return PromiseReject(new RangeError("sw has to be greater than 0")); } @@ -198,6 +198,7 @@ function createImageBitmap( } } + // 2. if (options.resizeWidth === 0) { return PromiseReject( new DOMException( @@ -217,70 +218,69 @@ function createImageBitmap( const imageBitmap = webidl.createBranded(ImageBitmap); - // 6. Switch on image + // 3. const isBlob = ObjectPrototypeIsPrototypeOf(BlobPrototype, image); const isImageData = ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image); - if ( - isImageData || - isBlob - ) { - return (async () => { - let width = 0; - let height = 0; - let mimeType = ""; - let imageBitmapSource, buf, predefinedColorSpace; - if (isBlob) { - imageBitmapSource = imageBitmapSources[0]; - buf = new Uint8Array(await image.arrayBuffer()); - mimeType = sniffImage(image.type); - } - if (isImageData) { - width = image[_width]; - height = image[_height]; - imageBitmapSource = imageBitmapSources[1]; - buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); - predefinedColorSpace = image[_colorSpace]; - } - - let sx; - if (typeof sxOrOptions === "number") { - sx = sxOrOptions; - } - - const processedImage = op_image_process( - buf, - { - width, - height, - sx, - sy, - sw, - sh, - imageOrientation: options.imageOrientation ?? "from-image", - premultiplyAlpha: options.premultiplyAlpha ?? "default", - predefinedColorSpace: predefinedColorSpace ?? "srgb", - colorSpaceConversion: options.colorSpaceConversion ?? "default", - resizeWidth: options.resizeWidth, - resizeHeight: options.resizeHeight, - resizeQuality: options.resizeQuality ?? "low", - imageBitmapSource, - mimeType, - }, - ); - imageBitmap[_bitmapData] = processedImage.data; - imageBitmap[_width] = processedImage.width; - imageBitmap[_height] = processedImage.height; - return imageBitmap; - })(); - } else { + if (!isBlob && !isImageData) { return PromiseReject( - new TypeError( - `${prefix}: The provided value is not of type '(${ + new DOMException( + `${prefix}: The provided value for 'image' is not of type '(${ ArrayPrototypeJoin(imageBitmapSources, " or ") })'.`, + "InvalidStateError", ), ); } + + // 4. + return (async () => { + let width = 0; + let height = 0; + let mimeType = ""; + let imageBitmapSource, buf, predefinedColorSpace; + if (isBlob) { + imageBitmapSource = imageBitmapSources[0]; + buf = new Uint8Array(await image.arrayBuffer()); + mimeType = sniffImage(image.type); + } + if (isImageData) { + width = image[_width]; + height = image[_height]; + imageBitmapSource = imageBitmapSources[1]; + buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); + predefinedColorSpace = image[_colorSpace]; + } + + let sx; + if (typeof sxOrOptions === "number") { + sx = sxOrOptions; + } + // TODO(Hajime-san): this should be real async + const processedImage = op_create_image_bitmap( + buf, + { + width, + height, + sx, + sy, + sw, + sh, + imageOrientation: options.imageOrientation ?? "from-image", + premultiplyAlpha: options.premultiplyAlpha ?? "default", + predefinedColorSpace: predefinedColorSpace ?? "srgb", + colorSpaceConversion: options.colorSpaceConversion ?? "default", + resizeWidth: options.resizeWidth, + resizeHeight: options.resizeHeight, + resizeQuality: options.resizeQuality ?? "low", + imageBitmapSource, + mimeType, + }, + ); + imageBitmap[_bitmapData] = processedImage.data; + imageBitmap[_width] = processedImage.width; + imageBitmap[_height] = processedImage.height; + return imageBitmap; + })(); } function getBitmapData(imageBitmap) { diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 1bf7323d7b3f4e..52b145f2ac852a 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -19,7 +19,7 @@ deno_core.workspace = true deno_terminal.workspace = true deno_webgpu.workspace = true image = { version = "0.25.2", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } -# NOTE: The qcms is a color space conversion crate which parses ICC profiles that used in Gecko, +# NOTE: The qcms is a color space conversion crate which parses ICC profiles that used in Gecko, # however it supports only 8-bit color depth currently. # https://searchfox.org/mozilla-central/rev/f09e3f9603a08b5b51bf504846091579bc2ff531/gfx/qcms/src/transform.rs#130-137 lcms2 = "6.1.0" diff --git a/ext/canvas/error.rs b/ext/canvas/error.rs index bd08f9e8d64910..e2d160534552dc 100644 --- a/ext/canvas/error.rs +++ b/ext/canvas/error.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use deno_core::error::AnyError; +use std::borrow::Cow; use std::fmt; #[derive(Debug)] @@ -28,3 +29,16 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { e.downcast_ref::() .map(|_| "DOMExceptionInvalidStateError") } + +/// Returns a string that represents the error message for the image. +pub(crate) fn image_error_message<'a, T: Into>>( + opreation: T, + reason: T, +) -> String { + format!( + "An error has occurred while {}. +reason: {}", + opreation.into(), + reason.into(), + ) +} diff --git a/ext/canvas/idl.rs b/ext/canvas/idl.rs new file mode 100644 index 00000000000000..23f33641c9bbf0 --- /dev/null +++ b/ext/canvas/idl.rs @@ -0,0 +1,11 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use serde::Deserialize; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum PredefinedColorSpace { + Srgb, + #[serde(rename = "display-p3")] + DisplayP3, +} diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs new file mode 100644 index 00000000000000..4b9e4fa53a7d3b --- /dev/null +++ b/ext/canvas/image_decoder.rs @@ -0,0 +1,89 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::io::BufRead; +use std::io::BufReader; +use std::io::Cursor; +use std::io::Seek; + +use deno_core::error::AnyError; +use image::codecs::bmp::BmpDecoder; +use image::codecs::gif::GifDecoder; +use image::codecs::ico::IcoDecoder; +use image::codecs::jpeg::JpegDecoder; +use image::codecs::png::PngDecoder; +use image::codecs::webp::WebPDecoder; +use image::DynamicImage; +use image::ImageDecoder; +use image::ImageError; + +use crate::error::image_error_message; +use crate::error::DOMExceptionInvalidStateError; + +// +// About the animated image +// > Blob .4 +// > ... If this is an animated image, imageBitmap's bitmap data must only be taken from +// > the default image of the animation (the one that the format defines is to be used when animation is +// > not supported or is disabled), or, if there is no such image, the first frame of the animation. +// https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html +// +// see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) +// https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 +// + +pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { + fn to_decoder(reader: R) -> Result + where + Self: Sized; + fn to_intermediate_image(self) -> Result; + fn get_icc_profile(&mut self) -> Option>; +} + +pub(crate) type ImageDecoderFromReaderType<'a> = BufReader>; + +pub(crate) fn image_decoding_error( + error: ImageError, +) -> DOMExceptionInvalidStateError { + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &error.to_string(), + )) +} + +macro_rules! impl_image_decoder_from_reader { + ($decoder:ty, $reader:ty) => { + impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { + fn to_decoder(reader: R) -> Result + where + Self: Sized, + { + match <$decoder>::new(reader) { + Ok(decoder) => Ok(decoder), + Err(err) => return Err(image_decoding_error(err).into()), + } + } + fn to_intermediate_image(self) -> Result { + match DynamicImage::from_decoder(self) { + Ok(image) => Ok(image), + Err(err) => Err(image_decoding_error(err).into()), + } + } + fn get_icc_profile(&mut self) -> Option> { + match self.icc_profile() { + Ok(profile) => profile, + Err(_) => None, + } + } + } + }; +} + +// If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. +impl_image_decoder_from_reader!(PngDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); +// The GifDecoder decodes the first frame. +impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); +impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); +// The WebPDecoder decodes the first frame. +impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs new file mode 100644 index 00000000000000..bbd6fcc55cf996 --- /dev/null +++ b/ext/canvas/image_ops.rs @@ -0,0 +1,597 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use bytemuck::cast_slice; +use deno_core::error::AnyError; +use image::ColorType; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageBuffer; +use image::Luma; +use image::LumaA; +use image::Pixel; +use image::Primitive; +use image::Rgb; +use image::Rgba; +use lcms2::PixelFormat; +use lcms2::Pod; +use lcms2::Profile; +use lcms2::Transform; +use num_traits::NumCast; +use num_traits::SaturatingMul; + +pub(crate) trait PremultiplyAlpha { + fn premultiply_alpha(&self) -> Self; +} + +impl PremultiplyAlpha for LumaA { + fn premultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); + + if normalized_alpha == 0.0 { + return LumaA::([pixel[0], pixel[alpha_index]]); + } + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) + .unwrap() + } + + LumaA::([pixel[0], pixel[alpha_index]]) + } +} + +impl PremultiplyAlpha for Rgba { + fn premultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); + + if normalized_alpha == 0.0 { + return Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]); + } + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) + .unwrap() + } + + Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + } +} + +// make public if needed +fn process_premultiply_alpha(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + PremultiplyAlpha + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + for (x, y, pixel) in image.pixels() { + let pixel = pixel.premultiply_alpha(); + + out.put_pixel(x, y, pixel); + } + + out +} + +/// Premultiply the alpha channel of the image. +pub(crate) fn premultiply_alpha( + image: DynamicImage, + unmatch: Option Result>, +) -> Result { + let color = image.color(); + match color { + ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( + &image.to_luma_alpha8(), + ))), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_premultiply_alpha(&image.to_rgba8()), + )), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16( + process_premultiply_alpha(&image.to_luma_alpha16()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_premultiply_alpha(&image.to_rgba16()), + )), + x => match unmatch { + Some(unmatch) => unmatch(x), + None => Ok(image), + }, + } +} + +pub(crate) trait UnpremultiplyAlpha { + /// To determine if the image is premultiplied alpha, + /// checking premultiplied RGBA value is one where any of the R/G/B channel values exceeds the alpha channel value.\ + /// https://www.w3.org/TR/webgpu/#color-spaces + fn is_premultiplied_alpha(&self) -> bool; + fn unpremultiply_alpha(&self) -> Self; +} + +impl UnpremultiplyAlpha for Rgba { + fn is_premultiplied_alpha(&self) -> bool { + let max_t = T::DEFAULT_MAX_VALUE; + + let pixel = [self.0[0], self.0[1], self.0[2]]; + let alpha_index = self.0.len() - 1; + let alpha = self.0[alpha_index]; + + match pixel.iter().max() { + Some(rgb_max) => rgb_max < &max_t.saturating_mul(&alpha), + // usually doesn't reach here + None => false, + } + } + + fn unpremultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from( + (rgb.to_f32().unwrap() + / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) + .round(), + ) + .unwrap(); + } + + Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + } +} + +impl UnpremultiplyAlpha for LumaA { + fn is_premultiplied_alpha(&self) -> bool { + let max_t = T::DEFAULT_MAX_VALUE; + + let pixel = [self.0[0]]; + let alpha_index = self.0.len() - 1; + let alpha = self.0[alpha_index]; + + pixel[0] < max_t.saturating_mul(&alpha) + } + + fn unpremultiply_alpha(&self) -> Self { + let max_t = T::DEFAULT_MAX_VALUE; + + let mut pixel = [self.0[0], self.0[1]]; + let alpha_index = pixel.len() - 1; + let alpha = pixel[alpha_index]; + + for rgb in pixel.iter_mut().take(alpha_index) { + *rgb = NumCast::from( + (rgb.to_f32().unwrap() + / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) + .round(), + ) + .unwrap(); + } + + LumaA::([pixel[0], pixel[alpha_index]]) + } +} + +// make public if needed +fn process_unpremultiply_alpha(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + UnpremultiplyAlpha + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + let is_premultiplied_alpha = image + .pixels() + .any(|(_, _, pixel)| pixel.is_premultiplied_alpha()); + + for (x, y, pixel) in image.pixels() { + let pixel = if is_premultiplied_alpha { + pixel.unpremultiply_alpha() + } else { + // return the original + pixel + }; + + out.put_pixel(x, y, pixel); + } + + out +} + +/// Invert the premultiplied alpha channel of the image. +pub(crate) fn unpremultiply_alpha( + image: DynamicImage, + unmatch: Option Result>, +) -> Result { + match image.color() { + ColorType::La8 => Ok(DynamicImage::ImageLumaA8( + process_unpremultiply_alpha(&image.to_luma_alpha8()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_unpremultiply_alpha(&image.to_rgba8()), + )), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16( + process_unpremultiply_alpha(&image.to_luma_alpha16()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_unpremultiply_alpha(&image.to_rgba16()), + )), + x => match unmatch { + Some(unmatch) => unmatch(x), + None => Ok(image), + }, + } +} + +// reference +// https://www.w3.org/TR/css-color-4/#color-conversion-code +fn srgb_to_linear(value: T) -> f32 { + if value.to_f32().unwrap() <= 0.04045 { + value.to_f32().unwrap() / 12.92 + } else { + ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) + } +} + +// reference +// https://www.w3.org/TR/css-color-4/#color-conversion-code +fn linear_to_display_p3(value: T) -> f32 { + if value.to_f32().unwrap() <= 0.0031308 { + value.to_f32().unwrap() * 12.92 + } else { + 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 + } +} + +fn normalize_value_to_0_1(value: T) -> f32 { + value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() +} + +fn unnormalize_value_from_0_1(value: f32) -> T { + NumCast::from( + (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), + ) + .unwrap() +} + +fn apply_conversion_matrix_srgb_to_display_p3( + r: T, + g: T, + b: T, +) -> (T, T, T) { + // normalize the value to 0.0 - 1.0 + let (r, g, b) = ( + normalize_value_to_0_1(r), + normalize_value_to_0_1(g), + normalize_value_to_0_1(b), + ); + + // sRGB -> Linear RGB + let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); + + // Display-P3 (RGB) -> Display-P3 (XYZ) + // + // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] + // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html + // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html + + // [ sRGB (D65) to XYZ ] + #[rustfmt::skip] + let (m1x, m1y, m1z) = ( + [0.4124564, 0.3575761, 0.1804375], + [0.2126729, 0.7151522, 0.0721750], + [0.0193339, 0.119_192, 0.9503041], + ); + + let (r, g, b) = ( + r * m1x[0] + g * m1x[1] + b * m1x[2], + r * m1y[0] + g * m1y[1] + b * m1y[2], + r * m1z[0] + g * m1z[1] + b * m1z[2], + ); + + // inv[ P3-D65 (D65) to XYZ ] + #[rustfmt::skip] + let (m2x, m2y, m2z) = ( + [ 2.493_497, -0.931_383_6, -0.402_710_8 ], + [ -0.829_489, 1.762_664_1, 0.023_624_687 ], + [ 0.035_845_83, -0.076_172_39, 0.956_884_5 ], + ); + + let (r, g, b) = ( + r * m2x[0] + g * m2x[1] + b * m2x[2], + r * m2y[0] + g * m2y[1] + b * m2y[2], + r * m2z[0] + g * m2z[1] + b * m2z[2], + ); + + // This calculation is similar as above that it is a little faster, but less accurate. + // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; + // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; + // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; + + // Display-P3 (Linear) -> Display-P3 + let (r, g, b) = ( + linear_to_display_p3(r), + linear_to_display_p3(g), + linear_to_display_p3(b), + ); + + // unnormalize the value from 0.0 - 1.0 + ( + unnormalize_value_from_0_1(r), + unnormalize_value_from_0_1(g), + unnormalize_value_from_0_1(b), + ) +} + +pub(crate) trait ColorSpaceConversion { + /// Display P3 Color Encoding (v 1.0) + /// https://www.color.org/chardata/rgb/DisplayP3.xalter + fn srgb_to_display_p3(&self) -> Self; +} + +impl ColorSpaceConversion for Rgb { + fn srgb_to_display_p3(&self) -> Self { + let (r, g, b) = (self.0[0], self.0[1], self.0[2]); + + let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); + + Rgb::([r, g, b]) + } +} + +impl ColorSpaceConversion for Rgba { + fn srgb_to_display_p3(&self) -> Self { + let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); + + let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); + + Rgba::([r, g, b, a]) + } +} + +// make public if needed +fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> +where + I: GenericImageView, + P: Pixel + ColorSpaceConversion + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + + for (x, y, pixel) in image.pixels() { + let pixel = pixel.srgb_to_display_p3(); + + out.put_pixel(x, y, pixel); + } + + out +} + +/// Convert the color space of the image from sRGB to Display-P3. +pub(crate) fn srgb_to_display_p3( + image: DynamicImage, + unmatch: Option Result>, +) -> Result { + match image.color() { + // The conversion of the lumincance color types to the display-p3 color space is meaningless. + ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), + ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), + ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), + ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), + ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( + &image.to_rgb8(), + ))), + ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( + process_srgb_to_display_p3(&image.to_rgb16()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_srgb_to_display_p3(&image.to_rgba8()), + )), + ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( + process_srgb_to_display_p3(&image.to_rgba16()), + )), + x => match unmatch { + Some(unmatch) => unmatch(x), + None => Ok(image), + }, + } +} + +pub(crate) trait SliceToPixel { + fn slice_to_pixel(pixel: &[u8]) -> Self; +} + +impl SliceToPixel for Luma { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0]]; + + Luma::(pixel) + } +} + +impl SliceToPixel for LumaA { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1]]; + + LumaA::(pixel) + } +} + +impl SliceToPixel for Rgb { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1], pixel[2]]; + + Rgb::(pixel) + } +} + +impl SliceToPixel for Rgba { + fn slice_to_pixel(pixel: &[u8]) -> Self { + let pixel: &[T] = cast_slice(pixel); + let pixel = [pixel[0], pixel[1], pixel[2], pixel[3]]; + + Rgba::(pixel) + } +} + +/// Convert the pixel slice to an array to avoid the copy to Vec. +/// I implemented this trait because of I couldn't find a way to effectively combine +/// the `Transform` struct of `lcms2` and `Pixel` trait of `image`. +/// If there is an implementation that is safer and can withstand changes, I would like to adopt it. +pub(crate) trait SliceToArray { + fn slice_to_array(pixel: &[u8]) -> [u8; N]; +} + +macro_rules! impl_slice_to_array { + ($type:ty, $n:expr) => { + impl SliceToArray<$n> for $type { + fn slice_to_array(pixel: &[u8]) -> [u8; $n] { + let mut dst = [0_u8; $n]; + dst.copy_from_slice(&pixel[..$n]); + + dst + } + } + }; +} + +impl_slice_to_array!(Luma, 1); +impl_slice_to_array!(Luma, 2); +impl_slice_to_array!(LumaA, 2); +impl_slice_to_array!(LumaA, 4); +impl_slice_to_array!(Rgb, 3); +impl_slice_to_array!(Rgb, 6); +impl_slice_to_array!(Rgba, 4); +impl_slice_to_array!(Rgba, 8); + +// make public if needed +fn process_icc_profile_conversion( + image: &DynamicImage, + input_icc_profile: Profile, + output_icc_profile: Profile, +) -> ImageBuffer> +where + P: Pixel + SliceToPixel + SliceToArray + 'static, + S: Primitive + 'static, +{ + let (width, height) = image.dimensions(); + let mut out = ImageBuffer::new(width, height); + let chunk_size = image.color().bytes_per_pixel() as usize; + let pixel_iter = image + .as_bytes() + .chunks_exact(chunk_size) + .zip(image.pixels()); + let pixel_format = match image.color() { + ColorType::L8 => PixelFormat::GRAY_8, + ColorType::L16 => PixelFormat::GRAY_16, + ColorType::La8 => PixelFormat::GRAYA_8, + ColorType::La16 => PixelFormat::GRAYA_16, + ColorType::Rgb8 => PixelFormat::RGB_8, + ColorType::Rgb16 => PixelFormat::RGB_16, + ColorType::Rgba8 => PixelFormat::RGBA_8, + ColorType::Rgba16 => PixelFormat::RGBA_16, + // This arm usually doesn't reach, but it should be handled with returning the original image. + _ => { + return { + for (pixel, (x, y, _)) in pixel_iter { + out.put_pixel(x, y, P::slice_to_pixel(pixel)); + } + out + } + } + }; + let transformer = Transform::new( + &input_icc_profile, + pixel_format, + &output_icc_profile, + pixel_format, + output_icc_profile.header_rendering_intent(), + ); + + for (pixel, (x, y, _)) in pixel_iter { + let pixel = match transformer { + Ok(ref transformer) => { + let mut dst = P::slice_to_array(pixel); + transformer.transform_in_place(&mut dst); + + dst + } + // This arm will reach when the ffi call fails. + Err(_) => P::slice_to_array(pixel), + }; + + out.put_pixel(x, y, P::slice_to_pixel(&pixel)); + } + + out +} + +#[rustfmt::skip] +/// Convert the color space of the image from the ICC profile to sRGB. +pub(crate) fn to_srgb_from_icc_profile( + image: DynamicImage, + icc_profile: Option>, + unmatch: Option Result>, +) -> Result { + match icc_profile { + // If there is no color profile information, return the image as is. + None => Ok(image), + Some(icc_profile) => match Profile::new_icc(&icc_profile) { + // If the color profile information is invalid, return the image as is. + Err(_) => Ok(image), + Ok(icc_profile) => { + let srgb_icc_profile = Profile::new_srgb(); + match image.color() { + ColorType::L8 => { + Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion::<_,_,1>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::L16 => { + Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion::<_,_,2>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::La8 => { + Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion::<_,_,2>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::La16 => { + Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion::<_, _, 4>(&image,icc_profile,srgb_icc_profile))) + }, + ColorType::Rgb8 => { + Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion::<_,_,3>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::Rgb16 => { + Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion::<_,_,6>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::Rgba8 => { + Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion::<_,_,4>(&image,icc_profile,srgb_icc_profile))) + } + ColorType::Rgba16 => { + Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion::<_,_,8>(&image,icc_profile,srgb_icc_profile))) + } + x => match unmatch { + Some(unmatch) => unmatch(x), + None => Ok(image), + }, + } + } + }, + } +} diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 80898d4788fdbf..a5d7f32c38c293 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -1,1138 +1,18 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use bytemuck::cast_slice; -use deno_core::error::type_error; -use deno_core::error::AnyError; -use deno_core::op2; -use deno_core::JsBuffer; -use deno_core::ToJsBuffer; -use deno_terminal::colors::cyan; -use image::codecs::bmp::BmpDecoder; -use image::codecs::gif::GifDecoder; -use image::codecs::ico::IcoDecoder; -use image::codecs::jpeg::JpegDecoder; -use image::codecs::png::PngDecoder; -use image::codecs::webp::WebPDecoder; -use image::imageops::overlay; -use image::imageops::FilterType; -use image::ColorType; -use image::DynamicImage; -use image::GenericImageView; -use image::ImageBuffer; -use image::ImageDecoder; -use image::ImageError; -use image::Luma; -use image::LumaA; -use image::Pixel; -use image::Primitive; -use image::Rgb; -use image::Rgba; -use image::RgbaImage; -use lcms2::PixelFormat; -use lcms2::Pod; -use lcms2::Profile; -use lcms2::Transform; -use num_traits::NumCast; -use num_traits::SaturatingMul; -use serde::Deserialize; -use serde::Serialize; -use std::borrow::Cow; -use std::io::BufRead; -use std::io::BufReader; -use std::io::Cursor; -use std::io::Seek; use std::path::PathBuf; pub mod error; -use error::DOMExceptionInvalidStateError; - -fn to_js_buffer(image: &DynamicImage) -> ToJsBuffer { - image.as_bytes().to_vec().into() -} - -fn image_error_message<'a, T: Into>>( - opreation: T, - reason: T, -) -> String { - format!( - "An error has occurred while {}. -reason: {}", - opreation.into(), - reason.into(), - ) -} - -// reference -// https://github.com/image-rs/image/blob/6d19ffa72756c1b00e7979a90f8794a0ef847b88/src/color.rs#L739 -trait ProcessPremultiplyAlpha { - fn premultiply_alpha(&self) -> Self; -} - -impl ProcessPremultiplyAlpha for LumaA { - fn premultiply_alpha(&self) -> Self { - let max_t = T::DEFAULT_MAX_VALUE; - - let mut pixel = [self.0[0], self.0[1]]; - let alpha_index = pixel.len() - 1; - let alpha = pixel[alpha_index]; - let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); - - if normalized_alpha == 0.0 { - return LumaA::([pixel[0], pixel[alpha_index]]); - } - - for rgb in pixel.iter_mut().take(alpha_index) { - *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) - .unwrap() - } - - LumaA::([pixel[0], pixel[alpha_index]]) - } -} - -impl ProcessPremultiplyAlpha for Rgba { - fn premultiply_alpha(&self) -> Self { - let max_t = T::DEFAULT_MAX_VALUE; - - let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; - let alpha_index = pixel.len() - 1; - let alpha = pixel[alpha_index]; - let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); - - if normalized_alpha == 0.0 { - return Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]); - } - - for rgb in pixel.iter_mut().take(alpha_index) { - *rgb = NumCast::from((rgb.to_f32().unwrap() * normalized_alpha).round()) - .unwrap() - } - - Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) - } -} - -fn process_premultiply_alpha(image: &I) -> ImageBuffer> -where - I: GenericImageView, - P: Pixel + ProcessPremultiplyAlpha + 'static, - S: Primitive + 'static, -{ - let (width, height) = image.dimensions(); - let mut out = ImageBuffer::new(width, height); - - for (x, y, pixel) in image.pixels() { - let pixel = pixel.premultiply_alpha(); - - out.put_pixel(x, y, pixel); - } - - out -} - -fn apply_premultiply_alpha( - image: &DynamicImage, -) -> Result { - match image.color() { - ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( - &image.to_luma_alpha8(), - ))), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_premultiply_alpha(&image.to_rgba8()), - )), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_premultiply_alpha(&image.to_luma_alpha16()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_premultiply_alpha(&image.to_rgba16()), - )), - _ => Err(type_error(image_error_message( - "apply premultiplyAlpha: premultiply", - "The color type is not supported.", - ))), - } -} - -trait ProcessUnpremultiplyAlpha { - /// To determine if the image is premultiplied alpha, - /// checking premultiplied RGBA value is one where any of the R/G/B channel values exceeds the alpha channel value.\ - /// https://www.w3.org/TR/webgpu/#color-spaces - fn is_premultiplied_alpha(&self) -> bool; - fn unpremultiply_alpha(&self) -> Self; -} - -impl ProcessUnpremultiplyAlpha for Rgba { - fn is_premultiplied_alpha(&self) -> bool { - let max_t = T::DEFAULT_MAX_VALUE; - - let pixel = [self.0[0], self.0[1], self.0[2]]; - let alpha_index = self.0.len() - 1; - let alpha = self.0[alpha_index]; - - match pixel.iter().max() { - Some(rgb_max) => rgb_max < &max_t.saturating_mul(&alpha), - // usually doesn't reach here - None => false, - } - } - - fn unpremultiply_alpha(&self) -> Self { - let max_t = T::DEFAULT_MAX_VALUE; - - let mut pixel = [self.0[0], self.0[1], self.0[2], self.0[3]]; - let alpha_index = pixel.len() - 1; - let alpha = pixel[alpha_index]; - - for rgb in pixel.iter_mut().take(alpha_index) { - *rgb = NumCast::from( - (rgb.to_f32().unwrap() - / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) - .round(), - ) - .unwrap(); - } - - Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) - } -} - -impl ProcessUnpremultiplyAlpha - for LumaA -{ - fn is_premultiplied_alpha(&self) -> bool { - let max_t = T::DEFAULT_MAX_VALUE; - - let pixel = [self.0[0]]; - let alpha_index = self.0.len() - 1; - let alpha = self.0[alpha_index]; - - pixel[0] < max_t.saturating_mul(&alpha) - } - - fn unpremultiply_alpha(&self) -> Self { - let max_t = T::DEFAULT_MAX_VALUE; - - let mut pixel = [self.0[0], self.0[1]]; - let alpha_index = pixel.len() - 1; - let alpha = pixel[alpha_index]; - - for rgb in pixel.iter_mut().take(alpha_index) { - *rgb = NumCast::from( - (rgb.to_f32().unwrap() - / (alpha.to_f32().unwrap() / max_t.to_f32().unwrap())) - .round(), - ) - .unwrap(); - } - - LumaA::([pixel[0], pixel[alpha_index]]) - } -} - -fn process_unpremultiply_alpha(image: &I) -> ImageBuffer> -where - I: GenericImageView, - P: Pixel + ProcessUnpremultiplyAlpha + 'static, - S: Primitive + 'static, -{ - let (width, height) = image.dimensions(); - let mut out = ImageBuffer::new(width, height); - - let is_premultiplied_alpha = image - .pixels() - .any(|(_, _, pixel)| pixel.is_premultiplied_alpha()); - - for (x, y, pixel) in image.pixels() { - let pixel = if is_premultiplied_alpha { - pixel.unpremultiply_alpha() - } else { - // return the original - pixel - }; - - out.put_pixel(x, y, pixel); - } - - out -} - -fn apply_unpremultiply_alpha( - image: &DynamicImage, -) -> Result { - match image.color() { - ColorType::La8 => Ok(DynamicImage::ImageLumaA8( - process_unpremultiply_alpha(&image.to_luma_alpha8()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_unpremultiply_alpha(&image.to_rgba8()), - )), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_unpremultiply_alpha(&image.to_luma_alpha16()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_unpremultiply_alpha(&image.to_rgba16()), - )), - _ => Err(type_error(image_error_message( - "apply premultiplyAlpha: none", - "The color type is not supported.", - ))), - } -} - -// reference -// https://www.w3.org/TR/css-color-4/#color-conversion-code -fn srgb_to_linear(value: T) -> f32 { - if value.to_f32().unwrap() <= 0.04045 { - value.to_f32().unwrap() / 12.92 - } else { - ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) - } -} - -// reference -// https://www.w3.org/TR/css-color-4/#color-conversion-code -fn linear_to_display_p3(value: T) -> f32 { - if value.to_f32().unwrap() <= 0.0031308 { - value.to_f32().unwrap() * 12.92 - } else { - 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 - } -} - -fn normalize_value_to_0_1(value: T) -> f32 { - value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() -} - -fn unnormalize_value_from_0_1(value: f32) -> T { - NumCast::from( - (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), - ) - .unwrap() -} - -fn srgb_to_display_p3(r: T, g: T, b: T) -> (T, T, T) { - // normalize the value to 0.0 - 1.0 - let (r, g, b) = ( - normalize_value_to_0_1(r), - normalize_value_to_0_1(g), - normalize_value_to_0_1(b), - ); - - // sRGB -> Linear RGB - let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); - - // Display-P3 (RGB) -> Display-P3 (XYZ) - // - // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] - // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html - // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html - - // [ sRGB (D65) to XYZ ] - #[rustfmt::skip] - let (m1x, m1y, m1z) = ( - [0.4124564, 0.3575761, 0.1804375], - [0.2126729, 0.7151522, 0.0721750], - [0.0193339, 0.1191920, 0.9503041], - ); - - let (r, g, b) = ( - r * m1x[0] + g * m1x[1] + b * m1x[2], - r * m1y[0] + g * m1y[1] + b * m1y[2], - r * m1z[0] + g * m1z[1] + b * m1z[2], - ); - - // inv[ P3-D65 (D65) to XYZ ] - #[rustfmt::skip] - let (m2x, m2y, m2z) = ( - [ 2.493496911941425, -0.9313836179191239, -0.40271078445071684 ], - [ -0.8294889695615747, 1.7626640603183463, 0.023624685841943577 ], - [ 0.03584583024378447,-0.07617238926804182, 0.9568845240076872 ], - ); - - let (r, g, b) = ( - r * m2x[0] + g * m2x[1] + b * m2x[2], - r * m2y[0] + g * m2y[1] + b * m2y[2], - r * m2z[0] + g * m2z[1] + b * m2z[2], - ); - - // This calculation is similar as above that it is a little faster, but less accurate. - // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; - // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; - // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; - - // Display-P3 (Linear) -> Display-P3 - let (r, g, b) = ( - linear_to_display_p3(r), - linear_to_display_p3(g), - linear_to_display_p3(b), - ); - - // unnormalize the value from 0.0 - 1.0 - ( - unnormalize_value_from_0_1(r), - unnormalize_value_from_0_1(g), - unnormalize_value_from_0_1(b), - ) -} - -trait ProcessColorSpaceConversion { - /// Display P3 Color Encoding (v 1.0) - /// https://www.color.org/chardata/rgb/DisplayP3.xalter - fn process_srgb_to_display_p3(&self) -> Self; -} - -impl ProcessColorSpaceConversion for Rgb { - fn process_srgb_to_display_p3(&self) -> Self { - let (r, g, b) = (self.0[0], self.0[1], self.0[2]); - - let (r, g, b) = srgb_to_display_p3(r, g, b); - - Rgb::([r, g, b]) - } -} - -impl ProcessColorSpaceConversion for Rgba { - fn process_srgb_to_display_p3(&self) -> Self { - let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); - - let (r, g, b) = srgb_to_display_p3(r, g, b); - - Rgba::([r, g, b, a]) - } -} - -fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> -where - I: GenericImageView, - P: Pixel + ProcessColorSpaceConversion + 'static, - S: Primitive + 'static, -{ - let (width, height) = image.dimensions(); - let mut out = ImageBuffer::new(width, height); - - for (x, y, pixel) in image.pixels() { - let pixel = pixel.process_srgb_to_display_p3(); - - out.put_pixel(x, y, pixel); - } - - out -} - -trait SliceToPixel { - fn slice_to_pixel(pixel: &[u8]) -> Self; -} - -impl SliceToPixel for Luma { - fn slice_to_pixel(pixel: &[u8]) -> Self { - let pixel: &[T] = cast_slice(pixel); - let pixel = [pixel[0]]; - - Luma::(pixel) - } -} - -impl SliceToPixel for LumaA { - fn slice_to_pixel(pixel: &[u8]) -> Self { - let pixel: &[T] = cast_slice(pixel); - let pixel = [pixel[0], pixel[1]]; - - LumaA::(pixel) - } -} - -impl SliceToPixel for Rgb { - fn slice_to_pixel(pixel: &[u8]) -> Self { - let pixel: &[T] = cast_slice(pixel); - let pixel = [pixel[0], pixel[1], pixel[2]]; - - Rgb::(pixel) - } -} - -impl SliceToPixel for Rgba { - fn slice_to_pixel(pixel: &[u8]) -> Self { - let pixel: &[T] = cast_slice(pixel); - let pixel = [pixel[0], pixel[1], pixel[2], pixel[3]]; - - Rgba::(pixel) - } -} - -/// Convert the pixel slice to an array to avoid the copy to Vec. -/// I implemented this trait because of I couldn't find a way to effectively combine -/// the `Transform` struct of `lcms2` and `Pixel` trait of `image`. -/// If there is an implementation that is safer and can withstand changes, I would like to adopt it. -trait SliceToArray { - fn slice_to_array(pixel: &[u8]) -> [u8; N]; -} - -macro_rules! impl_slice_to_array { - ($type:ty, $n:expr) => { - impl SliceToArray<$n> for $type { - fn slice_to_array(pixel: &[u8]) -> [u8; $n] { - let mut dst = [0_u8; $n]; - dst.copy_from_slice(&pixel[..$n]); - - dst - } - } - }; -} - -impl_slice_to_array!(Luma, 1); -impl_slice_to_array!(Luma, 2); -impl_slice_to_array!(LumaA, 2); -impl_slice_to_array!(LumaA, 4); -impl_slice_to_array!(Rgb, 3); -impl_slice_to_array!(Rgb, 6); -impl_slice_to_array!(Rgba, 4); -impl_slice_to_array!(Rgba, 8); - -fn process_color_space_from_icc_profile_to_srgb( - image: &DynamicImage, - icc_profile: Profile, -) -> ImageBuffer> -where - P: Pixel + SliceToPixel + SliceToArray + 'static, - S: Primitive + 'static, -{ - let (width, height) = image.dimensions(); - let mut out = ImageBuffer::new(width, height); - let chunk_size = image.color().bytes_per_pixel() as usize; - let pixel_iter = image - .as_bytes() - .chunks_exact(chunk_size) - .zip(image.pixels()); - let pixel_format = match image.color() { - ColorType::L8 => PixelFormat::GRAY_8, - ColorType::L16 => PixelFormat::GRAY_16, - ColorType::La8 => PixelFormat::GRAYA_8, - ColorType::La16 => PixelFormat::GRAYA_16, - ColorType::Rgb8 => PixelFormat::RGB_8, - ColorType::Rgb16 => PixelFormat::RGB_16, - ColorType::Rgba8 => PixelFormat::RGBA_8, - ColorType::Rgba16 => PixelFormat::RGBA_16, - // This arm usually doesn't reach, but it should be handled with returning the original image. - _ => { - return { - for (pixel, (x, y, _)) in pixel_iter { - out.put_pixel(x, y, P::slice_to_pixel(&pixel)); - } - out - } - } - }; - let srgb_icc_profile = Profile::new_srgb(); - let transformer = Transform::new( - &icc_profile, - pixel_format, - &srgb_icc_profile, - pixel_format, - srgb_icc_profile.header_rendering_intent(), - ); - - for (pixel, (x, y, _)) in pixel_iter { - let pixel = match transformer { - Ok(ref transformer) => { - let mut dst = P::slice_to_array(pixel); - transformer.transform_in_place(&mut dst); - - dst - } - // This arm will reach when the ffi call fails. - Err(_) => P::slice_to_array(pixel), - }; - - out.put_pixel(x, y, P::slice_to_pixel(&pixel)); - } - - out -} - -/// According to the spec, it's not clear how to handle the color space conversion. -/// -/// Therefore, if you interpret the specification description from the implementation and wpt results, it will be as follows. -/// -/// Let val be the value of the colorSpaceConversion member of options, and then run these substeps: -/// 1. If val is "default", to convert to the sRGB color space. -/// 2. If val is "none", to use the decoded image data as is. -/// -/// related issue in whatwg -/// https://github.com/whatwg/html/issues/10578 -/// -/// reference in wpt -/// https://github.com/web-platform-tests/wpt/blob/d575dc75ede770df322fbc5da3112dcf81f192ec/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html#L18 -/// https://wpt.live/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html -fn apply_color_space_conversion( - image: DynamicImage, - icc_profile: Option>, - image_bitmap_source: &ImageBitmapSource, - color_space_conversion: &ColorSpaceConversion, - predefined_color_space: &PredefinedColorSpace, -) -> Result { - match color_space_conversion { - // return the decoded image as is. - ColorSpaceConversion::None => Ok(image), - ColorSpaceConversion::Default => { - match image_bitmap_source { - ImageBitmapSource::Blob => match icc_profile { - // If there is no color profile information, return the image as is. - None => Ok(image), - Some(icc_profile) => match Profile::new_icc(&icc_profile) { - // If the color profile information is invalid, return the image as is. - Err(_) => Ok(image), - Ok(icc_profile) => match image.color() { - ColorType::L8 => Ok(DynamicImage::ImageLuma8( - process_color_space_from_icc_profile_to_srgb::<_, _, 1>( - &image, - icc_profile, - ), - )), - ColorType::L16 => Ok(DynamicImage::ImageLuma16( - process_color_space_from_icc_profile_to_srgb::<_, _, 2>( - &image, - icc_profile, - ), - )), - ColorType::La8 => Ok(DynamicImage::ImageLumaA8( - process_color_space_from_icc_profile_to_srgb::<_, _, 2>( - &image, - icc_profile, - ), - )), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_color_space_from_icc_profile_to_srgb::<_, _, 4>( - &image, - icc_profile, - ), - )), - ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8( - process_color_space_from_icc_profile_to_srgb::<_, _, 3>( - &image, - icc_profile, - ), - )), - ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( - process_color_space_from_icc_profile_to_srgb::<_, _, 6>( - &image, - icc_profile, - ), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_color_space_from_icc_profile_to_srgb::<_, _, 4>( - &image, - icc_profile, - ), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_color_space_from_icc_profile_to_srgb::<_, _, 8>( - &image, - icc_profile, - ), - )), - _ => Err(type_error(image_error_message( - "apply colorspaceConversion: default", - "The color type is not supported.", - ))), - }, - }, - }, - ImageBitmapSource::ImageData => match predefined_color_space { - // If the color space is sRGB, return the image as is. - PredefinedColorSpace::Srgb => Ok(image), - PredefinedColorSpace::DisplayP3 => { - match image.color() { - // The conversion of the lumincance color types to the display-p3 color space is meaningless. - ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), - ColorType::L16 => { - Ok(DynamicImage::ImageLuma16(image.to_luma16())) - } - ColorType::La8 => { - Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())) - } - ColorType::La16 => { - Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())) - } - ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8( - process_srgb_to_display_p3(&image.to_rgb8()), - )), - ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( - process_srgb_to_display_p3(&image.to_rgb16()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_srgb_to_display_p3(&image.to_rgba8()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_srgb_to_display_p3(&image.to_rgba16()), - )), - _ => Err(type_error(image_error_message( - "apply colorspace: display-p3", - "The color type is not supported.", - ))), - } - } - }, - } - } - } -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -enum ImageResizeQuality { - Pixelated, - Low, - Medium, - High, -} - -#[derive(Debug, Deserialize, PartialEq)] -// Follow the cases defined in the spec -enum ImageBitmapSource { - Blob, - ImageData, -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -enum PremultiplyAlpha { - Default, - Premultiply, - None, -} - -// https://github.com/gfx-rs/wgpu/blob/04618b36a89721c23dc46f5844c71c0e10fc7844/wgpu-types/src/lib.rs#L6948C10-L6948C30 -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -enum PredefinedColorSpace { - Srgb, - #[serde(rename = "display-p3")] - DisplayP3, -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -enum ColorSpaceConversion { - Default, - None, -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -enum ImageOrientation { - FlipY, - #[serde(rename = "from-image")] - FromImage, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ImageProcessArgs { - width: u32, - height: u32, - sx: Option, - sy: Option, - sw: Option, - sh: Option, - image_orientation: ImageOrientation, - premultiply_alpha: PremultiplyAlpha, - predefined_color_space: PredefinedColorSpace, - color_space_conversion: ColorSpaceConversion, - resize_width: Option, - resize_height: Option, - resize_quality: ImageResizeQuality, - image_bitmap_source: ImageBitmapSource, - mime_type: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ImageProcessResult { - data: ToJsBuffer, - width: u32, - height: u32, -} - -// -// About the animated image -// > Blob .4 -// > ... If this is an animated image, imageBitmap's bitmap data must only be taken from -// > the default image of the animation (the one that the format defines is to be used when animation is -// > not supported or is disabled), or, if there is no such image, the first frame of the animation. -// https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html -// -// see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) -// https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 -// - -trait ImageDecoderFromReader<'a, R: BufRead + Seek> { - fn to_decoder(reader: R) -> Result - where - Self: Sized; - fn to_intermediate_image(self) -> Result; - fn get_icc_profile(&mut self) -> Option>; -} - -type ImageDecoderFromReaderType<'a> = BufReader>; - -fn image_decoding_error(error: ImageError) -> DOMExceptionInvalidStateError { - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &error.to_string(), - )) -} - -macro_rules! impl_image_decoder_from_reader { - ($decoder:ty, $reader:ty) => { - impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { - fn to_decoder(reader: R) -> Result - where - Self: Sized, - { - match <$decoder>::new(reader) { - Ok(decoder) => Ok(decoder), - Err(err) => return Err(image_decoding_error(err).into()), - } - } - fn to_intermediate_image(self) -> Result { - match DynamicImage::from_decoder(self) { - Ok(image) => Ok(image), - Err(err) => Err(image_decoding_error(err).into()), - } - } - fn get_icc_profile(&mut self) -> Option> { - match self.icc_profile() { - Ok(profile) => profile, - Err(_) => None, - } - } - } - }; -} - -// If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. -impl_image_decoder_from_reader!(PngDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); -// The GifDecoder decodes the first frame. -impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); -// The WebPDecoder decodes the first frame. -impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); - -type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); - -fn decode_bitmap_data( - buf: &[u8], - width: u32, - height: u32, - image_bitmap_source: &ImageBitmapSource, - mime_type: String, -) -> Result { - let (view, width, height, icc_profile) = match image_bitmap_source { - ImageBitmapSource::Blob => { - let (image, icc_profile) = match &*mime_type { - // Should we support the "image/apng" MIME type here? - "image/png" => { - let mut decoder: PngDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "image/jpeg" => { - let mut decoder: JpegDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "image/gif" => { - let mut decoder: GifDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "image/bmp" => { - let mut decoder: BmpDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "image/x-icon" => { - let mut decoder: IcoDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "image/webp" => { - let mut decoder: WebPDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )))?; - let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) - } - "" => { - return Err( - DOMExceptionInvalidStateError::new( - &format!("The MIME type of source image is not specified. -INFO: The behavior of the Blob constructor in browsers is different from the spec. -It needs to specify the MIME type like {} that works well between Deno and browsers. -See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", - cyan("new Blob([blobParts], { type: 'image/png' })") - )).into(), - ) - } - // return an error if the MIME type is not supported in the variable list of ImageTypePatternTable below - // ext/web/01_mimesniff.js - // - // NOTE: Chromium supports AVIF - // https://source.chromium.org/chromium/chromium/src/+/ef3f4e4ed97079dc57861d1195fb2389483bc195:third_party/blink/renderer/platform/image-decoders/image_decoder.cc;l=311 - x => { - return Err( - DOMExceptionInvalidStateError::new( - &format!("The the MIME type {} of source image is not a supported format. -INFO: The following MIME types are supported: -See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", - x - )).into() - ) - } - }; - - let width = image.width(); - let height = image.height(); - - (image, width, height, icc_profile) - } - ImageBitmapSource::ImageData => { - // > 4.12.5.1.15 Pixel manipulation - // > imagedata.data - // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. - // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation - let image = match RgbaImage::from_raw(width, height, buf.into()) { - Some(image) => image.into(), - None => { - return Err(type_error(image_error_message( - "decoding", - "The Chunk Data is not big enough with the specified width and height.", - ))) - } - }; - - (image, width, height, None) - } - }; - - Ok((view, width, height, icc_profile)) -} - -#[op2] -#[serde] -fn op_image_process( - #[buffer] zero_copy: JsBuffer, - #[serde] args: ImageProcessArgs, -) -> Result { - let buf = &*zero_copy; - let ImageProcessArgs { - width, - height, - sh, - sw, - sx, - sy, - image_orientation, - premultiply_alpha, - predefined_color_space, - color_space_conversion, - resize_width, - resize_height, - resize_quality, - image_bitmap_source, - mime_type, - } = ImageProcessArgs { - width: args.width, - height: args.height, - sx: args.sx, - sy: args.sy, - sw: args.sw, - sh: args.sh, - image_orientation: args.image_orientation, - premultiply_alpha: args.premultiply_alpha, - predefined_color_space: args.predefined_color_space, - color_space_conversion: args.color_space_conversion, - resize_width: args.resize_width, - resize_height: args.resize_height, - resize_quality: args.resize_quality, - image_bitmap_source: args.image_bitmap_source, - mime_type: args.mime_type, - }; - - let (view, width, height, icc_profile) = - decode_bitmap_data(buf, width, height, &image_bitmap_source, mime_type)?; - - #[rustfmt::skip] - let source_rectangle: [[i32; 2]; 4] = - if let (Some(sx), Some(sy), Some(sw), Some(sh)) = (sx, sy, sw, sh) { - [ - [sx, sy], - [sx + sw, sy], - [sx + sw, sy + sh], - [sx, sy + sh] - ] - } else { - [ - [0, 0], - [width as i32, 0], - [width as i32, height as i32], - [0, height as i32], - ] - }; - - /* - * The cropping works differently than the spec specifies: - * The spec states to create an infinite surface and place the top-left corner - * of the image a 0,0 and crop based on sourceRectangle. - * - * We instead create a surface the size of sourceRectangle, and position - * the image at the correct location, which is the inverse of the x & y of - * sourceRectangle's top-left corner. - */ - let input_x = -(source_rectangle[0][0] as i64); - let input_y = -(source_rectangle[0][1] as i64); - - let surface_width = (source_rectangle[1][0] - source_rectangle[0][0]) as u32; - let surface_height = (source_rectangle[3][1] - source_rectangle[0][1]) as u32; - - let output_width = if let Some(resize_width) = resize_width { - resize_width - } else if let Some(resize_height) = resize_height { - (surface_width * resize_height).div_ceil(surface_height) - } else { - surface_width - }; - - let output_height = if let Some(resize_height) = resize_height { - resize_height - } else if let Some(resize_width) = resize_width { - (surface_height * resize_width).div_ceil(surface_width) - } else { - surface_height - }; - - let color = view.color(); - - let surface = if !(width == surface_width - && height == surface_height - && input_x == 0 - && input_y == 0) - { - let mut surface = DynamicImage::new(surface_width, surface_height, color); - overlay(&mut surface, &view, input_x, input_y); - - surface - } else { - view - }; - - let filter_type = match resize_quality { - ImageResizeQuality::Pixelated => FilterType::Nearest, - ImageResizeQuality::Low => FilterType::Triangle, - ImageResizeQuality::Medium => FilterType::CatmullRom, - ImageResizeQuality::High => FilterType::Lanczos3, - }; - - // should use resize_exact - // https://github.com/image-rs/image/issues/1220#issuecomment-632060015 - let image_out = - surface.resize_exact(output_width, output_height, filter_type); - - // - // FIXME: It also need to fix about orientation when the spec is updated. - // - // > Multiple browser vendors discussed this a while back and (99% sure, from recollection) - // > agreed to change createImageBitmap's behavior. - // > The HTML spec should be updated to say: - // > first EXIF orientation is applied, and then if imageOrientation is flipY, the image is flipped vertically - // https://github.com/whatwg/html/issues/8085#issuecomment-2204696312 - let image_out = if image_orientation == ImageOrientation::FlipY { - image_out.flipv() - } else { - image_out - }; - - // 9. - let image_out = apply_color_space_conversion( - image_out, - icc_profile, - &image_bitmap_source, - &color_space_conversion, - &predefined_color_space, - )?; - - // 10. - if color.has_alpha() { - match premultiply_alpha { - // 1. - PremultiplyAlpha::Default => { /* noop */ } - - // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied - - // 2. - PremultiplyAlpha::Premultiply => { - let result = apply_premultiply_alpha(&image_out)?; - let data = to_js_buffer(&result); - return Ok(ImageProcessResult { - data, - width: output_width, - height: output_height, - }); - } - // 3. - PremultiplyAlpha::None => { - // NOTE: It's not clear how to handle the case of ImageData. - // https://issues.chromium.org/issues/339759426 - // https://github.com/whatwg/html/issues/5365 - if image_bitmap_source == ImageBitmapSource::ImageData { - return Ok(ImageProcessResult { - data: image_out.clone().into_bytes().into(), - width: output_width, - height: output_height, - }); - } - - let result = apply_unpremultiply_alpha(&image_out)?; - let data = to_js_buffer(&result); - return Ok(ImageProcessResult { - data, - width: output_width, - height: output_height, - }); - } - } - } - - Ok(ImageProcessResult { - data: image_out.clone().into_bytes().into(), - width: output_width, - height: output_height, - }) -} +pub mod idl; +mod image_decoder; +mod image_ops; +mod op_create_image_bitmap; +use op_create_image_bitmap::op_create_image_bitmap; deno_core::extension!( deno_canvas, deps = [deno_webidl, deno_web, deno_webgpu], - ops = [op_image_process], + ops = [op_create_image_bitmap], lazy_loaded_esm = ["01_image.js"], ); diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs new file mode 100644 index 00000000000000..91cf68a08c5d72 --- /dev/null +++ b/ext/canvas/op_create_image_bitmap.rs @@ -0,0 +1,453 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::io::BufReader; +use std::io::Cursor; + +use deno_core::error::type_error; +use deno_core::error::AnyError; +use deno_core::op2; +use deno_core::JsBuffer; +use deno_core::ToJsBuffer; +use deno_terminal::colors::cyan; +use image::codecs::bmp::BmpDecoder; +use image::codecs::gif::GifDecoder; +use image::codecs::ico::IcoDecoder; +use image::codecs::jpeg::JpegDecoder; +use image::codecs::png::PngDecoder; +use image::codecs::webp::WebPDecoder; +use image::imageops::overlay; +use image::imageops::FilterType; +use image::ColorType; +use image::DynamicImage; +use image::RgbaImage; +use serde::Deserialize; +use serde::Serialize; + +use crate::error::image_error_message; +use crate::error::DOMExceptionInvalidStateError; +use crate::idl::PredefinedColorSpace; +use crate::image_decoder::ImageDecoderFromReader; +use crate::image_decoder::ImageDecoderFromReaderType; +use crate::image_ops::premultiply_alpha as process_premultiply_alpha; +use crate::image_ops::srgb_to_display_p3; +use crate::image_ops::to_srgb_from_icc_profile; +use crate::image_ops::unpremultiply_alpha; + +#[derive(Debug, Deserialize, PartialEq)] +// Follow the cases defined in the spec +enum ImageBitmapSource { + Blob, + ImageData, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum ImageOrientation { + FlipY, + #[serde(rename = "from-image")] + FromImage, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum PremultiplyAlpha { + Default, + Premultiply, + None, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum ColorSpaceConversion { + Default, + None, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum ResizeQuality { + Pixelated, + Low, + Medium, + High, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OpCreateImageBitmapArgs { + width: u32, + height: u32, + sx: Option, + sy: Option, + sw: Option, + sh: Option, + image_orientation: ImageOrientation, + premultiply_alpha: PremultiplyAlpha, + predefined_color_space: PredefinedColorSpace, + color_space_conversion: ColorSpaceConversion, + resize_width: Option, + resize_height: Option, + resize_quality: ResizeQuality, + image_bitmap_source: ImageBitmapSource, + mime_type: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpCreateImageBitmapReturn { + data: ToJsBuffer, + width: u32, + height: u32, +} + +type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); + +fn decode_bitmap_data( + buf: &[u8], + width: u32, + height: u32, + image_bitmap_source: &ImageBitmapSource, + mime_type: String, +) -> Result { + let (image, width, height, icc_profile) = match image_bitmap_source { + ImageBitmapSource::Blob => { + let (image, icc_profile) = match &*mime_type { + // Should we support the "image/apng" MIME type here? + "image/png" => { + let mut decoder: PngDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "image/jpeg" => { + let mut decoder: JpegDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "image/gif" => { + let mut decoder: GifDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "image/bmp" => { + let mut decoder: BmpDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "image/x-icon" => { + let mut decoder: IcoDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "image/webp" => { + let mut decoder: WebPDecoder = + ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( + buf, + )))?; + let icc_profile = decoder.get_icc_profile(); + (decoder.to_intermediate_image()?, icc_profile) + } + "" => { + return Err( + DOMExceptionInvalidStateError::new( + &format!("The MIME type of source image is not specified. +INFO: The behavior of the Blob constructor in browsers is different from the spec. +It needs to specify the MIME type like {} that works well between Deno and browsers. +See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", + cyan("new Blob([blobParts], { type: 'image/png' })") + )).into(), + ) + } + // return an error if the MIME type is not supported in the variable list of ImageTypePatternTable below + // ext/web/01_mimesniff.js + // + // NOTE: Chromium supports AVIF + // https://source.chromium.org/chromium/chromium/src/+/ef3f4e4ed97079dc57861d1195fb2389483bc195:third_party/blink/renderer/platform/image-decoders/image_decoder.cc;l=311 + x => { + return Err( + DOMExceptionInvalidStateError::new( + &format!("The the MIME type {} of source image is not a supported format. +INFO: The following MIME types are supported: +See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", + x + )).into() + ) + } + }; + + let width = image.width(); + let height = image.height(); + + (image, width, height, icc_profile) + } + ImageBitmapSource::ImageData => { + // > 4.12.5.1.15 Pixel manipulation + // > imagedata.data + // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. + // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation + let image = match RgbaImage::from_raw(width, height, buf.into()) { + Some(image) => image.into(), + None => { + return Err(type_error(image_error_message( + "decoding", + "The Chunk Data is not big enough with the specified width and height.", + ))) + } + }; + + (image, width, height, None) + } + }; + + Ok((image, width, height, icc_profile)) +} + +/// According to the spec, it's not clear how to handle the color space conversion. +/// +/// Therefore, if you interpret the specification description from the implementation and wpt results, it will be as follows. +/// +/// Let val be the value of the colorSpaceConversion member of options, and then run these substeps: +/// 1. If val is "default", to convert to the sRGB color space. +/// 2. If val is "none", to use the decoded image data as is. +/// +/// related issue in whatwg +/// https://github.com/whatwg/html/issues/10578 +/// +/// reference in wpt +/// https://github.com/web-platform-tests/wpt/blob/d575dc75ede770df322fbc5da3112dcf81f192ec/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html#L18 +/// https://wpt.live/html/canvas/element/manual/imagebitmap/createImageBitmap-colorSpaceConversion.html +fn apply_color_space_conversion( + image: DynamicImage, + icc_profile: Option>, + image_bitmap_source: &ImageBitmapSource, + color_space_conversion: &ColorSpaceConversion, + predefined_color_space: &PredefinedColorSpace, +) -> Result { + match color_space_conversion { + // return the decoded image as is. + ColorSpaceConversion::None => Ok(image), + ColorSpaceConversion::Default => { + match image_bitmap_source { + ImageBitmapSource::Blob => { + fn color_unmatch(x: ColorType) -> Result { + Err(type_error(image_error_message( + "apply colorspaceConversion: default", + &format!("The color type {:?} is not supported.", x), + ))) + } + to_srgb_from_icc_profile(image, icc_profile, Some(color_unmatch)) + } + ImageBitmapSource::ImageData => match predefined_color_space { + // If the color space is sRGB, return the image as is. + PredefinedColorSpace::Srgb => Ok(image), + PredefinedColorSpace::DisplayP3 => { + fn unmatch(x: ColorType) -> Result { + Err(type_error(image_error_message( + "apply colorspace: display-p3", + &format!("The color type {:?} is not supported.", x), + ))) + } + srgb_to_display_p3(image, Some(unmatch)) + } + }, + } + } + } +} + +fn apply_premultiply_alpha( + image: DynamicImage, + image_bitmap_source: &ImageBitmapSource, + premultiply_alpha: &PremultiplyAlpha, +) -> Result { + let color = image.color(); + if !color.has_alpha() { + Ok(image) + } else { + match premultiply_alpha { + // 1. + PremultiplyAlpha::Default => Ok(image), + + // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied + + // 2. + PremultiplyAlpha::Premultiply => process_premultiply_alpha(image, None), + // 3. + PremultiplyAlpha::None => { + // NOTE: It's not clear how to handle the case of ImageData. + // https://issues.chromium.org/issues/339759426 + // https://github.com/whatwg/html/issues/5365 + if *image_bitmap_source == ImageBitmapSource::ImageData { + return Ok(image); + } + + unpremultiply_alpha(image, None) + } + } + } +} + +#[op2] +#[serde] +pub(super) fn op_create_image_bitmap( + #[buffer] zero_copy: JsBuffer, + #[serde] args: OpCreateImageBitmapArgs, +) -> Result { + let buf = &*zero_copy; + let OpCreateImageBitmapArgs { + width, + height, + sh, + sw, + sx, + sy, + image_orientation, + premultiply_alpha, + predefined_color_space, + color_space_conversion, + resize_width, + resize_height, + resize_quality, + image_bitmap_source, + mime_type, + } = OpCreateImageBitmapArgs { + width: args.width, + height: args.height, + sx: args.sx, + sy: args.sy, + sw: args.sw, + sh: args.sh, + image_orientation: args.image_orientation, + premultiply_alpha: args.premultiply_alpha, + predefined_color_space: args.predefined_color_space, + color_space_conversion: args.color_space_conversion, + resize_width: args.resize_width, + resize_height: args.resize_height, + resize_quality: args.resize_quality, + image_bitmap_source: args.image_bitmap_source, + mime_type: args.mime_type, + }; + + // 6. Switch on image: + let (image, width, height, icc_profile) = + decode_bitmap_data(buf, width, height, &image_bitmap_source, mime_type)?; + + // crop bitmap data + // 2. + #[rustfmt::skip] + let source_rectangle: [[i32; 2]; 4] = + if let (Some(sx), Some(sy), Some(sw), Some(sh)) = (sx, sy, sw, sh) { + [ + [sx, sy], + [sx + sw, sy], + [sx + sw, sy + sh], + [sx, sy + sh] + ] + } else { + [ + [0, 0], + [width as i32, 0], + [width as i32, height as i32], + [0, height as i32], + ] + }; + + /* + * The cropping works differently than the spec specifies: + * The spec states to create an infinite surface and place the top-left corner + * of the image a 0,0 and crop based on sourceRectangle. + * + * We instead create a surface the size of sourceRectangle, and position + * the image at the correct location, which is the inverse of the x & y of + * sourceRectangle's top-left corner. + */ + let input_x = -(source_rectangle[0][0] as i64); + let input_y = -(source_rectangle[0][1] as i64); + + let surface_width = (source_rectangle[1][0] - source_rectangle[0][0]) as u32; + let surface_height = (source_rectangle[3][1] - source_rectangle[0][1]) as u32; + + // 3. + let output_width = if let Some(resize_width) = resize_width { + resize_width + } else if let Some(resize_height) = resize_height { + (surface_width * resize_height).div_ceil(surface_height) + } else { + surface_width + }; + + // 4. + let output_height = if let Some(resize_height) = resize_height { + resize_height + } else if let Some(resize_width) = resize_width { + (surface_height * resize_width).div_ceil(surface_width) + } else { + surface_height + }; + + // 5. + let image = if !(width == surface_width + && height == surface_height + && input_x == 0 + && input_y == 0) + { + let mut surface = + DynamicImage::new(surface_width, surface_height, image.color()); + overlay(&mut surface, &image, input_x, input_y); + + surface + } else { + image + }; + + // 7. + let filter_type = match resize_quality { + ResizeQuality::Pixelated => FilterType::Nearest, + ResizeQuality::Low => FilterType::Triangle, + ResizeQuality::Medium => FilterType::CatmullRom, + ResizeQuality::High => FilterType::Lanczos3, + }; + // should use resize_exact + // https://github.com/image-rs/image/issues/1220#issuecomment-632060015 + let image = image.resize_exact(output_width, output_height, filter_type); + + // 8. + let image = if image_orientation == ImageOrientation::FlipY { + image.flipv() + } else { + image + }; + + // 9. + let image = apply_color_space_conversion( + image, + icc_profile, + &image_bitmap_source, + &color_space_conversion, + &predefined_color_space, + )?; + + // 10. + let image = + apply_premultiply_alpha(image, &image_bitmap_source, &premultiply_alpha)?; + + Ok(OpCreateImageBitmapReturn { + data: image.into_bytes().into(), + width: output_width, + height: output_height, + }) +} From 98e9304f2ae5a2bb95ca20b46bf3187c665f55b0 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 8 Sep 2024 20:24:00 +0900 Subject: [PATCH 15/52] add unit test --- ext/canvas/image_ops.rs | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index bbd6fcc55cf996..2709df97aad2cb 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -595,3 +595,51 @@ pub(crate) fn to_srgb_from_icc_profile( }, } } + +#[cfg(test)] +mod tests { + use super::*; + use image::Rgba; + + #[test] + fn test_premultiply_alpha() { + let rgba = Rgba::([255, 128, 0, 128]); + let rgba = rgba.premultiply_alpha(); + assert_eq!(rgba, Rgba::([128, 64, 0, 128])); + + let rgba = Rgba::([255, 255, 255, 255]); + let rgba = rgba.premultiply_alpha(); + assert_eq!(rgba, Rgba::([255, 255, 255, 255])); + } + + #[test] + fn test_unpremultiply_alpha() { + let rgba = Rgba::([127, 0, 0, 127]); + let rgba = rgba.unpremultiply_alpha(); + assert_eq!(rgba, Rgba::([255, 0, 0, 127])); + } + + #[test] + fn test_apply_conversion_matrix_srgb_to_display_p3() { + let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(255_u8, 0, 0); + assert_eq!(r, 234); + assert_eq!(g, 51); + assert_eq!(b, 35); + + let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 255, 0); + assert_eq!(r, 117); + assert_eq!(g, 251); + assert_eq!(b, 76); + + let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 0, 255); + assert_eq!(r, 0); + assert_eq!(g, 0); + assert_eq!(b, 245); + + let (r, g, b) = + apply_conversion_matrix_srgb_to_display_p3(255_u8, 255, 255); + assert_eq!(r, 255); + assert_eq!(g, 255); + assert_eq!(b, 255); + } +} From 4bc517eab2593eac532f0332ca09c6a708ea338c Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 8 Sep 2024 20:26:53 +0900 Subject: [PATCH 16/52] update README --- ext/canvas/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ext/canvas/README.md b/ext/canvas/README.md index cf013677e786b1..f731953eb34718 100644 --- a/ext/canvas/README.md +++ b/ext/canvas/README.md @@ -1,3 +1,32 @@ # deno_canvas Extension that implements various OffscreenCanvas related APIs. + +## Image processing architecture in Rust + +```mermaid +flowchart LR + Input["input binary
( &[u8] )"] + II["intermediate image
( DynamicImage )"] + Ops["processing pixel
( ImageBuffer< P, S > )"] + Output["output binary
( &[u8] )"] + Input --> II + II --> Ops --> II + II --> Output +``` + +The architecture of image processing in Rust is rely on the structure of +[image](https://github.com/image-rs/image) crate.\ +If the input is a image of binary, it convert to an intermediate image +(`DynamicImage` in `image`) with using a decoder corresponding to its image +formats.\ +After converting to an intermediate image, it can process various way for +example, to use the pixel processong operation +[imageops](https://github.com/image-rs/image?tab=readme-ov-file#image-processing-functions) +supplied by `image`.\ +On the other hand, there can also to implement your own pixel processong +operation to refer to +[the implementation of imageops as here](https://github.com/image-rs/image/blob/4afe9572b5c867cf4d07cd88107e8c49354de9f3/src/imageops/colorops.rs#L156-L182) +or [image_ops.rs module](./image_ops.rs).\ +You can treat any bit depth that supported by `image` with generics in the +processing pixel layer. From f0895c6b7b2814450db609860aadf881a57f3727 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:32:22 +0900 Subject: [PATCH 17/52] fix error handling --- ext/canvas/image_decoder.rs | 36 ++++++++--------- ext/canvas/image_ops.rs | 37 ++++++++---------- ext/canvas/op_create_image_bitmap.rs | 58 +++++++++++++++++++--------- 3 files changed, 75 insertions(+), 56 deletions(-) diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs index 4b9e4fa53a7d3b..02680da8e367d9 100644 --- a/ext/canvas/image_decoder.rs +++ b/ext/canvas/image_decoder.rs @@ -16,9 +16,6 @@ use image::DynamicImage; use image::ImageDecoder; use image::ImageError; -use crate::error::image_error_message; -use crate::error::DOMExceptionInvalidStateError; - // // About the animated image // > Blob .4 @@ -32,40 +29,43 @@ use crate::error::DOMExceptionInvalidStateError; // pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { - fn to_decoder(reader: R) -> Result + fn to_decoder( + reader: R, + error_fn: fn(ImageError) -> AnyError, + ) -> Result where Self: Sized; - fn to_intermediate_image(self) -> Result; + fn to_intermediate_image( + self, + error_fn: fn(ImageError) -> AnyError, + ) -> Result; fn get_icc_profile(&mut self) -> Option>; } pub(crate) type ImageDecoderFromReaderType<'a> = BufReader>; -pub(crate) fn image_decoding_error( - error: ImageError, -) -> DOMExceptionInvalidStateError { - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &error.to_string(), - )) -} - macro_rules! impl_image_decoder_from_reader { ($decoder:ty, $reader:ty) => { impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { - fn to_decoder(reader: R) -> Result + fn to_decoder( + reader: R, + error_fn: fn(ImageError) -> AnyError, + ) -> Result where Self: Sized, { match <$decoder>::new(reader) { Ok(decoder) => Ok(decoder), - Err(err) => return Err(image_decoding_error(err).into()), + Err(err) => return Err(error_fn(err)), } } - fn to_intermediate_image(self) -> Result { + fn to_intermediate_image( + self, + error_fn: fn(ImageError) -> AnyError, + ) -> Result { match DynamicImage::from_decoder(self) { Ok(image) => Ok(image), - Err(err) => Err(image_decoding_error(err).into()), + Err(err) => Err(error_fn(err)), } } fn get_icc_profile(&mut self) -> Option> { diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 2709df97aad2cb..f984b039bea269 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -89,7 +89,10 @@ where /// Premultiply the alpha channel of the image. pub(crate) fn premultiply_alpha( image: DynamicImage, - unmatch: Option Result>, + unmatch_color_handler: fn( + ColorType, + DynamicImage, + ) -> Result, ) -> Result { let color = image.color(); match color { @@ -105,10 +108,7 @@ pub(crate) fn premultiply_alpha( ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( process_premultiply_alpha(&image.to_rgba16()), )), - x => match unmatch { - Some(unmatch) => unmatch(x), - None => Ok(image), - }, + x => unmatch_color_handler(x, image), } } @@ -217,7 +217,10 @@ where /// Invert the premultiplied alpha channel of the image. pub(crate) fn unpremultiply_alpha( image: DynamicImage, - unmatch: Option Result>, + unmatch_color_handler: fn( + ColorType, + DynamicImage, + ) -> Result, ) -> Result { match image.color() { ColorType::La8 => Ok(DynamicImage::ImageLumaA8( @@ -232,10 +235,7 @@ pub(crate) fn unpremultiply_alpha( ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( process_unpremultiply_alpha(&image.to_rgba16()), )), - x => match unmatch { - Some(unmatch) => unmatch(x), - None => Ok(image), - }, + x => unmatch_color_handler(x, image), } } @@ -387,7 +387,10 @@ where /// Convert the color space of the image from sRGB to Display-P3. pub(crate) fn srgb_to_display_p3( image: DynamicImage, - unmatch: Option Result>, + unmatch_color_handler: fn( + ColorType, + DynamicImage, + ) -> Result, ) -> Result { match image.color() { // The conversion of the lumincance color types to the display-p3 color space is meaningless. @@ -407,10 +410,7 @@ pub(crate) fn srgb_to_display_p3( ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( process_srgb_to_display_p3(&image.to_rgba16()), )), - x => match unmatch { - Some(unmatch) => unmatch(x), - None => Ok(image), - }, + x => unmatch_color_handler(x, image), } } @@ -551,7 +551,7 @@ where pub(crate) fn to_srgb_from_icc_profile( image: DynamicImage, icc_profile: Option>, - unmatch: Option Result>, + unmatch_color_handler: fn(ColorType, DynamicImage) -> Result, ) -> Result { match icc_profile { // If there is no color profile information, return the image as is. @@ -586,10 +586,7 @@ pub(crate) fn to_srgb_from_icc_profile( ColorType::Rgba16 => { Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion::<_,_,8>(&image,icc_profile,srgb_icc_profile))) } - x => match unmatch { - Some(unmatch) => unmatch(x), - None => Ok(image), - }, + x => unmatch_color_handler(x, image), } } }, diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 91cf68a08c5d72..a98469ee265949 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -19,6 +19,7 @@ use image::imageops::overlay; use image::imageops::FilterType; use image::ColorType; use image::DynamicImage; +use image::ImageError; use image::RgbaImage; use serde::Deserialize; use serde::Serialize; @@ -111,55 +112,62 @@ fn decode_bitmap_data( ) -> Result { let (image, width, height, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { + fn image_decoding_error(error: ImageError) -> AnyError { + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &error.to_string(), + )) + .into() + } let (image, icc_profile) = match &*mime_type { // Should we support the "image/apng" MIME type here? "image/png" => { let mut decoder: PngDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "image/jpeg" => { let mut decoder: JpegDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "image/gif" => { let mut decoder: GifDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "image/bmp" => { let mut decoder: BmpDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "image/x-icon" => { let mut decoder: IcoDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "image/webp" => { let mut decoder: WebPDecoder = ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( buf, - )))?; + )), image_decoding_error)?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image()?, icc_profile) + (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) } "" => { return Err( @@ -243,25 +251,31 @@ fn apply_color_space_conversion( ColorSpaceConversion::Default => { match image_bitmap_source { ImageBitmapSource::Blob => { - fn color_unmatch(x: ColorType) -> Result { + fn unmatch_color_handler( + x: ColorType, + _: DynamicImage, + ) -> Result { Err(type_error(image_error_message( "apply colorspaceConversion: default", &format!("The color type {:?} is not supported.", x), ))) } - to_srgb_from_icc_profile(image, icc_profile, Some(color_unmatch)) + to_srgb_from_icc_profile(image, icc_profile, unmatch_color_handler) } ImageBitmapSource::ImageData => match predefined_color_space { // If the color space is sRGB, return the image as is. PredefinedColorSpace::Srgb => Ok(image), PredefinedColorSpace::DisplayP3 => { - fn unmatch(x: ColorType) -> Result { + fn unmatch_color_handler( + x: ColorType, + _: DynamicImage, + ) -> Result { Err(type_error(image_error_message( "apply colorspace: display-p3", &format!("The color type {:?} is not supported.", x), ))) } - srgb_to_display_p3(image, Some(unmatch)) + srgb_to_display_p3(image, unmatch_color_handler) } }, } @@ -278,6 +292,12 @@ fn apply_premultiply_alpha( if !color.has_alpha() { Ok(image) } else { + fn unmatch_color_handler( + _: ColorType, + image: DynamicImage, + ) -> Result { + Ok(image) + } match premultiply_alpha { // 1. PremultiplyAlpha::Default => Ok(image), @@ -285,7 +305,9 @@ fn apply_premultiply_alpha( // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied // 2. - PremultiplyAlpha::Premultiply => process_premultiply_alpha(image, None), + PremultiplyAlpha::Premultiply => { + process_premultiply_alpha(image, unmatch_color_handler) + } // 3. PremultiplyAlpha::None => { // NOTE: It's not clear how to handle the case of ImageData. @@ -295,7 +317,7 @@ fn apply_premultiply_alpha( return Ok(image); } - unpremultiply_alpha(image, None) + unpremultiply_alpha(image, unmatch_color_handler) } } } From 86f0000bb6b7e9360aac61543ed44ac3f09de9df Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:32:34 +0900 Subject: [PATCH 18/52] update README --- ext/canvas/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/canvas/README.md b/ext/canvas/README.md index f731953eb34718..0303d1f739ad22 100644 --- a/ext/canvas/README.md +++ b/ext/canvas/README.md @@ -9,7 +9,7 @@ flowchart LR Input["input binary
( &[u8] )"] II["intermediate image
( DynamicImage )"] Ops["processing pixel
( ImageBuffer< P, S > )"] - Output["output binary
( &[u8] )"] + Output["output binary
( Box<[u8]> )"] Input --> II II --> Ops --> II II --> Output From 75d26df537ead5bbe9434e2f71b031e477d48002 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:26:26 +0900 Subject: [PATCH 19/52] revert "support colorspaceConversion for ImageData" --- ext/canvas/01_image.js | 5 +- ext/canvas/idl.rs | 11 - ext/canvas/image_ops.rs | 400 ++++++++++++++------------- ext/canvas/lib.rs | 1 - ext/canvas/op_create_image_bitmap.rs | 54 +--- ext/web/16_image_data.js | 11 +- tests/unit/image_bitmap_test.ts | 20 -- 7 files changed, 219 insertions(+), 283 deletions(-) delete mode 100644 ext/canvas/idl.rs diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 22ad44687dad84..4f54a2aadfbc75 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -18,7 +18,6 @@ const { ArrayPrototypeJoin, } = primordials; import { - _colorSpace, _data, _height, _width, @@ -237,7 +236,7 @@ function createImageBitmap( let width = 0; let height = 0; let mimeType = ""; - let imageBitmapSource, buf, predefinedColorSpace; + let imageBitmapSource, buf; if (isBlob) { imageBitmapSource = imageBitmapSources[0]; buf = new Uint8Array(await image.arrayBuffer()); @@ -248,7 +247,6 @@ function createImageBitmap( height = image[_height]; imageBitmapSource = imageBitmapSources[1]; buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); - predefinedColorSpace = image[_colorSpace]; } let sx; @@ -267,7 +265,6 @@ function createImageBitmap( sh, imageOrientation: options.imageOrientation ?? "from-image", premultiplyAlpha: options.premultiplyAlpha ?? "default", - predefinedColorSpace: predefinedColorSpace ?? "srgb", colorSpaceConversion: options.colorSpaceConversion ?? "default", resizeWidth: options.resizeWidth, resizeHeight: options.resizeHeight, diff --git a/ext/canvas/idl.rs b/ext/canvas/idl.rs deleted file mode 100644 index 23f33641c9bbf0..00000000000000 --- a/ext/canvas/idl.rs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use serde::Deserialize; - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub enum PredefinedColorSpace { - Srgb, - #[serde(rename = "display-p3")] - DisplayP3, -} diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index f984b039bea269..9e7cb97a9e1ade 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -239,181 +239,6 @@ pub(crate) fn unpremultiply_alpha( } } -// reference -// https://www.w3.org/TR/css-color-4/#color-conversion-code -fn srgb_to_linear(value: T) -> f32 { - if value.to_f32().unwrap() <= 0.04045 { - value.to_f32().unwrap() / 12.92 - } else { - ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) - } -} - -// reference -// https://www.w3.org/TR/css-color-4/#color-conversion-code -fn linear_to_display_p3(value: T) -> f32 { - if value.to_f32().unwrap() <= 0.0031308 { - value.to_f32().unwrap() * 12.92 - } else { - 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 - } -} - -fn normalize_value_to_0_1(value: T) -> f32 { - value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() -} - -fn unnormalize_value_from_0_1(value: f32) -> T { - NumCast::from( - (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), - ) - .unwrap() -} - -fn apply_conversion_matrix_srgb_to_display_p3( - r: T, - g: T, - b: T, -) -> (T, T, T) { - // normalize the value to 0.0 - 1.0 - let (r, g, b) = ( - normalize_value_to_0_1(r), - normalize_value_to_0_1(g), - normalize_value_to_0_1(b), - ); - - // sRGB -> Linear RGB - let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); - - // Display-P3 (RGB) -> Display-P3 (XYZ) - // - // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] - // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html - // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html - - // [ sRGB (D65) to XYZ ] - #[rustfmt::skip] - let (m1x, m1y, m1z) = ( - [0.4124564, 0.3575761, 0.1804375], - [0.2126729, 0.7151522, 0.0721750], - [0.0193339, 0.119_192, 0.9503041], - ); - - let (r, g, b) = ( - r * m1x[0] + g * m1x[1] + b * m1x[2], - r * m1y[0] + g * m1y[1] + b * m1y[2], - r * m1z[0] + g * m1z[1] + b * m1z[2], - ); - - // inv[ P3-D65 (D65) to XYZ ] - #[rustfmt::skip] - let (m2x, m2y, m2z) = ( - [ 2.493_497, -0.931_383_6, -0.402_710_8 ], - [ -0.829_489, 1.762_664_1, 0.023_624_687 ], - [ 0.035_845_83, -0.076_172_39, 0.956_884_5 ], - ); - - let (r, g, b) = ( - r * m2x[0] + g * m2x[1] + b * m2x[2], - r * m2y[0] + g * m2y[1] + b * m2y[2], - r * m2z[0] + g * m2z[1] + b * m2z[2], - ); - - // This calculation is similar as above that it is a little faster, but less accurate. - // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; - // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; - // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; - - // Display-P3 (Linear) -> Display-P3 - let (r, g, b) = ( - linear_to_display_p3(r), - linear_to_display_p3(g), - linear_to_display_p3(b), - ); - - // unnormalize the value from 0.0 - 1.0 - ( - unnormalize_value_from_0_1(r), - unnormalize_value_from_0_1(g), - unnormalize_value_from_0_1(b), - ) -} - -pub(crate) trait ColorSpaceConversion { - /// Display P3 Color Encoding (v 1.0) - /// https://www.color.org/chardata/rgb/DisplayP3.xalter - fn srgb_to_display_p3(&self) -> Self; -} - -impl ColorSpaceConversion for Rgb { - fn srgb_to_display_p3(&self) -> Self { - let (r, g, b) = (self.0[0], self.0[1], self.0[2]); - - let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); - - Rgb::([r, g, b]) - } -} - -impl ColorSpaceConversion for Rgba { - fn srgb_to_display_p3(&self) -> Self { - let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); - - let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); - - Rgba::([r, g, b, a]) - } -} - -// make public if needed -fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> -where - I: GenericImageView, - P: Pixel + ColorSpaceConversion + 'static, - S: Primitive + 'static, -{ - let (width, height) = image.dimensions(); - let mut out = ImageBuffer::new(width, height); - - for (x, y, pixel) in image.pixels() { - let pixel = pixel.srgb_to_display_p3(); - - out.put_pixel(x, y, pixel); - } - - out -} - -/// Convert the color space of the image from sRGB to Display-P3. -pub(crate) fn srgb_to_display_p3( - image: DynamicImage, - unmatch_color_handler: fn( - ColorType, - DynamicImage, - ) -> Result, -) -> Result { - match image.color() { - // The conversion of the lumincance color types to the display-p3 color space is meaningless. - ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), - ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), - ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), - ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( - &image.to_rgb8(), - ))), - ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( - process_srgb_to_display_p3(&image.to_rgb16()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_srgb_to_display_p3(&image.to_rgba8()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_srgb_to_display_p3(&image.to_rgba16()), - )), - x => unmatch_color_handler(x, image), - } -} - pub(crate) trait SliceToPixel { fn slice_to_pixel(pixel: &[u8]) -> Self; } @@ -593,6 +418,185 @@ pub(crate) fn to_srgb_from_icc_profile( } } +// NOTE: The following code is not used in the current implementation, +// but it is left as a reference for future use about implementing CanvasRenderingContext2D. +// https://github.com/denoland/deno/issues/5701#issuecomment-1833304511 + +// // reference +// // https://www.w3.org/TR/css-color-4/#color-conversion-code +// fn srgb_to_linear(value: T) -> f32 { +// if value.to_f32().unwrap() <= 0.04045 { +// value.to_f32().unwrap() / 12.92 +// } else { +// ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) +// } +// } + +// // reference +// // https://www.w3.org/TR/css-color-4/#color-conversion-code +// fn linear_to_display_p3(value: T) -> f32 { +// if value.to_f32().unwrap() <= 0.0031308 { +// value.to_f32().unwrap() * 12.92 +// } else { +// 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 +// } +// } + +// fn normalize_value_to_0_1(value: T) -> f32 { +// value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() +// } + +// fn unnormalize_value_from_0_1(value: f32) -> T { +// NumCast::from( +// (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), +// ) +// .unwrap() +// } + +// fn apply_conversion_matrix_srgb_to_display_p3( +// r: T, +// g: T, +// b: T, +// ) -> (T, T, T) { +// // normalize the value to 0.0 - 1.0 +// let (r, g, b) = ( +// normalize_value_to_0_1(r), +// normalize_value_to_0_1(g), +// normalize_value_to_0_1(b), +// ); + +// // sRGB -> Linear RGB +// let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); + +// // Display-P3 (RGB) -> Display-P3 (XYZ) +// // +// // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] +// // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html +// // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html + +// // [ sRGB (D65) to XYZ ] +// #[rustfmt::skip] +// let (m1x, m1y, m1z) = ( +// [0.4124564, 0.3575761, 0.1804375], +// [0.2126729, 0.7151522, 0.0721750], +// [0.0193339, 0.119_192, 0.9503041], +// ); + +// let (r, g, b) = ( +// r * m1x[0] + g * m1x[1] + b * m1x[2], +// r * m1y[0] + g * m1y[1] + b * m1y[2], +// r * m1z[0] + g * m1z[1] + b * m1z[2], +// ); + +// // inv[ P3-D65 (D65) to XYZ ] +// #[rustfmt::skip] +// let (m2x, m2y, m2z) = ( +// [ 2.493_497, -0.931_383_6, -0.402_710_8 ], +// [ -0.829_489, 1.762_664_1, 0.023_624_687 ], +// [ 0.035_845_83, -0.076_172_39, 0.956_884_5 ], +// ); + +// let (r, g, b) = ( +// r * m2x[0] + g * m2x[1] + b * m2x[2], +// r * m2y[0] + g * m2y[1] + b * m2y[2], +// r * m2z[0] + g * m2z[1] + b * m2z[2], +// ); + +// // This calculation is similar as above that it is a little faster, but less accurate. +// // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; +// // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; +// // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; + +// // Display-P3 (Linear) -> Display-P3 +// let (r, g, b) = ( +// linear_to_display_p3(r), +// linear_to_display_p3(g), +// linear_to_display_p3(b), +// ); + +// // unnormalize the value from 0.0 - 1.0 +// ( +// unnormalize_value_from_0_1(r), +// unnormalize_value_from_0_1(g), +// unnormalize_value_from_0_1(b), +// ) +// } + +// trait ColorSpaceConversion { +// /// Display P3 Color Encoding (v 1.0) +// /// https://www.color.org/chardata/rgb/DisplayP3.xalter +// fn srgb_to_display_p3(&self) -> Self; +// } + +// impl ColorSpaceConversion for Rgb { +// fn srgb_to_display_p3(&self) -> Self { +// let (r, g, b) = (self.0[0], self.0[1], self.0[2]); + +// let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); + +// Rgb::([r, g, b]) +// } +// } + +// impl ColorSpaceConversion for Rgba { +// fn srgb_to_display_p3(&self) -> Self { +// let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); + +// let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); + +// Rgba::([r, g, b, a]) +// } +// } + +// // make public if needed +// fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> +// where +// I: GenericImageView, +// P: Pixel + ColorSpaceConversion + 'static, +// S: Primitive + 'static, +// { +// let (width, height) = image.dimensions(); +// let mut out = ImageBuffer::new(width, height); + +// for (x, y, pixel) in image.pixels() { +// let pixel = pixel.srgb_to_display_p3(); + +// out.put_pixel(x, y, pixel); +// } + +// out +// } + +// /// Convert the color space of the image from sRGB to Display-P3. +// fn srgb_to_display_p3( +// image: DynamicImage, +// unmatch_color_handler: fn( +// ColorType, +// DynamicImage, +// ) -> Result, +// ) -> Result { +// match image.color() { +// // The conversion of the lumincance color types to the display-p3 color space is meaningless. +// ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), +// ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), +// ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), +// ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), +// ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( +// &image.to_rgb8(), +// ))), +// ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( +// process_srgb_to_display_p3(&image.to_rgb16()), +// )), +// ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( +// process_srgb_to_display_p3(&image.to_rgba8()), +// )), +// ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( +// process_srgb_to_display_p3(&image.to_rgba16()), +// )), +// x => unmatch_color_handler(x, image), +// } +// } + #[cfg(test)] mod tests { use super::*; @@ -616,27 +620,27 @@ mod tests { assert_eq!(rgba, Rgba::([255, 0, 0, 127])); } - #[test] - fn test_apply_conversion_matrix_srgb_to_display_p3() { - let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(255_u8, 0, 0); - assert_eq!(r, 234); - assert_eq!(g, 51); - assert_eq!(b, 35); - - let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 255, 0); - assert_eq!(r, 117); - assert_eq!(g, 251); - assert_eq!(b, 76); - - let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 0, 255); - assert_eq!(r, 0); - assert_eq!(g, 0); - assert_eq!(b, 245); - - let (r, g, b) = - apply_conversion_matrix_srgb_to_display_p3(255_u8, 255, 255); - assert_eq!(r, 255); - assert_eq!(g, 255); - assert_eq!(b, 255); - } + // #[test] + // fn test_apply_conversion_matrix_srgb_to_display_p3() { + // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(255_u8, 0, 0); + // assert_eq!(r, 234); + // assert_eq!(g, 51); + // assert_eq!(b, 35); + + // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 255, 0); + // assert_eq!(r, 117); + // assert_eq!(g, 251); + // assert_eq!(b, 76); + + // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 0, 255); + // assert_eq!(r, 0); + // assert_eq!(g, 0); + // assert_eq!(b, 245); + + // let (r, g, b) = + // apply_conversion_matrix_srgb_to_display_p3(255_u8, 255, 255); + // assert_eq!(r, 255); + // assert_eq!(g, 255); + // assert_eq!(b, 255); + // } } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index a5d7f32c38c293..462c85755a66ea 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; pub mod error; -pub mod idl; mod image_decoder; mod image_ops; mod op_create_image_bitmap; diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index a98469ee265949..e4334d9b20ee7d 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -26,11 +26,9 @@ use serde::Serialize; use crate::error::image_error_message; use crate::error::DOMExceptionInvalidStateError; -use crate::idl::PredefinedColorSpace; use crate::image_decoder::ImageDecoderFromReader; use crate::image_decoder::ImageDecoderFromReaderType; use crate::image_ops::premultiply_alpha as process_premultiply_alpha; -use crate::image_ops::srgb_to_display_p3; use crate::image_ops::to_srgb_from_icc_profile; use crate::image_ops::unpremultiply_alpha; @@ -84,7 +82,6 @@ struct OpCreateImageBitmapArgs { sh: Option, image_orientation: ImageOrientation, premultiply_alpha: PremultiplyAlpha, - predefined_color_space: PredefinedColorSpace, color_space_conversion: ColorSpaceConversion, resize_width: Option, resize_height: Option, @@ -241,44 +238,22 @@ See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n" fn apply_color_space_conversion( image: DynamicImage, icc_profile: Option>, - image_bitmap_source: &ImageBitmapSource, color_space_conversion: &ColorSpaceConversion, - predefined_color_space: &PredefinedColorSpace, ) -> Result { match color_space_conversion { // return the decoded image as is. ColorSpaceConversion::None => Ok(image), ColorSpaceConversion::Default => { - match image_bitmap_source { - ImageBitmapSource::Blob => { - fn unmatch_color_handler( - x: ColorType, - _: DynamicImage, - ) -> Result { - Err(type_error(image_error_message( - "apply colorspaceConversion: default", - &format!("The color type {:?} is not supported.", x), - ))) - } - to_srgb_from_icc_profile(image, icc_profile, unmatch_color_handler) - } - ImageBitmapSource::ImageData => match predefined_color_space { - // If the color space is sRGB, return the image as is. - PredefinedColorSpace::Srgb => Ok(image), - PredefinedColorSpace::DisplayP3 => { - fn unmatch_color_handler( - x: ColorType, - _: DynamicImage, - ) -> Result { - Err(type_error(image_error_message( - "apply colorspace: display-p3", - &format!("The color type {:?} is not supported.", x), - ))) - } - srgb_to_display_p3(image, unmatch_color_handler) - } - }, + fn unmatch_color_handler( + x: ColorType, + _: DynamicImage, + ) -> Result { + Err(type_error(image_error_message( + "apply colorspaceConversion: default", + &format!("The color type {:?} is not supported.", x), + ))) } + to_srgb_from_icc_profile(image, icc_profile, unmatch_color_handler) } } } @@ -339,7 +314,6 @@ pub(super) fn op_create_image_bitmap( sy, image_orientation, premultiply_alpha, - predefined_color_space, color_space_conversion, resize_width, resize_height, @@ -355,7 +329,6 @@ pub(super) fn op_create_image_bitmap( sh: args.sh, image_orientation: args.image_orientation, premultiply_alpha: args.premultiply_alpha, - predefined_color_space: args.predefined_color_space, color_space_conversion: args.color_space_conversion, resize_width: args.resize_width, resize_height: args.resize_height, @@ -455,13 +428,8 @@ pub(super) fn op_create_image_bitmap( }; // 9. - let image = apply_color_space_conversion( - image, - icc_profile, - &image_bitmap_source, - &color_space_conversion, - &predefined_color_space, - )?; + let image = + apply_color_space_conversion(image, icc_profile, &color_space_conversion)?; // 10. let image = diff --git a/ext/web/16_image_data.js b/ext/web/16_image_data.js index 1926aeffdc3eb9..2048f002d5923e 100644 --- a/ext/web/16_image_data.js +++ b/ext/web/16_image_data.js @@ -31,7 +31,6 @@ webidl.converters["ImageDataSettings"] = webidl.createDictionaryConverter( const _data = Symbol("[[data]]"); const _width = Symbol("[[width]]"); const _height = Symbol("[[height]]"); -const _colorSpace = Symbol("[[colorSpace]]"); class ImageData { /** @type {number} */ [_width]; @@ -40,7 +39,7 @@ class ImageData { /** @type {Uint8Array} */ [_data]; /** @type {'srgb' | 'display-p3'} */ - [_colorSpace]; + #colorSpace; constructor(arg0, arg1, arg2 = undefined, arg3 = undefined) { webidl.requiredArguments( @@ -134,7 +133,7 @@ class ImageData { this[_height] = sourceHeight; } - this[_colorSpace] = settings.colorSpace ?? "srgb"; + this.#colorSpace = settings.colorSpace ?? "srgb"; this[_width] = sourceWidth; this[_data] = data; return; @@ -172,7 +171,7 @@ class ImageData { ); } - this[_colorSpace] = settings.colorSpace ?? "srgb"; + this.#colorSpace = settings.colorSpace ?? "srgb"; this[_width] = sourceWidth; this[_height] = sourceHeight; this[_data] = new Uint8ClampedArray(sourceWidth * sourceHeight * 4); @@ -195,7 +194,7 @@ class ImageData { get colorSpace() { webidl.assertBranded(this, ImageDataPrototype); - return this[_colorSpace]; + return this.#colorSpace; } [SymbolFor("Deno.privateCustomInspect")](inspect, inspectOptions) { @@ -217,4 +216,4 @@ class ImageData { const ImageDataPrototype = ImageData.prototype; -export { _colorSpace, _data, _height, _width, ImageData, ImageDataPrototype }; +export { _data, _height, _width, ImageData, ImageDataPrototype }; diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index d244431b6e471b..67d163852e19b9 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -299,26 +299,6 @@ Deno.test("imageBitmapFromBlobAnimatedImage", async (t) => { }); }); -Deno.test(async function imageBitmapImageDataColorspaceConversion() { - const imageData = new ImageData( - new Uint8ClampedArray([ - 255, - 0, - 0, - 255, - ]), - 1, - 1, - { - colorSpace: "display-p3", - }, - ); - const imageBitmap = await createImageBitmap(imageData); - // @ts-ignore: Deno[Deno.internal].core allowed - // deno-fmt-ignore - assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([234, 51, 35, 255])); -}); - /** * extract high bytes from Uint16Array */ From 1de527ff7d6af54aac25c4f9e25c0ed762832fe5 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 15 Sep 2024 14:07:37 +0900 Subject: [PATCH 20/52] switch to static link for lcms2 --- ext/canvas/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 6422409dda9e04..7266923ba6f2c4 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -22,6 +22,8 @@ image = { version = "0.25.2", default-features = false, features = ["png", "jpeg # NOTE: The qcms is a color space conversion crate which parses ICC profiles that used in Gecko, # however it supports only 8-bit color depth currently. # https://searchfox.org/mozilla-central/rev/f09e3f9603a08b5b51bf504846091579bc2ff531/gfx/qcms/src/transform.rs#130-137 -lcms2 = "6.1.0" +# It seems to be failed to build for aarch64-unknown-linux-gnu with pkg-config. +# https://github.com/kornelski/rust-lcms2-sys/blob/b8e9c3efcf266b88600318fb519c073b9ebb61b7/README.md#L26 +lcms2 = { version = "6.1.0", features = ["static"] } num-traits = { version = "0.2.19" } serde = { workspace = true, features = ["derive"] } From bd77fb92f14dcc89422fd79b66528cc67a509f83 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sat, 21 Sep 2024 15:59:14 +0900 Subject: [PATCH 21/52] add comments for the reference of image mimesniff --- ext/canvas/op_create_image_bitmap.rs | 3 --- ext/web/01_mimesniff.js | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index e4334d9b20ee7d..3ce5d13b9ef483 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -179,9 +179,6 @@ See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", } // return an error if the MIME type is not supported in the variable list of ImageTypePatternTable below // ext/web/01_mimesniff.js - // - // NOTE: Chromium supports AVIF - // https://source.chromium.org/chromium/chromium/src/+/ef3f4e4ed97079dc57861d1195fb2389483bc195:third_party/blink/renderer/platform/image-decoders/image_decoder.cc;l=311 x => { return Err( DOMExceptionInvalidStateError::new( diff --git a/ext/web/01_mimesniff.js b/ext/web/01_mimesniff.js index e60783bbe171ac..67a0a24d639604 100644 --- a/ext/web/01_mimesniff.js +++ b/ext/web/01_mimesniff.js @@ -395,6 +395,10 @@ const ImageTypePatternTable = [ /** * Ref: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm + * NOTE: Some browsers have implementation-defined image formats. + * For example, The AVIF image format is supported by all browsers today. + * However, the standardization seems to have hard going. + * See: https://github.com/whatwg/mimesniff/issues/143 * @param {Uint8Array} input * @returns {string | undefined} */ From 2d58fd1d62c6b236b36db20fbf1f9d91ff1d652a Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 10:19:31 +0900 Subject: [PATCH 22/52] clean up `process_icc_profile_conversion` --- ext/canvas/image_ops.rs | 109 ++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 9e7cb97a9e1ade..d8240980dd9235 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use bytemuck::cast_slice; +use bytemuck::cast_slice_mut; use deno_core::error::AnyError; use image::ColorType; use image::DynamicImage; @@ -279,54 +280,60 @@ impl SliceToPixel for Rgba { } } -/// Convert the pixel slice to an array to avoid the copy to Vec. -/// I implemented this trait because of I couldn't find a way to effectively combine -/// the `Transform` struct of `lcms2` and `Pixel` trait of `image`. -/// If there is an implementation that is safer and can withstand changes, I would like to adopt it. -pub(crate) trait SliceToArray { - fn slice_to_array(pixel: &[u8]) -> [u8; N]; +pub(crate) trait TransformColorProfile { + fn transform_color_profile( + &mut self, + transformer: &Transform, + ) -> P + where + P: Pixel + SliceToPixel + 'static, + S: Primitive + 'static; } -macro_rules! impl_slice_to_array { - ($type:ty, $n:expr) => { - impl SliceToArray<$n> for $type { - fn slice_to_array(pixel: &[u8]) -> [u8; $n] { - let mut dst = [0_u8; $n]; - dst.copy_from_slice(&pixel[..$n]); - - dst +macro_rules! impl_transform_color_profile { + ($type:ty) => { + impl TransformColorProfile for $type { + fn transform_color_profile( + &mut self, + transformer: &Transform, + ) -> P + where + P: Pixel + SliceToPixel + 'static, + S: Primitive + 'static, + { + let mut pixel = cast_slice_mut(self.0.as_mut_slice()); + transformer.transform_in_place(&mut pixel); + + P::slice_to_pixel(&pixel) } } }; } -impl_slice_to_array!(Luma, 1); -impl_slice_to_array!(Luma, 2); -impl_slice_to_array!(LumaA, 2); -impl_slice_to_array!(LumaA, 4); -impl_slice_to_array!(Rgb, 3); -impl_slice_to_array!(Rgb, 6); -impl_slice_to_array!(Rgba, 4); -impl_slice_to_array!(Rgba, 8); +impl_transform_color_profile!(Luma); +impl_transform_color_profile!(Luma); +impl_transform_color_profile!(LumaA); +impl_transform_color_profile!(LumaA); +impl_transform_color_profile!(Rgb); +impl_transform_color_profile!(Rgb); +impl_transform_color_profile!(Rgba); +impl_transform_color_profile!(Rgba); // make public if needed -fn process_icc_profile_conversion( - image: &DynamicImage, +fn process_icc_profile_conversion( + image: &I, + color: ColorType, input_icc_profile: Profile, output_icc_profile: Profile, ) -> ImageBuffer> where - P: Pixel + SliceToPixel + SliceToArray + 'static, + I: GenericImageView, + P: Pixel + SliceToPixel + TransformColorProfile + 'static, S: Primitive + 'static, { let (width, height) = image.dimensions(); let mut out = ImageBuffer::new(width, height); - let chunk_size = image.color().bytes_per_pixel() as usize; - let pixel_iter = image - .as_bytes() - .chunks_exact(chunk_size) - .zip(image.pixels()); - let pixel_format = match image.color() { + let pixel_format = match color { ColorType::L8 => PixelFormat::GRAY_8, ColorType::L16 => PixelFormat::GRAY_16, ColorType::La8 => PixelFormat::GRAYA_8, @@ -337,12 +344,10 @@ where ColorType::Rgba16 => PixelFormat::RGBA_16, // This arm usually doesn't reach, but it should be handled with returning the original image. _ => { - return { - for (pixel, (x, y, _)) in pixel_iter { - out.put_pixel(x, y, P::slice_to_pixel(pixel)); - } - out + for (x, y, pixel) in image.pixels() { + out.put_pixel(x, y, pixel); } + return out; } }; let transformer = Transform::new( @@ -353,19 +358,14 @@ where output_icc_profile.header_rendering_intent(), ); - for (pixel, (x, y, _)) in pixel_iter { + for (x, y, mut pixel) in image.pixels() { let pixel = match transformer { - Ok(ref transformer) => { - let mut dst = P::slice_to_array(pixel); - transformer.transform_in_place(&mut dst); - - dst - } + Ok(ref transformer) => pixel.transform_color_profile(transformer), // This arm will reach when the ffi call fails. - Err(_) => P::slice_to_array(pixel), + Err(_) => pixel, }; - out.put_pixel(x, y, P::slice_to_pixel(&pixel)); + out.put_pixel(x, y, pixel); } out @@ -386,30 +386,31 @@ pub(crate) fn to_srgb_from_icc_profile( Err(_) => Ok(image), Ok(icc_profile) => { let srgb_icc_profile = Profile::new_srgb(); - match image.color() { + let color = image.color(); + match color { ColorType::L8 => { - Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion::<_,_,1>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion(&image.to_luma8(), color, icc_profile,srgb_icc_profile))) } ColorType::L16 => { - Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion::<_,_,2>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion(&image.to_luma16(),color, icc_profile,srgb_icc_profile))) } ColorType::La8 => { - Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion::<_,_,2>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion(&image.to_luma_alpha8(), color, icc_profile,srgb_icc_profile))) } ColorType::La16 => { - Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion::<_, _, 4>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion(&image.to_luma_alpha16(), color, icc_profile,srgb_icc_profile))) }, ColorType::Rgb8 => { - Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion::<_,_,3>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion(&image.to_rgb8(), color, icc_profile,srgb_icc_profile))) } ColorType::Rgb16 => { - Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion::<_,_,6>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion(&image.to_rgb16(), color, icc_profile,srgb_icc_profile))) } ColorType::Rgba8 => { - Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion::<_,_,4>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion(&image.to_rgba8(), color, icc_profile,srgb_icc_profile))) } ColorType::Rgba16 => { - Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion::<_,_,8>(&image,icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion(&image.to_rgba16(),color, icc_profile,srgb_icc_profile))) } x => unmatch_color_handler(x, image), } From d2589142f43707d17becdbebd2f368fce9e1104a Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 10:21:35 +0900 Subject: [PATCH 23/52] clean up --- ext/canvas/image_ops.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index d8240980dd9235..5c4cf94998f02e 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -34,7 +34,7 @@ impl PremultiplyAlpha for LumaA { let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); if normalized_alpha == 0.0 { - return LumaA::([pixel[0], pixel[alpha_index]]); + return LumaA([pixel[0], pixel[alpha_index]]); } for rgb in pixel.iter_mut().take(alpha_index) { @@ -42,7 +42,7 @@ impl PremultiplyAlpha for LumaA { .unwrap() } - LumaA::([pixel[0], pixel[alpha_index]]) + LumaA([pixel[0], pixel[alpha_index]]) } } @@ -56,7 +56,7 @@ impl PremultiplyAlpha for Rgba { let normalized_alpha = alpha.to_f32().unwrap() / max_t.to_f32().unwrap(); if normalized_alpha == 0.0 { - return Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]); + return Rgba([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]); } for rgb in pixel.iter_mut().take(alpha_index) { @@ -64,7 +64,7 @@ impl PremultiplyAlpha for Rgba { .unwrap() } - Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + Rgba([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) } } @@ -152,7 +152,7 @@ impl UnpremultiplyAlpha for Rgba { .unwrap(); } - Rgba::([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) + Rgba([pixel[0], pixel[1], pixel[2], pixel[alpha_index]]) } } @@ -183,7 +183,7 @@ impl UnpremultiplyAlpha for LumaA { .unwrap(); } - LumaA::([pixel[0], pixel[alpha_index]]) + LumaA([pixel[0], pixel[alpha_index]]) } } @@ -249,7 +249,7 @@ impl SliceToPixel for Luma { let pixel: &[T] = cast_slice(pixel); let pixel = [pixel[0]]; - Luma::(pixel) + Luma(pixel) } } @@ -258,7 +258,7 @@ impl SliceToPixel for LumaA { let pixel: &[T] = cast_slice(pixel); let pixel = [pixel[0], pixel[1]]; - LumaA::(pixel) + LumaA(pixel) } } @@ -267,7 +267,7 @@ impl SliceToPixel for Rgb { let pixel: &[T] = cast_slice(pixel); let pixel = [pixel[0], pixel[1], pixel[2]]; - Rgb::(pixel) + Rgb(pixel) } } @@ -276,7 +276,7 @@ impl SliceToPixel for Rgba { let pixel: &[T] = cast_slice(pixel); let pixel = [pixel[0], pixel[1], pixel[2], pixel[3]]; - Rgba::(pixel) + Rgba(pixel) } } @@ -535,7 +535,7 @@ pub(crate) fn to_srgb_from_icc_profile( // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); -// Rgb::([r, g, b]) +// Rgb([r, g, b]) // } // } @@ -545,7 +545,7 @@ pub(crate) fn to_srgb_from_icc_profile( // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); -// Rgba::([r, g, b, a]) +// Rgba([r, g, b, a]) // } // } From fa0132042e06510a65ad3f994e3680b0a335b05e Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 13:11:46 +0900 Subject: [PATCH 24/52] fmt --- ext/canvas/image_ops.rs | 64 ++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 5c4cf94998f02e..4db7243195be7c 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -371,12 +371,14 @@ where out } -#[rustfmt::skip] /// Convert the color space of the image from the ICC profile to sRGB. pub(crate) fn to_srgb_from_icc_profile( image: DynamicImage, icc_profile: Option>, - unmatch_color_handler: fn(ColorType, DynamicImage) -> Result, + unmatch_color_handler: fn( + ColorType, + DynamicImage, + ) -> Result, ) -> Result { match icc_profile { // If there is no color profile information, return the image as is. @@ -389,28 +391,68 @@ pub(crate) fn to_srgb_from_icc_profile( let color = image.color(); match color { ColorType::L8 => { - Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion(&image.to_luma8(), color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion( + &image.to_luma8(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::L16 => { - Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion(&image.to_luma16(),color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion( + &image.to_luma16(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::La8 => { - Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion(&image.to_luma_alpha8(), color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion( + &image.to_luma_alpha8(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::La16 => { - Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion(&image.to_luma_alpha16(), color, icc_profile,srgb_icc_profile))) - }, + Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion( + &image.to_luma_alpha16(), + color, + icc_profile, + srgb_icc_profile, + ))) + } ColorType::Rgb8 => { - Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion(&image.to_rgb8(), color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion( + &image.to_rgb8(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::Rgb16 => { - Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion(&image.to_rgb16(), color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion( + &image.to_rgb16(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::Rgba8 => { - Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion(&image.to_rgba8(), color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion( + &image.to_rgba8(), + color, + icc_profile, + srgb_icc_profile, + ))) } ColorType::Rgba16 => { - Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion(&image.to_rgba16(),color, icc_profile,srgb_icc_profile))) + Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion( + &image.to_rgba16(), + color, + icc_profile, + srgb_icc_profile, + ))) } x => unmatch_color_handler(x, image), } From 8d8d726e00e8ea9f1c914a9ef96aa45ee2f804eb Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 13:42:13 +0900 Subject: [PATCH 25/52] avoid copying vector --- ext/canvas/image_ops.rs | 100 +++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 4db7243195be7c..2d5017c233d6d2 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -68,7 +68,6 @@ impl PremultiplyAlpha for Rgba { } } -// make public if needed fn process_premultiply_alpha(image: &I) -> ImageBuffer> where I: GenericImageView, @@ -98,16 +97,16 @@ pub(crate) fn premultiply_alpha( let color = image.color(); match color { ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( - &image.to_luma_alpha8(), + image.as_luma_alpha8().unwrap(), ))), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_premultiply_alpha(&image.to_rgba8()), - )), ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_premultiply_alpha(&image.to_luma_alpha16()), + process_premultiply_alpha(image.as_luma_alpha16().unwrap()), + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + process_premultiply_alpha(image.as_rgba8().unwrap()), )), ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_premultiply_alpha(&image.to_rgba16()), + process_premultiply_alpha(image.as_rgba16().unwrap()), )), x => unmatch_color_handler(x, image), } @@ -187,7 +186,17 @@ impl UnpremultiplyAlpha for LumaA { } } -// make public if needed +fn is_premultiplied_alpha(image: &I) -> bool +where + I: GenericImageView, + P: Pixel + UnpremultiplyAlpha + 'static, + S: Primitive + 'static, +{ + image + .pixels() + .any(|(_, _, pixel)| pixel.is_premultiplied_alpha()) +} + fn process_unpremultiply_alpha(image: &I) -> ImageBuffer> where I: GenericImageView, @@ -197,17 +206,8 @@ where let (width, height) = image.dimensions(); let mut out = ImageBuffer::new(width, height); - let is_premultiplied_alpha = image - .pixels() - .any(|(_, _, pixel)| pixel.is_premultiplied_alpha()); - for (x, y, pixel) in image.pixels() { - let pixel = if is_premultiplied_alpha { - pixel.unpremultiply_alpha() - } else { - // return the original - pixel - }; + let pixel = pixel.unpremultiply_alpha(); out.put_pixel(x, y, pixel); } @@ -225,16 +225,32 @@ pub(crate) fn unpremultiply_alpha( ) -> Result { match image.color() { ColorType::La8 => Ok(DynamicImage::ImageLumaA8( - process_unpremultiply_alpha(&image.to_luma_alpha8()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_unpremultiply_alpha(&image.to_rgba8()), + if is_premultiplied_alpha(image.as_luma_alpha8().unwrap()) { + process_unpremultiply_alpha(image.as_luma_alpha8().unwrap()) + } else { + image.into_luma_alpha8() + }, )), ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_unpremultiply_alpha(&image.to_luma_alpha16()), + if is_premultiplied_alpha(image.as_luma_alpha16().unwrap()) { + process_unpremultiply_alpha(image.as_luma_alpha16().unwrap()) + } else { + image.into_luma_alpha16() + }, + )), + ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( + if is_premultiplied_alpha(image.as_rgba8().unwrap()) { + process_unpremultiply_alpha(image.as_rgba8().unwrap()) + } else { + image.into_rgba8() + }, )), ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_unpremultiply_alpha(&image.to_rgba16()), + if is_premultiplied_alpha(image.as_rgba16().unwrap()) { + process_unpremultiply_alpha(image.as_rgba16().unwrap()) + } else { + image.into_rgba16() + }, )), x => unmatch_color_handler(x, image), } @@ -319,7 +335,6 @@ impl_transform_color_profile!(Rgb); impl_transform_color_profile!(Rgba); impl_transform_color_profile!(Rgba); -// make public if needed fn process_icc_profile_conversion( image: &I, color: ColorType, @@ -392,7 +407,7 @@ pub(crate) fn to_srgb_from_icc_profile( match color { ColorType::L8 => { Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion( - &image.to_luma8(), + image.as_luma8().unwrap(), color, icc_profile, srgb_icc_profile, @@ -400,7 +415,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::L16 => { Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion( - &image.to_luma16(), + image.as_luma16().unwrap(), color, icc_profile, srgb_icc_profile, @@ -408,7 +423,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::La8 => { Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion( - &image.to_luma_alpha8(), + image.as_luma_alpha8().unwrap(), color, icc_profile, srgb_icc_profile, @@ -416,7 +431,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::La16 => { Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion( - &image.to_luma_alpha16(), + image.as_luma_alpha16().unwrap(), color, icc_profile, srgb_icc_profile, @@ -424,7 +439,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::Rgb8 => { Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion( - &image.to_rgb8(), + image.as_rgb8().unwrap(), color, icc_profile, srgb_icc_profile, @@ -432,7 +447,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::Rgb16 => { Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion( - &image.to_rgb16(), + image.as_rgb16().unwrap(), color, icc_profile, srgb_icc_profile, @@ -440,7 +455,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::Rgba8 => { Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion( - &image.to_rgba8(), + image.as_rgba8().unwrap(), color, icc_profile, srgb_icc_profile, @@ -448,7 +463,7 @@ pub(crate) fn to_srgb_from_icc_profile( } ColorType::Rgba16 => { Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion( - &image.to_rgba16(), + image.as_rgba16().unwrap(), color, icc_profile, srgb_icc_profile, @@ -591,7 +606,6 @@ pub(crate) fn to_srgb_from_icc_profile( // } // } -// // make public if needed // fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> // where // I: GenericImageView, @@ -620,21 +634,23 @@ pub(crate) fn to_srgb_from_icc_profile( // ) -> Result { // match image.color() { // // The conversion of the lumincance color types to the display-p3 color space is meaningless. -// ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.to_luma8())), -// ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.to_luma16())), -// ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.to_luma_alpha8())), -// ColorType::La16 => Ok(DynamicImage::ImageLumaA16(image.to_luma_alpha16())), +// ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.into_luma8())), +// ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.into_luma16())), +// ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.into_luma_alpha8())), +// ColorType::La16 => { +// Ok(DynamicImage::ImageLumaA16(image.into_luma_alpha16())) +// } // ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( -// &image.to_rgb8(), +// image.as_rgb8().unwrap(), // ))), // ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( -// process_srgb_to_display_p3(&image.to_rgb16()), +// process_srgb_to_display_p3(image.as_rgb16().unwrap()), // )), // ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( -// process_srgb_to_display_p3(&image.to_rgba8()), +// process_srgb_to_display_p3(image.as_rgba8().unwrap()), // )), // ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( -// process_srgb_to_display_p3(&image.to_rgba16()), +// process_srgb_to_display_p3(image.as_rgba16().unwrap()), // )), // x => unmatch_color_handler(x, image), // } From fc4064e0d73904244f5c3bc470453c9d8d7cd5eb Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 13:53:19 +0900 Subject: [PATCH 26/52] `image_ops` takes responsibility of treat alpha channel --- ext/canvas/image_ops.rs | 17 ++++------- ext/canvas/op_create_image_bitmap.rs | 45 ++++++++++------------------ 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 2d5017c233d6d2..cbe8b63174031b 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -89,13 +89,8 @@ where /// Premultiply the alpha channel of the image. pub(crate) fn premultiply_alpha( image: DynamicImage, - unmatch_color_handler: fn( - ColorType, - DynamicImage, - ) -> Result, ) -> Result { - let color = image.color(); - match color { + match image.color() { ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( image.as_luma_alpha8().unwrap(), ))), @@ -108,7 +103,8 @@ pub(crate) fn premultiply_alpha( ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( process_premultiply_alpha(image.as_rgba16().unwrap()), )), - x => unmatch_color_handler(x, image), + // If the image does not have an alpha channel, return the image as is. + _ => Ok(image), } } @@ -218,10 +214,6 @@ where /// Invert the premultiplied alpha channel of the image. pub(crate) fn unpremultiply_alpha( image: DynamicImage, - unmatch_color_handler: fn( - ColorType, - DynamicImage, - ) -> Result, ) -> Result { match image.color() { ColorType::La8 => Ok(DynamicImage::ImageLumaA8( @@ -252,7 +244,8 @@ pub(crate) fn unpremultiply_alpha( image.into_rgba16() }, )), - x => unmatch_color_handler(x, image), + // If the image does not have an alpha channel, return the image as is. + _ => Ok(image), } } diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 3ce5d13b9ef483..c11fb518d82d2e 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -260,37 +260,24 @@ fn apply_premultiply_alpha( image_bitmap_source: &ImageBitmapSource, premultiply_alpha: &PremultiplyAlpha, ) -> Result { - let color = image.color(); - if !color.has_alpha() { - Ok(image) - } else { - fn unmatch_color_handler( - _: ColorType, - image: DynamicImage, - ) -> Result { - Ok(image) - } - match premultiply_alpha { - // 1. - PremultiplyAlpha::Default => Ok(image), - - // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied - - // 2. - PremultiplyAlpha::Premultiply => { - process_premultiply_alpha(image, unmatch_color_handler) + match premultiply_alpha { + // 1. + PremultiplyAlpha::Default => Ok(image), + + // https://html.spec.whatwg.org/multipage/canvas.html#convert-from-premultiplied + + // 2. + PremultiplyAlpha::Premultiply => process_premultiply_alpha(image), + // 3. + PremultiplyAlpha::None => { + // NOTE: It's not clear how to handle the case of ImageData. + // https://issues.chromium.org/issues/339759426 + // https://github.com/whatwg/html/issues/5365 + if *image_bitmap_source == ImageBitmapSource::ImageData { + return Ok(image); } - // 3. - PremultiplyAlpha::None => { - // NOTE: It's not clear how to handle the case of ImageData. - // https://issues.chromium.org/issues/339759426 - // https://github.com/whatwg/html/issues/5365 - if *image_bitmap_source == ImageBitmapSource::ImageData { - return Ok(image); - } - unpremultiply_alpha(image, unmatch_color_handler) - } + unpremultiply_alpha(image) } } } From fb6b87d656a1911787f7daca55392ab2b09981b3 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 17:09:05 +0900 Subject: [PATCH 27/52] avoid copying string --- ext/canvas/01_image.js | 30 +++++------ ext/canvas/op_create_image_bitmap.rs | 76 +++++++--------------------- 2 files changed, 33 insertions(+), 73 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 4f54a2aadfbc75..0e0fa62ba4fdbe 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -256,22 +256,20 @@ function createImageBitmap( // TODO(Hajime-san): this should be real async const processedImage = op_create_image_bitmap( buf, - { - width, - height, - sx, - sy, - sw, - sh, - imageOrientation: options.imageOrientation ?? "from-image", - premultiplyAlpha: options.premultiplyAlpha ?? "default", - colorSpaceConversion: options.colorSpaceConversion ?? "default", - resizeWidth: options.resizeWidth, - resizeHeight: options.resizeHeight, - resizeQuality: options.resizeQuality ?? "low", - imageBitmapSource, - mimeType, - }, + width, + height, + sx, + sy, + sw, + sh, + options.imageOrientation ?? "from-image", + options.premultiplyAlpha ?? "default", + options.colorSpaceConversion ?? "default", + options.resizeWidth, + options.resizeHeight, + options.resizeQuality ?? "low", + imageBitmapSource, + mimeType, ); imageBitmap[_bitmapData] = processedImage.data; imageBitmap[_width] = processedImage.width; diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index c11fb518d82d2e..71bdf8203e47aa 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -71,25 +71,6 @@ enum ResizeQuality { High, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct OpCreateImageBitmapArgs { - width: u32, - height: u32, - sx: Option, - sy: Option, - sw: Option, - sh: Option, - image_orientation: ImageOrientation, - premultiply_alpha: PremultiplyAlpha, - color_space_conversion: ColorSpaceConversion, - resize_width: Option, - resize_height: Option, - resize_quality: ResizeQuality, - image_bitmap_source: ImageBitmapSource, - mime_type: String, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct OpCreateImageBitmapReturn { @@ -105,7 +86,7 @@ fn decode_bitmap_data( width: u32, height: u32, image_bitmap_source: &ImageBitmapSource, - mime_type: String, + mime_type: &str, ) -> Result { let (image, width, height, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { @@ -116,7 +97,7 @@ fn decode_bitmap_data( )) .into() } - let (image, icc_profile) = match &*mime_type { + let (image, icc_profile) = match mime_type { // Should we support the "image/apng" MIME type here? "image/png" => { let mut decoder: PngDecoder = @@ -284,46 +265,27 @@ fn apply_premultiply_alpha( #[op2] #[serde] +#[allow(clippy::too_many_arguments)] pub(super) fn op_create_image_bitmap( - #[buffer] zero_copy: JsBuffer, - #[serde] args: OpCreateImageBitmapArgs, + #[buffer] buf: JsBuffer, + width: u32, + height: u32, + sx: Option, + sy: Option, + sw: Option, + sh: Option, + #[serde] image_orientation: ImageOrientation, + #[serde] premultiply_alpha: PremultiplyAlpha, + #[serde] color_space_conversion: ColorSpaceConversion, + resize_width: Option, + resize_height: Option, + #[serde] resize_quality: ResizeQuality, + #[serde] image_bitmap_source: ImageBitmapSource, + #[string] mime_type: &str, ) -> Result { - let buf = &*zero_copy; - let OpCreateImageBitmapArgs { - width, - height, - sh, - sw, - sx, - sy, - image_orientation, - premultiply_alpha, - color_space_conversion, - resize_width, - resize_height, - resize_quality, - image_bitmap_source, - mime_type, - } = OpCreateImageBitmapArgs { - width: args.width, - height: args.height, - sx: args.sx, - sy: args.sy, - sw: args.sw, - sh: args.sh, - image_orientation: args.image_orientation, - premultiply_alpha: args.premultiply_alpha, - color_space_conversion: args.color_space_conversion, - resize_width: args.resize_width, - resize_height: args.resize_height, - resize_quality: args.resize_quality, - image_bitmap_source: args.image_bitmap_source, - mime_type: args.mime_type, - }; - // 6. Switch on image: let (image, width, height, icc_profile) = - decode_bitmap_data(buf, width, height, &image_bitmap_source, mime_type)?; + decode_bitmap_data(&buf, width, height, &image_bitmap_source, mime_type)?; // crop bitmap data // 2. From 3beb65b9f0af254c1c3c0d7b2c1f6ca778859437 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 29 Sep 2024 23:05:03 +0900 Subject: [PATCH 28/52] perf: return the value not a struct but a tuple --- ext/canvas/01_image.js | 6 +++--- ext/canvas/op_create_image_bitmap.rs | 17 ++--------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 0e0fa62ba4fdbe..2716e14bf51350 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -271,9 +271,9 @@ function createImageBitmap( imageBitmapSource, mimeType, ); - imageBitmap[_bitmapData] = processedImage.data; - imageBitmap[_width] = processedImage.width; - imageBitmap[_height] = processedImage.height; + imageBitmap[_bitmapData] = processedImage[0]; + imageBitmap[_width] = processedImage[1]; + imageBitmap[_height] = processedImage[2]; return imageBitmap; })(); } diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 71bdf8203e47aa..0b361eb0e9afc0 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -22,7 +22,6 @@ use image::DynamicImage; use image::ImageError; use image::RgbaImage; use serde::Deserialize; -use serde::Serialize; use crate::error::image_error_message; use crate::error::DOMExceptionInvalidStateError; @@ -71,14 +70,6 @@ enum ResizeQuality { High, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct OpCreateImageBitmapReturn { - data: ToJsBuffer, - width: u32, - height: u32, -} - type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); fn decode_bitmap_data( @@ -282,7 +273,7 @@ pub(super) fn op_create_image_bitmap( #[serde] resize_quality: ResizeQuality, #[serde] image_bitmap_source: ImageBitmapSource, #[string] mime_type: &str, -) -> Result { +) -> Result<(ToJsBuffer, u32, u32), AnyError> { // 6. Switch on image: let (image, width, height, icc_profile) = decode_bitmap_data(&buf, width, height, &image_bitmap_source, mime_type)?; @@ -381,9 +372,5 @@ pub(super) fn op_create_image_bitmap( let image = apply_premultiply_alpha(image, &image_bitmap_source, &premultiply_alpha)?; - Ok(OpCreateImageBitmapReturn { - data: image.into_bytes().into(), - width: output_width, - height: output_height, - }) + Ok((image.into_bytes().into(), output_width, output_height)) } From 829f523f897a5cb3bbc4a3a53acc65141cf82091 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:55:17 +0900 Subject: [PATCH 29/52] add unit test for `imageOrientation` --- ext/canvas/op_create_image_bitmap.rs | 10 ++++ tests/testdata/image/squares_6.jpg | Bin 0 -> 1227 bytes tests/unit/image_bitmap_test.ts | 66 +++++++++++++++++++++------ 3 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 tests/testdata/image/squares_6.jpg diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 0b361eb0e9afc0..daba9b4f209a15 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -361,7 +361,17 @@ pub(super) fn op_create_image_bitmap( let image = if image_orientation == ImageOrientation::FlipY { image.flipv() } else { + // TODO(Hajime-san): If the EXIF ​​contains image orientation information, decode the image in the appropriate orientation. + // The image create is expected to release this feature soon. + // https://github.com/image-rs/image/blob/bd0f7451a367de7ae3d898dcf1e96e9d0a1c4fa1/CHANGES.md#version-0253 + // https://github.com/image-rs/image/issues/1958 + // https://github.com/image-rs/image/pull/2299 + // https://github.com/image-rs/image/pull/2328 + // if image_bitmap_source == ImageBitmapSource::Blob { + // image.apply_orientation() + // } else { image + // } }; // 9. diff --git a/tests/testdata/image/squares_6.jpg b/tests/testdata/image/squares_6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f197760a111e07c54419d1993130aa1fee5982e8 GIT binary patch literal 1227 zcmdUrKS%>H6vkgJcV}{UZF6l!s30zac2z-fu%H$#R8Xpj;-VlbNL@s6a&vN$x(KBp z6?Jy3P{FyVi+=|TE>1eQ%9;2M|KQ}Lej&LZ-+k}p5%1Ew1CdG&B!LhBA{Tg7nCe@Z zn*m6tAr1fx3=#$^iGV4O7#MF1kRtW(JWQJXjO?LBK3Hazr2DbG8|VS5DXEi6>87s4 zOw%wqvn~30b6MlL`cuTnz{p_&v1LA!^)cNvdWI$x_lJ%UNn38B7g5_P zCZ7@Y^N|IC_*-q4@?E7Df5x53<9tB*JY1nw&51!Sk;xONQjAG;y%>J@OnvupqW literal 0 HcmV?d00001 diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 67d163852e19b9..bb703580fd9476 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertRejects } from "./test_util.ts"; +import { assertEquals, assertNotEquals, assertRejects } from "./test_util.ts"; const prefix = "tests/testdata/image"; @@ -78,19 +78,59 @@ Deno.test(async function imageBitmapScale() { ])); }); -Deno.test(async function imageBitmapFlipY() { - const data = generateNumberedData(9); - const imageData = new ImageData(data, 3, 3); - const imageBitmap = await createImageBitmap(imageData, { - imageOrientation: "flipY", +Deno.test("imageOrientation", async (t) => { + await t.step('"ImageData" imageOrientation: "flipY"', async () => { + const data = generateNumberedData(9); + const imageData = new ImageData(data, 3, 3); + const imageBitmap = await createImageBitmap(imageData, { + imageOrientation: "flipY", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + // deno-fmt-ignore + assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ + 7, 0, 0, 1, 8, 0, 0, 1, 9, 0, 0, 1, + 4, 0, 0, 1, 5, 0, 0, 1, 6, 0, 0, 1, + 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, + ])); + }); + + const imageData = new Blob( + [await Deno.readFile(`${prefix}/squares_6.jpg`)], + { type: "image/jpeg" }, + ); + const WIDTH = 320; + const CHANNELS = 3; + const TARGET_PIXEL_X = 40; + const START = TARGET_PIXEL_X * WIDTH * CHANNELS; + const END = START + CHANNELS; + // reference: + // https://github.com/web-platform-tests/wpt/blob/a1f4bbf4c6e1a9a861a145a34cd097ea260b5a49/html/canvas/element/manual/imagebitmap/createImageBitmap-exif-orientation.html#L30 + await t.step('"Blob" imageOrientation: "from-image"', async () => { + const imageBitmap = await createImageBitmap(imageData); + // @ts-ignore: Deno[Deno.internal].core allowed + const targetPixel = Deno[Deno.internal].getBitmapData(imageBitmap).slice( + START, + END, + ); + // FIXME: When the implementation is fixed, fix this to assertEquals + // However, in the case of jpg images, numerical errors may occur. + assertNotEquals(targetPixel, new Uint8Array([255, 1, 0])); + }); + // reference: + // https://github.com/web-platform-tests/wpt/blob/a1f4bbf4c6e1a9a861a145a34cd097ea260b5a49/html/canvas/element/manual/imagebitmap/createImageBitmap-exif-orientation.html#L55 + await t.step('"Blob" imageOrientation: "flipY"', async () => { + const imageBitmap = await createImageBitmap(imageData, { + imageOrientation: "flipY", + }); + // @ts-ignore: Deno[Deno.internal].core allowed + const targetPixel = Deno[Deno.internal].getBitmapData(imageBitmap).slice( + START, + END, + ); + // FIXME: When the implementation is fixed, fix this to assertEquals + // However, in the case of jpg images, numerical errors may occur. + assertNotEquals(targetPixel, new Uint8Array([254, 128, 129])); }); - // @ts-ignore: Deno[Deno.internal].core allowed - // deno-fmt-ignore - assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ - 7, 0, 0, 1, 8, 0, 0, 1, 9, 0, 0, 1, - 4, 0, 0, 1, 5, 0, 0, 1, 6, 0, 0, 1, - 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, - ])); }); Deno.test("imageBitmapPremultiplyAlpha", async (t) => { From 9a9664e38f1ffff1070c97cc13f3a5ada8cc2f47 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:11:41 +0900 Subject: [PATCH 30/52] add comments for the reference of issues about `imageOrientation` --- ext/canvas/op_create_image_bitmap.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index daba9b4f209a15..a7dfaabe81c67b 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -358,6 +358,10 @@ pub(super) fn op_create_image_bitmap( let image = image.resize_exact(output_width, output_height, filter_type); // 8. + // issues for imageOrientation + // https://github.com/whatwg/html/issues/8085 + // https://github.com/whatwg/html/issues/7210 + // https://github.com/whatwg/html/issues/8118 let image = if image_orientation == ImageOrientation::FlipY { image.flipv() } else { From 5e52b862fb17d46641a53a9271afdbe351b98e1d Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:30:46 +0900 Subject: [PATCH 31/52] update error message --- ext/canvas/op_create_image_bitmap.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index a7dfaabe81c67b..b5cb308cb29d62 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -9,6 +9,7 @@ use deno_core::op2; use deno_core::JsBuffer; use deno_core::ToJsBuffer; use deno_terminal::colors::cyan; +use deno_terminal::colors::yellow; use image::codecs::bmp::BmpDecoder; use image::codecs::gif::GifDecoder; use image::codecs::ico::IcoDecoder; @@ -142,10 +143,13 @@ fn decode_bitmap_data( return Err( DOMExceptionInvalidStateError::new( &format!("The MIME type of source image is not specified. -INFO: The behavior of the Blob constructor in browsers is different from the spec. -It needs to specify the MIME type like {} that works well between Deno and browsers. -See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", - cyan("new Blob([blobParts], { type: 'image/png' })") +{} When you want to get a `Blob` from `fetch`, make sure to go through a file server that returns the appropriate content-type response header, + and specify the URL to the file server like {}. + Alternatively, if you are reading a local file using `Deno.readFile` etc., + set the appropriate MIME type like {}.\n", + cyan("hint:"), + cyan("await(await fetch('http://localhost:8000/sample.png').blob()"), + cyan("new Blob([await Deno.readFile('sample.png')], { type: 'image/png' })") )).into(), ) } @@ -155,9 +159,10 @@ See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/type\n", return Err( DOMExceptionInvalidStateError::new( &format!("The the MIME type {} of source image is not a supported format. -INFO: The following MIME types are supported: -See: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", - x +{} The following MIME types are supported: + https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", + x, + yellow("info:") )).into() ) } From 1b545f980ad89c3b20ced67758b50c410c6cacb4 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 3 Oct 2024 18:27:32 +0900 Subject: [PATCH 32/52] handle 32-bit depth --- ext/canvas/image_ops.rs | 59 ++++++++++++++++------------ ext/canvas/op_create_image_bitmap.rs | 12 +----- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index cbe8b63174031b..1feb24c7234f32 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -2,6 +2,7 @@ use bytemuck::cast_slice; use bytemuck::cast_slice_mut; +use deno_core::error::type_error; use deno_core::error::AnyError; use image::ColorType; use image::DynamicImage; @@ -20,6 +21,15 @@ use lcms2::Transform; use num_traits::NumCast; use num_traits::SaturatingMul; +use crate::error::image_error_message; + +/// Image formats that is 32-bit depth are not supported currently due to the following reasons: +/// - e.g. OpenEXR, it's not covered by the spec. +/// - JPEG XL supported by WebKit, but it cannot be called a standard today. +/// https://github.com/whatwg/mimesniff/issues/143 +const NOT_SUPPORTED_BIT_DEPTH: &str = + "The 32-bit depth image format is not supported."; + pub(crate) trait PremultiplyAlpha { fn premultiply_alpha(&self) -> Self; } @@ -103,6 +113,12 @@ pub(crate) fn premultiply_alpha( ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( process_premultiply_alpha(image.as_rgba16().unwrap()), )), + ColorType::Rgb32F | ColorType::Rgba32F => { + Err(type_error(image_error_message( + "processing un-premultiply alpha", + NOT_SUPPORTED_BIT_DEPTH, + ))) + } // If the image does not have an alpha channel, return the image as is. _ => Ok(image), } @@ -244,6 +260,12 @@ pub(crate) fn unpremultiply_alpha( image.into_rgba16() }, )), + ColorType::Rgb32F | ColorType::Rgba32F => { + Err(type_error(image_error_message( + "processing un-premultiply alpha", + NOT_SUPPORTED_BIT_DEPTH, + ))) + } // If the image does not have an alpha channel, return the image as is. _ => Ok(image), } @@ -350,13 +372,7 @@ where ColorType::Rgb16 => PixelFormat::RGB_16, ColorType::Rgba8 => PixelFormat::RGBA_8, ColorType::Rgba16 => PixelFormat::RGBA_16, - // This arm usually doesn't reach, but it should be handled with returning the original image. - _ => { - for (x, y, pixel) in image.pixels() { - out.put_pixel(x, y, pixel); - } - return out; - } + _ => unreachable!("{}", NOT_SUPPORTED_BIT_DEPTH), }; let transformer = Transform::new( &input_icc_profile, @@ -383,10 +399,6 @@ where pub(crate) fn to_srgb_from_icc_profile( image: DynamicImage, icc_profile: Option>, - unmatch_color_handler: fn( - ColorType, - DynamicImage, - ) -> Result, ) -> Result { match icc_profile { // If there is no color profile information, return the image as is. @@ -462,7 +474,10 @@ pub(crate) fn to_srgb_from_icc_profile( srgb_icc_profile, ))) } - x => unmatch_color_handler(x, image), + _ => Err(type_error(image_error_message( + "processing un-premultiply alpha", + NOT_SUPPORTED_BIT_DEPTH, + ))), } } }, @@ -618,20 +633,11 @@ pub(crate) fn to_srgb_from_icc_profile( // } // /// Convert the color space of the image from sRGB to Display-P3. -// fn srgb_to_display_p3( -// image: DynamicImage, -// unmatch_color_handler: fn( -// ColorType, -// DynamicImage, -// ) -> Result, -// ) -> Result { +// fn srgb_to_display_p3(image: DynamicImage) -> Result { // match image.color() { // // The conversion of the lumincance color types to the display-p3 color space is meaningless. -// ColorType::L8 => Ok(DynamicImage::ImageLuma8(image.into_luma8())), -// ColorType::L16 => Ok(DynamicImage::ImageLuma16(image.into_luma16())), -// ColorType::La8 => Ok(DynamicImage::ImageLumaA8(image.into_luma_alpha8())), -// ColorType::La16 => { -// Ok(DynamicImage::ImageLumaA16(image.into_luma_alpha16())) +// ColorType::L8 | ColorType::L16 | ColorType::La8 | ColorType::La16 => { +// Ok(image) // } // ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( // image.as_rgb8().unwrap(), @@ -645,7 +651,10 @@ pub(crate) fn to_srgb_from_icc_profile( // ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( // process_srgb_to_display_p3(image.as_rgba16().unwrap()), // )), -// x => unmatch_color_handler(x, image), +// _ => Err(type_error(image_error_message( +// "processing ICC color profile conversion to sRGB", +// NOT_SUPPORTED_BIT_DEPTH, +// ))), // } // } diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index b5cb308cb29d62..bea9959523181b 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -18,7 +18,6 @@ use image::codecs::png::PngDecoder; use image::codecs::webp::WebPDecoder; use image::imageops::overlay; use image::imageops::FilterType; -use image::ColorType; use image::DynamicImage; use image::ImageError; use image::RgbaImage; @@ -218,16 +217,7 @@ fn apply_color_space_conversion( // return the decoded image as is. ColorSpaceConversion::None => Ok(image), ColorSpaceConversion::Default => { - fn unmatch_color_handler( - x: ColorType, - _: DynamicImage, - ) -> Result { - Err(type_error(image_error_message( - "apply colorspaceConversion: default", - &format!("The color type {:?} is not supported.", x), - ))) - } - to_srgb_from_icc_profile(image, icc_profile, unmatch_color_handler) + to_srgb_from_icc_profile(image, icc_profile) } } } From 977026c3324558c1f6ad612bbd341ab4aafb35f5 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:03:41 +0900 Subject: [PATCH 33/52] clean up --- ext/canvas/image_ops.rs | 203 ++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 100 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 1feb24c7234f32..8a8a8ef04e1f78 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -100,22 +100,22 @@ where pub(crate) fn premultiply_alpha( image: DynamicImage, ) -> Result { - match image.color() { - ColorType::La8 => Ok(DynamicImage::ImageLumaA8(process_premultiply_alpha( - image.as_luma_alpha8().unwrap(), - ))), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - process_premultiply_alpha(image.as_luma_alpha16().unwrap()), - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - process_premultiply_alpha(image.as_rgba8().unwrap()), - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - process_premultiply_alpha(image.as_rgba16().unwrap()), - )), - ColorType::Rgb32F | ColorType::Rgba32F => { + match image { + DynamicImage::ImageLumaA8(image) => { + Ok(process_premultiply_alpha(&image).into()) + } + DynamicImage::ImageLumaA16(image) => { + Ok(process_premultiply_alpha(&image).into()) + } + DynamicImage::ImageRgba8(image) => { + Ok(process_premultiply_alpha(&image).into()) + } + DynamicImage::ImageRgba16(image) => { + Ok(process_premultiply_alpha(&image).into()) + } + DynamicImage::ImageRgb32F(_) | DynamicImage::ImageRgba32F(_) => { Err(type_error(image_error_message( - "processing un-premultiply alpha", + "processing premultiply alpha", NOT_SUPPORTED_BIT_DEPTH, ))) } @@ -231,36 +231,30 @@ where pub(crate) fn unpremultiply_alpha( image: DynamicImage, ) -> Result { - match image.color() { - ColorType::La8 => Ok(DynamicImage::ImageLumaA8( - if is_premultiplied_alpha(image.as_luma_alpha8().unwrap()) { - process_unpremultiply_alpha(image.as_luma_alpha8().unwrap()) - } else { - image.into_luma_alpha8() - }, - )), - ColorType::La16 => Ok(DynamicImage::ImageLumaA16( - if is_premultiplied_alpha(image.as_luma_alpha16().unwrap()) { - process_unpremultiply_alpha(image.as_luma_alpha16().unwrap()) - } else { - image.into_luma_alpha16() - }, - )), - ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( - if is_premultiplied_alpha(image.as_rgba8().unwrap()) { - process_unpremultiply_alpha(image.as_rgba8().unwrap()) + match image { + DynamicImage::ImageLumaA8(image) => Ok(if is_premultiplied_alpha(&image) { + process_unpremultiply_alpha(&image).into() + } else { + image.into() + }), + DynamicImage::ImageLumaA16(image) => { + Ok(if is_premultiplied_alpha(&image) { + process_unpremultiply_alpha(&image).into() } else { - image.into_rgba8() - }, - )), - ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( - if is_premultiplied_alpha(image.as_rgba16().unwrap()) { - process_unpremultiply_alpha(image.as_rgba16().unwrap()) - } else { - image.into_rgba16() - }, - )), - ColorType::Rgb32F | ColorType::Rgba32F => { + image.into() + }) + } + DynamicImage::ImageRgba8(image) => Ok(if is_premultiplied_alpha(&image) { + process_unpremultiply_alpha(&image).into() + } else { + image.into() + }), + DynamicImage::ImageRgba16(image) => Ok(if is_premultiplied_alpha(&image) { + process_unpremultiply_alpha(&image).into() + } else { + image.into() + }), + DynamicImage::ImageRgb32F(_) | DynamicImage::ImageRgba32F(_) => { Err(type_error(image_error_message( "processing un-premultiply alpha", NOT_SUPPORTED_BIT_DEPTH, @@ -409,71 +403,79 @@ pub(crate) fn to_srgb_from_icc_profile( Ok(icc_profile) => { let srgb_icc_profile = Profile::new_srgb(); let color = image.color(); - match color { - ColorType::L8 => { - Ok(DynamicImage::ImageLuma8(process_icc_profile_conversion( - image.as_luma8().unwrap(), + match image { + DynamicImage::ImageLuma8(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::L16 => { - Ok(DynamicImage::ImageLuma16(process_icc_profile_conversion( - image.as_luma16().unwrap(), + ) + .into(), + ), + DynamicImage::ImageLuma16(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::La8 => { - Ok(DynamicImage::ImageLumaA8(process_icc_profile_conversion( - image.as_luma_alpha8().unwrap(), + ) + .into(), + ), + DynamicImage::ImageLumaA8(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::La16 => { - Ok(DynamicImage::ImageLumaA16(process_icc_profile_conversion( - image.as_luma_alpha16().unwrap(), + ) + .into(), + ), + DynamicImage::ImageLumaA16(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::Rgb8 => { - Ok(DynamicImage::ImageRgb8(process_icc_profile_conversion( - image.as_rgb8().unwrap(), + ) + .into(), + ), + DynamicImage::ImageRgb8(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::Rgb16 => { - Ok(DynamicImage::ImageRgb16(process_icc_profile_conversion( - image.as_rgb16().unwrap(), + ) + .into(), + ), + DynamicImage::ImageRgb16(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::Rgba8 => { - Ok(DynamicImage::ImageRgba8(process_icc_profile_conversion( - image.as_rgba8().unwrap(), + ) + .into(), + ), + DynamicImage::ImageRgba8(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } - ColorType::Rgba16 => { - Ok(DynamicImage::ImageRgba16(process_icc_profile_conversion( - image.as_rgba16().unwrap(), + ) + .into(), + ), + DynamicImage::ImageRgba16(image) => Ok( + process_icc_profile_conversion( + &image, color, icc_profile, srgb_icc_profile, - ))) - } + ) + .into(), + ), _ => Err(type_error(image_error_message( "processing un-premultiply alpha", NOT_SUPPORTED_BIT_DEPTH, @@ -634,23 +636,24 @@ pub(crate) fn to_srgb_from_icc_profile( // /// Convert the color space of the image from sRGB to Display-P3. // fn srgb_to_display_p3(image: DynamicImage) -> Result { -// match image.color() { +// match image { // // The conversion of the lumincance color types to the display-p3 color space is meaningless. -// ColorType::L8 | ColorType::L16 | ColorType::La8 | ColorType::La16 => { -// Ok(image) +// DynamicImage::ImageLuma8(_) +// | DynamicImage::ImageLuma16(_) +// | DynamicImage::ImageLumaA8(_) +// | DynamicImage::ImageLumaA16(_) => Ok(image), +// DynamicImage::ImageRgb8(image) => { +// Ok(process_srgb_to_display_p3(&image).into()) +// } +// DynamicImage::ImageRgb16(image) => { +// Ok(process_srgb_to_display_p3(&image).into()) +// } +// DynamicImage::ImageRgba8(image) => { +// Ok(process_srgb_to_display_p3(&image).into()) +// } +// DynamicImage::ImageRgba16(image) => { +// Ok(process_srgb_to_display_p3(&image).into()) // } -// ColorType::Rgb8 => Ok(DynamicImage::ImageRgb8(process_srgb_to_display_p3( -// image.as_rgb8().unwrap(), -// ))), -// ColorType::Rgb16 => Ok(DynamicImage::ImageRgb16( -// process_srgb_to_display_p3(image.as_rgb16().unwrap()), -// )), -// ColorType::Rgba8 => Ok(DynamicImage::ImageRgba8( -// process_srgb_to_display_p3(image.as_rgba8().unwrap()), -// )), -// ColorType::Rgba16 => Ok(DynamicImage::ImageRgba16( -// process_srgb_to_display_p3(image.as_rgba16().unwrap()), -// )), // _ => Err(type_error(image_error_message( // "processing ICC color profile conversion to sRGB", // NOT_SUPPORTED_BIT_DEPTH, From 69addc8faf2c6228309c9b7e162d8ce67f798942 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 6 Oct 2024 16:23:10 +0900 Subject: [PATCH 34/52] fix --- ext/canvas/01_image.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 2716e14bf51350..67b51729fcce14 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -249,10 +249,7 @@ function createImageBitmap( buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); } - let sx; - if (typeof sxOrOptions === "number") { - sx = sxOrOptions; - } + const sx = typeof sxOrOptions === "number" ? sxOrOptions : undefined; // TODO(Hajime-san): this should be real async const processedImage = op_create_image_bitmap( buf, From acc4119d7434bbc87463558c1cab6987f98c8df8 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:17:47 +0900 Subject: [PATCH 35/52] perf: pass `u8` to `op` --- ext/canvas/01_image.js | 107 +++++++-- ext/canvas/op_create_image_bitmap.rs | 323 ++++++++++++++++++++------- 2 files changed, 330 insertions(+), 100 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 67b51729fcce14..3755df1d6fb78b 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -233,38 +233,111 @@ function createImageBitmap( // 4. return (async () => { + // + // For performance reasons, the arguments passed to op are represented as numbers that don't need to be serialized. + // + let width = 0; let height = 0; - let mimeType = ""; + // If the of image doesn't have a MIME type, mark it as 0. + let mimeType = 0; let imageBitmapSource, buf; if (isBlob) { - imageBitmapSource = imageBitmapSources[0]; + imageBitmapSource = 0; buf = new Uint8Array(await image.arrayBuffer()); - mimeType = sniffImage(image.type); - } - if (isImageData) { + const mimeTypeString = sniffImage(image.type); + + if (mimeTypeString === "image/png") { + mimeType = 1; + } else if (mimeTypeString === "image/jpeg") { + mimeType = 2; + } else if (mimeTypeString === "image/gif") { + mimeType = 3; + } else if (mimeTypeString === "image/bmp") { + mimeType = 4; + } else if (mimeTypeString === "image/x-icon") { + mimeType = 5; + } else if (mimeTypeString === "image/webp") { + mimeType = 6; + } else if (mimeTypeString === "") { + return PromiseReject( + new DOMException( + `The MIME type of source image is not specified. +hint: When you want to get a "Blob" from "fetch", make sure to go through a file server that returns the appropriate content-type response header, + and specify the URL to the file server like "await(await fetch('http://localhost:8000/sample.png').blob()". + Alternatively, if you are reading a local file using 'Deno.readFile' etc., + set the appropriate MIME type like "new Blob([await Deno.readFile('sample.png')], { type: 'image/png' })".\n`, + "InvalidStateError", + ), + ); + } else { + return PromiseReject( + new DOMException( + `The the MIME type ${mimeTypeString} of source image is not a supported format. +info: The following MIME types are supported: + https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n`, + "InvalidStateError", + ), + ); + } + } else if (isImageData) { width = image[_width]; height = image[_height]; - imageBitmapSource = imageBitmapSources[1]; + imageBitmapSource = 1; buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); } - const sx = typeof sxOrOptions === "number" ? sxOrOptions : undefined; + // If those options are not provided, assign 0 to mean undefined(None). + const _sx = typeof sxOrOptions === "number" ? sxOrOptions : 0; + const _sy = sy ?? 0; + const _sw = sw ?? 0; + const _sh = sh ?? 0; + + // If those options are not provided, assign 0 to mean undefined(None). + const resizeWidth = options.resizeWidth ?? 0; + const resizeHeight = options.resizeHeight ?? 0; + + // If the imageOrientation option is set "from-image" or not set, assign 0. + const imageOrientation = options.imageOrientation === "flipY" ? 1 : 0; + + // If the premultiplyAlpha option is "default" or not set, assign 0. + let premultiplyAlpha = 0; + if (options.premultiplyAlpha === "premultiply") { + premultiplyAlpha = 1; + } else if (options.premultiplyAlpha === "none") { + premultiplyAlpha = 2; + } + + // If the colorSpaceConversion option is "default" or not set, assign 0. + const colorSpaceConversion = options.colorSpaceConversion === "none" + ? 1 + : 0; + + // If the resizeQuality option is "low" or not set, assign 0. + let resizeQuality = 0; + if (options.resizeQuality === "pixelated") { + resizeQuality = 1; + } else if (options.resizeQuality === "medium") { + resizeQuality = 2; + } else if (options.resizeQuality === "high") { + resizeQuality = 3; + } + // TODO(Hajime-san): this should be real async const processedImage = op_create_image_bitmap( buf, width, height, - sx, - sy, - sw, - sh, - options.imageOrientation ?? "from-image", - options.premultiplyAlpha ?? "default", - options.colorSpaceConversion ?? "default", - options.resizeWidth, - options.resizeHeight, - options.resizeQuality ?? "low", + _sx, + _sy, + _sw, + _sh, + imageOrientation, + premultiplyAlpha, + colorSpaceConversion, + resizeWidth, + resizeHeight, + resizeQuality, imageBitmapSource, mimeType, ); diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index bea9959523181b..849889d7133fb6 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -8,8 +8,6 @@ use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::ToJsBuffer; -use deno_terminal::colors::cyan; -use deno_terminal::colors::yellow; use image::codecs::bmp::BmpDecoder; use image::codecs::gif::GifDecoder; use image::codecs::ico::IcoDecoder; @@ -21,7 +19,6 @@ use image::imageops::FilterType; use image::DynamicImage; use image::ImageError; use image::RgbaImage; -use serde::Deserialize; use crate::error::image_error_message; use crate::error::DOMExceptionInvalidStateError; @@ -31,38 +28,32 @@ use crate::image_ops::premultiply_alpha as process_premultiply_alpha; use crate::image_ops::to_srgb_from_icc_profile; use crate::image_ops::unpremultiply_alpha; -#[derive(Debug, Deserialize, PartialEq)] -// Follow the cases defined in the spec +#[derive(Debug, PartialEq)] enum ImageBitmapSource { Blob, ImageData, } -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, PartialEq)] enum ImageOrientation { FlipY, - #[serde(rename = "from-image")] FromImage, } -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, PartialEq)] enum PremultiplyAlpha { Default, Premultiply, None, } -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, PartialEq)] enum ColorSpaceConversion { Default, None, } -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, PartialEq)] enum ResizeQuality { Pixelated, Low, @@ -70,6 +61,17 @@ enum ResizeQuality { High, } +#[derive(Debug, PartialEq)] +enum MimeType { + NoMatch, + Png, + Jpeg, + Gif, + Bmp, + Ico, + Webp, +} + type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); fn decode_bitmap_data( @@ -77,7 +79,7 @@ fn decode_bitmap_data( width: u32, height: u32, image_bitmap_source: &ImageBitmapSource, - mime_type: &str, + mime_type: MimeType, ) -> Result { let (image, width, height, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { @@ -90,81 +92,80 @@ fn decode_bitmap_data( } let (image, icc_profile) = match mime_type { // Should we support the "image/apng" MIME type here? - "image/png" => { + MimeType::Png => { let mut decoder: PngDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, + ) } - "image/jpeg" => { + MimeType::Jpeg => { let mut decoder: JpegDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, + ) } - "image/gif" => { + MimeType::Gif => { let mut decoder: GifDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, + ) } - "image/bmp" => { + MimeType::Bmp => { let mut decoder: BmpDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, + ) } - "image/x-icon" => { + MimeType::Ico => { let mut decoder: IcoDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, + ) } - "image/webp" => { + MimeType::Webp => { let mut decoder: WebPDecoder = - ImageDecoderFromReader::to_decoder(BufReader::new(Cursor::new( - buf, - )), image_decoding_error)?; + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; let icc_profile = decoder.get_icc_profile(); - (decoder.to_intermediate_image(image_decoding_error)?, icc_profile) - } - "" => { - return Err( - DOMExceptionInvalidStateError::new( - &format!("The MIME type of source image is not specified. -{} When you want to get a `Blob` from `fetch`, make sure to go through a file server that returns the appropriate content-type response header, - and specify the URL to the file server like {}. - Alternatively, if you are reading a local file using `Deno.readFile` etc., - set the appropriate MIME type like {}.\n", - cyan("hint:"), - cyan("await(await fetch('http://localhost:8000/sample.png').blob()"), - cyan("new Blob([await Deno.readFile('sample.png')], { type: 'image/png' })") - )).into(), - ) - } - // return an error if the MIME type is not supported in the variable list of ImageTypePatternTable below - // ext/web/01_mimesniff.js - x => { - return Err( - DOMExceptionInvalidStateError::new( - &format!("The the MIME type {} of source image is not a supported format. -{} The following MIME types are supported: - https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n", - x, - yellow("info:") - )).into() + ( + decoder.to_intermediate_image(image_decoding_error)?, + icc_profile, ) } + // This pattern is unreachable due to current block is already checked by the ImageBitmapSource above. + MimeType::NoMatch => unreachable!(), }; let width = image.width(); @@ -249,6 +250,107 @@ fn apply_premultiply_alpha( } } +#[derive(Debug, PartialEq)] +struct ParsedArgs { + resize_width: Option, + resize_height: Option, + sx: Option, + sy: Option, + sw: Option, + sh: Option, + image_orientation: ImageOrientation, + premultiply_alpha: PremultiplyAlpha, + color_space_conversion: ColorSpaceConversion, + resize_quality: ResizeQuality, + image_bitmap_source: ImageBitmapSource, + mime_type: MimeType, +} + +#[allow(clippy::too_many_arguments)] +fn parse_args( + sx: i32, + sy: i32, + sw: i32, + sh: i32, + image_orientation: u8, + premultiply_alpha: u8, + color_space_conversion: u8, + resize_width: u32, + resize_height: u32, + resize_quality: u8, + image_bitmap_source: u8, + mime_type: u8, +) -> ParsedArgs { + let resize_width = if resize_width == 0 { + None + } else { + Some(resize_width) + }; + let resize_height = if resize_height == 0 { + None + } else { + Some(resize_height) + }; + let sx = if sx == 0 { None } else { Some(sx) }; + let sy = if sy == 0 { None } else { Some(sy) }; + let sw = if sw == 0 { None } else { Some(sw) }; + let sh = if sh == 0 { None } else { Some(sh) }; + + // Their unreachable wildcard patterns are validated in JavaScript-side. + let image_orientation = match image_orientation { + 0 => ImageOrientation::FromImage, + 1 => ImageOrientation::FlipY, + _ => unreachable!(), + }; + let premultiply_alpha = match premultiply_alpha { + 0 => PremultiplyAlpha::Default, + 1 => PremultiplyAlpha::Premultiply, + 2 => PremultiplyAlpha::None, + _ => unreachable!(), + }; + let color_space_conversion = match color_space_conversion { + 0 => ColorSpaceConversion::Default, + 1 => ColorSpaceConversion::None, + _ => unreachable!(), + }; + let resize_quality = match resize_quality { + 0 => ResizeQuality::Low, + 1 => ResizeQuality::Pixelated, + 2 => ResizeQuality::Medium, + 3 => ResizeQuality::High, + _ => unreachable!(), + }; + let image_bitmap_source = match image_bitmap_source { + 0 => ImageBitmapSource::Blob, + 1 => ImageBitmapSource::ImageData, + _ => unreachable!(), + }; + let mime_type = match mime_type { + 0 => MimeType::NoMatch, + 1 => MimeType::Png, + 2 => MimeType::Jpeg, + 3 => MimeType::Gif, + 4 => MimeType::Bmp, + 5 => MimeType::Ico, + 6 => MimeType::Webp, + _ => unreachable!(), + }; + ParsedArgs { + resize_width, + resize_height, + sx, + sy, + sw, + sh, + image_orientation, + premultiply_alpha, + color_space_conversion, + resize_quality, + image_bitmap_source, + mime_type, + } +} + #[op2] #[serde] #[allow(clippy::too_many_arguments)] @@ -256,19 +358,47 @@ pub(super) fn op_create_image_bitmap( #[buffer] buf: JsBuffer, width: u32, height: u32, - sx: Option, - sy: Option, - sw: Option, - sh: Option, - #[serde] image_orientation: ImageOrientation, - #[serde] premultiply_alpha: PremultiplyAlpha, - #[serde] color_space_conversion: ColorSpaceConversion, - resize_width: Option, - resize_height: Option, - #[serde] resize_quality: ResizeQuality, - #[serde] image_bitmap_source: ImageBitmapSource, - #[string] mime_type: &str, + sx: i32, + sy: i32, + sw: i32, + sh: i32, + image_orientation: u8, + premultiply_alpha: u8, + color_space_conversion: u8, + resize_width: u32, + resize_height: u32, + resize_quality: u8, + image_bitmap_source: u8, + mime_type: u8, ) -> Result<(ToJsBuffer, u32, u32), AnyError> { + let ParsedArgs { + resize_width, + resize_height, + sx, + sy, + sw, + sh, + image_orientation, + premultiply_alpha, + color_space_conversion, + resize_quality, + image_bitmap_source, + mime_type, + } = parse_args( + sx, + sy, + sw, + sh, + image_orientation, + premultiply_alpha, + color_space_conversion, + resize_width, + resize_height, + resize_quality, + image_bitmap_source, + mime_type, + ); + // 6. Switch on image: let (image, width, height, icc_profile) = decode_bitmap_data(&buf, width, height, &image_bitmap_source, mime_type)?; @@ -383,3 +513,30 @@ pub(super) fn op_create_image_bitmap( Ok((image.into_bytes().into(), output_width, output_height)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_args() { + let parsed_args = parse_args(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq!( + parsed_args, + ParsedArgs { + resize_width: None, + resize_height: None, + sx: None, + sy: None, + sw: None, + sh: None, + image_orientation: ImageOrientation::FromImage, + premultiply_alpha: PremultiplyAlpha::Default, + color_space_conversion: ColorSpaceConversion::Default, + resize_quality: ResizeQuality::Low, + image_bitmap_source: ImageBitmapSource::Blob, + mime_type: MimeType::NoMatch, + } + ); + } +} From 83c40c4f1a14dae0ac4f0378bea25d5b237e385f Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 20 Oct 2024 20:17:05 +0900 Subject: [PATCH 36/52] support `imageOrientation` correctly --- Cargo.lock | 8 +- ext/canvas/Cargo.toml | 2 +- ext/canvas/op_create_image_bitmap.rs | 262 ++++++++++++++------------- tests/unit/image_bitmap_test.ts | 10 +- 4 files changed, 148 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59da8e2b6117b3..83712d6314e0c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3825,9 +3825,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" +checksum = "bc144d44a31d753b02ce64093d532f55ff8dc4ebf2ffb8a63c0dda691385acae" dependencies = [ "bytemuck", "byteorder-lite", @@ -3842,9 +3842,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f" dependencies = [ "byteorder-lite", "quick-error 2.0.1", diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 7266923ba6f2c4..8ac5bad5b0fe53 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -18,7 +18,7 @@ bytemuck = "1.17.1" deno_core.workspace = true deno_terminal.workspace = true deno_webgpu.workspace = true -image = { version = "0.25.2", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } +image = { version = "0.25.4", default-features = false, features = ["png", "jpeg", "bmp", "ico", "webp", "gif"] } # NOTE: The qcms is a color space conversion crate which parses ICC profiles that used in Gecko, # however it supports only 8-bit color depth currently. # https://searchfox.org/mozilla-central/rev/f09e3f9603a08b5b51bf504846091579bc2ff531/gfx/qcms/src/transform.rs#130-137 diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 849889d7133fb6..de671535301d43 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -16,7 +16,9 @@ use image::codecs::png::PngDecoder; use image::codecs::webp::WebPDecoder; use image::imageops::overlay; use image::imageops::FilterType; +use image::metadata::Orientation; use image::DynamicImage; +use image::ImageDecoder; use image::ImageError; use image::RgbaImage; @@ -72,7 +74,8 @@ enum MimeType { Webp, } -type DecodeBitmapDataReturn = (DynamicImage, u32, u32, Option>); +type DecodeBitmapDataReturn = + (DynamicImage, u32, u32, Option, Option>); fn decode_bitmap_data( buf: &[u8], @@ -81,104 +84,117 @@ fn decode_bitmap_data( image_bitmap_source: &ImageBitmapSource, mime_type: MimeType, ) -> Result { - let (image, width, height, icc_profile) = match image_bitmap_source { - ImageBitmapSource::Blob => { - fn image_decoding_error(error: ImageError) -> AnyError { - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &error.to_string(), - )) - .into() - } - let (image, icc_profile) = match mime_type { - // Should we support the "image/apng" MIME type here? - MimeType::Png => { - let mut decoder: PngDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) - } - MimeType::Jpeg => { - let mut decoder: JpegDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) - } - MimeType::Gif => { - let mut decoder: GifDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) - } - MimeType::Bmp => { - let mut decoder: BmpDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) - } - MimeType::Ico => { - let mut decoder: IcoDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) - } - MimeType::Webp => { - let mut decoder: WebPDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; - let icc_profile = decoder.get_icc_profile(); - ( - decoder.to_intermediate_image(image_decoding_error)?, - icc_profile, - ) + let (image, width, height, orientation, icc_profile) = + match image_bitmap_source { + ImageBitmapSource::Blob => { + fn image_decoding_error(error: ImageError) -> AnyError { + DOMExceptionInvalidStateError::new(&image_error_message( + "decoding", + &error.to_string(), + )) + .into() } - // This pattern is unreachable due to current block is already checked by the ImageBitmapSource above. - MimeType::NoMatch => unreachable!(), - }; - - let width = image.width(); - let height = image.height(); - - (image, width, height, icc_profile) - } - ImageBitmapSource::ImageData => { - // > 4.12.5.1.15 Pixel manipulation - // > imagedata.data - // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. - // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation - let image = match RgbaImage::from_raw(width, height, buf.into()) { + let (image, orientation, icc_profile) = match mime_type { + // Should we support the "image/apng" MIME type here? + MimeType::Png => { + let mut decoder: PngDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + MimeType::Jpeg => { + let mut decoder: JpegDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + MimeType::Gif => { + let mut decoder: GifDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + MimeType::Bmp => { + let mut decoder: BmpDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + MimeType::Ico => { + let mut decoder: IcoDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + MimeType::Webp => { + let mut decoder: WebPDecoder = + ImageDecoderFromReader::to_decoder( + BufReader::new(Cursor::new(buf)), + image_decoding_error, + )?; + let orientation = decoder.orientation()?; + let icc_profile = decoder.get_icc_profile(); + ( + decoder.to_intermediate_image(image_decoding_error)?, + orientation, + icc_profile, + ) + } + // This pattern is unreachable due to current block is already checked by the ImageBitmapSource above. + MimeType::NoMatch => unreachable!(), + }; + + let width = image.width(); + let height = image.height(); + + (image, width, height, Some(orientation), icc_profile) + } + ImageBitmapSource::ImageData => { + // > 4.12.5.1.15 Pixel manipulation + // > imagedata.data + // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. + // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation + let image = match RgbaImage::from_raw(width, height, buf.into()) { Some(image) => image.into(), None => { return Err(type_error(image_error_message( @@ -188,11 +204,11 @@ fn decode_bitmap_data( } }; - (image, width, height, None) - } - }; + (image, width, height, None, None) + } + }; - Ok((image, width, height, icc_profile)) + Ok((image, width, height, orientation, icc_profile)) } /// According to the spec, it's not clear how to handle the color space conversion. @@ -400,7 +416,7 @@ pub(super) fn op_create_image_bitmap( ); // 6. Switch on image: - let (image, width, height, icc_profile) = + let (image, width, height, orientation, icc_profile) = decode_bitmap_data(&buf, width, height, &image_bitmap_source, mime_type)?; // crop bitmap data @@ -480,27 +496,29 @@ pub(super) fn op_create_image_bitmap( }; // should use resize_exact // https://github.com/image-rs/image/issues/1220#issuecomment-632060015 - let image = image.resize_exact(output_width, output_height, filter_type); + let mut image = image.resize_exact(output_width, output_height, filter_type); // 8. - // issues for imageOrientation - // https://github.com/whatwg/html/issues/8085 - // https://github.com/whatwg/html/issues/7210 - // https://github.com/whatwg/html/issues/8118 - let image = if image_orientation == ImageOrientation::FlipY { - image.flipv() - } else { - // TODO(Hajime-san): If the EXIF ​​contains image orientation information, decode the image in the appropriate orientation. - // The image create is expected to release this feature soon. - // https://github.com/image-rs/image/blob/bd0f7451a367de7ae3d898dcf1e96e9d0a1c4fa1/CHANGES.md#version-0253 - // https://github.com/image-rs/image/issues/1958 - // https://github.com/image-rs/image/pull/2299 - // https://github.com/image-rs/image/pull/2328 - // if image_bitmap_source == ImageBitmapSource::Blob { - // image.apply_orientation() - // } else { - image - // } + let image = match image_bitmap_source { + ImageBitmapSource::Blob => { + // Note: According to browser behavior and wpt results, if Exif contains image orientation, + // it applies the rotation from it before following the value of imageOrientation. + // This is not stated in the spec but in MDN currently. + // https://github.com/mdn/content/pull/34366 + + // SAFETY: The orientation is always Some if the image is from a Blob. + let orientation = orientation.unwrap(); + DynamicImage::apply_orientation(&mut image, orientation); + + match image_orientation { + ImageOrientation::FlipY => image.flipv(), + ImageOrientation::FromImage => image, + } + } + ImageBitmapSource::ImageData => match image_orientation { + ImageOrientation::FlipY => image.flipv(), + ImageOrientation::FromImage => image, + }, }; // 9. diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index bb703580fd9476..08cba061006be1 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertNotEquals, assertRejects } from "./test_util.ts"; +import { assertEquals, assertRejects } from "./test_util.ts"; const prefix = "tests/testdata/image"; @@ -112,9 +112,7 @@ Deno.test("imageOrientation", async (t) => { START, END, ); - // FIXME: When the implementation is fixed, fix this to assertEquals - // However, in the case of jpg images, numerical errors may occur. - assertNotEquals(targetPixel, new Uint8Array([255, 1, 0])); + assertEquals(targetPixel, new Uint8Array([253, 0, 0])); }); // reference: // https://github.com/web-platform-tests/wpt/blob/a1f4bbf4c6e1a9a861a145a34cd097ea260b5a49/html/canvas/element/manual/imagebitmap/createImageBitmap-exif-orientation.html#L55 @@ -127,9 +125,7 @@ Deno.test("imageOrientation", async (t) => { START, END, ); - // FIXME: When the implementation is fixed, fix this to assertEquals - // However, in the case of jpg images, numerical errors may occur. - assertNotEquals(targetPixel, new Uint8Array([254, 128, 129])); + assertEquals(targetPixel, new Uint8Array([253, 127, 127])); }); }); From cb72235e2b6676813ed1ca2aa191a6552c8a86f4 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 20 Oct 2024 20:21:01 +0900 Subject: [PATCH 37/52] remove unnecessary boilerplate --- ext/canvas/image_decoder.rs | 8 -------- ext/canvas/op_create_image_bitmap.rs | 12 ++++++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs index 02680da8e367d9..31a6ac2c70f34c 100644 --- a/ext/canvas/image_decoder.rs +++ b/ext/canvas/image_decoder.rs @@ -13,7 +13,6 @@ use image::codecs::jpeg::JpegDecoder; use image::codecs::png::PngDecoder; use image::codecs::webp::WebPDecoder; use image::DynamicImage; -use image::ImageDecoder; use image::ImageError; // @@ -39,7 +38,6 @@ pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { self, error_fn: fn(ImageError) -> AnyError, ) -> Result; - fn get_icc_profile(&mut self) -> Option>; } pub(crate) type ImageDecoderFromReaderType<'a> = BufReader>; @@ -68,12 +66,6 @@ macro_rules! impl_image_decoder_from_reader { Err(err) => Err(error_fn(err)), } } - fn get_icc_profile(&mut self) -> Option> { - match self.icc_profile() { - Ok(profile) => profile, - Err(_) => None, - } - } } }; } diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index de671535301d43..2dc7d824973c2f 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -103,7 +103,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, @@ -117,7 +117,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, @@ -131,7 +131,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, @@ -145,7 +145,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, @@ -159,7 +159,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, @@ -173,7 +173,7 @@ fn decode_bitmap_data( image_decoding_error, )?; let orientation = decoder.orientation()?; - let icc_profile = decoder.get_icc_profile(); + let icc_profile = decoder.icc_profile()?; ( decoder.to_intermediate_image(image_decoding_error)?, orientation, From 3ef4cf97c85558d953561bcda707dcc0e89139d3 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sun, 20 Oct 2024 20:22:13 +0900 Subject: [PATCH 38/52] add comment --- ext/canvas/image_decoder.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs index 31a6ac2c70f34c..7e2464b22bd074 100644 --- a/ext/canvas/image_decoder.rs +++ b/ext/canvas/image_decoder.rs @@ -27,13 +27,18 @@ use image::ImageError; // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 // +/// A trait for converting a reader into an image decoder. +/// This trait is used to abstract over the different image decoders and +/// provide thin helpers to handle errors for the runtime. pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { + /// Create a new image decoder from a reader. fn to_decoder( reader: R, error_fn: fn(ImageError) -> AnyError, ) -> Result where Self: Sized; + /// Convert the image decoder into an intermediate image(DynamicImage). fn to_intermediate_image( self, error_fn: fn(ImageError) -> AnyError, From 50bee86fd56cd2e02de1940b9a9e926740578d30 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:40:31 +0900 Subject: [PATCH 39/52] update error --- ext/canvas/error.rs | 44 ----------------------- ext/canvas/image_decoder.rs | 19 +++++----- ext/canvas/image_ops.rs | 52 ++++++++++++---------------- ext/canvas/lib.rs | 13 +++++-- ext/canvas/op_create_image_bitmap.rs | 28 ++++++--------- runtime/errors.rs | 3 +- 6 files changed, 56 insertions(+), 103 deletions(-) delete mode 100644 ext/canvas/error.rs diff --git a/ext/canvas/error.rs b/ext/canvas/error.rs deleted file mode 100644 index e2d160534552dc..00000000000000 --- a/ext/canvas/error.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use deno_core::error::AnyError; -use std::borrow::Cow; -use std::fmt; - -#[derive(Debug)] -pub struct DOMExceptionInvalidStateError { - pub msg: String, -} - -impl DOMExceptionInvalidStateError { - pub fn new(msg: &str) -> Self { - DOMExceptionInvalidStateError { - msg: msg.to_string(), - } - } -} - -impl fmt::Display for DOMExceptionInvalidStateError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} - -impl std::error::Error for DOMExceptionInvalidStateError {} - -pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { - e.downcast_ref::() - .map(|_| "DOMExceptionInvalidStateError") -} - -/// Returns a string that represents the error message for the image. -pub(crate) fn image_error_message<'a, T: Into>>( - opreation: T, - reason: T, -) -> String { - format!( - "An error has occurred while {}. -reason: {}", - opreation.into(), - reason.into(), - ) -} diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs index 7e2464b22bd074..af56bff37f6bcf 100644 --- a/ext/canvas/image_decoder.rs +++ b/ext/canvas/image_decoder.rs @@ -5,7 +5,6 @@ use std::io::BufReader; use std::io::Cursor; use std::io::Seek; -use deno_core::error::AnyError; use image::codecs::bmp::BmpDecoder; use image::codecs::gif::GifDecoder; use image::codecs::ico::IcoDecoder; @@ -15,6 +14,8 @@ use image::codecs::webp::WebPDecoder; use image::DynamicImage; use image::ImageError; +use crate::CanvasError; + // // About the animated image // > Blob .4 @@ -34,15 +35,15 @@ pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { /// Create a new image decoder from a reader. fn to_decoder( reader: R, - error_fn: fn(ImageError) -> AnyError, - ) -> Result + error_fn: fn(ImageError) -> CanvasError, + ) -> Result where Self: Sized; /// Convert the image decoder into an intermediate image(DynamicImage). fn to_intermediate_image( self, - error_fn: fn(ImageError) -> AnyError, - ) -> Result; + error_fn: fn(ImageError) -> CanvasError, + ) -> Result; } pub(crate) type ImageDecoderFromReaderType<'a> = BufReader>; @@ -52,8 +53,8 @@ macro_rules! impl_image_decoder_from_reader { impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { fn to_decoder( reader: R, - error_fn: fn(ImageError) -> AnyError, - ) -> Result + error_fn: fn(ImageError) -> CanvasError, + ) -> Result where Self: Sized, { @@ -64,8 +65,8 @@ macro_rules! impl_image_decoder_from_reader { } fn to_intermediate_image( self, - error_fn: fn(ImageError) -> AnyError, - ) -> Result { + error_fn: fn(ImageError) -> CanvasError, + ) -> Result { match DynamicImage::from_decoder(self) { Ok(image) => Ok(image), Err(err) => Err(error_fn(err)), diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index 8a8a8ef04e1f78..bc2853455049f1 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -2,8 +2,6 @@ use bytemuck::cast_slice; use bytemuck::cast_slice_mut; -use deno_core::error::type_error; -use deno_core::error::AnyError; use image::ColorType; use image::DynamicImage; use image::GenericImageView; @@ -21,14 +19,7 @@ use lcms2::Transform; use num_traits::NumCast; use num_traits::SaturatingMul; -use crate::error::image_error_message; - -/// Image formats that is 32-bit depth are not supported currently due to the following reasons: -/// - e.g. OpenEXR, it's not covered by the spec. -/// - JPEG XL supported by WebKit, but it cannot be called a standard today. -/// https://github.com/whatwg/mimesniff/issues/143 -const NOT_SUPPORTED_BIT_DEPTH: &str = - "The 32-bit depth image format is not supported."; +use crate::CanvasError; pub(crate) trait PremultiplyAlpha { fn premultiply_alpha(&self) -> Self; @@ -99,7 +90,7 @@ where /// Premultiply the alpha channel of the image. pub(crate) fn premultiply_alpha( image: DynamicImage, -) -> Result { +) -> Result { match image { DynamicImage::ImageLumaA8(image) => { Ok(process_premultiply_alpha(&image).into()) @@ -113,11 +104,11 @@ pub(crate) fn premultiply_alpha( DynamicImage::ImageRgba16(image) => { Ok(process_premultiply_alpha(&image).into()) } - DynamicImage::ImageRgb32F(_) | DynamicImage::ImageRgba32F(_) => { - Err(type_error(image_error_message( - "processing premultiply alpha", - NOT_SUPPORTED_BIT_DEPTH, - ))) + DynamicImage::ImageRgb32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) + } + DynamicImage::ImageRgba32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) } // If the image does not have an alpha channel, return the image as is. _ => Ok(image), @@ -230,7 +221,7 @@ where /// Invert the premultiplied alpha channel of the image. pub(crate) fn unpremultiply_alpha( image: DynamicImage, -) -> Result { +) -> Result { match image { DynamicImage::ImageLumaA8(image) => Ok(if is_premultiplied_alpha(&image) { process_unpremultiply_alpha(&image).into() @@ -254,11 +245,11 @@ pub(crate) fn unpremultiply_alpha( } else { image.into() }), - DynamicImage::ImageRgb32F(_) | DynamicImage::ImageRgba32F(_) => { - Err(type_error(image_error_message( - "processing un-premultiply alpha", - NOT_SUPPORTED_BIT_DEPTH, - ))) + DynamicImage::ImageRgb32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) + } + DynamicImage::ImageRgba32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) } // If the image does not have an alpha channel, return the image as is. _ => Ok(image), @@ -366,7 +357,7 @@ where ColorType::Rgb16 => PixelFormat::RGB_16, ColorType::Rgba8 => PixelFormat::RGBA_8, ColorType::Rgba16 => PixelFormat::RGBA_16, - _ => unreachable!("{}", NOT_SUPPORTED_BIT_DEPTH), + _ => unreachable!("{}", CanvasError::UnsupportedColorType(color)), }; let transformer = Transform::new( &input_icc_profile, @@ -393,7 +384,7 @@ where pub(crate) fn to_srgb_from_icc_profile( image: DynamicImage, icc_profile: Option>, -) -> Result { +) -> Result { match icc_profile { // If there is no color profile information, return the image as is. None => Ok(image), @@ -476,10 +467,13 @@ pub(crate) fn to_srgb_from_icc_profile( ) .into(), ), - _ => Err(type_error(image_error_message( - "processing un-premultiply alpha", - NOT_SUPPORTED_BIT_DEPTH, - ))), + DynamicImage::ImageRgb32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) + } + DynamicImage::ImageRgba32F(_) => { + Err(CanvasError::UnsupportedColorType(image.color())) + } + _ => Err(CanvasError::UnsupportedColorType(image.color())), } } }, @@ -635,7 +629,7 @@ pub(crate) fn to_srgb_from_icc_profile( // } // /// Convert the color space of the image from sRGB to Display-P3. -// fn srgb_to_display_p3(image: DynamicImage) -> Result { +// fn srgb_to_display_p3(image: DynamicImage) -> Result { // match image { // // The conversion of the lumincance color types to the display-p3 color space is meaningless. // DynamicImage::ImageLuma8(_) diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 389ed58ff329f2..5124c60043432b 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; -pub mod error; mod image_decoder; mod image_ops; mod op_create_image_bitmap; @@ -11,9 +10,19 @@ use op_create_image_bitmap::op_create_image_bitmap; #[derive(Debug, thiserror::Error)] pub enum CanvasError { - #[error("Color type '{0:?}' not supported")] + /// Image formats that is 32-bit depth are not supported currently due to the following reasons: + /// - e.g. OpenEXR, it's not covered by the spec. + /// - JPEG XL supported by WebKit, but it cannot be called a standard today. + /// https://github.com/whatwg/mimesniff/issues/143 + /// + /// This error will be mapped to TypeError. + #[error("Unsupported color type and bit depth: '{0:?}'")] UnsupportedColorType(ColorType), + /// This error will be mapped to DOMExceptionInvalidStateError. + #[error("Cannot decode image '{0}'")] + InvalidImage(String), #[error(transparent)] + /// This error will be mapped to TypeError. Image(#[from] image::ImageError), } diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 2dc7d824973c2f..18f8fce976f5cd 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -3,8 +3,6 @@ use std::io::BufReader; use std::io::Cursor; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::ToJsBuffer; @@ -22,13 +20,12 @@ use image::ImageDecoder; use image::ImageError; use image::RgbaImage; -use crate::error::image_error_message; -use crate::error::DOMExceptionInvalidStateError; use crate::image_decoder::ImageDecoderFromReader; use crate::image_decoder::ImageDecoderFromReaderType; use crate::image_ops::premultiply_alpha as process_premultiply_alpha; use crate::image_ops::to_srgb_from_icc_profile; use crate::image_ops::unpremultiply_alpha; +use crate::CanvasError; #[derive(Debug, PartialEq)] enum ImageBitmapSource { @@ -83,16 +80,12 @@ fn decode_bitmap_data( height: u32, image_bitmap_source: &ImageBitmapSource, mime_type: MimeType, -) -> Result { +) -> Result { let (image, width, height, orientation, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { - fn image_decoding_error(error: ImageError) -> AnyError { - DOMExceptionInvalidStateError::new(&image_error_message( - "decoding", - &error.to_string(), - )) - .into() + fn image_decoding_error(error: ImageError) -> CanvasError { + CanvasError::InvalidImage(error.to_string()) } let (image, orientation, icc_profile) = match mime_type { // Should we support the "image/apng" MIME type here? @@ -197,10 +190,9 @@ fn decode_bitmap_data( let image = match RgbaImage::from_raw(width, height, buf.into()) { Some(image) => image.into(), None => { - return Err(type_error(image_error_message( - "decoding", - "The Chunk Data is not big enough with the specified width and height.", - ))) + return Err(CanvasError::InvalidImage( + "The Chunk Data is not big enough with the specified width and height.".to_string() + )) } }; @@ -229,7 +221,7 @@ fn apply_color_space_conversion( image: DynamicImage, icc_profile: Option>, color_space_conversion: &ColorSpaceConversion, -) -> Result { +) -> Result { match color_space_conversion { // return the decoded image as is. ColorSpaceConversion::None => Ok(image), @@ -243,7 +235,7 @@ fn apply_premultiply_alpha( image: DynamicImage, image_bitmap_source: &ImageBitmapSource, premultiply_alpha: &PremultiplyAlpha, -) -> Result { +) -> Result { match premultiply_alpha { // 1. PremultiplyAlpha::Default => Ok(image), @@ -386,7 +378,7 @@ pub(super) fn op_create_image_bitmap( resize_quality: u8, image_bitmap_source: u8, mime_type: u8, -) -> Result<(ToJsBuffer, u32, u32), AnyError> { +) -> Result<(ToJsBuffer, u32, u32), CanvasError> { let ParsedArgs { resize_width, resize_height, diff --git a/runtime/errors.rs b/runtime/errors.rs index 869f97968e7857..47cc40b1bdc884 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -602,7 +602,8 @@ pub fn get_cron_error_class(e: &CronError) -> &'static str { fn get_canvas_error(e: &CanvasError) -> &'static str { match e { CanvasError::UnsupportedColorType(_) => "TypeError", - CanvasError::Image(_) => "Error", + CanvasError::InvalidImage(_) => "DOMExceptionInvalidStateError", + CanvasError::Image(_) => "TypeError", } } From 1429013dd15708d9e0f26661f261639661553cd2 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:55:34 +0900 Subject: [PATCH 40/52] map lcms error --- ext/canvas/image_ops.rs | 10 ++++------ ext/canvas/lib.rs | 2 ++ runtime/errors.rs | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index bc2853455049f1..baf5b007019412 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -365,14 +365,12 @@ where &output_icc_profile, pixel_format, output_icc_profile.header_rendering_intent(), - ); + ) + .map_err(CanvasError::Lcms) + .unwrap(); for (x, y, mut pixel) in image.pixels() { - let pixel = match transformer { - Ok(ref transformer) => pixel.transform_color_profile(transformer), - // This arm will reach when the ffi call fails. - Err(_) => pixel, - }; + let pixel = pixel.transform_color_profile(&transformer); out.put_pixel(x, y, pixel); } diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 5124c60043432b..3e1f0424f969b9 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -22,6 +22,8 @@ pub enum CanvasError { #[error("Cannot decode image '{0}'")] InvalidImage(String), #[error(transparent)] + Lcms(#[from] lcms2::Error), + #[error(transparent)] /// This error will be mapped to TypeError. Image(#[from] image::ImageError), } diff --git a/runtime/errors.rs b/runtime/errors.rs index 47cc40b1bdc884..73f7a822f3480f 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -603,6 +603,7 @@ fn get_canvas_error(e: &CanvasError) -> &'static str { match e { CanvasError::UnsupportedColorType(_) => "TypeError", CanvasError::InvalidImage(_) => "DOMExceptionInvalidStateError", + CanvasError::Lcms(_) => "TypeError", CanvasError::Image(_) => "TypeError", } } From 40c89cc30a5a2f5a699f759da825ec4319bb5c28 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:06:33 +0900 Subject: [PATCH 41/52] remove TODO - https://github.com/denoland/deno/pull/25517#discussion_r1855085116 --- ext/canvas/01_image.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 3755df1d6fb78b..87446a6f1c0ca9 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -323,7 +323,6 @@ info: The following MIME types are supported: resizeQuality = 3; } - // TODO(Hajime-san): this should be real async const processedImage = op_create_image_bitmap( buf, width, From ad6775058740439b08cdc4cb1113eabf778ec9b4 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:36:10 +0900 Subject: [PATCH 42/52] move suggestion to `runtime/format_errors.rs` - https://github.com/denoland/deno/pull/25517#discussion_r1801666281 - https://github.com/denoland/deno/pull/25517#discussion_r1801666872 --- ext/canvas/01_image.js | 10 ++-------- runtime/fmt_errors.rs | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 87446a6f1c0ca9..2fb634aa892ac5 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -262,20 +262,14 @@ function createImageBitmap( } else if (mimeTypeString === "") { return PromiseReject( new DOMException( - `The MIME type of source image is not specified. -hint: When you want to get a "Blob" from "fetch", make sure to go through a file server that returns the appropriate content-type response header, - and specify the URL to the file server like "await(await fetch('http://localhost:8000/sample.png').blob()". - Alternatively, if you are reading a local file using 'Deno.readFile' etc., - set the appropriate MIME type like "new Blob([await Deno.readFile('sample.png')], { type: 'image/png' })".\n`, + `The MIME type of source image is not specified.`, "InvalidStateError", ), ); } else { return PromiseReject( new DOMException( - `The the MIME type ${mimeTypeString} of source image is not a supported format. -info: The following MIME types are supported: - https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n`, + `The MIME type ${mimeTypeString} of source image is not a supported format.`, "InvalidStateError", ), ); diff --git a/runtime/fmt_errors.rs b/runtime/fmt_errors.rs index 7c6cf6d397f2f5..743e33e6d9a24e 100644 --- a/runtime/fmt_errors.rs +++ b/runtime/fmt_errors.rs @@ -404,6 +404,28 @@ fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { ]), ]; } + // See: ext/canvas/01_image.js + else if msg.contains("The MIME type of source image is not specified") { + return vec![ + FixSuggestion::hint_multiline(&[ + cstr!("When you want to get a Blob from fetch, make sure to go through a file server that returns the appropriate content-type response header,"), + cstr!("and specify the URL to the file server like await(await fetch('http://localhost:8000/sample.png').blob()."), + cstr!("Alternatively, if you are reading a local file using Deno.readFile etc.,"), + cstr!("set the appropriate MIME type like new Blob([await Deno.readFile('sample.png')], { type: 'image/png' }).") + ]), + ]; + } + // See: ext/canvas/01_image.js + else if msg.contains("The MIME type") + && msg.contains("of source image is not a supported format") + { + return vec![ + FixSuggestion::info( + "The following MIME types are supported." + ), + FixSuggestion::docs("https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm"), + ]; + } } vec![] From 3fcc3ab8e486452a9b04a6bd0bda064993138b9c Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:37:14 +0900 Subject: [PATCH 43/52] remove comment out - https://github.com/denoland/deno/pull/25517#discussion_r1855078601 --- ext/canvas/image_ops.rs | 199 ---------------------------------------- 1 file changed, 199 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index baf5b007019412..c4cbc125d67c70 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -478,181 +478,6 @@ pub(crate) fn to_srgb_from_icc_profile( } } -// NOTE: The following code is not used in the current implementation, -// but it is left as a reference for future use about implementing CanvasRenderingContext2D. -// https://github.com/denoland/deno/issues/5701#issuecomment-1833304511 - -// // reference -// // https://www.w3.org/TR/css-color-4/#color-conversion-code -// fn srgb_to_linear(value: T) -> f32 { -// if value.to_f32().unwrap() <= 0.04045 { -// value.to_f32().unwrap() / 12.92 -// } else { -// ((value.to_f32().unwrap() + 0.055) / 1.055).powf(2.4) -// } -// } - -// // reference -// // https://www.w3.org/TR/css-color-4/#color-conversion-code -// fn linear_to_display_p3(value: T) -> f32 { -// if value.to_f32().unwrap() <= 0.0031308 { -// value.to_f32().unwrap() * 12.92 -// } else { -// 1.055 * value.to_f32().unwrap().powf(1.0 / 2.4) - 0.055 -// } -// } - -// fn normalize_value_to_0_1(value: T) -> f32 { -// value.to_f32().unwrap() / T::DEFAULT_MAX_VALUE.to_f32().unwrap() -// } - -// fn unnormalize_value_from_0_1(value: f32) -> T { -// NumCast::from( -// (value.clamp(0.0, 1.0) * T::DEFAULT_MAX_VALUE.to_f32().unwrap()).round(), -// ) -// .unwrap() -// } - -// fn apply_conversion_matrix_srgb_to_display_p3( -// r: T, -// g: T, -// b: T, -// ) -> (T, T, T) { -// // normalize the value to 0.0 - 1.0 -// let (r, g, b) = ( -// normalize_value_to_0_1(r), -// normalize_value_to_0_1(g), -// normalize_value_to_0_1(b), -// ); - -// // sRGB -> Linear RGB -// let (r, g, b) = (srgb_to_linear(r), srgb_to_linear(g), srgb_to_linear(b)); - -// // Display-P3 (RGB) -> Display-P3 (XYZ) -// // -// // inv[ P3-D65 (D65) to XYZ ] * [ sRGB (D65) to XYZ ] -// // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html -// // https://fujiwaratko.sakura.ne.jp/infosci/colorspace/colorspace2_e.html - -// // [ sRGB (D65) to XYZ ] -// #[rustfmt::skip] -// let (m1x, m1y, m1z) = ( -// [0.4124564, 0.3575761, 0.1804375], -// [0.2126729, 0.7151522, 0.0721750], -// [0.0193339, 0.119_192, 0.9503041], -// ); - -// let (r, g, b) = ( -// r * m1x[0] + g * m1x[1] + b * m1x[2], -// r * m1y[0] + g * m1y[1] + b * m1y[2], -// r * m1z[0] + g * m1z[1] + b * m1z[2], -// ); - -// // inv[ P3-D65 (D65) to XYZ ] -// #[rustfmt::skip] -// let (m2x, m2y, m2z) = ( -// [ 2.493_497, -0.931_383_6, -0.402_710_8 ], -// [ -0.829_489, 1.762_664_1, 0.023_624_687 ], -// [ 0.035_845_83, -0.076_172_39, 0.956_884_5 ], -// ); - -// let (r, g, b) = ( -// r * m2x[0] + g * m2x[1] + b * m2x[2], -// r * m2y[0] + g * m2y[1] + b * m2y[2], -// r * m2z[0] + g * m2z[1] + b * m2z[2], -// ); - -// // This calculation is similar as above that it is a little faster, but less accurate. -// // let r = 0.8225 * r + 0.1774 * g + 0.0000 * b; -// // let g = 0.0332 * r + 0.9669 * g + 0.0000 * b; -// // let b = 0.0171 * r + 0.0724 * g + 0.9108 * b; - -// // Display-P3 (Linear) -> Display-P3 -// let (r, g, b) = ( -// linear_to_display_p3(r), -// linear_to_display_p3(g), -// linear_to_display_p3(b), -// ); - -// // unnormalize the value from 0.0 - 1.0 -// ( -// unnormalize_value_from_0_1(r), -// unnormalize_value_from_0_1(g), -// unnormalize_value_from_0_1(b), -// ) -// } - -// trait ColorSpaceConversion { -// /// Display P3 Color Encoding (v 1.0) -// /// https://www.color.org/chardata/rgb/DisplayP3.xalter -// fn srgb_to_display_p3(&self) -> Self; -// } - -// impl ColorSpaceConversion for Rgb { -// fn srgb_to_display_p3(&self) -> Self { -// let (r, g, b) = (self.0[0], self.0[1], self.0[2]); - -// let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); - -// Rgb([r, g, b]) -// } -// } - -// impl ColorSpaceConversion for Rgba { -// fn srgb_to_display_p3(&self) -> Self { -// let (r, g, b, a) = (self.0[0], self.0[1], self.0[2], self.0[3]); - -// let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(r, g, b); - -// Rgba([r, g, b, a]) -// } -// } - -// fn process_srgb_to_display_p3(image: &I) -> ImageBuffer> -// where -// I: GenericImageView, -// P: Pixel + ColorSpaceConversion + 'static, -// S: Primitive + 'static, -// { -// let (width, height) = image.dimensions(); -// let mut out = ImageBuffer::new(width, height); - -// for (x, y, pixel) in image.pixels() { -// let pixel = pixel.srgb_to_display_p3(); - -// out.put_pixel(x, y, pixel); -// } - -// out -// } - -// /// Convert the color space of the image from sRGB to Display-P3. -// fn srgb_to_display_p3(image: DynamicImage) -> Result { -// match image { -// // The conversion of the lumincance color types to the display-p3 color space is meaningless. -// DynamicImage::ImageLuma8(_) -// | DynamicImage::ImageLuma16(_) -// | DynamicImage::ImageLumaA8(_) -// | DynamicImage::ImageLumaA16(_) => Ok(image), -// DynamicImage::ImageRgb8(image) => { -// Ok(process_srgb_to_display_p3(&image).into()) -// } -// DynamicImage::ImageRgb16(image) => { -// Ok(process_srgb_to_display_p3(&image).into()) -// } -// DynamicImage::ImageRgba8(image) => { -// Ok(process_srgb_to_display_p3(&image).into()) -// } -// DynamicImage::ImageRgba16(image) => { -// Ok(process_srgb_to_display_p3(&image).into()) -// } -// _ => Err(type_error(image_error_message( -// "processing ICC color profile conversion to sRGB", -// NOT_SUPPORTED_BIT_DEPTH, -// ))), -// } -// } - #[cfg(test)] mod tests { use super::*; @@ -675,28 +500,4 @@ mod tests { let rgba = rgba.unpremultiply_alpha(); assert_eq!(rgba, Rgba::([255, 0, 0, 127])); } - - // #[test] - // fn test_apply_conversion_matrix_srgb_to_display_p3() { - // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(255_u8, 0, 0); - // assert_eq!(r, 234); - // assert_eq!(g, 51); - // assert_eq!(b, 35); - - // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 255, 0); - // assert_eq!(r, 117); - // assert_eq!(g, 251); - // assert_eq!(b, 76); - - // let (r, g, b) = apply_conversion_matrix_srgb_to_display_p3(0_u8, 0, 255); - // assert_eq!(r, 0); - // assert_eq!(g, 0); - // assert_eq!(b, 245); - - // let (r, g, b) = - // apply_conversion_matrix_srgb_to_display_p3(255_u8, 255, 255); - // assert_eq!(r, 255); - // assert_eq!(g, 255); - // assert_eq!(b, 255); - // } } From 24306b3ed6d2cc358f0a868836522683341fe5de Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 11:46:42 +0900 Subject: [PATCH 44/52] remove comment - https://github.com/denoland/deno/pull/25517#discussion_r1855088648 --- ext/canvas/op_create_image_bitmap.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 18f8fce976f5cd..421a504081c237 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -88,7 +88,6 @@ fn decode_bitmap_data( CanvasError::InvalidImage(error.to_string()) } let (image, orientation, icc_profile) = match mime_type { - // Should we support the "image/apng" MIME type here? MimeType::Png => { let mut decoder: PngDecoder = ImageDecoderFromReader::to_decoder( From 332fc416e8d9de839aedea0f77120cf118592c65 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:24:45 +0900 Subject: [PATCH 45/52] remove boilerplate - https://github.com/denoland/deno/pull/25517#discussion_r1855114381 --- ext/canvas/image_decoder.rs | 87 ---------------------------- ext/canvas/lib.rs | 8 ++- ext/canvas/op_create_image_bitmap.rs | 82 +++++++++++++------------- 3 files changed, 47 insertions(+), 130 deletions(-) delete mode 100644 ext/canvas/image_decoder.rs diff --git a/ext/canvas/image_decoder.rs b/ext/canvas/image_decoder.rs deleted file mode 100644 index af56bff37f6bcf..00000000000000 --- a/ext/canvas/image_decoder.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::io::BufRead; -use std::io::BufReader; -use std::io::Cursor; -use std::io::Seek; - -use image::codecs::bmp::BmpDecoder; -use image::codecs::gif::GifDecoder; -use image::codecs::ico::IcoDecoder; -use image::codecs::jpeg::JpegDecoder; -use image::codecs::png::PngDecoder; -use image::codecs::webp::WebPDecoder; -use image::DynamicImage; -use image::ImageError; - -use crate::CanvasError; - -// -// About the animated image -// > Blob .4 -// > ... If this is an animated image, imageBitmap's bitmap data must only be taken from -// > the default image of the animation (the one that the format defines is to be used when animation is -// > not supported or is disabled), or, if there is no such image, the first frame of the animation. -// https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html -// -// see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) -// https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 -// - -/// A trait for converting a reader into an image decoder. -/// This trait is used to abstract over the different image decoders and -/// provide thin helpers to handle errors for the runtime. -pub(crate) trait ImageDecoderFromReader<'a, R: BufRead + Seek> { - /// Create a new image decoder from a reader. - fn to_decoder( - reader: R, - error_fn: fn(ImageError) -> CanvasError, - ) -> Result - where - Self: Sized; - /// Convert the image decoder into an intermediate image(DynamicImage). - fn to_intermediate_image( - self, - error_fn: fn(ImageError) -> CanvasError, - ) -> Result; -} - -pub(crate) type ImageDecoderFromReaderType<'a> = BufReader>; - -macro_rules! impl_image_decoder_from_reader { - ($decoder:ty, $reader:ty) => { - impl<'a, R: BufRead + Seek> ImageDecoderFromReader<'a, R> for $decoder { - fn to_decoder( - reader: R, - error_fn: fn(ImageError) -> CanvasError, - ) -> Result - where - Self: Sized, - { - match <$decoder>::new(reader) { - Ok(decoder) => Ok(decoder), - Err(err) => return Err(error_fn(err)), - } - } - fn to_intermediate_image( - self, - error_fn: fn(ImageError) -> CanvasError, - ) -> Result { - match DynamicImage::from_decoder(self) { - Ok(image) => Ok(image), - Err(err) => Err(error_fn(err)), - } - } - } - }; -} - -// If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. -impl_image_decoder_from_reader!(PngDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(JpegDecoder, ImageDecoderFromReaderType); -// The GifDecoder decodes the first frame. -impl_image_decoder_from_reader!(GifDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(BmpDecoder, ImageDecoderFromReaderType); -impl_image_decoder_from_reader!(IcoDecoder, ImageDecoderFromReaderType); -// The WebPDecoder decodes the first frame. -impl_image_decoder_from_reader!(WebPDecoder, ImageDecoderFromReaderType); diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 3e1f0424f969b9..347b12eb338d46 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; -mod image_decoder; mod image_ops; mod op_create_image_bitmap; use image::ColorType; @@ -28,6 +27,13 @@ pub enum CanvasError { Image(#[from] image::ImageError), } +impl CanvasError { + /// Convert an [`image::ImageError`] to an [`CanvasError::InvalidImage`]. + fn image_error_to_invalid_image(error: image::ImageError) -> Self { + Self::InvalidImage(error.to_string()) + } +} + deno_core::extension!( deno_canvas, deps = [deno_webidl, deno_web, deno_webgpu], diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 421a504081c237..d0e3a0e27ab684 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -17,11 +17,8 @@ use image::imageops::FilterType; use image::metadata::Orientation; use image::DynamicImage; use image::ImageDecoder; -use image::ImageError; use image::RgbaImage; -use crate::image_decoder::ImageDecoderFromReader; -use crate::image_decoder::ImageDecoderFromReaderType; use crate::image_ops::premultiply_alpha as process_premultiply_alpha; use crate::image_ops::to_srgb_from_icc_profile; use crate::image_ops::unpremultiply_alpha; @@ -84,90 +81,91 @@ fn decode_bitmap_data( let (image, width, height, orientation, icc_profile) = match image_bitmap_source { ImageBitmapSource::Blob => { - fn image_decoding_error(error: ImageError) -> CanvasError { - CanvasError::InvalidImage(error.to_string()) - } + // + // About the animated image + // > Blob .4 + // > ... If this is an animated image, imageBitmap's bitmap data must only be taken from + // > the default image of the animation (the one that the format defines is to be used when animation is + // > not supported or is disabled), or, if there is no such image, the first frame of the animation. + // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html + // + // see also browser implementations: (The implementation of Gecko and WebKit is hard to read.) + // https://source.chromium.org/chromium/chromium/src/+/bdbc054a6cabbef991904b5df9066259505cc686:third_party/blink/renderer/platform/image-decoders/image_decoder.h;l=175-189 + // let (image, orientation, icc_profile) = match mime_type { MimeType::Png => { - let mut decoder: PngDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + // If PngDecoder decodes an animated image, it returns the default image if one is set, or the first frame if not. + let mut decoder = PngDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) } MimeType::Jpeg => { - let mut decoder: JpegDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + let mut decoder = + JpegDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) } MimeType::Gif => { - let mut decoder: GifDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + // The GifDecoder decodes the first frame. + let mut decoder = GifDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) } MimeType::Bmp => { - let mut decoder: BmpDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + let mut decoder = BmpDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) } MimeType::Ico => { - let mut decoder: IcoDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + let mut decoder = IcoDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) } MimeType::Webp => { - let mut decoder: WebPDecoder = - ImageDecoderFromReader::to_decoder( - BufReader::new(Cursor::new(buf)), - image_decoding_error, - )?; + // The WebPDecoder decodes the first frame. + let mut decoder = + WebPDecoder::new(BufReader::new(Cursor::new(buf))) + .map_err(CanvasError::image_error_to_invalid_image)?; let orientation = decoder.orientation()?; let icc_profile = decoder.icc_profile()?; ( - decoder.to_intermediate_image(image_decoding_error)?, + DynamicImage::from_decoder(decoder) + .map_err(CanvasError::image_error_to_invalid_image)?, orientation, icc_profile, ) From d764f628f5d3f7dde916ea4d57a3cb960b6709cf Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 19:51:28 +0900 Subject: [PATCH 46/52] use an unique variant of `CanvasError` - https://github.com/denoland/deno/pull/25517#discussion_r1855116556 --- ext/canvas/lib.rs | 3 +++ ext/canvas/op_create_image_bitmap.rs | 12 +++++------- runtime/errors.rs | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 347b12eb338d46..602a7d18018de3 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -20,6 +20,9 @@ pub enum CanvasError { /// This error will be mapped to DOMExceptionInvalidStateError. #[error("Cannot decode image '{0}'")] InvalidImage(String), + /// This error will be mapped to DOMExceptionInvalidStateError. + #[error("The chunk data is not big enough with the specified width: {0} and height: {1}.")] + NotBigEnoughChunk(u32, u32), #[error(transparent)] Lcms(#[from] lcms2::Error), #[error(transparent)] diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index d0e3a0e27ab684..45484d17c036de 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -185,13 +185,11 @@ fn decode_bitmap_data( // > Returns the one-dimensional array containing the data in RGBA order, as integers in the range 0 to 255. // https://html.spec.whatwg.org/multipage/canvas.html#pixel-manipulation let image = match RgbaImage::from_raw(width, height, buf.into()) { - Some(image) => image.into(), - None => { - return Err(CanvasError::InvalidImage( - "The Chunk Data is not big enough with the specified width and height.".to_string() - )) - } - }; + Some(image) => image.into(), + None => { + return Err(CanvasError::NotBigEnoughChunk(width, height)); + } + }; (image, width, height, None, None) } diff --git a/runtime/errors.rs b/runtime/errors.rs index 73f7a822f3480f..3f0fbacecde3c1 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -603,6 +603,7 @@ fn get_canvas_error(e: &CanvasError) -> &'static str { match e { CanvasError::UnsupportedColorType(_) => "TypeError", CanvasError::InvalidImage(_) => "DOMExceptionInvalidStateError", + CanvasError::NotBigEnoughChunk(_, _) => "DOMExceptionInvalidStateError", CanvasError::Lcms(_) => "TypeError", CanvasError::Image(_) => "TypeError", } From 4b9cf867c32e16d27b44126b54eaef15ebafc058 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 22:28:53 +0900 Subject: [PATCH 47/52] implement the case of `ImageBitmap` - https://github.com/denoland/deno/pull/25517#discussion_r1855082101 --- ext/canvas/01_image.js | 12 ++- ext/canvas/image_ops.rs | 106 +++++++++++++++++++++++++++ ext/canvas/lib.deno_canvas.d.ts | 2 +- ext/canvas/lib.rs | 4 + ext/canvas/op_create_image_bitmap.rs | 18 ++++- runtime/errors.rs | 1 + tests/unit/image_bitmap_test.ts | 47 ++++++++---- 7 files changed, 168 insertions(+), 22 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 2fb634aa892ac5..9d266def96b8ac 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -166,6 +166,7 @@ function createImageBitmap( const imageBitmapSources = [ "Blob", "ImageData", + "ImageBitmap", ]; // Overload: createImageBitmap(image [, options ]) @@ -220,7 +221,11 @@ function createImageBitmap( // 3. const isBlob = ObjectPrototypeIsPrototypeOf(BlobPrototype, image); const isImageData = ObjectPrototypeIsPrototypeOf(ImageDataPrototype, image); - if (!isBlob && !isImageData) { + const isImageBitmap = ObjectPrototypeIsPrototypeOf( + ImageBitmapPrototype, + image, + ); + if (!isBlob && !isImageData && !isImageBitmap) { return PromiseReject( new DOMException( `${prefix}: The provided value for 'image' is not of type '(${ @@ -279,6 +284,11 @@ function createImageBitmap( height = image[_height]; imageBitmapSource = 1; buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_data])); + } else if (isImageBitmap) { + width = image[_width]; + height = image[_height]; + imageBitmapSource = 2; + buf = new Uint8Array(TypedArrayPrototypeGetBuffer(image[_bitmapData])); } // If those options are not provided, assign 0 to mean undefined(None). diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index c4cbc125d67c70..f948f29c7609c9 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -478,6 +478,97 @@ pub(crate) fn to_srgb_from_icc_profile( } } +/// Create an image buffer from raw bytes. +fn process_image_buffer_from_raw_bytes( + width: u32, + height: u32, + buffer: &[u8], + bytes_per_pixel: usize, +) -> ImageBuffer> +where + P: Pixel + SliceToPixel + 'static, + S: Primitive + 'static, +{ + let mut out = ImageBuffer::new(width, height); + for (index, buffer) in buffer.chunks_exact(bytes_per_pixel).enumerate() { + let pixel = P::slice_to_pixel(buffer); + + out.put_pixel(index as u32, index as u32, pixel); + } + + out +} + +pub(crate) fn create_image_from_raw_bytes( + width: u32, + height: u32, + buffer: &[u8], +) -> Result { + let total_pixels = (width * height) as usize; + // avoid to divide by zero + let bytes_per_pixel = buffer + .len() + .checked_div(total_pixels) + .ok_or(CanvasError::InvalidSizeZero(width, height))?; + // convert from a bytes per pixel to the color type of the image + // https://github.com/image-rs/image/blob/2c986d353333d2604f0c3f1fcef262cc763c0001/src/color.rs#L38-L49 + match bytes_per_pixel { + 1 => Ok(DynamicImage::ImageLuma8( + process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + ), + )), + 2 => Ok( + // NOTE: ImageLumaA8 is also the same bytes per pixel. + DynamicImage::ImageLuma16(process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + )), + ), + 3 => Ok(DynamicImage::ImageRgb8( + process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + ), + )), + 4 => Ok( + // NOTE: ImageLumaA16 is also the same bytes per pixel. + DynamicImage::ImageRgba8(process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + )), + ), + 6 => Ok(DynamicImage::ImageRgb16( + process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + ), + )), + 8 => Ok(DynamicImage::ImageRgba16( + process_image_buffer_from_raw_bytes( + width, + height, + buffer, + bytes_per_pixel, + ), + )), + 12 => Err(CanvasError::UnsupportedColorType(ColorType::Rgb32F)), + 16 => Err(CanvasError::UnsupportedColorType(ColorType::Rgba32F)), + _ => Err(CanvasError::UnsupportedColorType(ColorType::L8)), + } +} + #[cfg(test)] mod tests { use super::*; @@ -500,4 +591,19 @@ mod tests { let rgba = rgba.unpremultiply_alpha(); assert_eq!(rgba, Rgba::([255, 0, 0, 127])); } + + #[test] + fn test_process_image_buffer_from_raw_bytes() { + let buffer = &[255, 255, 0, 0, 0, 0, 255, 255]; + let color = ColorType::Rgba16; + let bytes_per_pixel = color.bytes_per_pixel() as usize; + let image = DynamicImage::ImageRgba16(process_image_buffer_from_raw_bytes( + 1, + 1, + buffer, + bytes_per_pixel, + )) + .to_rgba16(); + assert_eq!(image.get_pixel(0, 0), &Rgba::([65535, 0, 0, 65535])); + } } diff --git a/ext/canvas/lib.deno_canvas.d.ts b/ext/canvas/lib.deno_canvas.d.ts index c695ba5cd8a973..6014a12756aadf 100644 --- a/ext/canvas/lib.deno_canvas.d.ts +++ b/ext/canvas/lib.deno_canvas.d.ts @@ -42,7 +42,7 @@ type ResizeQuality = "high" | "low" | "medium" | "pixelated"; * used to create an `ImageBitmap`. * * @category Canvas */ -type ImageBitmapSource = Blob | ImageData; +type ImageBitmapSource = Blob | ImageData | ImageBitmap; /** * The options of {@linkcode createImageBitmap}. diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 602a7d18018de3..d20406f550cf0c 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -23,6 +23,10 @@ pub enum CanvasError { /// This error will be mapped to DOMExceptionInvalidStateError. #[error("The chunk data is not big enough with the specified width: {0} and height: {1}.")] NotBigEnoughChunk(u32, u32), + /// This error will be mapped to DOMExceptionInvalidStateError. + #[error("The width: {0} or height: {1} could not be zero.")] + InvalidSizeZero(u32, u32), + /// This error will be mapped to TypeError. #[error(transparent)] Lcms(#[from] lcms2::Error), #[error(transparent)] diff --git a/ext/canvas/op_create_image_bitmap.rs b/ext/canvas/op_create_image_bitmap.rs index 45484d17c036de..bc14535452521a 100644 --- a/ext/canvas/op_create_image_bitmap.rs +++ b/ext/canvas/op_create_image_bitmap.rs @@ -19,6 +19,7 @@ use image::DynamicImage; use image::ImageDecoder; use image::RgbaImage; +use crate::image_ops::create_image_from_raw_bytes; use crate::image_ops::premultiply_alpha as process_premultiply_alpha; use crate::image_ops::to_srgb_from_icc_profile; use crate::image_ops::unpremultiply_alpha; @@ -28,6 +29,7 @@ use crate::CanvasError; enum ImageBitmapSource { Blob, ImageData, + ImageBitmap, } #[derive(Debug, PartialEq)] @@ -191,6 +193,11 @@ fn decode_bitmap_data( } }; + (image, width, height, None, None) + } + ImageBitmapSource::ImageBitmap => { + let image = create_image_from_raw_bytes(width, height, buf)?; + (image, width, height, None, None) } }; @@ -326,6 +333,7 @@ fn parse_args( let image_bitmap_source = match image_bitmap_source { 0 => ImageBitmapSource::Blob, 1 => ImageBitmapSource::ImageData, + 2 => ImageBitmapSource::ImageBitmap, _ => unreachable!(), }; let mime_type = match mime_type { @@ -502,10 +510,12 @@ pub(super) fn op_create_image_bitmap( ImageOrientation::FromImage => image, } } - ImageBitmapSource::ImageData => match image_orientation { - ImageOrientation::FlipY => image.flipv(), - ImageOrientation::FromImage => image, - }, + ImageBitmapSource::ImageData | ImageBitmapSource::ImageBitmap => { + match image_orientation { + ImageOrientation::FlipY => image.flipv(), + ImageOrientation::FromImage => image, + } + } }; // 9. diff --git a/runtime/errors.rs b/runtime/errors.rs index 3f0fbacecde3c1..f1379bfeec7928 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -604,6 +604,7 @@ fn get_canvas_error(e: &CanvasError) -> &'static str { CanvasError::UnsupportedColorType(_) => "TypeError", CanvasError::InvalidImage(_) => "DOMExceptionInvalidStateError", CanvasError::NotBigEnoughChunk(_, _) => "DOMExceptionInvalidStateError", + CanvasError::InvalidSizeZero(_, _) => "DOMExceptionInvalidStateError", CanvasError::Lcms(_) => "TypeError", CanvasError::Image(_) => "TypeError", } diff --git a/tests/unit/image_bitmap_test.ts b/tests/unit/image_bitmap_test.ts index 08cba061006be1..14555b6f45a126 100644 --- a/tests/unit/image_bitmap_test.ts +++ b/tests/unit/image_bitmap_test.ts @@ -21,6 +21,21 @@ Deno.test(async function imageBitmapDirect() { ); }); +Deno.test(async function imageBitmapRecivesImageBitmap() { + const imageData = new Blob( + [await Deno.readFile(`${prefix}/1x1-red16.png`)], + { type: "image/png" }, + ); + const imageBitmap1 = await createImageBitmap(imageData); + const imageBitmap2 = await createImageBitmap(imageBitmap1); + assertEquals( + // @ts-ignore: Deno[Deno.internal].core allowed + Deno[Deno.internal].getBitmapData(imageBitmap1), + // @ts-ignore: Deno[Deno.internal].core allowed + Deno[Deno.internal].getBitmapData(imageBitmap2), + ); +}); + Deno.test(async function imageBitmapCrop() { const data = generateNumberedData(3 * 3); const imageData = new ImageData(data, 3, 3); @@ -39,8 +54,8 @@ Deno.test(async function imageBitmapCropPartialNegative() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 1 + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 1 ])); }); @@ -51,11 +66,11 @@ Deno.test(async function imageBitmapCropGreater() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 0, 0, 1, 5, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 7, 0, 0, 1, 8, 0, 0, 1, 9, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 0, 0, 1, 5, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 1, 8, 0, 0, 1, 9, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ])); }); @@ -70,11 +85,11 @@ Deno.test(async function imageBitmapScale() { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ - 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, - 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, - 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, - 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, - 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1 + 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, + 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, + 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, + 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1, + 1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 3, 0, 0, 1 ])); }); @@ -181,8 +196,8 @@ Deno.test("imageBitmapPremultiplyAlpha", async (t) => { // @ts-ignore: Deno[Deno.internal].core allowed // deno-fmt-ignore assertEquals(Deno[Deno.internal].getBitmapData(imageBitmap), new Uint8Array([ - 255, 0, 0, 255, 0, 255, 0, 255, - 0, 0, 255, 255, 255, 0, 0, 127 + 255, 0, 0, 255, 0, 255, 0, 255, + 0, 0, 255, 255, 255, 0, 0, 127 ])); }); }); @@ -211,8 +226,8 @@ Deno.test("imageBitmapFromBlob", async (t) => { new Uint8Array( [ 255, 255, // R - 0, 0, // G - 0, 0, // B + 0, 0, // G + 0, 0, // B 255, 255 // A ] ) From 5b112826a8cbaa02227a85db88f887f1e305cc03 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 22:56:48 +0900 Subject: [PATCH 48/52] better error handling --- ext/canvas/image_ops.rs | 43 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/ext/canvas/image_ops.rs b/ext/canvas/image_ops.rs index f948f29c7609c9..d275df15bcc978 100644 --- a/ext/canvas/image_ops.rs +++ b/ext/canvas/image_ops.rs @@ -340,7 +340,7 @@ fn process_icc_profile_conversion( color: ColorType, input_icc_profile: Profile, output_icc_profile: Profile, -) -> ImageBuffer> +) -> Result>, CanvasError> where I: GenericImageView, P: Pixel + SliceToPixel + TransformColorProfile + 'static, @@ -349,16 +349,16 @@ where let (width, height) = image.dimensions(); let mut out = ImageBuffer::new(width, height); let pixel_format = match color { - ColorType::L8 => PixelFormat::GRAY_8, - ColorType::L16 => PixelFormat::GRAY_16, - ColorType::La8 => PixelFormat::GRAYA_8, - ColorType::La16 => PixelFormat::GRAYA_16, - ColorType::Rgb8 => PixelFormat::RGB_8, - ColorType::Rgb16 => PixelFormat::RGB_16, - ColorType::Rgba8 => PixelFormat::RGBA_8, - ColorType::Rgba16 => PixelFormat::RGBA_16, - _ => unreachable!("{}", CanvasError::UnsupportedColorType(color)), - }; + ColorType::L8 => Ok(PixelFormat::GRAY_8), + ColorType::L16 => Ok(PixelFormat::GRAY_16), + ColorType::La8 => Ok(PixelFormat::GRAYA_8), + ColorType::La16 => Ok(PixelFormat::GRAYA_16), + ColorType::Rgb8 => Ok(PixelFormat::RGB_8), + ColorType::Rgb16 => Ok(PixelFormat::RGB_16), + ColorType::Rgba8 => Ok(PixelFormat::RGBA_8), + ColorType::Rgba16 => Ok(PixelFormat::RGBA_16), + _ => Err(CanvasError::UnsupportedColorType(color)), + }?; let transformer = Transform::new( &input_icc_profile, pixel_format, @@ -366,8 +366,7 @@ where pixel_format, output_icc_profile.header_rendering_intent(), ) - .map_err(CanvasError::Lcms) - .unwrap(); + .map_err(CanvasError::Lcms)?; for (x, y, mut pixel) in image.pixels() { let pixel = pixel.transform_color_profile(&transformer); @@ -375,7 +374,7 @@ where out.put_pixel(x, y, pixel); } - out + Ok(out) } /// Convert the color space of the image from the ICC profile to sRGB. @@ -399,7 +398,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageLuma16(image) => Ok( @@ -408,7 +407,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageLumaA8(image) => Ok( @@ -417,7 +416,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageLumaA16(image) => Ok( @@ -426,7 +425,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageRgb8(image) => Ok( @@ -435,7 +434,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageRgb16(image) => Ok( @@ -444,7 +443,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageRgba8(image) => Ok( @@ -453,7 +452,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageRgba16(image) => Ok( @@ -462,7 +461,7 @@ pub(crate) fn to_srgb_from_icc_profile( color, icc_profile, srgb_icc_profile, - ) + )? .into(), ), DynamicImage::ImageRgb32F(_) => { From e59be4a2c570f8f11d93f19151e4d522d8afd75a Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Tue, 26 Nov 2024 22:57:03 +0900 Subject: [PATCH 49/52] fix typo --- ext/canvas/01_image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 9d266def96b8ac..743b9bfe2b3fc8 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -244,7 +244,7 @@ function createImageBitmap( let width = 0; let height = 0; - // If the of image doesn't have a MIME type, mark it as 0. + // If the image doesn't have a MIME type, mark it as 0. let mimeType = 0; let imageBitmapSource, buf; if (isBlob) { From 7039401fbe58c226f8068f10139462b45ad9d068 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:41:25 +0900 Subject: [PATCH 50/52] use `ImageError` instead of string - https://github.com/denoland/deno/pull/25517#discussion_r1861402027 --- ext/canvas/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index d20406f550cf0c..7086ebdbfafa4e 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -19,7 +19,7 @@ pub enum CanvasError { UnsupportedColorType(ColorType), /// This error will be mapped to DOMExceptionInvalidStateError. #[error("Cannot decode image '{0}'")] - InvalidImage(String), + InvalidImage(image::ImageError), /// This error will be mapped to DOMExceptionInvalidStateError. #[error("The chunk data is not big enough with the specified width: {0} and height: {1}.")] NotBigEnoughChunk(u32, u32), @@ -37,7 +37,7 @@ pub enum CanvasError { impl CanvasError { /// Convert an [`image::ImageError`] to an [`CanvasError::InvalidImage`]. fn image_error_to_invalid_image(error: image::ImageError) -> Self { - Self::InvalidImage(error.to_string()) + CanvasError::InvalidImage(error) } } From 4cca9ae37054a1e6d6453c2fb91e3824d9631894 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:43:27 +0900 Subject: [PATCH 51/52] align error message - https://github.com/denoland/deno/pull/25517#issuecomment-2505095797 --- ext/canvas/01_image.js | 6 +++--- ext/canvas/lib.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 743b9bfe2b3fc8..4afce031461cd4 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -230,7 +230,7 @@ function createImageBitmap( new DOMException( `${prefix}: The provided value for 'image' is not of type '(${ ArrayPrototypeJoin(imageBitmapSources, " or ") - })'.`, + })'`, "InvalidStateError", ), ); @@ -267,14 +267,14 @@ function createImageBitmap( } else if (mimeTypeString === "") { return PromiseReject( new DOMException( - `The MIME type of source image is not specified.`, + `The MIME type of source image is not specified`, "InvalidStateError", ), ); } else { return PromiseReject( new DOMException( - `The MIME type ${mimeTypeString} of source image is not a supported format.`, + `The MIME type ${mimeTypeString} of source image is not a supported format`, "InvalidStateError", ), ); diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 7086ebdbfafa4e..1c990a5adc3ad2 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -21,10 +21,10 @@ pub enum CanvasError { #[error("Cannot decode image '{0}'")] InvalidImage(image::ImageError), /// This error will be mapped to DOMExceptionInvalidStateError. - #[error("The chunk data is not big enough with the specified width: {0} and height: {1}.")] + #[error("The chunk data is not big enough with the specified width: {0} and height: {1}")] NotBigEnoughChunk(u32, u32), /// This error will be mapped to DOMExceptionInvalidStateError. - #[error("The width: {0} or height: {1} could not be zero.")] + #[error("The width: {0} or height: {1} could not be zero")] InvalidSizeZero(u32, u32), /// This error will be mapped to TypeError. #[error(transparent)] From 6453b7fe54dae8144c748a77a2ab765d2afeb703 Mon Sep 17 00:00:00 2001 From: Hajime-san <41257923+Hajime-san@users.noreply.github.com> Date: Sat, 7 Dec 2024 18:59:44 +0900 Subject: [PATCH 52/52] Revert move suggestion to `runtime/fmt_errors.rs` --- ext/canvas/01_image.js | 10 ++++++++-- runtime/fmt_errors.rs | 22 ---------------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/ext/canvas/01_image.js b/ext/canvas/01_image.js index 4afce031461cd4..75ec1b80f9e89f 100644 --- a/ext/canvas/01_image.js +++ b/ext/canvas/01_image.js @@ -267,14 +267,20 @@ function createImageBitmap( } else if (mimeTypeString === "") { return PromiseReject( new DOMException( - `The MIME type of source image is not specified`, + `The MIME type of source image is not specified\n +hint: When you want to get a "Blob" from "fetch", make sure to go through a file server that returns the appropriate content-type response header, + and specify the URL to the file server like "await(await fetch('http://localhost:8000/sample.png').blob()". + Alternatively, if you are reading a local file using 'Deno.readFile' etc., + set the appropriate MIME type like "new Blob([await Deno.readFile('sample.png')], { type: 'image/png' })".\n`, "InvalidStateError", ), ); } else { return PromiseReject( new DOMException( - `The MIME type ${mimeTypeString} of source image is not a supported format`, + `The the MIME type ${mimeTypeString} of source image is not a supported format\n +info: The following MIME types are supported. +docs: https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm\n`, "InvalidStateError", ), ); diff --git a/runtime/fmt_errors.rs b/runtime/fmt_errors.rs index e04b2e988cc90f..6f120b5d4669bd 100644 --- a/runtime/fmt_errors.rs +++ b/runtime/fmt_errors.rs @@ -458,28 +458,6 @@ fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { ]), ]; } - // See: ext/canvas/01_image.js - else if msg.contains("The MIME type of source image is not specified") { - return vec![ - FixSuggestion::hint_multiline(&[ - cstr!("When you want to get a Blob from fetch, make sure to go through a file server that returns the appropriate content-type response header,"), - cstr!("and specify the URL to the file server like await(await fetch('http://localhost:8000/sample.png').blob()."), - cstr!("Alternatively, if you are reading a local file using Deno.readFile etc.,"), - cstr!("set the appropriate MIME type like new Blob([await Deno.readFile('sample.png')], { type: 'image/png' }).") - ]), - ]; - } - // See: ext/canvas/01_image.js - else if msg.contains("The MIME type") - && msg.contains("of source image is not a supported format") - { - return vec![ - FixSuggestion::info( - "The following MIME types are supported." - ), - FixSuggestion::docs("https://mimesniff.spec.whatwg.org/#image-type-pattern-matching-algorithm"), - ]; - } } vec![]