From 327c2afeb0360dc3702c08bfc0eda748be03b616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 12:47:54 +0900 Subject: [PATCH 01/63] rsm --- src/jsontypes.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jsontypes.rs b/src/jsontypes.rs index 00630c0..5ddd2c1 100644 --- a/src/jsontypes.rs +++ b/src/jsontypes.rs @@ -43,6 +43,8 @@ pub struct RawSourceMap { #[serde(skip_serializing_if = "Option::is_none")] pub names: Option>, #[serde(skip_serializing_if = "Option::is_none")] + pub range_mappings: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mappings: Option, #[serde(skip_serializing_if = "Option::is_none")] pub x_facebook_offsets: Option>>, From 879fb1bff97e50f3d654fc4e0daaf0470f64f6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 12:47:57 +0900 Subject: [PATCH 02/63] range_mapping --- src/decoder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/decoder.rs b/src/decoder.rs index e701c47..7d04cca 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -129,6 +129,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let names = rsm.names.unwrap_or_default(); let sources = rsm.sources.unwrap_or_default(); + let range_mappings = rsm.range_mappings.unwrap_or_default(); let mappings = rsm.mappings.unwrap_or_default(); let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10; let mut tokens = Vec::with_capacity(allocation_size); From 9de66925dc5eecc09d144cdbb88c1c7add278121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 12:50:53 +0900 Subject: [PATCH 03/63] range_mapping --- src/decoder.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 7d04cca..92f7dc7 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -129,14 +129,18 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let names = rsm.names.unwrap_or_default(); let sources = rsm.sources.unwrap_or_default(); - let range_mappings = rsm.range_mappings.unwrap_or_default(); + let mut range_mappings = rsm.range_mappings.unwrap_or_default(); let mappings = rsm.mappings.unwrap_or_default(); let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10; let mut tokens = Vec::with_capacity(allocation_size); let mut nums = Vec::with_capacity(6); - for (dst_line, line) in mappings.split(';').enumerate() { + for (dst_line, (line, range_mapping_index)) in mappings + .split(';') + .zip(range_mappings.split(';')) + .enumerate() + { if line.is_empty() { continue; } From b13e00caffd842907cb9b63e228d13c38296bcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 12:59:18 +0900 Subject: [PATCH 04/63] error variant --- src/errors.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/errors.rs b/src/errors.rs index 5c4fbe1..e51f398 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -45,6 +45,8 @@ pub enum Error { InvalidRamBundleEntry, /// Tried to operate on a non RAM bundle file NotARamBundle, + /// Range mapping index is invalid + InvalidRangeMappingIndex, } impl From for Error { @@ -114,6 +116,7 @@ impl fmt::Display for Error { Error::InvalidRamBundleIndex => write!(f, "invalid module index in ram bundle"), Error::InvalidRamBundleEntry => write!(f, "invalid ram bundle module entry"), Error::NotARamBundle => write!(f, "not a ram bundle"), + Error::InvalidRangeMappingIndex => write!(f, "invalid range mapping index"), } } } From 755833585669e3bc4d9fc5aec24b55b61b9c3aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 12:59:23 +0900 Subject: [PATCH 05/63] rmi_result --- src/decoder.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 92f7dc7..97079de 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -1,6 +1,7 @@ use std::io; use std::io::{BufReader, Read}; +use data_encoding::BASE64; use serde_json::Value; use crate::errors::{Error, Result}; @@ -135,8 +136,10 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mut tokens = Vec::with_capacity(allocation_size); let mut nums = Vec::with_capacity(6); + let mut range_mapping_buf = Vec::new(); + let mut is_range_mapping; - for (dst_line, (line, range_mapping_index)) in mappings + for (dst_line, (line, range_mapping_index_str)) in mappings .split(';') .zip(range_mappings.split(';')) .enumerate() @@ -147,12 +150,22 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; - for segment in line.split(',') { + let rmi_result = + BASE64.decode_mut(range_mapping_index_str.as_bytes(), &mut range_mapping_buf); + + let rmi_result = match rmi_result { + Ok(rmi_result) => rmi_result, + Err(_) => { + fail!(Error::InvalidRangeMappingIndex); + } + }; + + for (nth, segment) in line.split(',').enumerate() { if segment.is_empty() { continue; } - nums.clear(); + is_range_mapping = nums.clear(); parse_vlq_segment_into(segment, &mut nums)?; dst_col = (i64::from(dst_col) + nums[0]) as u32; From 502e9a903daac5a025a0880a15d37d6f99d5e333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 13:03:00 +0900 Subject: [PATCH 06/63] fix compilation --- src/encoder.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/encoder.rs b/src/encoder.rs index af080be..5137144 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -21,6 +21,11 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } +fn serialize_range_mappings(sm: &SourceMap) -> Option { + // TODO + None +} + fn serialize_mappings(sm: &SourceMap) -> String { let mut rv = String::new(); // dst == minified == generated @@ -89,6 +94,7 @@ impl Encodable for SourceMap { sources_content: if have_contents { Some(contents) } else { None }, sections: None, names: Some(self.names().map(|x| Value::String(x.to_string())).collect()), + range_mappings: serialize_range_mappings(self), mappings: Some(serialize_mappings(self)), x_facebook_offsets: None, x_metro_module_paths: None, @@ -121,6 +127,7 @@ impl Encodable for SourceMapIndex { .collect(), ), names: None, + range_mappings: None, mappings: None, x_facebook_offsets: None, x_metro_module_paths: None, From 92928158d08366d54e028455246d21905d96bcb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 13:05:19 +0900 Subject: [PATCH 07/63] json --- src/jsontypes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsontypes.rs b/src/jsontypes.rs index 5ddd2c1..59d893b 100644 --- a/src/jsontypes.rs +++ b/src/jsontypes.rs @@ -42,7 +42,7 @@ pub struct RawSourceMap { pub sections: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub names: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "rangeMappings", skip_serializing_if = "Option::is_none")] pub range_mappings: Option, #[serde(skip_serializing_if = "Option::is_none")] pub mappings: Option, From 0cc6b202b284c4fa3b8f8a923ae98691865d2e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 14:39:23 +0900 Subject: [PATCH 08/63] WIP --- src/decoder.rs | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 97079de..f073de9 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -1,14 +1,14 @@ use std::io; use std::io::{BufReader, Read}; -use data_encoding::BASE64; +use data_encoding::{BASE64, BASE64_NOPAD}; use serde_json::Value; use crate::errors::{Error, Result}; use crate::hermes::decode_hermes; use crate::jsontypes::RawSourceMap; use crate::types::{DecodedMap, RawToken, SourceMap, SourceMapIndex, SourceMapSection}; -use crate::vlq::parse_vlq_segment_into; +use crate::vlq::{parse_vlq_segment, parse_vlq_segment_into}; const DATA_PREAMBLE: &str = "data:application/json;base64,"; @@ -121,6 +121,27 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { Ok(&slice[slice.len()..]) } +/// Decodes range mappping index string into index +/// +/// When the bit is set, the mapping is a range mapping, otherwise it is a normal mapping. For every 6 bits a base64 encoded char is emitted. Zero bits can be padded or omitted, to create full base64 chars or make the encoding shorter. +fn decode_rmi(rmi_str: &str) -> Result { + let mut res = 0; + + // So A is 6 zero bits. B is 5 zero bits and a 1 + + for (idx, byte) in rmi_str.bytes().enumerate() { + let mut val = BASE64_NOPAD + .decode(&[byte]) + .map_err(|_| Error::InvalidRangeMappingIndex)?[0]; + if idx == rmi_str.len() - 1 { + val &= 0x1f; + } + res = (res << 5) | (val as usize); + } + + Ok(res) +} + pub fn decode_regular(rsm: RawSourceMap) -> Result { let mut dst_col; let mut src_id = 0; @@ -130,16 +151,14 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let names = rsm.names.unwrap_or_default(); let sources = rsm.sources.unwrap_or_default(); - let mut range_mappings = rsm.range_mappings.unwrap_or_default(); + let range_mappings = rsm.range_mappings.unwrap_or_default(); let mappings = rsm.mappings.unwrap_or_default(); let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10; let mut tokens = Vec::with_capacity(allocation_size); let mut nums = Vec::with_capacity(6); - let mut range_mapping_buf = Vec::new(); - let mut is_range_mapping; - for (dst_line, (line, range_mapping_index_str)) in mappings + for (dst_line, (line, rmi_str)) in mappings .split(';') .zip(range_mappings.split(';')) .enumerate() @@ -150,22 +169,14 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; - let rmi_result = - BASE64.decode_mut(range_mapping_index_str.as_bytes(), &mut range_mapping_buf); - - let rmi_result = match rmi_result { - Ok(rmi_result) => rmi_result, - Err(_) => { - fail!(Error::InvalidRangeMappingIndex); - } - }; + let rmi = decode_rmi(rmi_str)?; for (nth, segment) in line.split(',').enumerate() { if segment.is_empty() { continue; } - is_range_mapping = nums.clear(); + nums.clear(); parse_vlq_segment_into(segment, &mut nums)?; dst_col = (i64::from(dst_col) + nums[0]) as u32; From cecc1bfccd4c34fb350ddf17e412418e7869b502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 14:40:58 +0900 Subject: [PATCH 09/63] error variant --- src/errors.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index e51f398..ddfa776 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -46,7 +46,7 @@ pub enum Error { /// Tried to operate on a non RAM bundle file NotARamBundle, /// Range mapping index is invalid - InvalidRangeMappingIndex, + InvalidRangeMappingIndex(data_encoding::DecodeError), } impl From for Error { @@ -80,6 +80,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: data_encoding::DecodeError) -> Error { + Error::InvalidRangeMappingIndex(err) + } +} + impl error::Error for Error { fn cause(&self) -> Option<&dyn error::Error> { match *self { @@ -116,7 +122,7 @@ impl fmt::Display for Error { Error::InvalidRamBundleIndex => write!(f, "invalid module index in ram bundle"), Error::InvalidRamBundleEntry => write!(f, "invalid ram bundle module entry"), Error::NotARamBundle => write!(f, "not a ram bundle"), - Error::InvalidRangeMappingIndex => write!(f, "invalid range mapping index"), + Error::InvalidRangeMappingIndex(err) => write!(f, "invalid range mapping index: {err}"), } } } From 88a054c8c8dc9b3c2b9a212d93ece24ec7e525de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:16:12 +0900 Subject: [PATCH 10/63] decode_rmi --- src/decoder.rs | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index f073de9..465f49e 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -1,14 +1,13 @@ use std::io; use std::io::{BufReader, Read}; -use data_encoding::{BASE64, BASE64_NOPAD}; use serde_json::Value; use crate::errors::{Error, Result}; use crate::hermes::decode_hermes; use crate::jsontypes::RawSourceMap; use crate::types::{DecodedMap, RawToken, SourceMap, SourceMapIndex, SourceMapSection}; -use crate::vlq::{parse_vlq_segment, parse_vlq_segment_into}; +use crate::vlq::parse_vlq_segment_into; const DATA_PREAMBLE: &str = "data:application/json;base64,"; @@ -122,24 +121,31 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { } /// Decodes range mappping index string into index -/// -/// When the bit is set, the mapping is a range mapping, otherwise it is a normal mapping. For every 6 bits a base64 encoded char is emitted. Zero bits can be padded or omitted, to create full base64 chars or make the encoding shorter. fn decode_rmi(rmi_str: &str) -> Result { - let mut res = 0; + let mut val = 0usize; + + for &byte in rmi_str.as_bytes() { + // A: 0b000000 + // B: 0b000001 + // g: 0b100000 + if byte == b'A' { + val += 6; + continue; + } - // So A is 6 zero bits. B is 5 zero bits and a 1 + let byte = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + _ => unreachable!("invalid rmi"), + }; - for (idx, byte) in rmi_str.bytes().enumerate() { - let mut val = BASE64_NOPAD - .decode(&[byte]) - .map_err(|_| Error::InvalidRangeMappingIndex)?[0]; - if idx == rmi_str.len() - 1 { - val &= 0x1f; - } - res = (res << 5) | (val as usize); + let delta = (byte.trailing_zeros() as usize) + 1; + + val += delta; } - Ok(res) + Ok(val) } pub fn decode_regular(rsm: RawSourceMap) -> Result { @@ -158,6 +164,8 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mut nums = Vec::with_capacity(6); + // TODO: assert that the mappings are longer than range mappings + for (dst_line, (line, rmi_str)) in mappings .split(';') .zip(range_mappings.split(';')) From 9631444830beced36c143d55bc17b279b80cb96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:22:44 +0900 Subject: [PATCH 11/63] is_range --- src/decoder.rs | 6 +++++- src/types.rs | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 465f49e..53c3441 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -161,6 +161,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mappings = rsm.mappings.unwrap_or_default(); let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10; let mut tokens = Vec::with_capacity(allocation_size); + let mut range_tokens = vec![]; let mut nums = Vec::with_capacity(6); @@ -178,8 +179,11 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; let rmi = decode_rmi(rmi_str)?; + if rmi != 0 { + range_tokens.push(rmi); + } - for (nth, segment) in line.split(',').enumerate() { + for segment in line.split(',') { if segment.is_empty() { continue; } diff --git a/src/types.rs b/src/types.rs index ce7d22f..71a3152 100644 --- a/src/types.rs +++ b/src/types.rs @@ -153,6 +153,8 @@ pub struct Token<'a> { raw: &'a RawToken, i: &'a SourceMap, idx: u32, + + is_range: bool, } impl<'a> PartialEq for Token<'a> { @@ -187,6 +189,8 @@ impl<'a> Ord for Token<'a> { try_cmp!(self.get_src_line(), other.get_src_line()); try_cmp!(self.get_src_col(), other.get_src_col()); try_cmp!(self.get_name(), other.get_name()); + try_cmp!(self.is_range(), other.is_range()); + Ordering::Equal } } @@ -280,6 +284,13 @@ impl<'a> Token<'a> { pub fn get_source_view(&self) -> Option<&SourceView<'_>> { self.i.get_source_view(self.get_src_id()) } + + /// If true, this token is a range token. + /// + /// See https://github.com/tc39/source-map-rfc/blob/main/proposals/range-mappings.md + pub fn is_range(&self) -> bool { + self.is_range + } } pub fn idx_from_token(token: &Token<'_>) -> u32 { @@ -459,6 +470,7 @@ pub struct SourceMapIndex { pub struct SourceMap { pub(crate) file: Option, pub(crate) tokens: Vec, + pub(crate) range_tokens: Vec, pub(crate) index: Vec<(u32, u32, u32)>, pub(crate) names: Vec, pub(crate) source_root: Option, @@ -588,6 +600,7 @@ impl SourceMap { SourceMap { file, tokens, + range_tokens: vec![], index, names, source_root: None, @@ -660,9 +673,14 @@ impl SourceMap { /// Looks up a token by its index. pub fn get_token(&self, idx: u32) -> Option> { - self.tokens - .get(idx as usize) - .map(|raw| Token { raw, i: self, idx }) + let is_range = self.range_tokens.binary_search(&idx).is_ok(); + + self.tokens.get(idx as usize).map(|raw| Token { + raw, + i: self, + idx, + is_range, + }) } /// Returns the number of tokens in the sourcemap. From 4935322a791d8f16939a88168f82f9261b42a924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:24:10 +0900 Subject: [PATCH 12/63] u32 --- src/decoder.rs | 7 ++++--- src/types.rs | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 53c3441..9719abd 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -121,8 +121,8 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { } /// Decodes range mappping index string into index -fn decode_rmi(rmi_str: &str) -> Result { - let mut val = 0usize; +fn decode_rmi(rmi_str: &str) -> Result { + let mut val = 0u32; for &byte in rmi_str.as_bytes() { // A: 0b000000 @@ -140,7 +140,7 @@ fn decode_rmi(rmi_str: &str) -> Result { _ => unreachable!("invalid rmi"), }; - let delta = (byte.trailing_zeros() as usize) + 1; + let delta = byte.trailing_zeros() + 1; val += delta; } @@ -248,6 +248,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { }); let mut sm = SourceMap::new(file, tokens, names, sources, rsm.sources_content); + sm.set_range_tokens(range_tokens); sm.set_source_root(rsm.source_root); sm.set_debug_id(rsm.debug_id); diff --git a/src/types.rs b/src/types.rs index 71a3152..efcc3b2 100644 --- a/src/types.rs +++ b/src/types.rs @@ -671,6 +671,13 @@ impl SourceMap { } } + /// Sets + /// + /// `range_tokens` should be sorted + pub fn set_range_tokens(&mut self, range_tokens: Vec) { + self.range_tokens = range_tokens; + } + /// Looks up a token by its index. pub fn get_token(&self, idx: u32) -> Option> { let is_range = self.range_tokens.binary_search(&idx).is_ok(); From 24cda131ebfde04ce61682e3c34d3298466465d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:24:44 +0900 Subject: [PATCH 13/63] doc --- src/types.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types.rs b/src/types.rs index efcc3b2..41ccbd8 100644 --- a/src/types.rs +++ b/src/types.rs @@ -671,9 +671,9 @@ impl SourceMap { } } - /// Sets + /// Sets range_token indices. /// - /// `range_tokens` should be sorted + /// `range_tokens` should be sorted. pub fn set_range_tokens(&mut self, range_tokens: Vec) { self.range_tokens = range_tokens; } From 26eeeed699c25d308520da66906bc0ac1afba8a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:38:16 +0900 Subject: [PATCH 14/63] fix rmi decoding --- src/decoder.rs | 9 +++++---- src/types.rs | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 9719abd..cc881c4 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -179,11 +179,8 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; let rmi = decode_rmi(rmi_str)?; - if rmi != 0 { - range_tokens.push(rmi); - } - for segment in line.split(',') { + for (line_index, segment) in line.split(',').enumerate() { if segment.is_empty() { continue; } @@ -217,6 +214,10 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } + if rmi != 0 && (line_index as u32) == rmi { + range_tokens.push(tokens.len() as u32) + } + tokens.push(RawToken { dst_line: dst_line as u32, dst_col, diff --git a/src/types.rs b/src/types.rs index 41ccbd8..2f7ea53 100644 --- a/src/types.rs +++ b/src/types.rs @@ -673,20 +673,18 @@ impl SourceMap { /// Sets range_token indices. /// - /// `range_tokens` should be sorted. + /// A token is a range token if the index is element of range_tokens. pub fn set_range_tokens(&mut self, range_tokens: Vec) { self.range_tokens = range_tokens; } /// Looks up a token by its index. pub fn get_token(&self, idx: u32) -> Option> { - let is_range = self.range_tokens.binary_search(&idx).is_ok(); - self.tokens.get(idx as usize).map(|raw| Token { raw, i: self, idx, - is_range, + is_range: self.range_tokens.contains(&idx), }) } From 3d43ddf2137d14817a906c4e0a6d9c38694b35d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:39:25 +0900 Subject: [PATCH 15/63] minus one --- src/decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index cc881c4..3ff3fb4 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -214,7 +214,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - if rmi != 0 && (line_index as u32) == rmi { + if rmi != 0 && (line_index as u32) == rmi - 1 { range_tokens.push(tokens.len() as u32) } From f517b36c8d61b11b6e8fd4e9029b11de780c4318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:40:06 +0900 Subject: [PATCH 16/63] Add a test for decoder --- tests/test_decoder.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_decoder.rs b/tests/test_decoder.rs index 103d194..21b6726 100644 --- a/tests/test_decoder.rs +++ b/tests/test_decoder.rs @@ -158,3 +158,24 @@ fn test_sourcemap_nofiles() { assert_eq!(iter.next().unwrap().to_tuple(), ("", 2, 2, Some("alert"))); assert!(iter.next().is_none()); } + +#[test] +fn test_sourcemap_range_mappings() { + let input: &[_] = br#"{ + "version": 3, + "sources": [null], + "names": ["console","log","ab"], + "mappings": "AACAA,QAAQC,GAAG,CAAC,OAAM,OAAM,QACxBD,QAAQC,GAAG,CAAC,QAEZD,QAAQC,GAAG,CAJD;IAACC,IAAI;AAAI,IAKnBF,QAAQC,GAAG,CAAC,YACZD,QAAQC,GAAG,CAAC", + "rangeMappings": "AAB;;g" + }"#; + let sm = SourceMap::from_reader(input).unwrap(); + dbg!(&sm); + let mut iter = sm.tokens().filter(Token::is_range); + + assert_eq!(sm.tokens().filter(Token::is_range).count(), 2); + + assert_eq!(iter.next().unwrap().to_tuple(), ("", 4, 11, None)); + + assert_eq!(iter.next().unwrap().to_tuple(), ("", 6, 0, Some("console"))); + assert!(iter.next().is_none()); +} From e4ad541a4f41c6d034bf747bd5e8ac7cb65a118e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:40:12 +0900 Subject: [PATCH 17/63] Remove dbg --- tests/test_decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_decoder.rs b/tests/test_decoder.rs index 21b6726..9f89802 100644 --- a/tests/test_decoder.rs +++ b/tests/test_decoder.rs @@ -169,7 +169,7 @@ fn test_sourcemap_range_mappings() { "rangeMappings": "AAB;;g" }"#; let sm = SourceMap::from_reader(input).unwrap(); - dbg!(&sm); + let mut iter = sm.tokens().filter(Token::is_range); assert_eq!(sm.tokens().filter(Token::is_range).count(), 2); From a7f241ed59a36a4bfe74d8eb6d481b194033495d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:55:16 +0900 Subject: [PATCH 18/63] serialize range mappings --- src/encoder.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/encoder.rs b/src/encoder.rs index 5137144..8fee7bf 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -22,7 +22,18 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { } fn serialize_range_mappings(sm: &SourceMap) -> Option { - // TODO + let mut buf = String::new(); + let mut prev_line = 0; + + for &rt_idx in sm.range_tokens.iter() { + let token = sm.tokens.get(rt_idx as usize)?; + + while token.dst_line != prev_line { + buf.push(';'); + prev_line += 1; + } + } + None } From 46d337dfbcdb752670454eccf8158518693cb98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:55:18 +0900 Subject: [PATCH 19/63] Add a test --- tests/test_encoder.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_encoder.rs b/tests/test_encoder.rs index d38273c..682eb70 100644 --- a/tests/test_encoder.rs +++ b/tests/test_encoder.rs @@ -25,3 +25,23 @@ fn test_sourcemap_data_url() { let sm = SourceMap::from_reader(input).unwrap(); assert_eq!(sm.to_data_url().unwrap(), "data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYnVpbGQvZm9vLm1pbi5qcyIsInNvdXJjZXMiOlsic3JjL2Zvby5qcyJdLCJzb3VyY2VSb290IjoiLyIsIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBIn0="); } + +#[test] +fn test_basic_range() { + let input: &[_] = br#"{ + "version": 3, + "sources": [null], + "names": ["console","log","ab"], + "mappings": "AACAA,QAAQC,GAAG,CAAC,OAAM,OAAM,QACxBD,QAAQC,GAAG,CAAC,QAEZD,QAAQC,GAAG,CAJD;IAACC,IAAI;AAAI,IAKnBF,QAAQC,GAAG,CAAC,YACZD,QAAQC,GAAG,CAAC", + "rangeMappings": "AAB;;g" + }"#; + let sm = SourceMap::from_reader(input).unwrap(); + let mut out: Vec = vec![]; + sm.to_writer(&mut out).unwrap(); + + let sm2 = SourceMap::from_reader(&out[..]).unwrap(); + + for (tok1, tok2) in sm.tokens().zip(sm2.tokens()) { + assert_eq!(tok1, tok2); + } +} From 65ba538a4bfac2b23cc1de01334a7d70e0a2fff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 15:56:13 +0900 Subject: [PATCH 20/63] Remove useless TODO --- src/decoder.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 3ff3fb4..69a3d77 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -165,8 +165,6 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mut nums = Vec::with_capacity(6); - // TODO: assert that the mappings are longer than range mappings - for (dst_line, (line, rmi_str)) in mappings .split(';') .zip(range_mappings.split(';')) From 5cd2046de626ef8a124e09ec4171474f0914e811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 16:09:51 +0900 Subject: [PATCH 21/63] review --- src/decoder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index 69a3d77..646eed1 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -120,7 +120,7 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { Ok(&slice[slice.len()..]) } -/// Decodes range mappping index string into index +/// Decodes range mappping bitfield string into index fn decode_rmi(rmi_str: &str) -> Result { let mut val = 0u32; @@ -137,6 +137,8 @@ fn decode_rmi(rmi_str: &str) -> Result { b'A'..=b'Z' => byte - b'A', b'a'..=b'z' => byte - b'a' + 26, b'0'..=b'9' => byte - b'0' + 52, + b'+' => 62, + b'/' => 63, _ => unreachable!("invalid rmi"), }; From 4535a2b7ca3f26a1cf5e2d2b6a66e9a3ea5d0dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 16:11:00 +0900 Subject: [PATCH 22/63] repeat empty string --- src/decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index 646eed1..896395b 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -169,7 +169,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { for (dst_line, (line, rmi_str)) in mappings .split(';') - .zip(range_mappings.split(';')) + .zip(range_mappings.split(';').chain(std::iter::repeat(""))) .enumerate() { if line.is_empty() { From fc2ee728682a1934138e9be3866db66b8c71c7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 25 Jan 2024 16:14:13 +0900 Subject: [PATCH 23/63] Fix doc CI --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 2f7ea53..e95809b 100644 --- a/src/types.rs +++ b/src/types.rs @@ -287,7 +287,7 @@ impl<'a> Token<'a> { /// If true, this token is a range token. /// - /// See https://github.com/tc39/source-map-rfc/blob/main/proposals/range-mappings.md + /// See pub fn is_range(&self) -> bool { self.is_range } From 1a1dbc5050ebf96b97a9140a215e347c3ddb1e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Fri, 26 Jan 2024 11:28:29 +0900 Subject: [PATCH 24/63] Dep on bitvec --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index c242091..b86f323 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ scroll = { version = "0.10.1", features = ["derive"], optional = true } data-encoding = "2.3.3" debugid = {version = "0.8.0", features = ["serde"] } base64-simd = { version = "0.7" } +bitvec = "1.0.1" [build-dependencies] rustc_version = "0.2.3" From af1953717c66094dbc27a5cfb9daf1d479ef3bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 28 Jan 2024 23:55:44 +0900 Subject: [PATCH 25/63] fix decode_rmi --- src/decoder.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 896395b..67048b7 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -1,6 +1,10 @@ use std::io; use std::io::{BufReader, Read}; +use bitvec::bitvec; +use bitvec::field::BitField; +use bitvec::order::Msb0; +use bitvec::vec::BitVec; use serde_json::Value; use crate::errors::{Error, Result}; @@ -121,18 +125,16 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { } /// Decodes range mappping bitfield string into index -fn decode_rmi(rmi_str: &str) -> Result { - let mut val = 0u32; - - for &byte in rmi_str.as_bytes() { - // A: 0b000000 - // B: 0b000001 - // g: 0b100000 - if byte == b'A' { - val += 6; - continue; - } +fn decode_rmi(rmi_str: &str) -> Result> { + let mut val = bitvec![u8, Msb0; 0; rmi_str.len() * 6]; + + // A: 0b000000 + // B: 0b000001 + // While, + // AAB: 0b000000 0b000000 0b000001 => 13 + // so we iterate in the reverse order + for (idx, &byte) in rmi_str.as_bytes().iter().rev().enumerate() { let byte = match byte { b'A'..=b'Z' => byte - b'A', b'a'..=b'z' => byte - b'a' + 26, @@ -142,9 +144,7 @@ fn decode_rmi(rmi_str: &str) -> Result { _ => unreachable!("invalid rmi"), }; - let delta = byte.trailing_zeros() + 1; - - val += delta; + val[6 * idx..6 * (idx + 1)].store::(byte); } Ok(val) @@ -179,6 +179,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; let rmi = decode_rmi(rmi_str)?; + dbg!(format!("{rmi:b}")); for (line_index, segment) in line.split(',').enumerate() { if segment.is_empty() { @@ -214,9 +215,9 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - if rmi != 0 && (line_index as u32) == rmi - 1 { - range_tokens.push(tokens.len() as u32) - } + // if rmi != 0 && (line_index as u32) == rmi - 1 { + // range_tokens.push(tokens.len() as u32) + // } tokens.push(RawToken { dst_line: dst_line as u32, From be20676840d8eebf262ae208018a4fc6374db9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 28 Jan 2024 23:57:15 +0900 Subject: [PATCH 26/63] Use it --- src/decoder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 67048b7..da732d9 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -215,9 +215,9 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - // if rmi != 0 && (line_index as u32) == rmi - 1 { - // range_tokens.push(tokens.len() as u32) - // } + if rmi.trailing_zeros() != 0 && line_index == rmi.trailing_zeros() { + range_tokens.push(tokens.len() as u32) + } tokens.push(RawToken { dst_line: dst_line as u32, From 3b72696c87567e3eb2d2c2d666f3d0a55d4caf6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 15:45:32 +0900 Subject: [PATCH 27/63] to: &mut BitVec --- src/decoder.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index da732d9..c745746 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -1,7 +1,6 @@ use std::io; use std::io::{BufReader, Read}; -use bitvec::bitvec; use bitvec::field::BitField; use bitvec::order::Msb0; use bitvec::vec::BitVec; @@ -125,8 +124,14 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { } /// Decodes range mappping bitfield string into index -fn decode_rmi(rmi_str: &str) -> Result> { - let mut val = bitvec![u8, Msb0; 0; rmi_str.len() * 6]; +fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { + val.clear(); + if val.capacity() < rmi_str.len() * 6 { + val.reserve(rmi_str.len() * 6 - val.capacity()); + } + unsafe { + val.set_len(rmi_str.len() * 6); + }; // A: 0b000000 // B: 0b000001 @@ -147,7 +152,7 @@ fn decode_rmi(rmi_str: &str) -> Result> { val[6 * idx..6 * (idx + 1)].store::(byte); } - Ok(val) + Ok(()) } pub fn decode_regular(rsm: RawSourceMap) -> Result { @@ -166,6 +171,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mut range_tokens = vec![]; let mut nums = Vec::with_capacity(6); + let mut rmi = BitVec::new(); for (dst_line, (line, rmi_str)) in mappings .split(';') @@ -178,8 +184,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { dst_col = 0; - let rmi = decode_rmi(rmi_str)?; - dbg!(format!("{rmi:b}")); + decode_rmi(rmi_str, &mut rmi)?; for (line_index, segment) in line.split(',').enumerate() { if segment.is_empty() { From 59b3241edec28ae01bd3a140ea4e81fc63e02717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 16:48:58 +0900 Subject: [PATCH 28/63] Fix partial eq impl --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index e95809b..5812f30 100644 --- a/src/types.rs +++ b/src/types.rs @@ -159,7 +159,7 @@ pub struct Token<'a> { impl<'a> PartialEq for Token<'a> { fn eq(&self, other: &Token<'_>) -> bool { - self.raw == other.raw + self.raw == other.raw && self.is_range == other.is_range } } From 083405887cf3aa04c7ad3f71a83088ee5ada5201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 16:50:23 +0900 Subject: [PATCH 29/63] Debug impl --- src/types.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 5812f30..f6ab58b 100644 --- a/src/types.rs +++ b/src/types.rs @@ -421,7 +421,13 @@ impl<'a> fmt::Display for Token<'a> { .unwrap_or_default() )?; if f.alternate() { - write!(f, " ({}:{})", self.get_dst_line(), self.get_dst_col())?; + write!( + f, + " ({}:{}){}", + self.get_dst_line(), + self.get_dst_col(), + if self.is_range { " (range)" } else { "" } + )?; } Ok(()) } From 26c6280be5520ea1779875a04e709d1b447a5fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 17:00:49 +0900 Subject: [PATCH 30/63] Add safety comment --- src/decoder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/decoder.rs b/src/decoder.rs index c745746..19e9c73 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -130,6 +130,7 @@ fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { val.reserve(rmi_str.len() * 6 - val.capacity()); } unsafe { + // Safety: we just reserved enough space val.set_len(rmi_str.len() * 6); }; From 3f80f13c2e1557338e6682ac56b87b162e57c77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 17:36:17 +0900 Subject: [PATCH 31/63] Declare `encode_rmi` --- src/encoder.rs | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 8fee7bf..f79110b 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -1,5 +1,8 @@ use std::io::Write; +use bitvec::order::Msb0; +use bitvec::view::BitView; +use data_encoding::{BASE64, BASE64_NOPAD}; use serde_json::Value; use crate::errors::Result; @@ -21,20 +24,38 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } +fn encode_rmi(out: &mut Vec, mut i: usize) {} + fn serialize_range_mappings(sm: &SourceMap) -> Option { - let mut buf = String::new(); + if sm.range_tokens.is_empty() { + return None; + } + + let mut buf = Vec::new(); let mut prev_line = 0; - for &rt_idx in sm.range_tokens.iter() { - let token = sm.tokens.get(rt_idx as usize)?; + let mut idx_of_first_in_line = 0; + + for (idx, token) in sm.tokens().enumerate() { + if token.is_range() { + let num = idx - idx_of_first_in_line + 1; + + if num > 0 { + encode_rmi(&mut buf, num); + } + } - while token.dst_line != prev_line { - buf.push(';'); + while token.get_dst_line() != prev_line { + buf.push(b';'); prev_line += 1; + idx_of_first_in_line = idx; } } - None + Some(unsafe { + // Safety: We only push ASCII characters to the buffer + String::from_utf8(buf).expect("invalid utf8") + }) } fn serialize_mappings(sm: &SourceMap) -> String { @@ -105,7 +126,7 @@ impl Encodable for SourceMap { sources_content: if have_contents { Some(contents) } else { None }, sections: None, names: Some(self.names().map(|x| Value::String(x.to_string())).collect()), - range_mappings: serialize_range_mappings(self), + range_mappings: dbg!(serialize_range_mappings(self)), mappings: Some(serialize_mappings(self)), x_facebook_offsets: None, x_metro_module_paths: None, From 0be3f853c08fd790eef076da6e5af5f3140a26fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 17:39:39 +0900 Subject: [PATCH 32/63] Declare encode_byte --- src/encoder.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/encoder.rs b/src/encoder.rs index f79110b..243cc6a 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -24,7 +24,18 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } -fn encode_rmi(out: &mut Vec, mut i: usize) {} +fn encode_rmi(out: &mut Vec, mut i: usize) { + fn encode_byte(b: u8) -> u8 { + match b { + 0..=25 => b + b'A', + 26..=51 => b + b'a' - 26, + 52..=61 => b + b'0' - 52, + 62 => b'+', + 63 => b'/', + _ => panic!("invalid byte"), + } + } +} fn serialize_range_mappings(sm: &SourceMap) -> Option { if sm.range_tokens.is_empty() { From a8d6470ce41d3c6f85c4c2759a0f03cf08acc5ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 29 Jan 2024 18:13:26 +0900 Subject: [PATCH 33/63] encode_rmi --- src/encoder.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 243cc6a..6a7341a 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -1,8 +1,8 @@ use std::io::Write; +use bitvec::field::BitField; use bitvec::order::Msb0; use bitvec::view::BitView; -use data_encoding::{BASE64, BASE64_NOPAD}; use serde_json::Value; use crate::errors::Result; @@ -24,7 +24,7 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } -fn encode_rmi(out: &mut Vec, mut i: usize) { +fn encode_rmi(out: &mut Vec, i: usize) { fn encode_byte(b: u8) -> u8 { match b { 0..=25 => b + b'A', @@ -35,6 +35,31 @@ fn encode_rmi(out: &mut Vec, mut i: usize) { _ => panic!("invalid byte"), } } + + let mut data = [0u8; 8]; + + let bits = data.view_bits_mut::(); + bits.set(i - 1, true); + + // trim zero at the end + + let mut last = 0; + for (idx, bit) in bits.iter().enumerate() { + if *bit { + last = idx; + } + } + let bits = &mut bits[..last + 1]; + + for byte in bits.chunks_mut(6) { + byte.reverse(); + + let byte = byte.load::(); + + let encoded = encode_byte(byte); + + out.push(encoded); + } } fn serialize_range_mappings(sm: &SourceMap) -> Option { @@ -65,7 +90,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { Some(unsafe { // Safety: We only push ASCII characters to the buffer - String::from_utf8(buf).expect("invalid utf8") + String::from_utf8_unchecked(buf) }) } From a040b52b691a10bf213a294457037dd8a2858eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Tue, 30 Jan 2024 13:28:56 +0900 Subject: [PATCH 34/63] Patch encoder to support multiple mappings --- src/encoder.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 6a7341a..84d0793 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -24,7 +24,7 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } -fn encode_rmi(out: &mut Vec, i: usize) { +fn encode_rmi(out: &mut Vec, indices: &[usize]) { fn encode_byte(b: u8) -> u8 { match b { 0..=25 => b + b'A', @@ -39,7 +39,9 @@ fn encode_rmi(out: &mut Vec, i: usize) { let mut data = [0u8; 8]; let bits = data.view_bits_mut::(); - bits.set(i - 1, true); + for i in indices { + bits.set(i - 1, true); + } // trim zero at the end @@ -71,22 +73,31 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { let mut prev_line = 0; let mut idx_of_first_in_line = 0; + let mut indices = vec![]; for (idx, token) in sm.tokens().enumerate() { if token.is_range() { let num = idx - idx_of_first_in_line + 1; if num > 0 { - encode_rmi(&mut buf, num); + indices.push(num); } } while token.get_dst_line() != prev_line { + if !indices.is_empty() { + encode_rmi(&mut buf, &indices); + indices.clear(); + } + buf.push(b';'); prev_line += 1; idx_of_first_in_line = idx; } } + if !indices.is_empty() { + encode_rmi(&mut buf, &indices); + } Some(unsafe { // Safety: We only push ASCII characters to the buffer From 8383467e6ea24ebe05e94f9690a19add35d6f3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Tue, 30 Jan 2024 13:29:21 +0900 Subject: [PATCH 35/63] Remove dbg --- src/encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoder.rs b/src/encoder.rs index 84d0793..95f5463 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -173,7 +173,7 @@ impl Encodable for SourceMap { sources_content: if have_contents { Some(contents) } else { None }, sections: None, names: Some(self.names().map(|x| Value::String(x.to_string())).collect()), - range_mappings: dbg!(serialize_range_mappings(self)), + range_mappings: serialize_range_mappings(self), mappings: Some(serialize_mappings(self)), x_facebook_offsets: None, x_metro_module_paths: None, From 6f5a4ae2e29807c717c96fbac0d53ad8d7f30e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Tue, 30 Jan 2024 13:31:14 +0900 Subject: [PATCH 36/63] Fix decoder --- src/decoder.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 19e9c73..70a46be 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -2,7 +2,7 @@ use std::io; use std::io::{BufReader, Read}; use bitvec::field::BitField; -use bitvec::order::Msb0; +use bitvec::order::Lsb0; use bitvec::vec::BitVec; use serde_json::Value; @@ -124,7 +124,7 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { } /// Decodes range mappping bitfield string into index -fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { +fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { val.clear(); if val.capacity() < rmi_str.len() * 6 { val.reserve(rmi_str.len() * 6 - val.capacity()); @@ -140,7 +140,7 @@ fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { // While, // AAB: 0b000000 0b000000 0b000001 => 13 // so we iterate in the reverse order - for (idx, &byte) in rmi_str.as_bytes().iter().rev().enumerate() { + for (idx, &byte) in rmi_str.as_bytes().iter().enumerate() { let byte = match byte { b'A'..=b'Z' => byte - b'A', b'a'..=b'z' => byte - b'a' + 26, @@ -221,7 +221,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - if rmi.trailing_zeros() != 0 && line_index == rmi.trailing_zeros() { + if rmi.get(line_index as usize).map(|v| *v).unwrap_or_default() { range_tokens.push(tokens.len() as u32) } From 3cf9dd68ba1de1301d1725303d952462305dc074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 31 Jan 2024 16:56:17 +0900 Subject: [PATCH 37/63] Use binary search --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index f6ab58b..96b7d2f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -690,7 +690,7 @@ impl SourceMap { raw, i: self, idx, - is_range: self.range_tokens.contains(&idx), + is_range: self.range_tokens.binary_search(&idx).is_ok(), }) } From bc2a2373923e0201f5f857a111ec706ea61dc3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 31 Jan 2024 16:56:42 +0900 Subject: [PATCH 38/63] Remove comments --- src/decoder.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 70a46be..e94143f 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -134,12 +134,6 @@ fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { val.set_len(rmi_str.len() * 6); }; - // A: 0b000000 - // B: 0b000001 - - // While, - // AAB: 0b000000 0b000000 0b000001 => 13 - // so we iterate in the reverse order for (idx, &byte) in rmi_str.as_bytes().iter().enumerate() { let byte = match byte { b'A'..=b'Z' => byte - b'A', From 38bb516dfab566f740ffa49ba7e2f4910a1ac19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 31 Jan 2024 17:02:03 +0900 Subject: [PATCH 39/63] assert 1-based index --- src/encoder.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 95f5463..75602dd 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -39,7 +39,9 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { let mut data = [0u8; 8]; let bits = data.view_bits_mut::(); - for i in indices { + for &i in indices { + debug_assert!(i > 0, "0th mapping does not exist"); + // -1 to convert to 0-based index bits.set(i - 1, true); } @@ -77,11 +79,10 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { for (idx, token) in sm.tokens().enumerate() { if token.is_range() { + // 0th mapping does not exist, so we use 1-based index let num = idx - idx_of_first_in_line + 1; - if num > 0 { - indices.push(num); - } + indices.push(num); } while token.get_dst_line() != prev_line { From 2ab0d3fe7400f7077fe2092f1e9f08309845bf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 31 Jan 2024 17:04:09 +0900 Subject: [PATCH 40/63] Use lsb and remove reverse --- src/encoder.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 75602dd..8b85af0 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -1,7 +1,7 @@ use std::io::Write; use bitvec::field::BitField; -use bitvec::order::Msb0; +use bitvec::order::Lsb0; use bitvec::view::BitView; use serde_json::Value; @@ -38,7 +38,7 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { let mut data = [0u8; 8]; - let bits = data.view_bits_mut::(); + let bits = data.view_bits_mut::(); for &i in indices { debug_assert!(i > 0, "0th mapping does not exist"); // -1 to convert to 0-based index @@ -55,9 +55,7 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { } let bits = &mut bits[..last + 1]; - for byte in bits.chunks_mut(6) { - byte.reverse(); - + for byte in bits.chunks(6) { let byte = byte.load::(); let encoded = encode_byte(byte); From 37cdbc7917cf60bd71b1e35859957f993a35306b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 31 Jan 2024 17:11:16 +0900 Subject: [PATCH 41/63] Add tests --- src/decoder.rs | 21 +++++++++++++++++++++ src/encoder.rs | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/decoder.rs b/src/decoder.rs index e94143f..7d79558 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -355,3 +355,24 @@ fn test_bad_newline() { } } } + +#[test] +fn test_decode_rmi() { + fn decode(rmi_str: &str) -> Vec { + let mut out = bitvec::bitvec![u8, Lsb0; 0; 0]; + decode_rmi(rmi_str, &mut out).expect("failed to decode"); + + let mut res = vec![]; + for (idx, bit) in out.iter().enumerate() { + if *bit { + // 0-based index to 1-based index + res.push(idx + 1); + } + } + res + } + + assert_eq!(decode("AAB"), vec![13]); + assert_eq!(decode("g"), vec![6]); + assert_eq!(decode("Bg"), vec![1, 12]); +} diff --git a/src/encoder.rs b/src/encoder.rs index 8b85af0..d418f5e 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -224,3 +224,16 @@ impl Encodable for DecodedMap { } } } + +#[test] +fn test_encode_rmi() { + fn encode(indices: &[usize]) -> String { + let mut out = vec![]; + encode_rmi(&mut out, indices); + String::from_utf8(out).unwrap() + } + + assert_eq!(encode(&[13]), "AAB"); + assert_eq!(encode(&[6]), "g"); + assert_eq!(encode(&[1, 12]), "Bg"); +} From 3150d8369be9e9515ecfd517ce246af5da8ee096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 1 Feb 2024 11:09:34 +0900 Subject: [PATCH 42/63] Use 0-based index --- src/decoder.rs | 10 +++++----- src/encoder.rs | 7 ++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 7d79558..644704e 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -365,14 +365,14 @@ fn test_decode_rmi() { let mut res = vec![]; for (idx, bit) in out.iter().enumerate() { if *bit { - // 0-based index to 1-based index - res.push(idx + 1); + res.push(idx); } } res } - assert_eq!(decode("AAB"), vec![13]); - assert_eq!(decode("g"), vec![6]); - assert_eq!(decode("Bg"), vec![1, 12]); + // This is 0-based index of the bits + assert_eq!(decode("AAB"), vec![12]); + assert_eq!(decode("g"), vec![5]); + assert_eq!(decode("Bg"), vec![0, 11]); } diff --git a/src/encoder.rs b/src/encoder.rs index d418f5e..24b047c 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -40,9 +40,7 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { let bits = data.view_bits_mut::(); for &i in indices { - debug_assert!(i > 0, "0th mapping does not exist"); - // -1 to convert to 0-based index - bits.set(i - 1, true); + bits.set(i, true); } // trim zero at the end @@ -77,8 +75,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { for (idx, token) in sm.tokens().enumerate() { if token.is_range() { - // 0th mapping does not exist, so we use 1-based index - let num = idx - idx_of_first_in_line + 1; + let num = idx - idx_of_first_in_line; indices.push(num); } From 38785c30396e143a6df9769b16ee1b19fee6a139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 1 Feb 2024 11:11:12 +0900 Subject: [PATCH 43/63] fix test --- src/encoder.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index 24b047c..e368deb 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -230,7 +230,8 @@ fn test_encode_rmi() { String::from_utf8(out).unwrap() } - assert_eq!(encode(&[13]), "AAB"); - assert_eq!(encode(&[6]), "g"); - assert_eq!(encode(&[1, 12]), "Bg"); + // This is 0-based index + assert_eq!(encode(&[12]), "AAB"); + assert_eq!(encode(&[5]), "g"); + assert_eq!(encode(&[0, 11]), "Bg"); } From 09b72f306fb8c986efa802d7d3780a7da8c31204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Thu, 1 Feb 2024 11:11:21 +0900 Subject: [PATCH 44/63] lint --- src/decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index 644704e..5c0135f 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -215,7 +215,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - if rmi.get(line_index as usize).map(|v| *v).unwrap_or_default() { + if rmi.get(line_index).map(|v| *v).unwrap_or_default() { range_tokens.push(tokens.len() as u32) } From b0d9e1d22f02350f1747d762481485e598e17f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 28 Feb 2024 15:18:41 +0900 Subject: [PATCH 45/63] resize --- src/decoder.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/decoder.rs b/src/decoder.rs index 5c0135f..ac4f19c 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -126,13 +126,7 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> { /// Decodes range mappping bitfield string into index fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { val.clear(); - if val.capacity() < rmi_str.len() * 6 { - val.reserve(rmi_str.len() * 6 - val.capacity()); - } - unsafe { - // Safety: we just reserved enough space - val.set_len(rmi_str.len() * 6); - }; + val.resize(rmi_str.len() * 6, false); for (idx, &byte) in rmi_str.as_bytes().iter().enumerate() { let byte = match byte { From f4867b148c47b0526bf7254f8368411608a1dcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 28 Feb 2024 15:20:23 +0900 Subject: [PATCH 46/63] panic => error --- src/decoder.rs | 4 +++- src/errors.rs | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index ac4f19c..d812cfb 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -135,7 +135,9 @@ fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { b'0'..=b'9' => byte - b'0' + 52, b'+' => 62, b'/' => 63, - _ => unreachable!("invalid rmi"), + _ => { + fail!(Error::InvalidBase64(byte as char)); + } }; val[6 * idx..6 * (idx + 1)].store::(byte); diff --git a/src/errors.rs b/src/errors.rs index ddfa776..d833dc3 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -47,6 +47,8 @@ pub enum Error { NotARamBundle, /// Range mapping index is invalid InvalidRangeMappingIndex(data_encoding::DecodeError), + + InvalidBase64(char), } impl From for Error { @@ -123,6 +125,7 @@ impl fmt::Display for Error { Error::InvalidRamBundleEntry => write!(f, "invalid ram bundle module entry"), Error::NotARamBundle => write!(f, "not a ram bundle"), Error::InvalidRangeMappingIndex(err) => write!(f, "invalid range mapping index: {err}"), + Error::InvalidBase64(c) => write!(f, "invalid base64 character: {}", c), } } } From 46d64fa2f099c0046ebdeca2283225339a87e087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 28 Feb 2024 15:21:51 +0900 Subject: [PATCH 47/63] store_le --- src/decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decoder.rs b/src/decoder.rs index d812cfb..cd75006 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -140,7 +140,7 @@ fn decode_rmi(rmi_str: &str, val: &mut BitVec) -> Result<()> { } }; - val[6 * idx..6 * (idx + 1)].store::(byte); + val[6 * idx..6 * (idx + 1)].store_le::(byte); } Ok(()) From f70a7544982cc6234100e6864706e6a7655e8830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 28 Feb 2024 15:23:58 +0900 Subject: [PATCH 48/63] unsafe --- src/encoder.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index e368deb..cf328dc 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -95,10 +95,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { encode_rmi(&mut buf, &indices); } - Some(unsafe { - // Safety: We only push ASCII characters to the buffer - String::from_utf8_unchecked(buf) - }) + Some(String::from_utf8(buf).expect("invalid utf8")) } fn serialize_mappings(sm: &SourceMap) -> String { From a0252481afab46d986898480d520f6d7e37f3a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 10:47:48 +0900 Subject: [PATCH 49/63] `RawToken.is_range` --- src/builder.rs | 10 +++++++++- src/decoder.rs | 7 ++----- src/encoder.rs | 4 ---- src/types.rs | 32 ++++++++++++-------------------- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index b9a4791..f633e6c 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -189,8 +189,11 @@ impl SourceMapBuilder { src_col: u32, source: Option<&str>, name: Option<&str>, + is_range: bool, ) -> RawToken { - self.add_with_id(dst_line, dst_col, src_line, src_col, source, !0, name) + self.add_with_id( + dst_line, dst_col, src_line, src_col, source, !0, name, is_range, + ) } #[allow(clippy::too_many_arguments)] @@ -203,6 +206,7 @@ impl SourceMapBuilder { source: Option<&str>, source_id: u32, name: Option<&str>, + is_range: bool, ) -> RawToken { let src_id = match source { Some(source) => self.add_source_with_id(source, source_id), @@ -219,6 +223,7 @@ impl SourceMapBuilder { src_col, src_id, name_id, + is_range, }; self.tokens.push(raw); raw @@ -233,6 +238,7 @@ impl SourceMapBuilder { src_col: u32, source: Option, name: Option, + is_range: bool, ) -> RawToken { let src_id = source.unwrap_or(!0); let name_id = name.unwrap_or(!0); @@ -243,6 +249,7 @@ impl SourceMapBuilder { src_col, src_id, name_id, + is_range, }; self.tokens.push(raw); raw @@ -260,6 +267,7 @@ impl SourceMapBuilder { token.get_source(), token.get_src_id(), name, + token.is_range(), ) } diff --git a/src/decoder.rs b/src/decoder.rs index cd75006..55278b0 100644 --- a/src/decoder.rs +++ b/src/decoder.rs @@ -159,7 +159,6 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { let mappings = rsm.mappings.unwrap_or_default(); let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10; let mut tokens = Vec::with_capacity(allocation_size); - let mut range_tokens = vec![]; let mut nums = Vec::with_capacity(6); let mut rmi = BitVec::new(); @@ -211,9 +210,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { } } - if rmi.get(line_index).map(|v| *v).unwrap_or_default() { - range_tokens.push(tokens.len() as u32) - } + let is_range = rmi.get(line_index).map(|v| *v).unwrap_or_default(); tokens.push(RawToken { dst_line: dst_line as u32, @@ -222,6 +219,7 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { src_col, src_id: src, name_id: name, + is_range, }); } } @@ -246,7 +244,6 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result { }); let mut sm = SourceMap::new(file, tokens, names, sources, rsm.sources_content); - sm.set_range_tokens(range_tokens); sm.set_source_root(rsm.source_root); sm.set_debug_id(rsm.debug_id); diff --git a/src/encoder.rs b/src/encoder.rs index cf328dc..b4eb5e3 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -63,10 +63,6 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { } fn serialize_range_mappings(sm: &SourceMap) -> Option { - if sm.range_tokens.is_empty() { - return None; - } - let mut buf = Vec::new(); let mut prev_line = 0; diff --git a/src/types.rs b/src/types.rs index 96b7d2f..cfb1361 100644 --- a/src/types.rs +++ b/src/types.rs @@ -145,6 +145,11 @@ pub struct RawToken { pub src_id: u32, /// name identifier (`!0` in case there is no associated name) pub name_id: u32, + + /// If true, this token is a range token. + /// + /// See + pub is_range: bool, } /// Represents a token from a sourcemap @@ -153,13 +158,11 @@ pub struct Token<'a> { raw: &'a RawToken, i: &'a SourceMap, idx: u32, - - is_range: bool, } impl<'a> PartialEq for Token<'a> { fn eq(&self, other: &Token<'_>) -> bool { - self.raw == other.raw && self.is_range == other.is_range + self.raw == other.raw } } @@ -289,7 +292,7 @@ impl<'a> Token<'a> { /// /// See pub fn is_range(&self) -> bool { - self.is_range + self.raw.is_range } } @@ -426,7 +429,7 @@ impl<'a> fmt::Display for Token<'a> { " ({}:{}){}", self.get_dst_line(), self.get_dst_col(), - if self.is_range { " (range)" } else { "" } + if self.is_range() { " (range)" } else { "" } )?; } Ok(()) @@ -476,7 +479,6 @@ pub struct SourceMapIndex { pub struct SourceMap { pub(crate) file: Option, pub(crate) tokens: Vec, - pub(crate) range_tokens: Vec, pub(crate) index: Vec<(u32, u32, u32)>, pub(crate) names: Vec, pub(crate) source_root: Option, @@ -606,7 +608,6 @@ impl SourceMap { SourceMap { file, tokens, - range_tokens: vec![], index, names, source_root: None, @@ -677,21 +678,11 @@ impl SourceMap { } } - /// Sets range_token indices. - /// - /// A token is a range token if the index is element of range_tokens. - pub fn set_range_tokens(&mut self, range_tokens: Vec) { - self.range_tokens = range_tokens; - } - /// Looks up a token by its index. pub fn get_token(&self, idx: u32) -> Option> { - self.tokens.get(idx as usize).map(|raw| Token { - raw, - i: self, - idx, - is_range: self.range_tokens.binary_search(&idx).is_ok(), - }) + self.tokens + .get(idx as usize) + .map(|raw| Token { raw, i: self, idx }) } /// Returns the number of tokens in the sourcemap. @@ -1232,6 +1223,7 @@ impl SourceMapIndex { token.get_src_col(), token.get_source(), token.get_name(), + token.is_range(), ); if token.get_source().is_some() && !builder.has_source_contents(raw.src_id) { builder.set_source_contents( From 92ce26dbac96c3fe8163691692d62447fe82ef15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 13:29:31 +0900 Subject: [PATCH 50/63] Update test --- tests/test_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_builder.rs b/tests/test_builder.rs index 8d34a30..ad0716b 100644 --- a/tests/test_builder.rs +++ b/tests/test_builder.rs @@ -12,7 +12,7 @@ fn test_builder_into_sourcemap() { assert_eq!(sm.get_source(0), Some("/foo/bar/baz.js")); assert_eq!(sm.get_name(0), Some("x")); - let expected = br#"{"version":3,"sources":["baz.js"],"sourceRoot":"/foo/bar","names":["x"],"mappings":""}"#; + let expected = br#"{"version":3,"sources":["baz.js"],"sourceRoot":"/foo/bar","names":["x"],"rangeMappings":"","mappings":""}"#; let mut output: Vec = vec![]; sm.to_writer(&mut output).unwrap(); assert_eq!(output, expected); From af22bf63940eccd7bcb44dd459823285e19e8b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 13:39:01 +0900 Subject: [PATCH 51/63] Make encoding efficient --- src/encoder.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index b4eb5e3..ac7c089 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -2,6 +2,7 @@ use std::io::Write; use bitvec::field::BitField; use bitvec::order::Lsb0; +use bitvec::slice::BitSlice; use bitvec::view::BitView; use serde_json::Value; @@ -24,7 +25,7 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } -fn encode_rmi(out: &mut Vec, indices: &[usize]) { +fn encode_rmi(out: &mut Vec, bits: &mut BitSlice) { fn encode_byte(b: u8) -> u8 { match b { 0..=25 => b + b'A', @@ -36,15 +37,7 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { } } - let mut data = [0u8; 8]; - - let bits = data.view_bits_mut::(); - for &i in indices { - bits.set(i, true); - } - // trim zero at the end - let mut last = 0; for (idx, bit) in bits.iter().enumerate() { if *bit { @@ -65,30 +58,34 @@ fn encode_rmi(out: &mut Vec, indices: &[usize]) { fn serialize_range_mappings(sm: &SourceMap) -> Option { let mut buf = Vec::new(); let mut prev_line = 0; + let mut had_rmi = false; let mut idx_of_first_in_line = 0; - let mut indices = vec![]; + + let mut rmi_data = Vec::::new(); + + let rmi_bits = rmi_data.view_bits_mut::(); for (idx, token) in sm.tokens().enumerate() { if token.is_range() { let num = idx - idx_of_first_in_line; - indices.push(num); + rmi_bits.set(num, true); } while token.get_dst_line() != prev_line { - if !indices.is_empty() { - encode_rmi(&mut buf, &indices); - indices.clear(); + if had_rmi { + encode_rmi(&mut buf, rmi_bits); } buf.push(b';'); prev_line += 1; + had_rmi = false; idx_of_first_in_line = idx; } } - if !indices.is_empty() { - encode_rmi(&mut buf, &indices); + if had_rmi { + encode_rmi(&mut buf, rmi_bits); } Some(String::from_utf8(buf).expect("invalid utf8")) @@ -219,7 +216,15 @@ impl Encodable for DecodedMap { fn test_encode_rmi() { fn encode(indices: &[usize]) -> String { let mut out = vec![]; - encode_rmi(&mut out, indices); + + let mut data = [0u8; 8]; + + let bits = data.view_bits_mut::(); + for &i in indices { + bits.set(i, true); + } + + encode_rmi(&mut out, bits); String::from_utf8(out).unwrap() } From f80aa98c60fbda0119391cdbe0ff03bc6924a5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 13:43:20 +0900 Subject: [PATCH 52/63] dynamic size --- src/encoder.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/encoder.rs b/src/encoder.rs index ac7c089..d7e9dfc 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -2,7 +2,6 @@ use std::io::Write; use bitvec::field::BitField; use bitvec::order::Lsb0; -use bitvec::slice::BitSlice; use bitvec::view::BitView; use serde_json::Value; @@ -25,7 +24,7 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) { encode_vlq(out, i64::from(a) - i64::from(b)) } -fn encode_rmi(out: &mut Vec, bits: &mut BitSlice) { +fn encode_rmi(out: &mut Vec, data: &mut Vec) { fn encode_byte(b: u8) -> u8 { match b { 0..=25 => b + b'A', @@ -37,6 +36,8 @@ fn encode_rmi(out: &mut Vec, bits: &mut BitSlice) { } } + let bits = data.view_bits_mut::(); + // trim zero at the end let mut last = 0; for (idx, bit) in bits.iter().enumerate() { @@ -64,18 +65,19 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { let mut rmi_data = Vec::::new(); - let rmi_bits = rmi_data.view_bits_mut::(); - for (idx, token) in sm.tokens().enumerate() { if token.is_range() { let num = idx - idx_of_first_in_line; + rmi_data.reserve(8); + + let rmi_bits = rmi_data.view_bits_mut::(); rmi_bits.set(num, true); } while token.get_dst_line() != prev_line { if had_rmi { - encode_rmi(&mut buf, rmi_bits); + encode_rmi(&mut buf, &mut rmi_data); } buf.push(b';'); @@ -85,7 +87,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { } } if had_rmi { - encode_rmi(&mut buf, rmi_bits); + encode_rmi(&mut buf, &mut rmi_data); } Some(String::from_utf8(buf).expect("invalid utf8")) @@ -217,14 +219,16 @@ fn test_encode_rmi() { fn encode(indices: &[usize]) -> String { let mut out = vec![]; - let mut data = [0u8; 8]; + let mut data = Vec::with_capacity(256); + // Fill with zeros while testing + data.resize(256, 0); let bits = data.view_bits_mut::(); for &i in indices { bits.set(i, true); } - encode_rmi(&mut out, bits); + encode_rmi(&mut out, &mut data); String::from_utf8(out).unwrap() } From f8d1edd49827319a370fb5ec8ea25ed2d90f36a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 13:43:51 +0900 Subject: [PATCH 53/63] Use resize --- src/encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoder.rs b/src/encoder.rs index d7e9dfc..b43760e 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -69,7 +69,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { if token.is_range() { let num = idx - idx_of_first_in_line; - rmi_data.reserve(8); + rmi_data.resize(rmi_data.len() + 8, 0); let rmi_bits = rmi_data.view_bits_mut::(); rmi_bits.set(num, true); From 544605dfd13f7e0df62532eaedafdd3184066c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 16:41:54 +0900 Subject: [PATCH 54/63] fix --- src/encoder.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/encoder.rs b/src/encoder.rs index b43760e..ffcd670 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -67,9 +67,11 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { for (idx, token) in sm.tokens().enumerate() { if token.is_range() { + had_rmi = true; + let num = idx - idx_of_first_in_line; - rmi_data.resize(rmi_data.len() + 8, 0); + rmi_data.resize(rmi_data.len() + 2, 0); let rmi_bits = rmi_data.view_bits_mut::(); rmi_bits.set(num, true); @@ -78,6 +80,7 @@ fn serialize_range_mappings(sm: &SourceMap) -> Option { while token.get_dst_line() != prev_line { if had_rmi { encode_rmi(&mut buf, &mut rmi_data); + rmi_data.clear(); } buf.push(b';'); From e53ed5f831e4d1e8306033bfaf36ae797a61d106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 16:47:10 +0900 Subject: [PATCH 55/63] Fix unit test --- src/ram_bundle.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ram_bundle.rs b/src/ram_bundle.rs index 23f4401..fd3f63c 100644 --- a/src/ram_bundle.rs +++ b/src/ram_bundle.rs @@ -402,6 +402,7 @@ impl<'a> SplitRamBundleModuleIter<'a> { token.get_src_col(), token.get_source(), token.get_name(), + false, ); if token.get_source().is_some() && !builder.has_source_contents(raw.src_id) { builder.set_source_contents( From ca6d0c54e6a5df8b2bd0fe45dd305fe1db4c222d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Mon, 4 Mar 2024 16:47:57 +0900 Subject: [PATCH 56/63] fix clipy --- src/builder.rs | 2 ++ src/encoder.rs | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index f633e6c..92aaeb0 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -181,6 +181,7 @@ impl SourceMapBuilder { } /// Adds a new mapping to the builder. + #[allow(clippy::too_many_arguments)] pub fn add( &mut self, dst_line: u32, @@ -230,6 +231,7 @@ impl SourceMapBuilder { } /// Adds a new mapping to the builder. + #[allow(clippy::too_many_arguments)] pub fn add_raw( &mut self, dst_line: u32, diff --git a/src/encoder.rs b/src/encoder.rs index ffcd670..9a7fea0 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -222,9 +222,8 @@ fn test_encode_rmi() { fn encode(indices: &[usize]) -> String { let mut out = vec![]; - let mut data = Vec::with_capacity(256); // Fill with zeros while testing - data.resize(256, 0); + let mut data = vec![0; 256]; let bits = data.view_bits_mut::(); for &i in indices { From b6fd204b4aabb464124d3f8ffd8554a8f8a4a33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:16:33 +0900 Subject: [PATCH 57/63] Update test refs --- tests/test_encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_encoder.rs b/tests/test_encoder.rs index 682eb70..23adc7f 100644 --- a/tests/test_encoder.rs +++ b/tests/test_encoder.rs @@ -23,7 +23,7 @@ fn test_basic_sourcemap() { fn test_sourcemap_data_url() { let input: &[_] = br#"{"version":3,"file":"build/foo.min.js","sources":["src/foo.js"],"names":[],"mappings":"AAAA","sourceRoot":"/"}"#; let sm = SourceMap::from_reader(input).unwrap(); - assert_eq!(sm.to_data_url().unwrap(), "data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYnVpbGQvZm9vLm1pbi5qcyIsInNvdXJjZXMiOlsic3JjL2Zvby5qcyJdLCJzb3VyY2VSb290IjoiLyIsIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBIn0="); + assert_eq!(sm.to_data_url().unwrap(), "data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYnVpbGQvZm9vLm1pbi5qcyIsInNvdXJjZXMiOlsic3JjL2Zvby5qcyJdLCJzb3VyY2VSb290IjoiLyIsIm5hbWVzIjpbXSwicmFuZ2VNYXBwaW5ncyI6IiIsIm1hcHBpbmdzIjoiQUFBQSJ9"); } #[test] From d8e5bbea528dfd963d502702e13c8cdccafad353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:20:21 +0900 Subject: [PATCH 58/63] Add a test --- tests/test_regular.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_regular.rs b/tests/test_regular.rs index 141c3b5..d3dfd85 100644 --- a/tests/test_regular.rs +++ b/tests/test_regular.rs @@ -35,3 +35,28 @@ fn test_basic_sourcemap() { ("coolstuff.js", 2, 8, None) ); } + +#[test] +fn test_basic_range() { + let input: &[_] = br#"{ + "version": 3, + "sources": ["input.js"], + "names": ["console","log","ab"], + "mappings": "AACAA,QAAQC,GAAG,CAAC,OAAM,OAAM,QACxBD,QAAQC,GAAG,CAAC,QAEZD,QAAQC,GAAG,CAJD;IAACC,IAAI;AAAI,IAKnBF,QAAQC,GAAG,CAAC,YACZD,QAAQC,GAAG,CAAC", + "rangeMappings": "AAB;;g" + }"#; + let sm = SourceMap::from_reader(input).unwrap(); + + assert_eq!( + sm.lookup_token(1, 1).unwrap().to_tuple(), + ("input.js", 2, 2, None) + ); + assert_eq!( + sm.lookup_token(1, 8).unwrap().to_tuple(), + ("input.js", 2, 10, None) + ); + assert_eq!( + sm.lookup_token(1, 12).unwrap().to_tuple(), + ("input.js", 2, 14, None) + ); +} From 2fedd9fb07332b3c66531f7b544f27a9a0a4d658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:42:22 +0900 Subject: [PATCH 59/63] fixup --- tests/test_regular.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_regular.rs b/tests/test_regular.rs index d3dfd85..22c7495 100644 --- a/tests/test_regular.rs +++ b/tests/test_regular.rs @@ -1,4 +1,4 @@ -use sourcemap::SourceMap; +use sourcemap::{SourceMap, SourceMapBuilder}; #[test] fn test_basic_sourcemap() { @@ -38,14 +38,9 @@ fn test_basic_sourcemap() { #[test] fn test_basic_range() { - let input: &[_] = br#"{ - "version": 3, - "sources": ["input.js"], - "names": ["console","log","ab"], - "mappings": "AACAA,QAAQC,GAAG,CAAC,OAAM,OAAM,QACxBD,QAAQC,GAAG,CAAC,QAEZD,QAAQC,GAAG,CAJD;IAACC,IAAI;AAAI,IAKnBF,QAAQC,GAAG,CAAC,YACZD,QAAQC,GAAG,CAAC", - "rangeMappings": "AAB;;g" - }"#; - let sm = SourceMap::from_reader(input).unwrap(); + let mut b = SourceMapBuilder::new(Some("input.js")); + b.add_raw(1, 0, 2, 2, Some(0), None, true); + let sm = b.into_sourcemap(); assert_eq!( sm.lookup_token(1, 1).unwrap().to_tuple(), From 8538bd6623f1b6a854a869f61cea7443166190b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:48:55 +0900 Subject: [PATCH 60/63] offset --- src/types.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/types.rs b/src/types.rs index cfb1361..41d02db 100644 --- a/src/types.rs +++ b/src/types.rs @@ -157,6 +157,7 @@ pub struct RawToken { pub struct Token<'a> { raw: &'a RawToken, i: &'a SourceMap, + offset: u32, idx: u32, } @@ -221,7 +222,7 @@ impl<'a> Token<'a> { /// get the source column number pub fn get_src_col(&self) -> u32 { - self.raw.src_col + self.raw.src_col - self.offset } /// get the source line and column @@ -680,9 +681,12 @@ impl SourceMap { /// Looks up a token by its index. pub fn get_token(&self, idx: u32) -> Option> { - self.tokens - .get(idx as usize) - .map(|raw| Token { raw, i: self, idx }) + self.tokens.get(idx as usize).map(|raw| Token { + raw, + i: self, + idx, + offset: 0, + }) } /// Returns the number of tokens in the sourcemap. @@ -701,7 +705,14 @@ impl SourceMap { /// Looks up the closest token to a given 0-indexed line and column. pub fn lookup_token(&self, line: u32, col: u32) -> Option> { let ii = greatest_lower_bound(&self.index, &(line, col), |ii| (ii.0, ii.1))?; - self.get_token(ii.2) + + let mut token = self.get_token(ii.2)?; + + if token.is_range() { + token.offset = col - token.get_dst_col(); + } + + Some(token) } /// Given a location, name and minified source file resolve a minified From a7f33561c46745ef053ca7dfd329b03ea4e5b659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:49:13 +0900 Subject: [PATCH 61/63] saturate --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 41d02db..36e626d 100644 --- a/src/types.rs +++ b/src/types.rs @@ -222,7 +222,7 @@ impl<'a> Token<'a> { /// get the source column number pub fn get_src_col(&self) -> u32 { - self.raw.src_col - self.offset + self.raw.src_col.saturating_sub(self.offset) } /// get the source line and column From 95ff8b8195d56fdfad3b43f6ab2585a4886f7f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:50:17 +0900 Subject: [PATCH 62/63] Update test --- tests/test_regular.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_regular.rs b/tests/test_regular.rs index 22c7495..8bafbed 100644 --- a/tests/test_regular.rs +++ b/tests/test_regular.rs @@ -38,20 +38,21 @@ fn test_basic_sourcemap() { #[test] fn test_basic_range() { - let mut b = SourceMapBuilder::new(Some("input.js")); - b.add_raw(1, 0, 2, 2, Some(0), None, true); + let mut b = SourceMapBuilder::new(None); + let id = b.add_source("input.js"); + b.add_raw(1, 0, 2, 2, Some(id), None, true); let sm = b.into_sourcemap(); assert_eq!( - sm.lookup_token(1, 1).unwrap().to_tuple(), + sm.lookup_token(1, 0).unwrap().to_tuple(), ("input.js", 2, 2, None) ); assert_eq!( - sm.lookup_token(1, 8).unwrap().to_tuple(), + sm.lookup_token(1, 7).unwrap().to_tuple(), ("input.js", 2, 10, None) ); assert_eq!( - sm.lookup_token(1, 12).unwrap().to_tuple(), + sm.lookup_token(1, 11).unwrap().to_tuple(), ("input.js", 2, 14, None) ); } From b5c4ca84669062dd5be1134607148dd7afc4261a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Wed, 13 Mar 2024 11:52:45 +0900 Subject: [PATCH 63/63] Fix test & logic --- src/types.rs | 2 +- tests/test_regular.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/types.rs b/src/types.rs index 36e626d..4093a5a 100644 --- a/src/types.rs +++ b/src/types.rs @@ -222,7 +222,7 @@ impl<'a> Token<'a> { /// get the source column number pub fn get_src_col(&self) -> u32 { - self.raw.src_col.saturating_sub(self.offset) + self.raw.src_col.saturating_add(self.offset) } /// get the source line and column diff --git a/tests/test_regular.rs b/tests/test_regular.rs index 8bafbed..93db0d4 100644 --- a/tests/test_regular.rs +++ b/tests/test_regular.rs @@ -48,11 +48,11 @@ fn test_basic_range() { ("input.js", 2, 2, None) ); assert_eq!( - sm.lookup_token(1, 7).unwrap().to_tuple(), + sm.lookup_token(1, 8).unwrap().to_tuple(), ("input.js", 2, 10, None) ); assert_eq!( - sm.lookup_token(1, 11).unwrap().to_tuple(), + sm.lookup_token(1, 12).unwrap().to_tuple(), ("input.js", 2, 14, None) ); }