diff --git a/Cargo.lock b/Cargo.lock index 0b911538..ba3d573f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,37 +72,12 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bstr" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - [[package]] name = "clap" version = "4.3.19" @@ -163,35 +138,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "cpufeatures" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - [[package]] name = "encode_unicode" version = "0.3.6" @@ -225,30 +171,12 @@ dependencies = [ "libc", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "gengo" version = "0.1.0" dependencies = [ + "gengo-impl", "regex", - "serde", - "serde_yaml", - "tera", ] [[package]] @@ -261,27 +189,17 @@ dependencies = [ ] [[package]] -name = "globset" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df" -dependencies = [ - "aho-corasick", - "bstr", - "fnv", - "log", - "regex", -] - -[[package]] -name = "globwalk" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +name = "gengo-impl" +version = "0.1.0" dependencies = [ - "bitflags 1.3.2", - "ignore", - "walkdir", + "indexmap", + "insta", + "paste", + "proc-macro2", + "quote", + "serde", + "serde_yaml", + "syn", ] [[package]] @@ -302,23 +220,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" -[[package]] -name = "ignore" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" -dependencies = [ - "globset", - "lazy_static", - "log", - "memchr", - "regex", - "same-file", - "thread_local", - "walkdir", - "winapi-util", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -327,6 +228,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -400,12 +302,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" -[[package]] -name = "log" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" - [[package]] name = "memchr" version = "2.5.0" @@ -419,48 +315,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] -name = "pest" -version = "2.7.1" +name = "paste" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d2d1d55045829d65aad9d389139882ad623b33b904e7c9f1b10c5b8927298e5" -dependencies = [ - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f94bca7e7a599d89dea5dfa309e217e7906c3c007fb9c3299c40b10d6a315d3" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d490fe7e8556575ff6911e45567ab95e71617f43781e5c05490dc8d75c965c" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pest_meta" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674c66ebb4b4d9036012091b537aae5878970d6999f81a265034d85b136b341" -dependencies = [ - "once_cell", - "pest", - "sha2", -] +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "proc-macro2" @@ -542,15 +400,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "serde" version = "1.0.175" @@ -571,17 +420,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_json" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_yaml" version = "0.9.25" @@ -595,17 +433,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "sha2" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "similar" version = "2.2.1" @@ -629,23 +456,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tera" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ab29bb4f3e256ae6ad5c3e2775aa1f8829f2c0c101fc407bfd3a6df15c60c5" -dependencies = [ - "globwalk", - "lazy_static", - "pest", - "pest_derive", - "regex", - "serde", - "serde_json", - "thread_local", - "unic-segment", -] - [[package]] name = "terminal_size" version = "0.2.6" @@ -656,97 +466,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "thiserror" -version = "1.0.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" -dependencies = [ - "once_cell", -] - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - -[[package]] -name = "unic-char-property" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" -dependencies = [ - "unic-char-range", -] - -[[package]] -name = "unic-char-range" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" - -[[package]] -name = "unic-common" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" - -[[package]] -name = "unic-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" -dependencies = [ - "unic-ucd-segment", -] - -[[package]] -name = "unic-ucd-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-version" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" -dependencies = [ - "unic-common", -] - [[package]] name = "unicode-ident" version = "1.0.11" @@ -765,53 +484,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "walkdir" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index e5c9fbfc..41695916 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["gengo", "gengo-bin"] +members = ["gengo", "gengo-bin", "gengo-impl"] diff --git a/gengo-impl/Cargo.toml b/gengo-impl/Cargo.toml new file mode 100644 index 00000000..6207e3f6 --- /dev/null +++ b/gengo-impl/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "gengo-impl" +version = "0.1.0" +edition = "2021" + +exclude = ["fixtures/", "src/snapshots/"] + +[lib] +proc-macro = true + +[dependencies] +indexmap = { version = "2", features = ["serde"] } +proc-macro2 = "1" +quote = "1" +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +syn = "2" + +[dev-dependencies] +insta = "1" +paste = "1" diff --git a/gengo-impl/fixtures/default_analyzers.yaml b/gengo-impl/fixtures/default_analyzers.yaml new file mode 100644 index 00000000..db52301f --- /dev/null +++ b/gengo-impl/fixtures/default_analyzers.yaml @@ -0,0 +1,22 @@ +--- +Plain Text: + category: prose + color: "#000000" + matchers: + extensions: + - ".text" + - ".txt" + filenames: + - "LICENCE" + - "LICENSE" +Dockerfile: + category: programming + color: "#0000AA" + heuristics: + - "^(?m)ENTRYPOINT\\ " + matchers: + filenames: + - Dockerfile + patterns: + - "Dockerfile\\.[\\w\\d]+$" + priority: 0.75 diff --git a/gengo-impl/fixtures/language_enum.yaml b/gengo-impl/fixtures/language_enum.yaml new file mode 100644 index 00000000..b24eceb8 --- /dev/null +++ b/gengo-impl/fixtures/language_enum.yaml @@ -0,0 +1,17 @@ +--- +Plain Text: + category: prose + color: "#000000" + matchers: + extensions: + - ".text" + - ".txt" + filenames: + - "LICENCE" + - "LICENSE" +Rust: + category: programming + color: "#FF0000" + matchers: + extensions: + - ".rs" diff --git a/gengo/languages.yaml b/gengo-impl/languages.yaml similarity index 100% rename from gengo/languages.yaml rename to gengo-impl/languages.yaml diff --git a/gengo-impl/src/lib.rs b/gengo-impl/src/lib.rs new file mode 100644 index 00000000..ae3cea44 --- /dev/null +++ b/gengo-impl/src/lib.rs @@ -0,0 +1,207 @@ +//! Crate for reading the `languages.yaml` file to generate code. +//! +//! You probably don't want to use this. +use indexmap::IndexMap; +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use proc_macro2::{Ident, Literal, Span}; +use quote::quote; +use serde::{Deserialize, Serialize}; +use syn::{parse_macro_input, LitStr}; + +const LANGUAGES_YAML: &str = include_str!("../languages.yaml"); + +type Languages = IndexMap; + +#[derive(Debug, Deserialize, Serialize)] +struct Language { + category: LanguageCategory, + color: String, + matchers: Matchers, + #[serde(default)] + heuristics: Vec, + #[serde(default = "default_priority")] + priority: f32, +} + +fn default_priority() -> f32 { + 0.5 +} + +#[derive(Debug, Deserialize, Serialize)] +struct Matchers { + #[serde(default)] + extensions: Vec, + #[serde(default)] + filenames: Vec, + #[serde(default)] + patterns: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +enum LanguageCategory { + Data, + Markup, + Programming, + Prose, +} + +fn rustify(s: &str) -> String { + s.replace(' ', "") +} + +fn language_enum_impl(languages: &str) -> TokenStream2 { + let languages: Languages = serde_yaml::from_str(languages).unwrap(); + let variants: Vec = languages + .iter() + .map(|(name, _)| rustify(name)) + .map(|name| { + let variant = Ident::new(&name, Span::call_site()); + quote! { #variant, } + }) + .collect(); + quote! { + enum Language { + #(#variants)* + } + } +} + +#[proc_macro] +pub fn language_enum(input: TokenStream) -> TokenStream { + if !input.is_empty() { + panic!("language_enum takes no arguments"); + } + let tokens = language_enum_impl(LANGUAGES_YAML); + tokens.into() +} + +fn default_analyzers_impl(languages: &str) -> TokenStream2 { + let languages: Languages = serde_yaml::from_str(languages).unwrap(); + let initializers: Vec = languages + .iter() + .map(|(name, language)| { + let name = rustify(name); + let name = Ident::new(&name, Span::call_site()); + let name = quote! { Language::#name }; + + let category = match language.category { + LanguageCategory::Data => "Data", + LanguageCategory::Markup => "Markup", + LanguageCategory::Programming => "Programming", + LanguageCategory::Prose => "Prose", + }; + let category = Ident::new(category, Span::call_site()); + let category = quote! { Category::#category }; + + let color = Literal::string(&language.color); + + let extensions: Vec = language + .matchers + .extensions + .iter() + .map(|ext| { + Literal::string(ext); + quote! { #ext } + }) + .collect(); + let extensions = quote! { &[#(#extensions ,)*] }; + + let filenames: Vec = language + .matchers + .filenames + .iter() + .map(|filename| { + Literal::string(filename); + quote! { #filename } + }) + .collect(); + let filenames = quote! { &[#(#filenames ,)*] }; + + let patterns: Vec = language + .matchers + .patterns + .iter() + .map(|pattern| { + Literal::string(pattern); + quote! { #pattern } + }) + .collect(); + let patterns = quote! { &[#(#patterns ,)*] }; + + let heuristics: Vec = language + .heuristics + .iter() + .map(|heuristic| { + Literal::string(heuristic); + quote! { #heuristic } + }) + .collect(); + let heuristics = quote! { &[#(#heuristics ,)*] }; + + let priority = Literal::f32_suffixed(language.priority); + + quote! { + Analyzer::new( + #name, + #category, + #color, + #extensions, + #filenames, + #patterns, + #heuristics, + #priority, + ) + } + }) + .collect(); + quote! { + vec![ + #(#initializers ,)* + ] + } +} + +#[proc_macro] +pub fn default_analyzers(input: TokenStream) -> TokenStream { + if !input.is_empty() { + panic!("default_analyzers takes no arguments"); + } + let tokens = default_analyzers_impl(LANGUAGES_YAML); + tokens.into() +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::{assert_debug_snapshot, assert_snapshot}; + use paste::paste; + + macro_rules! fixture { + ($name:literal $(,)?) => { + include_str!(concat!("../fixtures/", $name, ".yaml")) + }; + } + + macro_rules! snapshot_test_macro_impl { + ($name:ident , $fixture:literal $(,)?) => { + paste! { + #[test] + fn [< $name _rendered >]() { + let tokens = $name(fixture!($fixture)); + assert_snapshot!(tokens.to_string()); + } + + #[test] + fn [< $name _tokens >]() { + let tokens = $name(fixture!($fixture)); + assert_debug_snapshot!(tokens); + } + } + }; + } + + snapshot_test_macro_impl!(language_enum_impl, "language_enum"); + snapshot_test_macro_impl!(default_analyzers_impl, "default_analyzers"); +} diff --git a/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_rendered.snap b/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_rendered.snap new file mode 100644 index 00000000..651507d4 --- /dev/null +++ b/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_rendered.snap @@ -0,0 +1,5 @@ +--- +source: gengo-impl/src/lib.rs +expression: tokens.to_string() +--- +vec ! [Analyzer :: new (Language :: PlainText , Category :: Prose , "#000000" , & [".text" , ".txt" ,] , & ["LICENCE" , "LICENSE" ,] , & [] , & [] , 0.5f32 ,) , Analyzer :: new (Language :: Dockerfile , Category :: Programming , "#0000AA" , & [] , & ["Dockerfile" ,] , & ["Dockerfile\\.[\\w\\d]+$" ,] , & ["^(?m)ENTRYPOINT\\ " ,] , 0.75f32 ,) ,] diff --git a/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_tokens.snap b/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_tokens.snap new file mode 100644 index 00000000..14406aa4 --- /dev/null +++ b/gengo-impl/src/snapshots/gengo_impl__tests__default_analyzers_impl_tokens.snap @@ -0,0 +1,314 @@ +--- +source: gengo-impl/src/lib.rs +expression: tokens +--- +TokenStream [ + Ident { + sym: vec, + }, + Punct { + char: '!', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Ident { + sym: Analyzer, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: new, + }, + Group { + delimiter: Parenthesis, + stream: TokenStream [ + Ident { + sym: Language, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: PlainText, + }, + Punct { + char: ',', + spacing: Alone, + }, + Ident { + sym: Category, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: Prose, + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: "#000000", + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Literal { + lit: ".text", + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: ".txt", + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Literal { + lit: "LICENCE", + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: "LICENSE", + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [], + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: 0.5f32, + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Ident { + sym: Analyzer, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: new, + }, + Group { + delimiter: Parenthesis, + stream: TokenStream [ + Ident { + sym: Language, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: Dockerfile, + }, + Punct { + char: ',', + spacing: Alone, + }, + Ident { + sym: Category, + }, + Punct { + char: ':', + spacing: Joint, + }, + Punct { + char: ':', + spacing: Alone, + }, + Ident { + sym: Programming, + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: "#0000AA", + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Literal { + lit: "Dockerfile", + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Literal { + lit: "Dockerfile\\.[\\w\\d]+$", + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Punct { + char: '&', + spacing: Alone, + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Literal { + lit: "^(?m)ENTRYPOINT\\ ", + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: 0.75f32, + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, +] diff --git a/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_rendered.snap b/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_rendered.snap new file mode 100644 index 00000000..f7436cfb --- /dev/null +++ b/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_rendered.snap @@ -0,0 +1,5 @@ +--- +source: gengo-impl/src/lib.rs +expression: tokens.to_string() +--- +enum Language { PlainText , Rust , } diff --git a/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_tokens.snap b/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_tokens.snap new file mode 100644 index 00000000..47237f77 --- /dev/null +++ b/gengo-impl/src/snapshots/gengo_impl__tests__language_enum_impl_tokens.snap @@ -0,0 +1,31 @@ +--- +source: gengo-impl/src/lib.rs +expression: tokens +--- +TokenStream [ + Ident { + sym: enum, + }, + Ident { + sym: Language, + }, + Group { + delimiter: Brace, + stream: TokenStream [ + Ident { + sym: PlainText, + }, + Punct { + char: ',', + spacing: Alone, + }, + Ident { + sym: Rust, + }, + Punct { + char: ',', + spacing: Alone, + }, + ], + }, +] diff --git a/gengo/Cargo.toml b/gengo/Cargo.toml index 8a338e8e..00f8c1a8 100644 --- a/gengo/Cargo.toml +++ b/gengo/Cargo.toml @@ -6,9 +6,5 @@ repository = "https://github.com/spenserblack/gengo" readme = "../README.md" [dependencies] +gengo-impl = { version = "0.1", path = "../gengo-impl" } regex = "1" - -[build-dependencies] -serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.9" -tera = { version = "1", default-features = false } diff --git a/gengo/build.rs b/gengo/build.rs deleted file mode 100644 index ee2122c4..00000000 --- a/gengo/build.rs +++ /dev/null @@ -1,77 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::error::Error; -use std::fs; -use std::path::Path; -use tera::{Context, Tera, Value}; - -const LANGUAGES: &'static str = include_str!("./languages.yaml"); - -#[derive(Debug, Deserialize, Serialize)] -struct Language { - category: LanguageCategory, - color: String, - matchers: Matchers, - #[serde(default)] - heuristics: Vec, - #[serde(default = "default_priority")] - priority: f32, -} - -fn default_priority() -> f32 { - 0.5 -} - -#[derive(Debug, Deserialize, Serialize)] -struct Matchers { - #[serde(default)] - extensions: Vec, - #[serde(default)] - filenames: Vec, - #[serde(default)] - patterns: Vec, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -enum LanguageCategory { - Data, - Markup, - Programming, - Prose, -} - -macro_rules! template { - ($name:literal) => { - include_str!(concat!("./templates/", $name, ".tera")) - }; -} - -fn main() -> Result<(), Box> { - let mut tera = Tera::default(); - tera.register_filter("rustify", rustify); - let languages: HashMap = dbg!(serde_yaml::from_str(LANGUAGES))?; - let mut context = Context::new(); - context.insert("languages", &languages); - let context = dbg!(context); - - let analyzer_target_path = Path::new(&env::var("OUT_DIR")?).join("analyzer.rs"); - let code = tera.render_str(template!("analyzer.rs"), &context)?; - fs::write(&analyzer_target_path, code)?; - - let languages_target_path = Path::new(&env::var("OUT_DIR")?).join("languages.rs"); - let code = tera.render_str(template!("languages.rs"), &context)?; - fs::write(&languages_target_path, code)?; - Ok(()) -} - -/// Takes a human readable string like `"Foo Bar"` and returns a Rust identifier like `FooBar`. -fn rustify(value: &Value, _args: &HashMap) -> tera::Result { - let value = match value { - Value::String(s) => s, - _ => return Err("rustify filter only accepts strings".into()), - }; - let rustified = value.replace(" ", ""); - Ok(Value::String(rustified)) -} diff --git a/gengo/src/languages/analyzer.rs b/gengo/src/languages/analyzer.rs index cdc1f184..48f3b42e 100644 --- a/gengo/src/languages/analyzer.rs +++ b/gengo/src/languages/analyzer.rs @@ -1,16 +1,17 @@ /// Analyzes a language. use super::{Category, Language}; -use std::collections::HashSet; +use gengo_impl::default_analyzers; use regex::Regex; -use std::path::Path; +use std::collections::HashSet; use std::ffi::{OsStr, OsString}; +use std::path::Path; pub struct Analyzers(Vec); impl Default for Analyzers { /// Create a new language analyzer with default values. fn default() -> Self { - Self(include!(concat!(env!("OUT_DIR"), "/analyzer.rs"))) + Self(default_analyzers!()) } } pub struct Analyzer { @@ -59,7 +60,8 @@ impl Analyzer { } pub fn matches_filename(&self, filename: &str) -> bool { - self.filenames.contains(Path::new(filename).file_name().unwrap_or_default()) + self.filenames + .contains(Path::new(filename).file_name().unwrap_or_default()) } } diff --git a/gengo/src/languages/mod.rs b/gengo/src/languages/mod.rs index 7b13354a..572337f6 100644 --- a/gengo/src/languages/mod.rs +++ b/gengo/src/languages/mod.rs @@ -1,6 +1,8 @@ +use gengo_impl::language_enum; + pub mod analyzer; -include!(concat!(env!("OUT_DIR"), "/languages.rs")); +language_enum!(); // A category for a language. pub enum Category {