From e9191a43cc47bba686dbe3f822ca46c0ce9f0f52 Mon Sep 17 00:00:00 2001 From: Tristan Hume Date: Wed, 28 Jun 2017 15:39:51 -0700 Subject: [PATCH] Add pure Rust versions of dump loading and creation Splits out the existing functionality into dump-load and dump-create features and adds dump-load-rs and dump-create-rs features which use the libflate crate but are respectively slower and worse at compressing. Also updates the LICENSE.txt file to comply with Google's preferences and updates .travis.yml to build the configuration used by Xi. --- .travis.yml | 2 + Cargo.toml | 25 ++++++++-- LICENSE.txt | 2 +- src/dumps.rs | 85 ++++++++++++++++++++++++--------- src/easy.rs | 2 +- src/highlighting/highlighter.rs | 4 +- src/lib.rs | 5 ++ 7 files changed, 95 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1a33d583..a0c1c27f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,4 +25,6 @@ script: # default features are required for examples to build - so remove them from sight. # Doc-tests may also use default features - rm -Rf examples && cargo test --lib --no-default-features + # Test the build configuration that Xi uses + - cargo test --lib --no-default-features --features "assets dump-load-rs" after_success: curl https://raw.githubusercontent.com/trishume/syntect/master/scripts/travis-doc-upload.sh | sh diff --git a/Cargo.toml b/Cargo.toml index a26447d7..168716cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,9 @@ regex-syntax = { version = "^0.4", optional = true } lazy_static = "^0.2" bitflags = "^0.8" plist = "0.2" -bincode = "0.8" -flate2 = "^0.2" +bincode = { version = "0.8", optional = true } +flate2 = { version = "^0.2", optional = true } +libflate = { version = "0.1.8", optional = true } fnv = { version = "^1.0", optional = true } serde = { version = "1.0", features = ["rc"] } serde_derive = "1.0" @@ -33,12 +34,28 @@ regex = "0.2.1" getopts = "0.2" [features] + +# If you enable two dump loading features or two creation features the build +# will fail, choose one of each. If you are using both creation and loading, +# your binary will be smaller if you choose the same one for each. + +# Pure Rust dump loading, slower than regular `dump-load` +dump-load-rs = ["libflate", "bincode"] +# Dump loading using flate2, which depends on the miniz C library. +dump-load = ["flate2", "bincode"] +# Dump creation using flate2, which depends on the miniz C library. +dump-create = ["flate2", "bincode"] +# Pure Rust dump creation, worse compressor so produces larger dumps than dump-create +dump-create-rs = ["libflate", "bincode"] + static-onig = ["onig/static-libonig"] parsing = ["onig", "regex-syntax", "fnv"] +# The `assets` feature enables inclusion of the default theme and syntax packages. +# For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set. assets = [] html = ["parsing", "assets"] -yaml-load = ["yaml-rust"] -default = ["parsing", "assets", "html", "yaml-load"] +yaml-load = ["yaml-rust", "parsing"] +default = ["parsing", "assets", "html", "yaml-load", "dump-load", "dump-create"] # [profile.release] # debug = true diff --git a/LICENSE.txt b/LICENSE.txt index d00992bd..7ddbcce5 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2016 Tristan Hume +Copyright (c) 2017 Tristan Hume, Keith Hall, Google Inc and other contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/dumps.rs b/src/dumps.rs index 2780a543..76e4e863 100644 --- a/src/dumps.rs +++ b/src/dumps.rs @@ -7,55 +7,95 @@ //! You can use these methods to manage your own caching of compiled syntaxes and //! themes. And even your own `serde::Serialize` structures if you want to //! be consistent with your format. -use bincode::{ErrorKind, Infinite, Result, deserialize_from, serialize_into}; +use bincode::{ErrorKind, Infinite, Result}; +#[cfg(any(feature = "dump-load", feature = "dump-load-rs"))] +use bincode::deserialize_from; +#[cfg(any(feature = "dump-create", feature = "dump-create-rs"))] +use bincode::serialize_into; use std::fs::File; -use std::io::{BufReader, BufWriter}; -#[cfg(all(feature = "parsing", feature = "assets"))] +#[cfg(any(feature = "dump-load", feature = "dump-load-rs"))] +use std::io::{BufReader, Read}; +#[cfg(any(feature = "dump-create", feature = "dump-create-rs"))] +use std::io::{BufWriter, Write}; +#[cfg(all(feature = "parsing", feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] use parsing::SyntaxSet; -#[cfg(feature = "assets")] +#[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] use highlighting::ThemeSet; use std::path::Path; +#[cfg(feature = "dump-create")] use flate2::write::ZlibEncoder; +#[cfg(feature = "dump-load")] use flate2::read::ZlibDecoder; +#[cfg(feature = "dump-create")] use flate2::Compression; +#[cfg(any(feature = "dump-create", feature = "dump-create-rs"))] use serde::Serialize; +#[cfg(any(feature = "dump-load", feature = "dump-load-rs"))] use serde::de::DeserializeOwned; +#[cfg(feature = "dump-load-rs")] +use libflate::zlib::Decoder; +#[cfg(feature = "dump-create-rs")] +use libflate::zlib::Encoder; + +#[cfg(feature = "dump-create")] +pub fn dump_to_writer(to_dump: &T, output: W) -> Result<()> { + let mut encoder = ZlibEncoder::new(output, Compression::Best); + serialize_into(&mut encoder, to_dump, Infinite) +} + +#[cfg(feature = "dump-create-rs")] +pub fn dump_to_writer(to_dump: &T, output: W) -> Result<()> { + let mut encoder = Encoder::new(output)?; + serialize_into(&mut encoder, to_dump, Infinite)?; + encoder.finish().into_result()?; + Ok(()) +} /// Dumps an object to a binary array in the same format as `dump_to_file` +#[cfg(any(feature = "dump-create", feature = "dump-create-rs"))] pub fn dump_binary(o: &T) -> Vec { let mut v = Vec::new(); - { - let mut encoder = ZlibEncoder::new(&mut v, Compression::Best); - serialize_into(&mut encoder, o, Infinite).unwrap(); - } + dump_to_writer(o, &mut v).unwrap(); v } /// Dumps an encodable object to a file at a given path. If a file already exists at that path /// it will be overwritten. The files created are encoded with the `bincode` crate and then /// compressed with the `flate2` crate. +#[cfg(any(feature = "dump-create", feature = "dump-create-rs"))] pub fn dump_to_file>(o: &T, path: P) -> Result<()> { - let f = BufWriter::new(try!(File::create(path).map_err(ErrorKind::IoError))); - let mut encoder = ZlibEncoder::new(f, Compression::Best); - serialize_into(&mut encoder, o, Infinite) + let out = BufWriter::new(try!(File::create(path).map_err(ErrorKind::IoError))); + dump_to_writer(o, out) +} + +#[cfg(feature = "dump-load")] +pub fn from_reader(input: R) -> Result { + let mut decoder = ZlibDecoder::new(input); + deserialize_from(&mut decoder, Infinite) +} + +#[cfg(feature = "dump-load-rs")] +pub fn from_reader(input: R) -> Result { + let mut decoder: Decoder = try!(Decoder::new(input)); + deserialize_from(&mut decoder, Infinite) } /// Returns a fully loaded syntax set from /// a binary dump. Panics if the dump is invalid. +#[cfg(any(feature = "dump-load", feature = "dump-load-rs"))] pub fn from_binary(v: &[u8]) -> T { - let mut decoder = ZlibDecoder::new(v); - deserialize_from(&mut decoder, Infinite).unwrap() + from_reader(v).unwrap() } -/// Returns a fully loaded syntax set from -/// a binary dump file. +/// Returns a fully loaded syntax set from a binary dump file. +#[cfg(any(feature = "dump-load", feature = "dump-load-rs"))] pub fn from_dump_file>(path: P) -> Result { let f = try!(File::open(path).map_err(ErrorKind::IoError)); - let mut decoder = ZlibDecoder::new(BufReader::new(f)); - deserialize_from(&mut decoder, Infinite) + let reader = BufReader::new(f); + from_reader(reader) } -#[cfg(all(feature = "parsing", feature = "assets"))] +#[cfg(all(feature = "parsing", feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] impl SyntaxSet { /// Instantiates a new syntax set from a binary dump of /// Sublime Text's default open source syntax definitions and then links it. @@ -89,7 +129,7 @@ impl SyntaxSet { } } -#[cfg(feature = "assets")] +#[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] impl ThemeSet { /// Loads the set of default themes /// Currently includes (these are the keys for the map): @@ -104,7 +144,7 @@ impl ThemeSet { #[cfg(test)] mod tests { - #[cfg(feature = "yaml-load")] + #[cfg(all(feature = "yaml-load", any(feature = "dump-create", feature = "dump-create-rs"), any(feature = "dump-load", feature = "dump-load-rs")))] #[test] fn can_dump_and_load() { use super::*; @@ -113,12 +153,13 @@ mod tests { ps.load_syntaxes("testdata/Packages", false).unwrap(); let bin = dump_binary(&ps); + println!("{:?}", bin.len()); let ps2: SyntaxSet = from_binary(&bin[..]); assert_eq!(ps.syntaxes().len(), ps2.syntaxes().len()); } - - #[cfg(feature = "assets")] + + #[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] #[test] fn has_default_themes() { use highlighting::ThemeSet; diff --git a/src/easy.rs b/src/easy.rs index 999f0f21..0aa1fb8b 100644 --- a/src/easy.rs +++ b/src/easy.rs @@ -174,7 +174,7 @@ impl<'a> Iterator for ScopeRegionIterator<'a> { } } -#[cfg(feature = "assets")] +#[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))] #[cfg(test)] mod tests { use super::*; diff --git a/src/highlighting/highlighter.rs b/src/highlighting/highlighter.rs index a8f271cc..a59f97c0 100644 --- a/src/highlighting/highlighter.rs +++ b/src/highlighting/highlighter.rs @@ -271,13 +271,13 @@ impl<'a> Highlighter<'a> { } } -#[cfg(feature = "assets")] +#[cfg(all(feature = "assets", feature = "parsing", any(feature = "dump-load", feature = "dump-load-rs")))] #[cfg(test)] mod tests { use super::*; use highlighting::{ThemeSet, Style, Color, FontStyle}; use parsing::{ SyntaxSet, ScopeStack, ParseState}; - + #[test] fn can_parse() { let ps = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 161248e7..b4d80612 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,10 +24,14 @@ extern crate regex_syntax; #[macro_use] extern crate lazy_static; extern crate plist; +#[cfg(any(feature = "dump-load-rs", feature = "dump-load", feature = "dump-create"))] extern crate bincode; #[macro_use] extern crate bitflags; +#[cfg(any(feature = "dump-load", feature = "dump-create"))] extern crate flate2; +#[cfg(any(feature = "dump-load-rs", feature = "dump-create-rs"))] +extern crate libflate; #[cfg(feature = "parsing")] extern crate fnv; extern crate serde; @@ -37,6 +41,7 @@ extern crate serde_json; pub mod highlighting; pub mod parsing; pub mod util; +#[cfg(any(feature = "dump-load-rs", feature = "dump-load", feature = "dump-create", feature = "dump-create-rs"))] pub mod dumps; #[cfg(feature = "parsing")] pub mod easy;