Pull the datastore module into its own crate

sources/Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "api/apiclient",
     "api/bork",
     "api/corndog",
+    "api/datastore",
     "api/early-boot-config",
     "api/ecs-settings-applier",
     "api/netdog",

sources/api/apiserver/Cargo.toml

@@ -12,6 +12,7 @@ exclude = ["README.md"]
 [dependencies]
 actix-web = { version = "3.2.0", default-features = false }
 bottlerocket-release = { path = "../../bottlerocket-release" }
+datastore = { path = "../datastore" }
 fs2 = "0.4.3"
 futures = { version = "0.3", default-features = false }
 http = "0.2.1"

sources/api/apiserver/README.md

@@ -55,18 +55,13 @@ The current data store implementation maps keys to filesystem paths and stores t
 Metadata about a data key is stored in a file at the data key path + "." + the metadata key.
 The default data store location is `/var/lib/bottlerocket/datastore/current`, and the filesystem format makes it fairly easy to inspect.
 
-### Serialization and deserialization
-
-The `datastore::serialization` module provides code to serialize Rust types into a mapping of datastore-acceptable keys (a.b.c) and values.
-
-The `datastore::deserialization` module provides code to deserialize datastore-acceptable keys (a.b.c) and values into Rust types.
+For more detail, see [datastore](../datastore).
 
 ## Current limitations
 
 * Data store locking is coarse; read requests can happen in parallel, but a write request will block everything else.
 * There's no support for rolling back commits.
 * There are no metrics.
-* `datastore::serialization` can't handle complex types under lists; it assumes lists can be serialized as scalars.
 
 ## Example usage
 

sources/api/apiserver/src/lib.rs

@@ -52,18 +52,13 @@ The current data store implementation maps keys to filesystem paths and stores t
 Metadata about a data key is stored in a file at the data key path + "." + the metadata key.
 The default data store location is `/var/lib/bottlerocket/datastore/current`, and the filesystem format makes it fairly easy to inspect.
 
-## Serialization and deserialization
-
-The `datastore::serialization` module provides code to serialize Rust types into a mapping of datastore-acceptable keys (a.b.c) and values.
-
-The `datastore::deserialization` module provides code to deserialize datastore-acceptable keys (a.b.c) and values into Rust types.
+For more detail, see [datastore](../datastore).
 
 # Current limitations
 
 * Data store locking is coarse; read requests can happen in parallel, but a write request will block everything else.
 * There's no support for rolling back commits.
 * There are no metrics.
-* `datastore::serialization` can't handle complex types under lists; it assumes lists can be serialized as scalars.
 
 # Example usage
 
@@ -79,7 +74,6 @@ See `../../apiclient/README.md` for client examples.
 #[macro_use]
 extern crate log;
 
-pub mod datastore;
 pub mod server;
 
 pub use server::serve;

sources/api/apiserver/src/server/controller.rs

@@ -8,13 +8,11 @@ use std::collections::{HashMap, HashSet};
 use std::io::Write;
 use std::process::{Command, Stdio};
 
-use crate::datastore::deserialization::{from_map, from_map_with_prefix};
-use crate::datastore::serialization::to_pairs;
-use crate::datastore::{
-    deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value,
-};
 use crate::server::error::{self, Result};
 use actix_web::HttpResponse;
+use datastore::deserialization::{from_map, from_map_with_prefix};
+use datastore::serialization::to_pairs;
+use datastore::{deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value};
 use model::{ConfigurationFiles, Services, Settings};
 use num::FromPrimitive;
 use std::os::unix::process::ExitStatusExt;
@@ -385,7 +383,7 @@ pub(crate) fn dispatch_update_command(args: &[&str]) -> Result<HttpResponse> {
         .status()
         .context(error::UpdateDispatcher)?;
     if status.success() {
-        return Ok(HttpResponse::NoContent().finish())
+        return Ok(HttpResponse::NoContent().finish());
     }
     let exit_status = match status.code() {
         Some(code) => code,
@@ -406,8 +404,8 @@ pub(crate) fn dispatch_update_command(args: &[&str]) -> Result<HttpResponse> {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::datastore::memory::MemoryDataStore;
-    use crate::datastore::{Committed, DataStore, Key, KeyType};
+    use datastore::memory::MemoryDataStore;
+    use datastore::{Committed, DataStore, Key, KeyType};
     use maplit::{hashmap, hashset};
     use model::Service;
     use std::convert::TryInto;

sources/api/apiserver/src/server/error.rs

@@ -1,4 +1,4 @@
-use crate::datastore::{self, deserialization, serialization};
+use datastore::{self, deserialization, serialization};
 use nix::unistd::Gid;
 use snafu::Snafu;
 use std::io;
@@ -55,9 +55,7 @@ pub enum Error {
     CommitWithNoPending,
 
     #[snafu(display("Unable to get OS release data: {}", source))]
-    ReleaseData {
-        source: bottlerocket_release::Error,
-    },
+    ReleaseData { source: bottlerocket_release::Error },
 
     // =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
 

sources/api/apiserver/src/server/mod.rs

@@ -5,11 +5,11 @@ mod controller;
 mod error;
 pub use error::Error;
 
-use crate::datastore::{Committed, FilesystemDataStore, Key, Value};
 use actix_web::{
     error::ResponseError, web, App, FromRequest, HttpRequest, HttpResponse, HttpServer, Responder,
 };
 use bottlerocket_release::BottlerocketRelease;
+use datastore::{Committed, FilesystemDataStore, Key, Value};
 use error::Result;
 use fs2::FileExt;
 use futures::future;
@@ -76,16 +76,15 @@ where
             // ResponseError implementation.  This configuration of the Json extractor allows us to
             // add the error message into the response.
             .app_data(web::Json::<Settings>::configure(|cfg| {
-                cfg.error_handler(|err, _req| HttpResponse::BadRequest().body(err.to_string()).into())
+                cfg.error_handler(|err, _req| {
+                    HttpResponse::BadRequest().body(err.to_string()).into()
+                })
             }))
-
             // This makes the data store available to API methods merely by having a Data
             // parameter.
             .app_data(shared_datastore.clone())
-
             // Retrieve the full API model; not all data is writable, so we only support GET.
             .route("/", web::get().to(get_model))
-
             .service(
                 web::scope("/settings")
                     .route("", web::get().to(get_settings))
@@ -161,7 +160,12 @@ async fn get_model(data: web::Data<SharedDataStore>) -> Result<ModelResponse> {
     let configuration_files = Some(controller::get_configuration_files(&*datastore)?);
     let os = Some(controller::get_os_info()?);
 
-    let model = Model {settings, services, configuration_files, os};
+    let model = Model {
+        settings,
+        services,
+        configuration_files,
+        os,
+    };
     Ok(ModelResponse(model))
 }
 

sources/api/datastore/.gitignore

@@ -0,0 +1,2 @@
+/target
+**/*.rs.bk

sources/api/datastore/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "datastore"
+version = "0.1.0"
+authors = ["Tom Kirchner <tjk@amazon.com>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+build = "build.rs"
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+libc = "0.2"
+log = "0.4"
+percent-encoding = "2.1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+snafu = "0.6"
+walkdir = "2.2"
+
+[build-dependencies]
+cargo-readme = "3.1"
+
+[dev-dependencies]
+maplit = "1.0"
+toml = "0.5"

sources/api/datastore/README.md

@@ -0,0 +1,34 @@
+# datastore
+
+Current version: 0.1.0
+
+## Background
+
+A 'data store' in Bottlerocket is responsible for storing key/value pairs and metadata about those pairs, with the ability to commit changes in transactions.
+
+For more detail about their usage, see [apiserver](../apiserver).
+
+## Library
+
+This library provides a trait defining the exact requirements, along with basic implementations for filesystem and memory data stores.
+
+There's also a common error type and some methods that implementations of DataStore should generally share, like scalar serialization.
+
+We represent scalars -- the actual values stored under a datastore key -- using JSON, just to have a convenient human-readable form.
+(TOML doesn't allow raw scalars.  The JSON spec doesn't seem to either, but this works, and the format is so simple for scalars that it could be easily swapped out if needed.)
+
+## Serialization and deserialization
+
+The `serialization` module provides code to serialize Rust types into a mapping of datastore-acceptable keys (a.b.c) and values.
+
+The `deserialization` module provides code to deserialize datastore-acceptable keys (a.b.c) and values into Rust types.
+
+## Current limitations
+
+* The user (e.g. apiserver) needs to handle locking.
+* There's no support for rolling back transactions.
+* The `serialization` module can't handle complex types under lists; it assumes lists can be serialized as scalars.
+
+## Colophon
+
+This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/lib.rs`.

sources/api/datastore/README.tpl

@@ -0,0 +1,9 @@
+# {{crate}}
+
+Current version: {{version}}
+
+{{readme}}
+
+## Colophon
+
+This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/lib.rs`.

sources/api/datastore/build.rs

@@ -0,0 +1,32 @@
+// Automatically generate README.md from rustdoc.
+
+use std::env;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+fn main() {
+    // Check for environment variable "SKIP_README". If it is set,
+    // skip README generation
+    if env::var_os("SKIP_README").is_some() {
+        return;
+    }
+
+    let mut lib = File::open("src/lib.rs").unwrap();
+    let mut template = File::open("README.tpl").unwrap();
+
+    let content = cargo_readme::generate_readme(
+        &PathBuf::from("."), // root
+        &mut lib,            // source
+        Some(&mut template), // template
+        // The "add x" arguments don't apply when using a template.
+        true,  // add title
+        false, // add badges
+        false, // add license
+        true,  // indent headings
+    )
+    .unwrap();
+
+    let mut readme = File::create("README.md").unwrap();
+    readme.write_all(content.as_bytes()).unwrap();
+}

sources/api/apiserver/src/datastore/deserialization/error.rs → sources/api/datastore/src/deserialization/error.rs

@@ -1,7 +1,7 @@
 use serde::de;
 use snafu::{IntoError, NoneError as NoSource, Snafu};
 
-use crate::datastore::{Error as DataStoreError, ScalarError};
+use crate::{Error as DataStoreError, ScalarError};
 
 /// Potential errors from deserialization.
 #[derive(Debug, Snafu)]

sources/api/apiserver/src/datastore/deserialization/pairs.rs → sources/api/datastore/src/deserialization/pairs.rs

@@ -31,6 +31,7 @@
 //! the field to our "path" string.  In the example above, when we're looking at field "c", path
 //! would be "a.b", so we know we should look for "a.b.c" in our input mapping.
 
+use log::{error, trace};
 use serde::de::{value::MapDeserializer, IntoDeserializer, Visitor};
 use serde::{forward_to_deserialize_any, Deserialize};
 use snafu::ResultExt;
@@ -39,7 +40,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use std::hash::Hash;
 
 use super::{error, Error, Result};
-use crate::datastore::{deserializer_for_scalar, Key, KeyType, ScalarDeserializer};
+use crate::{deserializer_for_scalar, Key, KeyType, ScalarDeserializer};
 
 /// This is the primary interface to deserialization.  We turn the input map into the requested
 /// output type, assuming all non-Option fields are provided, etc.
@@ -408,7 +409,7 @@ where
 #[cfg(test)]
 mod test {
     use super::{from_map, from_map_with_prefix};
-    use crate::datastore::{deserialization::Error, Key, KeyType};
+    use crate::{deserialization::Error, Key, KeyType};
 
     use maplit::hashmap;
     use serde::Deserialize;

sources/api/apiserver/src/datastore/filesystem.rs → sources/api/datastore/src/filesystem.rs

@@ -4,6 +4,7 @@
 //! Data is kept in files with paths resembling the keys, e.g. a/b/c for a.b.c, and metadata is
 //! kept in a suffixed file next to the data, e.g. a/b/c.meta for metadata "meta" about a.b.c
 
+use log::{debug, error, trace};
 use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};

sources/api/apiserver/src/datastore/key.rs → sources/api/datastore/src/key.rs

@@ -1,5 +1,6 @@
 // Note: this only allows reading and writing UTF-8 keys and values; is that OK?
 
+use log::trace;
 use serde::{Serialize, Serializer};
 use snafu::ensure;
 use std::fmt;