feat: allow comments in config.json, genesis.json, node_key.json and …

…validator.json (#8423)

* Use json-comments-rs crate to skip config file comments

* strip comments out from json for genesis_config.json, config.json, validator_key.json and node_key.json

* strip comments functions sit in near-config-utils (utils/config)

Cargo.toml

@@ -57,6 +57,7 @@ members = [
     "tools/state-viewer",
     "tools/storage-usage-delta-calculator",
     "tools/themis",
+    "utils/config",
     "utils/mainnet-res",
     "utils/near-cache",
     "utils/stdx",
@@ -124,6 +125,7 @@ indicatif = { version = "0.15.0", features = ["with_rayon"] }
 insta = { version = "1.26.0", features = ["json", "yaml"] }
 itertools = "0.10.0"
 itoa = "1.0"
+json_comments = "0.2.1"
 libc = "0.2.81"
 libfuzzer-sys = { version = "0.4", features = ["arbitrary-derive"] }
 log = "0.4"

core/chain-configs/Cargo.toml

@@ -25,6 +25,7 @@ tracing.workspace = true
 near-crypto = { path = "../crypto" }
 near-o11y = { path = "../o11y" }
 near-primitives = { path = "../primitives" }
+near-config-utils = { path = "../../utils/config" }
 
 [features]
 default = []

core/chain-configs/src/genesis_config.rs

@@ -258,21 +258,28 @@ pub struct Genesis {
 
 impl GenesisConfig {
     /// Parses GenesisConfig from a JSON string.
-    ///
+    /// The string can be a JSON with comments.
     /// It panics if the contents cannot be parsed from JSON to the GenesisConfig structure.
     pub fn from_json(value: &str) -> Self {
-        serde_json::from_str(value).expect("Failed to deserialize the genesis config.")
+        let json_str_without_comments: String =
+            near_config_utils::strip_comments_from_json_str(&value.to_string())
+                .expect("Failed to strip comments from genesis config.");
+        serde_json::from_str(&json_str_without_comments)
+            .expect("Failed to deserialize the genesis config.")
     }
 
     /// Reads GenesisConfig from a JSON file.
-    ///
+    /// The file can be a JSON with comments.
     /// It panics if file cannot be open or read, or the contents cannot be parsed from JSON to the
     /// GenesisConfig structure.
     pub fn from_file<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
-        let file = File::open(path).with_context(|| "Could not open genesis config file.")?;
-        let reader = BufReader::new(file);
-        let genesis_config: GenesisConfig = serde_json::from_reader(reader)
-            .with_context(|| "Failed to deserialize the genesis records.")?;
+        let mut file = File::open(path).with_context(|| "Could not open genesis config file.")?;
+        let mut json_str = String::new();
+        file.read_to_string(&mut json_str)?;
+        let json_str_without_comments: String =
+            near_config_utils::strip_comments_from_json_str(&json_str)?;
+        let genesis_config: GenesisConfig = serde_json::from_str(&json_str_without_comments)
+            .with_context(|| "Failed to deserialize the genesis config.")?;
         Ok(genesis_config)
     }
 
@@ -309,12 +316,19 @@ impl GenesisRecords {
     }
 
     /// Reads GenesisRecords from a JSON file.
-    ///
+    /// The file can be a JSON with comments.
     /// It panics if file cannot be open or read, or the contents cannot be parsed from JSON to the
     /// GenesisConfig structure.
     pub fn from_file<P: AsRef<Path>>(path: P) -> Self {
-        let reader = BufReader::new(File::open(path).expect("Could not open genesis config file."));
-        serde_json::from_reader(reader).expect("Failed to deserialize the genesis records.")
+        let mut file = File::open(path).expect("Failed to open genesis config file.");
+        let mut json_str = String::new();
+        file.read_to_string(&mut json_str)
+            .expect("Failed to read the genesis config file to string. ");
+        let json_str_without_comments: String =
+            near_config_utils::strip_comments_from_json_str(&json_str)
+                .expect("Failed to strip comments from Genesis config file.");
+        serde_json::from_str(&json_str_without_comments)
+            .expect("Failed to deserialize the genesis records.")
     }
 
     /// Writes GenesisRecords to the file.
@@ -394,11 +408,13 @@ impl<'de, F: FnMut(StateRecord)> DeserializeSeed<'de> for RecordsProcessor<&'_ m
     }
 }
 
+/// The file can be a JSON with comments
 pub fn stream_records_from_file(
     reader: impl Read,
     mut callback: impl FnMut(StateRecord),
 ) -> serde_json::Result<()> {
-    let mut deserializer = serde_json::Deserializer::from_reader(reader);
+    let reader_without_comments = near_config_utils::strip_comments_from_json_reader(reader);
+    let mut deserializer = serde_json::Deserializer::from_reader(reader_without_comments);
     let records_processor = RecordsProcessor { sink: &mut callback };
     deserializer.deserialize_any(records_processor)
 }
@@ -449,10 +465,16 @@ impl Genesis {
     }
 
     /// Reads Genesis from a single file.
+    /// the file can be JSON with comments
     pub fn from_file<P: AsRef<Path>>(path: P, genesis_validation: GenesisValidationMode) -> Self {
-        let reader = BufReader::new(File::open(path).expect("Could not open genesis config file."));
-        let genesis: Genesis =
-            serde_json::from_reader(reader).expect("Failed to deserialize the genesis records.");
+        let mut file = File::open(path).expect("Could not open genesis config file.");
+        let mut json_str = String::new();
+        file.read_to_string(&mut json_str).expect("Failed to read genesis config file to string. ");
+        let json_str_without_comments: String =
+            near_config_utils::strip_comments_from_json_str(&json_str)
+                .expect("Failed to strip comments from Genesis config file.");
+        let genesis: Genesis = serde_json::from_str(&json_str_without_comments)
+            .expect("Failed to deserialize the genesis records.");
         // As serde skips the `records_file` field, we can assume that `Genesis` has `records` and
         // doesn't have `records_file`.
         Self::new_validated(genesis.config, genesis.records, genesis_validation)

core/crypto/Cargo.toml

@@ -28,6 +28,7 @@ serde_json.workspace = true
 stdx.workspace = true
 subtle.workspace = true
 thiserror.workspace = true
+near-config-utils = { path = "../../utils/config" }
 
 [dev-dependencies]
 hex-literal = "0.2"

core/crypto/src/key_file.rs

@@ -1,12 +1,10 @@
+use serde::{Deserialize, Serialize};
 use std::fs::File;
 use std::io;
-use std::io::Write;
+use std::io::{Read, Write};
 use std::path::Path;
 
-use serde::{Deserialize, Serialize};
-
 use crate::{PublicKey, SecretKey};
-
 use near_account_id::AccountId;
 
 #[derive(Serialize, Deserialize)]
@@ -39,8 +37,13 @@ impl KeyFile {
     }
 
     pub fn from_file(path: &Path) -> io::Result<Self> {
-        let content = std::fs::read_to_string(path)?;
-        Ok(serde_json::from_str(&content)?)
+        let mut file = File::open(path)?;
+        let mut json_config_str = String::new();
+        file.read_to_string(&mut json_config_str)?;
+        let json_str_without_comments: String =
+            near_config_utils::strip_comments_from_json_str(&json_config_str)?;
+
+        Ok(serde_json::from_str(&json_str_without_comments)?)
     }
 }
 

nearcore/Cargo.toml

@@ -56,6 +56,7 @@ near-store = { path = "../core/store" }
 near-telemetry = { path = "../chain/telemetry" }
 near-vm-runner = { path = "../runtime/near-vm-runner"}
 node-runtime = { path = "../runtime/runtime" }
+near-config-utils = { path = "../utils/config" }
 
 delay-detector = { path = "../tools/delay-detector" }
 

nearcore/src/config.rs

@@ -395,11 +395,12 @@ impl Default for Config {
 
 impl Config {
     pub fn from_file(path: &Path) -> anyhow::Result<Self> {
-        let contents = std::fs::read_to_string(path)
+        let json_str = std::fs::read_to_string(path)
             .with_context(|| format!("Failed to read config from {}", path.display()))?;
         let mut unrecognised_fields = Vec::new();
+        let json_str_without_comments = near_config_utils::strip_comments_from_json_str(&json_str)?;
         let config: Config = serde_ignored::deserialize(
-            &mut serde_json::Deserializer::from_str(&contents),
+            &mut serde_json::Deserializer::from_str(&json_str_without_comments),
             |field| {
                 let field = field.to_string();
                 // TODO(mina86): Remove this deprecation notice some time by the
@@ -1297,11 +1298,15 @@ struct NodeKeyFile {
 }
 
 impl NodeKeyFile {
+    // the file can be JSON with comments
     fn from_file(path: &Path) -> std::io::Result<Self> {
         let mut file = File::open(path)?;
-        let mut content = String::new();
-        file.read_to_string(&mut content)?;
-        Ok(serde_json::from_str(&content)?)
+        let mut json_str = String::new();
+        file.read_to_string(&mut json_str)?;
+
+        let json_str_without_comments = near_config_utils::strip_comments_from_json_str(&json_str)?;
+
+        Ok(serde_json::from_str(&json_str_without_comments)?)
     }
 }
 

nearcore/src/dyn_config.rs

@@ -52,19 +52,29 @@ fn read_log_config(home_dir: &Path) -> Result<Option<LogConfig>, UpdateableConfi
     read_json_config::<LogConfig>(&home_dir.join(LOG_CONFIG_FILENAME))
 }
 
+// the file can be JSON with comments
 fn read_json_config<T: std::fmt::Debug>(
     path: &Path,
 ) -> Result<Option<T>, UpdateableConfigLoaderError>
 where
     for<'a> T: Deserialize<'a>,
 {
     match std::fs::read_to_string(path) {
-        Ok(config_str) => match serde_json::from_str::<T>(&config_str) {
-            Ok(config) => {
-                tracing::info!(target: "neard", config=?config, "Changing the config {path:?}.");
-                return Ok(Some(config));
+        Ok(config_str) => match near_config_utils::strip_comments_from_json_str(&config_str) {
+            Ok(config_str_without_comments) => {
+                match serde_json::from_str::<T>(&config_str_without_comments) {
+                    Ok(config) => {
+                        tracing::info!(target: "neard", config=?config, "Changing the config {path:?}.");
+                        return Ok(Some(config));
+                    }
+                    Err(err) => {
+                        Err(UpdateableConfigLoaderError::Parse { file: path.to_path_buf(), err })
+                    }
+                }
+            }
+            Err(err) => {
+                Err(UpdateableConfigLoaderError::OpenAndRead { file: path.to_path_buf(), err })
             }
-            Err(err) => Err(UpdateableConfigLoaderError::Parse { file: path.to_path_buf(), err }),
         },
         Err(err) => match err.kind() {
             std::io::ErrorKind::NotFound => {

utils/config/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "near-config-utils"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+authors.workspace = true
+publish = true
+# Please update rust-toolchain.toml as well when changing version here:
+rust-version.workspace = true
+edition.workspace = true
+repository = "https://github.com/near/nearcore"
+description = "This is an internal crate to provide utils for reading config files"
+
+[dependencies]
+json_comments.workspace = true