Env changes and bugfixes (#104)

* added replace function

* added testing readme

* changed env to be more performant

* In this commit:
readme added more limitations and a few infos to types
Ability to share rust type between payloads by giving the schema the same name prop
Validation using json schemas at runtime
Env variable to turn on and off validation
Fixed streams never being generated currently

* made service_port a port

* made basic example use validation

* fixed naming being inconsistent between schema and rust types

* added comment for how to use test command

* fix wrong env names in handler

* simplified message names

* fmt

* added performance hint for validation

* brought policy into scope

---------

Co-authored-by: Jacob Große <jacobgrosse@Jacobs-Spectre.fritz.box>

README.md

@@ -54,11 +54,16 @@ just service-doc {project-id} # Alternatively, you can use 'cd output/{project-i
 
 Remember to replace `{project-id}` with the name of your generated microservice (`title` field from the provided spec).
 
+## Types
+Rust types will be generated in the models folder according to the given payload json schema definitions. Names will be generated according to channels etc, if you want to share a payload type between two messages, make sure to use the same "name" property in the payload. Warning: This will not check if the types of those payloads are actually the same, so make sure to use the same schema or better even, simply a ref to the schema with the name. By default, all defined properties are required and no additional properties are allowed, if you want to use optional types, please modify the generated types after generation or use oneOf/anyOf/allOf to represent optional types.
+
 ## Limitations
 
 - Only json payloads are currently supported for automatic deserialization
 - Only one server is currently supported and only nats protocol is supported
 - Only one message is currently supported per channel, payloads can be choosen freely including anyOf/oneOf/allOf
+- The generated rust types are required by default, if you want to use optional types, please modify the generated types after generation or use oneOf/anyOf/allOf to represent optional types
+- references in the specification are only suppported inside the same file, external references are not supported
 
 ## Contribute
 

example/specs/basic.yaml

@@ -14,23 +14,35 @@ channels:
       summary: User signup notification
       message:
         payload:
+          name: userSignUpPayload
           type: object
           properties:
-            userSingnedUp:
+            userName:
               type: string
+              minLength: 3
+            password:
+              type: string
+              minLength: 8
+            age: 
+              type: number
+              minimum: 18
     publish:
       operationId: userSignedUp
       summary: send welcome email to user
       message:
-         payload:
+        payload:
+          name: userSignUpPayload
           type: object
           properties:
-            username:
+            userName:
               type: string
+              minLength: 3
             password:
               type: string
+              minLength: 8
             age: 
               type: number
+              minimum: 18
   user/buy:
     subscribe:
       summary: User bought something

src/asyncapi_model/message.rs

@@ -169,6 +169,9 @@ use super::{
 #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct Message {
+    #[serde(skip_serializing_if = "Option::is_none", rename = "schema")]
+    pub payload_schema: Option<String>,
+
     /// Schema definition of the application headers.
     /// Schema MUST be of type "object". It **MUST NOT** define the protocol headers.
     #[serde(skip_serializing_if = "Option::is_none")]

src/generator/common.rs

@@ -8,7 +8,7 @@ use std::io::{self, Error};
 use std::path::Path;
 use walkdir::WalkDir;
 
-use super::generate_models_folder;
+use super::{generate_models_folder, generate_schemas_folder};
 
 /// runs cargo command with options
 /// Example: ` cargo_command!("init","--bin","path"); `
@@ -167,6 +167,10 @@ fn separate_files(
         generate_models_folder(template_str, context, output_dir);
         return Ok(true);
     }
+    if template_path.contains("$$schemas$$") {
+        generate_schemas_folder(template_str, context, output_dir);
+        return Ok(true);
+    }
     Ok(false)
 }
 

src/generator/mod.rs

@@ -6,4 +6,7 @@ pub use common::{
 mod model;
 pub use model::generate_models_folder;
 
+mod schemas;
+pub use schemas::generate_schemas_folder;
+
 mod template_functions;

src/generator/schemas.rs

@@ -0,0 +1,33 @@
+use super::common::render_write_template;
+use crate::parser::common::validate_identifier_string;
+use crate::template_context::TemplateContext;
+use std::path::Path;
+
+pub fn generate_schemas_folder(
+    template: impl Into<String> + Clone,
+    async_config: &TemplateContext,
+    output_dir: &Path,
+) {
+    async_config
+        .publish_channels
+        .iter()
+        .chain(async_config.subscribe_channels.iter())
+        .for_each(|(_key, operation)| {
+            let message = operation.messages.first();
+            if message.is_none() {
+                return;
+            }
+            let message = message.unwrap();
+            if message.payload_schema.is_none() {
+                return;
+            }
+            render_write_template(
+                template.clone(),
+                &message.clone(),
+                &output_dir.join(format!(
+                    "{}_payload_schema.json",
+                    validate_identifier_string(&message.unique_id, false)
+                )),
+            );
+        });
+}

src/main.rs

@@ -51,7 +51,6 @@ fn main() {
         }
     };
     check_for_overwrite(output_path, title);
-
     // make output a compilable project in output_path
     cargo_command!("init", "--bin", output_path);
 

src/parser/asyncapi_model_parser/preprocessor.rs

@@ -6,8 +6,13 @@ use crate::parser::common::{self, validate_identifier_string};
 pub fn preprocess_schema(spec: serde_json::Value) -> serde_json::Value {
     let with_message_names = fill_message_and_payload_names(spec.clone(), spec, false, false, None);
     let resolved_refs = resolve_refs(with_message_names.clone(), with_message_names);
+    let with_payload_schemas = duplicate_payload_schemas(resolved_refs.clone(), resolved_refs);
     let mut seen = HashSet::new();
-    sanitize_operation_ids_and_check_duplicate(resolved_refs.clone(), resolved_refs, &mut seen)
+    sanitize_operation_ids_and_check_duplicate(
+        with_payload_schemas.clone(),
+        with_payload_schemas,
+        &mut seen,
+    )
 }
 
 pub fn sanitize_operation_ids_and_check_duplicate(
@@ -93,6 +98,41 @@ pub fn resolve_refs(json: serde_json::Value, root_json: serde_json::Value) -> se
     }
 }
 
+pub fn duplicate_payload_schemas(
+    json: serde_json::Value,
+    root_json: serde_json::Value,
+) -> serde_json::Value {
+    match json {
+        serde_json::Value::Object(map) => {
+            let mut new_map = serde_json::Map::new();
+            for (key, value) in map {
+                if key == "payload" {
+                    if let serde_json::Value::Object(schema) = value {
+                        // insert schema as json string
+                        new_map.insert(
+                            "schema".into(),
+                            serde_json::Value::String(serde_json::to_string(&schema).unwrap()),
+                        );
+                        new_map.insert("payload".into(), serde_json::Value::Object(schema.clone()));
+                    }
+                } else {
+                    let new_value = duplicate_payload_schemas(value, root_json.clone());
+                    new_map.insert(key, new_value);
+                }
+            }
+            serde_json::Value::Object(new_map)
+        }
+        serde_json::Value::Array(array) => {
+            let new_array = array
+                .into_iter()
+                .map(|value| duplicate_payload_schemas(value, root_json.clone()))
+                .collect();
+            serde_json::Value::Array(new_array)
+        }
+        _ => json,
+    }
+}
+
 pub fn fill_message_and_payload_names(
     json: serde_json::Value,
     root_json: serde_json::Value,

src/parser/json_schema_parser/array_schema.rs

@@ -35,6 +35,7 @@ pub fn parse_array_schema(
     // ))
     Ok(RustSchemaRepresentation {
         unique_id: identifyer,
+        original_key: property_name.to_string(),
         struct_reference: format!("Vec<{}>", item_type),
         model_definition: "".to_string(),
         related_models: vec![],

src/parser/json_schema_parser/enum_schema.rs

@@ -33,6 +33,7 @@ pub fn parse_enum_schema(
     string_builder.push_str("}\n");
     Ok(RustSchemaRepresentation {
         unique_id: identifyer.clone(),
+        original_key: property_name.to_string(),
         struct_reference: identifyer,
         model_definition: string_builder,
         model_type: "enum".to_string(),

src/parser/json_schema_parser/object_schema.rs

@@ -10,6 +10,7 @@ pub fn parse_object_schema(
     property_name: &str,
 ) -> Result<RustSchemaRepresentation, SchemaParserError> {
     let identifyer = validate_identifier_string(property_name, true);
+
     let before_string: String = format!(
         "#[derive(Clone, Debug, Deserialize, Serialize)]\npub struct {} {{\n",
         identifyer
@@ -39,13 +40,20 @@ pub fn parse_object_schema(
 
     let property_string = unwrapped_property_types
         .iter()
-        .map(|x| format!("pub {}: {}", x.unique_id, x.struct_reference))
+        .map(|x| {
+            let rename = match x.original_key == x.unique_id.as_str() {
+                true => "".to_string(),
+                false => format!("#[serde(rename = \"{}\")]\n", x.original_key),
+            };
+            format!("{}pub {}: {}", rename, x.unique_id, x.struct_reference)
+        })
         .collect::<Vec<String>>()
         .join(",\n");
     let full_struct = before_string + &property_string + &after_string;
 
     let representation: RustSchemaRepresentation = RustSchemaRepresentation {
         unique_id: identifyer.clone(),
+        original_key: property_name.to_string(),
         struct_reference: identifyer,
         model_definition: full_struct,
         related_models: unwrapped_property_types,

src/parser/json_schema_parser/primitive_schema.rs

@@ -21,6 +21,7 @@ pub fn primitive_type_to_string(
 
     Ok(RustSchemaRepresentation {
         unique_id: validate_identifier_string(&variable_name, false),
+        original_key: property_name.to_string(),
         struct_reference: format_to_rust_type(&schema_type),
         model_definition: "".to_string(),
         related_models: vec![],

src/parser/json_schema_parser/types/rust_schema_representation.rs

@@ -4,6 +4,7 @@ use serde::Serialize;
 pub struct RustSchemaRepresentation {
     // the unique identifier (e.g. UserSignupMessage)
     pub unique_id: String,
+    pub original_key: String,
     // used to reference the model (e.g. UserSignupMessage, but for primitive schemas simply the primitive type e.g. String/f64)
     pub struct_reference: String,
     // model definition (e.g. pub struct UserSignupMessage { ... } or pub enum UserSignupMessage { ... }, is empty for primitive types)

src/template_context/types.rs

@@ -44,16 +44,9 @@ pub struct SimplifiedMessage {
     pub unique_id: String,
     pub original_message: Message,
     pub payload: Option<RustSchemaRepresentation>,
+    pub payload_schema: Option<String>,
 }
-// #[derive(Serialize, Debug, Clone)]
 
-// pub struct SimplifiedSchema {
-//     pub unique_id: String,
-//     pub original_schema: Schema,
-//     pub struct_definition: String,
-//     pub struct_names: Vec<String>,
-//     // pub multiple_payload_enum: Option<MultiStructEnum>,
-// }
 /// FIXME: these are just a quick workaround until gtmpl::Value supports `From<impl Serialize> for gtmpl::Value`
 impl<'a> From<&TemplateContext<'a>> for gtmpl::Value {
     fn from(value: &TemplateContext<'a>) -> Self {
@@ -68,6 +61,13 @@ impl From<&SimplifiedOperation> for gtmpl::Value {
     }
 }
 
+impl From<&SimplifiedMessage> for gtmpl::Value {
+    fn from(value: &SimplifiedMessage) -> Self {
+        let json_value: serde_json::Value = serde_json::to_value(value).unwrap();
+        serde_value_to_gtmpl_value(&json_value)
+    }
+}
+
 /// converts any serde serializable value to a gtmpl value
 /// WARNING: clones objects, so not exactly zero cost abstraction 🤷‍♂️
 fn serde_value_to_gtmpl_value(value: &serde_json::Value) -> gtmpl::Value {

src/template_context/utilities.rs

@@ -50,21 +50,13 @@ pub fn simplify_message(
         let payload = match &message.payload {
             Some(schema) => {
                 if let Payload::Schema(schema) = schema {
-                    unique_id = validate_identifier_string(
-                        format!(
-                            "{}{}Message",
-                            message.name.as_ref().unwrap_or(
-                                schema
-                                    .schema_data
-                                    .name
-                                    .as_ref()
-                                    .unwrap_or(&String::from(""))
-                            ),
-                            unique_parent_id
-                        )
-                        .as_str(),
-                        false,
-                    );
+                    let message_name = match &message.name {
+                        Some(name) => name.to_string(),
+                        None => {
+                            format!("{}Message", unique_parent_id)
+                        }
+                    };
+                    unique_id = validate_identifier_string(&message_name, false);
                     let simplified_schema = simplify_schema(schema, &unique_id);
                     Some(simplified_schema)
                 } else {
@@ -77,39 +69,17 @@ pub fn simplify_message(
             unique_id,
             original_message: message.clone(),
             payload,
+            payload_schema: message.payload_schema.clone(),
         }
     } else {
         panic!("Refs should be resolved by now");
     }
 }
 
 pub fn simplify_schema(schema: &Schema, unique_parent_id: &str) -> RustSchemaRepresentation {
-    parse_json_schema_to_rust_type(schema, unique_parent_id).unwrap()
-    // let rust_schema = parse_json_schema_to_rust_type(schema, unique_parent_id).unwrap();
-    // let mut schema_source = rust_schema.related_models.clone();
-    // schema_source.push(rust_schema.clone());
-    // let schemas = schema_source
-    //     .into_iter().map(|s| s.model_definition).collect::<Vec<String>>().join("\n");
-    // let struct_name =rust_schema.identifier.clone();
-    // TODO: this whole thing will need to be refactored, there's no way this will work in this form
-    // the idea is that we need to get the payload enum and its members out of the schema
-    // but we save it as string only... so the whole parsing function will need to be restructured and way more modular
-    // why you ask? we want to automatically generate match code for the payload, but currently it wont work without refactor
-
-    // let payload_enum_name = format!("{}PayloadEnum", unique_parent_id);
-    // let mut multiple_payload_enum = None;
-    // if schemas.contains_key(&payload_enum_name) {
-    //     multiple_payload_enum = Some(MultiStructEnum {
-    //         unique_id: payload_enum_name,
-    //         messages: vec![],
-    //         struct_definition: "".to_string(),
-    //     });
-    // }
-    // RustSchemaRepresentation {
-    //     unique_id: unique_parent_id.to_string(),
-    //     original_schema: schema.clone(),
-    //     struct_definition: schemas,
-    //     struct_names: vec![struct_name],
-    //     // multiple_payload_enum: None,
-    // }
+    let schema_name = match &schema.schema_data.name {
+        Some(name) => validate_identifier_string(name, false),
+        None => validate_identifier_string(unique_parent_id, false),
+    };
+    parse_json_schema_to_rust_type(schema, &schema_name).unwrap()
 }

templates/.env.go

@@ -1,10 +1,11 @@
 ################General Config################
 
-SERVICE_PORT = "http://localhost:8080"
+SERVICE_PORT = "8080"
 SERVER_URL = "{{ .server.url }}"
 LOG_LEVEL = "DEBUG"
 OPA_RULES= "path/to/admin/policy"
 TRACING_ENABLED = false
+SCHEMA_VALIDATION_ENABLED = true
 
 ################Channel wise Config################
 {{ range .subscribe_channels }}

templates/Cargo.toml.go

@@ -24,4 +24,5 @@ opa-wasm = { git = "https://github.com/matrix-org/rust-opa-wasm.git" }
 cargo_metadata = "0.15.4"
 warp = "0.3.5"
 lazy_static = "1.4"
+jsonschema = "0.17.0"