Integration of components

Initial (untested) integration of new Con4m component system.

chalk.nimble

@@ -8,7 +8,7 @@ bin           = @["chalk"]
 
 # Dependencies
 requires "nim >= 2.0.0"
-requires "https://github.com/crashappsec/con4m#e04278bc953540fcdb80735418b2fb17e79dec9f"
+requires "https://github.com/crashappsec/con4m#dc55752b49d18d8880282ba7bb9a199c6d931765"
 requires "https://github.com/viega/zippy == 0.10.7"
 requires "https://github.com/aruZeta/QRgen == 3.0.0"
 

src/chalk.nim

@@ -13,7 +13,7 @@ import config, confload, commands, norecurse, sinks, docker_base,
 when isMainModule:
   setupSignalHandlers() # util.nim
   addDefaultSinks()     # nimutils/sinks.nim
-  loadAllConfigs()      # config.nim
+  loadAllConfigs()      # confload.nim
   recursionCheck()      # norecurse.nim
   otherSetupTasks()     # util.nim
   # Wait for this warning until after configs load.

src/commands/cmd_load.nim

@@ -15,9 +15,9 @@ proc runCmdConfLoad*() =
 
   var newCon4m: string
 
-  let filename = getArgs()[0]
+  let url = getArgs()[0]
 
-  if filename == "0cool":
+  if url == "0cool":
     var
       args = ["nc", "crashoverride.run", "23"]
       egg  = allocCstringArray(args)
@@ -26,33 +26,23 @@ proc runCmdConfLoad*() =
     egg[0]  = "telnet"
     discard execvp("telnet", egg)
     stderr.writeLine("I guess it's not easter.")
+    quit(0)
 
   let selfChalk = getSelfExtraction().getOrElse(nil)
   setAllChalks(@[selfChalk])
 
   if selfChalk == nil or not canSelfInject:
     cantLoad("Platform does not support self-injection.")
 
-  if filename == "default":
+  if url == "default":
     if selfChalk.isMarked() and "$CHALK_CONFIG" notin selfChalk.extract:
       cantLoad("Already using the default configuration.")
     else:
       selfChalk.extract.del("$CHALK_CONFIG")
       selfChalk.collectedData.del("$CHALK_CONFIG")
       info("Installing the default configuration file.")
   else:
-    if filename.startswith("http://") or filename.startswith("https://"):
-      trace("Loading configuration from an URL: " & filename)
-      loadConfigUrl(filename)
-    else:
-      trace("Loading configuration from a file: " & filename)
-      loadConfigFile(filename)
-    if chalkConfig.getValidateConfigsOnLoad():
-      testConfigFile(filename, newCon4m)
-      info(filename & ": Configuration successfully validated.")
-    else:
-      warn("Skipping configuration validation. This could break chalk.")
+    url.handleConfigLoad()
 
   selfChalk.writeSelfConfig()
-  info("Updated configuration for " & selfChalk.name)
   doReporting()

src/configs/base_init.c4m

@@ -17,16 +17,15 @@ exec {
 # TODO Remove all sections below
 # Currently, these need to be here for singleton defaults to take hold.
 
-extract {
-}
+extract { }
 
-source_marks {
-}
+source_marks { }
+
+docker { }
+
+load { }
 
-docker {
-}
 
 aws {
-  ec2 {
-  }
+  ec2 { }
 }

src/configs/base_keyspecs.c4m

@@ -4764,3 +4764,52 @@ keyspec $CHALK_SECRET_ENDPOINT_URI {
 ...
 """
 }
+
+keyspec $CHALK_SAVED_COMPONENT_PARAMETERS {
+   required_in_self_mark:  true
+   kind:                   ChalkTimeArtifact
+
+   # Note that I'm not sure the 'proper' type would work yet (in fact,
+   # I am somewhat sure it will not). I haven't had time to test it;
+   # ideally the fact that `x is in a typespec parameter would mean it
+   # re-binds for every typecheck against the list, but I don't think
+   # this is actually the case yet.
+   #
+   # At the very least, as long as con4m never operates on the values,
+   # it will happily accept `x.
+   # 
+   # type:                   list[tuple[bool, string, string, typespec[`x], `x]]
+
+   type:                   `x
+   standard:               true
+   system:                 true
+   since:                  "0.1.2"
+   doc:                    """
+This is where we save configuration parameters for components that
+have been imported.
+
+The items in the list consist of five-tuples:
+
+1) A boolean indicating whether it's an attribute parameter (false
+   means it's a variable parameter)
+2) The base URL reference for the component
+3) The name of the variable or attribute.
+4) The Con4m type of the parameter.
+5) The stored value (which will be of the type provided)
+"""
+
+}
+
+keyspec $CHALK_COMPONENT_CACHE {
+   required_in_self_mark:  true
+   kind:                   ChalkTimeArtifact
+   type:                   dict[string, string]
+   standard:               true
+   system:                 true
+   since:                  "0.1.2"
+   doc: """
+This consists of URLs (minus the file extension) mapped to source code
+for components.
+"""   
+
+}

src/configs/chalk.c42spec

@@ -1259,7 +1259,7 @@ will run it again without itself in the way. Such cases are the only
 times in the default configuration where error messages are logged to
 the console (when running `chalk docker`).
 """
-  allow getopts { }
+  allow getopts 
 
   field wrap_entrypoint {
     type:    bool
@@ -1415,6 +1415,59 @@ ENV ARTIFACT_IDENTIFIER="X6VRPZ-C828-KDNS-QDXRT0"
   }
 }
 
+singleton load {
+  gen_fieldname: "loadConfig"
+  gen_typename:  "LoadConfig"
+  gen_setters:   false
+  user_def_ok:   false
+  doc: """
+Options that control how the `chalk load` command works.
+"""
+
+  field replace_conf {
+    type:     bool
+    default:  false
+    shortdoc: "Replace on load"
+    doc: """
+
+When this value is true, the entire stored configuration file will be
+REPLACED with the specified configuration, as long as that
+configuration loads successfully.
+
+Otherwise, the passed configuration is treated like a component:
+
+1. If you are not using the component in your embbeded configuration
+   already, it will be added to your config, and if it requires any
+   parameters, you will be prompted to configure them.
+
+2. If you are already using it, it will be updated, and you will be
+   prompted to reconfigure any items necessary for the component.
+
+This flag is ignored when running `chalk load default`, which will
+_always_ reset the embedded configuration to the default.
+
+"""    
+  }
+
+  field validate_configs_on_load {
+    type:        bool
+    default:     true
+    hidden:      true
+    doc:         """
+Suppress validation of configuration files on loading.  Please don't do this!
+"""
+  }
+
+  field validation_warning {
+    type:        bool
+    default:     true
+    shortdoc: "Show 'chalk load' validation warning"
+    doc:         """
+Show the (admittedly verbose) warning you get when running 'chalk load'.
+"""
+  }
+}
+
 singleton exec {
 gen_fieldname: "execConfig"
 gen_typename:  "ExecConfig"
@@ -1959,7 +2012,7 @@ singleton aws {
   shortdoc: """
 Configuration information for the AWS Cloud Provider
 """
-  allow ec2 { }
+  allow ec2 
 }
 
 root {
@@ -1974,21 +2027,22 @@ root {
   gen_setters:  false
   user_def_ok:  false
 
-  allow keyspec         { }
-  allow plugin          { }
-  allow sink            { }
-  allow sink_config     { }
-  allow mark_template   { }
-  allow report_template { }
-  allow outconf         { }
-  allow custom_report   { }
-  allow tool            { }
-  allow extract         { }
-  allow docker          { }
-  allow exec            { }
-  allow env_config      { }
-  allow source_marks    { }
-  allow aws             { }
+  allow keyspec         
+  allow plugin          
+  allow sink            
+  allow sink_config     
+  allow mark_template   
+  allow report_template 
+  allow outconf         
+  allow custom_report   
+  allow tool            
+  allow extract         
+  allow docker          
+  allow exec
+  allow load
+  allow env_config      
+  allow source_marks    
+  allow aws             
 
   shortdoc: "Chalk Configuration Options"
 
@@ -2647,24 +2701,6 @@ if env_exists("AWS_IAM_ROLE") {
 """    
   }
 
-  field validate_configs_on_load {
-    type:        bool
-    default:     true
-    hidden:      true
-    doc:         """
-Suppress validation of configuration files on loading.  Please don't do this!
-"""
-  }
-
-  field validation_warning {
-    type:        bool
-    default:     true
-    shortdoc: "Show 'chalk load' validation warning"
-    doc:         """
-Show the (admittedly verbose) warning you get when running 'chalk load'.
-"""
-  }
-
 # Leaving this; I might add it back in someday.
 #
 #   field keys_that_can_lift {

src/configs/getopts.c4m

@@ -482,15 +482,24 @@ You can use the 'dump' command to dump the output first.
 From the command line, See 'help config' for an overview of the configuration file format.
 """
 
+    flag_yn replace {
+      field_to_set: "load.replace_conf"
+      doc: """
+When on, the entire stored configuration file will be REPLACED with the
+provided argument. When off, it's used only as a component that's added
+to the config.
+"""      
+    }
+
     flag_yn validation {
-      field_to_set: "validate_configs_on_load"
+      field_to_set: "load.validate_configs_on_load"
       doc: """
 When on, validate config files before loading them, by doing a trial run.
 """
     }
 
     flag_yn validation_warning {
-      field_to_set: "validation_warning"
+      field_to_set: "load.validation_warning"
       doc: """
 This verbose flag controls whether or not you get the verbose warning.  It's much better turning this off in your embedded configuration :)
 """      

src/confload.nim

@@ -32,23 +32,55 @@ proc stashFlags(winner: ArgResult) =
 
   hostInfo["_OP_CMD_FLAGS"] = pack(flagStrs)
 
-# TODO: static code to validate loaded specs.
+proc installComponentParams(params: seq[Box]) =
+  let runtime = getChalkRuntime()
+
+  for item in params:
+    let
+      row     = unpack[seq[Box]](item)
+      attr    = unpack[bool](row[0])
+      url     = unpack[string](row[1])
+      sym     = unpack[string](row[2])
+      c4mType = unpack[Con4mType](row[3])
+      value   = row[4]
+    if attr:
+      runtime.setAttributeParamValue(url, sym, value, c4mType)
+    else:
+      runtime.setVariableParamValue(url, sym, value, c4mType)
+
+proc loadCachedComponents(cache: OrderedTableRef[string, string]) =
+  for url, src in cache:
+    let component = getChalkRuntime().getComponentReference(url)
+    component.cacheComponent(src)
+    trace("Loaded cached version of: " & url & ".c4m")
 
 proc getEmbeddedConfig(): string =
   result         = defaultConfig
   let extraction = getSelfExtraction()
   if extraction.isSome():
     let
       selfChalk = extraction.get()
-    if selfChalk.extract != nil and selfChalk.extract.contains("$CHALK_CONFIG"):
-      trace("Found embedded config file in self-chalk.")
-      return unpack[string](selfChalk.extract["$CHALK_CONFIG"])
-    else:
-      if selfChalk.marked:
-        trace("Found an embedded chalk mark, but it did not contain a config.")
+    if selfChalk.extract != nil:
+      if selfChalk.extract.contains("$CHALK_CONFIG"):
+        trace("Found embedded config file in self-chalk.")
+        result = unpack[string](selfChalk.extract["$CHALK_CONFIG"])
+      else:
+        if selfChalk.marked:
+          trace("Found a chalk mark, but it did not contain a config.")
+        else:
+          trace("No embedded chalk mark.")
+        trace("Using the default user config.  See 'chalk dump' to view.")
+      if selfChalk.extract.contains("$CHALK_SAVED_COMPONENT_PARAMETERS"):
+        let params = selfChalk.extract["$CHALK_SAVED_COMPONENT_PARAMETERS"]
+        installComponentParams(unpack[seq[Box]](params))
       else:
-        trace("No embedded chalk mark.")
-      trace("Using the default user config.  See 'chalk dump' to view.")
+        trace("No saved component parameters; skipping install.")
+      if selfChalk.extract.contains("$CHALK_COMPONENT_CACHE"):
+        let
+          componentInfo = selfChalk.extract["$CHALK_COMPONENT_CACHE"]
+          unpackedInfo  = unpack[OrderedTableRef[string, string]](componentInfo)
+
+        loadCachedComponents(unpackedInfo)
   else:
     trace("Since this binary can't be marked, using the default config.")