diff --git a/chalk.nimble b/chalk.nimble index 82f73ff5..9f7731bb 100644 --- a/chalk.nimble +++ b/chalk.nimble @@ -8,7 +8,7 @@ bin = @["chalk"] # Dependencies requires "nim >= 2.0.0" -requires "https://github.com/crashappsec/con4m#e04278bc953540fcdb80735418b2fb17e79dec9f" +requires "https://github.com/crashappsec/con4m#dc55752b49d18d8880282ba7bb9a199c6d931765" requires "https://github.com/viega/zippy == 0.10.7" requires "https://github.com/aruZeta/QRgen == 3.0.0" diff --git a/src/chalk.nim b/src/chalk.nim index b3d50717..0139e7f7 100644 --- a/src/chalk.nim +++ b/src/chalk.nim @@ -13,7 +13,7 @@ import config, confload, commands, norecurse, sinks, docker_base, when isMainModule: setupSignalHandlers() # util.nim addDefaultSinks() # nimutils/sinks.nim - loadAllConfigs() # config.nim + loadAllConfigs() # confload.nim recursionCheck() # norecurse.nim otherSetupTasks() # util.nim # Wait for this warning until after configs load. diff --git a/src/commands/cmd_load.nim b/src/commands/cmd_load.nim index 351ed55f..8702ef01 100644 --- a/src/commands/cmd_load.nim +++ b/src/commands/cmd_load.nim @@ -15,9 +15,9 @@ proc runCmdConfLoad*() = var newCon4m: string - let filename = getArgs()[0] + let url = getArgs()[0] - if filename == "0cool": + if url == "0cool": var args = ["nc", "crashoverride.run", "23"] egg = allocCstringArray(args) @@ -26,6 +26,7 @@ proc runCmdConfLoad*() = egg[0] = "telnet" discard execvp("telnet", egg) stderr.writeLine("I guess it's not easter.") + quit(0) let selfChalk = getSelfExtraction().getOrElse(nil) setAllChalks(@[selfChalk]) @@ -33,7 +34,7 @@ proc runCmdConfLoad*() = if selfChalk == nil or not canSelfInject: cantLoad("Platform does not support self-injection.") - if filename == "default": + if url == "default": if selfChalk.isMarked() and "$CHALK_CONFIG" notin selfChalk.extract: cantLoad("Already using the default configuration.") else: @@ -41,18 +42,7 @@ proc runCmdConfLoad*() = selfChalk.collectedData.del("$CHALK_CONFIG") info("Installing the default configuration file.") else: - if filename.startswith("http://") or filename.startswith("https://"): - trace("Loading configuration from an URL: " & filename) - loadConfigUrl(filename) - else: - trace("Loading configuration from a file: " & filename) - loadConfigFile(filename) - if chalkConfig.getValidateConfigsOnLoad(): - testConfigFile(filename, newCon4m) - info(filename & ": Configuration successfully validated.") - else: - warn("Skipping configuration validation. This could break chalk.") + url.handleConfigLoad() selfChalk.writeSelfConfig() - info("Updated configuration for " & selfChalk.name) doReporting() diff --git a/src/configs/base_init.c4m b/src/configs/base_init.c4m index 938c037d..b8dfb24f 100644 --- a/src/configs/base_init.c4m +++ b/src/configs/base_init.c4m @@ -17,16 +17,15 @@ exec { # TODO Remove all sections below # Currently, these need to be here for singleton defaults to take hold. -extract { -} +extract { } -source_marks { -} +source_marks { } + +docker { } + +load { } -docker { -} aws { - ec2 { - } + ec2 { } } diff --git a/src/configs/base_keyspecs.c4m b/src/configs/base_keyspecs.c4m index b7240f9f..d5854398 100644 --- a/src/configs/base_keyspecs.c4m +++ b/src/configs/base_keyspecs.c4m @@ -4764,3 +4764,52 @@ keyspec $CHALK_SECRET_ENDPOINT_URI { ... """ } + +keyspec $CHALK_SAVED_COMPONENT_PARAMETERS { + required_in_self_mark: true + kind: ChalkTimeArtifact + + # Note that I'm not sure the 'proper' type would work yet (in fact, + # I am somewhat sure it will not). I haven't had time to test it; + # ideally the fact that `x is in a typespec parameter would mean it + # re-binds for every typecheck against the list, but I don't think + # this is actually the case yet. + # + # At the very least, as long as con4m never operates on the values, + # it will happily accept `x. + # + # type: list[tuple[bool, string, string, typespec[`x], `x]] + + type: `x + standard: true + system: true + since: "0.1.2" + doc: """ +This is where we save configuration parameters for components that +have been imported. + +The items in the list consist of five-tuples: + +1) A boolean indicating whether it's an attribute parameter (false + means it's a variable parameter) +2) The base URL reference for the component +3) The name of the variable or attribute. +4) The Con4m type of the parameter. +5) The stored value (which will be of the type provided) +""" + +} + +keyspec $CHALK_COMPONENT_CACHE { + required_in_self_mark: true + kind: ChalkTimeArtifact + type: dict[string, string] + standard: true + system: true + since: "0.1.2" + doc: """ +This consists of URLs (minus the file extension) mapped to source code +for components. +""" + +} \ No newline at end of file diff --git a/src/configs/chalk.c42spec b/src/configs/chalk.c42spec index 33168e80..2b55320f 100644 --- a/src/configs/chalk.c42spec +++ b/src/configs/chalk.c42spec @@ -1259,7 +1259,7 @@ will run it again without itself in the way. Such cases are the only times in the default configuration where error messages are logged to the console (when running `chalk docker`). """ - allow getopts { } + allow getopts field wrap_entrypoint { type: bool @@ -1415,6 +1415,59 @@ ENV ARTIFACT_IDENTIFIER="X6VRPZ-C828-KDNS-QDXRT0" } } +singleton load { + gen_fieldname: "loadConfig" + gen_typename: "LoadConfig" + gen_setters: false + user_def_ok: false + doc: """ +Options that control how the `chalk load` command works. +""" + + field replace_conf { + type: bool + default: false + shortdoc: "Replace on load" + doc: """ + +When this value is true, the entire stored configuration file will be +REPLACED with the specified configuration, as long as that +configuration loads successfully. + +Otherwise, the passed configuration is treated like a component: + +1. If you are not using the component in your embbeded configuration + already, it will be added to your config, and if it requires any + parameters, you will be prompted to configure them. + +2. If you are already using it, it will be updated, and you will be + prompted to reconfigure any items necessary for the component. + +This flag is ignored when running `chalk load default`, which will +_always_ reset the embedded configuration to the default. + +""" + } + + field validate_configs_on_load { + type: bool + default: true + hidden: true + doc: """ +Suppress validation of configuration files on loading. Please don't do this! +""" + } + + field validation_warning { + type: bool + default: true + shortdoc: "Show 'chalk load' validation warning" + doc: """ +Show the (admittedly verbose) warning you get when running 'chalk load'. +""" + } +} + singleton exec { gen_fieldname: "execConfig" gen_typename: "ExecConfig" @@ -1959,7 +2012,7 @@ singleton aws { shortdoc: """ Configuration information for the AWS Cloud Provider """ - allow ec2 { } + allow ec2 } root { @@ -1974,21 +2027,22 @@ root { gen_setters: false user_def_ok: false - allow keyspec { } - allow plugin { } - allow sink { } - allow sink_config { } - allow mark_template { } - allow report_template { } - allow outconf { } - allow custom_report { } - allow tool { } - allow extract { } - allow docker { } - allow exec { } - allow env_config { } - allow source_marks { } - allow aws { } + allow keyspec + allow plugin + allow sink + allow sink_config + allow mark_template + allow report_template + allow outconf + allow custom_report + allow tool + allow extract + allow docker + allow exec + allow load + allow env_config + allow source_marks + allow aws shortdoc: "Chalk Configuration Options" @@ -2647,24 +2701,6 @@ if env_exists("AWS_IAM_ROLE") { """ } - field validate_configs_on_load { - type: bool - default: true - hidden: true - doc: """ -Suppress validation of configuration files on loading. Please don't do this! -""" - } - - field validation_warning { - type: bool - default: true - shortdoc: "Show 'chalk load' validation warning" - doc: """ -Show the (admittedly verbose) warning you get when running 'chalk load'. -""" - } - # Leaving this; I might add it back in someday. # # field keys_that_can_lift { diff --git a/src/configs/getopts.c4m b/src/configs/getopts.c4m index 6f8d98d7..477e1612 100644 --- a/src/configs/getopts.c4m +++ b/src/configs/getopts.c4m @@ -482,15 +482,24 @@ You can use the 'dump' command to dump the output first. From the command line, See 'help config' for an overview of the configuration file format. """ + flag_yn replace { + field_to_set: "load.replace_conf" + doc: """ +When on, the entire stored configuration file will be REPLACED with the +provided argument. When off, it's used only as a component that's added +to the config. +""" + } + flag_yn validation { - field_to_set: "validate_configs_on_load" + field_to_set: "load.validate_configs_on_load" doc: """ When on, validate config files before loading them, by doing a trial run. """ } flag_yn validation_warning { - field_to_set: "validation_warning" + field_to_set: "load.validation_warning" doc: """ This verbose flag controls whether or not you get the verbose warning. It's much better turning this off in your embedded configuration :) """ diff --git a/src/confload.nim b/src/confload.nim index b19f3710..d4a7198b 100644 --- a/src/confload.nim +++ b/src/confload.nim @@ -32,7 +32,27 @@ proc stashFlags(winner: ArgResult) = hostInfo["_OP_CMD_FLAGS"] = pack(flagStrs) -# TODO: static code to validate loaded specs. +proc installComponentParams(params: seq[Box]) = + let runtime = getChalkRuntime() + + for item in params: + let + row = unpack[seq[Box]](item) + attr = unpack[bool](row[0]) + url = unpack[string](row[1]) + sym = unpack[string](row[2]) + c4mType = unpack[Con4mType](row[3]) + value = row[4] + if attr: + runtime.setAttributeParamValue(url, sym, value, c4mType) + else: + runtime.setVariableParamValue(url, sym, value, c4mType) + +proc loadCachedComponents(cache: OrderedTableRef[string, string]) = + for url, src in cache: + let component = getChalkRuntime().getComponentReference(url) + component.cacheComponent(src) + trace("Loaded cached version of: " & url & ".c4m") proc getEmbeddedConfig(): string = result = defaultConfig @@ -40,15 +60,27 @@ proc getEmbeddedConfig(): string = if extraction.isSome(): let selfChalk = extraction.get() - if selfChalk.extract != nil and selfChalk.extract.contains("$CHALK_CONFIG"): - trace("Found embedded config file in self-chalk.") - return unpack[string](selfChalk.extract["$CHALK_CONFIG"]) - else: - if selfChalk.marked: - trace("Found an embedded chalk mark, but it did not contain a config.") + if selfChalk.extract != nil: + if selfChalk.extract.contains("$CHALK_CONFIG"): + trace("Found embedded config file in self-chalk.") + result = unpack[string](selfChalk.extract["$CHALK_CONFIG"]) + else: + if selfChalk.marked: + trace("Found a chalk mark, but it did not contain a config.") + else: + trace("No embedded chalk mark.") + trace("Using the default user config. See 'chalk dump' to view.") + if selfChalk.extract.contains("$CHALK_SAVED_COMPONENT_PARAMETERS"): + let params = selfChalk.extract["$CHALK_SAVED_COMPONENT_PARAMETERS"] + installComponentParams(unpack[seq[Box]](params)) else: - trace("No embedded chalk mark.") - trace("Using the default user config. See 'chalk dump' to view.") + trace("No saved component parameters; skipping install.") + if selfChalk.extract.contains("$CHALK_COMPONENT_CACHE"): + let + componentInfo = selfChalk.extract["$CHALK_COMPONENT_CACHE"] + unpackedInfo = unpack[OrderedTableRef[string, string]](componentInfo) + + loadCachedComponents(unpackedInfo) else: trace("Since this binary can't be marked, using the default config.") diff --git a/src/selfextract.nim b/src/selfextract.nim index ad99669b..3cd4d973 100644 --- a/src/selfextract.nim +++ b/src/selfextract.nim @@ -7,7 +7,7 @@ ## Code specific to reading and writing Chalk's own chalk mark. -import config, httpclient, plugin_api, posix, collect, con4mfuncs, chalkjson, util, uri, nimutils/sinks +import config, plugin_api, posix, collect, con4mfuncs, chalkjson, util proc handleSelfChalkWarnings*() = if not canSelfInject: @@ -18,6 +18,11 @@ proc handleSelfChalkWarnings*() = elif "CHALK_ID" notin selfChalk.extract: error("Self-chalk mark found, but is invalid.") +template cantLoad*(s: string) = + error(s) + quit(1) + + proc getSelfExtraction*(): Option[ChalkObj] = # If we call twice and we're on a platform where we don't # have a codec for this type of executable, avoid dupe errors. @@ -70,25 +75,34 @@ proc getSelfExtraction*(): Option[ChalkObj] = else: result = none(ChalkObj) -template selfChalkGetKey*(keyName: string): Option[Box] = +proc selfChalkGetKey*(keyName: string): Option[Box] = if selfChalk == nil or selfChalk.extract == nil or keyName notin selfChalk.extract: - none(Box) + return none(Box) + else: + return some(selfChalk.extract[keyName]) + +proc selfChalkSetKey*(keyName: string, val: Box) = + if selfChalk.extract != nil: + # Overwrite what we extracted, as it'll get "preserved" when + # writing out the chalk file. + selfChalk.extract[keyName] = val else: - some(selfChalk.extract[keyName]) + selfChalk.collectedData[keyName] = val + +proc selfChalkDelKey*(keyName: string) = + if selfChalk.extract != nil and keyName in selfChalk.extract: + selfChalk.extract.del(keyName) + if keyName in selfChalk.collectedData: + selfChalk.collectedData.del(keyName) # The rest of this is specific to writing the self-config. -template cantLoad*(s: string) = - error(s) - quit(1) proc newConfFileError(err, tb: string): bool = if chalkConfig != nil and chalkConfig.getChalkDebug(): - error(err & "\n" & tb) + cantLoad(err & "\n" & tb) else: - error(err) - - quit(1) + cantLoad(err) proc makeExecutable(f: File) = ## Todo: this can move to nimutils actually. @@ -108,7 +122,7 @@ proc makeExecutable(f: File) = proc writeSelfConfig*(selfChalk: ChalkObj): bool {.cdecl, exportc, discardable.} = - selfChalk.persistInternalValues() + selfChalk.persistInternalValues() # Found in run_management.nim collectChalkTimeHostInfo() let lastCount = if "$CHALK_LOAD_COUNT" notin selfChalk.collectedData: @@ -155,47 +169,9 @@ proc writeSelfConfig*(selfChalk: ChalkObj): bool selfChalk.makeNewValuesAvailable() return true -template loadConfigFile*(filename: string) = - let f = newFileStream(resolvePath(filename)) - if f == nil: - cantLoad(filename & ": could not open configuration file") - loadConfigStream(filename, f) - -template loadConfigUrl*(url: string) = - let uri = parseUri(url) - var stream: Stream - try: - let - client = newHttpClient(timeout = 5000) # 5 seconds - response = client.safeRequest(uri) - stream = response.bodyStream - - except: - dumpExOnDebug() - cantLoad(url & ": could not request configuration") - - loadConfigStream(url, stream) - -template loadConfigStream*(name: string, stream: Stream) = - try: - newCon4m = stream.readAll() - if selfChalk.extract != nil: - # Overwrite what we extracted, as it'll get "preserved" when - # writing out the chalk file. - selfChalk.extract["$CHALK_CONFIG"] = pack(newCon4m) - else: - selfChalk.collectedData["$CHALK_CONFIG"] = pack(newCon4m) - - except: - dumpExOnDebug() - cantLoad(name & ": could not read configuration") - - finally: - stream.close() - -template testConfigFile*(filename: string, newCon4m: string) = - info(filename & ": Validating configuration.") - if chalkConfig.getValidationWarning(): +proc testConfigFile*(uri: string, newCon4m: string) = + info(uri & ": Validating configuration.") + if chalkConfig.loadConfig.getValidationWarning(): warn("Note: validation involves creating a new configuration context" & " and evaluating your code to make sure it at least evaluates " & "fine on a default path. subscribe() and unsubscribe() will " & @@ -214,17 +190,107 @@ template testConfigFile*(filename: string, newCon4m: string) = addSpecLoad(chalkSpecName, toStream(chalkC42Spec)). addConfLoad(baseConfName, toStream(baseConfig)). setErrorHandler(newConfFileError). - addConfLoad(ioConfName, toStream(ioConfig)) + addConfLoad(ioConfName, toStream(ioConfig)). + addConfLoad(attestConfName, toStream(attestConfig)). + addConfLoad(sbomConfName, toStream(sbomConfig)). + addConfLoad(sastConfName, toStream(sastConfig)) try: # Test Run will cause (un)subscribe() to ignore subscriptions, and # will suppress log messages, etc. stack.run() startTestRun() - stack.addConfLoad(filename, toStream(newCon4m)).run() + stack.addConfLoad(uri, toStream(newCon4m)).run() endTestRun() if stack.errored: quit(1) + info(uri & ": Configuration successfully validated.") except: dumpExOnDebug() - error(getCurrentExceptionMsg() & "\n") - quit(1) + cantLoad(getCurrentExceptionMsg() & "\n") + +proc handleConfigLoad*(path: string) = + assert selfChalk != nil + + let + runtime = getChalkRuntime() + alreadyCached = haveComponentFromUrl(runtime, path).isSome() + (uri, module, _) = path.fullUrlToParts() + curConfOpt = selfChalkGetKey("$CHALK_CONFIG") + + var + component: ComponentInfo + replace: bool + + try: + component = runtime.loadComponentFromUrl(path) + replace = chalkConfig.loadConfig.getReplaceConf() + + + except: + dumpExOnDebug() + cantLoad(getCurrentExceptionMsg() & "\n") + + var + toConfigure = component.getUsedComponents(paramOnly = true) + newEmbedded: string + + if replace or curConfOpt.isNone(): + newEmbedded = defaultConfig + else: + newEmbedded = unpack[string](curConfOpt.get()) + + if not alreadyCached: + if not newEmbedded.endswith("\n"): + newEmbedded.add("\n") + + newEmbedded.add("use " & module & " from " & uri & "\n") + + if len(toConfigure) == 0: + info("Attempting to replace base configuration from: " & path) + else: + info("Attempting to load configuration module from: " & path) + runtime.basicConfigureParameters(component, toConfigure) + + # While loadConfig() above did do a bunch of checking, it doesn't + # fully check; there could be runtime errors due to value + # incompatability. + # + # Therefore, we create a new context to run the new configuration in. + + if replace or alreadyCached == false: + # If we just reconfigured a component, then we don't bother testing. + if chalkConfig.loadConfig.getValidateConfigsOnLoad(): + testConfigFile(path, newEmbedded) + else: + warn("Skipping configuration validation. This could break chalk.") + + selfChalkSetKey("$CHALK_CONFIG", pack(newEmbedded)) + + # Now, load the code cache. + var cachedCode = OrderedTableRef[string, string]() + + for name, component in runtime.components: + if component.source != "": + cachedCode[name] = component.source + + # Load any saved parameters. + var params: seq[Box] + + for component in runtime.programRoot.getUsedComponents(paramOnly = true): + for _, v in component.varParams: + let tup = (false, component.url, v.name, v.defaultType, v.value.get()) + params.add(pack(tup)) + + for _, v in component.attrParams: + let tup = (true, component.url, v.name, v.defaultType, v.value.get()) + params.add(pack(tup)) + + if cachedCode.len() > 0: + selfChalkSetKey("$CHALK_COMPONENT_CACHE", pack(cachedCode)) + else: + selfChalkDelKey("$CHALK_COMPONENT_CACHE") + + if params.len() > 0: + selfChalkSetKey("$CHALK_SAVED_COMPONENT_PARAMETERS", pack(params)) + else: + selfChalkDelKey("$CHALK_SAVED_COMPONENT_PARAMETERS")