Jtv/releaseitems (#67)

* Use explicit targets for module imports

* Removing the default config store; for current  semantics the CWD default seems better than assuming a URL.

* Did a lint pass; dealt with some unused variables, including turning off the 'hint' messages for stuff we want to keep around

* Add in some of the new configs.

* A tiny bit of polishing

* tweak

* Some beautification

* Initial untested param work done

* fixing missing quote from the wrap_entrypoints.c4m

* ionly downloading chalk binaries when doing docker build

* bumping con4m which fixes is_file path resolution

* Have the arch binary updating happen any time an update happens.

* Add dumping for cached config modules too

* Get rid of old configs

* Incorporate new versions of con4m / nimu with ability to get an external IP associated w/ the host portably. Used it in the reporting server to make it not be hacky, and added PUBLIC_IPV4_ADDR_WHEN_CHALKED and _OP_PUBLIC_IPV4_ADDR.

* Clean up a bit

* Update release notes; integrate Mark's change in tag-line

* Update autocomplete.

* Fixed circular imports

* Remove the chalk report from `chalk load`. It isn't needed and is too much for good intros to the product.

* bump nimu

* fix(docgen): add con4m defintions to fix docgen cmd regression

* docs(typos): fixing various typos found via vale.sh (#46)

* fix(docker): wrap_entrypoint honors command args

As chalk exec would parse the full command, if it had any arguments
chalk was not familiar with, chalk would not parse it correctly
and therefore will call command incorrectly.
By adding -- after the command name, chalk ignores rest of the args
and passes them as-is to the command.

For example this now works:

```
ENTRYPOINT ["ls", "-la"]
```

As chalk will end up calling it as:

```
/chalk exec --exec-command-name=ls -- -la
```

* Updated two of the howtos

* reverting dockerfile change

* Re-add in the ability to dump to a file

* fix(docs): fix doc/link regressions

* fix(docs): link fix

* fix(docs): more reformatting / regression fixing

* fix(docs): shockingly, more formatting fixes

* fix(docs): reverting the multiple regression fixes because there wasn't a regression

* Don't let docgen output static docs

* fix(docs) formatting

* fix(README): fix link to docker howto

* ansi fix

---------

Co-authored-by: Miroslav Shubernetskiy <miroslav@miki725.com>
Co-authored-by: Rich Smith <rich@crashoverride.com>

README.md

@@ -2,7 +2,7 @@
 
 [![tests](https://github.com/crashappsec/chalk/actions/workflows/tests.yml/badge.svg?branch=main&event=push)](https://github.com/crashappsec/chalk/actions/workflows/tests.yml?query=branch%3Amain)
 
-# Total visibility of your software engineering lifecycle.
+# Telemetry & observability for the software development lifecycle.
 
 ## About Chalk
 
@@ -32,6 +32,10 @@ Understanding which services run in containers can help you build a service map.
 
 Automatically create SBOMs for every build of every code repo, including auto-deploying and using built-in SBOM generation and collection tools. Send these SBOMs to a central location for further analysis, and to maintain a record across your environment. Follow this how-to on our docs site [here](https://crashoverride.com/docs/how-to-guides/how-to-create-and-maintain-an-sbom-registry).
 
+#### How-to deploy Chalk globally using Docker
+
+You can deploy Chalk by setting a global alias for Docker and having it call Chalk, so that every build that runs through your build server using Docker, will automatically be 'chalked'. It's a technique that can be combined with chalks ability to deploy tools and configure monitoring, to automatically add security controls and collect information for every application. Follow this how-to on our docs site [here](https://crashoverride.com/docs/how-to-guides/how-to-deploy-chalk-globally-using-docker)]
+
 All documentation for Chalk is available at https://crashoverride.com/docs and is also fully accessible though the command line interface.
 
 ## Getting started

chalk.nimble

@@ -1,4 +1,4 @@
-version       = "0.1.2"
+version       = "0.1.3"
 author        = "John Viega"
 description   = "Software artifact metadata to make it easy to tie " &
                 "deployments to source code and collect metadata."
@@ -8,7 +8,7 @@ bin           = @["chalk"]
 
 # Dependencies
 requires "nim >= 2.0.0"
-requires "https://github.com/crashappsec/con4m#da5a430616ef2740da603438b35436d184c36938"
+requires "https://github.com/crashappsec/con4m#816585633835c30e5aaf4f53fdfb8eb4dd91f97a"
 requires "https://github.com/viega/zippy == 0.10.7"
 requires "https://github.com/aruZeta/QRgen == 3.0.0"
 
@@ -70,7 +70,7 @@ before install:
 after build:
   when not defined(debug):
     exec "set -x && strip " & bin[0]
-  exec "set -x && ./" & bin[0] & " --no-use-external-config --skip-command-report load default"
+  exec "set -x && ./" & bin[0] & " --debug --no-use-external-config --skip-command-report load default"
 
 task debug, "Get a debug build":
   # additional flags are configured in config.nims

configs/app_inventory.c4m

@@ -0,0 +1,3 @@
+use reporting_server from "https://chalkdust.io"
+use wrap_entrypoints from "https://chalkdust.io"
+use impersonate_docker from "https://chalkdust.io"

configs/basic_compliance.c4m

@@ -0,0 +1,2 @@
+use reporting_server from "https://chalkdust.io"
+run_sbom_tools: true

configs/compliance_docker.c4m

@@ -0,0 +1,3 @@
+use impersonate_docker from "https://chalkdust.io"
+use basic_compliance from "https://chalkdust.io"
+

configs/compliance-docker.c4m → configs/embed_sboms.c4m

@@ -1,9 +1,7 @@
-run_sbom_tools: true
+# Embed sboms in chalk marks.
 
 # `chalk insert` uses the mark_default template.
 mark_template.mark_default.key.SBOM.use: true
 
 # `chalk docker build` uses the `minimal` template.
-mark_template.minimal.key.SBOM.use: true
-
-default_command: "docker"
+mark_template.minimal.key.SBOM.use: true

configs/new/reporting_server.c4m → configs/reporting_server.c4m

@@ -1,5 +1,3 @@
-# TODO: default
-
 func validate_url(url) {
   result := ""
 
@@ -9,20 +7,20 @@ func validate_url(url) {
 }
 
 func get_local_url() {
-  out, code := system("ifconfig -a | grep inet | grep broadcast | head -1 | " +
-                     "awk '{ print $2 }'")
-  if code != 0 {
-    return "https://localhost:7890"
-  }
-
-  return "https://" + out.strip() + ":7890"
+  return "https://" + external_ip() + ":7890"
 }
 
 parameter sink_config.output_to_http.uri {
   shortdoc:  "URL for reporting server"
   doc: """
 A config for sending reports to a custom implementation of the test
 reporting server.
+
+Run the server via:
+
+```
+docker run --rm - -w /db -v $HOME/.local/c0/:/db -p 8585:8585  ghcr.io/crashappsec/chalk-test-server
+```
 """
   validator: func validate_url(string) -> string
   default: func get_local_url() -> string

configs/use_heartbeats.c4m

@@ -0,0 +1,29 @@
+# We use a con4m duration field for the actual `heartbeat_rate` field,
+# But I don't want people to have to worry about that.
+
+func validate_heartbeat_freq(f: float) {
+  if (f <= 0.0) {
+    return "Value must be greater than 0"
+  } else {
+    return ""
+  }
+}
+
+parameter var heartbeat_minute_frequency {
+  default:  30.0
+  validator: func validate_heartbeat_freq(float) -> string
+  shortdoc: "Heartbeat Frequency (minutes)"
+  doc: """
+This value sets how many minutes to wait between heartbeats. Fractions
+of a minute are okay.
+"""  
+}
+var heartbeat_minute_frequency: float
+
+minutes  := int(heartbeat_minute_frequency)
+sec_as_f := (heartbeat_minute_frequency - float(minutes)) * 60.0
+sec      := int(sec_as_f)
+
+duration := Duration($(minutes) + " min " + $(sec) + " sec")
+exec.heartbeat: true
+exec.heartbeat_rate: duration

configs/wrap_entrypoints.c4m

@@ -0,0 +1,30 @@
+# Ensures entrypoint wrapping is enabled in the config"
+docker.wrap_entrypoint: true
+
+myarch := arch()
+binary_dir := "~/.local/chalk/bin/linux-" + myarch + "/"
+
+if osname() == "macosx" {
+  if not is_dir(binary_dir) {
+    mkdir(binary_dir)
+  }
+
+  linux_chalk_location := binary_dir + "chalk"
+  docker.arch_binary_locations = { "linux/" + myarch : linux_chalk_location }
+
+  if not is_file(linux_chalk_location) {
+    echo("MacOS requires downloading a Linux binary to wrap " +
+    "docker entry points.")
+    chalk_url_base := "https://crashoverride.com/dl/chalk/chalk-"
+    chalk_url      := chalk_url_base + version() + "-linux-" + myarch
+
+    info("Downloading chalk from: " + chalk_url)
+
+    bits := url_get(chalk_url)
+
+    info("Writing to: " + linux_chalk_location)
+    write_file(linux_chalk_location, bits)
+    config := run(program_path() + " dump")
+    write_file(binary_dir + "config.c4m", config)
+  }
+}

src/api.nim

@@ -12,19 +12,18 @@ template jwtSplitAndDecode(jwtString: string, doDecode: bool): string =
   let parts = split(jwtString, '.')
   if len(parts) != 3:
     raise newException(Exception, "Invalid JWT format")
-  let apiJwtPayload = parts[1]
 
   if doDecode:
     let decodedApiJwt = decode(apiJwtPayload)
     $decodedApiJwt
   else:
-    $apiJwtPayload
+    $(parts[1]) #apiJwtPayload
 
 proc refreshAccessToken*(refresh_token: string): string =
 
   # Mechanism to support access_token refresh via OIDC
   let timeout:   int = cast[int](chalkConfig.getSecretManagerTimeout())
-  var 
+  var
       refresh_url = uri.parseUri(chalkConfig.getSecretManagerUrl())
       context:           SslContext
       client:            HttpClient
@@ -34,7 +33,7 @@ proc refreshAccessToken*(refresh_token: string): string =
   # request new access_token via refresh
   info("Refreshing API access token....")
   if refresh_url.scheme == "https":
-    let context = newContext(verifyMode = CVerifyPeer)
+    context = newContext(verifyMode = CVerifyPeer)
     client  = newHttpClient(sslContext = context, timeout = timeout)
   else:
     client  = newHttpClient(timeout = timeout)
@@ -43,9 +42,9 @@ proc refreshAccessToken*(refresh_token: string): string =
 
   if response.status.startswith("200"):
     # parse json response and save / return values
-    let jsonNode = parseJson(response.body())
-    let new_access_token = jsonNode["access_token"].getStr()
-    let new_id_token     = jsonNode["id_token"].getStr()
+    let
+      jsonNode         = parseJson(response.body())
+      new_access_token = jsonNode["access_token"].getStr()
 
     return new_access_token
 
@@ -67,7 +66,6 @@ proc getChalkApiToken*(): (string, string) =
     contextPoll:       SslContext
     frameIndex:        int    = 0
     framerate:         float
-    jwtString:         string
     pollPayloadBase64: string
     pollUri:           Uri
     pollUrl:           string
@@ -155,7 +153,6 @@ proc getChalkApiToken*(): (string, string) =
 
           # decode JWT
           pollPayloadBase64  = jwtSplitAndDecode($accessToken, true)
-          let decodedPollJwt = parseJson(pollPayloadBase64)
           ret = ($accessToken, $refreshToken)
 
         elif responsePoll.status.startswith("428") or responsePoll.status.startswith("403"):