Merge branch 'main' into cjen1-build-bundle

.CodeQL.yml

@@ -0,0 +1,11 @@
+# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/troubleshooting/bugs/generated-library-code
+path_classifiers:
+  # Everything under tests is test code
+  test:
+    - tests
+  # Everything under build is generated
+  generated:
+    - build
+  # Everything under 3rdparty is external libraries
+  library:
+    - 3rdparty

.azure-pipelines-release.yml

@@ -8,15 +8,15 @@ pr: none
 resources:
   containers:
     - container: virtual
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ghcr.io/microsoft/ccf/ci/sgx:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/sgx:build-05-12-2024
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:

.azure-pipelines-templates/deploy_aci.yml

@@ -50,7 +50,7 @@ jobs:
         env:
           ACR_REGISTRY_RESOURCE_NAME: ccfmsrc
           ACR_REGISTRY: ccfmsrc.azurecr.io
-          BASE_IMAGE: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+          BASE_IMAGE: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
       - script: |
           set -ex

.azure_pipelines_snp.yml

@@ -22,7 +22,7 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
 jobs:

.github/workflows/bencher.yml

@@ -1,6 +1,8 @@
 name: "Bencher: Run Benchmarks"
 
 on:
+  schedule:
+    - cron: "0 0 * * 0"
   push:
     branches:
       - main
@@ -11,7 +13,7 @@ jobs:
     name: Continuous Benchmarking with Bencher
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/ci-containers-ghcr.yml

@@ -49,7 +49,7 @@ jobs:
           labels: ${{ steps.meta_default.outputs.labels }}
 
       - name: Attest default container
-        uses: actions/attest-build-provenance@v1
+        uses: actions/attest-build-provenance@v2
         with:
           subject-name: ${{ env.REGISTRY }}/${{ github.repository }}/ci/default
           subject-digest: ${{ steps.push_default.outputs.digest }}

.github/workflows/ci-verification.yml

@@ -24,7 +24,7 @@ jobs:
     name: Model Checking - Consistency
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
     defaults:
       run:
         working-directory: tla
@@ -102,7 +102,7 @@ jobs:
     name: Model Checking - Consensus
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
     defaults:
       run:
         working-directory: tla
@@ -158,7 +158,7 @@ jobs:
     name: Trace Validation - Consensus
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
   checks:
     name: "Format and License Checks"
     runs-on: ubuntu-latest
-    container: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+    container: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
@@ -44,7 +44,7 @@ jobs:
             options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
     runs-on: ${{ matrix.platform.nodes }}
     container:
-      image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-05-12-2024
       options: ${{ matrix.platform.options }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/codeql-analysis.yml

@@ -25,7 +25,7 @@ jobs:
     # Insufficient space to run on public runner, so use custom pool
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
       options: --user root
 
     permissions:

.github/workflows/containers-ghcr.yml

@@ -68,7 +68,7 @@ jobs:
         if: ${{ steps.tref.outputs.major_release >= '5'}}
 
       - name: Attest container
-        uses: actions/attest-build-provenance@v1
+        uses: actions/attest-build-provenance@v2
         with:
           subject-name: ${{ steps.tref.outputs.img_name }}
           subject-digest: ${{ steps.push.outputs.digest }}

.github/workflows/long-test.yml

@@ -17,7 +17,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - uses: actions/checkout@v4
@@ -35,7 +35,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - uses: actions/checkout@v4
@@ -79,7 +79,7 @@ jobs:
     name: TSAN
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - uses: actions/checkout@v4
@@ -117,7 +117,7 @@ jobs:
     name: LTS
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/long-verification.yml

@@ -22,7 +22,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-verification') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
     defaults:
       run:
         working-directory: tla
@@ -50,7 +50,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-verification') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
     defaults:
       run:
         working-directory: tla

.github/workflows/release.yml

@@ -75,7 +75,7 @@ jobs:
             nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     runs-on: ${{ matrix.platform.nodes }}
     container:
-      image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-08-10-2024
+      image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-05-12-2024
       options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro ${{ matrix.platform.container_options }}"
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.0-dev10]
+
+[6.0.0-dev10]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev10
+
+### Added
+
+- Expose `ccf:http::parse_accept_header()` and `ccf::http::AcceptHeaderField` (#6706).
+
+## [6.0.0-dev9]
+
+[6.0.0-dev9]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev9
+
+### Changed
+
+- The `read_ledger.py` tool now has a `--quiet` option which avoids printing anything per-transaction, as well as other performance improvements, which should make it more useful in verifying the integrity of large ledgers.
+- COSE signatures now set a kid that is a hex-encoded SHA-256 of the DER representation of the key used to produce them (#6703).
+
+## [6.0.0-dev8]
+
+[6.0.0-dev8]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev8
+
+### Changed
+
+- All definitions in CCF's public headers are now under the `ccf::` namespace. Any application code which references any of these types directly (notably `StartupConfig`, `http_status`, `LoggerLevel`), they will now need to be prefixed with the `ccf::` namespace.
+- `cchost` now requires `--config`.
+
 ## [6.0.0-dev7]
 
 [6.0.0-dev7]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev7

CMakeLists.txt

@@ -59,7 +59,12 @@ function(message)
 endfunction()
 
 option(PROFILE_TESTS "Profile tests" OFF)
-set(PYTHON python3)
+
+if("$ENV{CI}" STREQUAL "")
+  set(PYTHON unbuffer python3)
+else()
+  set(PYTHON python3)
+endif()
 
 set(DISTRIBUTE_PERF_TESTS
     ""

cddl/ccf-merkle-tree-cose-signature.cddl

@@ -13,7 +13,7 @@ unprotected-headers = {
 
 protected-headers = {
   &(alg: 1) => int,                     ; signing algoritm ID, as per RFC8152
-  &(kid: 4) => bstr,                    ; signing key hash
+  &(kid: 4) => bstr,                    ; Opaque key identifier, hex-encoded SHA-256 of the public key encoded as DER.
   &(cwt: 15) => cwt-map,                ; CWT claims, as per RFC8392
   &(vds: 395) => int,                   ; verifiable data structure, as per COSE Receipts (draft) RFC (https://datatracker.ietf.org/doc/draft-ietf-cose-merkle-tree-proofs/)
   "ccf.v1" => ccf-map                   ; a set of CCF-specific parameters

doc/build_apps/api.rst

@@ -166,6 +166,16 @@ HTTP Entity Tags Matching
    :project: CCF
    :members:
 
+HTTP Accept Header Matching
+---------------------------
+
+.. doxygenstruct:: ccf::http::AcceptHeaderField
+   :project: CCF
+   :members:
+
+.. doxygenfunction:: ccf::http::parse_accept_header
+   :project: CCF
+
 COSE
 ----
 

doc/build_apps/kv/kv_how_to.rst

@@ -122,7 +122,7 @@ Note that, as in the sample above, it is possible to acquire different kinds of
 Removing a key
 --------------
 
-If a Key-Value pair was written to a :cpp:type:`ccf::kv::Map` by a previous :cpp:class:`ccf::kv::Tx`, it is possible to delete this key. Because of the append-only nature of the KV, this Key-Value pair is not actually removed from the :cpp:type:`ccf::kv::Map` but instead explicitly marked as deleted in the version that the deleting :cpp:class:`ccf::kv::Tx` is applied at.
+If a Key-Value pair was written to a :cpp:type:`ccf::kv::Map` by a previous :cpp:class:`ccf::kv::Tx`, it is possible to delete this key.
 
 .. code-block:: cpp
 

docker/ccf_ci_built

@@ -3,7 +3,7 @@
 # Also contains CCF source and build directory
 
 # Latest image as of this change
-ARG base=ghcr.io/microsoft/ccf/ci/default:build-08-10-2024
+ARG base=ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
 FROM ${base}
 
 # SSH. Note that this could (should) be done in the base ccf_ci image instead

include/ccf/ccf_deprecated.h

@@ -3,3 +3,7 @@
 #pragma once
 
 #define CCF_DEPRECATED(reason) [[deprecated(reason)]]
+
+// ci-checks exception - only defines a macro
+namespace ccf
+{}

include/ccf/ds/enum_formatter.h

@@ -27,3 +27,7 @@ struct formatter<E, std::enable_if_t<std::is_enum_v<E>, char>>
   }
 };
 FMT_END_NAMESPACE
+
+// ci-checks exception - defines a struct in the fmt namespace
+namespace ccf
+{}

include/ccf/ds/logger.h

@@ -344,9 +344,9 @@ namespace ccf::logger
 // This allows:
 // CCF_LOG_OUT(DEBUG, "foo") << "this " << "msg";
 #define CCF_LOG_OUT(LVL, TAG) \
-  ccf::logger::config::ok(LoggerLevel::LVL) && \
+  ccf::logger::config::ok(ccf::LoggerLevel::LVL) && \
     ccf::logger::Out() == \
-      ccf::logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__)
+      ccf::logger::LogLine(ccf::LoggerLevel::LVL, TAG, __FILE__, __LINE__)
 
 // To avoid repeating the (s, ...) args for every macro, we cheat with a curried
 // macro here by ending the macro with another macro name, which then accepts

include/ccf/ds/logger_level.h

@@ -2,12 +2,15 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
-enum LoggerLevel
+namespace ccf
 {
-  TRACE,
-  DEBUG, // events useful for debugging
-  INFO, // important events that should be logged even in release mode
-  FAIL, // survivable failures that should always be logged
-  FATAL, // fatal errors that may be non-recoverable
-  MAX_LOG_LEVEL
-};
+  enum LoggerLevel
+  {
+    TRACE,
+    DEBUG, // events useful for debugging
+    INFO, // important events that should be logged even in release mode
+    FAIL, // survivable failures that should always be logged
+    FATAL, // fatal errors that may be non-recoverable
+    MAX_LOG_LEVEL
+  };
+}

src/http/http_accept.h → include/ccf/http_accept.h

@@ -5,11 +5,11 @@
 #include "ccf/ds/nonstd.h"
 #include "ccf/http_status.h"
 #include "ccf/odata_error.h"
-#include "node/rpc/rpc_exception.h"
+#include "ccf/rpc_exception.h"
 
 #include <string_view>
 
-namespace http
+namespace ccf::http
 {
   struct AcceptHeaderField
   {

include/ccf/http_status.h

@@ -4,15 +4,18 @@
 
 #include <llhttp/llhttp.h>
 
-using http_status = llhttp_status;
-
-/* Returns a string version of the HTTP status code. */
-static inline const char* http_status_str(http_status s)
+namespace ccf
 {
-  return llhttp_status_name(s);
-}
+  using http_status = llhttp_status;
 
-static inline bool is_http_status_client_error(http_status s)
-{
-  return s >= 400 && s < 500;
-}
+  /* Returns a string version of the HTTP status code. */
+  static inline const char* http_status_str(http_status s)
+  {
+    return llhttp_status_name(s);
+  }
+
+  static inline bool is_http_status_client_error(http_status s)
+  {
+    return s >= 400 && s < 500;
+  }
+}