Configure hist. cache size on startup

.daily_canary

@@ -4,3 +4,7 @@
 /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 2024__
 !..!
+
+ ,--.  
+ ( o>
+//\\

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - The `cchost` configuration file now includes an `idle_connection_timeout` option. This controls how long the node will keep idle connections (for user TLS sessions) before automatically closing them. This may be set to `null` to restore the previous behaviour, where idle connections are never closed. By default connections will be closed after 60s of idle time.
 - New endpoints `GET /gov/service/javascript-modules` and `GET /gov/service/javascript-modules/{moduleName}` to retrieve the raw JS code of the currently installed app. Note that the `{moduleName}` path parameter will need to be URL-encoded to escape any `/` characters (eg - `/foo/bar.js` should become `%2Ffoo%2Fbar.js`).
 - New gov API version `2024-07-01`. This is near-identical to `2023-06-01-preview`, but additionally offers the new `javascript-modules` endpoints.
+- Historical cache soft limit now is a node-specific startup parameter.
 
 ### Changed
 

CMakeLists.txt

@@ -1574,6 +1574,11 @@ if(BUILD_TESTS)
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_perf.py
       LABEL perf PERF_LABEL "Historical Queries"
     )
+
+    add_e2e_test(
+      NAME historical_query_cache_test
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_cache.py
+    )
   endif()
 endif()
 

doc/host_config_schema/cchost_config.json

@@ -704,6 +704,11 @@
       "type": "integer",
       "description": "Cap at which node-to-node message channels will be closed, and a new channel will be created. Can be used to limit use of single cryptographic key",
       "minimum": 0
+    },
+    "historical_cache_soft_limit": {
+      "type": "string",
+      "default": "512MB",
+      "description": "Historical queries cache soft limit (as size string)"
     }
   },
   "required": ["enclave", "network", "command"],

include/ccf/node/startup_config.h

@@ -22,6 +22,8 @@ struct CCFConfig
   // 2**24.5 as per RFC8446 Section 5.5
   size_t node_to_node_message_limit = 23'726'566;
 
+  ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
+
   ccf::consensus::Configuration consensus = {};
   ccf::NodeInfoNetwork network = {};
 

samples/apps/logging/logging.cpp

@@ -460,12 +460,6 @@ namespace loggingapp
         PUBLIC_RECORDS, context, 10000, 20);
       context.get_indexing_strategies().install_strategy(index_per_public_key);
 
-      // According to manual obvervation it's enough to start evicting old
-      // requests on historical perf test, but not too small to get stuck
-      // because of a single request being larget than the cache.
-      constexpr size_t cache_limit = 1024 * 1024 * 10; // MB
-      context.get_historical_state().set_soft_cache_limit(cache_limit);
-
       const ccf::AuthnPolicies auth_policies = {
         ccf::jwt_auth_policy,
         ccf::user_cert_auth_policy,

src/common/configuration.h

@@ -92,7 +92,8 @@ DECLARE_JSON_OPTIONAL_FIELDS(
   ledger_signatures,
   jwt,
   attestation,
-  node_to_node_message_limit);
+  node_to_node_message_limit,
+  historical_cache_soft_limit);
 
 DECLARE_JSON_TYPE(StartupConfig::Start);
 DECLARE_JSON_REQUIRED_FIELDS(

src/enclave/enclave.h

@@ -238,6 +238,9 @@ namespace ccf
 
       node->set_n2n_message_limit(ccf_config_.node_to_node_message_limit);
 
+      historical_state_cache->set_soft_cache_limit(
+        ccf_config_.historical_cache_soft_limit);
+
       // If we haven't heard from a node for multiple elections, then cleanup
       // their node-to-node channel
       const auto idle_timeout =

src/node/historical_queries.h

@@ -509,7 +509,7 @@ namespace ccf::historical
     std::unordered_map<SeqNo, std::set<CompoundHandle>> store_to_requests;
     std::unordered_map<ccf::SeqNo, size_t> raw_store_sizes{};
 
-    CacheSize soft_store_cache_limit{1ll * 1024 * 1024 * 512 /*512 MB*/};
+    CacheSize soft_store_cache_limit{std::numeric_limits<size_t>::max()};
     CacheSize soft_store_cache_limit_raw =
       soft_store_cache_limit / soft_to_raw_ratio;
     CacheSize estimated_store_cache_size{0};

tests/config.jinja

@@ -102,5 +102,6 @@
     "max_fragment_size": "256KB"
   },
   "ignore_first_sigterm": {{ ignore_first_sigterm|tojson }}{% if node_to_node_message_limit %},
-  "node_to_node_message_limit": {{ node_to_node_message_limit|tojson }}{% endif %}
+  "node_to_node_message_limit": {{ node_to_node_message_limit|tojson }}{% endif %}{% if historical_cache_soft_limit %},
+  "historical_cache_soft_limit": {{ historical_cache_soft_limit|tojson }}{% endif %}
 }

tests/infra/network.py

@@ -194,6 +194,7 @@ class Network:
         "acme",
         "snp_endorsements_servers",
         "node_to_node_message_limit",
+        "historical_cache_soft_limit",
         "tick_ms",
         "max_msg_size_bytes",
         "snp_security_policy_file",

tests/infra/remote.py

@@ -630,6 +630,7 @@ def __init__(
         snp_security_policy_file=None,
         snp_uvm_endorsements_file=None,
         service_subject_name="CN=CCF Test Service",
+        historical_cache_soft_limit=None,
         **kwargs,
     ):
         """
@@ -823,6 +824,7 @@ def __init__(
                 snp_security_policy_file=snp_security_policy_file,
                 snp_uvm_endorsements_file=snp_uvm_endorsements_file,
                 service_subject_name=service_subject_name,
+                historical_cache_soft_limit=historical_cache_soft_limit,
                 **kwargs,
             )