Support big strings

Previously, all strings needed to be stored on the stack. Given that BPF
has a 512B stack limit, this put serious limits on strings. In practice,
it was limited to about 200B.

This commit raises the limit much higher to around 1024. The new limit
we hit is from the LLVM memset() builtin. It only supports up to 1K. To
go above this, we need to create our own memset() routine in BPF. This
is quite easy to do and will be attempted in a later commit.

The way big strings work is by assigning each str() call in the script a
unique ID. Then we create a percpu array map such that for each CPU,
there is a percpu entry that can accomodate each str() callsite. This
makes it so scripts can keep as many strings "on the stack" as they'd
like, with the tradeoff being the amount of memory we need to
preallocate.

As long as the kernel never allows BPF programs to nest (eg execute prog
A while prog A is already on the call stack), this approach is robust to
self-corruption.

This closes bpftrace#305.

src/ast/irbuilderbpf.cpp

@@ -375,6 +375,18 @@ CallInst *IRBuilderBPF::CreateGetStackScratchMap(StackType stack_type,
                              failure_callback);
 }
 
+CallInst *IRBuilderBPF::CreateGetStrScratchMap(int idx,
+                                               BasicBlock *failure_callback,
+                                               const location &loc)
+{
+  return createGetScratchMap(to_string(MapType::StrBuffer),
+                             "str",
+                             GET_PTR_TY(),
+                             loc,
+                             failure_callback,
+                             idx);
+}
+
 // createGetScratchMap will jump to failure_callback if it cannot find the map
 // value
 CallInst *IRBuilderBPF::createGetScratchMap(const std::string &map_name,

src/ast/irbuilderbpf.h

@@ -201,6 +201,9 @@ class IRBuilderBPF : public IRBuilder<> {
   CallInst *CreateGetStackScratchMap(StackType stack_type,
                                      BasicBlock *failure_callback,
                                      const location &loc);
+  CallInst *CreateGetStrScratchMap(int idx,
+                                   BasicBlock *failure_callback,
+                                   const location &loc);
   CallInst *CreateHelperCall(libbpf::bpf_func_id func_id,
                              FunctionType *helper_type,
                              ArrayRef<Value *> args,

src/ast/passes/codegen_llvm.cpp

@@ -708,9 +708,9 @@ void CodegenLLVM::visit(Call &call)
     }
     expr_ = nullptr;
   } else if (call.func == "str") {
+    uint64_t max_strlen = bpftrace_.config_.get(ConfigKeyInt::max_strlen);
     // Largest read we'll allow = our global string buffer size
-    Value *strlen = b_.getInt64(
-        bpftrace_.config_.get(ConfigKeyInt::max_strlen));
+    Value *strlen = b_.getInt64(max_strlen);
     if (call.vargs->size() > 1) {
       auto scoped_del = accept(call.vargs->at(1));
       expr_ = b_.CreateIntCast(expr_, b_.getInt64Ty(), true);
@@ -728,17 +728,35 @@ void CodegenLLVM::visit(Call &call)
       // maximum
       strlen = b_.CreateSelect(Cmp, proposed_strlen, strlen, "str.min.select");
     }
-    AllocaInst *buf = b_.CreateAllocaBPF(
-        bpftrace_.config_.get(ConfigKeyInt::max_strlen), "str");
-    b_.CREATE_MEMSET(
-        buf, b_.getInt8(0), bpftrace_.config_.get(ConfigKeyInt::max_strlen), 1);
+
+    Function *parent = b_.GetInsertBlock()->getParent();
+    BasicBlock *lookup_failure_block = BasicBlock::Create(
+        module_->getContext(), "scratch_lookup_failure", parent);
+    BasicBlock *lookup_merge_block = BasicBlock::Create(module_->getContext(),
+                                                        "scratch_lookup_merge",
+                                                        parent);
+
+    Value *buf = b_.CreateGetStrScratchMap(str_id_,
+                                           lookup_failure_block,
+                                           call.loc);
+    b_.CREATE_MEMSET(buf, b_.getInt8(0), max_strlen, 1);
     auto arg0 = call.vargs->front();
     auto scoped_del = accept(call.vargs->front());
     b_.CreateProbeReadStr(
         ctx_, buf, strlen, expr_, arg0->type.GetAS(), call.loc);
+    b_.CreateBr(lookup_merge_block);
 
+    // This of this like an assert(). In practice, we cannot fail to lookup a
+    // percpu array map unless we have a coding error. Rather than have some
+    // kind of complicated fallback path where we provide an error string for
+    // our caller, just indicate to verifier we want to terminate execution.
+    b_.SetInsertPoint(lookup_failure_block);
+    createRet();
+
+    b_.SetInsertPoint(lookup_merge_block);
+
+    str_id_++;
     expr_ = buf;
-    expr_deleter_ = [this, buf]() { b_.CreateLifetimeEnd(buf); };
   } else if (call.func == "buf") {
     Value *max_length = b_.getInt64(
         bpftrace_.config_.get(ConfigKeyInt::max_strlen));
@@ -3635,6 +3653,15 @@ void CodegenLLVM::generate_maps(const RequiredResources &resources)
                         MapKey({ CreateInt(loss_cnt_key_size) }),
                         CreateInt(loss_cnt_val_size));
   }
+
+  if (resources.str_buffers > 0) {
+    auto max_strlen = bpftrace_.config_.get(ConfigKeyInt::max_strlen);
+    createMapDefinition(to_string(MapType::StrBuffer),
+                        libbpf::BPF_MAP_TYPE_PERCPU_ARRAY,
+                        resources.str_buffers,
+                        MapKey({ CreateInt32() }),
+                        CreateArray(max_strlen, CreateInt8()));
+  }
 }
 
 void CodegenLLVM::emit_elf(const std::string &filename)
@@ -4262,7 +4289,8 @@ std::function<void()> CodegenLLVM::create_reset_ids()
           starting_non_map_print_id = this->non_map_print_id_,
           starting_watchpoint_id = this->watchpoint_id_,
           starting_cgroup_path_id = this->cgroup_path_id_,
-          starting_skb_output_id = this->skb_output_id_] {
+          starting_skb_output_id = this->skb_output_id_,
+          starting_str_id = this->str_id_] {
     this->b_.helper_error_id_ = starting_helper_error_id;
     this->printf_id_ = starting_printf_id;
     this->mapped_printf_id_ = starting_mapped_printf_id;
@@ -4275,6 +4303,7 @@ std::function<void()> CodegenLLVM::create_reset_ids()
     this->watchpoint_id_ = starting_watchpoint_id;
     this->cgroup_path_id_ = starting_cgroup_path_id;
     this->skb_output_id_ = starting_skb_output_id;
+    this->str_id_ = starting_str_id;
   };
 }
 

src/ast/passes/codegen_llvm.h

@@ -279,6 +279,7 @@ class CodegenLLVM : public Visitor {
   uint64_t watchpoint_id_ = 0;
   int cgroup_path_id_ = 0;
   int skb_output_id_ = 0;
+  int str_id_ = 0;
 
   std::unordered_map<std::string, libbpf::bpf_map_type> map_types_;
 

src/ast/passes/resource_analyser.cpp

@@ -155,6 +155,8 @@ void ResourceAnalyser::visit(Call &call)
       resources_.time_args.push_back(get_literal_string(*call.vargs->at(0)));
     else
       resources_.time_args.push_back("%H:%M:%S\n");
+  } else if (call.func == "str") {
+    resources_.str_buffers++;
   } else if (call.func == "strftime") {
     resources_.strftime_args.push_back(get_literal_string(*call.vargs->at(0)));
   } else if (call.func == "print") {

src/bpfmap.cpp

@@ -69,6 +69,8 @@ std::string to_string(MapType t)
       return "ringbuf";
     case MapType::RingbufLossCounter:
       return "ringbuf_loss_counter";
+    case MapType::StrBuffer:
+      return "str_buffer";
   }
   return {}; // unreached
 }

src/bpfmap.h

@@ -72,6 +72,7 @@ enum class MapType {
   MappedPrintfData,
   Ringbuf,
   RingbufLossCounter,
+  StrBuffer,
 };
 
 std::string to_string(MapType t);

src/main.cpp

@@ -269,25 +269,6 @@ static void parse_env(BPFtrace& bpftrace)
     config_setter.set(ConfigKeyInt::max_strlen, x);
   });
 
-  // in practice, the largest buffer I've seen fit into the BPF stack was 240
-  // bytes. I've set the bar lower, in case your program has a deeper stack than
-  // the one from my tests, in the hope that you'll get this instructive error
-  // instead of getting the BPF verifier's error.
-  uint64_t max_strlen = bpftrace.config_.get(ConfigKeyInt::max_strlen);
-  if (max_strlen > 200) {
-    // the verifier errors you would encounter when attempting larger
-    // allocations would be: >240=  <Looks like the BPF stack limit of 512 bytes
-    // is exceeded. Please move large on stack variables into BPF per-cpu array
-    // map.> ~1024= <A call to built-in function 'memset' is not supported.>
-    LOG(ERROR) << "'BPFTRACE_MAX_STRLEN' " << max_strlen
-               << " exceeds the current maximum of 200 bytes.\n"
-               << "This limitation is because strings are currently stored on "
-                  "the 512 byte BPF stack.\n"
-               << "Long strings will be pursued in: "
-                  "https://github.com/bpftrace/bpftrace/issues/305";
-    exit(1);
-  }
-
   if (const char* env_p = std::getenv("BPFTRACE_STR_TRUNC_TRAILER"))
     config_setter.set(ConfigKeyString::str_trunc_trailer, std::string(env_p));
 

src/required_resources.h

@@ -117,6 +117,7 @@ class RequiredResources {
   bool needs_elapsed_map = false;
   bool needs_data_map = false;
   bool needs_perf_event_map = false;
+  uint32_t str_buffers = 0;
 
   // Probe metadata
   //