Support big strings

Previously, all strings needed to be stored on the stack. Given that BPF
has a 512B stack limit, this put serious limits on strings. In practice,
it was limited to about 200B.

This commit raises the limit much higher to around 1024. The new limit
we hit is from the LLVM memset() builtin. It only supports up to 1K. To
go above this, we need to create our own memset() routine in BPF. This
is quite easy to do and will be attempted in a later commit.

The way big strings work is by assigning each str() call in the script a
unique ID. Then we create a percpu array map such that for each CPU,
there is a percpu entry that can accomodate each str() callsite. This
makes it so scripts can keep as many strings "on the stack" as they'd
like, with the tradeoff being the amount of memory we need to
preallocate.

As long as the kernel never allows BPF programs to nest (eg execute prog
A while prog A is already on the call stack), this approach is robust to
self-corruption.

Note we do not yet remove or change default value of
BPFTRACE_MAX_STRLEN. That is b/c a few other helpers (buf(), path(),
maybe more) also read the value and allocate space on stack accordingly.
We need to decouple or convert those helpers to scratch maps as well.

This closes bpftrace#305.

src/ast/irbuilderbpf.cpp

@@ -375,6 +375,18 @@ CallInst *IRBuilderBPF::CreateGetStackScratchMap(StackType stack_type,
                              failure_callback);
 }
 
+CallInst *IRBuilderBPF::CreateGetStrScratchMap(int idx,
+                                               BasicBlock *failure_callback,
+                                               const location &loc)
+{
+  return createGetScratchMap(to_string(MapType::StrBuffer),
+                             "str",
+                             GET_PTR_TY(),
+                             loc,
+                             failure_callback,
+                             idx);
+}
+
 // createGetScratchMap will jump to failure_callback if it cannot find the map
 // value
 CallInst *IRBuilderBPF::createGetScratchMap(const std::string &map_name,

src/ast/irbuilderbpf.h

@@ -196,6 +196,9 @@ class IRBuilderBPF : public IRBuilder<> {
   CallInst *CreateGetStackScratchMap(StackType stack_type,
                                      BasicBlock *failure_callback,
                                      const location &loc);
+  CallInst *CreateGetStrScratchMap(int idx,
+                                   BasicBlock *failure_callback,
+                                   const location &loc);
   CallInst *CreateHelperCall(libbpf::bpf_func_id func_id,
                              FunctionType *helper_type,
                              ArrayRef<Value *> args,

src/ast/passes/codegen_llvm.cpp

@@ -708,9 +708,9 @@ void CodegenLLVM::visit(Call &call)
     }
     expr_ = nullptr;
   } else if (call.func == "str") {
+    uint64_t max_strlen = bpftrace_.config_.get(ConfigKeyInt::max_strlen);
     // Largest read we'll allow = our global string buffer size
-    Value *strlen = b_.getInt64(
-        bpftrace_.config_.get(ConfigKeyInt::max_strlen));
+    Value *strlen = b_.getInt64(max_strlen);
     if (call.vargs->size() > 1) {
       auto scoped_del = accept(call.vargs->at(1));
       expr_ = b_.CreateIntCast(expr_, b_.getInt64Ty(), true);
@@ -728,17 +728,35 @@ void CodegenLLVM::visit(Call &call)
       // maximum
       strlen = b_.CreateSelect(Cmp, proposed_strlen, strlen, "str.min.select");
     }
-    AllocaInst *buf = b_.CreateAllocaBPF(
-        bpftrace_.config_.get(ConfigKeyInt::max_strlen), "str");
-    b_.CREATE_MEMSET(
-        buf, b_.getInt8(0), bpftrace_.config_.get(ConfigKeyInt::max_strlen), 1);
+
+    Function *parent = b_.GetInsertBlock()->getParent();
+    BasicBlock *lookup_failure_block = BasicBlock::Create(
+        module_->getContext(), "scratch_lookup_failure", parent);
+    BasicBlock *lookup_merge_block = BasicBlock::Create(module_->getContext(),
+                                                        "scratch_lookup_merge",
+                                                        parent);
+
+    Value *buf = b_.CreateGetStrScratchMap(str_id_,
+                                           lookup_failure_block,
+                                           call.loc);
+    b_.CREATE_MEMSET(buf, b_.getInt8(0), max_strlen, 1);
     auto arg0 = call.vargs->front();
     auto scoped_del = accept(call.vargs->front());
     b_.CreateProbeReadStr(
         ctx_, buf, strlen, expr_, arg0->type.GetAS(), call.loc);
+    b_.CreateBr(lookup_merge_block);
 
+    // Think of this like an assert(). In practice, we cannot fail to lookup a
+    // percpu array map unless we have a coding error. Rather than have some
+    // kind of complicated fallback path where we provide an error string for
+    // our caller, just indicate to verifier we want to terminate execution.
+    b_.SetInsertPoint(lookup_failure_block);
+    createRet();
+
+    b_.SetInsertPoint(lookup_merge_block);
+
+    str_id_++;
     expr_ = buf;
-    expr_deleter_ = [this, buf]() { b_.CreateLifetimeEnd(buf); };
   } else if (call.func == "buf") {
     Value *max_length = b_.getInt64(
         bpftrace_.config_.get(ConfigKeyInt::max_strlen));
@@ -3622,6 +3640,15 @@ void CodegenLLVM::generate_maps(const RequiredResources &resources)
                         MapKey({ CreateInt(loss_cnt_key_size) }),
                         CreateInt(loss_cnt_val_size));
   }
+
+  if (resources.str_buffers > 0) {
+    auto max_strlen = bpftrace_.config_.get(ConfigKeyInt::max_strlen);
+    createMapDefinition(to_string(MapType::StrBuffer),
+                        libbpf::BPF_MAP_TYPE_PERCPU_ARRAY,
+                        resources.str_buffers,
+                        MapKey({ CreateInt32() }),
+                        CreateArray(max_strlen, CreateInt8()));
+  }
 }
 
 void CodegenLLVM::emit_elf(const std::string &filename)
@@ -4250,7 +4277,8 @@ std::function<void()> CodegenLLVM::create_reset_ids()
           starting_non_map_print_id = this->non_map_print_id_,
           starting_watchpoint_id = this->watchpoint_id_,
           starting_cgroup_path_id = this->cgroup_path_id_,
-          starting_skb_output_id = this->skb_output_id_] {
+          starting_skb_output_id = this->skb_output_id_,
+          starting_str_id = this->str_id_] {
     this->b_.helper_error_id_ = starting_helper_error_id;
     this->printf_id_ = starting_printf_id;
     this->mapped_printf_id_ = starting_mapped_printf_id;
@@ -4263,6 +4291,7 @@ std::function<void()> CodegenLLVM::create_reset_ids()
     this->watchpoint_id_ = starting_watchpoint_id;
     this->cgroup_path_id_ = starting_cgroup_path_id;
     this->skb_output_id_ = starting_skb_output_id;
+    this->str_id_ = starting_str_id;
   };
 }
 

src/ast/passes/codegen_llvm.h

@@ -279,6 +279,7 @@ class CodegenLLVM : public Visitor {
   uint64_t watchpoint_id_ = 0;
   int cgroup_path_id_ = 0;
   int skb_output_id_ = 0;
+  int str_id_ = 0;
 
   std::unordered_map<std::string, libbpf::bpf_map_type> map_types_;
 

src/ast/passes/resource_analyser.cpp

@@ -155,6 +155,8 @@ void ResourceAnalyser::visit(Call &call)
       resources_.time_args.push_back(get_literal_string(*call.vargs->at(0)));
     else
       resources_.time_args.push_back("%H:%M:%S\n");
+  } else if (call.func == "str") {
+    resources_.str_buffers++;
   } else if (call.func == "strftime") {
     resources_.strftime_args.push_back(get_literal_string(*call.vargs->at(0)));
   } else if (call.func == "print") {

src/bpfmap.cpp

@@ -69,6 +69,8 @@ std::string to_string(MapType t)
       return "ringbuf";
     case MapType::RingbufLossCounter:
       return "ringbuf_loss_counter";
+    case MapType::StrBuffer:
+      return "str_buffer";
   }
   return {}; // unreached
 }

src/bpfmap.h

@@ -72,6 +72,7 @@ enum class MapType {
   MappedPrintfData,
   Ringbuf,
   RingbufLossCounter,
+  StrBuffer,
 };
 
 std::string to_string(MapType t);

src/main.cpp

@@ -269,25 +269,6 @@ static void parse_env(BPFtrace& bpftrace)
     config_setter.set(ConfigKeyInt::max_strlen, x);
   });
 
-  // in practice, the largest buffer I've seen fit into the BPF stack was 240
-  // bytes. I've set the bar lower, in case your program has a deeper stack than
-  // the one from my tests, in the hope that you'll get this instructive error
-  // instead of getting the BPF verifier's error.
-  uint64_t max_strlen = bpftrace.config_.get(ConfigKeyInt::max_strlen);
-  if (max_strlen > 200) {
-    // the verifier errors you would encounter when attempting larger
-    // allocations would be: >240=  <Looks like the BPF stack limit of 512 bytes
-    // is exceeded. Please move large on stack variables into BPF per-cpu array
-    // map.> ~1024= <A call to built-in function 'memset' is not supported.>
-    LOG(ERROR) << "'BPFTRACE_MAX_STRLEN' " << max_strlen
-               << " exceeds the current maximum of 200 bytes.\n"
-               << "This limitation is because strings are currently stored on "
-                  "the 512 byte BPF stack.\n"
-               << "Long strings will be pursued in: "
-                  "https://github.com/bpftrace/bpftrace/issues/305";
-    exit(1);
-  }
-
   if (const char* env_p = std::getenv("BPFTRACE_STR_TRUNC_TRAILER"))
     config_setter.set(ConfigKeyString::str_trunc_trailer, std::string(env_p));
 

src/required_resources.h

@@ -117,6 +117,7 @@ class RequiredResources {
   bool needs_elapsed_map = false;
   bool needs_data_map = false;
   bool needs_perf_event_map = false;
+  uint32_t str_buffers = 0;
 
   // Probe metadata
   //

tests/codegen/llvm/call_str.ll

@@ -6,53 +6,69 @@ target triple = "bpf-pc-linux"
 %"struct map_t" = type { i8*, i8*, i8*, i8* }
 %"struct map_t.0" = type { i8*, i8* }
 %"struct map_t.1" = type { i8*, i8*, i8*, i8* }
+%"struct map_t.2" = type { i8*, i8*, i8*, i8* }
 
 @LICENSE = global [4 x i8] c"GPL\00", section "license"
 @AT_x = dso_local global %"struct map_t" zeroinitializer, section ".maps", !dbg !0
 @ringbuf = dso_local global %"struct map_t.0" zeroinitializer, section ".maps", !dbg !25
 @ringbuf_loss_counter = dso_local global %"struct map_t.1" zeroinitializer, section ".maps", !dbg !39
+@str_buffer = dso_local global %"struct map_t.2" zeroinitializer, section ".maps", !dbg !53
 
 ; Function Attrs: nounwind
 declare i64 @llvm.bpf.pseudo(i64 %0, i64 %1) #0
 
-define i64 @kprobe_f_1(i8* %0) section "s_kprobe_f_1" !dbg !57 {
+define i64 @kprobe_f_1(i8* %0) section "s_kprobe_f_1" !dbg !66 {
 entry:
   %"@x_key" = alloca i64, align 8
-  %str = alloca [64 x i8], align 1
-  %1 = bitcast [64 x i8]* %str to i8*
+  %lookup_str_key = alloca i32, align 4
+  %1 = bitcast i32* %lookup_str_key to i8*
   call void @llvm.lifetime.start.p0i8(i64 -1, i8* %1)
-  %2 = bitcast [64 x i8]* %str to i8*
-  call void @llvm.memset.p0i8.i64(i8* align 1 %2, i8 0, i64 64, i1 false)
-  %3 = bitcast i8* %0 to i64*
-  %4 = getelementptr i64, i64* %3, i64 14
-  %arg0 = load volatile i64, i64* %4, align 8
-  %probe_read_kernel_str = call i64 inttoptr (i64 115 to i64 ([64 x i8]*, i32, i64)*)([64 x i8]* %str, i32 64, i64 %arg0)
-  %5 = bitcast i64* %"@x_key" to i8*
-  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %5)
+  store i32 0, i32* %lookup_str_key, align 4
+  %lookup_str_map = call i8* inttoptr (i64 1 to i8* (%"struct map_t.2"*, i32*)*)(%"struct map_t.2"* @str_buffer, i32* %lookup_str_key)
+  %2 = bitcast i32* %lookup_str_key to i8*
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %2)
+  %lookup_str_cond = icmp ne i8* %lookup_str_map, null
+  br i1 %lookup_str_cond, label %lookup_str_merge, label %lookup_str_failure
+
+scratch_lookup_failure:                           ; preds = %lookup_str_failure
+  ret i64 0
+
+scratch_lookup_merge:                             ; preds = %lookup_str_merge
+  %3 = bitcast i64* %"@x_key" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %3)
   store i64 0, i64* %"@x_key", align 8
-  %update_elem = call i64 inttoptr (i64 2 to i64 (%"struct map_t"*, i64*, [64 x i8]*, i64)*)(%"struct map_t"* @AT_x, i64* %"@x_key", [64 x i8]* %str, i64 0)
-  %6 = bitcast i64* %"@x_key" to i8*
-  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %6)
-  %7 = bitcast [64 x i8]* %str to i8*
-  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %7)
+  %update_elem = call i64 inttoptr (i64 2 to i64 (%"struct map_t"*, i64*, i8*, i64)*)(%"struct map_t"* @AT_x, i64* %"@x_key", i8* %lookup_str_map, i64 0)
+  %4 = bitcast i64* %"@x_key" to i8*
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %4)
   ret i64 0
+
+lookup_str_failure:                               ; preds = %entry
+  br label %scratch_lookup_failure
+
+lookup_str_merge:                                 ; preds = %entry
+  call void @llvm.memset.p0i8.i64(i8* align 1 %lookup_str_map, i8 0, i64 64, i1 false)
+  %5 = bitcast i8* %0 to i64*
+  %6 = getelementptr i64, i64* %5, i64 14
+  %arg0 = load volatile i64, i64* %6, align 8
+  %probe_read_kernel_str = call i64 inttoptr (i64 115 to i64 (i8*, i32, i64)*)(i8* %lookup_str_map, i32 64, i64 %arg0)
+  br label %scratch_lookup_merge
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
 declare void @llvm.lifetime.start.p0i8(i64 immarg %0, i8* nocapture %1) #1
 
-; Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
-declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly %0, i8 %1, i64 %2, i1 immarg %3) #2
-
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
 declare void @llvm.lifetime.end.p0i8(i64 immarg %0, i8* nocapture %1) #1
 
+; Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly %0, i8 %1, i64 %2, i1 immarg %3) #2
+
 attributes #0 = { nounwind }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
 attributes #2 = { argmemonly nofree nosync nounwind willreturn writeonly }
 
-!llvm.dbg.cu = !{!53}
-!llvm.module.flags = !{!56}
+!llvm.dbg.cu = !{!62}
+!llvm.module.flags = !{!65}
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "AT_x", linkageName: "global", scope: !2, file: !2, type: !3, isLocal: false, isDefinition: true)
@@ -107,13 +123,22 @@ attributes #2 = { argmemonly nofree nosync nounwind willreturn writeonly }
 !50 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !51, size: 64)
 !51 = !DIBasicType(name: "int32", size: 32, encoding: DW_ATE_signed)
 !52 = !DIDerivedType(tag: DW_TAG_member, name: "value", scope: !2, file: !2, baseType: !17, size: 64, offset: 192)
-!53 = distinct !DICompileUnit(language: DW_LANG_C, file: !2, producer: "bpftrace", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !54, globals: !55)
-!54 = !{}
-!55 = !{!0, !25, !39}
-!56 = !{i32 2, !"Debug Info Version", i32 3}
-!57 = distinct !DISubprogram(name: "kprobe_f_1", linkageName: "kprobe_f_1", scope: !2, file: !2, type: !58, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !53, retainedNodes: !61)
-!58 = !DISubroutineType(types: !59)
-!59 = !{!18, !60}
-!60 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64)
-!61 = !{!62}
-!62 = !DILocalVariable(name: "ctx", arg: 1, scope: !57, file: !2, type: !60)
+!53 = !DIGlobalVariableExpression(var: !54, expr: !DIExpression())
+!54 = distinct !DIGlobalVariable(name: "str_buffer", linkageName: "global", scope: !2, file: !2, type: !55, isLocal: false, isDefinition: true)
+!55 = !DICompositeType(tag: DW_TAG_structure_type, scope: !2, file: !2, size: 256, elements: !56)
+!56 = !{!57, !48, !49, !19}
+!57 = !DIDerivedType(tag: DW_TAG_member, name: "type", scope: !2, file: !2, baseType: !58, size: 64)
+!58 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !59, size: 64)
+!59 = !DICompositeType(tag: DW_TAG_array_type, baseType: !8, size: 192, elements: !60)
+!60 = !{!61}
+!61 = !DISubrange(count: 6, lowerBound: 0)
+!62 = distinct !DICompileUnit(language: DW_LANG_C, file: !2, producer: "bpftrace", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !63, globals: !64)
+!63 = !{}
+!64 = !{!0, !25, !39, !53}
+!65 = !{i32 2, !"Debug Info Version", i32 3}
+!66 = distinct !DISubprogram(name: "kprobe_f_1", linkageName: "kprobe_f_1", scope: !2, file: !2, type: !67, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !62, retainedNodes: !70)
+!67 = !DISubroutineType(types: !68)
+!68 = !{!18, !69}
+!69 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64)
+!70 = !{!71}
+!71 = !DILocalVariable(name: "ctx", arg: 1, scope: !66, file: !2, type: !69)