btf: Export and marshal DeclTags

This commit makes the DeclTag BTF type exported. The DeclTag has been
unexported because its encoding is a bit tricky.

DeclTags on the wire refer to a type by its ID, and an component index.
Unlike other BTF types, the type ID isn't always the actual "target",
instead if the component index is not `-1` it targets a child object of
the type. This commit adds a `Tags` field to all types that can have
DeclTags including Members.

Exception is function parameters. Due to the func -> funcProto ->
parameter encoding, it is possible that BTF deduplication reuses the
same funcProto for multiple functions. The declTags point to the func to
annotate the arguments. Thus, adding a `Tags` field to Param would
cause inaccurate BTF marshalling. Instead, a `ParamTags` field is added
the func as `[][]string` to match the write format more closely to
avoid issues caused by BTF deduplication. This is a compromise between
technical feasibility and user experience. Users need to manually match
up the `ParamTags` with the `Params`. This shouldn't be a huge issue
in most cases however.

This means we now effectively have less "types" in our Go BTF
representation then we have on the wire since declTags become strings
on other types. This might leave "holes" in our BTF type numbering
if we were to remove them. So instead we leave the `declTag` types in
the BTF type list, but we hide them from the user during iteration.
Technically, users can still access them by ID if they know the ID, but
like before this commit, the returned value can only be printed.

Marshalling relies on the iteration of the BTF types. When go to load
a BTF blob from a collection, we iterate from a collection of root types
which are directly used. This causes us to load only BTF info that is
actually in use. This commit introduces a trick into the `children`
logic used during iteration. When we iterate over a type with declTags,
we create internal `declTag` types at-hoc with the correct component
index. So even though the tags are just strings on other types, when
iterating they appear as actual types. This causes only declTags that
are on types which are in use to be marshalled.

Signed-off-by: Dylan Reimerink <dylan.reimerink@isovalent.com>

Makefile

@@ -49,6 +49,7 @@ TARGETS := \
 	btf/testdata/relocs_read \
 	btf/testdata/relocs_read_tgt \
 	btf/testdata/relocs_enum \
+	btf/testdata/tags \
 	cmd/bpf2go/testdata/minimal
 
 .PHONY: all clean container-all container-shell generate

btf/btf.go

@@ -695,5 +695,13 @@ func (iter *TypesIterator) Next() bool {
 	iter.Type, ok = iter.spec.typeByID(iter.id)
 	iter.id++
 	iter.done = !ok
+	if !iter.done {
+		// Skip declTags, during unmarshaling declTags become `Tags` fields of other types.
+		// We keep them in the spec to avoid holes in the ID space, but for the purposes of
+		// iteration, they are not useful to the user.
+		if _, ok := iter.Type.(*declTag); ok {
+			return iter.Next()
+		}
+	}
 	return !iter.done
 }

btf/marshal.go

@@ -521,7 +521,7 @@ func (e *encoder) deflateEnum64(raw *rawType, enum *Enum) (err error) {
 			})
 		}
 
-		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members})
+		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members, nil})
 	}
 
 	raw.SetKind(kindEnum64)

btf/marshal_test.go

@@ -24,7 +24,7 @@ func TestBuilderMarshal(t *testing.T) {
 		(*Void)(nil),
 		typ,
 		&Pointer{typ},
-		&Typedef{"baz", typ},
+		&Typedef{"baz", typ, nil},
 	}
 
 	b, err := NewBuilder(want)
@@ -65,7 +65,7 @@ func TestBuilderAdd(t *testing.T) {
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(2)), qt.Commentf("Adding a type twice returns different ids"))
 
-	id, err = b.Add(&Typedef{"baz", i})
+	id, err = b.Add(&Typedef{"baz", i, nil})
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(3)))
 }

btf/testdata/tags.c

@@ -0,0 +1,37 @@
+#include "../../testdata/common.h"
+
+#define tagA __attribute__((btf_decl_tag("a")))
+#define tagB __attribute__((btf_decl_tag("b")))
+#define tagC __attribute__((btf_decl_tag("c")))
+#define tagD __attribute__((btf_decl_tag("d")))
+#define tagE __attribute__((btf_decl_tag("e")))
+
+struct s {
+	char tagA foo;
+	char tagB bar;
+} tagC;
+
+union u {
+	char tagA foo;
+	char tagB bar;
+} tagC;
+
+typedef tagB char td;
+
+struct s tagD s1;
+union u tagE u1;
+td tagA t1;
+
+int tagA tagB fwdDecl(char tagC x, char tagD y);
+
+int tagE normalDecl1(char tagB x, char tagC y) {
+	return fwdDecl(x, y);
+}
+
+int tagE normalDecl2(char tagB x, char tagC y) {
+	return fwdDecl(x, y);
+}
+
+__section("syscall") int prog(char *ctx) {
+	return normalDecl1(ctx[0], ctx[1]) + normalDecl2(ctx[2], ctx[3]);
+}

btf/traversal.go

@@ -40,9 +40,12 @@ func children(typ Type, yield func(child *Type) bool) bool {
 	// Explicitly type switch on the most common types to allow the inliner to
 	// do its work. This avoids allocating intermediate slices from walk() on
 	// the heap.
+	var tags []string
 	switch v := typ.(type) {
-	case *Void, *Int, *Enum, *Fwd, *Float:
+	case *Void, *Int, *Enum, *Fwd, *Float, *declTag:
 		// No children to traverse.
+		// declTags is declared as a leaf type since it's parsed into .Tags fields of other types
+		// during unmarshaling.
 	case *Pointer:
 		if !yield(&v.Target) {
 			return false
@@ -59,17 +62,32 @@ func children(typ Type, yield func(child *Type) bool) bool {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+			for _, t := range v.Members[i].Tags {
+				var tag Type = &declTag{v, t, i}
+				if !yield(&tag) {
+					return false
+				}
+			}
 		}
+		tags = v.Tags
 	case *Union:
 		for i := range v.Members {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+			for _, t := range v.Members[i].Tags {
+				var tag Type = &declTag{v, t, i}
+				if !yield(&tag) {
+					return false
+				}
+			}
 		}
+		tags = v.Tags
 	case *Typedef:
 		if !yield(&v.Type) {
 			return false
 		}
+		tags = v.Tags
 	case *Volatile:
 		if !yield(&v.Type) {
 			return false
@@ -86,6 +104,20 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		if fp, ok := v.Type.(*FuncProto); ok {
+			for i := range fp.Params {
+				if len(v.ParamTags) <= i {
+					continue
+				}
+				for _, t := range v.ParamTags[i] {
+					var tag Type = &declTag{v, t, i}
+					if !yield(&tag) {
+						return false
+					}
+				}
+			}
+		}
+		tags = v.Tags
 	case *FuncProto:
 		if !yield(&v.Return) {
 			return false
@@ -99,16 +131,13 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		tags = v.Tags
 	case *Datasec:
 		for i := range v.Vars {
 			if !yield(&v.Vars[i].Type) {
 				return false
 			}
 		}
-	case *declTag:
-		if !yield(&v.Type) {
-			return false
-		}
 	case *TypeTag:
 		if !yield(&v.Type) {
 			return false
@@ -119,5 +148,12 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		panic(fmt.Sprintf("don't know how to walk Type %T", v))
 	}
 
+	for _, t := range tags {
+		var tag Type = &declTag{typ, t, -1}
+		if !yield(&tag) {
+			return false
+		}
+	}
+
 	return true
 }

btf/types.go

@@ -169,6 +169,7 @@ type Struct struct {
 	// The size of the struct including padding, in bytes
 	Size    uint32
 	Members []Member
+	Tags    []string
 }
 
 func (s *Struct) Format(fs fmt.State, verb rune) {
@@ -195,6 +196,7 @@ type Union struct {
 	// The size of the union including padding, in bytes.
 	Size    uint32
 	Members []Member
+	Tags    []string
 }
 
 func (u *Union) Format(fs fmt.State, verb rune) {
@@ -247,6 +249,7 @@ type Member struct {
 	Type         Type
 	Offset       Bits
 	BitfieldSize Bits
+	Tags         []string
 }
 
 // Enum lists possible values.
@@ -334,6 +337,7 @@ func (f *Fwd) matches(typ Type) bool {
 type Typedef struct {
 	Name string
 	Type Type
+	Tags []string
 }
 
 func (td *Typedef) Format(fs fmt.State, verb rune) {
@@ -403,6 +407,12 @@ type Func struct {
 	Name    string
 	Type    Type
 	Linkage FuncLinkage
+	Tags    []string
+	// ParamTags holds a list of tags for each parameter of the FuncProto to which `Type` points.
+	// If no tags are present for any param, the outer slice will be nil/len(ParamTags)==0.
+	// If at least 1 param has a tag, the outer slice will have the same length as the number of params.
+	// The inner slice contains the tags and may be nil/len(ParamTags[i])==0 if no tags are present for that param.
+	ParamTags [][]string
 }
 
 func FuncMetadata(ins *asm.Instruction) *Func {
@@ -456,6 +466,7 @@ type Var struct {
 	Name    string
 	Type    Type
 	Linkage VarLinkage
+	Tags    []string
 }
 
 func (v *Var) Format(fs fmt.State, verb rune) {
@@ -924,7 +935,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err)
 			}
-			typ = &Struct{name, header.Size(), members}
+			typ = &Struct{name, header.Size(), members, nil}
 
 		case kindUnion:
 			vlen := header.Vlen()
@@ -941,7 +952,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("union %s (id %d): %w", name, id, err)
 			}
-			typ = &Union{name, header.Size(), members}
+			typ = &Union{name, header.Size(), members, nil}
 
 		case kindEnum:
 			vlen := header.Vlen()
@@ -974,7 +985,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = &Fwd{name, header.FwdKind()}
 
 		case kindTypedef:
-			typedef := &Typedef{name, nil}
+			typedef := &Typedef{name, nil, nil}
 			fixup(header.Type(), &typedef.Type)
 			typ = typedef
 
@@ -994,7 +1005,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = restrict
 
 		case kindFunc:
-			fn := &Func{name, nil, header.Linkage()}
+			fn := &Func{name, nil, header.Linkage(), nil, nil}
 			fixup(header.Type(), &fn.Type)
 			typ = fn
 
@@ -1036,7 +1047,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 				return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err)
 			}
 
-			v := &Var{name, nil, VarLinkage(bVariable.Linkage)}
+			v := &Var{name, nil, VarLinkage(bVariable.Linkage), nil}
 			fixup(header.Type(), &v.Type)
 			typ = v
 
@@ -1148,26 +1159,69 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 
 	for _, dt := range declTags {
 		switch t := dt.Type.(type) {
-		case *Var, *Typedef:
+		case *Var:
 			if dt.Index != -1 {
-				return nil, fmt.Errorf("type %s: index %d is not -1", dt, dt.Index)
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
 			}
+			t.Tags = append(t.Tags, dt.Value)
+
+		case *Typedef:
+			if dt.Index != -1 {
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
+			}
+			t.Tags = append(t.Tags, dt.Value)
 
 		case composite:
-			if dt.Index >= len(t.members()) {
-				return nil, fmt.Errorf("type %s: index %d exceeds members of %s", dt, dt.Index, t)
+			if dt.Index >= 0 {
+				members := t.members()
+				if dt.Index >= len(members) {
+					return nil, fmt.Errorf("type %s: component idx %d exceeds members of %s", dt, dt.Index, t)
+				}
+
+				members[dt.Index].Tags = append(members[dt.Index].Tags, dt.Value)
+				continue
+			}
+
+			if dt.Index == -1 {
+				switch t2 := t.(type) {
+				case *Struct:
+					t2.Tags = append(t2.Tags, dt.Value)
+				case *Union:
+					t2.Tags = append(t2.Tags, dt.Value)
+				}
+
+				continue
 			}
 
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		case *Func:
 			fp, ok := t.Type.(*FuncProto)
 			if !ok {
 				return nil, fmt.Errorf("type %s: %s is not a FuncProto", dt, t.Type)
 			}
 
-			if dt.Index >= len(fp.Params) {
-				return nil, fmt.Errorf("type %s: index %d exceeds params of %s", dt, dt.Index, t)
+			// Ensure the number of argument tag lists equals the number of arguments
+			if len(t.ParamTags) == 0 {
+				t.ParamTags = make([][]string, len(fp.Params))
 			}
 
+			if dt.Index >= 0 {
+				if dt.Index >= len(fp.Params) {
+					return nil, fmt.Errorf("type %s: component idx %d exceeds params of %s", dt, dt.Index, t)
+				}
+
+				t.ParamTags[dt.Index] = append(t.ParamTags[dt.Index], dt.Value)
+				continue
+			}
+
+			if dt.Index == -1 {
+				t.Tags = append(t.Tags, dt.Value)
+				continue
+			}
+
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		default:
 			return nil, fmt.Errorf("type %s: decl tag for type %s is not supported", dt, t)
 		}