btf: Export and marshal DeclTags

This commit makes the DeclTag BTF type exported. The DeclTag has been
unexported because its encoding is a bit tricky.

DeclTags on the wire refer to a type by its ID, and an component index.
Unlike other BTF types, the type ID isn't always the actual "target",
instead if the component index is not `-1` it targets a child object of
the type. For unions and structs, the `Target` can be a member, for
functions it can be a FuncParam. Upon marshalling the correct component
index is calculated. This allows for modifying BTF types or creating
them without having to think about the component index.

In addition, the BTF marshalling logic that creates a minimal BTF blob
for the program has been updated to include DeclTags associated with
FuncInfos. Before we were not adding the DeclTags to the generated BTF
even if the ELF included them.

Signed-off-by: Dylan Reimerink <dylan.reimerink@isovalent.com>

btf/marshal.go

@@ -521,7 +521,7 @@ func (e *encoder) deflateEnum64(raw *rawType, enum *Enum) (err error) {
 			})
 		}
 
-		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members})
+		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members, nil, nil})
 	}
 
 	raw.SetKind(kindEnum64)

btf/marshal_test.go

@@ -24,7 +24,7 @@ func TestBuilderMarshal(t *testing.T) {
 		(*Void)(nil),
 		typ,
 		&Pointer{typ},
-		&Typedef{"baz", typ},
+		&Typedef{"baz", typ, nil},
 	}
 
 	b, err := NewBuilder(want)
@@ -65,7 +65,7 @@ func TestBuilderAdd(t *testing.T) {
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(2)), qt.Commentf("Adding a type twice returns different ids"))
 
-	id, err = b.Add(&Typedef{"baz", i})
+	id, err = b.Add(&Typedef{"baz", i, nil})
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(3)))
 }
@@ -144,6 +144,7 @@ func TestMarshalEnum64(t *testing.T) {
 			{Name: "A", Type: placeholder},
 			{Name: "B", Type: placeholder},
 		},
+		MemberTags: make([][]string, 2),
 	}))
 }
 

btf/traversal.go

@@ -41,7 +41,7 @@ func children(typ Type, yield func(child *Type) bool) bool {
 	// do its work. This avoids allocating intermediate slices from walk() on
 	// the heap.
 	switch v := typ.(type) {
-	case *Void, *Int, *Enum, *Fwd, *Float:
+	case *Void, *Int, *Enum, *Fwd, *Float, *declTag:
 		// No children to traverse.
 	case *Pointer:
 		if !yield(&v.Target) {
@@ -59,12 +59,41 @@ func children(typ Type, yield func(child *Type) bool) bool {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+			if len(v.MemberTags) > i {
+				for _, t := range v.MemberTags[i] {
+					var tag Type = &declTag{t, v, i}
+					if !yield(&tag) {
+						return false
+					}
+				}
+			}
+		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
 		}
 	case *Union:
 		for i := range v.Members {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+
+			if len(v.MemberTags) > i {
+				for _, t := range v.MemberTags[i] {
+					var tag Type = &declTag{t, v, i}
+					if !yield(&tag) {
+						return false
+					}
+				}
+			}
+		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
 		}
 	case *Typedef:
 		if !yield(&v.Type) {
@@ -86,6 +115,22 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
+		}
+		if fp, ok := v.Type.(*FuncProto); ok {
+			for i, param := range fp.Params {
+				for _, t := range param.Tags {
+					var tag Type = &declTag{t, v, i}
+					if !yield(&tag) {
+						return false
+					}
+				}
+			}
+		}
 	case *FuncProto:
 		if !yield(&v.Return) {
 			return false
@@ -99,16 +144,18 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
+		}
 	case *Datasec:
 		for i := range v.Vars {
 			if !yield(&v.Vars[i].Type) {
 				return false
 			}
 		}
-	case *declTag:
-		if !yield(&v.Type) {
-			return false
-		}
 	case *TypeTag:
 		if !yield(&v.Type) {
 			return false

btf/types.go

@@ -66,7 +66,6 @@ var (
 	_ Type = (*Var)(nil)
 	_ Type = (*Datasec)(nil)
 	_ Type = (*Float)(nil)
-	_ Type = (*declTag)(nil)
 	_ Type = (*TypeTag)(nil)
 	_ Type = (*cycle)(nil)
 )
@@ -167,8 +166,10 @@ func (arr *Array) copy() Type {
 type Struct struct {
 	Name string
 	// The size of the struct including padding, in bytes
-	Size    uint32
-	Members []Member
+	Size       uint32
+	Members    []Member
+	Tags       []string
+	MemberTags [][]string
 }
 
 func (s *Struct) Format(fs fmt.State, verb rune) {
@@ -193,8 +194,10 @@ func (s *Struct) members() []Member {
 type Union struct {
 	Name string
 	// The size of the union including padding, in bytes.
-	Size    uint32
-	Members []Member
+	Size       uint32
+	Members    []Member
+	Tags       []string
+	MemberTags [][]string
 }
 
 func (u *Union) Format(fs fmt.State, verb rune) {
@@ -334,6 +337,7 @@ func (f *Fwd) matches(typ Type) bool {
 type Typedef struct {
 	Name string
 	Type Type
+	Tags []string
 }
 
 func (td *Typedef) Format(fs fmt.State, verb rune) {
@@ -403,6 +407,7 @@ type Func struct {
 	Name    string
 	Type    Type
 	Linkage FuncLinkage
+	Tags    []string
 }
 
 func FuncMetadata(ins *asm.Instruction) *Func {
@@ -449,13 +454,15 @@ func (fp *FuncProto) copy() Type {
 type FuncParam struct {
 	Name string
 	Type Type
+	Tags []string
 }
 
 // Var is a global variable.
 type Var struct {
 	Name    string
 	Type    Type
 	Linkage VarLinkage
+	Tags    []string
 }
 
 func (v *Var) Format(fs fmt.State, verb rune) {
@@ -522,8 +529,8 @@ func (f *Float) copy() Type {
 
 // declTag associates metadata with a declaration.
 type declTag struct {
-	Type  Type
 	Value string
+	Type  Type
 	// The index this tag refers to in the target type. For composite types,
 	// a value of -1 indicates that the tag refers to the whole type. Otherwise
 	// it indicates which member or argument the tag applies to.
@@ -918,7 +925,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err)
 			}
-			typ = &Struct{name, header.Size(), members}
+			typ = &Struct{name, header.Size(), members, nil, make([][]string, len(members))}
 
 		case kindUnion:
 			vlen := header.Vlen()
@@ -935,7 +942,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("union %s (id %d): %w", name, id, err)
 			}
-			typ = &Union{name, header.Size(), members}
+			typ = &Union{name, header.Size(), members, nil, make([][]string, len(members))}
 
 		case kindEnum:
 			vlen := header.Vlen()
@@ -968,7 +975,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = &Fwd{name, header.FwdKind()}
 
 		case kindTypedef:
-			typedef := &Typedef{name, nil}
+			typedef := &Typedef{name, nil, nil}
 			fixup(header.Type(), &typedef.Type)
 			typ = typedef
 
@@ -988,7 +995,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = restrict
 
 		case kindFunc:
-			fn := &Func{name, nil, header.Linkage()}
+			fn := &Func{name, nil, header.Linkage(), nil}
 			fixup(header.Type(), &fn.Type)
 			typ = fn
 
@@ -1030,7 +1037,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 				return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err)
 			}
 
-			v := &Var{name, nil, VarLinkage(bVariable.Linkage)}
+			v := &Var{name, nil, VarLinkage(bVariable.Linkage), nil}
 			fixup(header.Type(), &v.Type)
 			typ = v
 
@@ -1074,9 +1081,9 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 				return nil, fmt.Errorf("type id %d: index exceeds int", id)
 			}
 
-			dt := &declTag{nil, name, int(int32(btfIndex))}
+			dt := &declTag{name, nil, int(int32(btfIndex))}
 			fixup(header.Type(), &dt.Type)
-			typ = dt
+			typ = &Void{}
 
 			declTags = append(declTags, dt)
 
@@ -1142,26 +1149,77 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 
 	for _, dt := range declTags {
 		switch t := dt.Type.(type) {
-		case *Var, *Typedef:
+		case *Var:
+			if dt.Index != -1 {
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
+			}
+			t.Tags = append(t.Tags, dt.Value)
+
+		case *Typedef:
 			if dt.Index != -1 {
-				return nil, fmt.Errorf("type %s: index %d is not -1", dt, dt.Index)
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
 			}
+			t.Tags = append(t.Tags, dt.Value)
 
 		case composite:
-			if dt.Index >= len(t.members()) {
-				return nil, fmt.Errorf("type %s: index %d exceeds members of %s", dt, dt.Index, t)
+			switch t2 := t.(type) {
+			case *Struct:
+				if dt.Index >= 0 {
+					members := t.members()
+					if dt.Index >= len(members) {
+						return nil, fmt.Errorf("type %s: component idx %d exceeds members of %s", dt, dt.Index, t)
+					}
+
+					t2.MemberTags[dt.Index] = append(t2.MemberTags[dt.Index], dt.Value)
+					continue
+				}
+
+				if dt.Index == -1 {
+					t2.Tags = append(t2.Tags, dt.Value)
+					continue
+				}
+
+			case *Union:
+				if dt.Index >= 0 {
+					members := t.members()
+					if dt.Index >= len(members) {
+						return nil, fmt.Errorf("type %s: component idx %d exceeds members of %s", dt, dt.Index, t)
+					}
+
+					t2.MemberTags[dt.Index] = append(t2.MemberTags[dt.Index], dt.Value)
+					continue
+				}
+
+				if dt.Index == -1 {
+					t2.Tags = append(t2.Tags, dt.Value)
+					continue
+				}
 			}
 
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		case *Func:
 			fp, ok := t.Type.(*FuncProto)
 			if !ok {
 				return nil, fmt.Errorf("type %s: %s is not a FuncProto", dt, t.Type)
 			}
 
-			if dt.Index >= len(fp.Params) {
-				return nil, fmt.Errorf("type %s: index %d exceeds params of %s", dt, dt.Index, t)
+			if dt.Index >= 0 {
+				if dt.Index >= len(fp.Params) {
+					return nil, fmt.Errorf("type %s: component idx %d exceeds params of %s", dt, dt.Index, t)
+				}
+
+				fp.Params[dt.Index].Tags = append(fp.Params[dt.Index].Tags, dt.Value)
+				continue
 			}
 
+			if dt.Index == -1 {
+				t.Tags = append(t.Tags, dt.Value)
+				continue
+			}
+
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		default:
 			return nil, fmt.Errorf("type %s: decl tag for type %s is not supported", dt, t)
 		}

btf/types_test.go

@@ -174,7 +174,6 @@ func TestType(t *testing.T) {
 			}
 		},
 		func() Type { return &Float{} },
-		func() Type { return &declTag{Type: &Void{}} },
 		func() Type { return &TypeTag{Type: &Void{}} },
 		func() Type { return &cycle{&Void{}} },
 	}
@@ -213,7 +212,8 @@ func TestType(t *testing.T) {
 
 func TestTagMarshaling(t *testing.T) {
 	for _, typ := range []Type{
-		&declTag{&Struct{Members: []Member{}}, "foo", -1},
+		&Struct{Members: []Member{}, Tags: []string{"foo"}, MemberTags: make([][]string, 0)},
+		&Struct{Members: []Member{{Name: "field1", Type: &Int{}}}, Tags: nil, MemberTags: [][]string{{"foo"}}},
 		&TypeTag{&Int{}, "foo"},
 	} {
 		t.Run(fmt.Sprint(typ), func(t *testing.T) {

btf/workarounds_test.go

@@ -17,7 +17,7 @@ func TestDatasecResolveWorkaround(t *testing.T) {
 	i := &Int{Size: 1}
 
 	for _, typ := range []Type{
-		&Typedef{"foo", i},
+		&Typedef{"foo", i, nil},
 		&Volatile{i},
 		&Const{i},
 		&Restrict{i},