btf: Export and marshal DeclTags

This commit makes the DeclTag BTF type exported. The DeclTag has been
unexported because its encoding is a bit tricky.

DeclTags on the wire refer to a type by its ID, and an component index.
Unlike other BTF types, the type ID isn't always the actual "target",
instead if the component index is not `-1` it targets a child object of
the type. This commit adds a `Tags` field to all types that can have
DeclTags including Members.

Exception is function parameters. Due to the func -> funcProto ->
parameter encoding, it is possible that BTF deduplication reuses the
same funcProto for multiple functions. The declTags point to the func to
annotate the arguments. Thus, adding a `Tags` field to Param would
cause inaccurate BTF marshalling. Instead, a `ParamTags` field is added
the func as `[][]string` to match the write format more closely to
avoid issues caused by BTF deduplication.

In addition, the BTF marshalling logic that creates a minimal BTF blob
for the program has been updated to include DeclTags associated with
FuncInfos. Before we were not adding the DeclTags to the generated BTF
even if the ELF included them.

Signed-off-by: Dylan Reimerink <dylan.reimerink@isovalent.com>

btf/marshal.go

@@ -521,7 +521,7 @@ func (e *encoder) deflateEnum64(raw *rawType, enum *Enum) (err error) {
 			})
 		}
 
-		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members})
+		return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members, nil})
 	}
 
 	raw.SetKind(kindEnum64)

btf/marshal_test.go

@@ -24,7 +24,7 @@ func TestBuilderMarshal(t *testing.T) {
 		(*Void)(nil),
 		typ,
 		&Pointer{typ},
-		&Typedef{"baz", typ},
+		&Typedef{"baz", typ, nil},
 	}
 
 	b, err := NewBuilder(want)
@@ -65,7 +65,7 @@ func TestBuilderAdd(t *testing.T) {
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(2)), qt.Commentf("Adding a type twice returns different ids"))
 
-	id, err = b.Add(&Typedef{"baz", i})
+	id, err = b.Add(&Typedef{"baz", i, nil})
 	qt.Assert(t, qt.IsNil(err))
 	qt.Assert(t, qt.Equals(id, TypeID(3)))
 }

btf/traversal.go

@@ -41,7 +41,7 @@ func children(typ Type, yield func(child *Type) bool) bool {
 	// do its work. This avoids allocating intermediate slices from walk() on
 	// the heap.
 	switch v := typ.(type) {
-	case *Void, *Int, *Enum, *Fwd, *Float:
+	case *Void, *Int, *Enum, *Fwd, *Float, *declTag:
 		// No children to traverse.
 	case *Pointer:
 		if !yield(&v.Target) {
@@ -59,17 +59,47 @@ func children(typ Type, yield func(child *Type) bool) bool {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+			for _, t := range v.Members[i].Tags {
+				var tag Type = &declTag{t, v, i}
+				if !yield(&tag) {
+					return false
+				}
+			}
+		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
 		}
 	case *Union:
 		for i := range v.Members {
 			if !yield(&v.Members[i].Type) {
 				return false
 			}
+			for _, t := range v.Members[i].Tags {
+				var tag Type = &declTag{t, v, i}
+				if !yield(&tag) {
+					return false
+				}
+			}
+		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
 		}
 	case *Typedef:
 		if !yield(&v.Type) {
 			return false
 		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
+		}
 	case *Volatile:
 		if !yield(&v.Type) {
 			return false
@@ -86,6 +116,25 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
+		}
+		if fp, ok := v.Type.(*FuncProto); ok {
+			for i := range fp.Params {
+				if len(v.ParamTags) <= i {
+					continue
+				}
+				for _, t := range v.ParamTags[i] {
+					var tag Type = &declTag{t, v, i}
+					if !yield(&tag) {
+						return false
+					}
+				}
+			}
+		}
 	case *FuncProto:
 		if !yield(&v.Return) {
 			return false
@@ -99,16 +148,18 @@ func children(typ Type, yield func(child *Type) bool) bool {
 		if !yield(&v.Type) {
 			return false
 		}
+		for _, t := range v.Tags {
+			var tag Type = &declTag{t, v, -1}
+			if !yield(&tag) {
+				return false
+			}
+		}
 	case *Datasec:
 		for i := range v.Vars {
 			if !yield(&v.Vars[i].Type) {
 				return false
 			}
 		}
-	case *declTag:
-		if !yield(&v.Type) {
-			return false
-		}
 	case *TypeTag:
 		if !yield(&v.Type) {
 			return false

btf/types.go

@@ -66,7 +66,6 @@ var (
 	_ Type = (*Var)(nil)
 	_ Type = (*Datasec)(nil)
 	_ Type = (*Float)(nil)
-	_ Type = (*declTag)(nil)
 	_ Type = (*TypeTag)(nil)
 	_ Type = (*cycle)(nil)
 )
@@ -169,6 +168,7 @@ type Struct struct {
 	// The size of the struct including padding, in bytes
 	Size    uint32
 	Members []Member
+	Tags    []string
 }
 
 func (s *Struct) Format(fs fmt.State, verb rune) {
@@ -195,6 +195,7 @@ type Union struct {
 	// The size of the union including padding, in bytes.
 	Size    uint32
 	Members []Member
+	Tags    []string
 }
 
 func (u *Union) Format(fs fmt.State, verb rune) {
@@ -247,6 +248,7 @@ type Member struct {
 	Type         Type
 	Offset       Bits
 	BitfieldSize Bits
+	Tags         []string
 }
 
 // Enum lists possible values.
@@ -334,6 +336,7 @@ func (f *Fwd) matches(typ Type) bool {
 type Typedef struct {
 	Name string
 	Type Type
+	Tags []string
 }
 
 func (td *Typedef) Format(fs fmt.State, verb rune) {
@@ -400,9 +403,11 @@ func (r *Restrict) copy() Type {
 
 // Func is a function definition.
 type Func struct {
-	Name    string
-	Type    Type
-	Linkage FuncLinkage
+	Name      string
+	Type      Type
+	Linkage   FuncLinkage
+	Tags      []string
+	ParamTags [][]string
 }
 
 func FuncMetadata(ins *asm.Instruction) *Func {
@@ -456,6 +461,7 @@ type Var struct {
 	Name    string
 	Type    Type
 	Linkage VarLinkage
+	Tags    []string
 }
 
 func (v *Var) Format(fs fmt.State, verb rune) {
@@ -522,8 +528,8 @@ func (f *Float) copy() Type {
 
 // declTag associates metadata with a declaration.
 type declTag struct {
-	Type  Type
 	Value string
+	Type  Type
 	// The index this tag refers to in the target type. For composite types,
 	// a value of -1 indicates that the tag refers to the whole type. Otherwise
 	// it indicates which member or argument the tag applies to.
@@ -918,7 +924,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err)
 			}
-			typ = &Struct{name, header.Size(), members}
+			typ = &Struct{name, header.Size(), members, nil}
 
 		case kindUnion:
 			vlen := header.Vlen()
@@ -935,7 +941,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			if err != nil {
 				return nil, fmt.Errorf("union %s (id %d): %w", name, id, err)
 			}
-			typ = &Union{name, header.Size(), members}
+			typ = &Union{name, header.Size(), members, nil}
 
 		case kindEnum:
 			vlen := header.Vlen()
@@ -968,7 +974,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = &Fwd{name, header.FwdKind()}
 
 		case kindTypedef:
-			typedef := &Typedef{name, nil}
+			typedef := &Typedef{name, nil, nil}
 			fixup(header.Type(), &typedef.Type)
 			typ = typedef
 
@@ -988,7 +994,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 			typ = restrict
 
 		case kindFunc:
-			fn := &Func{name, nil, header.Linkage()}
+			fn := &Func{name, nil, header.Linkage(), nil, nil}
 			fixup(header.Type(), &fn.Type)
 			typ = fn
 
@@ -1030,7 +1036,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 				return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err)
 			}
 
-			v := &Var{name, nil, VarLinkage(bVariable.Linkage)}
+			v := &Var{name, nil, VarLinkage(bVariable.Linkage), nil}
 			fixup(header.Type(), &v.Type)
 			typ = v
 
@@ -1074,9 +1080,9 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 				return nil, fmt.Errorf("type id %d: index exceeds int", id)
 			}
 
-			dt := &declTag{nil, name, int(int32(btfIndex))}
+			dt := &declTag{name, nil, int(int32(btfIndex))}
 			fixup(header.Type(), &dt.Type)
-			typ = dt
+			typ = &Void{}
 
 			declTags = append(declTags, dt)
 
@@ -1142,26 +1148,69 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt
 
 	for _, dt := range declTags {
 		switch t := dt.Type.(type) {
-		case *Var, *Typedef:
+		case *Var:
 			if dt.Index != -1 {
-				return nil, fmt.Errorf("type %s: index %d is not -1", dt, dt.Index)
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
 			}
+			t.Tags = append(t.Tags, dt.Value)
+
+		case *Typedef:
+			if dt.Index != -1 {
+				return nil, fmt.Errorf("type %s: component idx %d is not -1", dt, dt.Index)
+			}
+			t.Tags = append(t.Tags, dt.Value)
 
 		case composite:
-			if dt.Index >= len(t.members()) {
-				return nil, fmt.Errorf("type %s: index %d exceeds members of %s", dt, dt.Index, t)
+			if dt.Index >= 0 {
+				members := t.members()
+				if dt.Index >= len(members) {
+					return nil, fmt.Errorf("type %s: component idx %d exceeds members of %s", dt, dt.Index, t)
+				}
+
+				members[dt.Index].Tags = append(members[dt.Index].Tags, dt.Value)
+				continue
+			}
+
+			if dt.Index == -1 {
+				switch t2 := t.(type) {
+				case *Struct:
+					t2.Tags = append(t2.Tags, dt.Value)
+				case *Union:
+					t2.Tags = append(t2.Tags, dt.Value)
+				}
+
+				continue
 			}
 
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		case *Func:
 			fp, ok := t.Type.(*FuncProto)
 			if !ok {
 				return nil, fmt.Errorf("type %s: %s is not a FuncProto", dt, t.Type)
 			}
 
-			if dt.Index >= len(fp.Params) {
-				return nil, fmt.Errorf("type %s: index %d exceeds params of %s", dt, dt.Index, t)
+			// Ensure the number of argument tag lists equals the number of arguments
+			if len(t.ParamTags) < len(fp.Params) {
+				t.ParamTags = append(t.ParamTags, make([][]string, len(fp.Params)-len(t.ParamTags))...)
 			}
 
+			if dt.Index >= 0 {
+				if dt.Index >= len(fp.Params) {
+					return nil, fmt.Errorf("type %s: component idx %d exceeds params of %s", dt, dt.Index, t)
+				}
+
+				t.ParamTags[dt.Index] = append(t.ParamTags[dt.Index], dt.Value)
+				continue
+			}
+
+			if dt.Index == -1 {
+				t.Tags = append(t.Tags, dt.Value)
+				continue
+			}
+
+			return nil, fmt.Errorf("type %s: decl tag for type %s has invalid component idx", dt, t)
+
 		default:
 			return nil, fmt.Errorf("type %s: decl tag for type %s is not supported", dt, t)
 		}

btf/types_test.go

@@ -174,7 +174,6 @@ func TestType(t *testing.T) {
 			}
 		},
 		func() Type { return &Float{} },
-		func() Type { return &declTag{Type: &Void{}} },
 		func() Type { return &TypeTag{Type: &Void{}} },
 		func() Type { return &cycle{&Void{}} },
 	}
@@ -213,8 +212,19 @@ func TestType(t *testing.T) {
 
 func TestTagMarshaling(t *testing.T) {
 	for _, typ := range []Type{
-		&declTag{&Struct{Members: []Member{}}, "foo", -1},
 		&TypeTag{&Int{}, "foo"},
+		&Struct{Members: []Member{
+			{Type: &Int{}, Tags: []string{"bar"}},
+		}, Tags: []string{"foo"}},
+		&Union{Members: []Member{
+			{Type: &Int{}, Tags: []string{"bar"}},
+			{Type: &Int{}, Tags: []string{"baz"}},
+		}, Tags: []string{"foo"}},
+		&Func{Type: &FuncProto{Return: &Int{}, Params: []FuncParam{
+			{Name: "param1", Type: &Int{}},
+		}}, Tags: []string{"foo"}, ParamTags: [][]string{{"bar"}}},
+		&Var{Name: "var1", Type: &Int{}, Tags: []string{"foo"}},
+		&Typedef{Name: "baz", Type: &Int{}, Tags: []string{"foo"}},
 	} {
 		t.Run(fmt.Sprint(typ), func(t *testing.T) {
 			s := specFromTypes(t, []Type{typ})

btf/workarounds_test.go

@@ -17,7 +17,7 @@ func TestDatasecResolveWorkaround(t *testing.T) {
 	i := &Int{Size: 1}
 
 	for _, typ := range []Type{
-		&Typedef{"foo", i},
+		&Typedef{"foo", i, nil},
 		&Volatile{i},
 		&Const{i},
 		&Restrict{i},