fix: support pointer to structs in marshal/unmarshal

For siderolabs/talos#6057

Signed-off-by: Dmitriy Matrenichev <dmitry.matrenichev@siderolabs.com>

marshal.go

@@ -218,6 +218,17 @@ func (m *marshaller) encodeValue(num protowire.Number, val reflect.Value) {
 			return
 		}
 
+		// If the pointer is to a struct
+		if deref(val.Type()).Kind() == reflect.Struct {
+			b, ok := tryEncodeFunc(val)
+			if ok {
+				putTag(m, num, protowire.BytesType)
+				putBytes(m, b)
+
+				return
+			}
+		}
+
 		m.encodeValue(num, val.Elem())
 
 	case reflect.Interface:

marshal_test.go

@@ -489,16 +489,16 @@ func TestCustomEcnoders(t *testing.T) {
 		},
 		"should use custom encoder on pointer": {
 			testCustomEncodersDecoders(
-				encodeCustomEncoderStruct,
-				decodeCustomEncoderStruct,
+				encodeCustomEncoderStructPtr,
+				decodeCustomEncoderStructPtr,
 				OneFieldStruct[*CustomEncoderStruct]{
 					Field: &CustomEncoderStruct{
 						Value: 150,
 					},
 				},
 				OneFieldStruct[*CustomEncoderStruct]{
 					Field: &CustomEncoderStruct{
-						Value: 152,
+						Value: 156,
 					},
 				},
 			),
@@ -547,6 +547,21 @@ func decodeCustomEncoderStruct(slc []byte) (CustomEncoderStruct, error) {
 	}, err
 }
 
+func encodeCustomEncoderStructPtr(v *CustomEncoderStruct) ([]byte, error) {
+	return []byte(strconv.Itoa(v.Value + 3)), nil
+}
+
+func decodeCustomEncoderStructPtr(slc []byte) (*CustomEncoderStruct, error) {
+	res, err := strconv.Atoi(string(slc))
+	if err != nil {
+		return &CustomEncoderStruct{}, err
+	}
+
+	return &CustomEncoderStruct{
+		Value: res + 3,
+	}, err
+}
+
 func testCustomEncodersDecoders[V any, T any](
 	enc func(T) ([]byte, error),
 	dec func([]byte) (T, error),

unmarshal.go

@@ -349,7 +349,7 @@ func instantiate(dst reflect.Value) error {
 	return nil
 }
 
-//nolint:cyclop,gocyclo
+//nolint:cyclop,gocognit,gocyclo
 func unmarshalBytes(dst reflect.Value, value complexValue) (err error) {
 	defer func() {
 		if err != nil {
@@ -413,6 +413,18 @@ func unmarshalBytes(dst reflect.Value, value complexValue) (err error) {
 			}
 		}
 
+		// If the pointer is to a struct
+		if deref(dst.Type()).Kind() == reflect.Struct {
+			ok, err := tryDecodeFunc(bytes, dst)
+			if err != nil {
+				return err
+			}
+
+			if ok {
+				return nil
+			}
+		}
+
 		return unmarshalBytes(dst.Elem(), value)
 
 	case reflect.String: