fix: xml marshal byte array fields as vCenter does

Go's encoding/xml package and vCenter marshal '[]byte' differently:
- Go encodes the entire array in a single xml element, for example:
  <foo>
    <bar>hello</bar>
  </foo>

- vCenter encodes each byte of the array in its own xml element, example with same data as above:
  <foo>
    <bar>104</bar>
    <bar>101</bar>
    <bar>108</bar>
    <bar>108</bar>
    <bar>111</bar>
  </foo>

This behavior is hardwired, see the xml/encoding source for the handful of []byte special cases along the lines of:

  if reflect.Slice && slice element type == reflect.Uint8

While govmomi maintains a fork of Go's xml/encoding package, we prefer to further diverge.
Proposed solution is to use a 'ByteSlice' type that implements xml marshaling as vCenter does,
but otherwise behaves just as '[]byte' does.

Commits that follow enhance vcsim and govc support around various []byte fields.

Fixes #1977
Fixes #3469

gen/gen_from_vmodl.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2014 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -86,6 +86,9 @@ def var_type
       when "dateTime"
         type ="time.Time"
       when "byte"
+        if slice?
+          return "types.ByteSlice"
+        end
       when "double"
         type ="float64"
       when "float"

gen/vim_wsdl.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2021 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -321,6 +321,10 @@ def var_type
           self.need_omitempty = false
         end
       when "byte"
+        if slice?
+          prefix = ""
+          t = "#{pkg}ByteSlice"
+        end
       when "double"
         t = "float64"
       when "float"

govc/object/collect.go

@@ -18,6 +18,7 @@ package object
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -111,8 +112,8 @@ Examples:
   govc object.collect -R create-filter-request.xml -O # convert filter to Go code
   govc object.collect -s vm/my-vm summary.runtime.host | xargs govc ls -L # inventory path of VM's host
   govc object.collect -dump -o "network/VM Network" # output Managed Object structure as Go code
-  govc object.collect -json $vm config | \ # use -json + jq to search array elements
-    jq -r '.[] | select(.val.hardware.device[].macAddress == "00:0c:29:0c:73:c0") | .val.name'`, atable)
+  govc object.collect -json -s $vm config | \ # use -json + jq to search array elements
+    jq -r 'select(.hardware.device[].macAddress == "00:50:56:99:c4:27") | .name'`, atable)
 }
 
 var stringer = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
@@ -123,11 +124,11 @@ type change struct {
 }
 
 func (pc *change) MarshalJSON() ([]byte, error) {
-	if len(pc.cmd.kind) == 0 {
+	if len(pc.cmd.kind) == 0 && !pc.cmd.simple {
 		return json.Marshal(pc.Update.ChangeSet)
 	}
 
-	return json.Marshal(pc.Update)
+	return json.Marshal(pc.Dump())
 }
 
 func (pc *change) output(name string, rval reflect.Value, rtype reflect.Type) {
@@ -154,6 +155,15 @@ func (pc *change) output(name string, rval reflect.Value, rtype reflect.Type) {
 
 		etype := rtype.Elem()
 
+		if etype.Kind() == reflect.Uint8 {
+			if v, ok := rval.Interface().(types.ByteSlice); ok {
+				s = base64.StdEncoding.EncodeToString(v) // ArrayOfByte
+			} else {
+				s = fmt.Sprintf("%x", rval.Interface().([]byte)) // ArrayOfBase64Binary
+			}
+			break
+		}
+
 		if etype.Kind() != reflect.Interface && etype.Kind() != reflect.Struct || etype.Implements(stringer) {
 			var val []string
 

govc/test/host.bats

@@ -159,6 +159,17 @@ load test_helper
 
     run govc host.storage.info -t hba
     assert_success
+
+    names=$(govc host.storage.info -json | jq -r .storageDeviceInfo.scsiLun[].alternateName[].data)
+    # given data is hex encoded []byte and:
+    #   [0] == encoding
+    #   [1] == type
+    #   [2] == ?
+    #   [3] == length
+    # validate name is at least 2 char x 4
+    for name in $names; do
+      [ "${#name}" -ge 8 ]
+    done
 }
 
 @test "host.options" {

govc/test/object.bats

@@ -338,6 +338,28 @@ load test_helper
   assert_success
 }
 
+@test "object.collect bytes" {
+  vcsim_env
+
+  host=$(govc find / -type h | head -1)
+
+  # ArrayOfByte with PEM encoded cert
+  govc object.collect -s "$host" config.certificate | \
+    base64 -d | openssl x509 -text
+
+  # []byte field with PEM encoded cert
+  govc object.collect -s -json "$host" config | jq -r .certificate | \
+    base64 -d | openssl x509 -text
+
+  # ArrayOfByte with DER encoded cert
+  govc object.collect -s CustomizationSpecManager:CustomizationSpecManager encryptionKey | \
+    base64 -d | openssl x509 -inform DER -text
+
+  # []byte field with DER encoded cert
+  govc object.collect -o -json CustomizationSpecManager:CustomizationSpecManager | jq -r .encryptionKey | \
+    base64 -d | openssl x509 -inform DER -text
+}
+
 @test "object.collect view" {
   vcsim_env -dc 2 -folder 1
 

object/host_system_test.go

@@ -1,9 +1,12 @@
 /*
-Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+Copyright (c) 2019-2024 VMware, Inc. All Rights Reserved.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
+
+http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,12 +17,16 @@ limitations under the License.
 package object_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/pem"
 	"testing"
 
 	"github.com/vmware/govmomi/find"
 	"github.com/vmware/govmomi/simulator"
+	"github.com/vmware/govmomi/simulator/esx"
 	"github.com/vmware/govmomi/vim25"
+	"github.com/vmware/govmomi/vim25/mo"
 )
 
 func TestHostSystemManagementIPs(t *testing.T) {
@@ -59,3 +66,26 @@ func TestHostSystemManagementIPs(t *testing.T) {
 		}
 	})
 }
+
+func TestHostSystemConfig(t *testing.T) {
+	simulator.Test(func(ctx context.Context, c *vim25.Client) {
+		host, err := find.NewFinder(c).HostSystem(ctx, "DC0_C0_H0")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var props mo.HostSystem
+		if err := host.Properties(ctx, host.Reference(), []string{"config"}, &props); err != nil {
+			t.Fatal(err)
+		}
+
+		if !bytes.Equal(props.Config.Certificate, esx.HostConfigInfo.Certificate) {
+			t.Errorf("certificate=%s", string(props.Config.Certificate))
+		}
+
+		b, _ := pem.Decode(props.Config.Certificate)
+		if b == nil {
+			t.Error("failed to parse certificate")
+		}
+	})
+}

simulator/customization_spec_manager.go

@@ -1,11 +1,11 @@
 /*
-Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+Copyright (c) 2019-2024 VMware, Inc. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,10 +17,12 @@ limitations under the License.
 package simulator
 
 import (
+	"encoding/pem"
 	"fmt"
 	"sync/atomic"
 	"time"
 
+	"github.com/vmware/govmomi/simulator/internal"
 	"github.com/vmware/govmomi/vim25/methods"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
@@ -169,7 +171,7 @@ var DefaultCustomizationSpec = []types.CustomizationSpecItem{
 					},
 				},
 			},
-			EncryptionKey: []uint8{0x30},
+			EncryptionKey: nil,
 		},
 	},
 	{
@@ -236,7 +238,7 @@ var DefaultCustomizationSpec = []types.CustomizationSpecItem{
 					},
 				},
 			},
-			EncryptionKey: []uint8{0x30},
+			EncryptionKey: nil,
 		},
 	},
 }
@@ -249,6 +251,13 @@ type CustomizationSpecManager struct {
 
 func (m *CustomizationSpecManager) init(r *Registry) {
 	m.items = DefaultCustomizationSpec
+
+	// Real VC is different DN, X509v3 extensions, etc.
+	// This is still useful for testing []byte of DER encoded cert over SOAP
+	if len(m.EncryptionKey) == 0 {
+		block, _ := pem.Decode(internal.LocalhostCert)
+		m.EncryptionKey = block.Bytes
+	}
 }
 
 var customizeNameCounter uint64

simulator/esx/host_config_info.go

@@ -16,7 +16,10 @@ limitations under the License.
 
 package esx
 
-import "github.com/vmware/govmomi/vim25/types"
+import (
+	"github.com/vmware/govmomi/simulator/internal"
+	"github.com/vmware/govmomi/vim25/types"
+)
 
 // HostConfigInfo is the default template for the HostSystem config property.
 // Capture method:
@@ -998,7 +1001,7 @@ var HostConfigInfo = types.HostConfigInfo{
 	Ipmi:              (*types.HostIpmiInfo)(nil),
 	SslThumbprintInfo: (*types.HostSslThumbprintInfo)(nil),
 	SslThumbprintData: nil,
-	Certificate:       []uint8{0x31, 0x30},
+	Certificate:       internal.LocalhostCert,
 	PciPassthruInfo:   nil,
 	AuthenticationManagerInfo: &types.HostAuthenticationManagerInfo{
 		AuthConfig: []types.BaseHostAuthenticationStoreInfo{

simulator/esx/host_storage_device_info.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2017-2023 VMware, Inc. All Rights Reserved.
+Copyright (c) 2017-2024 VMware, Inc. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -93,15 +93,15 @@ var HostStorageDeviceInfo = types.HostStorageDeviceInfo{
 				{
 					Namespace:   "GENERIC_VPD",
 					NamespaceId: 0x5,
-					Data:        []uint8{0x2d, 0x37, 0x39},
+					Data:        []uint8{0x0, 0x0, 0x0, 0x4, 0x0, 0xb0, 0xb1, 0xb2},
 				},
 				{
 					Namespace:   "GENERIC_VPD",
 					NamespaceId: 0x5,
-					Data:        []uint8{0x30},
+					Data:        []uint8{0x0, 0xb2, 0x0, 0x4, 0x1, 0x60, 0x0, 0x0},
 				},
 			},
-			StandardInquiry:  []uint8{0x30},
+			StandardInquiry:  []uint8{0x0, 0x0, 0x6, 0x2, 0x1f, 0x0, 0x0, 0x72},
 			QueueDepth:       0,
 			OperationalState: []string{"ok"},
 			Capabilities:     &types.ScsiLunCapabilities{},
@@ -143,15 +143,15 @@ var HostStorageDeviceInfo = types.HostStorageDeviceInfo{
 					{
 						Namespace:   "GENERIC_VPD",
 						NamespaceId: 0x5,
-						Data:        []uint8{0x2d, 0x37, 0x39},
+						Data:        []uint8{0x0, 0x0, 0x0, 0x4, 0x0, 0xb0, 0xb1, 0xb2},
 					},
 					{
 						Namespace:   "GENERIC_VPD",
 						NamespaceId: 0x5,
-						Data:        []uint8{0x30},
+						Data:        []uint8{0x0, 0xb2, 0x0, 0x4, 0x1, 0x60, 0x0, 0x0},
 					},
 				},
-				StandardInquiry:  []uint8{0x30},
+				StandardInquiry:  []uint8{0x0, 0x0, 0x6, 0x2, 0x1f, 0x0, 0x0, 0x72},
 				QueueDepth:       1024,
 				OperationalState: []string{"ok"},
 				Capabilities:     &types.ScsiLunCapabilities{},

simulator/property_collector.go

@@ -131,6 +131,10 @@ func wrapValue(rval reflect.Value, rtype reflect.Type) interface{} {
 			pval = &types.ArrayOfByte{
 				Byte: v,
 			}
+		case types.ByteSlice:
+			pval = &types.ArrayOfByte{
+				Byte: v,
+			}
 		case []int16:
 			pval = &types.ArrayOfShort{
 				Short: v,

vim25/mo/mo.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -210,7 +210,7 @@ type CustomizationSpecManager struct {
 	Self types.ManagedObjectReference `json:"self"`
 
 	Info          []types.CustomizationSpecInfo `json:"info"`
-	EncryptionKey []byte                        `json:"encryptionKey"`
+	EncryptionKey types.ByteSlice               `json:"encryptionKey"`
 }
 
 func (m CustomizationSpecManager) Reference() types.ManagedObjectReference {