Add size/md5 to Azure metadata

First this changes ore to write a JSON file that the Python side
parses, so we don't need to re-synthesize the URL on the Python
side.

Gather Azure Blob metadata (size+md5) and add that to our metadata,
This is mainly useful so that one can use the Azure API to "offline validate"
against our metadata snapshot.  I'd also like to add our sha256
checksum for consistency, but that can come later.

This is prep for coreos/stream-metadata-go#13

mantle/cmd/ore/azure/upload-blob-arm.go

@@ -100,7 +100,7 @@ func runUploadBlobARM(cmd *cobra.Command, args []string) {
 	//only use the first service key to avoid uploading twice
 	//see https://github.com/coreos/coreos-assembler/pull/1849
 	k := (*kr.Keys)[0]
-	if err := api.UploadBlob(ubo.storageacct, *k.Value, ubo.vhd, ubo.container, ubo.blob, ubo.overwrite); err != nil {
+	if _, err := api.UploadBlob(ubo.storageacct, *k.Value, ubo.vhd, ubo.container, ubo.blob, ubo.overwrite); err != nil {
 		plog.Fatalf("Uploading blob failed: %v", err)
 	}
 

mantle/cmd/ore/azure/upload-blob.go

@@ -15,13 +15,22 @@
 package azure
 
 import (
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"strings"
 
 	"github.com/Microsoft/azure-vhd-utils/vhdcore/validator"
 	"github.com/spf13/cobra"
 )
 
+type azureBlobMeta struct {
+	Url  string `json:"url"`
+	Size string `json:"size"`
+	Md5  string `json:"md5"`
+}
+
 var (
 	cmdUploadBlob = &cobra.Command{
 		Use:   "upload-blob storage-account container blob-name file",
@@ -37,14 +46,17 @@ var (
 		vhd         string
 		overwrite   bool
 		validate    bool
+		metaOutput  string
 	}
 )
 
 func init() {
 	bv := cmdUploadBlob.Flags().BoolVar
+	sv := cmdUploadBlob.Flags().StringVar
 
 	bv(&ubo.overwrite, "overwrite", false, "overwrite blob")
 	bv(&ubo.validate, "validate", true, "validate blob as VHD file")
+	sv(&ubo.metaOutput, "meta-output", "", "File to write JSON with uploaded blob properties")
 
 	Azure.AddCommand(cmdUploadBlob)
 }
@@ -79,11 +91,25 @@ func runUploadBlob(cmd *cobra.Command, args []string) {
 		plog.Fatalf("Fetching storage service keys failed: %v", err)
 	}
 
-	if err := api.UploadBlob(ubo.storageacct, kr.PrimaryKey, ubo.vhd, ubo.container, ubo.blob, ubo.overwrite); err != nil {
+	meta, err := api.UploadBlob(ubo.storageacct, kr.PrimaryKey, ubo.vhd, ubo.container, ubo.blob, ubo.overwrite)
+	if err != nil {
 		plog.Fatalf("Uploading blob failed: %v", err)
 	}
 
 	uri := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", ubo.storageacct, ubo.container, ubo.blob)
-
 	plog.Printf("Blob uploaded to %q", uri)
+
+	if ubo.metaOutput != "" {
+		metad := azureBlobMeta{
+			Url: uri,
+			// We always turn the size into a string to avoid JSON+64 bit issues
+			Size: fmt.Sprintf("%d", meta.FileMetaData.FileSize),
+			Md5:  hex.Dump(meta.FileMetaData.MD5Hash),
+		}
+		buf, err := json.Marshal(metad)
+		if err != nil {
+			plog.Fatalf("%v", err)
+		}
+		ioutil.WriteFile(ubo.metaOutput, buf, 0644)
+	}
 }

mantle/cmd/plume/prerelease.go

@@ -219,7 +219,7 @@ func uploadAzureBlob(spec *channelSpec, api *azure.API, storageKey storageservic
 		return nil
 	}
 
-	if err := api.UploadBlob(spec.Azure.StorageAccount, storageKey.PrimaryKey, vhdfile, container, blobName, false); err != nil {
+	if _, err := api.UploadBlob(spec.Azure.StorageAccount, storageKey.PrimaryKey, vhdfile, container, blobName, false); err != nil {
 		if _, ok := err.(azure.BlobExistsError); !ok {
 			return fmt.Errorf("uploading file %q to account %q container %q failed: %v", vhdfile, spec.Azure.StorageAccount, container, err)
 		}

mantle/cosa/cosa_v1.go

@@ -1,6 +1,6 @@
 package cosa
 
-// generated by "schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT
+// generated by "../tools/bin/schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT
 
 type AliyunImage struct {
 	ImageID string `json:"id"`
@@ -21,11 +21,18 @@ type Artifact struct {
 	UncompressedSize   int    `json:"uncompressed-size,omitempty"`
 }
 
+type Azureartifact struct {
+	Md5                    string `json:"md5,omitempty"`
+	RedundantNameOfTheFile string `json:"image,omitempty"`
+	SizeInBytes            int    `json:"size,omitempty"`
+	URL                    string `json:"url"`
+}
+
 type Build struct {
 	AlibabaAliyunUploads      []AliyunImage         `json:"aliyun,omitempty"`
 	Amis                      []Amis                `json:"amis,omitempty"`
 	Architecture              string                `json:"coreos-assembler.basearch,omitempty"`
-	Azure                     *Cloudartifact        `json:"azure,omitempty"`
+	Azure                     *Azureartifact        `json:"azure,omitempty"`
 	BuildArtifacts            *BuildArtifacts       `json:"images,omitempty"`
 	BuildID                   string                `json:"buildid"`
 	BuildRef                  string                `json:"ref,omitempty"`

mantle/cosa/schema_doc.go

@@ -13,16 +13,16 @@ var generatedSchemaJSON = `{
              "type":"string",
              "title":"Path"
             },
-           "sha256": {
-             "$id": "#/artifact/sha256",
-             "type":"string",
-             "title":"SHA256"
-            },
-           "size": {
-             "$id": "#/artifact/size",
-             "type":"integer",
-             "title":"Size in bytes"
-            },
+            "sha256": {
+              "$id": "#/artifact/sha256",
+              "type":"string",
+              "title":"SHA256"
+             },
+             "size": {
+              "$id": "#/artifact/size",
+              "type":"integer",
+              "title":"Size in bytes"
+             },
            "uncompressed-sha256": {
              "$id": "#/artifact/uncompressed-sha256",
              "type":"string",
@@ -35,10 +35,10 @@ var generatedSchemaJSON = `{
             }
           },
           "optional": [
-              "size",
-              "uncompressed-sha256",
-              "uncompressed-size"
-          ],
+            "size",
+            "uncompressed-sha256",
+            "uncompressed-size"
+        ],
           "required": [
               "path",
               "sha256"
@@ -71,25 +71,58 @@ var generatedSchemaJSON = `{
             }
           }
       },
-      "cloudartifact": {
-         "type": "object",
-         "required": [
-             "image",
-             "url"
-         ],
-         "properties": {
+      "azureartifact": {
+        "type": "object",
+        "required": [
+            "url"
+        ],
+        "optional": [
+          "image",
+          "size",
+          "md5"
+        ],
+        "properties": {
+          "url": {
+            "$id":"#/azureartifact/url",
+            "type":"string",
+            "title":"URL"
+           },
            "image": {
-             "$id":"#/cloudartifact/image",
-             "type":"string",
-             "title":"Image"
-            },
-           "url": {
-             "$id":"#/cloudartifact/url",
-             "type":"string",
-             "title":"URL"
-            }
-          }
-     },
+            "$id":"#/azureartifact/image",
+            "type":"string",
+            "title":"Redundant name of the file"
+           },
+           "md5": {
+            "$id":"#/azureartifact/md5",
+            "type":"string",
+            "title":"MD5"
+           },
+           "size": {
+            "$id": "#/azureartifact/size",
+            "type":"integer",
+            "title":"Size in bytes"
+           }
+         }
+      },
+      "cloudartifact": {
+        "type": "object",
+        "required": [
+            "image",
+            "url"
+        ],
+        "properties": {
+          "image": {
+            "$id":"#/cloudartifact/image",
+            "type":"string",
+            "title":"Image"
+           },
+          "url": {
+            "$id":"#/cloudartifact/url",
+            "type":"string",
+            "title":"URL"
+           }
+         }
+    },
      "git": {
          "type": "object",
          "required": [
@@ -686,7 +719,7 @@ var generatedSchemaJSON = `{
      "$id":"#/properties/azure",
      "type":"object",
      "title":"Azure",
-     "$ref": "#/definitions/cloudartifact"
+     "$ref": "#/definitions/azureartifact"
     },
    "gcp": {
      "$id":"#/properties/gcp",

mantle/platform/api/azure/storage_mit.go

@@ -96,29 +96,29 @@ func (a *API) GetBlob(storageaccount, storagekey, container, name string) (io.Re
 	return bsc.GetBlob(container, name)
 }
 
-// UploadBlob uploads vhd to the given storage account, container, and blob name.
-//
+// UploadBlob uploads vhd to the given storage account, container, and blob name,
+// returning the metadata for the blob.
 // It returns BlobExistsError if the blob exists and overwrite is not true.
-func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string, overwrite bool) error {
+func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string, overwrite bool) (*metadata.MetaData, error) {
 	ds, err := diskstream.CreateNewDiskStream(vhd)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer ds.Close()
 
 	sc, err := storage.NewClient(storageaccount, storagekey, a.opts.StorageEndpointSuffix, storage.DefaultAPIVersion, true)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	bsc := sc.GetBlobService()
 	if _, err = bsc.CreateContainerIfNotExists(container, storage.ContainerAccessTypePrivate); err != nil {
-		return err
+		return nil, err
 	}
 
 	blobExists, err := bsc.BlobExists(container, blob)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	resume := false
@@ -127,7 +127,7 @@ func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string
 		if !overwrite {
 			bm, err := getBlobMetaData(bsc, container, blob)
 			if err != nil {
-				return err
+				return nil, err
 			}
 			blobMetaData = bm
 			resume = true
@@ -137,32 +137,32 @@ func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string
 
 	localMetaData, err := getLocalVHDMetaData(vhd)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	var rangesToSkip []*common.IndexRange
 	if resume {
 		if errs := metadata.CompareMetaData(blobMetaData, localMetaData); len(errs) != 0 {
-			return multierror.Error(errs)
+			return nil, multierror.Error(errs)
 		}
 		ranges, err := getAlreadyUploadedBlobRanges(bsc, container, blob)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		rangesToSkip = ranges
 	} else {
 		if err := createBlob(bsc, container, blob, ds.GetSize(), localMetaData); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	uploadableRanges, err := upload.LocateUploadableRanges(ds, rangesToSkip, pageBlobPageSize)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	uploadableRanges, err = upload.DetectEmptyRanges(ds, uploadableRanges)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	cxt := &upload.DiskUploadContext{
@@ -177,7 +177,17 @@ func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string
 		MD5Hash:               localMetaData.FileMetaData.MD5Hash,
 	}
 
-	return upload.Upload(cxt)
+	err = upload.Upload(cxt)
+	if err != nil {
+		return nil, err
+	}
+
+	bm, err := getBlobMetaData(bsc, container, blob)
+	if err != nil {
+		return nil, err
+	}
+
+	return bm, nil
 }
 
 // getBlobMetaData returns the custom metadata associated with a page blob which is set by createBlob method.

src/cosalib/azure.py

@@ -1,5 +1,7 @@
+import json
 import os
 import urllib
+import tempfile
 from cosalib.cmdlib import run_verbose
 from tenacity import (
     retry,
@@ -36,6 +38,7 @@ def azure_run_ore(build, args):
     :type build: Build
     """
     azure_vhd_name = f"{build.image_name_base}.vhd"
+    tmp_metaf = tempfile.NamedTemporaryFile(suffix="oremeta")
     ore_args = [
         'ore',
         '--log-level', args.log_level,
@@ -47,20 +50,20 @@ def azure_run_ore(build, args):
         '--file', f"{build.image_path}",
         '--container', args.container,
         '--resource-group', args.resource_group,
-        '--storage-account', args.storage_account
+        '--storage-account', args.storage_account,
+        '--meta-output', tmp_metaf.name,
     ]
     if args.force:
         ore_args.append('--overwrite')
     run_verbose(ore_args)
 
-    url_path = urllib.parse.quote((
-        f"{args.storage_account}.blob.core.windows.net/"
-        f"{args.container}/{azure_vhd_name}"
-    ))
-    build.meta['azure'] = {
-        'image': azure_vhd_name,
-        'url': f"https://{url_path}",
-    }
+    tmp_metaf.seek(0)
+    meta = json.load(tmp_metaf)
+    tmp_metaf.close()
+    # Translate the ad-hoc ore JSON to coreos-assembler schema, also
+    # add 'image'
+    meta['image'] = azure_vhd_name
+    build.meta['azure'] = meta
     build.meta_write()  # update build metadata