Remove resource uri hostname parsing (#231)

* keep original error

* Remove dependency of storage URI structure.

* Update CHANGELOG

CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## Unreleased
 
+### Fixed
+
+- Fix storage URI parsing for hostname with more than 5 parts.
+- Keep original error when refreshing ingestion resources fails.
+
 ## [0.15.1] - 2024-03-04
 
 ### Changed

kusto/ingest/internal/resources/resources.go

@@ -31,25 +31,19 @@ type mgmter interface {
 	Mgmt(ctx context.Context, db string, query kusto.Statement, options ...kusto.MgmtOption) (*kusto.RowIterator, error)
 }
 
-var objectTypes = map[string]bool{
-	"queue": true,
-	"blob":  true,
-	"table": true,
-}
-
 // URI represents a resource URI for an ingestion command.
 type URI struct {
-	u                               *url.URL
-	account, objectType, objectName string
-	sas                             url.Values
+	u                   *url.URL
+	account, objectName string
+	sas                 url.Values
 }
 
 // Parse parses a string representing a Kutso resource URI.
-func Parse(uri string) (*URI, error) {
+func Parse(resourceUri string) (*URI, error) {
 	// Example for a valid url:
 	// https://fkjsalfdks.blob.core.windows.com/sdsadsadsa?sas=asdasdasd
 
-	u, err := url.Parse(uri)
+	u, err := url.Parse(resourceUri)
 	if err != nil {
 		return nil, err
 	}
@@ -58,28 +52,11 @@ func Parse(uri string) (*URI, error) {
 		return nil, fmt.Errorf("URI scheme must be 'https', was '%s'", u.Scheme)
 	}
 
-	hostSplit := strings.Split(u.Hostname(), ".")
-	if len(hostSplit) < 5 {
-		return nil, fmt.Errorf("error: Storage URI (%s) is invalid'", uri)
-	}
-
-	var v *URI
-	if len(hostSplit) == 5 {
-		v = &URI{
-			u:          u,
-			account:    hostSplit[0],
-			objectType: hostSplit[1],
-			objectName: strings.TrimLeft(u.EscapedPath(), "/"),
-			sas:        u.Query(),
-		}
-	} else {
-		v = &URI{
-			u:          u,
-			account:    hostSplit[0] + "." + hostSplit[1],
-			objectType: hostSplit[2],
-			objectName: strings.TrimLeft(u.EscapedPath(), "/"),
-			sas:        u.Query(),
-		}
+	v := &URI{
+		u:          u,
+		account:    u.Hostname(),
+		objectName: strings.TrimLeft(u.EscapedPath(), "/"),
+		sas:        u.Query(),
 	}
 
 	if err := v.validate(); err != nil {
@@ -89,20 +66,10 @@ func Parse(uri string) (*URI, error) {
 	return v, nil
 }
 
-// validate validates that the URI was valid.
-// TODO(Daniel): You could add deep validation of each value we have split to give better diagnostic info on an error.
-// I put in the most basic evalutation, but you might want to put checks for the account format or objectName foramt.
 func (u *URI) validate() error {
-	if u.account == "" {
-		return fmt.Errorf("account name was not provided")
-	}
-	if !objectTypes[u.objectType] {
-		return fmt.Errorf("object type was not valid(queue|blob|table), was: %q", u.objectType)
-	}
 	if u.objectName == "" {
 		return fmt.Errorf("object name was not provided")
 	}
-
 	return nil
 }
 
@@ -111,11 +78,6 @@ func (u *URI) Account() string {
 	return u.account
 }
 
-// ObjectType returns the type of object that will be ingested: queue, blob or table.
-func (u *URI) ObjectType() string {
-	return u.objectType
-}
-
 // ObjectName returns the object name of the resource, i.e container name.
 func (u *URI) ObjectName() string {
 	return u.objectName
@@ -398,7 +360,7 @@ func (m *Manager) fetchRetry(ctx context.Context) error {
 		if err != nil {
 			attempts++
 			if attempts > retryCount {
-				return fmt.Errorf("failed to fetch ingestion resources")
+				return fmt.Errorf("failed to fetch ingestion resources: %w", err)
 			}
 			time.Sleep(10 * time.Second)
 			continue

kusto/ingest/internal/resources/resources_test.go

@@ -19,24 +19,8 @@ func TestParse(t *testing.T) {
 		url            string
 		err            bool
 		wantAccount    string
-		wantObjectType string
 		wantObjectName string
 	}{
-		{
-			desc: "account is missing, but has leading dot",
-			url:  "https://.queue.core.windows.net/objectname",
-			err:  true,
-		},
-		{
-			desc: "account is missing",
-			url:  "https://queue.core.windows.net/objectname",
-			err:  true,
-		},
-		{
-			desc: "invalid object type",
-			url:  "https://account.invalid.core.windows.net/objectname",
-			err:  true,
-		},
 		{
 			desc: "no object name provided",
 			url:  "https://account.invalid.core.windows.net/",
@@ -50,22 +34,19 @@ func TestParse(t *testing.T) {
 		{
 			desc:           "success",
 			url:            "https://account.table.core.windows.net/objectname",
-			wantAccount:    "account",
-			wantObjectType: "table",
+			wantAccount:    "account.table.core.windows.net",
 			wantObjectName: "objectname",
 		},
 		{
 			desc:           "success non-public",
 			url:            "https://account.table.kusto.chinacloudapi.cn/objectname",
-			wantAccount:    "account",
-			wantObjectType: "table",
+			wantAccount:    "account.table.kusto.chinacloudapi.cn",
 			wantObjectName: "objectname",
 		},
 		{
 			desc:           "success dns zone",
-			url:            "https://account.zone1.blob.storage.azure.net/objectname",
-			wantAccount:    "account.zone1",
-			wantObjectType: "blob",
+			url:            "https://account.z01.blob.storage.azure.net/objectname",
+			wantAccount:    "account.z01.blob.storage.azure.net",
 			wantObjectName: "objectname",
 		},
 	}
@@ -84,7 +65,6 @@ func TestParse(t *testing.T) {
 			assert.NoError(t, err)
 
 			assert.Equal(t, test.wantAccount, got.Account())
-			assert.Equal(t, test.wantObjectType, got.ObjectType())
 			assert.Equal(t, test.wantObjectName, got.ObjectName())
 			assert.Equal(t, test.url, got.String())
 		})
@@ -221,7 +201,7 @@ func TestResources(t *testing.T) {
 						},
 						value.String{
 							Valid: true,
-							Value: "https://.blob.core.windows.net/storageroot",
+							Value: "https://.blob.core.windows.net/",
 						},
 					},
 				},