Fix query quota call by path

api/v1/api_v1_quotas.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2022-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@ package v1
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	"github.com/dell/goisilon/api"
@@ -29,23 +28,22 @@ func GetIsiQuota(
 	client api.Client,
 	path string) (quota *IsiQuota, err error) {
 
-	// PAPI call: GET https://1.2.3.4:8080/platform/1/quota/quotas
-	// This will list out all quotas on the cluster
+	// PAPI call: GET https://1.2.3.4:8080/platform/1/quota/quotas?path=/path/to/volume
+	// This will list the quota by path on the cluster
 
 	var quotaResp isiQuotaListResp
-	err = client.Get(ctx, quotaPath, "", nil, nil, &quotaResp)
+	var pathWithQueryParam = quotaPath + "?path=" + path
+	err = client.Get(ctx, pathWithQueryParam, "", nil, nil, &quotaResp)
 	if err != nil {
 		return nil, err
 	}
 
-	// find the specific quota we are looking for
-	for _, quota := range quotaResp.Quotas {
-		if quota.Path == path {
-			return &quota, nil
-		}
+	if quotaResp.Quotas != nil && len(quotaResp.Quotas) > 0 {
+		quota = &quotaResp.Quotas[0]
+		return quota, nil
 	}
 
-	return nil, errors.New(fmt.Sprintf("Quota not found: %s", path))
+	return nil, fmt.Errorf("Quota not found: %s", path)
 }
 
 // GetAllIsiQuota queries all quotas on the cluster

api/v11/api_v11_replication.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2022-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -82,6 +82,7 @@ type Policy struct {
 	Id           string    `json:"id,omitempty"`
 	Name         string    `json:"name,omitempty"`
 	Enabled      bool      `json:"enabled"`
+	Conflicted   bool      `json:"conflicted"`
 	TargetPath   string    `json:"target_path,omitempty"`
 	SourcePath   string    `json:"source_root_path,omitempty"`
 	TargetHost   string    `json:"target_host,omitempty"`
@@ -137,6 +138,7 @@ type Report struct {
 	JobId   int64     `json:"job_id"`
 	State   JOB_STATE `json:"state,omitempty"`
 	EndTime int64     `json:"end_time"`
+	Errors  []string  `json:"errors"`
 }
 
 // GetPolicyByName returns policy by name

env.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright © 2019-2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright © 2019-2023 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ GOISILON_USERNAME="admin"
 GOISILON_GROUP=""
 GOISILON_PASSWORD="admin"
 GOISILON_INSECURE="true"
+GOISILON_UNRESOLVABLE_HOSTS="false"
 GOISILON_VOLUMEPATH="/ifs/data/csi"
 GOISILON_VOLUMEPATH_PERMISSIONS="0777"
 GOISILON_AUTHTYPE=0

exports_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2022-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -347,13 +347,13 @@ func TestAddExportClientsByID(t *testing.T) {
 }
 
 func testAddExportClientsByID(t *testing.T, exportID int, export Export, addExportClientsByID func(
-	ctx context.Context, id int, clients []string) error) {
+	ctx context.Context, id int, clients []string, ignoreUnresolvableHosts bool) error) {
 
 	var clientsToAdd = []string{"192.168.1.110", "192.168.1.110", "192.168.1.111", "192.168.1.112", "192.168.1.113"}
 
 	log.Debug(defaultCtx, "add '%v' to '%v' for export '%d'", clientsToAdd, *export.Clients, exportID)
 
-	err = addExportClientsByID(defaultCtx, exportID, clientsToAdd)
+	err = addExportClientsByID(defaultCtx, exportID, clientsToAdd, false)
 	assert.Nil(t, err)
 }
 
@@ -372,8 +372,8 @@ func TestRemoveExportClientsByName(t *testing.T) {
 }
 
 func testRemoveExportClients(t *testing.T,
-	removeExportClientsByIDFunc func(ctx context.Context, id int, clientsToRemove []string) error,
-	removeExportClientsByNameFunc func(ctx context.Context, name string, clientsToRemove []string) error) {
+	removeExportClientsByIDFunc func(ctx context.Context, id int, clientsToRemove []string, ignoreUnresolvableHosts bool) error,
+	removeExportClientsByNameFunc func(ctx context.Context, name string, clientsToRemove []string, ignoreUnresolvableHosts bool) error) {
 
 	volumeName1 := "test_get_exports1"
 
@@ -388,10 +388,10 @@ func testRemoveExportClients(t *testing.T,
 	log.Debug(defaultCtx, "remove '%v' from '%v' for export '%d'", clientsToRemove, *export.Clients, exportID)
 
 	if removeExportClientsByIDFunc != nil {
-		err = removeExportClientsByIDFunc(defaultCtx, exportID, clientsToRemove)
+		err = removeExportClientsByIDFunc(defaultCtx, exportID, clientsToRemove, false)
 		assert.Nil(t, err)
 	} else {
-		err = removeExportClientsByNameFunc(defaultCtx, exportName, clientsToRemove)
+		err = removeExportClientsByNameFunc(defaultCtx, exportName, clientsToRemove, false)
 		assert.Nil(t, err)
 	}
 

goisilon_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2019-2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2019-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -64,8 +64,14 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.WithError(err).Panic(defaultCtx, "error fetching environment variable GOISILON_INSECURE")
 	}
-
+	ignoreUnresolvableHosts, err := strconv.ParseBool(os.Getenv("GOISILON_UNRESOLVABLE_HOSTS"))
+	if err != nil {
+		log.WithError(err).Panic(defaultCtx, "error fetching environment variable GOISILON_UNRESOLVABLE_HOSTS")
+	}
 	authType, err := strconv.Atoi(os.Getenv("GOISILON_AUTHTYPE"))
+	if err != nil {
+		log.WithError(err).Panic(defaultCtx, "error fetching environment variable GOISILON_AUTHTYPE")
+	}
 
 	client, err = NewClientWithArgs(
 		defaultCtx,
@@ -77,15 +83,12 @@ func TestMain(m *testing.M) {
 		os.Getenv("GOISILON_PASSWORD"),
 		os.Getenv("GOISILON_VOLUMEPATH"),
 		os.Getenv("GOISILON_VOLUMEPATH_PERMISSIONS"),
+		ignoreUnresolvableHosts,
 		uint8(authType))
-
 	if err != nil {
 		log.WithError(err).Panic(defaultCtx, "error creating test client")
 	}
 
-	if err != nil {
-		log.WithError(err).Panic(defaultCtx, "error creating test client")
-	}
 	os.Exit(m.Run())
 }
 

replication.go

@@ -1,7 +1,7 @@
 package goisilon
 
 /*
-Copyright (c) 2021-2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2021-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@ limitations under the License.
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"time"
 
 	log "github.com/akutz/gournal"
@@ -28,6 +30,12 @@ import (
 const defaultPoll = 5 * time.Second
 const defaultTimeout = 10 * time.Minute // set high timeout, we expect to be canceled via context before
 
+const retryInterval = 15 * time.Second
+const maxRetries = 20 // with 15 sec, it is 4 retries per min. For 5 minutes, it is 20 retries
+const retryablePolicyError = "is in an error state. Please resolve it and retry"
+const retryableReportError = "A new quota domain that has not finished QuotaScan has been found"
+const resolveErrorToIgnore = "The policy was not conflicted, so no change was made"
+
 const (
 	RESYNC_PREP            apiv11.JOB_ACTION              = "resync_prep"
 	ALLOW_WRITE            apiv11.JOB_ACTION              = "allow_write"
@@ -144,6 +152,23 @@ func (c *Client) ModifyPolicy(ctx context.Context, name string, schedule string,
 	return apiv11.UpdatePolicy(ctx, c.API, p)
 }
 
+// Resolves the policy that is in error state due to QuotaScan requirement.
+func (c *Client) ResolvePolicy(ctx context.Context, name string) error {
+	pp, err := c.GetPolicyByName(ctx, name)
+	if err != nil {
+		return err
+	}
+
+	p := &apiv11.Policy{
+		Id:         pp.Id,
+		Conflicted: false,
+		Enabled:    pp.Enabled,  // keep existing enabled state, otherwise it will be cleared
+		Schedule:   pp.Schedule, // keep existing schedule, otherwise it will be cleared
+	}
+
+	return apiv11.UpdatePolicy(ctx, c.API, p)
+}
+
 func (c *Client) AllowWrites(ctx context.Context, policyName string) error {
 	targetPolicy, err := c.GetTargetPolicyByName(ctx, policyName)
 	if err != nil {
@@ -319,7 +344,7 @@ func (c *Client) SyncPolicy(ctx context.Context, policyName string) error {
 		return err
 	}
 	if !policy.Enabled {
-		return nil
+		return fmt.Errorf("cannot run sync on disabled policy %s", policyName)
 	}
 
 	runningJobs, err := c.GetJobsByPolicyName(ctx, policyName)
@@ -345,10 +370,42 @@ func (c *Client) SyncPolicy(ctx context.Context, policyName string) error {
 			Id: policyName,
 		}
 		log.Info(ctx, "found no active sync jobs, starting a new one")
-		_, err := c.StartSyncIQJob(ctx, jobReq)
-		if err != nil {
-			return err
+
+		// workaround for PowerScale KB article
+		// https://www.dell.com/support/kbdoc/en-us/000019414/quotas-on-synciq-source-directories
+		for i := 0; i < maxRetries; i++ {
+			_, err := c.StartSyncIQJob(ctx, jobReq)
+			if err == nil {
+				break
+			}
+			if strings.Contains(err.Error(), retryablePolicyError) {
+				if i+1 == maxRetries {
+					return err
+				}
+
+				reports, err := c.GetReportsByPolicyName(ctx, policyName, 1)
+				if err != nil {
+					return fmt.Errorf("error while retrieving reports for failed sync job %s %s", policyName, err.Error())
+				}
+				if !(len(reports.Reports) > 0 && len(reports.Reports[0].Errors) > 0 &&
+					strings.Contains(reports.Reports[0].Errors[0], retryableReportError)) {
+					return fmt.Errorf("found no retryable error in reports for failed sync job %s", policyName)
+				}
+
+				log.Info(ctx, "Sync job failed with error: %s. %v of %v - retrying in %v...",
+					reports.Reports[0].Errors[0], i+1, maxRetries, retryInterval)
+				time.Sleep(retryInterval)
+
+				// Resolve policy with error before retrying
+				err = c.ResolvePolicy(ctx, policyName)
+				if err != nil && !strings.Contains(err.Error(), resolveErrorToIgnore) {
+					return err
+				}
+			} else { // not a retryable error
+				return err
+			}
 		}
+
 		time.Sleep(3 * time.Second)
 		err = c.WaitForNoActiveJobs(ctx, policyName)
 		if err != nil {

replication_test.go

@@ -1,7 +1,7 @@
 package goisilon_test
 
 /*
-Copyright (c) 2021-2022 Dell Inc, or its subsidiaries.
+Copyright (c) 2021-2023 Dell Inc, or its subsidiaries.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -45,7 +45,7 @@ func (suite *ReplicationTestSuite) SetupSuite() {
 		"",
 		"dangerous",
 		"/ifs/data/test-goisilon",
-		"0777", 0)
+		"0777", false, 0)
 	if err != nil {
 		panic(err)
 	}
@@ -61,7 +61,7 @@ func (suite *ReplicationTestSuite) SetupSuite() {
 		"",
 		"dangerous",
 		"/ifs/data/test-goisilon",
-		"0777", 0)
+		"0777", false, 0)
 	if err != nil {
 		panic(err)
 	}