Merge pull request RamenDR#310 from red-hat-storage/sync_us--main

Syncing latest changes from upstream main for ramen

test/README.md

@@ -175,6 +175,29 @@ drenv cleanup
 
 This should not be needed.
 
+## Caching resources
+
+If you run the drenv tool with a flaky network you can improve
+reliability of starting the environment by caching resources.
+
+To cache resources for the `regional-dr.yaml` environment run:
+
+```
+drenv cache envs/regional-dr.yaml
+```
+
+The cache expires in 2 days. To refresh the cache daily, you can install
+a cron job to run `scripts/refresh-cache` daily as the user used to run
+the environment.
+
+See the `scripts/refresh-cache.crontab` for example user crontab.
+
+To clear the cached resources run:
+
+```
+drenv clear
+```
+
 ## The environment file
 
 To create an environment you need an yaml file describing the

test/addons/recipe/fetch → test/addons/argocd/cache

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-path = cache.path("addons/recipe.yaml")
-cache.fetch(".", path)
+cache.refresh(".", "addons/argocd-2.11.yaml")

test/addons/argocd/start

@@ -3,6 +3,7 @@
 # SPDX-FileCopyrightText: The RamenDR authors
 # SPDX-License-Identifier: Apache-2.0
 
+import os
 import sys
 
 from drenv import temporary_kubeconfig
@@ -20,8 +21,7 @@ def wait_for_clusters(clusters):
 
 def deploy_argocd(cluster):
     print("Deploying argocd")
-    path = cache.path("addons/argocd-2.11.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/argocd-2.11.yaml")
     kubectl.apply("--filename", path, "--namespace", "argocd", context=cluster)
 
 
@@ -70,6 +70,7 @@ if len(sys.argv) != 4:
     print(f"Usage: {sys.argv[0]} hub cluster1 cluster2")
     sys.exit(1)
 
+os.chdir(os.path.dirname(__file__))
 hub, *clusters = sys.argv[1:]
 
 wait_for_clusters(clusters)

test/addons/cdi/cache

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh("operator", "addons/cdi-operator-1.59.0.yaml")
+cache.refresh("cr", "addons/cdi-cr-1.59.0.yaml")

test/addons/cdi/start

@@ -14,8 +14,7 @@ NAMESPACE = "cdi"
 
 def deploy(cluster):
     print("Deploying cdi operator")
-    path = cache.path("addons/cdi-operator-1.59.0.yaml")
-    cache.fetch("operator", path)
+    path = cache.get("operator", "addons/cdi-operator-1.59.0.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
     print("Waiting until cdi-operator is rolled out")
@@ -28,8 +27,7 @@ def deploy(cluster):
     )
 
     print("Deploying cdi cr")
-    path = cache.path("addons/cdi-cr-1.59.0.yaml")
-    cache.fetch("cr", path)
+    path = cache.get("cr", "addons/cdi-cr-1.59.0.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/csi-addons/fetch → test/addons/csi-addons/cache

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-path = cache.path("addons/csi-addons.yaml")
-cache.fetch(".", path)
+cache.refresh(".", "addons/csi-addons.yaml")

test/addons/csi-addons/start

@@ -12,8 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying csi addon for volume replication")
-    path = cache.path("addons/csi-addons.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/csi-addons.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/kubevirt/cache

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh("operator", "addons/kubevirt-operator-1.2.1.yaml")
+cache.refresh("cr", "addons/kubevirt-cr-1.2.1.yaml")

test/addons/kubevirt/start

@@ -14,8 +14,7 @@ NAMESPACE = "kubevirt"
 
 def deploy(cluster):
     print("Deploying kubevirt operator")
-    path = cache.path("addons/kubevirt-operator-1.2.1.yaml")
-    cache.fetch("operator", path)
+    path = cache.get("operator", "addons/kubevirt-operator-1.2.1.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
     print("Waiting until virt-operator is rolled out")
@@ -28,8 +27,7 @@ def deploy(cluster):
     )
 
     print("Deploying kubevirt cr")
-    path = cache.path("addons/kubevirt-cr-1.2.1.yaml")
-    cache.fetch("cr", path)
+    path = cache.get("cr", "addons/kubevirt-cr-1.2.1.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/ocm-controller/fetch → test/addons/ocm-controller/cache

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-path = cache.path("addons/ocm-controller.yaml")
-cache.fetch(".", path)
+cache.refresh(".", "addons/ocm-controller.yaml")

test/addons/ocm-controller/start

@@ -12,8 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying ocm controller")
-    path = cache.path("addons/ocm-controller.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/ocm-controller.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/olm/cache

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh("crds", "addons/olm-crds-0.27.yaml")
+cache.refresh("operators", "addons/olm-operators-0.27.yaml")

test/addons/olm/start

@@ -20,8 +20,7 @@ def deploy(cluster):
     #   The CustomResourceDefinition "clusterserviceversions.operators.coreos.com"
     #   is invalid: metadata.annotations: Too long: must have at most 262144 bytes
     # See https://medium.com/pareture/kubectl-install-crd-failed-annotations-too-long-2ebc91b40c7d
-    path = cache.path("addons/olm-crds-0.27.yaml")
-    cache.fetch("crds", path)
+    path = cache.get("crds", "addons/olm-crds-0.27.yaml")
     kubectl.apply("--filename", path, "--server-side=true", context=cluster)
 
     print("Waiting until cdrs are established")
@@ -33,8 +32,7 @@ def deploy(cluster):
     )
 
     print("Deploying olm")
-    path = cache.path("addons/olm-operators-0.27.yaml")
-    cache.fetch("operators", path)
+    path = cache.get("operators", "addons/olm-operators-0.27.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-cephfs/fetch → test/addons/recipe/cache

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-path = cache.path("addons/rook-cephfs-1.14.yaml")
-cache.fetch(".", path)
+cache.refresh(".", "addons/recipe.yaml")

test/addons/recipe/start

@@ -16,6 +16,5 @@ os.chdir(os.path.dirname(__file__))
 cluster = sys.argv[1]
 
 print("Deploying recipe crd")
-path = cache.path("addons/recipe.yaml")
-cache.fetch(".", path)
+path = cache.get(".", "addons/recipe.yaml")
 kubectl.apply("--filename", path, context=cluster)

test/addons/rook-cephfs/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-cephfs-1.14.yaml")

test/addons/rook-cephfs/start

@@ -13,8 +13,7 @@ from drenv import kubectl
 
 def deploy(cluster):
     print("Creating CephFS instance and storage/snapshot classes")
-    path = cache.path("addons/rook-cephfs-1.14.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/rook-cephfs-1.14.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-cluster/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-cluster-1.14.yaml")

test/addons/rook-cluster/start

@@ -20,8 +20,7 @@ TIMEOUT = 600
 
 def deploy(cluster):
     print("Deploying rook ceph cluster")
-    path = cache.path("addons/rook-cluster-1.14.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/rook-cluster-1.14.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-operator/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-operator-1.14.yaml")

test/addons/rook-operator/start

@@ -12,8 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying rook ceph operator")
-    path = cache.path("addons/rook-operator-1.14.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/rook-operator-1.14.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-toolbox/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-toolbox-1.14.yaml")

test/addons/rook-toolbox/start

@@ -13,8 +13,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying rook ceph toolbox")
-    path = cache.path("addons/rook-toolbox-1.14.yaml")
-    cache.fetch(".", path)
+    path = cache.get(".", "addons/rook-toolbox-1.14.yaml")
     kubectl.apply("--filename", path, context=cluster)