Merge pull request #35 from caktus/external-backup-bucket-tests

Allow custom backup bucket

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Test code
+
+on:
+  pull_request:
+  schedule:
+    # run once a week on early monday mornings
+    - cron: '22 2 * * 1'
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements-tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install -e .
+          pip install -r requirements-tests.txt
+      - name: Run Tests
+        run: pytest

.gitignore

@@ -3,4 +3,6 @@
 __pycache__/
 *.pyc
 *.egg-info
-.idea
+.envrc
+.direnv
+dist

MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include README.rst
+recursive-exclude tests *
+recursive-exclude * *.pyc *.pyo
+recursive-exclude **/__pycache__ *

RELEASES.rst

@@ -1,7 +1,20 @@
 Releases
 ========
 
-v0.0.13, 2021-FIXME
+
+v0.0.15, 2021-03-24
+~~~~~~~~~~~~~~~~~~~~
+
+* Add support for using any source S3 backup bucket with `config.hosting_services_backup_bucket`
+
+
+v0.0.14, 2021-03-02
+~~~~~~~~~~~~~~~~~~~~
+
+* Adds get_backup_from_hosting to allow retrieving backups that are created and stored using caktus-hosting-backups
+
+
+v0.0.13, 2021-02-22
 ~~~~~~~~~~~~~~~~~~~~
 * Make custom playbooks compatible with boto. This requires that you add a
   ``aws.profile_name`` key in your tasks definition which points to an AWS_PROFILE that

kubesae/utils.py

@@ -45,20 +45,23 @@ def get_backup_from_hosting(c, latest="daily", profile="caktus", backup_name=Non
     """
     valid_periods = ['daily', 'weekly', 'monthly', 'yearly']
 
-    if "hosting_services_backup_folder" not in c.config.keys():
+    if c.config.get("hosting_services_backup_bucket"):
+        bucket = f"s3://{c.config.hosting_services_backup_bucket.strip('/')}"
+    else:
+        bucket = f"s3://{BASE_BACKUP_BUCKET.strip('/')}"
+
+    if c.config.get("hosting_services_backup_folder"):
+        bucket_folder = f"{bucket}/{c.config.hosting_services_backup_folder.strip('/')}"
+    else:
         print("A hosting services backup folder has not been defined in tasks.py for this project.")
-        print(f"Here are a list of the currently defined backup folders:")
-        c.run(
-            f"aws s3 ls s3://{BASE_BACKUP_BUCKET}/ --profile {profile}"
-        )
-        print(f"If the project is not listed it will need to be set up with Hosting services, "
-              f"see: https://github.com/caktus/ansible-role-k8s-hosting-services")
+        print("Here are a list of the currently defined backup folders:")
+        c.run(f"aws s3 ls {bucket} --profile {profile}")
+        print("If the project is not listed it will need to be set up with Hosting services, "
+              "see: https://github.com/caktus/ansible-role-k8s-hosting-services")
         return
 
     if list:
-        c.run(
-            f"aws s3 ls s3://{BASE_BACKUP_BUCKET}/{c.config.hosting_services_backup_folder}/ --profile {profile}"
-        )
+        c.run(f"aws s3 ls {bucket_folder}/ --profile {profile}")
         return
 
     if latest not in valid_periods:
@@ -67,7 +70,7 @@ def get_backup_from_hosting(c, latest="daily", profile="caktus", backup_name=Non
 
     if not backup_name:
         listing = c.run(
-            f"aws s3 ls s3://{BASE_BACKUP_BUCKET}/{c.config.hosting_services_backup_folder}/ --profile {profile}",
+            f"aws s3 ls {bucket_folder}/ --profile {profile}",
             pty=False,
             hide="out",
         ).stdout.strip()
@@ -81,7 +84,7 @@ def get_backup_from_hosting(c, latest="daily", profile="caktus", backup_name=Non
         backup_name = f"{latest}-{c.config.hosting_services_backup_folder}-{dates[-1]}.pgdump"
 
     c.run(
-        f"aws s3 cp s3://{BASE_BACKUP_BUCKET}/{c.config.hosting_services_backup_folder}/{backup_name} ./{backup_name} --profile {profile}"
+        f"aws s3 cp {bucket_folder}/{backup_name} ./{backup_name} --profile {profile}"
     )
 
 

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -W ignore::DeprecationWarning:invoke.loader -W ignore::DeprecationWarning:invoke.tasks

requirements-tests.txt

@@ -0,0 +1 @@
+pytest==6.2.2

setup.py

@@ -3,24 +3,24 @@
 
 
 setup(
-    name='invoke-kubesae',
-    version='0.0.11',
-    packages=find_packages(),
-    url='https://github.com/caktus/invoke-kubesae',
-    author='Caktus Group',
-    author_email='',
-    description='',
+    name="invoke-kubesae",
+    version="0.0.15",
+    packages=find_packages(exclude=["tests"]),
+    url="https://github.com/caktus/invoke-kubesae",
+    author="Caktus Group",
+    author_email="",
+    description="",
     install_requires=[
-        'boto3>=1.16',
-        'invoke>=1.4',
-        'colorama>=0.4',
-        'ansible>=2.9',
+        "boto3>=1.16",
+        "invoke>=1.4",
+        "colorama>=0.4",
+        "ansible>=2.9",
     ],
-    python_requires='>=3.5',
+    python_requires=">=3.5",
     classifiers=[
-        'Intended Audience :: Developers',
-        'Topic :: Software Development :: Build Tools',
-        'License :: OSI Approved :: BSD License',
-        'Programming Language :: Python :: 3',
-    ]
+        "Intended Audience :: Developers",
+        "Topic :: Software Development :: Build Tools",
+        "License :: OSI Approved :: BSD License",
+        "Programming Language :: Python :: 3",
+    ],
 )

tests/conftest.py

@@ -0,0 +1,10 @@
+from unittest import mock
+import pytest
+from invoke.context import Context
+
+
+@pytest.fixture
+def c():
+    context = Context()
+    context.run = mock.Mock()
+    return context

tests/test_utils_bucket.py

@@ -0,0 +1,53 @@
+import pytest
+
+import invoke
+
+from kubesae.utils import BASE_BACKUP_BUCKET, get_backup_from_hosting
+
+
+@pytest.fixture
+def test_bucket(c):
+    backup_bucket = "test-bucket"
+    c.config.hosting_services_backup_bucket = backup_bucket
+    return backup_bucket
+
+
+@pytest.fixture
+def test_folder(c):
+    backup_folder = "test-project"
+    c.config.hosting_services_backup_folder = backup_folder
+    return backup_folder
+
+
+@pytest.fixture
+def s3_filename(c, test_folder):
+    filename = f"daily-{test_folder}-202101010000.pgdump"
+    listing = f"2021-01-01 08:00:00 111266 {filename}"
+    c.run.return_value.stdout.strip.return_value = listing
+    return filename
+
+
+def test_backup_bucket__default(c, test_folder):
+    get_backup_from_hosting(c, backup_name="-")
+    assert f"aws s3 cp s3://{BASE_BACKUP_BUCKET}" in c.run.call_args.args[0]
+
+
+def test_backup_bucket__custom(c, test_bucket, test_folder):
+    get_backup_from_hosting(c, backup_name="-")
+    assert f"aws s3 cp s3://{test_bucket}" in c.run.call_args.args[0]
+
+
+def test_bucket_folder(c, test_bucket, test_folder):
+    get_backup_from_hosting(c, backup_name="-")
+    cmd = c.run.call_args.args[0]
+    assert f"aws s3 cp s3://{test_bucket}/{test_folder}" in cmd
+
+
+def test_backup_name__default(c, test_bucket, test_folder, s3_filename):
+    get_backup_from_hosting(c)
+    assert f"./{s3_filename}" in c.run.call_args.args[0]
+
+
+def test_backup_name__custom(c, test_bucket, test_folder):
+    get_backup_from_hosting(c, backup_name="mydb.pgdump")
+    assert "./mydb.pgdump" in c.run.call_args.args[0]