feat(compose): Backup to local and object storage

Automatically backup 2 different places.
5ouma · Aug 8, 2024 · 4810cd5 · 4810cd5
1 parent 44c9d29
commit 4810cd5
Show file tree

Hide file tree

Showing 11 changed files with 120 additions and 47 deletions.
diff --git a/.env.tmpl b/.env.tmpl
@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_ENDPOINT=
diff --git a/.github/README.md b/.github/README.md
@@ -29,7 +29,15 @@
    git clone https://github.com/5ouma/homelab.git
    ```
 
-2. 📝 Follow the instructions below
+2. 🏔️ Copy the [`.env.tmpl`](../.env.tmpl) file to `.env` and Edit it
+
+   |          Name           |              Value              |
+   | :---------------------: | :-----------------------------: |
+   |   `AWS_ACCESS_KEY_ID`   |   Cloudflare R2 Access Key ID   |
+   | `AWS_SECRET_ACCESS_KEY` | Cloudflare R2 Secret Access Key |
+   |     `AWS_ENDPOINT`      |  Cloudflare R2 Endpoint Domain  |
+
+3. 📝 Follow the instructions below
 
    - [🪐 **Misskey**](../misskey)
    - [🌸 **Immich**](../immich)

diff --git a/immich/.env.tmpl b/immich/.env.tmpl
@@ -2,4 +2,7 @@ DB_HOSTNAME=immich_postgres
 DB_DATABASE_NAME=immich
 DB_USERNAME=postgres
 DB_PASSWORD=
+R2_BUCKET_NAME=
+R2_PATH=backup
+BACKUP_DIR=
 TUNNEL_TOKEN=
diff --git a/immich/README.md b/immich/README.md
@@ -12,10 +12,12 @@
 
 1. 🏔️ Copy the [`.env.tmpl`](./.env.tmpl) file to `.env` and Edit it
 
-   |      Name      |           Value           |
-   | :------------: | :-----------------------: |
-   | `DB_PASSWORD`  | Random Database Password  |
-   | `TUNNEL_TOKEN` | [Cloudflare Tunnel Token] |
+   |       Name       |           Value           |
+   | :--------------: | :-----------------------: |
+   |  `DB_PASSWORD`   | Random Database Password  |
+   | `R2_BUCKET_NAME` | Cloudflare R2 Bucket Name |
+   |   `BACKUP_DIR`   |  Local Backup Directory   |
+   |  `TUNNEL_TOKEN`  | [Cloudflare Tunnel Token] |
 
    [Cloudflare Tunnel Token]: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/
 

diff --git a/immich/compose.yml b/immich/compose.yml
@@ -41,8 +41,11 @@ services:
       POSTGRES_PASSWORD: $DB_PASSWORD
     volumes:
       - database:/var/lib/postgresql/data
+      - backup:/tmp/backup
     healthcheck:
       test: pg_isready -d '$DB_DATABASE_NAME' -U '$DB_USERNAME' || exit 1
+    labels:
+      docker-volume-backup.archive-pre: pg_dumpall -l '$DB_DATABASE_NAME' -U '$DB_USERNAME' -f /tmp/backup/immich.sql -c --if-exists
     restart: always
 
   redis:
@@ -52,25 +55,44 @@ services:
       test: redis-cli ping || exit 1
     restart: always
 
-  backup:
-    container_name: immich_backup
-    image: prodrigestivill/postgres-backup-local:16@sha256:f873bbfb406db633ab879628b11ef8092279639c467a2ea5f5bfa08b776252fb
+  backup-database: &backup
+    container_name: immich_backup-database
+    image: offen/docker-volume-backup:v2.40.0@sha256:f9cf83d63a1a9129b7c421090b584f11732fc201283f415552b14113f2cbf674
     environment:
-      POSTGRES_HOST: $DB_HOSTNAME
-      POSTGRES_DB: $DB_DATABASE_NAME
-      POSTGRES_USER: $DB_USERNAME
-      POSTGRES_PASSWORD: $DB_PASSWORD
-      POSTGRES_CLUSTER: TRUE
-      POSTGRES_EXTRA_OPTS: --clean --if-exists
-      TZ: JST
-      SCHEDULE: 0 5 * * *
+      BACKUP_CRON_EXPRESSION: "0 5 * * *"
+      BACKUP_FILENAME: immich_database_%Y%m%d.{{ .Extension }}
+      BACKUP_SKIP_BACKENDS_FROM_PRUNE: s3
+      AWS_S3_BUCKET_NAME: $R2_BUCKET_NAME
+      AWS_S3_PATH: $R2_PATH
+      BACKUP_RETENTION_DAYS: 7
+    env_file:
+      - path: ../.env
+        required: true
     volumes:
-      - ./backups:/backups
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - backup:/backup:ro
+      - $BACKUP_DIR/database:/archive
+    configs:
+      - source: timezone
+        target: /etc/localtime
     depends_on:
       database:
         condition: service_healthy
     restart: always
 
+  backup-library:
+    <<: *backup
+    container_name: immich_backup-library
+    environment:
+      BACKUP_CRON_EXPRESSION: "0 5 * * *"
+      BACKUP_FILENAME: immich_library_%Y%m%d.{{ .Extension }}
+      BACKUP_SOURCES: /library
+      BACKUP_ARCHIVE: /archive-library
+      BACKUP_RETENTION_DAYS: 7
+    volumes:
+      - ./library:/library:ro
+      - $BACKUP_DIR/library:/archive-library
+
   cloudflare-tunnel:
     container_name: immich_cloudflare_tunnel
     image: cloudflare/cloudflared:2024.8.2@sha256:004f4b7b60bab652d478148c138843c24eae1feee4c58fddd435b9b79c953957
@@ -82,6 +104,10 @@ services:
 volumes:
   model-cache:
   database:
+  backup:
+
 configs:
   immich_config:
     file: ./config/immich.json
+  timezone:
+    file: /etc/localtime
diff --git a/misskey/.env.tmpl b/misskey/.env.tmpl
@@ -3,4 +3,7 @@ DB_HOSTNAME=database
 DB_DATABASE_NAME=misskey
 DB_USERNAME=postgres
 DB_PASSWORD=
+R2_BUCKET_NAME=
+R2_PATH=backup
+BACKUP_DIR=
 TUNNEL_TOKEN=
diff --git a/misskey/README.md b/misskey/README.md
@@ -12,11 +12,13 @@
 
 1. 🏔️ Copy the [`.env.tmpl`](./.env.tmpl) file to `.env` and Edit it
 
-   |      Name      |           Value           |
-   | :------------: | :-----------------------: |
-   | `MISSKEY_URL`  |        Misskey URL        |
-   | `DB_PASSWORD`  | Random Database Password  |
-   | `TUNNEL_TOKEN` | [Cloudflare Tunnel Token] |
+   |       Name       |           Value           |
+   | :--------------: | :-----------------------: |
+   |  `MISSKEY_URL`   |        Misskey URL        |
+   |  `DB_PASSWORD`   | Random Database Password  |
+   | `R2_BUCKET_NAME` | Cloudflare R2 Bucket Name |
+   |   `BACKUP_DIR`   |  Local Backup Directory   |
+   |  `TUNNEL_TOKEN`  | [Cloudflare Tunnel Token] |
 
 [Cloudflare Tunnel Token]: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/
 

diff --git a/misskey/compose.yml b/misskey/compose.yml
@@ -29,8 +29,11 @@ services:
       POSTGRES_PASSWORD: $DB_PASSWORD
     volumes:
       - database:/var/lib/postgresql/data
+      - backup:/tmp/backup
     healthcheck:
       test: pg_isready -d '$DB_DATABASE_NAME' -U '$DB_USERNAME' || exit 1
+    labels:
+      docker-volume-backup.archive-pre: pg_dumpall -l '$DB_DATABASE_NAME' -U '$DB_USERNAME' -f /tmp/backup/misskey.sql -c --if-exists
     restart: always
 
   redis:
@@ -42,18 +45,24 @@ services:
 
   backup:
     container_name: misskey_backup
-    image: prodrigestivill/postgres-backup-local:16@sha256:f873bbfb406db633ab879628b11ef8092279639c467a2ea5f5bfa08b776252fb
+    image: offen/docker-volume-backup:v2.40.0@sha256:f9cf83d63a1a9129b7c421090b584f11732fc201283f415552b14113f2cbf674
     environment:
-      POSTGRES_HOST: $DB_HOSTNAME
-      POSTGRES_DB: $DB_DATABASE_NAME
-      POSTGRES_USER: $DB_USERNAME
-      POSTGRES_PASSWORD: $DB_PASSWORD
-      POSTGRES_CLUSTER: TRUE
-      POSTGRES_EXTRA_OPTS: --clean --if-exists
-      TZ: JST
-      SCHEDULE: 0 5 * * *
+      BACKUP_CRON_EXPRESSION: "0 5 * * *"
+      BACKUP_FILENAME: misskey_database_%Y%m%d.{{ .Extension }}
+      BACKUP_SKIP_BACKENDS_FROM_PRUNE: s3
+      AWS_S3_BUCKET_NAME: $R2_BUCKET_NAME
+      AWS_S3_PATH: $R2_PATH
+      BACKUP_RETENTION_DAYS: 7
+    env_file:
+      - path: ../.env
+        required: true
     volumes:
-      - ./backups:/backups
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - backup:/backup:ro
+      - $BACKUP_DIR/database:/archive
+    configs:
+      - source: timezone
+        target: /etc/localtime
     depends_on:
       database:
         condition: service_healthy
@@ -69,7 +78,10 @@ services:
 
 volumes:
   database:
+  backup:
 
 configs:
   misskey_config:
     file: ./config/misskey.yml
+  timezone:
+    file: /etc/localtime
diff --git a/speedtest-tracker/.env.tmpl b/speedtest-tracker/.env.tmpl
@@ -4,4 +4,7 @@ DB_HOSTNAME=speedtest_tracker_database
 DB_DATABASE_NAME=speedtest_tracker
 DB_USERNAME=postgres
 DB_PASSWORD=
+R2_BUCKET_NAME=
+R2_PATH=backup
+BACKUP_DIR=
 TUNNEL_TOKEN=
diff --git a/speedtest-tracker/README.md b/speedtest-tracker/README.md
@@ -12,11 +12,13 @@
 
 1. 🏔️ Copy the [`.env.tmpl`](./.env.tmpl) file to `.env` and Edit it
 
-   |      Name      |           Value           |
-   | :------------: | :-----------------------: |
-   |   `APP_KEY`    |     [Encryption Key]      |
-   | `DB_PASSWORD`  | Random Database Password  |
-   | `TUNNEL_TOKEN` | [Cloudflare Tunnel Token] |
+   |       Name       |           Value           |
+   | :--------------: | :-----------------------: |
+   |    `APP_KEY`     |     [Encryption Key]      |
+   |  `DB_PASSWORD`   | Random Database Password  |
+   | `R2_BUCKET_NAME` | Cloudflare R2 Bucket Name |
+   |   `BACKUP_DIR`   |  Local Backup Directory   |
+   |  `TUNNEL_TOKEN`  | [Cloudflare Tunnel Token] |
 
    [Encryption Key]: https://speedtest-tracker.dev
    [Cloudflare Tunnel Token]: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/

diff --git a/speedtest-tracker/compose.yml b/speedtest-tracker/compose.yml
@@ -31,24 +31,33 @@ services:
       POSTGRES_PASSWORD: $DB_PASSWORD
     volumes:
       - database:/var/lib/postgresql/data
+      - backup:/tmp/backup
     healthcheck:
       test: pg_isready -d '$DB_DATABASE_NAME' -U '$DB_USERNAME' || exit 1
+    labels:
+      docker-volume-backup.archive-pre: pg_dumpall -l '$DB_DATABASE_NAME' -U '$DB_USERNAME' -f /tmp/backup/speedtest-tracker.sql -c --if-exists
     restart: always
 
   backup:
     container_name: speedtest-tracker_backup
-    image: prodrigestivill/postgres-backup-local:16@sha256:f873bbfb406db633ab879628b11ef8092279639c467a2ea5f5bfa08b776252fb
+    image: offen/docker-volume-backup:v2.40.0@sha256:f9cf83d63a1a9129b7c421090b584f11732fc201283f415552b14113f2cbf674
     environment:
-      POSTGRES_HOST: $DB_HOSTNAME
-      POSTGRES_DB: $DB_DATABASE_NAME
-      POSTGRES_USER: $DB_USERNAME
-      POSTGRES_PASSWORD: $DB_PASSWORD
-      POSTGRES_CLUSTER: TRUE
-      POSTGRES_EXTRA_OPTS: --clean --if-exists
-      TZ: JST
-      SCHEDULE: 0 5 * * *
+      BACKUP_CRON_EXPRESSION: "0 5 * * *"
+      BACKUP_FILENAME: speedtest-tracker_database_%Y%m%d.{{ .Extension }}
+      BACKUP_SKIP_BACKENDS_FROM_PRUNE: s3
+      AWS_S3_BUCKET_NAME: $R2_BUCKET_NAME
+      AWS_S3_PATH: $R2_PATH
+      BACKUP_RETENTION_DAYS: 7
+    env_file:
+      - path: ../.env
+        required: true
     volumes:
-      - ./backups:/backups
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - backup:/backup:ro
+      - $BACKUP_DIR/database:/archive
+    configs:
+      - source: timezone
+        target: /etc/localtime
     depends_on:
       database:
         condition: service_healthy