From dcf35804127c5c2732d51b3aeb787288d0e1293d Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Thu, 5 Oct 2017 13:39:07 +0200 Subject: [PATCH 01/53] Add LICENSE.txt --- LICENSE.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..a52c1e4 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) 2017 Jarek Lipski + +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 4d42b327a3926102de7b8e305272ffa2ac29eb0a Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Thu, 5 Oct 2017 13:43:37 +0200 Subject: [PATCH 02/53] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..dc68780 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Jarek Lipski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From ea9e96573f1596491cf1721f55aaa635473de170 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Thu, 5 Oct 2017 13:44:14 +0200 Subject: [PATCH 03/53] Remove old invalid LICENSE.txt --- LICENSE.txt | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index a52c1e4..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -MIT License - -Copyright (c) 2017 Jarek Lipski - -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 91f4149c9d413564c0de2e5321708497a4f45ee9 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 14 Jan 2018 13:39:30 +0100 Subject: [PATCH 04/53] Allow backing-up and restoring from stdin --- volume-backup.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/volume-backup.sh b/volume-backup.sh index b31f183..e99f557 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -6,18 +6,23 @@ usage() { } backup() { - mkdir -p `dirname /backup/$ARCHIVE` - tar -cjf /backup/$ARCHIVE -C /volume ./ + if ! [ "$ARCHIVE" == "-" ]; then + mkdir -p `dirname /backup/$ARCHIVE` + fi + + tar -cjf $ARCHIVE_PATH -C /volume ./ } restore() { - if ! [ -e /backup/$ARCHIVE ]; then - echo "Archive file $ARCHIVE does not exist" - exit 1 + if ! [ "$ARCHIVE" == "-" ]; then + if ! [ -e $ARCHIVE_PATH ]; then + echo "Archive file $ARCHIVE does not exist" + exit 1 + fi fi rm -rf /volume/* /volume/..?* /volume/.[!.]* - tar -C /volume/ -xjf /backup/$ARCHIVE + tar -C /volume/ -xjf $ARCHIVE_PATH } # Needed because sometimes pty is not ready when executing docker-compose run @@ -31,7 +36,13 @@ fi OPERATION=$1 -ARCHIVE=${2%%.tar.bz2}.tar.bz2 +if [ "$2" == "-" ]; then + ARCHIVE=$2 + ARCHIVE_PATH=$ARCHIVE +else + ARCHIVE=${2%%.tar.bz2}.tar.bz2 + ARCHIVE_PATH=/backup/$ARCHIVE +fi case "$OPERATION" in "backup" ) From 3b59c8e418a26cb36e4a8413dfbdc340fe47b777 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 14 Jan 2018 14:21:49 +0100 Subject: [PATCH 05/53] Update README to add info about stdin/stdout --- README.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7da327a..f0ba357 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,21 @@ Syntax: For example: - docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup backup archive1 + docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup backup some_archive -will archive volume named `some_volume` to `/tmp/archive.tar.bz2` archive file. +will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive file. + +It's also possible to backup a volume to standard output, which avoids mounting a second backup volume and allows to redirect it to a file, network, etc. + +Syntax: + + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - > [archive-name] + +For example: + + docker run -v some_volume:/volume --rm loomchild/volume-backup backup - > some_archive.tar.bz2 + +will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. ## Restore @@ -26,6 +38,20 @@ Syntax: For example: - docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup restore archive1 + docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup restore some_archive + +will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. + +It's also possible to restore a volume from standard input, which avoids mounting a second backup volume. + +**Note**: Don't forget the `-i` switch for interactive operation. + +Syntax: + + cat [archive-name] | docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - + +For example: + + cat some_archive.tar.bz2 | docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore - -will clean and restore volume named `some_volume` from `/tmp/archive.tar.bz2` archive file. +will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. From 026832a24bc4d45bbad5b5988286786c4ae6c402 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 14 Jan 2018 14:25:01 +0100 Subject: [PATCH 06/53] Update README --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f0ba357..fb6fc83 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ For example: will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive file. -It's also possible to backup a volume to standard output, which avoids mounting a second backup volume and allows to redirect it to a file, network, etc. +### Backup to standard output + +This avoids mounting a second backup volume and allows to redirect it to a file, network, etc. Syntax: @@ -42,7 +44,9 @@ For example: will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. -It's also possible to restore a volume from standard input, which avoids mounting a second backup volume. +### Restore from standard input + +This avoids mounting a second backup volume. **Note**: Don't forget the `-i` switch for interactive operation. From 21d40355b086198ba66afe950d0c7bb32aa6cc27 Mon Sep 17 00:00:00 2001 From: Damian Bushong Date: Mon, 14 Jan 2019 10:08:14 -0600 Subject: [PATCH 07/53] Advise Docker for Windows users against involving stdin/stdout --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fb6fc83..5c18756 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive f This avoids mounting a second backup volume and allows to redirect it to a file, network, etc. +**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. + Syntax: docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - > [archive-name] @@ -50,6 +52,8 @@ This avoids mounting a second backup volume. **Note**: Don't forget the `-i` switch for interactive operation. +**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. + Syntax: cat [archive-name] | docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - From ca8268850690e9c6e8aed0de44af7f2c7348bf9d Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 9 Feb 2019 21:37:14 +0100 Subject: [PATCH 08/53] mention stidin/stdout in usage --- volume-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volume-backup.sh b/volume-backup.sh index e99f557..f21d54c 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -1,7 +1,7 @@ #!/bin/sh usage() { - echo "Usage: volume-backup " + echo "Usage: volume-backup " exit } From 222b9b170ec42f8c94607cb2466a089436b2b1b0 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 10 Feb 2019 11:17:40 +0100 Subject: [PATCH 09/53] write errors to stderr to show them on redirect --- volume-backup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volume-backup.sh b/volume-backup.sh index f21d54c..550f776 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -1,8 +1,8 @@ #!/bin/sh usage() { - echo "Usage: volume-backup " - exit + >&2 echo "Usage: volume-backup " + exit 1 } backup() { @@ -16,7 +16,7 @@ backup() { restore() { if ! [ "$ARCHIVE" == "-" ]; then if ! [ -e $ARCHIVE_PATH ]; then - echo "Archive file $ARCHIVE does not exist" + >&2 echo "Archive file $ARCHIVE does not exist" exit 1 fi fi From 3b77acf1c16d1f111a5c71196e128d533d7dcc8d Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 10 Feb 2019 11:26:09 +0100 Subject: [PATCH 10/53] avoid backing-up empty volumes to avoid typos --- volume-backup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/volume-backup.sh b/volume-backup.sh index 550f776..463a8af 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -6,6 +6,11 @@ usage() { } backup() { + if [ -z "$(ls -A /volume)" ]; then + >&2 echo "Volume is empty or missing, check if you specified a correct name" + exit 1 + fi + if ! [ "$ARCHIVE" == "-" ]; then mkdir -p `dirname /backup/$ARCHIVE` fi From f12f5865e55f5e88e3e36be5aff4c7da5233f6d7 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 12:08:32 +0200 Subject: [PATCH 11/53] add misc section to README --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c18756..bb61b3c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). -**Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. +**Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. See [Miscellaneous](#miscellaneous) for instructions. ## Backup @@ -63,3 +63,13 @@ For example: cat some_archive.tar.bz2 | docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore - will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. + +### Miscellaneous + +1. Upgrade / update volume-backup + + docker pull loomchild/volume-backup + +1. Find all containers using a volume (to stop them before backing-up) + + docker ps -a --filter volume=[volume-name] From 0c3fddb08e1d3011e85cc1191e1dee301f52fda1 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 12:11:46 +0200 Subject: [PATCH 12/53] fix formatting --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb61b3c..ac1aed0 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,11 @@ will clean and restore volume named `some_volume` from `some_archive.tar.bz2` ar ### Miscellaneous 1. Upgrade / update volume-backup - + ``` docker pull loomchild/volume-backup + ``` 1. Find all containers using a volume (to stop them before backing-up) - + ``` docker ps -a --filter volume=[volume-name] + ``` From 82d3800b4581b91fc45b99a353bace68ca4ab93a Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 13:48:27 +0200 Subject: [PATCH 13/53] exclude files from backup --- README.md | 5 +++++ volume-backup.sh | 44 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ac1aed0..4faf2b9 100644 --- a/README.md +++ b/README.md @@ -75,3 +75,8 @@ will clean and restore volume named `some_volume` from `some_archive.tar.bz2` ar ``` docker ps -a --filter volume=[volume-name] ``` + +1. Exclude some files from the backup and output archive to stdout + ``` + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] + ``` diff --git a/volume-backup.sh b/volume-backup.sh index 463a8af..c1c17d7 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -1,8 +1,10 @@ #!/bin/sh usage() { - >&2 echo "Usage: volume-backup " - exit 1 + >&2 echo "Usage: volume-backup [options] " + >&2 echo "" + >&2 echo "Options:" + >&2 echo " -e exclude files or directories (only for backup operation)" } backup() { @@ -15,7 +17,7 @@ backup() { mkdir -p `dirname /backup/$ARCHIVE` fi - tar -cjf $ARCHIVE_PATH -C /volume ./ + tar -cjf $ARCHIVE_PATH -C /volume $TAROPTS ./ } restore() { @@ -27,7 +29,7 @@ restore() { fi rm -rf /volume/* /volume/..?* /volume/.[!.]* - tar -C /volume/ -xjf $ARCHIVE_PATH + tar -C /volume/ -xjf $TAROPTS $ARCHIVE_PATH } # Needed because sometimes pty is not ready when executing docker-compose run @@ -35,17 +37,39 @@ restore() { # TODO: remove after above pull request or equivalent is merged sleep 1 -if [ $# -ne 2 ]; then +OPERATION=$1 + +TAROPTS="" + +OPTIND=2 + +while getopts "h?e:" OPTION; do + case "$OPTION" in + h|\?) + usage + exit 0 + ;; + e) if [ -z "$OPTARG" -o "$OPERATION" != "backup" ]; then + usage + exit 1 + fi + TAROPTS="$TAROPTS --exclude $OPTARG" + ;; + esac +done + +shift $((OPTIND - 1)) + +if [ $# -lt 1 ]; then usage + exit 1 fi -OPERATION=$1 - -if [ "$2" == "-" ]; then - ARCHIVE=$2 +if [ "$1" == "-" ]; then + ARCHIVE=$1 ARCHIVE_PATH=$ARCHIVE else - ARCHIVE=${2%%.tar.bz2}.tar.bz2 + ARCHIVE=${1%%.tar.bz2}.tar.bz2 ARCHIVE_PATH=/backup/$ARCHIVE fi From 9c38e057e95eb2fcc4a35ed744158ac1abc701d2 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 13:59:26 +0200 Subject: [PATCH 14/53] allow aborting the operation --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3075837..b9f3b57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM alpine +RUN apk update && apk add --no-cache dumb-init + COPY volume-backup.sh / ENTRYPOINT [ "/bin/sh", "/volume-backup.sh" ] From 566915065f2d39a93f4d8b53e11818bbee56a92d Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 14:09:55 +0200 Subject: [PATCH 15/53] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4faf2b9..9d56c10 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ will clean and restore volume named `some_volume` from `some_archive.tar.bz2` ar docker ps -a --filter volume=[volume-name] ``` -1. Exclude some files from the backup and output archive to stdout +1. Exclude some files from the backup and send the archive to stdout ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] ``` From f5e0a593e3534b14a27a20ef5e9d0cb87686a131 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 16:38:21 +0200 Subject: [PATCH 16/53] reorganize README --- README.md | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9d56c10..b579509 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,10 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine ## Backup -Syntax: - - docker run -v [volume-name]:/volume -v [output-dir]:/backup --rm loomchild/volume-backup backup [archive-name] - -For example: - - docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup backup some_archive - -will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive file. - -### Backup to standard output +### To standard output This avoids mounting a second backup volume and allows to redirect it to a file, network, etc. -**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. - Syntax: docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - > [archive-name] @@ -32,27 +20,29 @@ For example: will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. -## Restore +**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. -**Note**: This operation will delete all contents of the volume +### To a file Syntax: - docker run -v [volume-name]:/volume -v [output-dir]:/backup --rm loomchild/volume-backup restore [archive-name] + docker run -v [volume-name]:/volume -v [output-dir]:/backup --rm loomchild/volume-backup backup [archive-name] For example: - docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup restore some_archive + docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup backup some_archive -will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. +will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive file. + +## Restore + +**WARNING**: This operation will delete all contents of the volume ### Restore from standard input This avoids mounting a second backup volume. -**Note**: Don't forget the `-i` switch for interactive operation. - -**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. +*NOTE*: Don't forget the `-i` switch for interactive operation. Syntax: @@ -64,7 +54,21 @@ For example: will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. -### Miscellaneous +**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. + +### From a file + +Syntax: + + docker run -v [volume-name]:/volume -v [output-dir]:/backup --rm loomchild/volume-backup restore [archive-name] + +For example: + + docker run -v some_volume:/volume -v /tmp:/backup --rm loomchild/volume-backup restore some_archive + +will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. + +## Miscellaneous 1. Upgrade / update volume-backup ``` From c3858340e0e6eac187a475fd242c40fbb30b0e7c Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 16:39:34 +0200 Subject: [PATCH 17/53] improve README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b579509..079811c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine ## Backup -### To standard output +### Backup to standard output This avoids mounting a second backup volume and allows to redirect it to a file, network, etc. @@ -22,7 +22,7 @@ will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. **WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. -### To a file +### Backup to a file Syntax: @@ -56,7 +56,7 @@ will clean and restore volume named `some_volume` from `some_archive.tar.bz2` ar **WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. -### From a file +### Restore from a file Syntax: From dc846456899d1aa30a05980a0a4287dab573c7ba Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 16:41:14 +0200 Subject: [PATCH 18/53] update LICENSE year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index dc68780..9d19387 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Jarek Lipski +Copyright (c) 2017 - 2019 Jarek Lipski Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 102b6ba816457349b807f2b3ba9852f7d56c426a Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 13 Apr 2019 17:18:00 +0200 Subject: [PATCH 19/53] choose compression algorithm or no algorithm --- README.md | 5 +++++ volume-backup.sh | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 079811c..86b1d32 100644 --- a/README.md +++ b/README.md @@ -84,3 +84,8 @@ will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] ``` + +1. Use different compression algorithm for better performance + ``` + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -c gz - > [archive-name] + ``` diff --git a/volume-backup.sh b/volume-backup.sh index c1c17d7..b36490b 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -4,6 +4,7 @@ usage() { >&2 echo "Usage: volume-backup [options] " >&2 echo "" >&2 echo "Options:" + >&2 echo " -c chooose compression algorithm: bz2 (default), gz and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" } @@ -17,7 +18,7 @@ backup() { mkdir -p `dirname /backup/$ARCHIVE` fi - tar -cjf $ARCHIVE_PATH -C /volume $TAROPTS ./ + tar -C /volume $TAROPTS -cf $ARCHIVE_PATH ./ } restore() { @@ -29,7 +30,7 @@ restore() { fi rm -rf /volume/* /volume/..?* /volume/.[!.]* - tar -C /volume/ -xjf $TAROPTS $ARCHIVE_PATH + tar -C /volume/ $TAROPTS -xf $ARCHIVE_PATH } # Needed because sometimes pty is not ready when executing docker-compose run @@ -40,16 +41,25 @@ sleep 1 OPERATION=$1 TAROPTS="" +COMPRESSION="bz2" OPTIND=2 -while getopts "h?e:" OPTION; do +while getopts "h?c:e:" OPTION; do case "$OPTION" in h|\?) usage exit 0 ;; - e) if [ -z "$OPTARG" -o "$OPERATION" != "backup" ]; then + c) + if [ -z "$OPTARG" ]; then + usage + exit 1 + fi + COMPRESSION=$OPTARG + ;; + e) + if [ -z "$OPTARG" -o "$OPERATION" != "backup" ]; then usage exit 1 fi @@ -65,11 +75,29 @@ if [ $# -lt 1 ]; then exit 1 fi +case "$COMPRESSION" in +bz2) + TAROPTS="$TAROPTS -j" + EXTENSION=.tar.bz2 + ;; +gz) + TAROPTS="$TAROPTS -z" + EXTENSION=.tar.gz + ;; +none|0) + EXTENSION=.tar + ;; +*) + usage + exit 1 + ;; +esac + if [ "$1" == "-" ]; then ARCHIVE=$1 ARCHIVE_PATH=$ARCHIVE else - ARCHIVE=${1%%.tar.bz2}.tar.bz2 + ARCHIVE=${1%%$EXTENSION}$EXTENSION ARCHIVE_PATH=/backup/$ARCHIVE fi From 5efdf1f5a14c91b62ce49e5e62260ac6fd0bc5dc Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 2 Jun 2019 15:35:51 +0200 Subject: [PATCH 20/53] improve Windows README warning --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 86b1d32..2290b86 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ For example: will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. -**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. +**WARNING**: This method should not be used under PowerShell on Windows as no usable backup will be generated. ### Backup to a file @@ -54,8 +54,6 @@ For example: will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. -**WARNING**: This method should not be used with a Docker for Windows installation; no usable backup will be generated. - ### Restore from a file Syntax: From 7794551e317984263dd315ed2f42150531c32e56 Mon Sep 17 00:00:00 2001 From: Lennart Weller Date: Fri, 14 Jun 2019 11:33:41 +0200 Subject: [PATCH 21/53] Add a new use case to the README Add an example of how to use the image to directly copy volumes between hosts. --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 2290b86..3ce3376 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,22 @@ For example: will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. +### Copy volume between hosts + +One good example of how you can use the output to stdout is directly migrating the volume to a new host + +Syntax: + + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - |\ + ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - + +*Note* In case traffic is not an issue it is recommended to turn of compression for this process + +For example: + + docker run -v some_volume:/volume --rm loomchild/volume-backup backup -c none - |\ + ssh user@new.machine docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore -c none - + ## Miscellaneous 1. Upgrade / update volume-backup From 18ee3743cdfecf28ff69ad9b2f16c80e48fff6a6 Mon Sep 17 00:00:00 2001 From: Lennart Weller Date: Wed, 19 Jun 2019 14:06:13 +0200 Subject: [PATCH 22/53] Clarify note on cpu/traffic trade-off --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ce3376..fa9a8da 100644 --- a/README.md +++ b/README.md @@ -68,14 +68,14 @@ will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz ### Copy volume between hosts -One good example of how you can use the output to stdout is directly migrating the volume to a new host +One good example of how you can use the output to stdout would be directly migrating the volume to a new host Syntax: docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - |\ ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - -*Note* In case traffic is not an issue it is recommended to turn of compression for this process +*Note* In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression as shown in the example below. For example: From 01a5fdb7cb94e32f56c1f4f067e7f00a182f093c Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Wed, 19 Jun 2019 17:17:22 +0200 Subject: [PATCH 23/53] uniform note syntax --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa9a8da..8307ed2 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive f This avoids mounting a second backup volume. -*NOTE*: Don't forget the `-i` switch for interactive operation. +**Note**: Don't forget the `-i` switch for interactive operation. Syntax: @@ -75,7 +75,7 @@ Syntax: docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - |\ ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - -*Note* In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression as shown in the example below. +**Note**: In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression as shown in the example below. For example: From 872720b0cc1bb89713c06083e8a7e02618274205 Mon Sep 17 00:00:00 2001 From: Paul Kronenwetter Date: Thu, 12 Sep 2019 09:42:01 +0000 Subject: [PATCH 24/53] Add xz compression (#18) --- Dockerfile | 2 +- volume-backup.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b9f3b57..d8246ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init +RUN apk update && apk add --no-cache dumb-init xz COPY volume-backup.sh / diff --git a/volume-backup.sh b/volume-backup.sh index b36490b..feabd18 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -4,7 +4,7 @@ usage() { >&2 echo "Usage: volume-backup [options] " >&2 echo "" >&2 echo "Options:" - >&2 echo " -c chooose compression algorithm: bz2 (default), gz and 0 (none)" + >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" } @@ -76,6 +76,10 @@ if [ $# -lt 1 ]; then fi case "$COMPRESSION" in +xz) + TAROPTS="$TAROPTS -J" + EXTENSION=.tar.xz + ;; bz2) TAROPTS="$TAROPTS -j" EXTENSION=.tar.bz2 From c7eaf8881048b84b436deb6025e3b2d564cedd70 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 11 Jan 2020 19:52:58 +0100 Subject: [PATCH 25/53] use GNU tar --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d8246ea..c025d44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz +RUN apk update && apk add --no-cache dumb-init xz tar COPY volume-backup.sh / From 189c389dbed6212e6e6e983fb46d85ec21760b3e Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 11 Jan 2020 19:54:19 +0100 Subject: [PATCH 26/53] add verbose flag --- README.md | 4 ++++ volume-backup.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8307ed2..71f77ce 100644 --- a/README.md +++ b/README.md @@ -103,3 +103,7 @@ For example: ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -c gz - > [archive-name] ``` +1. Show simple progress indicator using verbose `-v` flag (works both for backup and restore) + ``` + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -v > [archive-name] + ``` diff --git a/volume-backup.sh b/volume-backup.sh index feabd18..f8415e8 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -6,6 +6,7 @@ usage() { >&2 echo "Options:" >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" + >&2 echo " -v verbose" } backup() { @@ -45,7 +46,7 @@ COMPRESSION="bz2" OPTIND=2 -while getopts "h?c:e:" OPTION; do +while getopts "h?vc:e:" OPTION; do case "$OPTION" in h|\?) usage @@ -65,6 +66,10 @@ while getopts "h?c:e:" OPTION; do fi TAROPTS="$TAROPTS --exclude $OPTARG" ;; + v) + TAROPTS="$TAROPTS --checkpoint=.1000" + EOLN=1 + ;; esac done @@ -116,3 +121,7 @@ restore usage ;; esac + +if ! [ -z "$EOLN" ]; then + >&2 echo +fi From d04843942053632ad3d1c400abdb9e417abdcbd0 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 12 Jan 2020 11:43:39 +0100 Subject: [PATCH 27/53] handle signals via dumb init (Ctrl-C et al) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c025d44..5a21b38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ RUN apk update && apk add --no-cache dumb-init xz tar COPY volume-backup.sh / -ENTRYPOINT [ "/bin/sh", "/volume-backup.sh" ] +ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/volume-backup.sh" ] From f8b07e804e5c43d0c7530b2f357006ac2a13e118 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Thu, 18 Feb 2021 10:51:35 +0100 Subject: [PATCH 28/53] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71f77ce..3dcc8fa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # volume-backup -An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). +An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). For more info, read my article on [Medium](https://medium.com/@jareklipski/backup-restore-docker-named-volumes-350397b8e362) **Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. See [Miscellaneous](#miscellaneous) for instructions. From 0cec0d76e32ac1ad314199cea4bb15fa8e34ca82 Mon Sep 17 00:00:00 2001 From: ksurl Date: Sun, 14 Mar 2021 10:38:23 -0700 Subject: [PATCH 29/53] add zstd (#36) --- Dockerfile | 2 +- volume-backup.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5a21b38..a5efc95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz tar +RUN apk update && apk add --no-cache dumb-init xz tar zstd COPY volume-backup.sh / diff --git a/volume-backup.sh b/volume-backup.sh index f8415e8..07405f4 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -93,6 +93,10 @@ gz) TAROPTS="$TAROPTS -z" EXTENSION=.tar.gz ;; +zstd) + TAROPTS="$TAROPTS -I zstd" + EXTENSION=.tar.zstd + ;; none|0) EXTENSION=.tar ;; From f62b9a70cf01d532b2fc26b62a4f941a6501ca1f Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 14 Mar 2021 18:39:11 +0100 Subject: [PATCH 30/53] add zstd to usage --- volume-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volume-backup.sh b/volume-backup.sh index 07405f4..2efe35e 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -4,7 +4,7 @@ usage() { >&2 echo "Usage: volume-backup [options] " >&2 echo "" >&2 echo "Options:" - >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz and 0 (none)" + >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, zstd and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" >&2 echo " -v verbose" } From 7e8b6b2d57975ed7d81809cf1e03d4438707ad2b Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 20 Mar 2021 19:35:48 +0100 Subject: [PATCH 31/53] pass additional args to Tar --- README.md | 7 +++++++ volume-backup.sh | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3dcc8fa..9f6f044 100644 --- a/README.md +++ b/README.md @@ -107,3 +107,10 @@ For example: ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -v > [archive-name] ``` + +1. Pass additional arguments to the Tar utility using `-x` option + ``` + docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -x --verbose > [archive-name] + ``` + + diff --git a/volume-backup.sh b/volume-backup.sh index 2efe35e..d06f16b 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -7,6 +7,7 @@ usage() { >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, zstd and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" >&2 echo " -v verbose" + >&2 echo " -x pass additional arguments to the Tar utility" } backup() { @@ -46,7 +47,7 @@ COMPRESSION="bz2" OPTIND=2 -while getopts "h?vc:e:" OPTION; do +while getopts "h?vc:e:x:" OPTION; do case "$OPTION" in h|\?) usage @@ -70,6 +71,12 @@ while getopts "h?vc:e:" OPTION; do TAROPTS="$TAROPTS --checkpoint=.1000" EOLN=1 ;; + x) + if [ -z "$OPTARG" ]; then + usage + exit 1 + fi + TAROPTS="$TAROPTS $OPTARG" esac done From 41074d29715b50714f8b97404f5bdb887189778b Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 20 Mar 2021 20:07:35 +0100 Subject: [PATCH 32/53] avoid overwriting data when restoring --- README.md | 2 +- volume-backup.sh | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f6f044..d26eb40 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive f ## Restore -**WARNING**: This operation will delete all contents of the volume +Restore will fail if the target volume is not empty (use `-f` flag to override). ### Restore from standard input diff --git a/volume-backup.sh b/volume-backup.sh index d06f16b..19f7ec2 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -6,8 +6,9 @@ usage() { >&2 echo "Options:" >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, zstd and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" - >&2 echo " -v verbose" + >&2 echo " -f force overwrite even if target volume is not empty during restore" >&2 echo " -x pass additional arguments to the Tar utility" + >&2 echo " -v verbose" } backup() { @@ -31,6 +32,12 @@ restore() { fi fi + + if ! [ -z "$(ls -A /volume)" -o -n "$FORCE" ]; then + >&2 echo "Target volume is not empty, aborting; use -f to override" + exit 1 + fi + rm -rf /volume/* /volume/..?* /volume/.[!.]* tar -C /volume/ $TAROPTS -xf $ARCHIVE_PATH } @@ -44,10 +51,11 @@ OPERATION=$1 TAROPTS="" COMPRESSION="bz2" +FORCE="" OPTIND=2 -while getopts "h?vc:e:x:" OPTION; do +while getopts "h?vfc:e:x:" OPTION; do case "$OPTION" in h|\?) usage @@ -67,6 +75,13 @@ while getopts "h?vc:e:x:" OPTION; do fi TAROPTS="$TAROPTS --exclude $OPTARG" ;; + f) + if [ "$OPERATION" != "restore" ]; then + usage + exit 1 + fi + FORCE=1 + ;; v) TAROPTS="$TAROPTS --checkpoint=.1000" EOLN=1 @@ -77,6 +92,7 @@ while getopts "h?vc:e:x:" OPTION; do exit 1 fi TAROPTS="$TAROPTS $OPTARG" + ;; esac done From 3bb7e1886101d2e50cd802b7da5aad7ded3f4cf3 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 20 Mar 2021 20:16:13 +0100 Subject: [PATCH 33/53] remove unnecessary 1 second delay --- volume-backup.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volume-backup.sh b/volume-backup.sh index 19f7ec2..ccf5807 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -42,11 +42,6 @@ restore() { tar -C /volume/ $TAROPTS -xf $ARCHIVE_PATH } -# Needed because sometimes pty is not ready when executing docker-compose run -# See https://github.com/docker/compose/pull/4738 for more details -# TODO: remove after above pull request or equivalent is merged -sleep 1 - OPERATION=$1 TAROPTS="" From 8f24efce1f2111a927b430bfec3b1af97d82ec50 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 20 Mar 2021 20:39:40 +0100 Subject: [PATCH 34/53] added info about restoring labels --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d26eb40..1e70001 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). For more info, read my article on [Medium](https://medium.com/@jareklipski/backup-restore-docker-named-volumes-350397b8e362) **Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. See [Miscellaneous](#miscellaneous) for instructions. +**Note**: When using docker-compose, make sure to backup and restore volume labels. See [Miscellaneous](#miscellaneous) for more information. ## Backup @@ -107,10 +108,8 @@ For example: ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -v > [archive-name] ``` - 1. Pass additional arguments to the Tar utility using `-x` option ``` docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -x --verbose > [archive-name] ``` - - +1. Volume labels are not backed-up or restored automatically, but they might be required for your application to work (e.g. when using docker-compose). If you need to preserve them, create a label backup file as follows: `docker inspect [volume-name] -f "{{json .Labels}}" > labels.json`. When restoring your data, target volume needs to be created manually with labels before launching the restore script: `docker volume create --label "label1" --label "label2" [volume-name]`. From e63597e72587d6b1ed01aec43a166b41dcfd35ef Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 20 Mar 2021 20:40:39 +0100 Subject: [PATCH 35/53] formatting --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1e70001..ddce7d7 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). For more info, read my article on [Medium](https://medium.com/@jareklipski/backup-restore-docker-named-volumes-350397b8e362) **Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. See [Miscellaneous](#miscellaneous) for instructions. + **Note**: When using docker-compose, make sure to backup and restore volume labels. See [Miscellaneous](#miscellaneous) for more information. ## Backup From e09c1a77c780afd14b65afd8631b0db8076fbdf7 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Wed, 26 May 2021 20:03:06 +0200 Subject: [PATCH 36/53] mention --log-driver none --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ddce7d7..29052e0 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,16 @@ This avoids mounting a second backup volume and allows to redirect it to a file, Syntax: - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup - > [archive-name] For example: - docker run -v some_volume:/volume --rm loomchild/volume-backup backup - > some_archive.tar.bz2 + docker run -v some_volume:/volume --rm --log-driver none loomchild/volume-backup backup - > some_archive.tar.bz2 will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. +*Note*: `--log-driver none` option is necessary to avoid storing an entire backup in a temporary stdout JSON file. More info: https://docs.docker.com/config/containers/logging/configure/, https://github.com/loomchild/volume-backup/issues/39. Thanks @pschonmann for raising this. + **WARNING**: This method should not be used under PowerShell on Windows as no usable backup will be generated. ### Backup to a file @@ -74,14 +76,14 @@ One good example of how you can use the output to stdout would be directly migra Syntax: - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup - |\ + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup - |\ ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - **Note**: In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression as shown in the example below. For example: - docker run -v some_volume:/volume --rm loomchild/volume-backup backup -c none - |\ + docker run -v some_volume:/volume --rm --log-driver none loomchild/volume-backup backup -c none - |\ ssh user@new.machine docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore -c none - ## Miscellaneous @@ -98,19 +100,19 @@ For example: 1. Exclude some files from the backup and send the archive to stdout ``` - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] ``` 1. Use different compression algorithm for better performance ``` - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -c gz - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -c gz - > [archive-name] ``` 1. Show simple progress indicator using verbose `-v` flag (works both for backup and restore) ``` - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -v > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -v - > [archive-name] ``` 1. Pass additional arguments to the Tar utility using `-x` option ``` - docker run -v [volume-name]:/volume --rm loomchild/volume-backup backup -x --verbose > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -x --verbose - > [archive-name] ``` 1. Volume labels are not backed-up or restored automatically, but they might be required for your application to work (e.g. when using docker-compose). If you need to preserve them, create a label backup file as follows: `docker inspect [volume-name] -f "{{json .Labels}}" > labels.json`. When restoring your data, target volume needs to be created manually with labels before launching the restore script: `docker volume create --label "label1" --label "label2" [volume-name]`. From 3d7b8d3e4b4822901a2e718ad620f82d1825e750 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Wed, 26 May 2021 20:04:38 +0200 Subject: [PATCH 37/53] improve README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29052e0..7ae98ca 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For example: will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. -*Note*: `--log-driver none` option is necessary to avoid storing an entire backup in a temporary stdout JSON file. More info: https://docs.docker.com/config/containers/logging/configure/, https://github.com/loomchild/volume-backup/issues/39. Thanks @pschonmann for raising this. +*Note*: `--log-driver none` option is necessary to avoid storing an entire backup in a temporary stdout JSON file. More info in [Docker logging documentation](https://docs.docker.com/config/containers/logging/configure/) and in [this issue](https://github.com/loomchild/volume-backup/issues/39). **WARNING**: This method should not be used under PowerShell on Windows as no usable backup will be generated. From a3b2d9f94373cbba561712b0c2013c0e15567aa3 Mon Sep 17 00:00:00 2001 From: Azri Kahar <42867097+azrikahar@users.noreply.github.com> Date: Sat, 7 Aug 2021 22:13:40 +0800 Subject: [PATCH 38/53] add workflow for multiarch & ghcr support (#40) --- .github/workflows/publish-image.yml | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/publish-image.yml diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..9ba87dd --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,40 @@ +name: publish-image + +on: + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: | + loomchild/volume-backup:latest + ghcr.io/loomchild/volume-backup:latest From 89a50cb9d18dcece5c1f7a0f1e11b0a9591248da Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 16:20:39 +0200 Subject: [PATCH 39/53] hardcode username (already used in tags) --- .github/workflows/publish-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 9ba87dd..48a3f88 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -20,14 +20,14 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: loomchild password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: loomchild password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push From 4c9a91afc0043006c00b79495af9f814704dd3df Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 16:27:35 +0200 Subject: [PATCH 40/53] release on every push --- .github/workflows/publish-image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 48a3f88..310b231 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -1,7 +1,9 @@ name: publish-image on: - workflow_dispatch: + push: + branches: + - 'master' jobs: docker: From c5a7dcbf67ae96efed52b20d9e04b7357e585d7e Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 16:41:05 +0200 Subject: [PATCH 41/53] add pbzip2 --- Dockerfile | 2 +- volume-backup.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a5efc95..fc6b4e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz tar zstd +RUN apk update && apk add --no-cache dumb-init xz tar zstd pbzip2 COPY volume-backup.sh / diff --git a/volume-backup.sh b/volume-backup.sh index ccf5807..21c36ec 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -115,6 +115,10 @@ zstd) TAROPTS="$TAROPTS -I zstd" EXTENSION=.tar.zstd ;; +pbz2) + TAROPTS="$TAROPTS -I pbzip2" + EXTENSION=.tar.bz2 + ;; none|0) EXTENSION=.tar ;; From faa7828a26fc98bf6f83852e7d2aed71e108db27 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 16:49:32 +0200 Subject: [PATCH 42/53] Revert "add pbzip2" This reverts commit c5a7dcbf67ae96efed52b20d9e04b7357e585d7e. --- Dockerfile | 2 +- volume-backup.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc6b4e7..a5efc95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz tar zstd pbzip2 +RUN apk update && apk add --no-cache dumb-init xz tar zstd COPY volume-backup.sh / diff --git a/volume-backup.sh b/volume-backup.sh index 21c36ec..ccf5807 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -115,10 +115,6 @@ zstd) TAROPTS="$TAROPTS -I zstd" EXTENSION=.tar.zstd ;; -pbz2) - TAROPTS="$TAROPTS -I pbzip2" - EXTENSION=.tar.bz2 - ;; none|0) EXTENSION=.tar ;; From 124377e8f6ba7a27802f3fcf331a03f2803f7ae4 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 17:08:48 +0200 Subject: [PATCH 43/53] Add ghcr comment in README --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae98ca..d968a49 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For example: will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. -*Note*: `--log-driver none` option is necessary to avoid storing an entire backup in a temporary stdout JSON file. More info in [Docker logging documentation](https://docs.docker.com/config/containers/logging/configure/) and in [this issue](https://github.com/loomchild/volume-backup/issues/39). +**Note**: `--log-driver none` option is necessary to avoid storing an entire backup in a temporary stdout JSON file. More info in [Docker logging documentation](https://docs.docker.com/config/containers/logging/configure/) and in [this issue](https://github.com/loomchild/volume-backup/issues/39). **WARNING**: This method should not be used under PowerShell on Windows as no usable backup will be generated. @@ -92,6 +92,11 @@ For example: ``` docker pull loomchild/volume-backup ``` + Alternatively volume-backup is also available from ghcr.io (GitHub container registry), to avoid DockerHub usage limits: + ``` + docker pull ghcr.io/loomchild/volume-backup + ``` + **Note**: you'll need to use ghcr.io/loomchild/volume-backup instead of just loomchild/volume-backup when running the utility. 1. Find all containers using a volume (to stop them before backing-up) ``` From 9195dd942c1f698d061842872afce149eb68ac61 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 17:09:53 +0200 Subject: [PATCH 44/53] README formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d968a49..f4fd91b 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ For example: ``` docker pull ghcr.io/loomchild/volume-backup ``` - **Note**: you'll need to use ghcr.io/loomchild/volume-backup instead of just loomchild/volume-backup when running the utility. + **Note**: you'll need to write `ghcr.io/loomchild/volume-backup` instead of just `loomchild/volume-backup` when running the utility. 1. Find all containers using a volume (to stop them before backing-up) ``` From eacf96f904f6c7e2bdecf070bb9d518a9e5a7d0f Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 7 Aug 2021 17:12:08 +0200 Subject: [PATCH 45/53] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f4fd91b..b44d3bb 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,8 @@ For example: ``` docker pull loomchild/volume-backup ``` - Alternatively volume-backup is also available from ghcr.io (GitHub container registry), to avoid DockerHub usage limits: + +1. volume-backup is also available from GitHub Container Registry (ghcr.io), to avoid DockerHub usage limits: ``` docker pull ghcr.io/loomchild/volume-backup ``` From 0c12c4401032056f0e041303ec00c1c029cf9219 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Mon, 9 Aug 2021 17:12:06 +0200 Subject: [PATCH 46/53] add pigz --- Dockerfile | 2 +- README.md | 2 +- volume-backup.sh | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a5efc95..708da23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz tar zstd +RUN apk update && apk add --no-cache dumb-init xz tar pigz zstd COPY volume-backup.sh / diff --git a/README.md b/README.md index b44d3bb..5cadf6b 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ For example: 1. Use different compression algorithm for better performance ``` - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -c gz - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -c pigz - > [archive-name] ``` 1. Show simple progress indicator using verbose `-v` flag (works both for backup and restore) ``` diff --git a/volume-backup.sh b/volume-backup.sh index ccf5807..bfe5795 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -4,7 +4,7 @@ usage() { >&2 echo "Usage: volume-backup [options] " >&2 echo "" >&2 echo "Options:" - >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, zstd and 0 (none)" + >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, pigz, zstd and 0 (none)" >&2 echo " -e exclude files or directories (only for backup operation)" >&2 echo " -f force overwrite even if target volume is not empty during restore" >&2 echo " -x pass additional arguments to the Tar utility" @@ -111,6 +111,10 @@ gz) TAROPTS="$TAROPTS -z" EXTENSION=.tar.gz ;; +pigz) + TAROPTS="$TAROPTS -I pigz" + EXTENSION=.tar.gz + ;; zstd) TAROPTS="$TAROPTS -I zstd" EXTENSION=.tar.zstd From 5c41e8770855b2919481446589a72e7d41d41fc9 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Tue, 9 Nov 2021 11:58:50 +0100 Subject: [PATCH 47/53] schedule build every 1st day of the month --- .github/workflows/publish-image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 310b231..343df9b 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -4,7 +4,8 @@ on: push: branches: - 'master' - + schedule: + - cron: '0 4 1 * *' jobs: docker: runs-on: ubuntu-latest From 8b478fd48e89fabb611ef5ade8807cba481168b5 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Thu, 2 Jun 2022 13:41:20 +0200 Subject: [PATCH 48/53] replace cats by pipes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5cadf6b..da58a52 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,11 @@ This avoids mounting a second backup volume. Syntax: - cat [archive-name] | docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - + docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - < [archive-name] For example: - cat some_archive.tar.bz2 | docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore - + docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore - < some_archive.tar.bz2 will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. From 83bff02b5db54bcf6a841772d74b3c08df11a90f Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 9 Oct 2022 15:54:54 +0200 Subject: [PATCH 49/53] use bash shell, adaptive zstd --- Dockerfile | 2 +- volume-backup.sh | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 708da23..4a6bb5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk update && apk add --no-cache dumb-init xz tar pigz zstd +RUN apk update && apk add --no-cache dumb-init bash xz tar pigz zstd COPY volume-backup.sh / diff --git a/volume-backup.sh b/volume-backup.sh index bfe5795..f965f13 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash usage() { >&2 echo "Usage: volume-backup [options] " @@ -21,7 +21,7 @@ backup() { mkdir -p `dirname /backup/$ARCHIVE` fi - tar -C /volume $TAROPTS -cf $ARCHIVE_PATH ./ + tar -C /volume "${TAROPTS[@]}" -cf $ARCHIVE_PATH ./ } restore() { @@ -39,12 +39,12 @@ restore() { fi rm -rf /volume/* /volume/..?* /volume/.[!.]* - tar -C /volume/ $TAROPTS -xf $ARCHIVE_PATH + tar -C /volume/ "${TAROPTS[@]}" -xf $ARCHIVE_PATH } OPERATION=$1 -TAROPTS="" +TAROPTS=() COMPRESSION="bz2" FORCE="" @@ -68,7 +68,7 @@ while getopts "h?vfc:e:x:" OPTION; do usage exit 1 fi - TAROPTS="$TAROPTS --exclude $OPTARG" + TAROPTS+=(--exclude $OPTARG) ;; f) if [ "$OPERATION" != "restore" ]; then @@ -78,7 +78,7 @@ while getopts "h?vfc:e:x:" OPTION; do FORCE=1 ;; v) - TAROPTS="$TAROPTS --checkpoint=.1000" + TAROPTS+=(--checkpoint=.1000) EOLN=1 ;; x) @@ -86,7 +86,9 @@ while getopts "h?vfc:e:x:" OPTION; do usage exit 1 fi - TAROPTS="$TAROPTS $OPTARG" + # Note: it doesn't support nested quotes, e.g. -x '-I "zstd -10"' + OPTARR=($OPTARG) + TAROPTS=(${TAROPTS[@]} ${OPTARR[@]}) ;; esac done @@ -100,23 +102,23 @@ fi case "$COMPRESSION" in xz) - TAROPTS="$TAROPTS -J" + TAROPTS+=(-J) EXTENSION=.tar.xz ;; bz2) - TAROPTS="$TAROPTS -j" + TAROPTS+=(-j) EXTENSION=.tar.bz2 ;; gz) - TAROPTS="$TAROPTS -z" + TAROPTS+=(-z) EXTENSION=.tar.gz ;; pigz) - TAROPTS="$TAROPTS -I pigz" + TAROPTS+=(-I pigz) EXTENSION=.tar.gz ;; zstd) - TAROPTS="$TAROPTS -I zstd" + TAROPTS+=(-I "zstd --adapt") EXTENSION=.tar.zstd ;; none|0) From fdffb03c4c337fb46518e8fd91975c0e97401e42 Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sun, 9 Oct 2022 16:23:57 +0200 Subject: [PATCH 50/53] make stdin/out syntax default, - optional --- README.md | 62 ++++++++++++++++++------------------------------ volume-backup.sh | 11 +++------ 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index da58a52..4713a94 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,13 @@ An utility to backup and restore [docker volumes](https://docs.docker.com/engine ## Backup -### Backup to standard output - -This avoids mounting a second backup volume and allows to redirect it to a file, network, etc. - Syntax: - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup > [archive-path] For example: - docker run -v some_volume:/volume --rm --log-driver none loomchild/volume-backup backup - > some_archive.tar.bz2 + docker run -v some_volume:/volume --rm --log-driver none loomchild/volume-backup backup > some_archive.tar.bz2 will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. @@ -26,7 +22,7 @@ will archive volume named `some_volume` to `some_archive.tar.bz2` archive file. **WARNING**: This method should not be used under PowerShell on Windows as no usable backup will be generated. -### Backup to a file +### Backup to a file (deprecated) Syntax: @@ -40,25 +36,20 @@ will archive volume named `some_volume` to `/tmp/some_archive.tar.bz2` archive f ## Restore -Restore will fail if the target volume is not empty (use `-f` flag to override). - -### Restore from standard input - -This avoids mounting a second backup volume. - -**Note**: Don't forget the `-i` switch for interactive operation. - Syntax: - docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - < [archive-name] + docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore < [archive-path] For example: - docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore - < some_archive.tar.bz2 + docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore < some_archive.tar.bz2 will clean and restore volume named `some_volume` from `some_archive.tar.bz2` archive file. -### Restore from a file +**Note**: Don't forget the `-i` switch for interactive operation. +**Note** Restore will fail if the target volume is not empty (use `-f` flag to override). + +### Restore from a file (deprecated) Syntax: @@ -70,22 +61,6 @@ For example: will clean and restore volume named `some_volume` from `/tmp/some_archive.tar.bz2` archive file. -### Copy volume between hosts - -One good example of how you can use the output to stdout would be directly migrating the volume to a new host - -Syntax: - - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup - |\ - ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore - - -**Note**: In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression as shown in the example below. - -For example: - - docker run -v some_volume:/volume --rm --log-driver none loomchild/volume-backup backup -c none - |\ - ssh user@new.machine docker run -i -v some_volume:/volume --rm loomchild/volume-backup restore -c none - - ## Miscellaneous 1. Upgrade / update volume-backup @@ -106,19 +81,28 @@ For example: 1. Exclude some files from the backup and send the archive to stdout ``` - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -e [excluded-glob] - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -e [excluded-glob] > [archive-path] ``` 1. Use different compression algorithm for better performance ``` - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -c pigz - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -c pigz > [archive-path] ``` + 1. Show simple progress indicator using verbose `-v` flag (works both for backup and restore) ``` - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -v - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -v > [archive-path] ``` + 1. Pass additional arguments to the Tar utility using `-x` option ``` - docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -x --verbose - > [archive-name] + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup -x --verbose > [archive-path] + ``` + +1. Directly migrate the volume to a new host ``` -1. Volume labels are not backed-up or restored automatically, but they might be required for your application to work (e.g. when using docker-compose). If you need to preserve them, create a label backup file as follows: `docker inspect [volume-name] -f "{{json .Labels}}" > labels.json`. When restoring your data, target volume needs to be created manually with labels before launching the restore script: `docker volume create --label "label1" --label "label2" [volume-name]`. + docker run -v [volume-name]:/volume --rm --log-driver none loomchild/volume-backup backup | ssh [receiver] docker run -i -v [volume-name]:/volume --rm loomchild/volume-backup restore + ``` + **Note**: In case there are no traffic limitations between the hosts you can trade CPU time for bandwidth by turning off compression via `-c none` option. + +1. Volume labels are not backed-up or restored automatically, but they might be required for your application to work (e.g. when using `docker-compose`). If you need to preserve them, create a label backup file as follows: `docker inspect [volume-name] -f "{{json .Labels}}" > labels.json`. When restoring your data, target volume needs to be created manually with labels before launching the restore script: `docker volume create --label "label1" --label "label2" [volume-name]`. diff --git a/volume-backup.sh b/volume-backup.sh index f965f13..922b0ff 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -1,7 +1,7 @@ #!/bin/bash usage() { - >&2 echo "Usage: volume-backup [options] " + >&2 echo "Usage: volume-backup [options]. Reads from stdin and writes to stdout. Can also read/write files (deprecated)." >&2 echo "" >&2 echo "Options:" >&2 echo " -c chooose compression algorithm: bz2 (default), gz, xz, pigz, zstd and 0 (none)" @@ -95,11 +95,6 @@ done shift $((OPTIND - 1)) -if [ $# -lt 1 ]; then - usage - exit 1 -fi - case "$COMPRESSION" in xz) TAROPTS+=(-J) @@ -130,8 +125,8 @@ none|0) ;; esac -if [ "$1" == "-" ]; then - ARCHIVE=$1 +if [ -z "$1" ] || [ "$1" == "-" ]; then + ARCHIVE="-" ARCHIVE_PATH=$ARCHIVE else ARCHIVE=${1%%$EXTENSION}$EXTENSION From 6a4b21f48285c918fcfba817652f4bb2c6d51bae Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 17 Dec 2022 17:23:22 +0100 Subject: [PATCH 51/53] disable zstd adapt option (performance) --- volume-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volume-backup.sh b/volume-backup.sh index 922b0ff..b4d481a 100755 --- a/volume-backup.sh +++ b/volume-backup.sh @@ -113,7 +113,7 @@ pigz) EXTENSION=.tar.gz ;; zstd) - TAROPTS+=(-I "zstd --adapt") + TAROPTS+=(-I zstd) EXTENSION=.tar.zstd ;; none|0) From d31abfbdae62c68e619b629d9769c9f13944e7cd Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Fri, 15 Sep 2023 20:25:26 +0200 Subject: [PATCH 52/53] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..be6afbc --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: loomchild From 4d7158e177c0d61924fcbc1cb2e9bd188f3159ea Mon Sep 17 00:00:00 2001 From: Jarek Lipski Date: Sat, 27 Jan 2024 20:34:26 +0100 Subject: [PATCH 53/53] mention Podman --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4713a94..97d6a49 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # volume-backup -An utility to backup and restore [docker volumes](https://docs.docker.com/engine/reference/commandline/volume/). For more info, read my article on [Medium](https://medium.com/@jareklipski/backup-restore-docker-named-volumes-350397b8e362) +An utility to backup and restore [Docker volumes](https://docs.docker.com/storage/volumes/). For more info, read my article on [Medium](https://medium.com/@jareklipski/backup-restore-docker-named-volumes-350397b8e362). + +It works just as well with [Podman](https://podman.io/) volumes, just prefix each command with `podman` instead of `docker`. **Note**: Make sure no container is using the volume before backup or restore, otherwise your data might be damaged. See [Miscellaneous](#miscellaneous) for instructions.