From 626d276c685bc4c77ebd6446822cc2175d29644a Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Sat, 25 May 2024 12:48:47 -0700 Subject: [PATCH] Release 4.1.0 - See CHANGELOG.md --- .github/workflows/main.yml | 4 +-- .github/workflows/manual.yml | 4 +-- CHANGELOG.md | 12 +++++++++ Dockerfile | 36 ++++++++++++++++----------- README.md | 1 + install/assets/defaults/10-db-backup | 2 +- install/assets/functions/10-db-backup | 29 +++++++++++++++------ 7 files changed, 61 insertions(+), 27 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bb3b61f..f2681e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ on: jobs: build: - uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 9463b9f..2b4ebfa 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -9,7 +9,7 @@ on: jobs: build: - uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit diff --git a/CHANGELOG.md b/CHANGELOG.md index ca07664..da69447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## 4.1.0 2024-05-25 + + Note that arm/v7 builds have been removed from this release going forward + + ### Added + - Introduce DEFAULT/DBXX_MYSQL_CLIENT option to use mariadb or mysql for client dumping to solve incompatibility issues + - Alpine 3.20 Base + - MariaDB 10.11.8 Client + - AWS Client 1.32.113 + - MySQL Client 8.4.0 + + ## 4.0.35 2024-01-14 ### Changed diff --git a/Dockerfile b/Dockerfile index 6739547..8c3999b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DISTRO=alpine -ARG DISTRO_VARIANT=3.19 +ARG DISTRO_VARIANT=3.20 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT} LABEL maintainer="Dave Conroy (github.com/tiredofit)" @@ -9,7 +9,9 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \ INFLUX2_CLIENT_VERSION=2.7.3 \ MSODBC_VERSION=18.3.2.1-1 \ MSSQL_VERSION=18.3.1.1-1 \ - AWS_CLI_VERSION=1.31.5 \ + MYSQL_VERSION=mysql-8.4.0 \ + MYSQL_REPO_URL=https://github.com/mysql/mysql-server \ + AWS_CLI_VERSION=1.32.113 \ CONTAINER_ENABLE_MESSAGING=TRUE \ CONTAINER_ENABLE_MONITORING=TRUE \ IMAGE_NAME="tiredofit/db-backup" \ @@ -27,11 +29,13 @@ RUN source /assets/functions/00-container && \ build-base \ bzip2-dev \ cargo \ + cmake \ git \ go \ libarchive-dev \ openssl-dev \ libffi-dev \ + ncurses-dev \ python3-dev \ py3-pip \ xz-dev \ @@ -47,8 +51,10 @@ RUN source /assets/functions/00-container && \ mariadb-client \ mariadb-connector-c \ mongodb-tools \ + ncurses \ openssl \ pigz \ + pixz \ postgresql16 \ postgresql16-client \ pv \ @@ -81,24 +87,24 @@ RUN source /assets/functions/00-container && \ clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \ go build -o /usr/sbin/influxd ./cmd/influxd && \ strip /usr/sbin/influxd && \ + \ + clone_git_repo "${MYSQL_REPO_URL}" "${MYSQL_VERSION}" && \ + cmake \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_INSTALL_PREFIX=/opt/mysql \ + -DFORCE_INSOURCE_BUILD=1 \ + -DWITHOUT_SERVER:BOOL=ON \ + && \ + make -j$(nproc) install && \ + \ + pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \ + pip3 install --break-system-packages blobxfer && \ + \ mkdir -p /usr/src/pbzip2 && \ curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \ cd /usr/src/pbzip2 && \ make && \ make install && \ - mkdir -p /usr/src/pixz && \ - curl -sSL https://github.com/vasi/pixz/releases/download/v1.0.7/pixz-1.0.7.tar.xz | tar xvfJ - --strip 1 -C /usr/src/pixz && \ - cd /usr/src/pixz && \ - ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --localstatedir=/var \ - && \ - make && \ - make install && \ - \ - pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \ - pip3 install --break-system-packages blobxfer && \ \ package remove .db-backup-build-deps && \ package cleanup && \ diff --git a/README.md b/README.md index 0f964ea..e14f7d1 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,7 @@ Encryption occurs after compression and the encrypted filename will have a `.gpg | `DEFAULT_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | | `DEFAULT_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | | `DEFAULT_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DEFAULT_MYSQL_CLIENT` | Choose between `mariadb` or `mysql` client to perform dump operations for compatibility purposes | `mariadb` | | | `DEFAULT_MYSQL_EVENTS` | Backup Events | `TRUE` | | | `DEFAULT_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet | `512M` | | | `DEFAULT_MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction | `TRUE` | | diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 1aa74e7..0645a68 100644 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -6,7 +6,6 @@ DBBACKUP_USER=${DBBACKUP_USER:-"dbbackup"} DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0} DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} -DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"} DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"} @@ -20,6 +19,7 @@ DEFAULT_FILESYSTEM_PATH_PERMISSION=${DEFAULT_FILESYSTEM_PATH_PERMISSION:-"700"} DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"600"} DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"} DEFAULT_LOG_LEVEL=${DEFAULT_LOG_LEVEL:-"notice"} +DEFAULT_MYSQL_CLIENT=${DEFAULT_MYSQL_CLIENT:-"mariadb"} DEFAULT_MYSQL_ENABLE_TLS=${DEFAULT_MYSQL_ENABLE_TLS:-"FALSE"} DEFAULT_MYSQL_EVENTS=${DEFAULT_MYSQL_EVENTS:-"TRUE"} DEFAULT_MYSQL_MAX_ALLOWED_PACKET=${DEFAULT_MYSQL_MAX_ALLOWED_PACKET:-"512M"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 1cc986d..237f0bb 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -221,6 +221,7 @@ bootstrap_variables() { transform_backup_instance_variable "${backup_instance_number}" INFLUX_VERSION backup_job_influx_version transform_backup_instance_variable "${backup_instance_number}" LOG_LEVEL backup_job_log_level transform_backup_instance_variable "${backup_instance_number}" MONGO_CUSTOM_URI backup_job_mongo_custom_uri + transform_backup_instance_variable "${backup_instance_number}" MYSQL_CLIENT backup_job_mysql_client transform_backup_instance_variable "${backup_instance_number}" MYSQL_ENABLE_TLS backup_job_mysql_enable_tls transform_backup_instance_variable "${backup_instance_number}" MYSQL_EVENTS backup_job_mysql_events transform_backup_instance_variable "${backup_instance_number}" MYSQL_MAX_ALLOWED_PACKET backup_job_mysql_max_allowed_packet @@ -401,6 +402,20 @@ EOF dbtype=mysql backup_job_db_port=${backup_job_db_port:-3306} check_var backup_job_db_name DB"${v_instance}"_NAME "database name. Seperate multiple with commas" + case "${backup_job_mysql_client,,}" in + mariadb ) + _mysql_prefix=/usr/bin/ + ;; + mysql ) + _mysql_prefix=/opt/mysql/bin/ + ;; + * ) + print_error "I don't understand '${backup_job_mysql_client,,}' as a client. Exiting.." + exit 99 + ;; + esac + + print_debug "Using '${backup_job_mysql_client,,}' as client" if [ -n "${backup_job_db_pass}" ] ; then export MYSQL_PWD=${backup_job_db_pass} ; fi if var_true "${backup_job_mysql_enable_tls}" ; then @@ -665,7 +680,7 @@ backup_mysql() { if [ "${backup_job_db_name,,}" = "all" ] ; then write_log debug "Preparing to back up everything except for information_schema and _* prefixes" - db_names=$(run_as_user mysql -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_enumeration_opts} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) + db_names=$(run_as_user ${_mysql_prefix}mysql -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_enumeration_opts} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) if [ -n "${backup_job_db_name_exclude}" ] ; then db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n') for db_exclude in ${db_names_exclusions} ; do @@ -682,13 +697,13 @@ backup_mysql() { if var_true "${backup_job_split_db}" ; then for db in ${db_names} ; do prepare_dbbackup - backup_job_filename=mysql_${db}_${backup_job_db_host,,}_${now}.sql - backup_job_filename_base=mysql_${db}_${backup_job_db_host,,} + backup_job_filename=${backup_job_mysql_client,,}_${db}_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=${backup_job_mysql_client,,}_${db}_${backup_job_db_host,,} compression pre_dbbackup "${db}" write_log notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi - run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} $db | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null + run_as_user ${play_fair} ${_mysql_prefix}mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} $db | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null exit_code=$? if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi check_exit_code backup "${backup_job_filename}" @@ -703,13 +718,13 @@ backup_mysql() { else write_log debug "Not splitting database dumps into their own files" prepare_dbbackup - backup_job_filename=mysql_all_${backup_job_db_host,,}_${now}.sql - backup_job_filename_base=mysql_all_${backup_job_db_host,,} + backup_job_filename=${backup_job_mysql_client,,}_all_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=${backup_job_mysql_client,,}_all_${backup_job_db_host,,} compression pre_dbbackup all write_log notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi - run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null + run_as_user ${play_fair} ${_mysql_prefix}mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null exit_code=$? if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi check_exit_code backup "${backup_job_filename}"