From 19fcb7f6fdf9d9f2d478836eea65adfde20ae23e Mon Sep 17 00:00:00 2001 From: Muthu Chidambaram Date: Mon, 22 Aug 2022 11:18:44 -0700 Subject: [PATCH] [BACKPORT 2.6][PLAT-511] Automating generation of platform support packages Summary: migrating s3 support packages to 2.6 branch Test Plan: verify packages/conf files correct in dev-itest-pipeline build Reviewers: sanketh, amalyshev Reviewed By: amalyshev Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D19087 --- managed/devops/opscli/ybops/utils/__init__.py | 2 +- managed/support/packages/backup.sh | 248 ++++++++++++++++++ managed/support/packages/cleanup.sh | 24 ++ managed/support/packages/install.sh | 67 +++++ managed/support/packages/platform.2.6.1.conf | 92 +++++++ managed/support/packages/platform.conf | 1 + managed/support/packages/prometheus.service | 18 ++ managed/support/packages/prometheus.yml | 61 +++++ managed/support/packages/yb-platform.service | 29 ++ .../packages/yugaware-nginx-https.conf | 46 ++++ managed/support/packages/yugaware-nginx.conf | 36 +++ managed/support/thirdparty-dependencies.txt | 11 + managed/support/version.txt | 1 + managed/support/yb_release | 94 +++++++ managed/support/yb_release.py | 34 +++ 15 files changed, 763 insertions(+), 1 deletion(-) create mode 100755 managed/support/packages/backup.sh create mode 100755 managed/support/packages/cleanup.sh create mode 100755 managed/support/packages/install.sh create mode 100755 managed/support/packages/platform.2.6.1.conf create mode 100755 managed/support/packages/platform.conf create mode 100755 managed/support/packages/prometheus.service create mode 100755 managed/support/packages/prometheus.yml create mode 100755 managed/support/packages/yb-platform.service create mode 100755 managed/support/packages/yugaware-nginx-https.conf create mode 100755 managed/support/packages/yugaware-nginx.conf create mode 100644 managed/support/thirdparty-dependencies.txt create mode 120000 managed/support/version.txt create mode 100755 managed/support/yb_release create mode 100755 managed/support/yb_release.py diff --git a/managed/devops/opscli/ybops/utils/__init__.py b/managed/devops/opscli/ybops/utils/__init__.py index 0e4d62426a17..3f5038e092bd 100644 --- a/managed/devops/opscli/ybops/utils/__init__.py +++ b/managed/devops/opscli/ybops/utils/__init__.py @@ -55,7 +55,7 @@ RSA_KEY_LENGTH = 2048 RELEASE_VERSION_FILENAME = "version.txt" RELEASE_VERSION_PATTERN = "\d+.\d+.\d+.\d+" -RELEASE_REPOS = set(["devops", "yugaware", "yugabyte"]) +RELEASE_REPOS = set(["devops", "yugaware", "yugabyte", "yugabundle_support"]) # Home directory of node instances. Try to read home dir from env, else assume it's /home/yugabyte. YB_HOME_DIR = os.environ.get("YB_HOME_DIR") or "/home/yugabyte" diff --git a/managed/support/packages/backup.sh b/managed/support/packages/backup.sh new file mode 100755 index 000000000000..e7cd45ee7bfa --- /dev/null +++ b/managed/support/packages/backup.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# Copyright 2019 YugaByte, Inc. and Contributors +# +# Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# https://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt + +set -euo pipefail + +YUGAWARE_DUMP_FNAME="yugaware_dump.sql" + +# When false, we won't stop/start platform and prometheus services when executing the script +RESTART_PROCESSES=true + +set +e +DOCKER_CMD=$(command -v docker) +DOCKER_BASED="" +if [[ ! -z "$DOCKER_CMD" ]]; then + DOCKER_BASED="$(docker ps -a | grep yugaware)" +fi +set -e + +# VM based default. +PROMETHEUS_DATA_DIR="/var/lib/prometheus" +# Docker (Replicated) based default. +if [[ -z $DOCKER_BASED ]]; then + PROMETHEUS_DATA_DIR="/opt/yugabyte/prometheusv2" +fi +PROMETHEUS_SNAPSHOT_DIR="prometheus_snapshot" + +# Takes docker container and command as arguments. Executes docker cmd if docker-based or not. +docker_aware_cmd() { + if [[ -z "$DOCKER_BASED" ]]; then + sh -c "$2" + else + docker exec -i $1 $2 + fi +} + +create_backup() { + now=$(date +"%y-%m-%d-%H-%M") + output_path="$1" + data_dir="$2" + exclude_prometheus="$3" + exclude_prometheus_flag=" " + if [[ "$exclude_prometheus" = true ]]; then + exclude_prometheus_flag=" --exclude prometheus* " + fi + + modify_service yb-platform stop + + tarname="${output_path}/backup_${now}.tgz" + trap "cleanup ${data_dir}/${YUGAWARE_DUMP_FNAME}" EXIT + echo "Creating snapshot of platform data" + docker_aware_cmd "postgres" "pg_dump -U postgres -Fc yugaware" > \ + "${data_dir}/${YUGAWARE_DUMP_FNAME}" + + # Backup prometheus data. + if [[ "$exclude_prometheus" = false ]]; then + trap "sudo rm -rf ${data_dir}/${PROMETHEUS_SNAPSHOT_DIR}" RETURN + echo "Creating prometheus snapshot" + set_prometheus_data_dir + snapshot_dir=$(curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot | + python -c "import sys, json; print(json.load(sys.stdin)['data']['name'])") + mkdir -p "$data_dir/$PROMETHEUS_SNAPSHOT_DIR" + sudo cp -aR "$PROMETHEUS_DATA_DIR/snapshots/$snapshot_dir" "$data_dir/$PROMETHEUS_SNAPSHOT_DIR" + sudo rm -rf "$PROMETHEUS_DATA_DIR/snapshots/$snapshot_dir" + fi + echo "Creating platform backup package" + tar $exclude_prometheus_flag --exclude "postgresql" -czf $tarname -C $data_dir . + echo "Finished creating backup $tarname" + + modify_service yb-platform restart + echo "Done!" +} + +restore_backup() { + input_path="$1" + destination="$2" + is_prometheus=false + + prometheus_dir_regex="${PROMETHEUS_SNAPSHOT_DIR}/$" + if tar -tf $input_path | grep $prometheus_dir_regex; then + is_prometheus=true + set_prometheus_data_dir + fi + + modify_service yb-platform stop + + if [[ "$is_prometheus" = true ]]; then + modify_service prometheus stop + fi + + yugaware_dump="${destination}/${YUGAWARE_DUMP_FNAME}" + trap "cleanup $yugaware_dump" EXIT + sudo tar -xzf $input_path --directory $destination + echo "Restoring platform data to database" + docker_aware_cmd "postgres" "pg_restore -U postgres -d yugaware -c" < "${yugaware_dump}" + # Restore prometheus data. + if [[ "$is_prometheus" = true ]]; then + if [[ -z "$PROMETHEUS_DATA_DIR" ]]; then + echo "Failed to find prometheus data directory." + exit 1 + fi + sudo rm -rf ${PROMETHEUS_DATA_DIR}/* + sudo mv $destination/$PROMETHEUS_SNAPSHOT_DIR/* $PROMETHEUS_DATA_DIR + if [[ -z "$DOCKER_BASED" ]]; then + sudo chown -R prometheus:prometheus "$PROMETHEUS_DATA_DIR" + else + sudo chown -R nobody:nogroup "$PROMETHEUS_DATA_DIR" + fi + fi + # Create following directory if it wasn't created yet so restore will succeed. + mkdir -p "$destination/release" + + modify_service yb-platform restart + + if [[ "$is_prometheus" = true ]]; then + modify_service prometheus restart + fi + + echo "Finished restoring backup" +} + +set_prometheus_data_dir() { + PROMETHEUS_DATA_DIR=$(curl http://localhost:9090/api/v1/status/flags | + python -c "import sys, json; print(json.load(sys.stdin)['data']['storage.tsdb.path'])") +} + +modify_service() { + if [[ -z "$DOCKER_BASED" ]] && [[ "$RESTART_PROCESSES" = true ]]; then + set +e + service="$1" + operation="$2" + echo "Performing operation $operation on service $service" + sudo systemctl "$operation" "$service" + fi +} + +print_backup_usage() { + echo "$0 backup --output [--data_dir ] [--exclude_prometheus] [--skip_restart]" + echo "Backup YW to a specified output location" +} + +print_restore_usage() { + echo "$0 restore --input [--destination ] [--skip_restart]" + echo "Restore YW from a specified input location" +} + +cleanup () { + rm -f "$1" +} + +if [[ $# -eq 0 ]]; then + echo "ERROR: Please use one of the following:" + echo "" + print_backup_usage + echo "" + print_restore_usage + exit 1 +fi + +command=$1 +shift + +case $command in + backup) + exclude_prometheus=false + data_dir=/opt/yugabyte + while (( "$#" )); do + case "$1" in + --output) + output_path=$2 + shift 2 + ;; + --exclude_prometheus) + exclude_prometheus=true + shift + ;; + --data_dir) + data_dir=$2 + shift 2 + ;; + --skip_restart) + RESTART_PROCESSES=false + shift + ;; + --verbose) + set -x + shift + ;; + *) + echo "$1" + echo "ERROR: Backup Usage" + print_backup_usage + exit 1 + esac + done + + if [[ -z "$output_path" ]]; then + echo "ERROR: Backup Usage" + print_backup_usage + exit 1 + fi + create_backup "$output_path" "$data_dir" "$exclude_prometheus" + exit 0 + ;; + restore) + destination=/opt/yugabyte + while (( "$#" )); do + case "$1" in + --input) + input_path=$2 + shift 2 + ;; + --destination) + destination=$2 + shift 2 + ;; + --skip_restart) + RESTART_PROCESSES=false + shift + ;; + --verbose) + set -x + shift + ;; + *) + echo "ERROR: Restore usage" + print_restore_usage + exit 1 + esac + done + if [[ -z "$input_path" ]]; then + echo "ERROR: Restore usage" + print_restore_usage + exit 1 + fi + restore_backup "$input_path" "$destination" + exit 0 + ;; + *) + echo "ERROR: Command must be either 'backup' or 'restore'" + exit 1 +esac diff --git a/managed/support/packages/cleanup.sh b/managed/support/packages/cleanup.sh new file mode 100755 index 000000000000..7aa8c757395c --- /dev/null +++ b/managed/support/packages/cleanup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +read -p "Are you sure? Y/N: " -n 1 -r +echo + +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + exit 1 +fi + +echo "Stop services" +sudo service yb-platform stop +sudo service prometheus stop +sudo service nginx stop + +echo "Drop platform database" +dropdb -U postgres yugaware + +echo "Cleanup data directories" +rm -rf /opt/yugabyte + + +echo "Great you want to cleanup!!!" diff --git a/managed/support/packages/install.sh b/managed/support/packages/install.sh new file mode 100755 index 000000000000..ae9c007cf487 --- /dev/null +++ b/managed/support/packages/install.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -euo pipefail + +function print_usage { + echo "Usage: ./install.sh version" + printf "Available versions..:\t" + for package in `ls yugabundle-*` + do + release=`echo $package | cut -d- -f2` + build=`echo $package | cut -d- -f3 | cut -d. -f1` + printf "$release-$build\t" + done + printf "\n" + exit 1 +} + +if [ $# -lt 1 ]; then + echo "Please specify a valid version." + print_usage +fi + +if [ "$1" == "--help" ]; then + print_usage +fi + +version=$1 +package=`ls yugabundle-${version}.tar.gz` +package_folder=`tar tf ${package} | sed -e 's@/.*@@' | uniq` +install_path=/opt/yugabyte + +echo "Create necessary directories" +mkdir -p ${install_path}/releases/$version +mkdir -p ${install_path}/swamper_targets +mkdir -p ${install_path}/data +mkdir -p ${install_path}/third-party + +echo "Unpacking package ${package} inside ${package_folder}" +tar -xvf $package + +echo "Create Devops and Yugaware directories" +mkdir -p ${package_folder}/devops +mkdir -p ${package_folder}/yugaware + +echo "Untarring devops package" +tar -xf ${package_folder}/devops*.tar.gz -C ./${package_folder}/devops +echo "Untarring yugaware package" +tar -xf ${package_folder}/yugaware*.tar.gz -C ./${package_folder}/yugaware + +echo "Copy yugabyte release file" +cp ${package_folder}/yugabyte*.tar.gz ${install_path}/releases/$version + +echo "Installing devops environment" +tar -xf ./${package_folder}/devops/yb_platform_virtualenv.tar.gz -C ./${package_folder}/devops || \ + ./${package_folder}/devops/bin/install_python_requirements.sh --use_package + +if [ -L ${install_path}/yugaware ]; then + echo "Renaming current symlink" + mv ${install_path}/yugaware ${install_path}/yugaware_backup +fi +if [ -L ${install_path}/devops ]; then + echo "Renaming current symlink" + mv ${install_path}/devops ${install_path}/devops_backup +fi + +echo "Creating symlinks" +ln -s $(pwd)/${package_folder}/yugaware ${install_path}/yugaware +ln -s $(pwd)/${package_folder}/devops ${install_path}/devops diff --git a/managed/support/packages/platform.2.6.1.conf b/managed/support/packages/platform.2.6.1.conf new file mode 100755 index 000000000000..2b4f2b153515 --- /dev/null +++ b/managed/support/packages/platform.2.6.1.conf @@ -0,0 +1,92 @@ +include classpath("application.common.conf") + +play.crypto.secret=${PLATFORM_APP_SECRET} + +play.i18n.langs = [ "en" ] +pidfile.path = "/dev/null" +db { + default.url="jdbc:postgresql://127.0.0.1:5432/yugaware" + default.driver=org.postgresql.Driver + default.username=${PLATFORM_DB_USER} + default.password=${PLATFORM_DB_PASSWORD} + default.logStatements=true + default.migration.initOnMigrate=true + default.migration.auto=true +} +ebean { + default = ["com.yugabyte.yw.models.*"] +} + +play.evolutions.enabled=false +play.modules.enabled += "org.flywaydb.play.PlayModule" + +yb { + devops.home = ${?DEVOPS_HOME} + swamper.targetPath = ${?SWAMPER_TARGET_PATH} + metrics.url = ${?METRICS_URL} + storage.path = /opt/yugabyte/data + seedData = false + multiTenant = false + releases.path = "/opt/yugabyte/releases" + thirdparty.packagePath = /opt/yugabyte/third-party + + # Interval at which to check the status of every universe. Default: 5 minutes. + health.check_interval_ms = 300000 + # Interval at which to send a status report email. Default: 12 hours. + health.status_interval_ms = 43200000 + # If SSO needs to be enabled on the platform. + security.use_oauth = false + security.use_oauth = ${?USE_OAUTH} + security.type = "" + # Override in case SECURITY_TYPE is set. + security.type = ${?YB_SECURITY_TYPE} + security.clientID = "" + security.clientID = ${?YB_OIDC_CLIENT_ID} + security.secret = "" + security.secret = ${?YB_OIDC_SECRET} + security.discoveryURI = "" + security.discoveryURI = ${?YB_OIDC_DISCOVERY_URI} + security.oidcScope = "" + security.oidcScope = ${?YB_OIDC_SCOPE} + security.oidcEmailAttribute = "" + security.oidcEmailAttribute = ${?YB_OIDC_EMAIL_ATTR} + taskGC.gc_check_interval = 1 day + taskGC.task_retention_duration = 30 days + # The IP of the platform. + url = "" + url = ${?YW_URL} +} + +play.filters { + # CSRF config + csrf { + cookie { + # If non null, the CSRF token will be placed in a cookie with this name + name = "csrfCookie" + # Whether the cookie should be set to secure + secure = false + # Whether the cookie should have the HTTP only flag set + httpOnly = false + } + # Whether to bypass CSRF check if CORS check is satisfied + bypassCorsTrustedOrigins = false + header { + # The name of the header to accept CSRF tokens from. + name = "Csrf-Token" + } + } + cors { + pathPrefixes = ["/"] + allowedOrigins = [${?CORS_ORIGIN}] + supportsCredentials=true + allowedHttpMethods = ["GET", "POST", "PUT", "OPTIONS", "DELETE"] + allowedHttpHeaders = [ + "Accept", + "Origin", + "Content-Type", + "X-Auth-Token", + "X-AUTH-YW-API-TOKEN", + ${play.filters.csrf.header.name} + ] + } +} diff --git a/managed/support/packages/platform.conf b/managed/support/packages/platform.conf new file mode 100755 index 000000000000..1d6bbdcf6048 --- /dev/null +++ b/managed/support/packages/platform.conf @@ -0,0 +1 @@ +include classpath("application.yugabundle.conf") diff --git a/managed/support/packages/prometheus.service b/managed/support/packages/prometheus.service new file mode 100755 index 000000000000..0cf759dad0b7 --- /dev/null +++ b/managed/support/packages/prometheus.service @@ -0,0 +1,18 @@ +[Unit] +Description=Prometheus +Wants=network-online.target +After=network-online.target + +[Service] +User=prometheus +Group=prometheus +Type=simple +ExecStart=/usr/local/bin/prometheus \ +--config.file /etc/prometheus/prometheus.yml \ +--storage.tsdb.path /var/lib/prometheus/ \ +--web.console.templates=/etc/prometheus/consoles \ +--web.console.libraries=/etc/prometheus/console_libraries \ +--web.enable-admin-api + +[Install] +WantedBy=multi-user.target diff --git a/managed/support/packages/prometheus.yml b/managed/support/packages/prometheus.yml new file mode 100755 index 000000000000..74639f880386 --- /dev/null +++ b/managed/support/packages/prometheus.yml @@ -0,0 +1,61 @@ +# Copyright (c) YugaByte, Inc. + +# This is a template for auto-generated file for prometheus configuration + +global: + # By default, scrape targets every 15 seconds. + scrape_interval: 15s + evaluation_interval: 15s + # scrape_timeout is set to the global default (10s). + + # The labels to add to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'swamper' + + +# A list of scrape configurations. +scrape_configs: + + - job_name: 'prometheus' + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: ['localhost:9090'] + + - job_name: "node" + file_sd_configs: + - files: + - '/opt/yugabyte/swamper_targets/node.*.json' + metric_relabel_configs: + - source_labels: ["__name__"] + regex: "(.*)" + target_label: "saved_name" + replacement: "$1" + + - job_name: "yugabyte" + metrics_path: "/prometheus-metrics" + file_sd_configs: + - files: + - '/opt/yugabyte/swamper_targets/yugabyte.*.json' + metric_relabel_configs: + - source_labels: ["__name__"] + regex: "handler_latency_(yb_[^_]*)_([^_]*)_([^_]*)(.*)" + target_label: "server_type" + replacement: "$1" + - source_labels: ["__name__"] + regex: "handler_latency_(yb_[^_]*)_([^_]*)_([^_]*)(.*)" + target_label: "service_type" + replacement: "$2" + - source_labels: ["__name__"] + regex: "handler_latency_(yb_[^_]*)_([^_]*)_([^_]*)(_sum|_count)" + target_label: "service_method" + replacement: "$3" + - source_labels: ["__name__"] + regex: "handler_latency_(yb_[^_]*)_([^_]*)_([^_]*)(_sum|_count)" + target_label: "__name__" + replacement: "rpc_latency$4" + - source_labels: ["__name__"] + regex: "(.*)" + target_label: "saved_name" + replacement: "$1" diff --git a/managed/support/packages/yb-platform.service b/managed/support/packages/yb-platform.service new file mode 100755 index 000000000000..583a899bd79a --- /dev/null +++ b/managed/support/packages/yb-platform.service @@ -0,0 +1,29 @@ +[Unit] +Description=Yugabyte Platform +Wants=network-online.target +After=network-online.target + +[Service] +User=yugabyte +Group=yugabyte +Type=simple + +Environment="PLATFORM_APP_SECRET=REPLACE_PLATFORM_APP_SECRET" +Environment="DEVOPS_HOME=/opt/yugabyte/devops" +Environment="METRICS_URL=http://127.0.0.1:9090/api/v1" +Environment="SWAMPER_TARGET_PATH=/opt/yugabyte/swamper_targets" +Environment="PLATFORM_DB_USER=postgres" +Environment="PLATFORM_DB_PASSWORD=" +Environment="USE_OAUTH=true" +Environment="YB_SECURITY_TYPE=OIDC +Environment="YB_OIDC_CLIENT_ID=" +Environment="YB_OIDC_SECRET=" +Environment="YB_OIDC_DISCOVERY_URI=" +Environment="YW_URL=" +Environment="YB_OIDC_SCOPE=openid " +Environment="YB_OIDC_EMAIL_ATTR=" + +ExecStart=/opt/yugabyte/yugaware/bin/yugaware -Dconfig.file=/opt/yugabyte/platform.conf + +[Install] +WantedBy=multi-user.target diff --git a/managed/support/packages/yugaware-nginx-https.conf b/managed/support/packages/yugaware-nginx-https.conf new file mode 100755 index 000000000000..18539d602e00 --- /dev/null +++ b/managed/support/packages/yugaware-nginx-https.conf @@ -0,0 +1,46 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + server_tokens off; + client_header_timeout 4s; + client_body_timeout 4s; + keepalive_timeout 10s 10s; + send_timeout 10s; + client_body_buffer_size 8k; + client_header_buffer_size 1k; + large_client_header_buffers 2 8k; + + server { + listen 80; + server_name _; + return 301 https://$host$request_uri; + } + server { + listen 443 ssl; + ssl_certificate /opt/yugabyte/certs/server.crt; + ssl_certificate_key /opt/yugabyte/certs/server.key; + server_name _; + proxy_http_version 1.1; + proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + + location / { + proxy_pass http://127.0.0.1:9000; + } + location ~ /settings/ha/internal/upload$ { + proxy_pass http://127.0.0.1:9000; + # HA backp upload endpoint needs to allow bigger request size + # Broaden the location match regex to cover more endpoints as needed + client_max_body_size 100G; + } + add_header X-Frame-Options "SAMEORIGIN"; + } +} diff --git a/managed/support/packages/yugaware-nginx.conf b/managed/support/packages/yugaware-nginx.conf new file mode 100755 index 000000000000..6ce3a644f68b --- /dev/null +++ b/managed/support/packages/yugaware-nginx.conf @@ -0,0 +1,36 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + server_tokens off; + client_header_timeout 10s; + client_body_timeout 10s; + keepalive_timeout 10s 10s; + send_timeout 10s; + + server { + listen 80; + server_name _; + + proxy_http_version 1.1; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $host; + + location / { + proxy_pass http://127.0.0.1:9000; + } + location ~ /settings/ha/internal/upload$ { + proxy_pass http://127.0.0.1:9000; + # HA backp upload endpoint needs to allow bigger request size + # Broaden the location match regex to cover more endpoints as needed + client_max_body_size 100G; + } + add_header X-Frame-Options "SAMEORIGIN"; + } +} diff --git a/managed/support/thirdparty-dependencies.txt b/managed/support/thirdparty-dependencies.txt new file mode 100644 index 000000000000..6f20407fc6db --- /dev/null +++ b/managed/support/thirdparty-dependencies.txt @@ -0,0 +1,11 @@ +https://github.com/prometheus/node_exporter/releases/download/v0.13.0/node_exporter-0.13.0.linux-amd64.tar.gz +https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz +https://azcopyvnext.azureedge.net/release20220315/azcopy_linux_amd64_10.14.1.tar.gz +https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gz +https://files.pythonhosted.org/packages/be/ed/5bbc91f03fa4c839c4c7360375da77f9659af5f7086b7a7bdda65771c8e0/python-dateutil-2.8.1.tar.gz +https://files.pythonhosted.org/packages/e3/85/1aff76b966622868a73717abd8b501a3c91890e23a65e5f574ff6df1970f/python-magic-0.4.18.tar.gz +https://files.pythonhosted.org/packages/a7/52/5f84da9ee2780682795550ddea20bec3e604a8a82600f4e5d3a6ca0bcbcd/setuptools_scm-1.17.0.tar.gz#sha256=70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a +https://storage.googleapis.com/cloud-sdk-release/google-cloud-sdk-286.0.0-linux-x86_64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.2.1/prometheus-2.2.1.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.5.1/alertmanager-0.5.1.linux-amd64.tar.gz +https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip diff --git a/managed/support/version.txt b/managed/support/version.txt new file mode 120000 index 000000000000..7fa4c866fa1b --- /dev/null +++ b/managed/support/version.txt @@ -0,0 +1 @@ +../../version.txt \ No newline at end of file diff --git a/managed/support/yb_release b/managed/support/yb_release new file mode 100755 index 000000000000..7c3867cbd6b7 --- /dev/null +++ b/managed/support/yb_release @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# Copyright 2022 YugaByte, Inc. and Contributors +# +# Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# https://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt +# +# This script generates the platform support packages tar.gz in the required destination. + +set -euo pipefail + +print_help() { + cat <<-EOT +Generates platform-support-packages.tar.gz in destination. +Config files and binaries necessary for yugabundle install. +Usage: ${0##*/} +Options: + -h, --help + Show usage. + -d, --destination + Directory into which the support packages should be copied. +EOT +} + +export DEVOPS_HOME="${BASH_SOURCE%/*}"/../devops +export YB_MANAGED_DEVOPS_USE_PYTHON3=1 + +. "$DEVOPS_HOME/bin/"/common.sh + + +readonly yb_support_home=$( cd "${BASH_SOURCE%/*}" && pwd ) + +# verify packages directory exists +if [[ ! -d $yb_support_home/packages ]]; then + fatal "No 'packages' subdirectory found inside yb_support_home ('$yb_support_home')" +fi + +PACKAGES="packages" +THIRDPARTY="thirdparty" +THIRDPARTY_TARGZ="thirdparty-deps.tar.gz" +PACKAGE_TARGZ="yugabundle_support.tar.gz" +destination="" +package_path=$yb_support_home/$PACKAGES + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + print_help + exit 0 + ;; + -d|--destination) + destination="$2" + shift + esac + shift +done + +# verify destination directory exists +if [[ ! -d $destination ]]; then + fatal "No destination directory found ('$destination')" +fi + +thirdparty_path="$package_path/$THIRDPARTY" + +# ensure thirdparty directory exists +if [[ ! -d $thirdparty_path ]]; then + mkdir -p $thirdparty_path +fi + +cd $thirdparty_path +# This downloads all thirdparty dependencies into packages/thirdparty/ +wget -qi $yb_support_home/thirdparty-dependencies.txt + +# Recompress setuptools +unzip -q setuptools-44.1.1.zip && rm setuptools-44.1.1.zip +tar -czf setuptools-44.1.1.tar.gz setuptools-44.1.1 +rm -rf setuptools-44.1.1 + +# Generate thirdparty tar.gz with downloaded dependencies +cd $package_path +tar -czvf $THIRDPARTY_TARGZ $THIRDPARTY +# Remove thirdparty directory +rm -rf $thirdparty_path +# Generate platform-support-packages with packages/ conf files and thirdparty deps +cd $yb_support_home +tar -czvf $PACKAGE_TARGZ $PACKAGES +# Remove generated thirdparty tar.gz +rm "$package_path/$THIRDPARTY_TARGZ" + +activate_virtualenv +"$( dirname "${BASH_SOURCE[0]}" )"/yb_release.py --package $PACKAGE_TARGZ --destination $destination diff --git a/managed/support/yb_release.py b/managed/support/yb_release.py new file mode 100755 index 000000000000..190814e0c015 --- /dev/null +++ b/managed/support/yb_release.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) YugaByte, Inc. + +import argparse +import logging +import os +import shutil + +from ybops.utils import init_env, log_message, get_release_file +from ybops.common.exceptions import YBOpsRuntimeError + +"""This script packages Anywhere's support packages according to release info. + - Rename the package file to have the commit sha in it + - Move the package file to the required destination + - A higher level user, such as itest, will upload all release packages to s3 release bucket +""" + +parser = argparse.ArgumentParser() +parser.add_argument('--destination', help='Copy release to Destination folder.') +parser.add_argument('--package', help='Package to rename and copy to Destination') +args = parser.parse_args() + +try: + init_env(logging.INFO) + script_dir = os.path.dirname(os.path.realpath(__file__)) + release_file = get_release_file(script_dir, "yugabundle_support") + shutil.copyfile(args.package, release_file) + if args.destination: + if not os.path.exists(args.destination): + raise YBOpsRuntimeError("Destination {} not a directory.".format(args.destination)) + shutil.copy(release_file, args.destination) +except (OSError, shutil.SameFileError) as e: + log_message(logging.ERROR, e) + raise e