Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: object storage cluster init. #4510

Merged
merged 29 commits into from
Feb 26, 2024
Merged
Show file tree
Hide file tree
Changes from 26 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 147 additions & 0 deletions .github/workflows/objectstorage.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
name: Build Object Storage Cluster image

on:
workflow_call:
inputs:
push_image:
description: 'Push image'
required: false
type: boolean
default: false
push_image_tag:
description: 'Push all-in-one image tag, default is latest'
default: 'latest'
required: false
type: string
build_from:
description: 'Build all-in-one image from components image tag, default is latest'
default: 'latest'
required: false
type: string
workflow_dispatch:
inputs:
push_image:
description: 'Push image'
required: false
type: boolean
default: false
push_image_tag:
description: 'Push all-in-one image tag, default is latest'
default: 'latest'
required: false
type: string
build_from:
description: 'Build all-in-one image from components image tag, default is latest'
default: 'latest'
required: false
type: string
push:
branches: [ "main" ]
paths:
- "deploy/objectstorage/**"
- ".github/workflows/objectstorage.yml"
- "!**/*.md"
- "!**/*.yaml"
pull_request:
branches: [ "*" ]
paths:
- "deploy/objectstorage/**"
- ".github/workflows/objectstorage.yml"
- "!**/*.md"
- "!**/*.yaml"

env:
# Common versions
GO_VERSION: "1.20"
DEFAULT_OWNER: "labring"

jobs:
save-sealos:
uses: ./.github/workflows/import-save-sealos.yml

build-cluster-image:
if: ${{ (github.event_name == 'release') ||(github.event_name == 'push') || (inputs.push_image == true) }}
needs:
- save-sealos
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Expose git commit data
uses: rlespinasse/git-commit-data-action@v1
- name: Check if tag
id: check_tag
run: |
if [[ "${{ github.ref }}" == refs/tags/* ]]; then
echo "isTag=true" >> "$GITHUB_OUTPUT"
else
echo "isTag=false" >> "$GITHUB_OUTPUT"
fi
- name: Prepare
id: prepare
run: |
bash ./scripts/resolve-tag-image.sh "${{ inputs.push_image }}" "${{ steps.check_tag.outputs.isTag }}" "${{ inputs.push_image_tag }}"
echo repo=ghcr.io/${{ github.repository_owner }}/sealos-cloud-objectstorage >> $GITHUB_OUTPUT
- name: Download sealos
uses: actions/download-artifact@v3
with:
name: sealos
path: /tmp/
- name: Verify sealos
run: |
sudo chmod a+x /tmp/sealos
sudo mv /tmp/sealos /usr/bin/sealos
sudo sealos version

# todo: mutate image tag in images/shim and scripts or change scripts to use changeable tags

- name: Sealos login to ghcr.io
# if push to master, then login to ghcr.io
run: |
sudo sealos login -u ${{ github.repository_owner }} -p ${{ secrets.GH_PAT }} --debug ghcr.io

- name: Build sealos cloud cluster image
working-directory: deploy/objectstorage
run: |
[ -z "${{ inputs.build_from }}" ] && BuildFromTag="latest" || BuildFromTag="${{ inputs.build_from }}"; echo "BuildFromTag=${BuildFromTag}"
sed -i "s#labring#${{ github.repository_owner }}#g" init.sh
sed -i "s#latest#${BuildFromTag}#g" init.sh

sudo bash init.sh amd64
sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-amd64 --platform linux/amd64 -f Kubefile
sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-amd64 --platform linux/amd64 -f Kubefile


# delete old registry cache
sudo rm -rf registry
sudo rm -rf tars

sudo bash init.sh arm64
sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-arm64 --platform linux/arm64 -f Kubefile
sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-arm64 --platform linux/arm64 -f Kubefile

- name: Manifest Cluster Images
# if push to master, then patch images to ghcr.io
run: |
sudo sealos images
bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}
bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:latest
env:
OWNER: ${{ github.repository_owner }}

- name: Renew issue and Sync Images
uses: labring/gh-rebot@v0.0.6
if: ${{ github.repository_owner == env.DEFAULT_OWNER }}
with:
version: v0.0.8-rc1
env:
GH_TOKEN: "${{ secrets.GH_PAT }}"
SEALOS_TYPE: "issue_renew"
SEALOS_ISSUE_TITLE: "[DaylyReport] Auto build for sealos"
SEALOS_ISSUE_BODYFILE: "scripts/ISSUE_RENEW.md"
SEALOS_ISSUE_LABEL: "dayly-report"
SEALOS_ISSUE_TYPE: "day"
SEALOS_ISSUE_REPO: "labring-actions/cluster-image"
SEALOS_COMMENT_BODY: "/imagesync ghcr.io/${{ github.repository_owner }}/sealos-cloud:${{ steps.prepare.outputs.tag_name }}"
14 changes: 14 additions & 0 deletions deploy/objectstorage/Kubefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
FROM scratch
COPY tars tars
COPY etc etc
COPY scripts scripts
COPY manifests manifests

ENV cloudDomain=${cloudDomain:-"127.0.0.1.nip.io"}
ENV cloudPort=""
ENV minioStorageSize=${minioStorageSize:-1Gi}
ENV promStorageSize=${promStorageSize:-1Gi}
ENV minioAdminUser=${minioAdminUser:-"username"}
lingdie marked this conversation as resolved.
Show resolved Hide resolved
ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"}

CMD ["bash scripts/init.sh"]
6 changes: 6 additions & 0 deletions deploy/objectstorage/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# sealos cloud object storage cluster image
## prepare

1. install minio operator
2. install prometheus operator
3. run object storage cluster image
44 changes: 44 additions & 0 deletions deploy/objectstorage/etc/minio/policy/kubeblocks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketTagging",
"s3:ListBucket",
"s3:ListMultipartUploadParts",
"s3:PutBucketTagging",
"s3:PutObject",
"s3:CreateBucket",
"s3:DeleteObject",
"s3:GetBucketLocation",
"s3:GetBucketPolicy",
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::file-migration/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:CreateBucket",
"s3:DeleteObject",
"s3:GetBucketLocation",
"s3:PutObject",
"s3:GetBucketPolicy",
"s3:GetBucketTagging",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:PutBucketTagging"
],
"Resource": [
"arn:aws:s3:::file-backup/*"
]
}
]
}
14 changes: 14 additions & 0 deletions deploy/objectstorage/etc/minio/policy/user_deny_write.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Deny",
"Action": [
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::${aws:username}-*"
]
}
]
}
23 changes: 23 additions & 0 deletions deploy/objectstorage/etc/minio/policy/user_normal.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:GetBucketPolicy",
"s3:GetBucketLocation",
"s3:GetBucketTagging",
"s3:PutBucketTagging",
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::${aws:username}-*"
]
}
]
}
2 changes: 2 additions & 0 deletions deploy/objectstorage/images/shim/imageList
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
quay.io/prometheus/prometheus:v2.45.0
minio/minio:RELEASE.2023-11-11T08-14-41Z
33 changes: 33 additions & 0 deletions deploy/objectstorage/init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash
set -e
export readonly ARCH=${1:-amd64}
mkdir -p tars

RetryPullImageInterval=3
RetrySleepSeconds=3

retryPullImage() {
local image=$1
local retry=0
local retryMax=3
set +e
while [ $retry -lt $RetryPullImageInterval ]; do
sealos pull --policy=always --platform=linux/"${ARCH}" $image >/dev/null && break
retry=$(($retry + 1))
echo "retry pull image $image, retry times: $retry"
sleep $RetrySleepSeconds
done
set -e
if [ $retry -eq $retryMax ]; then
echo "pull image $image failed"
exit 1
fi
}

retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-controller:latest
retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest
retryPullImage ghcr.io/labring/sealos-cloud-minio-service:latest

sealos save -o tars/objectstorage-controller.tar ghcr.io/labring/sealos-cloud-objectstorage-controller:latest
sealos save -o tars/objectstorage-frontend.tar ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest
sealos save -o tars/objectstorage-service.tar ghcr.io/labring/sealos-cloud-minio-service:latest
Loading
Loading