From 62c6f7b79650915048592dd92aa4060a62491f55 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 17 Dec 2024 17:31:10 -0700 Subject: [PATCH 1/2] ci: fix the docker driver build (#15354) (cherry picked from commit 4d71ca3bbaf54841bfbcfbd7ca9503b481ee058a) --- .github/jsonnetfile.json | 2 +- .github/jsonnetfile.lock.json | 4 +- .github/release-workflows.jsonnet | 19 +- .../loki-release/workflows/build.libsonnet | 104 +---- .../loki-release/workflows/main.jsonnet | 5 +- .../loki-release/workflows/release.libsonnet | 68 ++- .../loki-release/workflows/workflows.jsonnet | 21 +- .github/workflows/check.yml | 2 +- .github/workflows/images.yml | 2 +- .github/workflows/minor-release-pr.yml | 38 +- .github/workflows/patch-release-pr.yml | 38 +- .github/workflows/release.yml | 43 ++ Makefile | 398 +++++++++--------- 13 files changed, 377 insertions(+), 367 deletions(-) diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json index 731d3c3c82974..e75ed698ab80f 100644 --- a/.github/jsonnetfile.json +++ b/.github/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "workflows" } }, - "version": "21f1189544e3976070cbdb6463f64c7a32dcc176" + "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae" } ], "legacyImports": true diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index d11f133725822..0f978214e1e77 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "21f1189544e3976070cbdb6463f64c7a32dcc176", - "sum": "IPS1oGR8k7jk6J2snciTycWFgtISCwXSPhJ3A+nEGvY=" + "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae", + "sum": "tml1dcFlo15kEE6JvN/nPY2xkhfeF3ERZjAyFbnguHA=" } ], "legacyImports": false diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index c0a4b5475fed8..9c7aeb2cd1e0d 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -1,9 +1,14 @@ local lokiRelease = import 'workflows/main.jsonnet'; -local build = lokiRelease.build; +local build = lokiRelease.build; local releaseLibRef = 'main'; - local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; +local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION'); +local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion; +local golangCiLintVersion = 'v1.60.3'; +local imageBuildTimeoutMin = 60; +local imagePrefix = 'grafana'; +local dockerPluginDir = 'clients/cmd/docker-driver'; local imageJobs = { loki: build.image('loki', 'cmd/loki'), @@ -15,7 +20,7 @@ local imageJobs = { 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), promtail: build.image('promtail', 'clients/cmd/promtail'), querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), - 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', 'clients/cmd/docker-driver'), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }; local weeklyImageJobs = { @@ -25,13 +30,6 @@ local weeklyImageJobs = { promtail: build.weeklyImage('promtail', 'clients/cmd/promtail'), }; -local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION'); -local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion; -local golangCiLintVersion = 'v1.60.3'; - -local imageBuildTimeoutMin = 60; -local imagePrefix = 'grafana'; - { 'patch-release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( @@ -77,6 +75,7 @@ local imagePrefix = 'grafana'; getDockerCredsFromVault=true, imagePrefix='grafana', releaseLibRef=releaseLibRef, + pluginBuildDir=dockerPluginDir, releaseRepo='grafana/loki', useGitHubAppToken=true, ), false, false diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet index 72a47e901e890..d174c362c691a 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -108,14 +108,14 @@ local releaseLibStep = common.releaseLibStep; dockerPlugin: function( name, path, + buildImage, dockerfile='Dockerfile', context='release', platform=[ 'linux/amd64', 'linux/arm64', - 'linux/arm', ] - ) + ) job.new() + job.withStrategy({ 'fail-fast': true, @@ -136,6 +136,7 @@ local releaseLibStep = common.releaseLibStep; + step.withId('platform') + step.withRun(||| mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -150,108 +151,39 @@ local releaseLibStep = common.releaseLibStep; step.new('Build and export', 'docker/build-push-action@v6') + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') - + step.withEnv({ - IMAGE_TAG: '${{ needs.version.outputs.version }}', - }) + step.with({ context: context, file: 'release/%s/%s' % [path, dockerfile], platforms: '${{ matrix.platform }}', push: false, tags: '${{ env.IMAGE_PREFIX }}/%s:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}' % [name], - outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - 'build-args': 'IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}', + outputs: 'type=local,dest=release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}' % name, + 'build-args': ||| + %s + ||| % std.rstripChars(std.lines([ + 'IMAGE_TAG=${{ needs.version.outputs.version }}', + 'GOARCH=${{ steps.platform.outputs.platform_short }}', + ('BUILD_IMAGE=%s' % buildImage), + ]), '\n'), }), - releaseStep('Package as Docker plugin') + step.new('compress rootfs') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') - + step.withEnv({ - IMAGE_TAG: '${{ needs.version.outputs.version }}', - BUILD_DIR: 'release/%s' % [path], - }) + step.withRun(||| - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - |||), + tar -cf release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . + ||| % [name, name]), step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + step.with({ - path: 'release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - destination: '${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images', //TODO: make bucket configurable + path: 'release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + destination: '${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins', process_gcloudignore: false, }), ]), - weeklyDockerPlugin: function( - name, - path, - dockerfile='Dockerfile', - context='release', - platform=[ - 'linux/amd64', - 'linux/arm64', - 'linux/arm', - ] - ) - job.new() - + job.withStrategy({ - matrix: { - platform: platform, - }, - }) - + job.withSteps([ - common.fetchReleaseLib, - common.fetchReleaseRepo, - common.setupNode, - - step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), - step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), - step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), - - releaseStep('Get weekly version') - + step.withId('weekly-version') - + step.withRun(||| - echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT - - platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" - echo "platform=${platform}" >> $GITHUB_OUTPUT - echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT - if [[ "${platform}" == "linux/arm64" ]]; then - echo "plugin_arch=-arm64" >> $GITHUB_OUTPUT - else - echo "plugin_arch=" >> $GITHUB_OUTPUT - fi - |||), - - step.new('Build and export', 'docker/build-push-action@v6') - + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') - + step.with({ - context: context, - file: 'release/%s/%s' % [path, dockerfile], - platforms: '${{ matrix.platform }}', - push: false, - tags: '${{ env.IMAGE_PREFIX }}/%s:${{ steps.weekly-version.outputs.version }}' % [name], - outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - 'build-args': 'IMAGE_TAG=${{ steps.weekly-version.outputs.version }},GOARCH=${{ steps.weekly-version.outputs.platform_short }}', - }), - - releaseStep('Package and push as Docker plugin') - + step.withEnv({ - IMAGE_TAG: '${{ steps.weekly-version.outputs.version }}', - BUILD_DIR: 'release/%s' % [path], - }) - + step.withRun(||| - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - docker plugin push "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" - |||), - ]), - version: job.new() + job.withSteps([ diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet index b4d7b24246cf9..a642294498558 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet @@ -83,11 +83,13 @@ dockerUsername='grafanabot', getDockerCredsFromVault=false, imagePrefix='grafana', + pluginBuildDir='release/plugin-tmp-dir', publishBucket='', publishToGCS=false, releaseLibRef='main', releaseRepo='grafana/loki-release', useGitHubAppToken=true, + dockerPluginPath='clients/cmd/docker-driver', ) { name: 'create release', on: { @@ -119,7 +121,8 @@ shouldRelease: $.release.shouldRelease, createRelease: $.release.createRelease, publishImages: $.release.publishImages(getDockerCredsFromVault, dockerUsername), - publishRelease: $.release.publishRelease, + publishDockerPlugins: $.release.publishDockerPlugins(pluginBuildDir, getDockerCredsFromVault, dockerUsername), + publishRelease: $.release.publishRelease(['createRelease', 'publishImages', 'publishDockerPlugins']), }, }, check: { diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet index b77d8fe615ec9..d35c8a76661e8 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -186,19 +186,57 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// ] ), - publishRelease: job.new() - + job.withNeeds(['createRelease', 'publishImages']) - + job.withSteps([ - common.fetchReleaseRepo, - common.githubAppToken, - common.setToken, - releaseStep('publish release') - + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') - + step.withEnv({ - GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', - }) - + step.withRun(||| - gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} - |||), - ]), + publishDockerPlugins: function(path, getDockerCredsFromVault=false, dockerUsername='grafanabot') + job.new() + + job.withNeeds(['createRelease']) + + job.withSteps( + [ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.googleAuth, + common.setupGoogleCloudSdk, + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + ] + (if getDockerCredsFromVault then [ + step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), + ] else [ + step.new('Login to DockerHub (from secrets)', 'docker/login-action@v3') + + step.with({ + username: dockerUsername, + password: '${{ secrets.DOCKER_PASSWORD }}', + }), + ]) + + [ + step.new('download and prepare plugins') + + step.withRun(||| + echo "downloading images to $(pwd)/plugins" + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/plugins . + mkdir -p "release/%s" + ||| % path), + step.new('publish docker driver', './lib/actions/push-images') + + step.with({ + imageDir: 'plugins', + imagePrefix: '${{ env.IMAGE_PREFIX }}', + isPlugin: true, + buildDir: 'release/%s' % path, + }), + ] + ), + + publishRelease: function(dependencies=['createRelease']) + job.new() + + job.withNeeds(dependencies) + + job.withSteps([ + common.fetchReleaseRepo, + common.githubAppToken, + common.setToken, + releaseStep('publish release') + + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} + |||), + ]), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet index 1992d23f0d3b9..930aa539f25c6 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet @@ -1,15 +1,22 @@ local lokiRelease = import 'main.jsonnet'; local build = lokiRelease.build; + + +local buildImage = 'grafana/loki-build-image:0.34.3'; +local dockerPluginDir = 'clients/cmd/docker-driver'; + { '.github/workflows/release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( imageJobs={ loki: build.image('fake-loki', 'cmd/loki'), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }, + buildImage=buildImage, buildArtifactsBucket='loki-build-artifacts', branches=['release-[0-9]+.[0-9]+.x'], imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', skipValidation=false, versioningStrategy='always-bump-patch', @@ -21,12 +28,14 @@ local build = lokiRelease.build; lokiRelease.releasePRWorkflow( imageJobs={ loki: build.image('fake-loki', 'cmd/loki'), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }, + buildImage=buildImage, buildArtifactsBucket='loki-build-artifacts', branches=['release-[0-9]+.[0-9]+.x'], dryRun=true, imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', skipValidation=false, versioningStrategy='always-bump-patch', @@ -41,11 +50,13 @@ local build = lokiRelease.build; lokiRelease.releaseWorkflow( branches=['release-[0-9]+.[0-9]+.x'], buildArtifactsBucket='loki-build-artifacts', - getDockerCredsFromVault=true, + dockerUsername='trevorwhitney075', + getDockerCredsFromVault=false, imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + pluginBuildDir=dockerPluginDir, + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', - useGitHubAppToken=false, + useGitHubAppToken=true, ) + { name: 'Create Release', on+: { diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5e90ce2c01920..82a801ec8668a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ "check": "uses": "grafana/loki-release/.github/workflows/check.yml@main" "with": - "build_image": "grafana/loki-build-image:0.34.1" + "build_image": "grafana/loki-build-image:0.34.3" "golang_ci_lint_version": "v1.60.3" "release_lib_ref": "main" "skip_validation": false diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 55bbd64d6c3f9..cc56042b4420f 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -2,7 +2,7 @@ "check": "uses": "grafana/loki-release/.github/workflows/check.yml@main" "with": - "build_image": "grafana/loki-build-image:0.34.1" + "build_image": "grafana/loki-build-image:0.34.3" "golang_ci_lint_version": "v1.60.3" "release_lib_ref": "main" "skip_validation": false diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index bd3eb329d5977..9d7ae022341d9 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.34.1" + build_image: "grafana/loki-build-image:0.34.3" golang_ci_lint_version: "v1.60.3" release_lib_ref: "main" skip_validation: false @@ -144,7 +144,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.34.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.34.3" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -631,6 +631,7 @@ jobs: name: "parse image platform" run: | mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -641,37 +642,33 @@ jobs: echo "plugin_arch=" >> $GITHUB_OUTPUT fi working-directory: "release" - - env: - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v6" with: - build-args: "IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}" + build-args: | + IMAGE_TAG=${{ needs.version.outputs.version }} + GOARCH=${{ steps.platform.outputs.platform_short }} + BUILD_IMAGE=grafana/loki-build-image:0.34.3 context: "release" file: "release/clients/cmd/docker-driver/Dockerfile" - outputs: "type=docker,dest=release/images/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=local,dest=release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}" platforms: "${{ matrix.platform }}" push: false tags: "${{ env.IMAGE_PREFIX }}/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - env: - BUILD_DIR: "release/clients/cmd/docker-driver" - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Package as Docker plugin" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "compress rootfs" run: | - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - working-directory: "release" + tar -cf release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: - destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" - path: "release/images/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins" + path: "release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -679,7 +676,6 @@ jobs: platform: - "linux/amd64" - "linux/arm64" - - "linux/arm" promtail: needs: - "version" @@ -910,4 +906,4 @@ name: "Prepare Minor Release PR from Weekly" permissions: contents: "write" id-token: "write" - pull-requests: "write" \ No newline at end of file + pull-requests: "write" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index f12377a941ecf..c0e5bad0e5d1e 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.34.1" + build_image: "grafana/loki-build-image:0.34.3" golang_ci_lint_version: "v1.60.3" release_lib_ref: "main" skip_validation: false @@ -144,7 +144,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.34.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.34.3" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -631,6 +631,7 @@ jobs: name: "parse image platform" run: | mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -641,37 +642,33 @@ jobs: echo "plugin_arch=" >> $GITHUB_OUTPUT fi working-directory: "release" - - env: - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v6" with: - build-args: "IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}" + build-args: | + IMAGE_TAG=${{ needs.version.outputs.version }} + GOARCH=${{ steps.platform.outputs.platform_short }} + BUILD_IMAGE=grafana/loki-build-image:0.34.3 context: "release" file: "release/clients/cmd/docker-driver/Dockerfile" - outputs: "type=docker,dest=release/images/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=local,dest=release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}" platforms: "${{ matrix.platform }}" push: false tags: "${{ env.IMAGE_PREFIX }}/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - env: - BUILD_DIR: "release/clients/cmd/docker-driver" - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Package as Docker plugin" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "compress rootfs" run: | - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - working-directory: "release" + tar -cf release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: - destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" - path: "release/images/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins" + path: "release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -679,7 +676,6 @@ jobs: platform: - "linux/amd64" - "linux/arm64" - - "linux/arm" promtail: needs: - "version" @@ -910,4 +906,4 @@ name: "Prepare Patch Release PR" permissions: contents: "write" id-token: "write" - pull-requests: "write" \ No newline at end of file + pull-requests: "write" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1628a0b57dfb5..426f523f80da7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -110,6 +110,48 @@ jobs: parent: false path: "release/dist" process_gcloudignore: false + publishDockerPlugins: + needs: + - "createRelease" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Login to DockerHub (from vault)" + uses: "grafana/shared-workflows/actions/dockerhub-login@main" + - name: "download and prepare plugins" + run: | + echo "downloading images to $(pwd)/plugins" + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/plugins . + mkdir -p "release/clients/cmd/docker-driver" + - name: "publish docker driver" + uses: "./lib/actions/push-images" + with: + buildDir: "release/clients/cmd/docker-driver" + imageDir: "plugins" + imagePrefix: "${{ env.IMAGE_PREFIX }}" + isPlugin: true publishImages: needs: - "createRelease" @@ -148,6 +190,7 @@ jobs: needs: - "createRelease" - "publishImages" + - "publishDockerPlugins" runs-on: "ubuntu-latest" steps: - name: "pull code to release" diff --git a/Makefile b/Makefile index 979d63119244f..76de2fac98dfc 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,104 @@ -# Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/ -.PHONY: help -help: ## Display this help and any documented user-facing targets. Other undocumented targets may be present in the Makefile. -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST) +# ______ ____ __ __ _ +# / ____/________ _/ __/___ _____ ____ _ / / ____ / /__(_) +# / / __/ ___/ __ `/ /_/ __ `/ __ \/ __ `/ / / / __ \/ //_/ / +# / /_/ / / / /_/ / __/ /_/ / / / / /_/ / / /___/ /_/ / ,< / / +# \____/_/ \__,_/_/ \__,_/_/ /_/\__,_/ /_____/\____/_/|_/_/ +# +# Loki Project Makefile + +SHELL = /usr/bin/env bash -o pipefail .DEFAULT_GOAL := all + +# Certain aspects of the build are done in containers for consistency (e.g. +# yacc/protobuf generation) If you have the correct tools installed and you +# want to speed up development you can run make BUILD_IN_CONTAINER=false target +# or you can override this with an environment variable. +BUILD_IN_CONTAINER ?= true +CI ?= false + +# Ensure you run `make release-workflows` after changing this +GO_VERSION := 1.23.1 +BUILD_IMAGE_TAG := 0.34.3 + +IMAGE_TAG ?= $(shell ./tools/image-tag) +GIT_REVISION := $(shell git rev-parse --short HEAD) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) + +# Golang environment +GOOS ?= $(shell go env GOOS) +GOHOSTOS ?= $(shell go env GOHOSTOS) +GOARCH ?= $(shell go env GOARCH) +GOARM ?= $(shell go env GOARM) +GOEXPERIMENT ?= $(shell go env GOEXPERIMENT) +CGO_ENABLED := 0 +GO_ENV := GOEXPERIMENT=$(GOEXPERIMENT) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) CGO_ENABLED=$(CGO_ENABLED) + +GOTEST ?= go test + +# Build flags +VPREFIX := github.com/grafana/loki/v3/pkg/util/build +GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) \ + -X $(VPREFIX).Version=$(IMAGE_TAG) \ + -X $(VPREFIX).Revision=$(GIT_REVISION) \ + -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) \ + -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") + +GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo +DYN_GO_FLAGS := -ldflags "-s -w $(GO_LDFLAGS)" -tags netgo + +# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented +# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. +# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo +DEBUG_DYN_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo + +# Image names +IMAGE_PREFIX ?= grafana +BUILD_IMAGE := $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_TAG) +LOKI_IMAGE := $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) +CANARY_IMAGE := $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) +QUERY_TEE_IMAGE := $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) +LOGCLI_IMAGE := $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) +PROMTAIL_IMAGE := $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) +LOGQL_ANALYZER_IMAGE := $(IMAGE_PREFIX)/logql-analyzer:$(IMAGE_TAG) +OPERATOR_IMAGE := $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) + +# OCI (Docker) setup +OCI_PLATFORMS := --platform=linux/amd64,linux/arm64 +OCI_BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) +OCI_PUSH_ARGS := -o type=registry +OCI_PUSH := docker push +OCI_TAG := docker tag + +ifeq ($(CI),true) + # ensure buildx is set up + _ := $(shell ./tools/ensure-buildx-builder.sh) + OCI_BUILD := DOCKER_BUILDKIT=1 docker buildx build $(OCI_PLATFORMS) $(OCI_BUILD_ARGS) +else + OCI_BUILD := DOCKER_BUILDKIT=1 docker build $(OCI_BUILD_ARGS) +endif + +BUILD_OCI_PUSH := $(OCI_BUILD) $(OCI_PUSH_ARGS) + +# Docker mount flag, ignored on native docker host. +# See https://docs.docker.com/docker-for-mac/osxfs-caching/#delegated +MOUNT_FLAGS := :delegated + +define run_in_container + @mkdir -p $(shell pwd)/.pkg $(shell pwd)/.cache + @echo ">>> Running make $@ in container ..." + docker run --rm --tty --interactive \ + -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(BUILD_IMAGE) -f Makefile $@; +endef + +# Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/ +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-45s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + .PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary loki-canary-boringcrypto lint test clean yacc protos touch-protobuf-sources .PHONY: format check-format .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push @@ -20,54 +114,16 @@ help: .PHONY: k3d-loki k3d-enterprise-logs k3d-down .PHONY: helm-test helm-lint -SHELL = /usr/bin/env bash -o pipefail - -GOTEST ?= go test - ############# # Variables # ############# DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) -# Certain aspects of the build are done in containers for consistency (e.g. yacc/protobuf generation) -# If you have the correct tools installed and you want to speed up development you can run -# make BUILD_IN_CONTAINER=false target -# or you can override this with an environment variable -BUILD_IN_CONTAINER ?= true - -# ensure you run `make release-workflows` after changing this -BUILD_IMAGE_VERSION ?= 0.34.1 -GO_VERSION := 1.23.1 - -# Docker image info -IMAGE_PREFIX ?= grafana - -BUILD_IMAGE_PREFIX ?= grafana - -IMAGE_TAG ?= $(shell ./tools/image-tag) - -# Version info for binaries -GIT_REVISION := $(shell git rev-parse --short HEAD) -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) - # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/src/loki' phase. DONT_FIND := -name tools -prune -o -name vendor -prune -o -name operator -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o -# Build flags -VPREFIX := github.com/grafana/loki/v3/pkg/util/build -GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo -DYN_GO_FLAGS := -ldflags "-s -w $(GO_LDFLAGS)" -tags netgo -# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented -# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. -# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. -DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo -# Docker mount flag, ignored on native docker host. see (https://docs.docker.com/docker-for-mac/osxfs-caching/#delegated) -MOUNT_FLAGS := :delegated - # Protobuf files PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print) PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) @@ -96,28 +152,8 @@ DOC_FLAGS := $(DOC_SOURCES_PATH)/shared/configuration.md # Docker # ########## -# RM is parameterized to allow CircleCI to run builds, as it -# currently disallows `docker run --rm`. This value is overridden -# in circle.yml -RM := --rm -# TTY is parameterized to allow Google Cloud Builder to run builds, -# as it currently disallows TTY devices. This value needs to be overridden -# in any custom cloudbuild.yaml files -TTY := --tty - -DOCKER_BUILDKIT ?= 1 -BUILD_IMAGE = BUILD_IMAGE=$(BUILD_IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) -PUSH_OCI=docker push -TAG_OCI=docker tag -ifeq ($(CI), true) - OCI_PLATFORMS=--platform=linux/amd64,linux/arm64 - BUILD_OCI=DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker buildx build $(OCI_PLATFORMS) --build-arg $(BUILD_IMAGE) -else - BUILD_OCI=DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker build --build-arg $(BUILD_IMAGE) -endif - binfmt: - $(SUDO) docker run --privileged linuxkit/binfmt:v0.6 + docker run --privileged linuxkit/binfmt:v0.6 ################ # Main Targets # @@ -142,13 +178,14 @@ logcli: cmd/logcli/logcli ## build logcli executable logcli-debug: cmd/logcli/logcli-debug ## build debug logcli executable logcli-image: ## build logcli docker image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile . + $(OCI_BUILD) -t $(LOGCLI_IMAGE) -f cmd/logcli/Dockerfile . cmd/logcli/logcli: CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./cmd/logcli cmd/logcli/logcli-debug: CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o ./cmd/logcli/logcli-debug ./cmd/logcli + ######## # Loki # ######## @@ -171,7 +208,6 @@ loki-canary: cmd/loki-canary/loki-canary ## build loki-canary executable cmd/loki-canary/loki-canary: CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) - ############### # Loki-Canary (BoringCrypto)# ############### @@ -180,6 +216,7 @@ loki-canary-boringcrypto: cmd/loki-canary-boringcrypto/loki-canary-boringcrypto cmd/loki-canary-boringcrypto/loki-canary-boringcrypto: CGO_ENABLED=1 GOOS=linux GOARCH=$(GOARCH) GOEXPERIMENT=boringcrypto go build $(GO_FLAGS) -o $@ ./$(@D)/../loki-canary + ############### # Helm # ############### @@ -223,13 +260,13 @@ PROMTAIL_GO_FLAGS := $(GO_FLAGS) PROMTAIL_DEBUG_GO_FLAGS := $(DEBUG_GO_FLAGS) # Validate GOHOSTOS=linux && GOOS=linux to use CGO. -ifeq ($(shell go env GOHOSTOS),linux) -ifeq ($(shell go env GOOS),linux) +ifeq ($(GOHOSTOS),linux) +ifeq ($(GOOS),linux) ifneq ($(CGO_ENABLED), 0) PROMTAIL_CGO = 1 endif PROMTAIL_GO_FLAGS = $(DYN_GO_FLAGS) -PROMTAIL_DEBUG_GO_FLAGS = $(DYN_DEBUG_GO_FLAGS) +PROMTAIL_DEBUG_GO_FLAGS = $(DEBUG_DYN_GO_FLAGS) endif endif ifeq ($(PROMTAIL_JOURNAL_ENABLED), true) @@ -263,9 +300,7 @@ MIXIN_OUT_PATH_SSD := production/loki-mixin-compiled-ssd loki-mixin: ## compile the loki mixin ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else @rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH) @cd $(MIXIN_PATH) && jb install @@ -332,10 +367,7 @@ publish: packages # docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.24.1 lint lint: ## run linters ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else go version golangci-lint version @@ -352,7 +384,7 @@ test: all ## run the unit tests cd tools/lambda-promtail/ && $(GOTEST) -covermode=atomic -coverprofile=lambda-promtail-coverage.txt -p=4 ./... | tee lambda_promtail_test_results.txt test-integration: - $(GOTEST) -count=1 -v -tags=integration -timeout 10m ./integration + $(GOTEST) -count=1 -v -tags=integration -timeout 15m ./integration compare-coverage: ./tools/diff_coverage.sh $(old) $(new) $(packages) @@ -365,18 +397,18 @@ clean-protos: rm -rf $(PROTO_GOS) clean: ## clean the generated files - rm -rf clients/cmd/promtail/promtail - rm -rf cmd/loki/loki - rm -rf cmd/logcli/logcli - rm -rf cmd/loki-canary/loki-canary - rm -rf cmd/querytee/querytee rm -rf .cache rm -rf clients/cmd/docker-driver/rootfs - rm -rf dist/ rm -rf clients/cmd/fluent-bit/out_grafana_loki.h rm -rf clients/cmd/fluent-bit/out_grafana_loki.so - rm -rf cmd/migrate/migrate + rm -rf clients/cmd/promtail/promtail + rm -rf cmd/logcli/logcli rm -rf cmd/logql-analyzer/logql-analyzer + rm -rf cmd/loki-canary/loki-canary + rm -rf cmd/loki/loki + rm -rf cmd/migrate/migrate + rm -rf cmd/querytee/querytee + rm -rf dist/ $(MAKE) -BC clients/cmd/fluentd $@ go clean ./... @@ -388,14 +420,7 @@ yacc: $(YACC_GOS) %.y.go: %.y ifeq ($(BUILD_IN_CONTAINER),true) - # I wish we could make this a multiline variable however you can't pass more than simple arguments to them - @mkdir -p $(shell pwd)/.pkg - @mkdir -p $(shell pwd)/.cache - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else goyacc -p $(basename $(notdir $<)) -o $@ $< sed -i.back '/^\/\/line/ d' $@ @@ -410,13 +435,7 @@ ragel: $(RAGEL_GOS) %.rl.go: %.rl ifeq ($(BUILD_IN_CONTAINER),true) - @mkdir -p $(shell pwd)/.pkg - @mkdir -p $(shell pwd)/.cache - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else ragel -Z $< -o $@ endif @@ -429,13 +448,7 @@ protos: clean-protos $(PROTO_GOS) %.pb.go: ifeq ($(BUILD_IN_CONTAINER),true) - @mkdir -p $(shell pwd)/.pkg - @mkdir -p $(shell pwd)/.cache - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else @# The store-gateway RPC is based on Thanos which uses relative references to other protos, so we need @# to configure all such relative paths. `gogo/protobuf` is used by it. @@ -458,13 +471,17 @@ endif LOKI_DOCKER_DRIVER ?= "grafana/loki-docker-driver" PLUGIN_TAG ?= $(IMAGE_TAG) PLUGIN_ARCH ?= +PLUGIN_BUILD_ARGS ?= +ifeq ("$(PLUGIN_ARCH)", "-arm64") + PLUGIN_BUILD_ARGS = --build-arg GOARCH=arm64 +endif # build-rootfs # builds the plugin rootfs define build-rootfs rm -rf clients/cmd/docker-driver/rootfs || true mkdir clients/cmd/docker-driver/rootfs - docker build --build-arg $(BUILD_IMAGE) -t rootfsimage -f clients/cmd/docker-driver/Dockerfile . + $(OCI_BUILD) $(PLUGIN_BUILD_ARGS) -t rootfsimage -f clients/cmd/docker-driver/Dockerfile . ID=$$(docker create rootfsimage true) && \ (docker export $$ID | tar -x -C clients/cmd/docker-driver/rootfs) && \ @@ -510,17 +527,16 @@ fluent-bit-plugin: ## build the fluent-bit plugin go build $(DYN_GO_FLAGS) -buildmode=c-shared -o clients/cmd/fluent-bit/out_grafana_loki.so ./clients/cmd/fluent-bit/ fluent-bit-image: ## build the fluent-bit plugin docker image - $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . + $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . fluent-bit-image-cross: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . + $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . fluent-bit-push: fluent-bit-image-cross ## push the fluent-bit plugin docker image - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) + $(OCI_PUSH) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) fluent-bit-test: LOKI_URL ?= http://localhost:3100/loki/api/ fluent-bit-test: - docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \ - $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) + docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) ################## @@ -533,11 +549,11 @@ fluentd-plugin-push: ## push the fluentd plugin $(MAKE) -BC clients/cmd/fluentd $@ fluentd-image: ## build the fluentd docker image - $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile . + $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile . fluentd-push: fluentd-image-push: ## push the fluentd docker image - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) + $(OCI_PUSH) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) fluentd-test: LOKI_URL ?= http://loki:3100 fluentd-test: @@ -547,20 +563,21 @@ fluentd-test: # logstash plugin # ################## logstash-image: ## build the logstash image - $(SUDO) docker build -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./ + $(OCI_BUILD) -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./ # Send 10 lines to the local Loki instance. logstash-push-test-logs: LOKI_URL ?= http://host.docker.internal:3100/loki/api/v1/push logstash-push-test-logs: - $(SUDO) docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \ + docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \ $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f loki.conf logstash-push: ## push the logstash image - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) + $(OCI_PUSH) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) # Enter an env already configure to build and test logstash output plugin. logstash-env: - $(SUDO) docker run -v `pwd`/clients/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) + docker run -v `pwd`/clients/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh \ + $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) ######################## # Bigtable Backup Tool # @@ -570,11 +587,11 @@ BIGTABLE_BACKUP_TOOL_FOLDER = ./tools/bigtable-backup BIGTABLE_BACKUP_TOOL_TAG ?= $(IMAGE_TAG) bigtable-backup: - docker build -t $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(BIGTABLE_BACKUP_TOOL_FOLDER) - docker tag $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) + $(OCI_BUILD) -t $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(BIGTABLE_BACKUP_TOOL_FOLDER) + $(OCI_TAG) $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) push-bigtable-backup: bigtable-backup - docker push $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) + $(OCI_PUSH) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) ########## # Images # @@ -585,8 +602,8 @@ images: promtail-image loki-image loki-canary-image helm-test-image docker-drive # push(app, optional tag) # pushes the app, optionally tagging it differently before define push - $(SUDO) $(TAG_OCI) $(IMAGE_PREFIX)/$(1):$(IMAGE_TAG) $(IMAGE_PREFIX)/$(1):$(2) - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/$(1):$(2) + $(OCI_TAG) $(IMAGE_PREFIX)/$(1):$(IMAGE_TAG) $(IMAGE_PREFIX)/$(1):$(2) + $(OCI_PUSH) $(IMAGE_PREFIX)/$(1):$(2) endef # push-image(app) @@ -596,91 +613,79 @@ define push-image $(call push,$(1),main) endef -# promtail +# Promtail image promtail-image: ## build the promtail docker image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile . + $(OCI_BUILD) -t $(PROMTAIL_IMAGE) -f clients/cmd/promtail/Dockerfile . promtail-image-cross: - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile.cross . - + $(OCI_BUILD) -t $(PROMTAIL_IMAGE) -f clients/cmd/promtail/Dockerfile.cross . promtail-debug-image: ## build the promtail debug docker image - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG)-debug -f clients/cmd/promtail/Dockerfile.debug . - + $(OCI_BUILD) -t $(PROMTAIL_IMAGE)-debug -f clients/cmd/promtail/Dockerfile.debug . promtail-push: promtail-image-cross $(call push-image,promtail) -# loki +# Loki image loki-image: ## build the loki docker image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . + $(OCI_BUILD) -t $(LOKI_IMAGE) -f cmd/loki/Dockerfile . loki-image-cross: - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross . - -loki-debug-image: ## build the debug loki docker image - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)-debug -f cmd/loki/Dockerfile.debug . - + $(OCI_BUILD) -t $(LOKI_IMAGE) -f cmd/loki/Dockerfile.cross . +loki-debug-image: ## build the loki debug docker image + $(OCI_BUILD) -t $(LOKI_IMAGE)-debug -f cmd/loki/Dockerfile.debug . loki-push: loki-image-cross $(call push-image,loki) -# loki-canary -loki-canary-image: ## build the loki canary docker image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . +# Canary image +loki-canary-image: ## build the canary docker image + $(OCI_BUILD) -t $(LOKI_CANARY_IMAGE) -f cmd/loki-canary/Dockerfile . loki-canary-image-cross: - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . -loki-canary-image-cross-boringcrypto: - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) -f cmd/loki-canary-boringcrypto/Dockerfile . + $(OCI_BUILD) -t $(LOKI_CANARY_IMAGE) -f cmd/loki-canary/Dockerfile.cross . loki-canary-push: loki-canary-image-cross - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) + $(OCI_PUSH) $(LOKI_CANARY_IMAGE) +loki-canary-image-cross-boringcrypto: + $(OCI_BUILD) -t $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) -f cmd/loki-canary-boringcrypto/Dockerfile . loki-canary-push-boringcrypto: loki-canary-image-cross-boringcrypto - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) -helm-test-image: ## build the helm test image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile . -helm-test-push: helm-test-image ## push the helm test image - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) - -# loki-querytee -loki-querytee-image: - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile . + $(OCI_PUSH) $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) + +# Helm test image +helm-test-image: ## build the helm test docker image + $(OCI_BUIILD) -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile . +helm-test-push: helm-test-image + $(OCI_PUSH) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) + +# Query Tee image +loki-querytee-image: ## build the querytee docker image + $(OCI_BUILD) -t $(QUERY_TEE_IMAGE) -f cmd/querytee/Dockerfile . loki-querytee-image-cross: - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile.cross . + $(OCI_BUILD) -t $(QUERY_TEE_IMAGE) -f cmd/querytee/Dockerfile.cross . loki-querytee-push: loki-querytee-image-cross - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) + $(OCI_PUSH) $(QUERY_TEE_IMAGE) -# migrate-image -migrate-image: - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile . +# Migrate image +migrate-image: ## build the migrate docker image + $(OCI_BUILD) -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile . # LogQL Analyzer -logql-analyzer-image: ## build the LogQL Analyzer image - $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/logql-analyzer:$(IMAGE_TAG) -f cmd/logql-analyzer/Dockerfile . -logql-analyzer-push: logql-analyzer-image ## push the LogQL Analyzer image +logql-analyzer-image: ## build the logql analyzer docker image + $(OCI_BUILD) -t $(LOGQL_ANALYZER_IMAGE) -f cmd/logql-analyzer/Dockerfile . +logql-analyzer-push: logql-analyzer-image $(call push-image,logql-analyzer) - -# build-image -ensure-buildx-builder: -ifeq ($(CI),true) - ./tools/ensure-buildx-builder.sh -else - @echo "skipping buildx setup" -endif - -build-image: ensure-buildx-builder - $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) ./loki-build-image -build-image-push: build-image ## push the docker build image +# Build image +build-image: ## build the build docker image + $(OCI_BUILD) -t $(BUILD_IMAGE) ./loki-build-image +build-image-push: ifneq (,$(findstring WIP,$(IMAGE_TAG))) @echo "Cannot push a WIP image, commit changes first"; \ false; endif - echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin - $(SUDO) DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker buildx build $(OCI_PLATFORMS) \ - -o type=registry -t $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) ./loki-build-image + DOCKER_BUILDKIT=1 docker buildx build $(OCI_PLATFORMS) $(OCI_BUILD_ARGS) $(OCI_PUSH_ARGS) -t $(BUILD_IMAGE) ./loki-build-image -# loki-operator -loki-operator-image: - $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile operator/ +# Loki Operator +loki-operator-image: ## build the operator docker image + $(OCI_BUILD) -t $(OPERATOR_IMAGE) -f operator/Dockerfile operator/ loki-operator-image-cross: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile.cross operator/ + $(OCI_BUILD) -t $(OPERATOR_IMAGE) -f operator/Dockerfile.cross operator/ loki-operator-push: loki-operator-image-cross - $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) + $(OCI_PUSH) $(OPERATOR_IMAGE) ################# # Documentation # @@ -702,10 +707,7 @@ benchmark-store: # support go modules check-mod: ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else GO111MODULE=on GOPROXY=https://proxy.golang.org go mod download GO111MODULE=on GOPROXY=https://proxy.golang.org go mod verify @@ -743,14 +745,7 @@ fmt-jsonnet: fmt-proto: ifeq ($(BUILD_IN_CONTAINER),true) - # I wish we could make this a multiline variable however you can't pass more than simple arguments to them - @mkdir -p $(shell pwd)/.pkg - @mkdir -p $(shell pwd)/.cache - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else echo '$(PROTO_DEFS)' | \ xargs -n 1 -- buf format -w @@ -768,9 +763,7 @@ lint-scripts: # see https://github.com/settings/tokens lint-markdown: ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else lychee --verbose --config .lychee.toml ./*.md ./docs/**/*.md ./production/**/*.md ./cmd/**/*.md ./clients/**/*.md ./tools/**/*.md endif @@ -798,9 +791,7 @@ check-format: format doc: ## Generates the config file documentation ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else go run ./tools/doc-generator $(DOC_FLAGS_TEMPLATE) > $(DOC_FLAGS) endif @@ -882,16 +873,17 @@ scan-vulnerabilities: trivy snyk .PHONY: release-workflows release-workflows: +ifeq ($(BUILD_IN_CONTAINER),true) + $(run_in_container) +else pushd $(CURDIR)/.github && jb update && popd - jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_VERSION) .github/release-workflows.jsonnet + jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_TAG) .github/release-workflows.jsonnet +endif .PHONY: release-workflows-check release-workflows-check: ifeq ($(BUILD_IN_CONTAINER),true) - $(SUDO) docker run $(RM) $(TTY) -i \ - -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; + $(run_in_container) else @$(MAKE) release-workflows @echo "Checking diff" From f29d45d2a8afe8a0f6e90c772f21b15a07df0e89 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Wed, 18 Dec 2024 08:59:21 -0700 Subject: [PATCH 2/2] chore: revert Makefile change --- .github/workflows/minor-release-pr.yml | 2 +- .github/workflows/patch-release-pr.yml | 2 +- Makefile | 398 +++++++++++++------------ 3 files changed, 205 insertions(+), 197 deletions(-) diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 9d7ae022341d9..b8e46bf6816df 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -906,4 +906,4 @@ name: "Prepare Minor Release PR from Weekly" permissions: contents: "write" id-token: "write" - pull-requests: "write" + pull-requests: "write" \ No newline at end of file diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index c0e5bad0e5d1e..cfd24aefb9c20 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -906,4 +906,4 @@ name: "Prepare Patch Release PR" permissions: contents: "write" id-token: "write" - pull-requests: "write" + pull-requests: "write" \ No newline at end of file diff --git a/Makefile b/Makefile index 76de2fac98dfc..1286e7583afaf 100644 --- a/Makefile +++ b/Makefile @@ -1,104 +1,10 @@ -# ______ ____ __ __ _ -# / ____/________ _/ __/___ _____ ____ _ / / ____ / /__(_) -# / / __/ ___/ __ `/ /_/ __ `/ __ \/ __ `/ / / / __ \/ //_/ / -# / /_/ / / / /_/ / __/ /_/ / / / / /_/ / / /___/ /_/ / ,< / / -# \____/_/ \__,_/_/ \__,_/_/ /_/\__,_/ /_____/\____/_/|_/_/ -# -# Loki Project Makefile - -SHELL = /usr/bin/env bash -o pipefail - -.DEFAULT_GOAL := all - -# Certain aspects of the build are done in containers for consistency (e.g. -# yacc/protobuf generation) If you have the correct tools installed and you -# want to speed up development you can run make BUILD_IN_CONTAINER=false target -# or you can override this with an environment variable. -BUILD_IN_CONTAINER ?= true -CI ?= false - -# Ensure you run `make release-workflows` after changing this -GO_VERSION := 1.23.1 -BUILD_IMAGE_TAG := 0.34.3 - -IMAGE_TAG ?= $(shell ./tools/image-tag) -GIT_REVISION := $(shell git rev-parse --short HEAD) -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) - -# Golang environment -GOOS ?= $(shell go env GOOS) -GOHOSTOS ?= $(shell go env GOHOSTOS) -GOARCH ?= $(shell go env GOARCH) -GOARM ?= $(shell go env GOARM) -GOEXPERIMENT ?= $(shell go env GOEXPERIMENT) -CGO_ENABLED := 0 -GO_ENV := GOEXPERIMENT=$(GOEXPERIMENT) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) CGO_ENABLED=$(CGO_ENABLED) - -GOTEST ?= go test - -# Build flags -VPREFIX := github.com/grafana/loki/v3/pkg/util/build -GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) \ - -X $(VPREFIX).Version=$(IMAGE_TAG) \ - -X $(VPREFIX).Revision=$(GIT_REVISION) \ - -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) \ - -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") - -GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo -DYN_GO_FLAGS := -ldflags "-s -w $(GO_LDFLAGS)" -tags netgo - -# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented -# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. -# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. -DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -DEBUG_DYN_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo - -# Image names -IMAGE_PREFIX ?= grafana -BUILD_IMAGE := $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_TAG) -LOKI_IMAGE := $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -CANARY_IMAGE := $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -QUERY_TEE_IMAGE := $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -LOGCLI_IMAGE := $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -PROMTAIL_IMAGE := $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -LOGQL_ANALYZER_IMAGE := $(IMAGE_PREFIX)/logql-analyzer:$(IMAGE_TAG) -OPERATOR_IMAGE := $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) - -# OCI (Docker) setup -OCI_PLATFORMS := --platform=linux/amd64,linux/arm64 -OCI_BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) -OCI_PUSH_ARGS := -o type=registry -OCI_PUSH := docker push -OCI_TAG := docker tag - -ifeq ($(CI),true) - # ensure buildx is set up - _ := $(shell ./tools/ensure-buildx-builder.sh) - OCI_BUILD := DOCKER_BUILDKIT=1 docker buildx build $(OCI_PLATFORMS) $(OCI_BUILD_ARGS) -else - OCI_BUILD := DOCKER_BUILDKIT=1 docker build $(OCI_BUILD_ARGS) -endif - -BUILD_OCI_PUSH := $(OCI_BUILD) $(OCI_PUSH_ARGS) - -# Docker mount flag, ignored on native docker host. -# See https://docs.docker.com/docker-for-mac/osxfs-caching/#delegated -MOUNT_FLAGS := :delegated - -define run_in_container - @mkdir -p $(shell pwd)/.pkg $(shell pwd)/.cache - @echo ">>> Running make $@ in container ..." - docker run --rm --tty --interactive \ - -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(BUILD_IMAGE) -f Makefile $@; -endef - # Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/ -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-45s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +.PHONY: help +help: ## Display this help and any documented user-facing targets. Other undocumented targets may be present in the Makefile. +help: + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST) +.DEFAULT_GOAL := all .PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary loki-canary-boringcrypto lint test clean yacc protos touch-protobuf-sources .PHONY: format check-format .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push @@ -114,16 +20,54 @@ help: ## Display this help .PHONY: k3d-loki k3d-enterprise-logs k3d-down .PHONY: helm-test helm-lint +SHELL = /usr/bin/env bash -o pipefail + +GOTEST ?= go test + ############# # Variables # ############# DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) +# Certain aspects of the build are done in containers for consistency (e.g. yacc/protobuf generation) +# If you have the correct tools installed and you want to speed up development you can run +# make BUILD_IN_CONTAINER=false target +# or you can override this with an environment variable +BUILD_IN_CONTAINER ?= true + +# ensure you run `make release-workflows` after changing this +BUILD_IMAGE_VERSION ?= 0.34.3 +GO_VERSION := 1.23.1 + +# Docker image info +IMAGE_PREFIX ?= grafana + +BUILD_IMAGE_PREFIX ?= grafana + +IMAGE_TAG ?= $(shell ./tools/image-tag) + +# Version info for binaries +GIT_REVISION := $(shell git rev-parse --short HEAD) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) + # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/src/loki' phase. DONT_FIND := -name tools -prune -o -name vendor -prune -o -name operator -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o +# Build flags +VPREFIX := github.com/grafana/loki/v3/pkg/util/build +GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") +GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo +DYN_GO_FLAGS := -ldflags "-s -w $(GO_LDFLAGS)" -tags netgo +# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented +# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. +# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo +DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo +# Docker mount flag, ignored on native docker host. see (https://docs.docker.com/docker-for-mac/osxfs-caching/#delegated) +MOUNT_FLAGS := :delegated + # Protobuf files PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print) PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) @@ -152,8 +96,28 @@ DOC_FLAGS := $(DOC_SOURCES_PATH)/shared/configuration.md # Docker # ########## +# RM is parameterized to allow CircleCI to run builds, as it +# currently disallows `docker run --rm`. This value is overridden +# in circle.yml +RM := --rm +# TTY is parameterized to allow Google Cloud Builder to run builds, +# as it currently disallows TTY devices. This value needs to be overridden +# in any custom cloudbuild.yaml files +TTY := --tty + +DOCKER_BUILDKIT ?= 1 +BUILD_IMAGE = BUILD_IMAGE=$(BUILD_IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) +PUSH_OCI=docker push +TAG_OCI=docker tag +ifeq ($(CI), true) + OCI_PLATFORMS=--platform=linux/amd64,linux/arm64 + BUILD_OCI=DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker buildx build $(OCI_PLATFORMS) --build-arg $(BUILD_IMAGE) +else + BUILD_OCI=DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker build --build-arg $(BUILD_IMAGE) +endif + binfmt: - docker run --privileged linuxkit/binfmt:v0.6 + $(SUDO) docker run --privileged linuxkit/binfmt:v0.6 ################ # Main Targets # @@ -178,14 +142,13 @@ logcli: cmd/logcli/logcli ## build logcli executable logcli-debug: cmd/logcli/logcli-debug ## build debug logcli executable logcli-image: ## build logcli docker image - $(OCI_BUILD) -t $(LOGCLI_IMAGE) -f cmd/logcli/Dockerfile . + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile . cmd/logcli/logcli: CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./cmd/logcli cmd/logcli/logcli-debug: CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o ./cmd/logcli/logcli-debug ./cmd/logcli - ######## # Loki # ######## @@ -208,6 +171,7 @@ loki-canary: cmd/loki-canary/loki-canary ## build loki-canary executable cmd/loki-canary/loki-canary: CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) + ############### # Loki-Canary (BoringCrypto)# ############### @@ -216,7 +180,6 @@ loki-canary-boringcrypto: cmd/loki-canary-boringcrypto/loki-canary-boringcrypto cmd/loki-canary-boringcrypto/loki-canary-boringcrypto: CGO_ENABLED=1 GOOS=linux GOARCH=$(GOARCH) GOEXPERIMENT=boringcrypto go build $(GO_FLAGS) -o $@ ./$(@D)/../loki-canary - ############### # Helm # ############### @@ -260,13 +223,13 @@ PROMTAIL_GO_FLAGS := $(GO_FLAGS) PROMTAIL_DEBUG_GO_FLAGS := $(DEBUG_GO_FLAGS) # Validate GOHOSTOS=linux && GOOS=linux to use CGO. -ifeq ($(GOHOSTOS),linux) -ifeq ($(GOOS),linux) +ifeq ($(shell go env GOHOSTOS),linux) +ifeq ($(shell go env GOOS),linux) ifneq ($(CGO_ENABLED), 0) PROMTAIL_CGO = 1 endif PROMTAIL_GO_FLAGS = $(DYN_GO_FLAGS) -PROMTAIL_DEBUG_GO_FLAGS = $(DEBUG_DYN_GO_FLAGS) +PROMTAIL_DEBUG_GO_FLAGS = $(DYN_DEBUG_GO_FLAGS) endif endif ifeq ($(PROMTAIL_JOURNAL_ENABLED), true) @@ -300,7 +263,9 @@ MIXIN_OUT_PATH_SSD := production/loki-mixin-compiled-ssd loki-mixin: ## compile the loki mixin ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else @rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH) @cd $(MIXIN_PATH) && jb install @@ -367,7 +332,10 @@ publish: packages # docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.24.1 lint lint: ## run linters ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else go version golangci-lint version @@ -384,7 +352,7 @@ test: all ## run the unit tests cd tools/lambda-promtail/ && $(GOTEST) -covermode=atomic -coverprofile=lambda-promtail-coverage.txt -p=4 ./... | tee lambda_promtail_test_results.txt test-integration: - $(GOTEST) -count=1 -v -tags=integration -timeout 15m ./integration + $(GOTEST) -count=1 -v -tags=integration -timeout 10m ./integration compare-coverage: ./tools/diff_coverage.sh $(old) $(new) $(packages) @@ -397,18 +365,18 @@ clean-protos: rm -rf $(PROTO_GOS) clean: ## clean the generated files - rm -rf .cache - rm -rf clients/cmd/docker-driver/rootfs - rm -rf clients/cmd/fluent-bit/out_grafana_loki.h - rm -rf clients/cmd/fluent-bit/out_grafana_loki.so rm -rf clients/cmd/promtail/promtail + rm -rf cmd/loki/loki rm -rf cmd/logcli/logcli - rm -rf cmd/logql-analyzer/logql-analyzer rm -rf cmd/loki-canary/loki-canary - rm -rf cmd/loki/loki - rm -rf cmd/migrate/migrate rm -rf cmd/querytee/querytee + rm -rf .cache + rm -rf clients/cmd/docker-driver/rootfs rm -rf dist/ + rm -rf clients/cmd/fluent-bit/out_grafana_loki.h + rm -rf clients/cmd/fluent-bit/out_grafana_loki.so + rm -rf cmd/migrate/migrate + rm -rf cmd/logql-analyzer/logql-analyzer $(MAKE) -BC clients/cmd/fluentd $@ go clean ./... @@ -420,7 +388,14 @@ yacc: $(YACC_GOS) %.y.go: %.y ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + # I wish we could make this a multiline variable however you can't pass more than simple arguments to them + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else goyacc -p $(basename $(notdir $<)) -o $@ $< sed -i.back '/^\/\/line/ d' $@ @@ -435,7 +410,13 @@ ragel: $(RAGEL_GOS) %.rl.go: %.rl ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else ragel -Z $< -o $@ endif @@ -448,7 +429,13 @@ protos: clean-protos $(PROTO_GOS) %.pb.go: ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else @# The store-gateway RPC is based on Thanos which uses relative references to other protos, so we need @# to configure all such relative paths. `gogo/protobuf` is used by it. @@ -471,17 +458,13 @@ endif LOKI_DOCKER_DRIVER ?= "grafana/loki-docker-driver" PLUGIN_TAG ?= $(IMAGE_TAG) PLUGIN_ARCH ?= -PLUGIN_BUILD_ARGS ?= -ifeq ("$(PLUGIN_ARCH)", "-arm64") - PLUGIN_BUILD_ARGS = --build-arg GOARCH=arm64 -endif # build-rootfs # builds the plugin rootfs define build-rootfs rm -rf clients/cmd/docker-driver/rootfs || true mkdir clients/cmd/docker-driver/rootfs - $(OCI_BUILD) $(PLUGIN_BUILD_ARGS) -t rootfsimage -f clients/cmd/docker-driver/Dockerfile . + docker build --build-arg $(BUILD_IMAGE) -t rootfsimage -f clients/cmd/docker-driver/Dockerfile . ID=$$(docker create rootfsimage true) && \ (docker export $$ID | tar -x -C clients/cmd/docker-driver/rootfs) && \ @@ -527,16 +510,17 @@ fluent-bit-plugin: ## build the fluent-bit plugin go build $(DYN_GO_FLAGS) -buildmode=c-shared -o clients/cmd/fluent-bit/out_grafana_loki.so ./clients/cmd/fluent-bit/ fluent-bit-image: ## build the fluent-bit plugin docker image - $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . fluent-bit-image-cross: - $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile . fluent-bit-push: fluent-bit-image-cross ## push the fluent-bit plugin docker image - $(OCI_PUSH) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) fluent-bit-test: LOKI_URL ?= http://localhost:3100/loki/api/ fluent-bit-test: - docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) + docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \ + $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) ################## @@ -549,11 +533,11 @@ fluentd-plugin-push: ## push the fluentd plugin $(MAKE) -BC clients/cmd/fluentd $@ fluentd-image: ## build the fluentd docker image - $(OCI_BUILD) -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile . fluentd-push: fluentd-image-push: ## push the fluentd docker image - $(OCI_PUSH) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) fluentd-test: LOKI_URL ?= http://loki:3100 fluentd-test: @@ -563,21 +547,20 @@ fluentd-test: # logstash plugin # ################## logstash-image: ## build the logstash image - $(OCI_BUILD) -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./ + $(SUDO) docker build -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./ # Send 10 lines to the local Loki instance. logstash-push-test-logs: LOKI_URL ?= http://host.docker.internal:3100/loki/api/v1/push logstash-push-test-logs: - docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \ + $(SUDO) docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \ $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f loki.conf logstash-push: ## push the logstash image - $(OCI_PUSH) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) # Enter an env already configure to build and test logstash output plugin. logstash-env: - docker run -v `pwd`/clients/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh \ - $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) + $(SUDO) docker run -v `pwd`/clients/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) ######################## # Bigtable Backup Tool # @@ -587,11 +570,11 @@ BIGTABLE_BACKUP_TOOL_FOLDER = ./tools/bigtable-backup BIGTABLE_BACKUP_TOOL_TAG ?= $(IMAGE_TAG) bigtable-backup: - $(OCI_BUILD) -t $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(BIGTABLE_BACKUP_TOOL_FOLDER) - $(OCI_TAG) $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) + docker build -t $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(BIGTABLE_BACKUP_TOOL_FOLDER) + docker tag $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) push-bigtable-backup: bigtable-backup - $(OCI_PUSH) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) + docker push $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) ########## # Images # @@ -602,8 +585,8 @@ images: promtail-image loki-image loki-canary-image helm-test-image docker-drive # push(app, optional tag) # pushes the app, optionally tagging it differently before define push - $(OCI_TAG) $(IMAGE_PREFIX)/$(1):$(IMAGE_TAG) $(IMAGE_PREFIX)/$(1):$(2) - $(OCI_PUSH) $(IMAGE_PREFIX)/$(1):$(2) + $(SUDO) $(TAG_OCI) $(IMAGE_PREFIX)/$(1):$(IMAGE_TAG) $(IMAGE_PREFIX)/$(1):$(2) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/$(1):$(2) endef # push-image(app) @@ -613,79 +596,91 @@ define push-image $(call push,$(1),main) endef -# Promtail image +# promtail promtail-image: ## build the promtail docker image - $(OCI_BUILD) -t $(PROMTAIL_IMAGE) -f clients/cmd/promtail/Dockerfile . + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile . promtail-image-cross: - $(OCI_BUILD) -t $(PROMTAIL_IMAGE) -f clients/cmd/promtail/Dockerfile.cross . + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile.cross . + promtail-debug-image: ## build the promtail debug docker image - $(OCI_BUILD) -t $(PROMTAIL_IMAGE)-debug -f clients/cmd/promtail/Dockerfile.debug . + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG)-debug -f clients/cmd/promtail/Dockerfile.debug . + promtail-push: promtail-image-cross $(call push-image,promtail) -# Loki image +# loki loki-image: ## build the loki docker image - $(OCI_BUILD) -t $(LOKI_IMAGE) -f cmd/loki/Dockerfile . + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . loki-image-cross: - $(OCI_BUILD) -t $(LOKI_IMAGE) -f cmd/loki/Dockerfile.cross . -loki-debug-image: ## build the loki debug docker image - $(OCI_BUILD) -t $(LOKI_IMAGE)-debug -f cmd/loki/Dockerfile.debug . + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross . + +loki-debug-image: ## build the debug loki docker image + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)-debug -f cmd/loki/Dockerfile.debug . + loki-push: loki-image-cross $(call push-image,loki) -# Canary image -loki-canary-image: ## build the canary docker image - $(OCI_BUILD) -t $(LOKI_CANARY_IMAGE) -f cmd/loki-canary/Dockerfile . +# loki-canary +loki-canary-image: ## build the loki canary docker image + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . loki-canary-image-cross: - $(OCI_BUILD) -t $(LOKI_CANARY_IMAGE) -f cmd/loki-canary/Dockerfile.cross . -loki-canary-push: loki-canary-image-cross - $(OCI_PUSH) $(LOKI_CANARY_IMAGE) + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . loki-canary-image-cross-boringcrypto: - $(OCI_BUILD) -t $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) -f cmd/loki-canary-boringcrypto/Dockerfile . + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) -f cmd/loki-canary-boringcrypto/Dockerfile . +loki-canary-push: loki-canary-image-cross + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) loki-canary-push-boringcrypto: loki-canary-image-cross-boringcrypto - $(OCI_PUSH) $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) - -# Helm test image -helm-test-image: ## build the helm test docker image - $(OCI_BUIILD) -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile . -helm-test-push: helm-test-image - $(OCI_PUSH) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) - -# Query Tee image -loki-querytee-image: ## build the querytee docker image - $(OCI_BUILD) -t $(QUERY_TEE_IMAGE) -f cmd/querytee/Dockerfile . + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary-boringcrypto:$(IMAGE_TAG) +helm-test-image: ## build the helm test image + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile . +helm-test-push: helm-test-image ## push the helm test image + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) + +# loki-querytee +loki-querytee-image: + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile . loki-querytee-image-cross: - $(OCI_BUILD) -t $(QUERY_TEE_IMAGE) -f cmd/querytee/Dockerfile.cross . + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile.cross . loki-querytee-push: loki-querytee-image-cross - $(OCI_PUSH) $(QUERY_TEE_IMAGE) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -# Migrate image -migrate-image: ## build the migrate docker image - $(OCI_BUILD) -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile . +# migrate-image +migrate-image: + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile . # LogQL Analyzer -logql-analyzer-image: ## build the logql analyzer docker image - $(OCI_BUILD) -t $(LOGQL_ANALYZER_IMAGE) -f cmd/logql-analyzer/Dockerfile . -logql-analyzer-push: logql-analyzer-image +logql-analyzer-image: ## build the LogQL Analyzer image + $(SUDO) docker build --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/logql-analyzer:$(IMAGE_TAG) -f cmd/logql-analyzer/Dockerfile . +logql-analyzer-push: logql-analyzer-image ## push the LogQL Analyzer image $(call push-image,logql-analyzer) -# Build image -build-image: ## build the build docker image - $(OCI_BUILD) -t $(BUILD_IMAGE) ./loki-build-image -build-image-push: + +# build-image +ensure-buildx-builder: +ifeq ($(CI),true) + ./tools/ensure-buildx-builder.sh +else + @echo "skipping buildx setup" +endif + +build-image: ensure-buildx-builder + $(SUDO) $(BUILD_OCI) --build-arg=GO_VERSION=$(GO_VERSION) -t $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) ./loki-build-image +build-image-push: build-image ## push the docker build image ifneq (,$(findstring WIP,$(IMAGE_TAG))) @echo "Cannot push a WIP image, commit changes first"; \ false; endif - DOCKER_BUILDKIT=1 docker buildx build $(OCI_PLATFORMS) $(OCI_BUILD_ARGS) $(OCI_PUSH_ARGS) -t $(BUILD_IMAGE) ./loki-build-image + echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin + $(SUDO) DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker buildx build $(OCI_PLATFORMS) \ + -o type=registry -t $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) ./loki-build-image -# Loki Operator -loki-operator-image: ## build the operator docker image - $(OCI_BUILD) -t $(OPERATOR_IMAGE) -f operator/Dockerfile operator/ +# loki-operator +loki-operator-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile operator/ loki-operator-image-cross: - $(OCI_BUILD) -t $(OPERATOR_IMAGE) -f operator/Dockerfile.cross operator/ + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile.cross operator/ loki-operator-push: loki-operator-image-cross - $(OCI_PUSH) $(OPERATOR_IMAGE) + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) ################# # Documentation # @@ -707,7 +702,10 @@ benchmark-store: # support go modules check-mod: ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else GO111MODULE=on GOPROXY=https://proxy.golang.org go mod download GO111MODULE=on GOPROXY=https://proxy.golang.org go mod verify @@ -745,7 +743,14 @@ fmt-jsonnet: fmt-proto: ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + # I wish we could make this a multiline variable however you can't pass more than simple arguments to them + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else echo '$(PROTO_DEFS)' | \ xargs -n 1 -- buf format -w @@ -763,7 +768,9 @@ lint-scripts: # see https://github.com/settings/tokens lint-markdown: ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else lychee --verbose --config .lychee.toml ./*.md ./docs/**/*.md ./production/**/*.md ./cmd/**/*.md ./clients/**/*.md ./tools/**/*.md endif @@ -791,7 +798,9 @@ check-format: format doc: ## Generates the config file documentation ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else go run ./tools/doc-generator $(DOC_FLAGS_TEMPLATE) > $(DOC_FLAGS) endif @@ -873,17 +882,16 @@ scan-vulnerabilities: trivy snyk .PHONY: release-workflows release-workflows: -ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) -else pushd $(CURDIR)/.github && jb update && popd - jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_TAG) .github/release-workflows.jsonnet -endif + jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_VERSION) .github/release-workflows.jsonnet .PHONY: release-workflows-check release-workflows-check: ifeq ($(BUILD_IN_CONTAINER),true) - $(run_in_container) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else @$(MAKE) release-workflows @echo "Checking diff"