fix(io): file source ignore endLine buffer (#3431) #9466
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build packages | |
| concurrency: | |
| group: build-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| release: | |
| types: | |
| - published | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| arch: ${{ steps.arch.outputs.arch }} | |
| steps: | |
| - id: arch | |
| run: | | |
| if ${{ contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name) }}; then | |
| echo "arch=[\"linux/amd64\", \"linux/arm64\", \"linux/arm/v7\"]" >> $GITHUB_OUTPUT | |
| else | |
| echo "arch=[\"linux/amd64\"]" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| golang: | |
| - 1.23.1 | |
| arch: ${{fromJson(needs.prepare.outputs.arch)}} | |
| os: | |
| - debian | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - name: build | |
| if: matrix.os == 'debian' | |
| run: | | |
| docker run -i --rm \ | |
| -v $(pwd):/ekuiper \ | |
| --workdir /ekuiper \ | |
| --env KUIPER_SOURCE='/ekuiper' \ | |
| --platform ${{ matrix.arch }} \ | |
| ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os }} \ | |
| bash -euc "git config --global --add safe.directory /ekuiper && make pkg && make pkg_core && make pkg_full" | |
| - name: create sha file | |
| run: | | |
| cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - | |
| - name: Set artifact name | |
| id: artifact_name | |
| env: | |
| ARCH: ${{ matrix.arch }} | |
| REF: ${{ github.ref_name }} | |
| run: | | |
| arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g') | |
| ref=$(echo ${REF/\//-}) | |
| artifact_name="kuiper-${ref}-${{ matrix.os }}-${arch}" | |
| echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.artifact_name.outputs.artifact_name }} | |
| path: _packages/ | |
| build-on-mac: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23.1' | |
| - name: prepare | |
| run: | | |
| brew install zip unzip gnu-sed zmq | |
| echo "/usr/local/bin:$PATH" >> ~/.bashrc | |
| - name: build | |
| run: | | |
| make pkg | |
| cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd - | |
| - name: Set artifact name | |
| id: artifact_name | |
| env: | |
| REF: ${{ github.ref_name }} | |
| run: | | |
| arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g') | |
| ref=$(echo ${REF/\//-}) | |
| artifact_name="kuiper-${ref}-darwin-${arch}" | |
| echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.artifact_name.outputs.artifact_name }} | |
| path: _packages/ | |
| build-docker-images: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suffix: | |
| - "" | |
| - "-alpine" | |
| - "-alpine-python" | |
| - "-dev" | |
| - "-slim" | |
| - "-slim-python" | |
| - "-full" | |
| golang: | |
| - 1.23.1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/setup-qemu-action@v3 | |
| with: | |
| image: tonistiigi/binfmt:latest | |
| platforms: all | |
| - name: Build single platform image | |
| if: endsWith( matrix.suffix, 'python') == false | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: docker.io/lfedge/ekuiper | |
| build-args: GO_VERSION=${{ matrix.golang }} | |
| file: deploy/docker/Dockerfile${{ matrix.suffix }} | |
| - name: Test docker image | |
| run: | | |
| docker run -d --name ekuiper docker.io/lfedge/ekuiper | |
| ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ekuiper) | |
| sleep 5 | |
| if ! curl ${ip_address}:9081 >/dev/null 2>&1; then | |
| echo "docker image failed" | |
| docker logs ekuiper | |
| exit 1 | |
| fi | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: docker.io/lfedge/ekuiper | |
| flavor: | | |
| latest=${{ github.event_name == 'release' && matrix.suffix == '-alpine' && !github.event.release.prerelease}} | |
| suffix=${{ matrix.suffix }} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=ref,event=tag | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - uses: docker/login-action@v2 | |
| if: github.event_name == 'release' | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB_TOKEN }} | |
| - name: Build multi platform image | |
| if: contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name) | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64,linux/arm/v7 | |
| push: ${{ github.event_name == 'release' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| build-args: GO_VERSION=${{ matrix.golang }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| file: deploy/docker/Dockerfile${{ matrix.suffix }} | |
| build-plugins: | |
| runs-on: ubuntu-latest | |
| needs: prepare | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| plugin: | |
| - sinks/image | |
| - sinks/influx | |
| - sinks/influx2 | |
| - sinks/zmq | |
| - sinks/kafka | |
| - sinks/sql | |
| - sources/random | |
| - sources/zmq | |
| - sources/sql | |
| - sources/video | |
| - sources/kafka | |
| - functions/accumulateWordCount | |
| - functions/countPlusOne | |
| - functions/echo | |
| - functions/image | |
| - functions/geohash | |
| - functions/labelImage | |
| - functions/tfLite | |
| - functions/onnx | |
| arch: ${{fromJson(needs.prepare.outputs.arch)}} | |
| os: | |
| - [ debian, "slim" ] | |
| - [ alpine, "alpine" ] | |
| golang: | |
| - 1.23.1 | |
| exclude: | |
| - arch: linux/arm/v7 | |
| - os: [ alpine,"alpine" ] | |
| plugin: functions/labelImage | |
| - os: [ alpine,"alpine" ] | |
| plugin: functions/tfLite | |
| - os: [ alpine,"alpine" ] | |
| plugin: functions/onnx | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/setup-qemu-action@v3 | |
| with: | |
| image: tonistiigi/binfmt:latest | |
| platforms: all | |
| - name: build plugins | |
| run: | | |
| docker run -i --rm \ | |
| -v $(pwd):/ekuiper \ | |
| --workdir /ekuiper \ | |
| --platform ${{ matrix.arch }} \ | |
| ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os[0] }} \ | |
| bash -euc "git config --global --add safe.directory /ekuiper && make ${{ matrix.plugin }}" | |
| - name: Build ekuiper image | |
| uses: docker/build-push-action@v6 | |
| if: matrix.arch == 'linux/amd64' | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: docker.io/lfedge/ekuiper-${{ matrix.os[0] }} | |
| file: deploy/docker/Dockerfile-${{ matrix.os[1] }} | |
| - name: test docker and plugins | |
| if: matrix.arch == 'linux/amd64' | |
| env: | |
| PLUGIN: ${{ matrix.plugin }} | |
| run: | | |
| set -e -x -u | |
| plugin_type=$(echo ${PLUGIN%%/*}) | |
| plugin_name=$(echo ${PLUGIN##*/}) | |
| container_id=$(docker run -u 0 -d -v $(pwd)/_plugins:/var/plugins docker.io/lfedge/ekuiper-${{ matrix.os[0] }}) | |
| ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id) | |
| os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" ) | |
| sleep 5 | |
| if ! curl ${ip_address}:9081 >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi | |
| if [ "${plugin_name}" = "image" ]; then | |
| curl \ | |
| ${ip_address}:9081/plugins/${plugin_type} \ | |
| -X POST \ | |
| -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"resize\",\"thumbnail\"]}" | |
| elif [ "${plugin_name}" = "geohash" ]; then | |
| curl \ | |
| ${ip_address}:9081/plugins/${plugin_type} \ | |
| -X POST \ | |
| -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"geohashEncode\", \"geohashEncodeInt\", \"geohashDecode\", \"geohashDecodeInt\", \"geohashBoundingBox\", \"geohashBoundingBoxInt\", \"geohashNeighbor\", \"geohashNeighborInt\", \"geohashNeighbors\", \"geohashNeighborsInt\"]}" | |
| else | |
| curl \ | |
| ${ip_address}:9081/plugins/${plugin_type} \ | |
| -X POST \ | |
| -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}" | |
| fi | |
| docker logs ${container_id} | |
| [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1 | |
| - name: Set artifact name | |
| id: artifact_name | |
| env: | |
| PLUGIN: ${{ matrix.plugin }} | |
| ARCH: ${{ matrix.arch }} | |
| REF: ${{ github.ref_name }} | |
| run: | | |
| plugin_type=$(echo ${PLUGIN%%/*}) | |
| plugin_name=$(echo ${PLUGIN##*/}) | |
| arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g') | |
| ref=$(echo ${REF/\//-}) | |
| artifact_name="plugin-${ref}-${{ matrix.os[0] }}-${plugin_type}-${plugin_name}-${arch}" | |
| echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.artifact_name.outputs.artifact_name }} | |
| path: _plugins/ | |
| build-kubernetes-tool: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/setup-qemu-action@v3 | |
| with: | |
| image: tonistiigi/binfmt:latest | |
| platforms: all | |
| - name: Build single platform image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: docker.io/lfedge/ekuiper-kubernetes-tool | |
| file: deploy/docker/Dockerfile-kubernetes-tool | |
| - name: Test docker image | |
| run: | | |
| docker run -d --name kuiper-kubernetes-tool docker.io/lfedge/ekuiper-kubernetes-tool | |
| sleep 5 | |
| if [[ "$(docker logs kuiper-kubernetes-tool)" != *"Kuiper kubernetes tool is started successfully!"* ]]; then exit 1; fi | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: docker.io/lfedge/ekuiper-kubernetes-tool | |
| flavor: | | |
| latest=${{ github.event_name == 'release' && !github.event.release.prerelease}} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=ref,event=tag | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - uses: docker/login-action@v2 | |
| if: github.event_name == 'release' | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB_TOKEN }} | |
| - name: Update version for setup.py | |
| if: github.event_name == 'release' | |
| run: sed -i -r "s|([ \t]*version=).*|\1'${GITHUB_REF#refs/tags/}',|1" sdk/python/setup.py | |
| - name: Build multi platform image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386 | |
| push: ${{ github.event_name == 'release' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| file: deploy/docker/Dockerfile-kubernetes-tool | |
| release: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build | |
| - build-on-mac | |
| - build-plugins | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: kuiper-* | |
| path: _packages | |
| merge-multiple: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: plugin-* | |
| path: _plugins | |
| merge-multiple: true | |
| - name: check packages | |
| run: | | |
| cd _packages && for var in $( ls |grep -v sha256); do | |
| echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 | |
| done | |
| - uses: Rory-Z/upload-release-asset@v1 | |
| if: github.event_name == 'release' | |
| with: | |
| repo: ekuiper | |
| path: "_packages/kuiper*" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: upload packages to s3 | |
| if: github.event_name == 'release' | |
| run: | | |
| version=$(echo ${{ github.ref }} | sed -r "s .*/.*/(.*) \1 g") | |
| aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws configure set default.region us-west-2 | |
| aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version | |
| aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version | |
| aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version | |
| aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*" | |
| - name: update ekuiper.org | |
| if: github.event_name == 'release' | |
| run: | | |
| set -e -x -u | |
| curl -fvs -w %{http_code} \ | |
| --insecure \ | |
| -H "Content-Type: application/json" \ | |
| -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ | |
| -X POST \ | |
| -d "{\"repo\":\"lf-edge/ekuiper\", \"tag\": \"${{ github.ref_name }}\" }" \ | |
| ${{ secrets.EMQX_IO_RELEASE_API }} | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: "*" |