Add ARM build image

.github/ISSUE_TEMPLATE/awesome-ilogtail.md

@@ -0,0 +1,10 @@
+---
+name: Awesome iLogtail
+about: Join the iLogtail ambanssadar program
+title: "[Publicity]:"
+labels: awesome ilogtail
+assignees: ''
+
+---
+
+Please post your Zhihu article link, WeChat article link or any proof of publicity here.

.github/workflows/build.yaml

@@ -27,7 +27,7 @@ jobs:
       matrix:
         go-version: [ 1.15, 1.16 ]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [  ubuntu-latest, macos-latest, windows-2019 ]
+        runner: [ ubuntu-latest, macos-latest, windows-2019 ]
       fail-fast: true
     steps:
 
@@ -59,6 +59,10 @@ jobs:
 
       - name: Build Docker
         if: matrix.runner == 'ubuntu-latest'
+        env:
+            BUILD_LOGTAIL_UT: ON
+            ENABLE_COMPATIBLE_MODE: ON
+            ENABLE_STATIC_LINK_CRT: ON
         run: make dist && scripts/check_glibc.sh && make docker
 
   result:

core/CMakeLists.txt

@@ -17,8 +17,8 @@ project(logtail)
 
 # Options.
 option(BUILD_LOGTAIL_UT "Build unit test for Logtail" OFF)
-option(ENABLE_COMPATIBLE_MODE "Build Logtail in compatible mode (for low version Linux)" ON)
-option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT statically" ON)
+option(ENABLE_COMPATIBLE_MODE "Build Logtail in compatible mode (for low version Linux)" OFF)
+option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT statically" OFF)
 
 # Name/Version information.
 if (NOT DEFINED LOGTAIL_VERSION)

docker/Dockerfile.ilogtail-build-linux-arm64

@@ -0,0 +1,15 @@
+FROM local/ilogtail-toolchain-linux-arm64 as dep-build
+
+RUN curl -sSfL https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/toolchain/goc-v1.4.1-linux-arm64.tar.gz -o goc-v1.4.1-linux-arm64.tar.gz && \
+    tar -xzf goc-v1.4.1-linux-arm64.tar.gz
+
+RUN curl -sSfL https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/prebuilt-dependencies/ilogtail-deps.linux-arm64.tar.gz -o ilogtail-deps.linux-arm64.tar.gz && \
+    tar -xzf ilogtail-deps.linux-arm64.tar.gz
+
+FROM local/ilogtail-toolchain-linux-arm64
+RUN mkdir -p /usr/local/bin /opt/logtail
+# install goc for coverage
+COPY --from=dep-build goc /usr/local/bin
+# install c++ deps
+COPY --from=dep-build ilogtail-deps.linux-arm64 /opt/logtail/deps
+

docker/Dockerfile.ilogtail-toolchain-linux-arm64

@@ -0,0 +1,81 @@
+FROM local/c7-systemd-linux-arm64 as toolchain-build
+
+# install dependencies
+RUN yum -y install epel-release centos-release-scl
+RUN yum -y install devtoolset-8-gcc devtoolset-8-gcc-c++ make
+RUN yum -y install curl-devel expat-devel gettext-devel openssl-devel perl-devel python3-devel zlib-devel \
+    asciidoc xmlto docbook2X wget
+
+# using gcc8
+ENV MANPATH=/opt/rh/devtoolset-8/root/usr/share/man \
+    PERL5LIB=/opt/rh/devtoolset-8/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-8/root/usr/lib/perl5:/opt/rh/devtoolset-8/root//usr/share/perl5/vendor_perl \
+    X_SCLS=devtoolset-8 \
+    PCP_DIR=/opt/rh/devtoolset-8/root \
+    LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib \
+    PATH=/usr/ali/bin:/opt/rh/devtoolset-8/root/usr/bin:$PATH \
+    PYTHONPATH=/opt/rh/devtoolset-8/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-8/root/usr/lib/python2.7/site-packages \
+    PKG_CONFIG_PATH=/opt/rh/devtoolset-8/root/usr/lib64/pkgconfig \
+    INFOPATH=/opt/rh/devtoolset-8/root/usr/share/info
+
+WORKDIR /
+RUN wget -nv https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/toolchain/cmake-3.23.2-linux-aarch64.tar.gz && \
+    wget -nv https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/toolchain/llvm-project-14.0.0.src.tar.xz && \
+    wget -nv https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/toolchain/go1.18.5.linux-arm64.tar.gz && \
+    wget -nv https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/toolchain/git-2.29.3.tar.xz
+
+# prepare cmake
+RUN tar -xzf cmake-3.23.2-linux-aarch64.tar.gz
+ENV CMAKE_ROOT=/cmake-3.23.2-linux-aarch64 \
+    PATH=/cmake-3.23.2-linux-aarch64/bin:$PATH
+
+# build clang
+RUN tar -xf llvm-project-14.0.0.src.tar.xz && \
+    mkdir llvm-project-14.0.0.src/build && \
+    cd llvm-project-14.0.0.src/build && \
+    cmake -G 'Unix Makefiles' -D CMAKE_C_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/gcc \
+        -D CMAKE_CXX_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/g++ \
+        -D LLVM_ENABLE_PROJECTS='clang' \
+        -D CMAKE_BUILD_TYPE=Release ../llvm && \
+    make -sj$(nproc) clang-format && \
+    cd /
+
+# build git
+RUN tar -xf git-2.29.3.tar.xz && \
+    cd git-2.29.3 && \
+    make -sj$(nproc) install install-man prefix=$PWD/build && \
+    cd /
+
+# prepare go
+RUN tar -xzf go1.18.5.linux-arm64.tar.gz
+
+FROM local/c7-systemd-linux-arm64
+
+# install dev tool set and debug utilities
+RUN yum -y install gcc gcc-c++ make libuuid-devel libstdc++-static systemd-devel iproute gdb net-tools which wget vim tree man openssh-clients
+RUN debuginfo-install -y glibc-2.17-326.el7_9.aarch64 libuuid-2.23.2-65.el7_9.1.aarch64
+
+# install cmake
+COPY --from=toolchain-build /cmake-3.23.2-linux-aarch64 /usr/
+
+# install clang-format
+COPY --from=toolchain-build /llvm-project-14.0.0.src/build/bin/clang-format /usr/bin/
+RUN chmod +x /usr/bin/clang-format
+
+# install golang
+WORKDIR /
+COPY --from=toolchain-build /go /go
+ENV GOROOT=/go GOPATH=/root/go PATH=/go/bin:$PATH
+
+# install go language server
+RUN go env -w GOPROXY="https://goproxy.cn,direct"
+RUN go install golang.org/x/tools/gopls@v0.9.1 && \
+    go install github.com/go-delve/delve/cmd/dlv@v1.6.1 && \
+    go install github.com/josharian/impl@v1.1.0 && \
+    go install github.com/fatih/gomodifytags@v1.16.0 && \
+    go install github.com/cweill/gotests/gotests@v1.6.0 && \
+    go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
+
+# install git
+COPY --from=toolchain-build /git-2.29.3/git /usr/bin
+RUN chmod +x /usr/bin/git
+

docker/Dockerfile_build

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux-amd64:latest as build
+FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux:latest as build
 
 WORKDIR /src
 

docker/Dockerfile_development_part

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux-amd64:latest
+FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux:latest
 
 ARG HOST_OS=Linux
 ARG VERSION=1.1.1

docker/Dockerfile_goc

@@ -14,6 +14,6 @@
 
 # goc server is only for e2e test to analysis code coverage.
 
-FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux-amd64:latest
+FROM sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux:latest
 
 ENTRYPOINT ["goc","server"]

docker/build-mono-arch-build-image.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ue
+set -o pipefail
+
+machine=`uname -m`
+[[ $machine = 'aarch64' ]] && arch=arm64 || arch=amd64
+
+docker build --rm -t local/c7-systemd-linux-${arch} . -f Dockerfile.c7-systemd-linux
+docker build --rm -t local/ilogtail-toolchain-linux-${arch} . -f Dockerfile.ilogtail-toolchain-linux-${arch}
+docker build --rm -t local/ilogtail-build-linux-${arch} . -f Dockerfile.ilogtail-build-linux-${arch}

docker/build-multi-arch-build-image.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ue
+set -o pipefail
+
+docker manifest create \
+sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux:latest \
+--amend sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux-amd64:latest \
+--amend sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux-arm64:latest
+
+docker manifest push sls-registry.cn-beijing.cr.aliyuncs.com/sls-microservices/ilogtail-build-linux:latest

external/sync_vendor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # This script is used after calling `go mod vendor`, it will use files in external directory to override
 # corresponding files in vendor directory.

scripts/gen_build_scripts.sh

@@ -28,19 +28,23 @@ VERSION=${3:-1.1.1}
 REPOSITORY=${4:-aliyun/ilogtail}
 OUT_DIR=${5:-output}
 
+BUILD_LOGTAIL_UT=${BUILD_LOGTAIL_UT:-OFF}
+ENABLE_COMPATIBLE_MODE=${ENABLE_COMPATIBLE_MODE:-OFF}
+ENABLE_STATIC_LINK_CRT=${ENABLE_STATIC_LINK_CRT:-OFF}
+
 BUILD_SCRIPT_FILE=$GENERATED_HOME/gen_build.sh
 COPY_SCRIPT_FILE=$GENERATED_HOME/gen_copy_docker.sh
 
 function generateBuildScript() {
   rm -rf $BUILD_SCRIPT_FILE && echo -e "#!/bin/bash\nset -ue\nset -o pipefail\n" > $BUILD_SCRIPT_FILE && chmod 755 $BUILD_SCRIPT_FILE
   if [ $CATEGORY = "plugin" ]; then
-    echo './scripts/plugin_build.sh vendor c-shared '${OUT_DIR} >> $BUILD_SCRIPT_FILE;
+    echo "mkdir -p core/build && cd core/build && cmake -D CMAKE_BUILD_TYPE=Release -D LOGTAIL_VERSION=${VERSION} .. && cd plugin && make -s PluginAdapter && cd ../../.. && ./scripts/plugin_build.sh vendor c-shared ${OUT_DIR}" >> $BUILD_SCRIPT_FILE;
   elif [ $CATEGORY = "core" ]; then
-    echo "mkdir -p core/build && cd core/build && cmake -D CMAKE_BUILD_TYPE=Release -D LOGTAIL_VERSION=${VERSION} .. && make -sj\$(nproc)" >>  $BUILD_SCRIPT_FILE;
+    echo "mkdir -p core/build && cd core/build && cmake -DCMAKE_BUILD_TYPE=Release -DLOGTAIL_VERSION=${VERSION} -DBUILD_LOGTAIL_UT=${BUILD_LOGTAIL_UT} -DENABLE_COMPATIBLE_MODE=${ENABLE_COMPATIBLE_MODE} -DENABLE_STATIC_LINK_CRT=${ENABLE_STATIC_LINK_CRT} .. && make -sj\$(nproc)" >>  $BUILD_SCRIPT_FILE;
   elif [ $CATEGORY = "all" ]; then
-    echo "./scripts/plugin_build.sh vendor c-shared ${OUT_DIR} && mkdir -p core/build && cd core/build && cmake -D CMAKE_BUILD_TYPE=Release -D LOGTAIL_VERSION=${VERSION} .. && make -sj\$(nproc)" >> $BUILD_SCRIPT_FILE;
+    echo "./scripts/plugin_build.sh vendor c-shared ${OUT_DIR} && mkdir -p core/build && cd core/build && cmake -DCMAKE_BUILD_TYPE=Release -DLOGTAIL_VERSION=${VERSION} -DBUILD_LOGTAIL_UT=${BUILD_LOGTAIL_UT} -DENABLE_COMPATIBLE_MODE=${ENABLE_COMPATIBLE_MODE} -DENABLE_STATIC_LINK_CRT=${ENABLE_STATIC_LINK_CRT} .. && make -sj\$(nproc)" >> $BUILD_SCRIPT_FILE;
   elif [ $CATEGORY = "e2e" ]; then
-    echo "./scripts/plugin_gocbuild.sh ${OUT_DIR} && mkdir -p core/build && cd core/build && cmake -D LOGTAIL_VERSION=${VERSION} .. && make -sj\$(nproc)" >> $BUILD_SCRIPT_FILE;
+    echo "./scripts/plugin_gocbuild.sh ${OUT_DIR} && mkdir -p core/build && cd core/build && cmake -DLOGTAIL_VERSION=${VERSION} -DBUILD_LOGTAIL_UT=${BUILD_LOGTAIL_UT} -DENABLE_COMPATIBLE_MODE=${ENABLE_COMPATIBLE_MODE} -DENABLE_STATIC_LINK_CRT=${ENABLE_STATIC_LINK_CRT} .. && make -sj\$(nproc)" >> $BUILD_SCRIPT_FILE;
   fi
 }
 

scripts/plugin_build.sh

@@ -39,6 +39,7 @@ mkdir -p "$ROOTDIR"/bin
 if [ $OS_FLAG = 1 ]; then
   IDFLAGS='-extldflags "-Wl,--wrap=memcpy"'
   if [ $BUILDMODE = "c-shared" ]; then
+    cp ${ROOTDIR}/core/build/plugin/libPluginAdapter.so ${ROOTDIR}/vendor/github.com/alibaba/ilogtail/pkg/logtail/libPluginAdapter.so
     NAME=libPluginBase.so
   fi
 elif [ $OS_FLAG = 3 ]; then

scripts/plugin_gocbuild.sh

@@ -35,6 +35,7 @@ cd "$ROOTDIR"/plugin_main
 pwd
 
 if uname -s | grep Linux; then
+  cp ${ROOTDIR}/core/build/plugin/libPluginAdapter.so ${ROOTDIR}/vendor/github.com/alibaba/ilogtail/pkg/logtail/libPluginAdapter.so
   goc build '--buildflags=-mod=vendor -buildmode=c-shared -ldflags="-extldflags "-Wl,--wrap=memcpy""' --center=http://goc:7777 -o "$ROOTDIR/$OUT_DIR/${NAME}"
 else
   echo "goc build only build a dynamic library in linux platform"