Merge pull request #1 from paramite/sft-519-scheduler

[STF-519][STF-520][STF-521] basic functionality

.github/workflows/smoke.yml

@@ -0,0 +1,32 @@
+name: Smoke testing
+env:
+  PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-agent
+
+  TEST_IMAGE: registry.access.redhat.com/ubi8
+
+on: push
+
+jobs:
+  smoketest:
+    name: "basic functionality test"
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Check if sg-core repository has same topic branch
+        id: core_branch
+        continue-on-error: true
+        run: |
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-core.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Check if apputils repository has same topic branch
+        id: apputils_branch
+        continue-on-error: true
+        run: |
+          git ls-remote --exit-code --heads https://github.com/infrawatch/apputils.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Run sg-agent tests
+        run: |
+          docker run --name=sgagent -uroot --network host \
+            -e GITHUB_REF -e FOUND_CORE_BRANCH=${{ steps.core_branch.outcome }} \
+            -e FOUND_APPUTILS_BRANCH=${{ steps.apputils_branch.outcome }} \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/smoke/run.sh

.github/workflows/tests.yml

@@ -0,0 +1,67 @@
+name: CI
+env:
+  PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-agent
+
+  TEST_IMAGE: registry.access.redhat.com/ubi8
+  COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+
+on: [push, pull_request]
+
+jobs:
+  golangci:
+    name: Linting
+    runs-on: ubuntu-20.04
+    container: 'quay.io/plmr/sg-core-ci'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - name: tidy
+        run: go mod tidy
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Caching conflicts happen in GHA, so just disable for now
+          skip-pkg-cache: true
+          skip-build-cache: true
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.33
+  unit-tests:
+    name: Unit tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Check if apputils repository has same topic branch
+        id: apputils_branch
+        continue-on-error: true
+        run: |
+          git ls-remote --exit-code --heads https://github.com/infrawatch/apputils.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Check if sg-core repository has same topic branch
+        id: core_branch
+        continue-on-error: true
+        run: |
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-core.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Run sg-agent unit test suite
+        run: |
+          docker run --name=testsuite -uroot --network host -e COVERALLS_REPO_TOKEN \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            -e GITHUB_REF -e FOUND_CORE_BRANCH=${{ steps.core_branch.outcome }} \
+            -e FOUND_APPUTILS_BRANCH=${{ steps.apputils_branch.outcome }} \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/unit/run_tests.sh
+      - name: Send coverage
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          path-to-profile: ${{ github.workspace }}/profile.cov
+  image-build:
+    name: Image build
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Verify image builds
+        run: |
+          docker build --tag infrawatch/sg-agent:latest --file build/Dockerfile .
+      - name: List images
+        run: |
+          docker images

.golangci.yaml

@@ -0,0 +1,58 @@
+run:
+  skip-dirs:
+    - plugins/transport/dummy-alertmanager
+    - plugins/transport/dummy-events
+    - plugins/transport/dummy-metrics
+    - plugins/transport/dummy-logs
+    - plugins/application/print
+    - devenv
+issues:
+  exclude-rules:
+    - linters:
+      - errcheck
+      text: "[a-zA-Z]+.[a-zA-Z]+.(Error|Info|Debug|Warn)" # from logger
+    - text: "[A-Z]+" #omit enums
+      linters:
+        - deadcode
+    - text: New
+      linters:
+        - deadcode
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    - dupl
+    - errcheck
+    # - exhaustive
+    # - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - golint
+    - goprintffuncname
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    # - unused
+    - unconvert
+    - unparam
+    - varcheck
+    # - whitespace
+

README.md

@@ -1,2 +1,21 @@
 # sg-agent
-Availability monitoring agent for SG
+
+## Set of sg-core application/transport plugins for spawning availability monitoring agent STF
+
+sg-agent serves as availability monitoring agent for STF. It executes
+self-scheduled and/or sg-core requested checks and reports results back.
+
+As a matter of fact sg-agent will not need sg-core as a server side component.
+The plan for sg-agent is to be self sufficient without server side or cooperate
+well with other availability monitoring solutions such as sensu-core, sensu-go,
+Nagios, Icinga, Zabbix or even Datadog.
+
+TO-DO:
+
+  - [ ] basic functionality
+  - [ ] sensu-core mediator (via RabbitMQ)
+  - [ ] sensu-go mediator (via websockets)
+  - [ ] DataDog mediator (via DogStatsD)
+  - [ ] Nagios mediator (via special Nagios plugin)
+  - [ ] Icinga mediator (via Icinga API)
+  - [ ] Zabbix mediator (via Zabbix API)

build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+base=$(pwd)
+
+PLUGIN_DIR=${PLUGIN_DIR:-"/tmp/plugins/"}
+
+build_plugin() {
+  plugin=$1
+
+  cd "$base/plugins/$plugin"
+  echo "building $plugin"
+  go build -o "$PLUGIN_DIR/$plugin.so" -buildmode=plugin
+}
+
+build_plugin "scheduler"
+build_plugin "executor"

build/Dockerfile

@@ -0,0 +1,35 @@
+# --- build smart gateway and agent plugins ---
+FROM registry.access.redhat.com/ubi8 AS builder
+ENV GOPATH=/go
+ENV CORE=/go/src/github.com/infrawatch/sg-core
+ENV WD=/go/src/github.com/infrawatch/sg-agent
+
+WORKDIR $WD
+COPY . $WD/
+
+# dependencies for qpid-proton-c
+COPY build/repos/opstools.repo /etc/yum.repos.d/opstools.repo
+RUN dnf install golang git qpid-proton-c-devel -y --setopt=tsflags=nodocs
+
+# build sg-core
+RUN git clone https://github.com/infrawatch/sg-core $CORE && pushd $CORE && PRODUCTION_BUILD=true ./build.sh && popd
+
+# sync go.mod with local sg-core
+RUN sed -i "s/go [0-9\.]\{1,\}/$(grep -e '^go [0-9\.]' ../sg-core/go.mod)/" go.mod
+RUN echo "replace $(grep sg-core go.mod | tr '\n' ' ' | tr '\t' ' ')=> $CORE" >> go.mod
+
+# build agent plugins
+RUN ./build.sh
+
+# --- end build, create smart gateway layer ---
+
+FROM registry.access.redhat.com/ubi8-minimal
+
+LABEL io.k8s.display-name="Smart Gateway Agent" \
+      io.k8s.description="A component of the Service Telemetry Framework on the client side that provides closed loop remediation" \
+      maintainer="Martin Mágr <mmagr@redhat.com>"
+
+COPY --from=builder /tmp/sg-core /
+COPY --from=builder /tmp/plugins/*.so /usr/lib64/sg-core/
+
+ENTRYPOINT ["/sg-core"]

build/repos/opstools.repo

@@ -0,0 +1,6 @@
+[centos-opstools]
+name=opstools
+baseurl=http://mirror.centos.org/centos/8/opstools/$basearch/collectd-5/
+gpgcheck=0
+enabled=1
+module_hotfixes=1

ci/smoke/config.yaml

@@ -0,0 +1,45 @@
+---
+
+pluginDir: /usr/lib64/sg-core
+logLevel: debug
+handlerErrors: false
+
+transports: []
+applications:
+  - name: scheduler
+    config:
+      logActions: true
+      tasks:
+        - name: test1
+          command: "echo 'test1'"
+        - name: test2
+          command: "echo 'test2' && exit 1"
+        - name: test3
+          command: "echo 'test3' && exit 3"
+        - name: test4
+          command: "echo 'test4'"
+      schedule:
+        - task: test1
+          interval: "1s"
+        - task: test2
+          interval: "1s"
+          instructions:
+            retries: 4
+            coolDown: 1
+        - task: test3
+          interval: "3s"
+          instructions:
+            muteOn:
+              - 3
+      reactions:
+        - ofTask: test3
+          condition: status=warning
+          reaction: test4
+  - name: executor
+    config:
+      logActions: true
+      workDirectory: /var/tmp
+  - name: print
+    config:
+      eventsoutput: /tmp/sg-agent.log
+      metricoutput: /dev/null

ci/smoke/run.sh

@@ -0,0 +1,66 @@
+#!/bin/env bash
+# CI script for UBI8 job
+# purpose: spawn sg-core with scheduler plugin and verify task executions are being requested
+
+set -ex
+
+# enable required repo(s)
+cat > /etc/yum.repos.d/fedora-eln.repo <<EOF
+[centos-opstools]
+name=opstools
+baseurl=http://mirror.centos.org/centos/8/opstools/\$basearch/collectd-5/
+gpgcheck=0
+enabled=1
+module_hotfixes=1
+EOF
+
+dnf install -y git golang gcc make qpid-proton-c-devel python3
+
+export APPUTILSPATH=$PWD/apputils
+export COREPATH=$PWD/sg-core
+
+mkdir -p /usr/lib64/sg-core
+
+# install sg-core
+git clone https://github.com/infrawatch/sg-core $COREPATH
+pushd $COREPATH
+
+if [[ $FOUND_CORE_BRANCH == 'success' ]]; then
+  git checkout "${GITHUB_REF#refs/heads/}"
+fi
+
+go mod tidy
+
+if [[ $FOUND_APPUTILS_BRANCH == 'success' ]]; then
+  git clone https://github.com/infrawatch/apputils $APPUTILSPATH
+  pushd $APPUTILSPATH
+  git checkout "${GITHUB_REF#refs/heads/}"
+  popd
+  echo "replace $(grep apputils go.mod | tr '\n' ' ' | tr '\t' ' ')=> $APPUTILSPATH" >> go.mod
+fi
+
+PRODUCTION_BUILD=true PLUGIN_DIR=/usr/lib64/sg-core/ ./build.sh
+popd
+
+# sync go.mod with local sg-core and apputils
+sed -i "s/go [0-9\.]\{1,\}/$(grep -e '^go [0-9\.]' $COREPATH/go.mod)/" go.mod
+echo "replace $(grep sg-core go.mod | tr '\n' ' ' | tr '\t' ' ')=> $COREPATH" >> go.mod
+if [[ $FOUND_APPUTILS_BRANCH == 'success' ]]; then
+  echo "replace $(grep apputils go.mod | tr '\n' ' ' | tr '\t' ' ')=> $APPUTILSPATH" >> go.mod
+fi
+
+# build sg-agent
+PLUGIN_DIR=/usr/lib64/sg-core/ ./build.sh
+
+# produce some events
+/tmp/sg-core -config ./ci/smoke/config.yaml &>/tmp/sg-core.log &
+CORE_PID=$!
+sleep 12
+kill -9 $CORE_PID
+
+# debug output
+cat /tmp/sg-core.log
+cat /tmp/sg-agent.log
+
+# validate produced events
+./ci/smoke/validate.py /tmp/sg-agent.log