Merge pull request #228 from EA31337/dev

Development improvements

.github/workflows/check.yml

@@ -10,8 +10,8 @@ jobs:
   Pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v1
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ hashFiles('.pre-commit-config.yaml') }}
@@ -22,5 +22,5 @@ jobs:
         env:
           # yamllint disable-line rule:line-length
           SHFMT_URL: https://github.com/mvdan/sh/releases/download/v3.1.1/shfmt_v3.1.1_linux_amd64
-      - uses: actions/setup-python@v2
-      - uses: pre-commit/action@v2.0.3
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.0

.github/workflows/compile.yml

@@ -20,7 +20,7 @@ jobs:
   Compile:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: fx31337/mql-compile-action@master
         with:
           verbose: true

.github/workflows/docker.yml

@@ -27,15 +27,18 @@ jobs:
   Hadolint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: brpaz/hadolint-action@master
         with:
           dockerfile: Dockerfile
   Docker:
+    defaults:
+      run:
+        shell: bash
     runs-on: ubuntu-latest
     needs: Hadolint
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Build ubuntu-base
         run: docker build --target ubuntu-base -t ubuntu-base .
       - name: Build ubuntu-provisioned
@@ -53,30 +56,40 @@ jobs:
           --build-arg VERSION=${GITHUB_REF}
           --target ea-tester .
       - run: docker images
-      - name: Prints GitHub actor and repository
-        run: echo "${{ github.actor }} @ ${{ github.repository }}"
+      - name: Print GitHub actor, repository and ref
+        run: echo "${{ github.actor }} @ ${{ github.repository }}:${{ github.ref }}"
+
+      - name: Get the current branch name
+        id: myref
+        run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Push to Docker Hub
         if: >
           github.repository == 'EA31337/EA-Tester'
           && github.event_name != 'pull_request'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: ea31337/ea-tester
-          tag_with_ref: true
+          tags: ea31337/ea-tester:${{ steps.myref.outputs.branch }}
+
+      - name: Login to GitHub Packages Docker Registry
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Push to GitHub Packages
         if: >
           github.repository == 'EA31337/EA-Tester'
           && github.event_name != 'pull_request'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2
         with:
-          add_git_labels: true
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
           registry: docker.pkg.github.com
-          repository: ea31337/ea-tester/ea-tester
-          tag_with_ref: true
+          tags: ea31337/ea-tester/ea-tester:${{ steps.myref.outputs.branch }}
+
   Test-Docker:
     runs-on: ubuntu-latest
     needs: Docker

.github/workflows/sync-fork.yml

@@ -14,11 +14,11 @@ jobs:
     if: github.repository != 'EA31337/EA-Tester'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           path: upstream

.github/workflows/tests-mql.yml

@@ -37,7 +37,7 @@ jobs:
           - TestModellingQuality
           - TestTimeframes
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Check dependencies
         if: github.actor == 'nektos/act'
         run: >
@@ -61,7 +61,7 @@ jobs:
           - TestLotstep5  # @fixme: GH-208
           - TestSpread    # @fixme: GH-209
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Check dependencies
         if: github.actor == 'nektos/act'
         run: >

.github/workflows/tests-shell.yml

@@ -32,5 +32,5 @@ jobs:
           - ./test_eval.sh
           - ./test_vars.sh
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: ${{ matrix.test }}

.shellcheckrc

@@ -35,3 +35,4 @@ disable=SC2221
 disable=SC2222
 disable=SC2230
 disable=SC2251
+disable=SC2294

Dockerfile

@@ -52,7 +52,7 @@ VOLUME $BT_DEST
 FROM ea-tester-base AS ea-tester-with-mt4
 
 # Install platform.
-ARG MT_VER=4.0.0.1359
+ARG MT_VER=4.0.0.1382
 RUN eval.sh install_mt $MT_VER && \
     run_backtest.sh -s PrintPaths -v
 
@@ -65,7 +65,7 @@ RUN eval.sh clean_bt && \
 FROM ea-tester-base AS ea-tester-with-mt5
 
 # Install platform.
-ARG MT_VER=5.0.0.3379
+ARG MT_VER=5.0.0.3802
 ENV MT_VER $MT_VER
 RUN eval.sh install_mt $MT_VER
 #RUN run_backtest.sh -s PrintPaths -v

docker-compose.yml

@@ -14,7 +14,7 @@ services:
     volumes:
       - ./scripts:/opt/scripts
   TestSyntaxShellcheck:
-    command: -e SC1004,SC1083,SC1090,SC1091,SC1117,SC2004,SC2005,SC2015,SC2027,SC2032,SC2033,SC2034,SC2035,SC2046,SC2068,SC2070,SC2071,SC2086,SC2091,SC2103,SC2116,SC2124,SC2128,SC2145,SC2154,SC2155,SC2162,SC2164,SC2166,SC2181,SC2188,SC2206,SC2221,SC2222,SC2230 /opt/scripts/eval.sh /opt/scripts/run_backtest.sh /opt/scripts/provision.sh /opt/scripts/.vars.inc.sh /opt/scripts/install_mt5.sh /opt/scripts/.aliases.inc.sh /opt/scripts/bt_data_get.sh /opt/scripts/get_gh_asset.sh /opt/scripts/push_repo.sh /opt/scripts/run_terminal.sh /opt/scripts/.funcs.inc.sh /opt/scripts/dl_ea.sh /opt/scripts/clone_repo.sh /opt/scripts/.funcs.cmds.inc.sh /opt/scripts/install_mt4.sh /opt/scripts/install_mt4x.sh
+    command: -e SC1004,SC1083,SC1090,SC1091,SC1117,SC2004,SC2005,SC2015,SC2027,SC2032,SC2033,SC2034,SC2035,SC2046,SC2068,SC2070,SC2071,SC2086,SC2091,SC2103,SC2116,SC2124,SC2128,SC2145,SC2154,SC2155,SC2162,SC2164,SC2166,SC2181,SC2188,SC2206,SC2221,SC2222,SC2230,SC2294 /opt/scripts/eval.sh /opt/scripts/run_backtest.sh /opt/scripts/provision.sh /opt/scripts/.vars.inc.sh /opt/scripts/install_mt5.sh /opt/scripts/.aliases.inc.sh /opt/scripts/bt_data_get.sh /opt/scripts/get_gh_asset.sh /opt/scripts/push_repo.sh /opt/scripts/run_terminal.sh /opt/scripts/.funcs.inc.sh /opt/scripts/dl_ea.sh /opt/scripts/clone_repo.sh /opt/scripts/.funcs.cmds.inc.sh /opt/scripts/install_mt4.sh /opt/scripts/install_mt4x.sh
     image: koalaman/shellcheck
     volumes:
       - ./scripts:/opt/scripts

scripts/.funcs.cmds.inc.sh

@@ -236,15 +236,21 @@ install_mt()
         exit 1
       }
       cd "$dir_dest"
-      mt_releases_json=$(curl -s https://api.github.com/repos/${REPO_MT-"EA31337/MT-Platforms"}/releases)
+      header=$([ -n "${GITHUB_API_TOKEN}" ] && echo "Authorization: Bearer ${GITHUB_API_TOKEN}" || echo "")
+      mt_releases_json="$(curl -H "Content-Type: application/json; $header" -s https://api.github.com/repos/${REPO_MT-"EA31337/MT-Platforms"}/releases)"
+      jq ".[]" <<< "$mt_releases_json" > /dev/null || true # Test JSON syntax.
       mapfile -t mt_releases_list < <(jq -r '.[]["tag_name"]' <<< "$mt_releases_json")
       if [[ " ${mt_releases_list[*]} " =~ ${mt_ver} ]]; then
         mt_release_url=$(jq -r '.[]|select(.tag_name == "'${mt_ver}'")["assets"][0]["browser_download_url"]' <<< "$mt_releases_json")
         wget -nv -c "$mt_release_url"
         (unzip -ou "mt-$mt_ver.zip" && rm $VFLAG "mt-$mt_ver.zip") 1>&2
         clean_bt . '*'
       else
-        echo "Error: Not supported platform version. Supported: ${mt_releases_list[@]}" >&2
+        echo "Error: Cannot find supported platform version. Supported: ${mt_releases_list[@]}" >&2
+        if [ -z "${mt_releases_list[*]}" ]; then
+          echo "Error: Empty release list!"
+          echo $mt_releases_json
+        fi
       fi
       cd - &> /dev/null
       ;;
@@ -551,10 +557,10 @@ ea_find()
       echo "$file"
       return
     }
-  result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.mq${mt_ver}" ')' -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -name "${file%.*}.ex${mt_ver}" ')' -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.mq${mt_ver}" -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.ex${mt_ver}" -print -quit)
+  result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f '(' -iname "$file" -o -iname "${file%.*}.mq${mt_ver}" ')' -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f '(' -iname "$file" -o -name "${file%.*}.ex${mt_ver}" ')' -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f -iname "*${file%.*}*.mq${mt_ver}" -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f -iname "*${file%.*}*.ex${mt_ver}" -print -quit)
   echo ${result#./}
   cd - &> /dev/null
 }
@@ -580,10 +586,10 @@ script_find()
       echo "$file"
       return
     }
-  result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.mq${mt_ver}" ')' -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -name "${file%.*}.ex${mt_ver}" ')' -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.mq${mt_ver}" -print -quit)
-  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.ex${mt_ver}" -print -quit)
+  result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f '(' -iname "$file" -o -iname "${file%.*}.mq${mt_ver}" ')' -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f '(' -iname "$file" -o -name "${file%.*}.ex${mt_ver}" ')' -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f -iname "*${file%.*}*.mq${mt_ver}" -print -quit)
+  [ -z "$result" ] && result=$(find -L . "$WORKDIR" "$ROOT" ~ -maxdepth 5 -type f -iname "*${file%.*}*.ex${mt_ver}" -print -quit)
   echo ${result#./}
   cd - &> /dev/null
 }

scripts/options.txt

@@ -127,6 +127,8 @@ Other supported variables (without options assigned):
   Values: 0 (default) - Every tick, 1 - Control points, 2 - Open prices only
 - EA_WHITELIST_URLS (string)
   Adds URLs to whitelist. Separate by a semicolon. Required for WebRequest().
+- GITHUB_API_TOKEN (string)
+  Personal access token (PAT) used for GitHub API.
 - OPT_NOERR (bool)
   Disables errexit flag to prevent exiting script on failure.
 - RUN_ON_START (string)

tests/requirements.pip.txt

@@ -0,0 +1 @@
+docker-compose