feat: apply changes across all formulas to work with GitLab CI

Split this into multiple commits.
myii · Dec 16, 2020 · f5c9af4 · f5c9af4
1 parent ee92ec3
commit f5c9af4
Show file tree

Hide file tree

Showing 12 changed files with 443 additions and 204 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -5,27 +5,24 @@
 # Define all YAML node anchors
 ###############################################################################
 .node_anchors:
+  # `only` (also used for `except` where applicable)
+  only_branch_master_parent_repo: &only_branch_master_parent_repo
+    - 'master@myii/ssf-formula'
   # `stage`
   stage_lint: &stage_lint 'lint'
   stage_release: &stage_release 'release'
-  stage_test: &stage_test 'test'
   # `image`
   image_commitlint: &image_commitlint 'myii/ssf-commitlint:11'
-  image_dindruby: &image_dindruby 'myii/ssf-dind-ruby:2.7.1-r3'
   image_precommit: &image_precommit
     name: 'myii/ssf-pre-commit:2.9.2'
     entrypoint: ['/bin/bash', '-c']
   image_semantic-release: &image_semanticrelease 'myii/ssf-semantic-release:15.14'
-  # `only` (also used for `except` where applicable)
-  only_branch_master_parent_repo: &only_branch_master_parent_repo
-    - 'master@myii/ssf-formula'
 
 ###############################################################################
 # Define stages and global variables
 ###############################################################################
 stages:
   - *stage_lint
-  - *stage_test
   - *stage_release
 variables:
   DOCKER_DRIVER: 'overlay2'
@@ -40,9 +37,21 @@ commitlint:
     # Add `upstream` remote to get access to `upstream/master`
     - 'git remote add upstream ${CI_PROJECT_URL}.git'
     - 'git fetch --all'
+    # Set default commit hashes for `--from` and `--to`
+    - 'export COMMITLINT_FROM="$(git merge-base upstream/master HEAD)"'
+    - 'export COMMITLINT_TO="${CI_COMMIT_SHA}"'
+    # `coqbot` adds a merge commit to test PRs on top of the latest commit in
+    # the repo; amend this merge commit message to avoid failure
+    - |
+      if [ "${GITLAB_USER_LOGIN}" = "coqbot" ] \
+      && [ "${CI_COMMIT_BRANCH}" != "master" ]; then
+        git commit --amend -m \
+          'chore: reword coqbot merge commit message for commitlint'
+        export COMMITLINT_TO=HEAD
+      fi
     # Run `commitlint`
-    - 'commitlint --from "$(git merge-base upstream/master HEAD)"
-                  --to   "${CI_COMMIT_SHA}"
+    - 'commitlint --from "${COMMITLINT_FROM}"
+                  --to   "${COMMITLINT_TO}"
                   --verbose'
 
 pre-commit:

diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst
@@ -172,4 +172,3 @@ An example of that:
    BREAKING CHANGE: With the removal of all of the `.sls` files under
    `template package`, this formula no longer supports the installation of
    packages.
-
diff --git a/ssf/config/formulas.sls b/ssf/config/formulas.sls
@@ -14,7 +14,6 @@
 {%-   set context = semrel_formula_specs.context %}
 {%-   set inspec_suites_kitchen = context.inspec_suites_kitchen %}
 {%-   set use_cirrus_ci = context.use_cirrus_ci %}
-{%-   set use_gitlab_ci = context.use_gitlab_ci %}
 {%-   set use_libsaltcli = context.use_libsaltcli %}
 {%-   set use_tofs = context.use_tofs %}
 {%-   set owner = context.git.github.owner %}
@@ -102,12 +101,12 @@ prepare-git-branch-for-{{ formula }}:
 {#-           Or if the file is `libsaltcli.jinja` and `use_libsaltcli` is `False` #}
 {#-           Likewise, if running the state for TOFS files when `use_tofs` is `False` #}
 {#-           Also remove the local `CONTRIBUTING` file to use the org-level file instead #}
-{#-           Also always remove `.travis.yml` since move to GitLab CI (or alternative) #}
+{#-           Furthermore, remove `.travis.yml` for the `ssf-formula` #}
 {%-           if (semrel_file == '.cirrus.yml' and not use_cirrus_ci) or
                  (semrel_file == 'formula/libsaltcli.jinja' and not use_libsaltcli) or
                  (semrel_file in ['docs/TOFS_pattern.rst', 'formula/libtofs.jinja'] and not use_tofs) or
                  (semrel_file in ['docs/CONTRIBUTING.rst'] and formula not in ['.github', 'ssf-formula']) or
-                 (semrel_file == '.travis.yml' and use_gitlab_ci)
+                 (semrel_file in ['.travis.yml'] and formula in ['ssf-formula'])
 %}
 {%-             set add_or_rm = ['rm', 'remove', 'absent'] %}
 {%-           endif %}

diff --git a/ssf/defaults.yaml b/ssf/defaults.yaml
@@ -22,8 +22,8 @@ ssf_node_anchors:
               - '/docs/AUTHORS.rst': '@saltstack-formulas/ssf'
               - '/docs/CHANGELOG.rst': '@saltstack-formulas/ssf'
               - '/docs/TOFS_pattern.rst': '@saltstack-formulas/ssf'
-              - '/./libsaltcli.jinja': '@saltstack-formulas/ssf'
-              - '/./libtofs.jinja': '@saltstack-formulas/ssf'
+              - '/*/libsaltcli.jinja': '@saltstack-formulas/ssf'
+              - '/*/libtofs.jinja': '@saltstack-formulas/ssf'
               - '/test/integration/**/inspec.yml': '@saltstack-formulas/ssf'
               - '/test/integration/**/README.md': '@saltstack-formulas/ssf'
               - '/.gitignore': '@saltstack-formulas/ssf'
@@ -59,7 +59,7 @@ ssf_node_anchors:
             #       An alternative method could be to use:
             #       `git describe --abbrev=0 --tags`
             # yamllint disable rule:line-length rule:quoted-strings
-            title: "feat(gitlab-ci): use GitLab CI as Travis CI replacement"
+            title: "ci(gitlab-ci): use GitLab CI as Travis CI replacement"
             body: '* Automated using https://github.com/myii/ssf-formula/pull/275'
             # yamllint enable rule:line-length rule:quoted-strings
           github:
@@ -191,7 +191,7 @@ ssf_node_anchors:
           - [amazonlinux  ,    2   ,   2019.2,      3,         default]
           - [fedora       ,   30   ,   2018.3,      3,         default]
           - [arch-base    ,  latest,   2018.3,      2,         default]
-          - [centos       ,    6   ,   2017.7,      2,         default]
+          # # - [centos       ,    6   ,   2017.7,      2,         default]
         # To deal with excessive instances when mimicking `kitchen list -b`
         # If values are set, only use these as commented entries in the matrix
         platforms_matrix_commented_includes: []
@@ -221,7 +221,6 @@ ssf_node_anchors:
           dist: 'bionic'
           use_single_job_for_linters: true
         use_cirrus_ci: false
-        use_gitlab_ci: false
         use_libsaltcli: false
         use_tofs: false
         yamllint:
@@ -508,7 +507,21 @@ ssf:
             name: 'ubuntu'
     dhcpd: *formula_default
     django: *formula_default
-    docker: *formula_default
+    docker:
+      <<: *formula_default
+      context:
+        <<: *context_default
+        inspec_suites_kitchen:
+          <<: *isk_default
+          0:
+            <<: *isk_suite_default
+            name: 'archive'
+          1:
+            <<: *isk_suite_default
+            name: 'package'
+          2:
+            <<: *isk_suite_default
+            name: 'clean'
     eclipse:
       <<: *formula_default
       context:
@@ -931,12 +944,6 @@ ssf:
           2:
             <<: *isk_suite_default
             name: 'v3000-py2'
-          3:
-            <<: *isk_suite_default
-            name: 'v201902-py3'
-          4:
-            <<: *isk_suite_default
-            name: 'v201902-py2'
     sqldeveloper: *formula_default
     sqlplus: *formula_default
     ssf: *formula_default

diff --git a/ssf/files/default/.gitlab-ci.yml b/ssf/files/default/.gitlab-ci.yml
@@ -3,7 +3,6 @@
 ---
 {%- from tplroot ~ "/libcimatrix.jinja" import format_ci_matrix with context %}
 
-
 {#-   Prepare variable used for `saltcheck` #}
 {%-   set use_saltcheck = False %}
 {%-   if semrel_formula in ['cron'] %}
@@ -13,27 +12,52 @@
 # Define all YAML node anchors
 ###############################################################################
 .node_anchors:
+  # `only` (also used for `except` where applicable)
+  only_branch_master_parent_repo: &only_branch_master_parent_repo
+    - 'master@{{ gitlab.owner }}/{{ gitlab.repo or formula }}'
   # `stage`
   stage_lint: &stage_lint 'lint'
   stage_release: &stage_release 'release'
+  {%- if semrel_formula != 'ssf' %}
   stage_test: &stage_test 'test'
+  {%- endif  %}
   # `image`
   image_commitlint: &image_commitlint 'myii/ssf-commitlint:11'
+  {%- if semrel_formula != 'ssf' %}
   image_dindruby: &image_dindruby 'myii/ssf-dind-ruby:2.7.1-r3'
+  {%-   if semrel_formula == 'template' %}
+  image_dindrubybionic: &image_dindrubybionic 'myii/ssf-dind-ruby-bionic:1_2.5.1'
+  {%-   endif  %}
+  {%- endif  %}
   image_precommit: &image_precommit
     name: 'myii/ssf-pre-commit:2.9.2'
     entrypoint: ['/bin/bash', '-c']
   image_semantic-release: &image_semanticrelease 'myii/ssf-semantic-release:15.14'
-  # `only` (also used for `except` where applicable)
-  only_branch_master_parent_repo: &only_branch_master_parent_repo
-    - 'master@{{ gitlab.owner }}/{{ gitlab.repo or formula }}'
+  {%- if semrel_formula != 'ssf' %}
+  # `services`
+  services_docker_dind: &services_docker_dind
+    - 'docker:dind'
+  # `variables`
+  # https://forum.gitlab.com/t/gitlab-com-ci-caching-rubygems/5627/3
+  # https://bundler.io/v1.16/bundle_config.html
+  variables_bundler: &variables_bundler
+    BUNDLE_CACHE_PATH: '${CI_PROJECT_DIR}/.cache/bundler'
+    BUNDLE_WITHOUT: 'production'
+  # `cache`
+  cache_bundler: &cache_bundler
+    key: '${CI_JOB_STAGE}'
+    paths:
+      - '${BUNDLE_CACHE_PATH}'
+  {%- endif  %}
 
 ###############################################################################
 # Define stages and global variables
 ###############################################################################
 stages:
   - *stage_lint
+  {%- if semrel_formula != 'ssf' %}
   - *stage_test
+  {%- endif  %}
   - *stage_release
 variables:
   DOCKER_DRIVER: 'overlay2'
@@ -56,7 +80,7 @@ commitlint:
     - |
       if [ "${GITLAB_USER_LOGIN}" = "coqbot" ] \
       && [ "${CI_COMMIT_BRANCH}" != "master" ]; then
-        git -c user.name='Null' -c user.email='null@example.com' commit --amend -m \
+        git commit --amend -m \
           'chore: reword coqbot merge commit message for commitlint'
         export COMMITLINT_TO=HEAD
       fi
@@ -86,18 +110,13 @@ pre-commit:
 .test_instance:
   stage: *stage_test
   image: *image_dindruby
-  services:
-    - 'docker:dind'
-  # https://forum.gitlab.com/t/gitlab-com-ci-caching-rubygems/5627/3
-  # https://bundler.io/v1.16/bundle_config.html
-  variables:
-    BUNDLE_CACHE_PATH: '${CI_PROJECT_DIR}/.cache/bundler'
-    BUNDLE_WITHOUT: 'production'
-  cache:
-    key: '${CI_JOB_STAGE}'
-    paths:
-      - '${BUNDLE_CACHE_PATH}'
+  services: *services_docker_dind
+  variables: *variables_bundler
+  cache: *cache_bundler
   before_script:
+    {%- if semrel_formula == 'keepalived' %}
+    - 'apk --no-cache add ipvsadm'
+    {%- endif %}
     # TODO: This should work from the env vars above automatically
     - 'bundle config set path "${BUNDLE_CACHE_PATH}"'
     - 'bundle config set without "${BUNDLE_WITHOUT}"'
@@ -106,13 +125,46 @@ pre-commit:
     # Alternative value to consider: `${CI_JOB_NAME}`
     - '{{ script_kitchen.bin }} {{ script_kitchen.cmd }} "${DOCKER_ENV_CI_JOB_NAME}"'
 
+{%-   if semrel_formula == 'template' %}
+# <REMOVEME
+
+###############################################################################
+# Define `test_conversion` template
+###############################################################################
+.test_conversion:
+  stage: *stage_test
+  image: *image_dindrubybionic
+  services: *services_docker_dind
+  variables: *variables_bundler
+  cache: *cache_bundler
+  before_script:
+    - 'export CONVERTED=test-the-use_this_template-button'
+    - 'git clone . tmp/"${CONVERTED}"-formula'
+    - 'cd tmp/"${CONVERTED}"-formula'
+    # Install `pre-commit` hooks
+    - 'bin/install-hooks'
+    # Run the conversion script with debug output
+    - 'DEBUG=true bin/convert-formula.sh "${CONVERTED}"'
+    - '[ $(git rev-list HEAD --count) -eq 2 ]'
+    # Quick visual check that correct files have been updated
+    - 'git show --pretty="" --name-status'
+    # TODO: This should work from the env vars above automatically
+    - 'bundle config set path "${BUNDLE_CACHE_PATH}"'
+    - 'bundle config set without "${BUNDLE_WITHOUT}"'
+    - 'bundle install'
+  script:
+    - '{{ script_kitchen.bin }} {{ script_kitchen.cmd }} default-debian-10-tiamat-py3'
+# REMOVEME>
+{%-   endif %}
+
 ###############################################################################
 # `test` stage: each instance below uses the `test` template above
 ###############################################################################
 ## Define the rest of the matrix based on Kitchen testing
 # Make sure the instances listed below match up with
 # the `platforms` defined in `kitchen.yml`
 {%-   if semrel_formula == 'template' %}
+# <REMOVEME
 # NOTE: Please try to select up to six instances that add some meaningful
 #       testing of the formula's behaviour. If possible, try to refrain from
 #       the classical "chosing all the instances because I want to test on
@@ -124,6 +176,8 @@ pre-commit:
 #       to allow for comprehensive local testing
 #       Ref: https://github.com/saltstack-formulas/template-formula/issues/118
 #       Ref: https://github.com/saltstack-formulas/template-formula/issues/121
+test-formula-conversion: {extends: '.test_conversion'}
+# REMOVEME>
 {%-   endif %}
 {{- format_ci_matrix(platforms, inspec_suites_kitchen, inspec_suites_matrix, platforms_matrix, platforms_matrix_commented_includes, semrel_formula, width=0, use_gitlab_format=True) }}
 {%- endif %}

diff --git a/ssf/files/default/.travis.yml b/ssf/files/default/.travis.yml
@@ -1,6 +1,15 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
+################################################################################
+# NOTE: This file is UNMAINTAINED; it is provided for references purposes only.
+#       No guarantees are tendered that this structure will work after 2020.
+################################################################################
+# * https://en.wikipedia.org/wiki/Travis_CI:
+#   - "... free open-source plans were removed in [sic] the end of 2020"
+#   - https://blog.travis-ci.com/2020-11-02-travis-ci-new-billing
+#   - https://ropensci.org/technotes/2020/11/19/moving-away-travis/
+################################################################################
 {%- from tplroot ~ "/libcimatrix.jinja" import format_ci_matrix with context %}
 
 {%- macro format_allow_failures(use_single_job_for_linters) %}
@@ -104,8 +113,10 @@ script:
 ## Stages and jobs matrix
 stages:
   - test
-  - name: 'release'
-    if: 'branch = master AND type != pull_request'
+  # # As part of the switch away from Travis CI, ensure that the `release` stage
+  # # is not run inadvertently
+  # - name: 'release'
+  #   if: 'branch = master AND type != pull_request'
 jobs:
   {{- format_allow_failures(travis.use_single_job_for_linters) }}
   include:
@@ -200,9 +211,13 @@ jobs:
     - env: 'Conversion'
       name: 'Test: bin/convert-formula.sh'
       script:
-        - git clone . tmp/converted-formula
-        - cd tmp/converted-formula
-        - DEBUG=true bin/convert-formula.sh converted
+        - export CONVERTED=test-the-use_this_template-button
+        - git clone . tmp/"$CONVERTED"-formula
+        - cd tmp/"$CONVERTED"-formula
+        - pyenv global 3.8
+        - pip install pre-commit==2.7.1
+        - bin/install-hooks
+        - DEBUG=true bin/convert-formula.sh "$CONVERTED"
         - '[ $(git rev-list HEAD --count) -eq 2 ]'
         # Quick visual check that correct files have been updated
         - git show --pretty="" --name-status

diff --git a/ssf/files/default/inspec/inspec.yml b/ssf/files/default/inspec/inspec.yml
@@ -7,7 +7,8 @@ maintainer: SaltStack Formulas
 license: Apache-2.0
 {%- set summary_heading = 'summary: ' %}
 {%- set summary_chars = summary_heading | length + suite.inspec_yml.summary | length %}
-{%- if summary_chars > yamllint.rules.get('line-length').max %}
+{%- if (summary_chars > yamllint.rules.get('line-length').max) or
+       ([semrel_formula, suite.name] == ['TEMPLATE', 'default'])  %}
 # yamllint disable-line rule:line-length
 {%- endif %}
 {{ summary_heading }}{{ suite.inspec_yml.summary }}

diff --git a/ssf/files/tofs_arvados-formula/.salt-lint b/ssf/files/tofs_arvados-formula/.salt-lint
@@ -7,6 +7,7 @@ rules:
     ignore: |
       arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja
       test/salt/pillar/examples/nginx_webshell_configuration.sls
+      test/salt/pillar/examples/nginx_passenger.sls
 skip_list:
   # Using `salt-lint` for linting other files as well, such as Jinja macros/templates
   - 205  # Use ".sls" as a Salt State file extension
Original file line number	Diff line number	Diff line change
Expand Up		@@ -172,4 +172,3 @@ An example of that:
		BREAKING CHANGE: With the removal of all of the `.sls` files under
		`template package`, this formula no longer supports the installation of
		packages.