Merge branch 'master' into feat/rust-wasm

.github/workflows/ci.yml

@@ -203,4 +203,4 @@ jobs:
       deployment_repo: ${{ inputs.deployment_repo }}
       images: ${{ inputs.deployment_images }}
     secrets:
-      token: ${{ secrets.deployment_token }}
+      token: ${{ secrets.deployment_token }}

.github/workflows/deploy.yml

@@ -50,4 +50,4 @@ jobs:
             author_name: catalyst-cibot
             author_email: ci@projectcatalyst.io
             message: "chore: updates ${{ github.event.repository.name }} deployments"
-            push: true
+            push: true

.github/workflows/dogfood.yml

@@ -15,7 +15,7 @@ jobs:
       aws_ecr_registry: 332405224602.dkr.ecr.eu-central-1.amazonaws.com
       aws_role_arn: arn:aws:iam::332405224602:role/ci
       aws_region: eu-central-1
-      ci_cli_version: 2.2.1
+      ci_cli_version: 2.3.0
     secrets:
       dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
       dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}

.github/workflows/publish.yml

@@ -172,4 +172,4 @@ jobs:
             ${{ inputs.aws_ecr_registry }}
             ghcr.io/${{ github.repository }}
           tags: |
-            ${{ steps.fetch_tag.outputs.tag }}
+            ${{ steps.fetch_tag.outputs.tag }}

.github/workflows/release.yml

@@ -170,4 +170,4 @@ jobs:
       - name: Create release
         uses: softprops/action-gh-release@v1
         with:
-          files: ${{ steps.collect.outputs.artifacts }}
+          files: ${{ steps.collect.outputs.artifacts }}

Earthfile

@@ -1,5 +1,5 @@
 # Set the Earthly version to 0.7
-VERSION --global-cache 0.7
+VERSION --global-cache --use-function-keyword 0.7
 
 # cspell: words livedocs sitedocs
 
@@ -48,4 +48,4 @@ repo-config:
 edit-docs:
     LOCALLY
 
-    RUN ./earthly/docs/dev/local.py cat-ci-docs:latest
+    RUN ./earthly/docs/dev/local.py cat-ci-docs:latest

cli/go.mod

@@ -6,11 +6,10 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/alecthomas/kong v0.7.1
 	github.com/aws/aws-sdk-go v1.44.274
-	github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b
+	github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb
 	github.com/onsi/ginkgo/v2 v2.9.7
 	github.com/onsi/gomega v1.27.7
 	github.com/spf13/afero v1.9.5
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -24,8 +23,9 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

cli/go.sum

@@ -61,8 +61,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b h1:kwdLqircFHqYNLrBuSOybPpGPnqaqXEcRvJOWiSqRZw=
-github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b/go.mod h1:OoxU/3mRER8gdd4l1lxF6EgK+0vf7zQ8m9g269y6hWI=
+github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb h1:dP0Wcrfbq5whzPCT/ZHGTvWVvx7S6BoF32nRHqhKt7E=
+github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb/go.mod h1:UX9246125xgKvMIKim7WTxR8nookovxWx8/Ze8kpYlQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -164,7 +164,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/poy/onpar v0.3.1 h1:iqwl2S3U08PY8Bm9Hfh9v3fxdig7hZy9bsZfTephng8=
+github.com/poy/onpar v0.3.2 h1:yo8ZRqU3C4RlvkXPWUWfonQiTodAgpKQZ1g8VTNU9xU=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
@@ -325,8 +325,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

cli/pkg/earthfile.go

@@ -14,7 +14,7 @@ type Earthfile struct {
 	Path           string               `json:"path,omitempty"`
 	SourceLocation *spec.SourceLocation `json:"sourceLocation,omitempty"`
 	Targets        []spec.Target        `json:"targets,omitempty"`
-	UserCommands   []spec.UserCommand   `json:"userCommands,omitempty"`
+	Functions      []spec.Function      `json:"functions,omitempty"`
 	Version        *spec.Version        `json:"version,omitempty"`
 }
 

cli/pkg/parsers/earthly_parser.go

@@ -24,7 +24,7 @@ func (e EarthlyParser) Parse(path string) (pkg.Earthfile, error) {
 		Path:           path,
 		SourceLocation: ef.SourceLocation,
 		Targets:        ef.Targets,
-		UserCommands:   ef.UserCommands,
+		Functions:      ef.Functions,
 		Version:        ef.Version,
 	}, nil
 }

docs/src/guides/docs.md

@@ -52,7 +52,7 @@ src:
 
 The first step of building process it preparing a source files.
 It is mandatory to have a `src` directory with all documentation md files in it and `mkdocs.yml` file.
-This directory and file will be picked during the execution of `+SRC` UDC target.
+This directory and file will be picked during the execution of `+SRC` Function.
 Also it is possible to replace defined `includes`, `macros` and `overrides` dirs
 to customize some docs appearance and configuration.
 
@@ -78,7 +78,7 @@ docs:
     DO ../earthly/docs+BUILD
 ```
 
-To build a docs artifact which will be used later just invoke `+BUILD` UDC target
+To build a docs artifact which will be used later just invoke `+BUILD` Function
 on the already prepared docs environment `+src` target which we have discussed before.
 
 ```Earthfile
@@ -93,7 +93,7 @@ local:
 ```
 
 To finally build a docker image which is pretty strait forward process,
-you should firstly invoke `+PACKAGE` UDC target which will prepare an environment for future docs image,
+you should firstly invoke `+PACKAGE` Function which will prepare an environment for future docs image,
 next step is to copy builded artifact from the previous step to `/usr/share/nginx/html` folder.
 And the last step is to save a docker image with the specified name, tag and registry if it is needed.
 

docs/src/guides/languages/go.md

@@ -56,7 +56,7 @@ deps:
     # Any build dependencies should also be captured in this target.
     RUN apk add --no-cache gcc musl-dev
 
-    # This UDC automatically copies the go.mod and go.sum files and runs
+    # This Function automatically copies the go.mod and go.sum files and runs
     # `go mod download` to install the dependencies.
     DO ../../earthly/go+DEPS --ginkgo="false"
 ```
@@ -75,11 +75,11 @@ This target is also going to build responsible for installing external build dep
 These are dependencies that are not specific to a language and usually get installed system-wide.
 In our case, since we're building a static binary, we need `gcc` and `musl`.
 
-Finally, the actual logic we will be using is encapsulate in a UDC.
+Finally, the actual logic we will be using is encapsulate in a Function.
 This is a very common pattern, as an `Earthfile` can get repetitive across a repository.
-In our case, we use the `go+DEPS` UDC that will automatically copy our `go.mod` and `go.sum` files and then execute
+In our case, we use the `go+DEPS` Function that will automatically copy our `go.mod` and `go.sum` files and then execute
 `go mod download`.
-The UDC will also establish a cache for the Go tooling.
+The Function will also establish a cache for the Go tooling.
 This means that, even if our source code changes, we'll see a substantial speed boost when compiling because the cache is preserved
 across Earthly runs.
 
@@ -96,10 +96,10 @@ check:
     # This target checks the overall health of the source code.
     FROM +src
 
-    # This UDC validates the code is formatted according to Go standards.
+    # This Function validates the code is formatted according to Go standards.
     DO ../../earthly/go+FMT --src="go.mod go.sum cmd"
 
-    # This UDC runs golangci-lint to check for common errors.
+    # This Function runs golangci-lint to check for common errors.
     DO ../../earthly/go+LINT --src="go.mod go.sum cmd"
 ```
 
@@ -111,8 +111,8 @@ Any future targets which need access to the source code will inherit from this t
 
 Now that the source code is available, we can begin performing static checks.
 These checks are intended to verify the code is healthy and conforms to a certain standard.
-As we did in the previous section, here we rely on UDCs again to perform these checks.
-These two UDCs will validate the code formatting is correct and also perform a series of lints to validate code quality.
+As we did in the previous section, here we rely on Functions again to perform these checks.
+These two Functions will validate the code formatting is correct and also perform a series of lints to validate code quality.
 
 Note that these checks are fast (compared to later steps) and perform quick feedback on code quality.
 Since this is the first target run in CI, we want to fail the CI as quickly as possible if we can easily find code quality issues.

docs/src/guides/languages/postgresql.md

@@ -62,7 +62,7 @@ builder:
 
 The first target we are going to consider will be responsible to prepare a PostgreSQL environment (Earthly `+postgres-base` target),
 migrations, migrations configuration and seed data (`COPY --dir ./migrations ./data ./refinery.toml .`),
-doing some final build step (Earthly `+BUILDER` UDC target).
+doing some final build step (Earthly `+BUILDER` Function).
 
 In the next steps we are going to inheriting from this `+builder` target which contains all necessary data,
 dependencies, environment to properly run PostgreSQL database.
@@ -92,8 +92,8 @@ These checks are intended to verify the code is healthy and well formatted to a
 and done with the help of the `sqlfluff` tool which is already configured during the `+postgres-base` target.
 
 To apply and fix some formatting issues you can run `+format` target which will picks up directory
-where your Earthly file lies in as a source dir for formatting and run `+FORMAT` UDC target.
-Under the hood `+FORMAT` UDC target runs `sqlfluff-image` docker image,
+where your Earthly file lies in as a source dir for formatting and run `+FORMAT` Function.
+Under the hood `+FORMAT` Function runs `sqlfluff-image` docker image,
 which contains the same configuration and setup which is applied during the `+check`.
 
 <!-- markdownlint-disable max-one-sentence-per-line -->
@@ -118,7 +118,7 @@ With the `*.sql` files validation out of the way, we can finally build our Postg
 Since we need migration and seed data files,
 we'll inherit from the `builder` target.
 The actual image build process is pretty straight-forward
-and fully defined under the `+BUILD` UDC target.
+and fully defined under the `+BUILD` Function.
 The only thing it is needed to specify is a few arguments:
 
 * `tag` - the tag of the image, default value `latest`.

docs/src/guides/languages/python.md

@@ -62,7 +62,7 @@ so it is mandatory to have `pyproject.toml` and `poetry.lock` files in the root
 The fist step of the `builder` target is prepare a Python environment
 with poetry via `+python-base` target.
 Next step is to copy source code of the project and finally finalize the build
-with some poetry project setup which is done with `+BUILDER` UDC target.
+with some poetry project setup which is done with `+BUILDER` Function.
 
 ### Running checks
 

docs/src/guides/languages/rust.md

@@ -69,8 +69,8 @@ The fist step of the `builder` target is to prepare a Rust environment via `+rus
 Next step is to copy source code of the project.
 Note that you need to copy only needed files for Rust build process,
 any other irrelevant stuff should omitted.
-And finally finalize the build with `+SETUP` UDC target.
-The `+SETUP` UDC target requires `rust-toolchain.toml` file,
+And finally finalize the build with `+SETUP` Function.
+The `+SETUP` Function requires `rust-toolchain.toml` file,
 with the specified `channel` option in it.
 This `rust-toolchain.toml` file could be specified
 via the `toolchain` argument of the `+SETUP` target like this

docs/src/guides/markdown.md

@@ -4,14 +4,14 @@ icon: simple/markdown
 
 # Markdown Check
 
-This Earthly Target and UDC enables uniform linting of Markdown files to maintain consistency and quality.
+This Earthly Target and Function enables uniform linting of Markdown files to maintain consistency and quality.
 
-This UDC is **NOT** intended to be used inside container builds.
+This Function is **NOT** intended to be used inside container builds.
 Its sole purpose is to enforce uniform style rules for all markdown files in a repository.
 It makes no assumptions about which files may or may not end up inside a container or are part of a build.
 This is *INTENTIONAL*.
 
-IF this UDC is used inside a container build, it is **NOT** a bug if it does not do the correct thing.
+IF this Function is used inside a container build, it is **NOT** a bug if it does not do the correct thing.
 
 ## Introduction
 
@@ -83,7 +83,7 @@ markdown-check-fix:
     DO github.com/input-output-hk/catalyst-ci/earthly/mdlint:<tag>+MDLINT_LOCALLY --src=$(echo ${PWD}) --fix=--fix
 ```
 
-In this use case, the UDC is run Locally, so that the markdown in the repo can be directly checked.
+In this use case, the Function is run Locally, so that the markdown in the repo can be directly checked.
 
 <!-- markdownlint-disable max-one-sentence-per-line -->
 !!! Note
@@ -105,4 +105,4 @@ For more example, please refer to this [doc](https://github.com/DavidAnson/markd
 mdlint-cli2 is integrated into VSCode and may be integrated into other Editors.
 
 The editor integration should pick up both the `.markdownlint.jsonc` and `.markdownlint-cli2.jsonc` configuration files.
-It will then behave exactly the same as the Earthly UDC.
+It will then behave exactly the same as the Earthly Function.

docs/src/guides/spellcheck.md

@@ -142,4 +142,4 @@ For these files/paths, exclude them from the spell check by adding their filenam
 
 `cspell` is integrated into VSCode and may be integrated into other Editors.
 
-The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly UDC.
+The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly Function.

docs/src/reference/function.md

@@ -0,0 +1,38 @@
+# Functions
+
+## Overview
+
+The Catalyst CI repository provides a number of Earthly  
+[Functions](https://docs.earthly.dev/docs/guides/functions).
+You can think of a Function as a reusable snippet of Earthly code that serves the same purpose as functions in a common programming
+language.
+Functions are helpful for several reasons:
+
+1. They keep Earthfiles DRY
+2. They can encapsulate complex logic into a simple contractual interface
+3. They enforce standardization and prevent solving the same problem in multiple different ways
+
+The third reason is particularly useful for Catalyst as we have multiple repositories with dozens of Earthfiles often solving
+similar problems.
+
+## Usage
+
+You are highly encouraged to review the currently available Functions in the
+[Catalyst CI repository](https://github.com/input-output-hk/catalyst-ci/tree/master/earthly).
+The folder structure is broken out by language/technology and should be relatively easy to navigate.
+You can incorporate these Functions into your Earthfiles by using something like below:
+
+```Earthfile
+DO github.com/input-output-hk/catalyst-ci/earthly/<folder>+<Function_NAME> --arg1=value1
+```
+
+Replacing `folder` and `Function_NAME` respectively.
+The passing of an argument is optional, as some Functions do not require any input arguments.
+
+## Contributing
+
+Please feel encouraged to contribute Functions to the repository.  
+If you're seeing the same logic being re-used across multiple Earthfiles in a  
+repository, this is a good candidate for refactoring into a Function.  
+Additionally, for SMEs who are aware of language best practices, encoding those  
+into a Function will help increase the overall health of the CI process.