diff --git a/docker_auth_test.go b/docker_auth_test.go index 33bdcf40f8..2ce7c223b9 100644 --- a/docker_auth_test.go +++ b/docker_auth_test.go @@ -272,6 +272,7 @@ func prepareLocalRegistryWithAuth(t *testing.T) { ctx := context.Background() wd, err := os.Getwd() assert.NoError(t, err) + // bindMounts { req := ContainerRequest{ Image: "registry:2", ExposedPorts: []string{"5000:5000/tcp"}, @@ -297,6 +298,7 @@ func prepareLocalRegistryWithAuth(t *testing.T) { }, WaitingFor: wait.ForExposedPort(), } + // } genContainerReq := GenericContainerRequest{ ProviderType: providerType, diff --git a/docs/features/files_and_mounts.md b/docs/features/files_and_mounts.md new file mode 100644 index 0000000000..12adae3081 --- /dev/null +++ b/docs/features/files_and_mounts.md @@ -0,0 +1,20 @@ +# Files and volumes + +## File mapping + +It is possible to map a file or directory from your FileSystem into the container as a volume using the `Mounts` attribute at the container request struct: + + +[Bind mounts](../../docker_auth_test.go) inside_block:bindMounts + + +## Volume mapping + +It is also possible to map a volume from your Docker host into the container using the `Mounts` attribute at the container request struct: + + +[Volume mounts](../../mounts_test.go) inside_block:volumeMounts + + +!!!tip + This ability of creating volumes is also available for remote Docker hosts. diff --git a/docs/system_requirements/ci/aws_codebuild.md b/docs/system_requirements/ci/aws_codebuild.md new file mode 100644 index 0000000000..d453debf72 --- /dev/null +++ b/docs/system_requirements/ci/aws_codebuild.md @@ -0,0 +1,18 @@ +# AWS CodeBuild + +To enable access to Docker in AWS CodeBuild, go to `Privileged` section and check +`Enable this flag if you want to build Docker images or want your builds to get elevated privileges`. + +This is a sample `buildspec.yml` config: + +```yaml +version: 0.2 + +phases: + install: + runtime-versions: + golang: 1.20 + build: + commands: + - go test ./... +``` diff --git a/docs/system_requirements/ci/bitbucket_pipelines.md b/docs/system_requirements/ci/bitbucket_pipelines.md new file mode 100644 index 0000000000..cbbc705ee3 --- /dev/null +++ b/docs/system_requirements/ci/bitbucket_pipelines.md @@ -0,0 +1,26 @@ +# Bitbucket Pipelines + +To enable access to Docker in Bitbucket Pipelines, you need to add `docker` as a service on the step. + +Furthermore, Ryuk needs to be turned off since Bitbucket Pipelines does not allow starting privileged containers (see [Disabling Ryuk](../../features/configuration.md#disabling-ryuk)). This can either be done by setting a repository variable in Bitbucket's project settings or by explicitly exporting the variable on a step. + +In some cases the memory available to Docker needs to be increased. + +Here is a sample Bitbucket Pipeline configuration that does a checkout of a project and runs Go tests: + +```yml +image: golang:1.19 + +pipelines: + default: + - step: + script: + - export TESTCONTAINERS_RYUK_DISABLED=true + - go test ./... + services: + - docker +definitions: + services: + docker: + memory: 2048 +``` diff --git a/docs/system_requirements/ci/circle_ci.md b/docs/system_requirements/ci/circle_ci.md new file mode 100644 index 0000000000..0c1fe107ae --- /dev/null +++ b/docs/system_requirements/ci/circle_ci.md @@ -0,0 +1,23 @@ +# CircleCI + +!!!info + This document applies to Circle CI Cloud, Server v4.x and Server v3.x. + +Your CircleCI configuration should use a dedicated VM for Testcontainers to work. You can achieve this by specifying the +executor type in your `.circleci/config.yml` to be `machine` instead of the default `docker` executor (see [Choosing an Executor Type](https://circleci.com/docs/executor-intro/) for more info). + +Here is a sample CircleCI configuration that does a checkout of a project and runs Maven: + +```yml +jobs: + build: + # Check https://circleci.com/docs/executor-intro#linux-vm for more details + machine: true + image: ubuntu-2204:2023.04.2 + steps: + # install Go 1.19 + # checkout the project + - run: go test./... +``` + +You can learn more about the best practices of using Testcontainers together with CircleCI in [this article](https://www.atomicjar.com/2022/12/testcontainers-with-circleci/) for Java. diff --git a/docs/system_requirements/ci/concourse_ci.md b/docs/system_requirements/ci/concourse_ci.md new file mode 100644 index 0000000000..5861c13138 --- /dev/null +++ b/docs/system_requirements/ci/concourse_ci.md @@ -0,0 +1,48 @@ +# Concourse CI + +This is an example to run Testcontainers tests on [Concourse CI](https://concourse-ci.org/). + +A possible `pipeline.yml` config looks like this: + +```yaml +resources: +- name: repo + type: git + source: + uri: # URL of your project + +jobs: +- name: testcontainers-job + plan: + # Add a get step referencing the resource + - get: repo + - task: testcontainers-task + privileged: true + config: + platform: linux + image_resource: + type: docker-image + source: + repository: amidos/dcind + tag: 2.1.0 + inputs: + - name: repo + run: + path: /bin/sh + args: + - -c + - | + source /docker-lib.sh + start_docker + + cd repo + docker run -it --rm -v "$PWD:$PWD" -w "$PWD" -v /var/run/docker.sock:/var/run/docker.sock golang:1.19 go test ./... +``` + +Finally, you can use Concourse's [fly CLI](https://concourse-ci.org/fly.html) to set the pipeline and trigger the job: + +```bash +fly -t tutorial set-pipeline -p testcontainers-pipeline -c pipeline.yml +fly -t tutorial unpause-pipeline -p testcontainers-pipeline +fly -t tutorial trigger-job --job testcontainers-pipeline/testcontainers-job --watch +``` diff --git a/docs/system_requirements/ci/dind_patterns.md b/docs/system_requirements/ci/dind_patterns.md new file mode 100644 index 0000000000..681dddb0d5 --- /dev/null +++ b/docs/system_requirements/ci/dind_patterns.md @@ -0,0 +1,64 @@ +# Patterns for running tests inside a Docker container + +## 'Docker wormhole' pattern - Sibling docker containers + +Testcontainers itself can be used from inside a container. +This is very useful for different CI scenarios like running everything in containers on Jenkins, or Docker-based CI tools such as Drone. + +Testcontainers will automatically detect if it's inside a container and instead of "localhost" will use the default gateway's IP. + +However, additional configuration is required if you use [volume mapping](../../features/files_and_mounts.md#volume-mapping). The following points need to be considered: + +* The docker socket must be available via a volume mount +* The 'local' source code directory must be volume mounted *at the same path* inside the container that Testcontainers runs in, so that Testcontainers is able to set up the correct volume mounts for the containers it spawns. + +### Docker-only example +If you run the tests with just `docker run ...` then make sure you add `-v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock` to the command, so it will look like this: +```bash +$ tree . +. +├── go.mod +└── internal + └── platform + └── integration_test.go +└── platform + └── integration_test.go + +$ docker run -it --rm -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock golang:1.19 go test ./... -v +``` + +Where: + +* `-v $PWD:$PWD` will add your current directory as a volume inside the container +* `-w $PWD` will set the current directory to this volume +* `-v /var/run/docker.sock:/var/run/docker.sock` will map the Docker socket + + +!!! warning + If you are using Docker Desktop, you need to configure the `TESTCONTAINERS_HOST_OVERRIDE` environment variable to use the special DNS name + `host.docker.internal` for accessing the host from within a container, which is provided by Docker Desktop: + `-e TESTCONTAINERS_HOST_OVERRIDE=host.docker.internal` + +### Docker Compose example + +The same can be achieved with Docker Compose: + +```yaml +tests: + image: golang:1.19 + stop_signal: SIGKILL + stdin_open: true + tty: true + working_dir: $PWD + volumes: + - $PWD:$PWD + - /var/run/docker.sock:/var/run/docker.sock + command: go test ./... -v +``` + +## Docker-in-Docker + +While Docker-in-Docker (DinD) is generally considered an instrument of last resort, it is necessary for some CI environments. + +[Drone CI](./drone.md) is one such example. Testcontainers has a Docker-in-Docker plugin (build image) for use with Drone, +which could be used as inspiration for setting up other similar testing using DinD. diff --git a/docs/system_requirements/ci/drone.md b/docs/system_requirements/ci/drone.md new file mode 100644 index 0000000000..f4a1dd2bc6 --- /dev/null +++ b/docs/system_requirements/ci/drone.md @@ -0,0 +1,6 @@ +# Drone CI + +Drone CI 0.8 is supported via the use of a general purpose Docker-in-Docker plugin. + +Please see [testcontainers/dind-drone-plugin](https://github.com/testcontainers/dind-drone-plugin) for further details and usage instructions. + diff --git a/docs/system_requirements/ci/gitlab_ci.md b/docs/system_requirements/ci/gitlab_ci.md new file mode 100644 index 0000000000..c3ae6cf030 --- /dev/null +++ b/docs/system_requirements/ci/gitlab_ci.md @@ -0,0 +1,63 @@ +# GitLab CI + +## Example using Docker socket +This applies if you have your own GitlabCI runner installed, use the Docker executor and you have `/var/run/docker.sock` mounted in the runner configuration. + +See below for an example runner configuration: + +```toml +[[runners]] + name = "MACHINE_NAME" + url = "https://gitlab.com/" + token = "GENERATED_GITLAB_RUNNER_TOKEN" + executor = "docker" + [runners.docker] + tls_verify = false + image = "docker:latest" + privileged = false + disable_entrypoint_overwrite = false + oom_kill_disable = false + disable_cache = false + volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"] + shm_size = 0 +``` + +!!! warning + The environment variable `TESTCONTAINERS_HOST_OVERRIDE` needs to be configured, otherwise, a wrong IP address would be used to resolve the Docker host, which will likely lead to failing tests. + +Please also include the following in your GitlabCI pipeline definitions (`.gitlab-ci.yml`) that use Testcontainers: + +```yml +variables: + TESTCONTAINERS_HOST_OVERRIDE: "host.docker.internal" +``` + +## Example using DinD (Docker-in-Docker) + +In order to use Testcontainers in a Gitlab CI pipeline, you need to run the job as a Docker container (see [Patterns for running inside Docker](dind_patterns.md)). +So edit your `.gitlab-ci.yml` to include the [Docker-In-Docker service](https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-workflow-with-docker-executor) (`docker:dind`) and set the `DOCKER_HOST` variable to `tcp://docker:2375` and `DOCKER_TLS_CERTDIR` to empty string. + +Caveat: Current Docker releases (verified for 20.10.9) intentionally delay the startup, if the Docker API is bound to a network address but not TLS protected. To avoid this delay, the Docker process needs to be started with the argument `--tls=false`. Otherwise jobs which access the Docker API at the very beginning might fail. + +Here is a sample `.gitlab-ci.yml` that executes test with gradle: + +```yml +# DinD service is required for Testcontainers +services: + - name: docker:dind + # explicitly disable tls to avoid docker startup interruption + command: ["--tls=false"] + +variables: + # Instruct Testcontainers to use the daemon of DinD, use port 2375 for non-tls connections. + DOCKER_HOST: "tcp://docker:2375" + # Instruct Docker not to start over TLS. + DOCKER_TLS_CERTDIR: "" + # Improve performance with overlayfs. + DOCKER_DRIVER: overlay2 + +test: + image: golang:1.19 + stage: test + script: go test ./... -v +``` diff --git a/docs/system_requirements/ci/tekton.md b/docs/system_requirements/ci/tekton.md new file mode 100644 index 0000000000..b53161038a --- /dev/null +++ b/docs/system_requirements/ci/tekton.md @@ -0,0 +1,94 @@ +# Tekton + +To enable access to Docker in Tekton, a dind sidecar needs to be added. An example of it can be found +[here](https://github.com/tektoncd/pipeline/blob/main/examples/v1beta1/taskruns/dind-sidecar.yaml) + +This is an example + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: run-tests + description: Run Tests +spec: + workspaces: + - name: source + steps: + - name: read + image: golang:1.19 + workingDir: $(workspaces.source.path) + script: go test ./... -v + volumeMounts: + - mountPath: /var/run/ + name: dind-socket + sidecars: + - image: docker:20.10-dind + name: docker + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/docker + name: dind-storage + - mountPath: /var/run/ + name: dind-socket + volumes: + - name: dind-storage + emptyDir: { } + - name: dind-socket + emptyDir: { } +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: testcontainers-demo +spec: + description: | + This pipeline clones a git repo, run testcontainers. + params: + - name: repo-url + type: string + description: The git repo URL to clone from. + workspaces: + - name: shared-data + description: | + This workspace contains the cloned repo files, so they can be read by the + next task. + tasks: + - name: fetch-source + taskRef: + name: git-clone + workspaces: + - name: output + workspace: shared-data + params: + - name: url + value: $(params.repo-url) + - name: run-tests + runAfter: ["fetch-source"] + taskRef: + name: run-tests + workspaces: + - name: source + workspace: shared-data +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: testcontainers-demo-run +spec: + pipelineRef: + name: testcontainers-demo + workspaces: + - name: shared-data + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + params: + - name: repo-url + value: # URL of the repo to clone +``` diff --git a/docs/system_requirements/ci/travis.md b/docs/system_requirements/ci/travis.md new file mode 100644 index 0000000000..b07d023fc2 --- /dev/null +++ b/docs/system_requirements/ci/travis.md @@ -0,0 +1,16 @@ +# Travis + +To run Testcontainers on TravisCI, docker needs to be installed. The configuration below +is the minimal required config. + +```yaml +language: go +go: +- 1.x +- "1.19" + +services: +- docker + +script: go test ./... -v +``` diff --git a/mkdocs.yml b/mkdocs.yml index 852674bffa..9097328919 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,7 @@ nav: - Quickstart: quickstart.md - Features: - features/creating_container.md + - features/files_and_mounts.md - features/configuration.md - features/networking.md - features/garbage_collector.md @@ -84,6 +85,16 @@ nav: - System Requirements: - system_requirements/index.md - system_requirements/docker.md + - Continuous Integration: + - system_requirements/ci/aws_codebuild.md + - system_requirements/ci/bitbucket_pipelines.md + - system_requirements/ci/circle_ci.md + - system_requirements/ci/concourse_ci.md + - system_requirements/ci/dind_patterns.md + - system_requirements/ci/drone.md + - system_requirements/ci/gitlab_ci.md + - system_requirements/ci/tekton.md + - system_requirements/ci/travis.md - system_requirements/using_colima.md - system_requirements/using_podman.md - Contributing: diff --git a/modulegen/mkdocs.go b/modulegen/mkdocs.go index 7d532335aa..7dc5b8d4a5 100644 --- a/modulegen/mkdocs.go +++ b/modulegen/mkdocs.go @@ -34,7 +34,7 @@ type MkDocsConfig struct { Features []interface{} `yaml:"Features,omitempty"` Examples []string `yaml:"Examples,omitempty"` Modules []string `yaml:"Modules,omitempty"` - SystemRequirements []string `yaml:"System Requirements,omitempty"` + SystemRequirements []interface{} `yaml:"System Requirements,omitempty"` Contributing []string `yaml:"Contributing,omitempty"` GettingHelp string `yaml:"Getting help,omitempty"` } `yaml:"nav"` diff --git a/mounts_test.go b/mounts_test.go index 952868b057..c69a430b8b 100644 --- a/mounts_test.go +++ b/mounts_test.go @@ -1,6 +1,7 @@ package testcontainers import ( + "context" "testing" "github.com/docker/docker/api/types/mount" @@ -174,3 +175,35 @@ func TestContainerMounts_PrepareMounts(t *testing.T) { }) } } + +func TestCreateContainerWithVolume(t *testing.T) { + // volumeMounts { + req := ContainerRequest{ + Image: "alpine", + Mounts: ContainerMounts{ + { + Source: GenericVolumeMountSource{ + Name: "test-volume", + }, + Target: "/data", + }, + }, + } + // } + + ctx := context.Background() + c, err := GenericContainer(ctx, GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + assert.NoError(t, err) + terminateContainerOnEnd(t, ctx, c) + + // Check if volume is created + client, err := NewDockerClient() + assert.NoError(t, err) + + volume, err := client.VolumeInspect(ctx, "test-volume") + assert.NoError(t, err) + assert.Equal(t, "test-volume", volume.Name) +}