From e51a59c1bf2fbbec02115c94a106f09a24e0ae61 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 19 Nov 2020 18:54:57 +0100 Subject: [PATCH] Deprecate testing/environment in favor of elastic-package stack (#405) * Deprecate testing/environment in favor of elastic-package stack * Address PR comments --- CONTRIBUTING.md | 54 +++++++++++++------ magefile.go | 13 ----- testing/environments/7.9.0.yml | 48 ----------------- .../environments/Dockerfile.package-registry | 4 -- testing/environments/README.md | 51 +++++++++++++----- testing/environments/agent.yml | 15 ------ testing/environments/kibana.config.yml | 16 ------ .../environments/package-registry.config.yml | 5 -- testing/environments/snapshot.yml | 50 ----------------- 9 files changed, 76 insertions(+), 180 deletions(-) delete mode 100644 testing/environments/7.9.0.yml delete mode 100644 testing/environments/Dockerfile.package-registry delete mode 100644 testing/environments/agent.yml delete mode 100644 testing/environments/kibana.config.yml delete mode 100644 testing/environments/package-registry.config.yml delete mode 100644 testing/environments/snapshot.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d0e9f1f61ab..d04f59c0aa8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,14 +88,14 @@ feel free to review the script's [README](https://github.com/elastic/integration ```bash $ go get -u -d github.com/magefile/mage ``` -3. Boot up required dependencies: +3. Use the `elastic-package stack up -v -d` command to boot up required dependencies: 1. Elasticseach instance: * Kibana's dependency 2. Kibana instance: * used to migrate dashboards, if not available, you can skip the generation (`SKIP_KIBANA=true`) - _Hint_. There is dockerized environment in beats (`cd testing/environments`). Boot it up with the following command: - `docker-compose -f snapshot.yml up --force-recreate`. + _Hint_. There is the `elastic-package` cheat sheet available [here](https://github.com/elastic/integrations/blob/master/testing/environments/README.md). + 4. Create a new branch for the integration in `integrations` repository (diverge from master). 5. Run the command: `mage ImportBeats` to start the import process (note that the import script assumes the projects checked out in step 2 are at `../{project-name}`). @@ -123,7 +123,7 @@ Most of migration work has been done by the `import-beats` script, but there're interaction. It may happen that your integration misses a screenshot or an icon, it's a good moment to add missing resources to -Beats/Kibana repositories and re-import the integration (idempotent). +Beats/Kibana repositories and re-import the integration (idempotent). #### Checklist @@ -160,7 +160,7 @@ what's been already fixed, as the script has overridden part of it). The README template is used to render the final README file including exported fields. The template should be placed in the `dev/import-beats-resources//docs/README.md`. - Review the MySQL docs template to see how to use template functions (e.g. `{{fields "dataset-name"}}`). + Review the MySQL docs template to see how to use template functions (e.g. `{{fields "dataset-name"}}`). If the same dataset name is used in both metrics and logs, please add `-metrics` and `-logs` in the template. For example, `elb` is a dataset for log and also a dataset for metrics. In README.md template, `{{fields "elb_logs"}}` and `{{fields "elb_metrics"}}` are used to separate them. 5. Review fields file and exported fields in docs. @@ -264,21 +264,33 @@ what's been already fixed, as the script has overridden part of it). ### Run the whole setup -1. Build `public` directory with package data: +_The `elastic-package stack` provides an enrolled instance of the Elastic Agent. Use that one instead of a local application +if you can run the service (you're integrating with) in the Docker network. The service Docker image can be used for [system +testing](https://github.com/elastic/elastic-package/blob/master/docs/howto/system_testing.md). + +1. Build `packages` directory with package data: ```bash $ mage build ``` 2. Start testing environment: + + _Run from inside the Integrations repository._ + ```bash - $ cd testing/environments - $ docker-compose -f snapshot.yml up + $ elastic-package stack up -d -v ``` - The command will boot up a docker cluster with Elasticsearch, Kibana and Package Registry. After every time you - rebuild and reload packages (`mage Reload`), all adjustments in packages will be propagated to the registry. + The command above will boot up the Elastic stack (Elasticsearch, Kibana, Package Registry) using Docker containers. + It rebuilds the Package Registry Docker image using local packages and boots up the Package Registry. + + To reload the already deployed Package Registry use the following command: -3. Verify that your integration is available (in the right version), e.g. MySQL: http://localhost:8080/search?package=mysql (use + ```bash + $ elastic-package stack up -v -d --services package-registry + ``` + +3. Verify that your integration is available (in the right version), e.g. MySQL: http://localhost:8080/search?package=mysql (use `experimental=true` parameter if the package is in experimental version. Alternatively set `release` to `beta` or higher in your package's `manifest.yml`, if appropriate.) @@ -381,9 +393,21 @@ on the business or technical requirements for the entire platform (Elastic Packa #### Development -1. When you're developing integrations and you'd like to propagate your changes to the package registry, - use `mage Reload` to rebuild and reload the package registry. - +1. When you're developing integrations and you'd like to propagate your changes to the package registry, first rebuild the package: + + ```bash + $ cd packages/apache + $ elastic-package build + ``` + + Then, rebuild and redeploy the Package Registry: + + _It's important to execute the following command in the Integrations repository._ + + ```bash + $ elastic-package stack up -v -d --services package-registry + ``` + Explanation: it's much faster to rebuild and restart the container with the Package Registry, than work with mounted volumes. @@ -437,7 +461,7 @@ on the business or technical requirements for the entire platform (Elastic Packa If you notice that fields file (e.g. `package-fields.yml`) doesn't contain any field definitions or it defines root only, feel free to remove it. - Bad candidate: + Bad candidate: ```yaml - name: mypackage.mydataset type: group diff --git a/magefile.go b/magefile.go index cdd32824fbfd..acab9c74456d 100644 --- a/magefile.go +++ b/magefile.go @@ -133,19 +133,6 @@ func UpdatePackageStorage() error { return sh.Run("go", args...) } -func Reload() error { - err := Build() - if err != nil { - return err - } - - err = sh.RunV("docker-compose", "-f", "testing/environments/snapshot.yml", "build", "package-registry") - if err != nil { - return err - } - return sh.RunV("docker-compose", "-f", "testing/environments/snapshot.yml", "up", "-d", "package-registry") -} - // Format method formats integrations. func formatIntegrations() error { return runElasticPackageOnAllIntegrations(true, "format") diff --git a/testing/environments/7.9.0.yml b/testing/environments/7.9.0.yml deleted file mode 100644 index 0f3b0d026a27..000000000000 --- a/testing/environments/7.9.0.yml +++ /dev/null @@ -1,48 +0,0 @@ -# This should start the environment with the 7.9.0 releases. - -version: '2.3' -services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 - healthcheck: - test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"] - retries: 300 - interval: 1s - environment: - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "network.host=" - - "transport.host=127.0.0.1" - - "http.host=0.0.0.0" - - "indices.id_field_data.enabled=true" - - "xpack.license.self_generated.type=trial" - - "xpack.security.enabled=true" - - "xpack.security.authc.api_key.enabled=true" - - "ELASTIC_PASSWORD=changeme" - ports: - - "127.0.0.1:9200:9200" - - kibana: - image: docker.elastic.co/kibana/kibana:7.9.0 - depends_on: - elasticsearch: - condition: service_healthy - package-registry: - condition: service_healthy - healthcheck: - test: "curl -f http://127.0.0.1:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null" - retries: 600 - interval: 1s - volumes: - - ./kibana.config.yml:/usr/share/kibana/config/kibana.yml - ports: - - "127.0.0.1:5601:5601" - - package-registry: - image: docker.elastic.co/package-registry/distribution:staging - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] - retries: 300 - interval: 1s - restart: always - ports: - - "127.0.0.1:8080:8080" diff --git a/testing/environments/Dockerfile.package-registry b/testing/environments/Dockerfile.package-registry deleted file mode 100644 index ef24482eb968..000000000000 --- a/testing/environments/Dockerfile.package-registry +++ /dev/null @@ -1,4 +0,0 @@ -FROM docker.elastic.co/package-registry/distribution:snapshot - -COPY testing/environments/package-registry.config.yml /package-registry/config.yml -COPY build/integrations /packages/integrations diff --git a/testing/environments/README.md b/testing/environments/README.md index 267c1ba5875f..403434bddc0b 100644 --- a/testing/environments/README.md +++ b/testing/environments/README.md @@ -1,26 +1,49 @@ -Before using the Package Registry, remember to `mage build` the project to prepare the volume with packages -(`public` directory). +## Cheat sheet: elastic-package -Refresh docker images: +Update already downloaded Docker images: -```bash -$ docker-compose -f snapshot.yml pull -``` +`elastic-package stack update -v` -Run docker containers (Elasticsearch, Kibana, Package Registry): +Quickly boot up the stack use: -```bash -$ docker-compose -f snapshot.yml up --force-recreate -``` +_Run from within the Integrations repository to consider local package sources (expected for package development)._ + +`elastic-package stack up -d -v` + +Take down the stack: + +`elastic-package stack down -v` -... or with Elastic Agent: +Advanced: if you need to modify the internal Docker compose definition, edit files in `~/.elastic-package/stack`, but +keep in mind that these files shouldn't be modified and your changes will be reverted once you update the `elastic-package`: ```bash -$ docker-compose -f snapshot.yml -f agent.yml up --force-recreate +$tree ~/.elastic-package/stack +/Users/JohnDoe/.elastic-package/stack +├── Dockerfile.package-registry +├── development +├── kibana.config.yml +├── package-registry.config.yml +└── snapshot.yml ``` -Use this command to spawn more agents: +## Cheat sheet: reload local changes in Kibana + +Rebuild the modified package: + +`mage build` (for all packages) + +or ```bash -$ docker-compose -f snapshot.yml -f agent.yml up --scale elastic-agent=10 --no-recreate -d +$ cd packages/apache +$ elastic-package build ``` + +(for single package, in this sample - _Apache_). + +Rebuild and restart the package-registry image: + +`elastic-package stack up -v -d --services package-registry` + +You should see your latest changes in the Kibana UI. \ No newline at end of file diff --git a/testing/environments/agent.yml b/testing/environments/agent.yml deleted file mode 100644 index 0570b69c4a95..000000000000 --- a/testing/environments/agent.yml +++ /dev/null @@ -1,15 +0,0 @@ -# This should start the environment with the latest snapshots. - -version: '2.3' -services: - elastic-agent: - image: docker.elastic.co/beats/elastic-agent:7.9.0-SNAPSHOT - depends_on: - elasticsearch: - condition: service_healthy - kibana: - condition: service_healthy - environment: - - "FLEET_ENROLL=1" - - "FLEET_SETUP=1" - - "KIBANA_HOST=http://kibana:5601" diff --git a/testing/environments/kibana.config.yml b/testing/environments/kibana.config.yml deleted file mode 100644 index 61f927473a07..000000000000 --- a/testing/environments/kibana.config.yml +++ /dev/null @@ -1,16 +0,0 @@ -server.name: kibana -server.host: "0" - -elasticsearch.hosts: [ "http://elasticsearch:9200" ] -elasticsearch.username: elastic -elasticsearch.password: changeme -xpack.monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.enabled: true -xpack.fleet.registryUrl: "http://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.host: "http://localhost:9200" -xpack.fleet.agents.kibana.host: "http://localhost:5601" -xpack.fleet.agents.tlsCheckDisabled: true - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" diff --git a/testing/environments/package-registry.config.yml b/testing/environments/package-registry.config.yml deleted file mode 100644 index 38b61a310af4..000000000000 --- a/testing/environments/package-registry.config.yml +++ /dev/null @@ -1,5 +0,0 @@ -package_paths: - - /packages/integrations - - /packages/production - - /packages/staging - - /packages/snapshot \ No newline at end of file diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml deleted file mode 100644 index 80e396aa1967..000000000000 --- a/testing/environments/snapshot.yml +++ /dev/null @@ -1,50 +0,0 @@ -# This should start the environment with the latest snapshots. - -version: '2.3' -services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0-SNAPSHOT - healthcheck: - test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"] - retries: 300 - interval: 1s - environment: - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "network.host=" - - "transport.host=127.0.0.1" - - "http.host=0.0.0.0" - - "indices.id_field_data.enabled=true" - - "xpack.license.self_generated.type=trial" - - "xpack.security.enabled=true" - - "xpack.security.authc.api_key.enabled=true" - - "ELASTIC_PASSWORD=changeme" - ports: - - "127.0.0.1:9200:9200" - - kibana: - image: docker.elastic.co/kibana/kibana:7.10.0-SNAPSHOT - depends_on: - elasticsearch: - condition: service_healthy - package-registry: - condition: service_healthy - healthcheck: - test: "curl -f http://127.0.0.1:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null" - retries: 600 - interval: 1s - volumes: - - ./kibana.config.yml:/usr/share/kibana/config/kibana.yml - ports: - - "127.0.0.1:5601:5601" - - package-registry: - build: - context: ../.. - dockerfile: testing/environments/Dockerfile.package-registry - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] - retries: 300 - interval: 1s - restart: always - ports: - - "127.0.0.1:8080:8080"