[Ingest Manager] Use dockerized package registry in api integration tests

.eslintignore

@@ -35,7 +35,7 @@ target
 /x-pack/legacy/plugins/infra/public/graphql/types.ts
 /x-pack/legacy/plugins/infra/server/graphql/types.ts
 /x-pack/legacy/plugins/maps/public/vendor/**
-
+/x-pack/test/epm_api_integration/apis/fixtures/**
 # package overrides
 /packages/eslint-config-kibana
 /packages/kbn-interpreter/src/common/lib/grammar.js

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -61,6 +61,9 @@ const dockerServerSchema = () =>
       portInContainer: Joi.number().required(),
       waitForLogLine: Joi.alternatives(Joi.object().type(RegExp), Joi.string()).optional(),
       waitFor: Joi.func().optional(),
+      args: Joi.array()
+        .items(Joi.string())
+        .optional(),
     })
     .default();
 

packages/kbn-test/src/functional_test_runner/lib/docker_servers/define_docker_servers_config.ts

@@ -24,8 +24,10 @@ export interface DockerServerSpec {
   port: number;
   image: string;
   waitForLogLine?: RegExp | string;
-  /** a function that should return an obeservable that will allow the tests to execute as soon as it emits anything */
+  /** a function that should return an observable that will allow the tests to execute as soon as it emits anything */
   waitFor?: (server: DockerServer, logLine$: Rx.Observable<string>) => Rx.Observable<unknown>;
+  /* additional command line arguments passed to docker run */
+  args?: string[];
 }
 
 export interface DockerServer extends DockerServerSpec {
@@ -37,6 +39,6 @@ export interface DockerServer extends DockerServerSpec {
  * Helper that helps authors use the type definitions for the section of the FTR config
  * under the `dockerServers` key.
  */
-export function defineDockerServersConfig(config: { [name: string]: DockerServerSpec }) {
+export function defineDockerServersConfig(config: { [name: string]: DockerServerSpec } | {}) {
   return config;
 }

packages/kbn-test/src/functional_test_runner/lib/docker_servers/docker_servers_service.ts

@@ -68,16 +68,19 @@ export class DockerServersService {
   }
 
   private async dockerRun(server: DockerServer) {
+    const { args } = server;
     try {
       this.log.info(`[docker:${server.name}] running image "${server.image}"`);
 
-      const res = await execa('docker', [
+      const dockerArgs = [
         'run',
         '-dit',
+        args || [],
         '-p',
         `${server.port}:${server.portInContainer}`,
         server.image,
-      ]);
+      ].flat();
+      const res = await execa('docker', dockerArgs);
 
       return res.stdout.trim();
     } catch (error) {
@@ -117,8 +120,12 @@ export class DockerServersService {
     lifecycle.cleanup.add(() => {
       try {
         execa.sync('docker', ['kill', containerId]);
+        execa.sync('docker', ['rm', containerId]);
       } catch (error) {
-        if (error.message.includes(`Container ${containerId} is not running`)) {
+        if (
+          error.message.includes(`Container ${containerId} is not running`) ||
+          error.message.includes(`No such container: ${containerId}`)
+        ) {
           return;
         }
 

src/dev/precommit_hook/casing_check_config.js

@@ -60,6 +60,10 @@ export const IGNORE_FILE_GLOBS = [
   // filename required by api-extractor
   'api-documenter.json',
 
+  // test cases come from elastic/package-storage repo, which uses
+  // nameing conventions from elastic/beats modules
+  'x-pack/test/epm_api_integration/apis/fixtures/**',
+
   // TODO fix file names in APM to remove these
   'x-pack/plugins/apm/public/**/*',
   'x-pack/plugins/apm/scripts/**/*',

x-pack/test/epm_api_integration/apis/fixtures/registry/public/package/base/0.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# Base package
+
+This is installed in the background by EPM to setup the Elastic Stack.

x-pack/test/epm_api_integration/apis/fixtures/registry/public/package/base/0.1.0/elasticsearch/component-template/events-mappings.json

@@ -0,0 +1,148 @@
+{
+    "template": {
+        "mappings": {
+            "_meta": {
+            },
+            "dynamic_templates": [
+                {
+                    "strings_as_keyword": {
+                        "mapping": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "match_mapping_type": "string"
+                    }
+                }
+            ],
+            "date_detection": false,
+            "properties": {
+                "@timestamp": {
+                    "type": "date"
+                },
+                "stream": {
+                    "properties": {
+                        "type": {
+                            "type": "constant_keyword"
+                        },
+                        "dataset": {
+                            "type": "constant_keyword"
+                        },
+                        "namespace": {
+                            "type": "constant_keyword"
+                        }
+                    }
+                },
+                "agent": {
+                    "properties": {
+                        "hostname": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "name": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "id": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "ephemeral_id": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "type": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "version": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        }
+                    }
+                },
+                "ecs": {
+                    "properties": {
+                        "version": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        }
+                    }
+                },
+                "host": {
+                    "properties": {
+                        "hostname": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "os": {
+                            "properties": {
+                                "build": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "kernel": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "codename": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "name": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "family": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "version": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "platform": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "full": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "ip": {
+                            "type": "ip"
+                        },
+                        "containerized": {
+                            "type": "boolean"
+                        },
+                        "name": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "id": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "type": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "mac": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "architecture": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        }
+                    }
+                },
+                "message": {
+                    "type": "text"
+                }
+            }
+        },
+        "aliases": {}
+    }
+}

x-pack/test/epm_api_integration/apis/fixtures/registry/public/package/base/0.1.0/elasticsearch/component-template/events-settings.json

@@ -0,0 +1,21 @@
+{
+    "template": {
+        "settings": {
+            "index": {
+                "lifecycle": {
+                    "name": "events-default"
+                },
+                "codec": "best_compression",
+                "refresh_interval": "5s",
+                "number_of_shards": "1",
+                "query": {
+                    "default_field": [
+                        "message"
+                    ]
+                },
+                "number_of_routing_shards": "30"
+            }
+        },
+        "aliases": {}
+    }
+}