k8s v2 backend

.github/workflows/test.yml

@@ -259,6 +259,48 @@ jobs:
         run: yarn --silent test:k8s --node-version ${{ matrix.node-version }}
         working-directory: ./e2e
 
+  e2e-k8s-v2-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
+      matrix:
+        node-version: [18.19.1, 20.11.1]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup Node ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+
+      # we login to docker to avoid docker pull limit rates
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install and build packages
+        run: yarn setup
+        env:
+          YARN_SETUP_ARGS: "--prod=false --silent"
+
+      - name: Compile e2e code
+        run: yarn build
+        working-directory: ./e2e
+
+      - name: Install Kind and Kubectl
+        uses: helm/kind-action@v1.8.0
+        with:
+          install_only: "true"
+
+      - name: Test k8s V2 elasticsearch7
+        run: yarn --silent test:k8sV2 --node-version ${{ matrix.node-version }}
+        working-directory: ./e2e
+
   e2e-external-storage-tests:
     runs-on: ubuntu-latest
     strategy:

docs/configuration/overview.md

@@ -58,7 +58,7 @@ NOTE: All `asset_storage` related fields are deprecated. Please use the fields i
 | **asset_storage_connection_type** |  `String`  | `"elasticsearch-next"` |       Name of the connection type that will store asset bundles. options: `elasticsearch-next`, `s3`.        |
 |                **assets_volume**                |              `String`              |             -              |                                                      name of shared asset volume (k8s)                                                      |
 |             **autoload_directory**              |              `String`              |     `"$PWD/autoload"`      |                                     directory to look for assets to auto deploy when teraslice boots up                                     |
-|            **cluster_manager_type**             |     `"native"`, `"kubernetes"`     |         `"native"`         |                                               determines which cluster system should be used                                                |
+|            **cluster_manager_type**             |     `"native"`, `"kubernetes", "kubernetesV2"`     |         `"native"`         |                                               determines which cluster system should be used                                                |
 |                     **cpu**                     |              `Number`              |             -              |                                        number of cpus to reserve per teraslice worker in kubernetes                                         |
 |                  **hostname**                   |              `String`              |        `"$HOST_IP"`        |                                                          IP or hostname for server                                                          |
 |     **index_rollover_frequency.analytics**      | `"daily"`, `"monthly"`, `"yearly"` |        `"monthly"`         |                                              How frequently the analytics indices are created                                               |

e2e/jest.config.js

@@ -12,7 +12,7 @@ const config = module.default(dirPath);
 // TODO: update arrays to run tests specific to platform.
 // First array is for tests skipped in kubernetes.
 // Second array is for tests skipped in native.
-config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : [];
+config.testPathIgnorePatterns = process.env.CLUSTERING_TYPE !== 'native' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : [];
 config.collectCoverage = false;
 config.testTimeout = 3 * 60 * 1000;
 

e2e/k8s/masterConfig/teraslice.yaml

@@ -25,7 +25,7 @@ teraslice:
   shutdown_timeout: 30000
   assets_directory: "/app/assets/"
   autoload_directory: "/app/autoload"
-  cluster_manager_type: "kubernetes"
+  cluster_manager_type: "" # set dynamically w/ scripts/src/helpers/config.ts CLUSTERING_TYPE, defaults to 'native'
   master: true
   master_hostname: "127.0.0.1"
   kubernetes_image: "" # set dynamically w/ scripts/src/helpers/config.ts NODE_VERSION: teraslice-workspace:e2e-nodev${NODE_VERSION}

e2e/k8s/workerConfig/teraslice.yaml

@@ -25,7 +25,7 @@ teraslice:
   shutdown_timeout: 30000
   assets_directory: "/app/assets/"
   autoload_directory: "/app/autoload"
-  cluster_manager_type: "kubernetes"
+  cluster_manager_type: "" # set dynamically w/ scripts/src/helpers/config.ts CLUSTERING_TYPE, defaults to 'native'
   master: false
   master_hostname: "teraslice-master"
   kubernetes_image: "" # set dynamically w/ scripts/src/helpers/config.ts NODE_VERSION: teraslice-workspace:e2e-nodev${NODE_VERSION}

e2e/package.json

@@ -33,6 +33,7 @@
         "test:elasticsearch7": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:k8s": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
         "test:k8sNoBuild": "NODE_OPTIONS='--experimental-vm-modules' SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
+        "test:k8sV2": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' ts-scripts test --suite e2e --",
         "test:opensearch1": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:opensearch2": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:s3AssetStorage": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' TEST_KAFKA='true' TEST_MINIO='true' ASSET_STORAGE_CONNECTION_TYPE='s3' ASSET_STORAGE_CONNECTION='default' ts-scripts test --suite e2e --",

e2e/test/cases/assets/simple-spec.ts

@@ -74,7 +74,7 @@ describe('assets', () => {
         // the asset on this job already points to 'ex1' so it should use the latest available asset
         const jobSpec = terasliceHarness.newJob('generator-asset');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         const { workers } = jobSpec;
@@ -102,7 +102,7 @@ describe('assets', () => {
     it('can directly ask for a specific asset version to be used', async () => {
         const jobSpec = terasliceHarness.newJob('generator-asset');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         // the previous test confirms the newer version will be used by default
@@ -271,7 +271,7 @@ describe('s3 asset storage', () => {
 
             const jobSpec = terasliceHarness.newJob('generator-large-asset');
             // // Set resource constraints on workers within CI
-            if (TEST_PLATFORM === 'kubernetes') {
+            if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
                 jobSpec.resources_requests_cpu = 0.1;
             }
 

e2e/test/cases/cluster/api-spec.ts

@@ -18,7 +18,7 @@ describe('cluster api', () => {
         const testStream = createReadStream(assetPath);
         const jobSpec = terasliceHarness.newJob('generator-asset');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.05;
         }
         await terasliceHarness.teraslice.assets.upload(testStream, {
@@ -35,7 +35,7 @@ describe('cluster api', () => {
         const jobSpec = terasliceHarness.newJob('generator-asset');
         const { workers, slicers } = jobSpec;
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.05;
         }
         const alteredJob: Partial<JobConfig> = cloneDeep(jobSpec);
@@ -65,7 +65,7 @@ describe('cluster api', () => {
         const specIndex = terasliceHarness.newSpecIndex('api');
         jobSpec.name = 'basic reindex for lifecycle';
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.05;
         }
 

e2e/test/cases/cluster/job-state-spec.ts

@@ -14,7 +14,7 @@ describe('job state', () => {
         const jobSpec1 = terasliceHarness.newJob('generator');
         const jobSpec2 = terasliceHarness.newJob('generator');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec1.resources_requests_cpu = 0.05;
             jobSpec1.cpu_execution_controller = 0.4;
             jobSpec2.resources_requests_cpu = 0.05;

e2e/test/cases/cluster/scale-workers-spec.ts

@@ -0,0 +1,55 @@
+import { Ex } from 'teraslice-client-js';
+import { WorkerNode } from '@terascope/types/dist/src/teraslice.js';
+import { pDelay } from '@terascope/utils';
+import { TerasliceHarness } from '../../teraslice-harness.js';
+import { DEFAULT_WORKERS, TEST_PLATFORM } from '../../config.js';
+
+describe('scale execution', () => {
+    let terasliceHarness: TerasliceHarness;
+    let job: Ex;
+
+    beforeAll(async () => {
+        terasliceHarness = new TerasliceHarness();
+        await terasliceHarness.init();
+        await terasliceHarness.resetState();
+
+        const jobSpec = terasliceHarness.newJob('generator');
+        // Set resource constraints on workers within CI
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
+            jobSpec.resources_requests_cpu = 0.1;
+        }
+        jobSpec.name = 'scale execution';
+
+        jobSpec.workers = DEFAULT_WORKERS;
+
+        job = await terasliceHarness.submitAndStart(jobSpec);
+    });
+
+    afterAll(async () => {
+        await job.stop();
+    });
+
+    it(`should start with ${DEFAULT_WORKERS} workers`, async () => {
+        let runningWorkers: WorkerNode[] = [];
+        while (runningWorkers.length === 0) {
+            await pDelay(100);
+            runningWorkers = await job.workers();
+        }
+        expect(runningWorkers).toBeArrayOfSize(DEFAULT_WORKERS);
+    });
+
+    it('should add 1 worker', async () => {
+        const runningWorkers = await terasliceHarness.addWorkers(job.id(), 1);
+        expect(runningWorkers).toBe(DEFAULT_WORKERS + 1);
+    });
+
+    it('should remove 2 workers', async () => {
+        const runningWorkers = await terasliceHarness.removeWorkers(job.id(), 2);
+        expect(runningWorkers).toBe(DEFAULT_WORKERS - 1);
+    });
+
+    it('should set workers to 2', async () => {
+        const runningWorkers = await terasliceHarness.setWorkers(job.id(), 2);
+        expect(runningWorkers).toBe(2);
+    });
+});

e2e/test/cases/cluster/state-spec.ts

@@ -96,7 +96,7 @@ describe('cluster state', () => {
         const jobSpec = terasliceHarness.newJob('reindex');
         const specIndex = terasliceHarness.newSpecIndex('state');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'cluster state with 1 worker';
@@ -145,7 +145,7 @@ describe('cluster state', () => {
         const jobSpec = terasliceHarness.newJob('reindex');
         const specIndex = terasliceHarness.newSpecIndex('state');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'cluster state with 4 workers';

e2e/test/cases/cluster/worker-allocation-spec.ts

@@ -10,7 +10,7 @@ async function workersTest(
     const jobSpec = harness.newJob('reindex');
     const specIndex = harness.newSpecIndex('worker-allocation');
     // Set resource constraints on workers within CI
-    if (TEST_PLATFORM === 'kubernetes') {
+    if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
         jobSpec.resources_requests_cpu = 0.1;
     }
     jobSpec.name = 'worker allocation';

e2e/test/cases/data/elasticsearch-bulk-spec.ts

@@ -14,7 +14,7 @@ describe('elasticsearch bulk', () => {
         const jobSpec = terasliceHarness.newJob('multisend');
         const specIndex = terasliceHarness.newSpecIndex('elasticsearch-bulk');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'multisend';

e2e/test/cases/data/id-reader-spec.ts

@@ -22,7 +22,7 @@ xdescribe('id reader', () => {
         const jobSpec = terasliceHarness.newJob('id');
         const specIndex = terasliceHarness.newSpecIndex('id-reader');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
 
@@ -42,7 +42,7 @@ xdescribe('id reader', () => {
         const jobSpec = terasliceHarness.newJob('id');
         const specIndex = terasliceHarness.newSpecIndex('id-reader');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'reindex by hex id';
@@ -63,7 +63,7 @@ xdescribe('id reader', () => {
         const jobSpec = terasliceHarness.newJob('id');
         const specIndex = terasliceHarness.newSpecIndex('id-reader');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'reindex by hex id (range=a..e)';
@@ -86,7 +86,7 @@ xdescribe('id reader', () => {
         const jobSpec = terasliceHarness.newJob('id');
         const specIndex = terasliceHarness.newSpecIndex('id-reader');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         // Job needs to be able to run long enough to cycle

e2e/test/cases/data/recovery-spec.ts

@@ -42,7 +42,7 @@ describe('recovery', () => {
     beforeEach(async () => {
         const jobSpec = terasliceHarness.newJob('generate-to-es');
         // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
         }
         jobSpec.name = 'test recovery job';

e2e/test/cases/data/reindex-spec.ts

@@ -14,7 +14,7 @@ describe('reindex', () => {
     it('should work for simple case', async () => {
         const jobSpec = terasliceHarness.newJob('reindex');
         // Set resource constraints on workers and ex controllers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
             jobSpec.cpu_execution_controller = 0.4;
         }
@@ -37,7 +37,7 @@ describe('reindex', () => {
         jobSpec.name = 'basic reindex';
         const specIndex = terasliceHarness.newSpecIndex('reindex');
         // Set resource constraints on workers and ex controllers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
             jobSpec.cpu_execution_controller = 0.4;
         }
@@ -82,7 +82,7 @@ describe('reindex', () => {
         const specIndex = terasliceHarness.newSpecIndex('reindex');
         jobSpec.name = `reindex ${iterations} times`;
         // Set resource constraints on workers and ex controllers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
             jobSpec.cpu_execution_controller = 0.4;
         }
@@ -112,7 +112,7 @@ describe('reindex', () => {
         const index = terasliceHarness.newSpecIndex('reindex');
         jobSpec.name = 'reindex (with recovery)';
         // Set resource constraints on workers and ex controllers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             jobSpec.resources_requests_cpu = 0.1;
             jobSpec.cpu_execution_controller = 0.4;
         }

e2e/test/cases/kafka/kafka-spec.ts

@@ -22,7 +22,7 @@ describe('kafka', () => {
         const readerSpec = terasliceHarness.newJob('kafka-reader');
 
         // Set resource constraints on workers and ex controllers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
+        if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
             senderSpec.resources_requests_cpu = 0.05;
             senderSpec.cpu_execution_controller = 0.4;
             readerSpec.resources_requests_cpu = 0.05;