Skip to content

Commit

Permalink
add types as scripts dependency, add kubernetesv2 to all cluster type…
Browse files Browse the repository at this point in the history 
… checks, refactor k8s-env and test-runner code
  • Loading branch information
@busma13
busma13 committed May 30, 2024
1 parent 0be5924 commit ab7f634
Show file tree
Hide file tree
Showing 36 changed files with 286 additions and 123 deletions.
2 changes: 1 addition & 1 deletion docs/configuration/overview.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ NOTE: All `asset_storage` related fields are deprecated. Please use the fields i
| **asset_storage_connection_type** | `String` | `"elasticsearch-next"` | Name of the connection type that will store asset bundles. options: `elasticsearch-next`, `s3`. |
| **assets_volume** | `String` | - | name of shared asset volume (k8s) |
| **autoload_directory** | `String` | `"$PWD/autoload"` | directory to look for assets to auto deploy when teraslice boots up |
| **cluster_manager_type** | `"native"`, `"kubernetes"` | `"native"` | determines which cluster system should be used |
| **cluster_manager_type** | `"native"`, `"kubernetes", "kubernetesV2"` | `"native"` | determines which cluster system should be used |
| **cpu** | `Number` | - | number of cpus to reserve per teraslice worker in kubernetes |
| **hostname** | `String` | `"$HOST_IP"` | IP or hostname for server |
| **index_rollover_frequency.analytics** | `"daily"`, `"monthly"`, `"yearly"` | `"monthly"` | How frequently the analytics indices are created |
Expand Down
2 changes: 1 addition & 1 deletion e2e/jest.config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ const config = module.default(dirPath);
// TODO: update arrays to run tests specific to platform.
// First array is for tests skipped in kubernetes.
// Second array is for tests skipped in native.
config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : [];
config.testPathIgnorePatterns = process.env.CLUSTERING_TYPE !== 'native' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : [];
config.collectCoverage = false;
config.testTimeout = 3 * 60 * 1000;

Expand Down
2 changes: 1 addition & 1 deletion e2e/k8s/masterConfig/teraslice.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ teraslice:
shutdown_timeout: 30000
assets_directory: "/app/assets/"
autoload_directory: "/app/autoload"
cluster_manager_type: "kubernetes"
cluster_manager_type: "" # set dynamically w/ scripts/src/helpers/config.ts CLUSTERING_TYPE, defaults to 'native'
master: true
master_hostname: "127.0.0.1"
kubernetes_image: "" # set dynamically w/ scripts/src/helpers/config.ts NODE_VERSION: teraslice-workspace:e2e-nodev${NODE_VERSION}
Expand Down
2 changes: 1 addition & 1 deletion e2e/k8s/workerConfig/teraslice.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ teraslice:
shutdown_timeout: 30000
assets_directory: "/app/assets/"
autoload_directory: "/app/autoload"
cluster_manager_type: "kubernetes"
cluster_manager_type: "" # set dynamically w/ scripts/src/helpers/config.ts CLUSTERING_TYPE, defaults to 'native'
master: false
master_hostname: "teraslice-master"
kubernetes_image: "" # set dynamically w/ scripts/src/helpers/config.ts NODE_VERSION: teraslice-workspace:e2e-nodev${NODE_VERSION}
Expand Down
8 changes: 4 additions & 4 deletions e2e/test/cases/assets/simple-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe('assets', () => {
const fileStream = fs.createReadStream(assetPath);
const jobSpec = terasliceHarness.newJob(jobSpecName);
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
const { workers } = jobSpec; // save for comparison
Expand Down Expand Up @@ -123,7 +123,7 @@ describe('assets', () => {
// the asset on this job already points to 'ex1' so it should use the latest available asset
const jobSpec = terasliceHarness.newJob('generator-asset');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
const { workers } = jobSpec;
Expand Down Expand Up @@ -151,7 +151,7 @@ describe('assets', () => {
it('can directly ask for the new asset to be used', async () => {
const jobSpec = terasliceHarness.newJob('generator-asset');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.assets = ['ex1:0.1.1', 'standard', 'elasticsearch'];
Expand Down Expand Up @@ -315,7 +315,7 @@ describe('s3 asset storage', () => {

const jobSpec = terasliceHarness.newJob('generator-large-asset');
// // Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}

Expand Down
6 changes: 3 additions & 3 deletions e2e/test/cases/cluster/api-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ describe('cluster api', () => {
const testStream = createReadStream(assetPath);
const jobSpec = terasliceHarness.newJob('generator-asset');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.05;
}
await terasliceHarness.teraslice.assets.upload(testStream, {
Expand All @@ -34,7 +34,7 @@ describe('cluster api', () => {
const jobSpec = terasliceHarness.newJob('generator-asset');
const { workers, slicers } = jobSpec;
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.05;
}
const alteredJob = cloneDeep(jobSpec);
Expand Down Expand Up @@ -64,7 +64,7 @@ describe('cluster api', () => {
const specIndex = terasliceHarness.newSpecIndex('api');
jobSpec.name = 'basic reindex for lifecycle';
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.05;
}

Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/cluster/job-state-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ describe('job state', () => {
const jobSpec1 = terasliceHarness.newJob('generator');
const jobSpec2 = terasliceHarness.newJob('generator');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec1.resources_requests_cpu = 0.05;
jobSpec1.cpu_execution_controller = 0.4;
jobSpec2.resources_requests_cpu = 0.05;
Expand Down
4 changes: 2 additions & 2 deletions e2e/test/cases/cluster/state-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ describe('cluster state', () => {
const jobSpec = terasliceHarness.newJob('reindex');
const specIndex = terasliceHarness.newSpecIndex('state');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'cluster state with 1 worker';
Expand Down Expand Up @@ -145,7 +145,7 @@ describe('cluster state', () => {
const jobSpec = terasliceHarness.newJob('reindex');
const specIndex = terasliceHarness.newSpecIndex('state');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'cluster state with 4 workers';
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/cluster/worker-allocation-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ async function workersTest(
const jobSpec = harness.newJob('reindex');
const specIndex = harness.newSpecIndex('worker-allocation');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'worker allocation';
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/data/elasticsearch-bulk-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ describe('elasticsearch bulk', () => {
const jobSpec = terasliceHarness.newJob('multisend');
const specIndex = terasliceHarness.newSpecIndex('elasticsearch-bulk');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'multisend';
Expand Down
8 changes: 4 additions & 4 deletions e2e/test/cases/data/id-reader-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ xdescribe('id reader', () => {
const jobSpec = terasliceHarness.newJob('id');
const specIndex = terasliceHarness.newSpecIndex('id-reader');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}

Expand All @@ -42,7 +42,7 @@ xdescribe('id reader', () => {
const jobSpec = terasliceHarness.newJob('id');
const specIndex = terasliceHarness.newSpecIndex('id-reader');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'reindex by hex id';
Expand All @@ -63,7 +63,7 @@ xdescribe('id reader', () => {
const jobSpec = terasliceHarness.newJob('id');
const specIndex = terasliceHarness.newSpecIndex('id-reader');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'reindex by hex id (range=a..e)';
Expand All @@ -86,7 +86,7 @@ xdescribe('id reader', () => {
const jobSpec = terasliceHarness.newJob('id');
const specIndex = terasliceHarness.newSpecIndex('id-reader');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
// Job needs to be able to run long enough to cycle
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/data/recovery-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ describe('recovery', () => {
beforeEach(async () => {
const jobSpec = terasliceHarness.newJob('generate-to-es');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.name = 'test recovery job';
Expand Down
8 changes: 4 additions & 4 deletions e2e/test/cases/data/reindex-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ describe('reindex', () => {
it('should work for simple case', async () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers and ex controllers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
jobSpec.cpu_execution_controller = 0.4;
}
Expand All @@ -37,7 +37,7 @@ describe('reindex', () => {
jobSpec.name = 'basic reindex';
const specIndex = terasliceHarness.newSpecIndex('reindex');
// Set resource constraints on workers and ex controllers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
jobSpec.cpu_execution_controller = 0.4;
}
Expand Down Expand Up @@ -82,7 +82,7 @@ describe('reindex', () => {
const specIndex = terasliceHarness.newSpecIndex('reindex');
jobSpec.name = `reindex ${iterations} times`;
// Set resource constraints on workers and ex controllers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
jobSpec.cpu_execution_controller = 0.4;
}
Expand Down Expand Up @@ -112,7 +112,7 @@ describe('reindex', () => {
const index = terasliceHarness.newSpecIndex('reindex');
jobSpec.name = 'reindex (with recovery)';
// Set resource constraints on workers and ex controllers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
jobSpec.cpu_execution_controller = 0.4;
}
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/cases/kafka/kafka-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ describe('kafka', () => {
const readerSpec = terasliceHarness.newJob('kafka-reader');

// Set resource constraints on workers and ex controllers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
senderSpec.resources_requests_cpu = 0.05;
senderSpec.cpu_execution_controller = 0.4;
readerSpec.resources_requests_cpu = 0.05;
Expand Down
12 changes: 6 additions & 6 deletions e2e/test/cases/validation/job-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ describe('job validation', () => {
it('should be rejected with empty index selector index name', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}

Expand All @@ -34,7 +34,7 @@ describe('job validation', () => {
it('should be rejected with empty reader index name', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}

Expand All @@ -54,7 +54,7 @@ describe('job validation', () => {
it('should be rejected with slicers = 0', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.slicers = 0;
Expand All @@ -70,7 +70,7 @@ describe('job validation', () => {
it('should be rejected with slicers < 0', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.slicers = -1;
Expand All @@ -88,7 +88,7 @@ describe('job validation', () => {
it('should be rejected with negative workers == 0', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.workers = 0;
Expand All @@ -104,7 +104,7 @@ describe('job validation', () => {
it('should be rejected with invalid lifecycle', () => {
const jobSpec = terasliceHarness.newJob('reindex');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
jobSpec.resources_requests_cpu = 0.1;
}
// @ts-expect-error
Expand Down
4 changes: 2 additions & 2 deletions e2e/test/global.setup.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ export default async () => {
fse.ensureDir(CONFIG_PATH),
]);

if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
await downloadAssets();
const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER);
await k8s.deployK8sTeraslice(true);
await k8s.deployK8sTeraslice(TEST_PLATFORM, true);
await teraslice.waitForTeraslice();
await setAlias(TERASLICE_PORT);
} else {
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/teardown.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export async function teardown(testClient?: Client) {
const errors = [];

try {
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER);
await k8s.deleteTerasliceNamespace('ts-ns.yaml');
await cleanupIndex(client, 'ts-dev1_*');
Expand Down
8 changes: 4 additions & 4 deletions e2e/test/teraslice-harness.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export class TerasliceHarness {
async resetState() {
const startTime = Date.now();

if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
try {
cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`);
await showState(TERASLICE_PORT);
Expand Down Expand Up @@ -468,7 +468,7 @@ export class TerasliceHarness {
return _waitForClusterState();
}

if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
// A get request to 'cluster/state' will return an empty object in kubernetes.
// Therefore nodes will be 0.
if (nodes === 0) return nodes;
Expand Down Expand Up @@ -517,7 +517,7 @@ export class TerasliceHarness {

const nodes = await this.waitForClusterState();

if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
signale.success('Teraslice is ready to go', getElapsed(startTime));
} else {
signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime));
Expand Down Expand Up @@ -559,7 +559,7 @@ export class TerasliceHarness {
} as any;

try {
if (TEST_PLATFORM === 'kubernetes') {
if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') {
// Set resource constraints on workers and ex controllers within CI
jobSpec.resources_requests_cpu = 0.05;
jobSpec.cpu_execution_controller = 0.4;
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
"bump": "ts-scripts bump",
"docs": "ts-scripts docs",
"k8s": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 ts-scripts k8s-env",
"k8sV2": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 CLUSTERING_TYPE='kubernetesV2' ts-scripts k8s-env",
"k8s:kafka": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 TEST_KAFKA=true KAFKA_PORT=9092 ts-scripts k8s-env",
"k8s:minio": "TEST_MINIO=true MINIO_PORT=9000 TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 ts-scripts k8s-env",
"k8s:noBuild": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 SKIP_DOCKER_BUILD_IN_K8S=true ts-scripts k8s-env",
Expand Down
Loading

0 comments on commit ab7f634

Please sign in to comment.