Skip to content

Commit

Permalink
Merge pull request #3097 from snyk/test/update-iac-test-exit-code-assert
Browse files Browse the repository at this point in the history 
test: get stdout failure text before exit code
  • Loading branch information
@ipapast
ipapast authored Apr 5, 2022
2 parents bf8281a + 8c392a4 commit 4cd5f66
Show file tree
Hide file tree
Showing 12 changed files with 69 additions and 143 deletions.
27 changes: 9 additions & 18 deletions test/jest/acceptance/iac/cli-share-results.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ describe('CLI Share Results', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);
expect(exitCode).toBe(2);

expect(stdout).toMatch(
'Flag "--report" is only supported if feature flag "iacCliShareResults" is enabled. To enable it, please contact Snyk support.',
);
expect(exitCode).toBe(2);
});
});

Expand All @@ -49,20 +49,18 @@ describe('CLI Share Results', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);
expect(exitCode).toBe(1);

expect(stdout).toContain(
`Your test results are available at: http://localhost:${server.getPort()}/org/test-org/projects under the name snyk/cli`,
);
expect(exitCode).toBe(1);
});

it('forwards value to iac-cli-share-results endpoint', async () => {
const { exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);

expect(exitCode).toEqual(1);

const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));
Expand All @@ -88,15 +86,14 @@ describe('CLI Share Results', () => {
],
}),
);
expect(exitCode).toEqual(1);
});

it('forwards project tags', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --tags=foo=bar',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -108,15 +105,14 @@ describe('CLI Share Results', () => {
expect(request.body).toMatchObject({
tags: [{ key: 'foo', value: 'bar' }],
});
expect(exitCode).toEqual(1);
});

it('forwards project environment', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-environment=saas',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -130,15 +126,14 @@ describe('CLI Share Results', () => {
environment: ['saas'],
},
});
expect(exitCode).toEqual(1);
});

it('forwards project lifecycle', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-lifecycle=sandbox',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -152,28 +147,26 @@ describe('CLI Share Results', () => {
lifecycle: ['sandbox'],
},
});
expect(exitCode).toEqual(1);
});

it('forwards project business criticality', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));

expect(requests.length).toEqual(1);

const [request] = requests;

expect(request.body).toMatchObject({
attributes: {
criticality: ['high'],
},
});
expect(exitCode).toEqual(1);
});

it('should print an error message if test usage is exceeded', async () => {
Expand All @@ -183,8 +176,8 @@ describe('CLI Share Results', () => {
'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high',
);

expect(exitCode).toEqual(2);
expect(stdout).toMatch(/test limit reached/i);
expect(exitCode).toEqual(2);
});

describe('with target reference', () => {
Expand All @@ -195,12 +188,9 @@ describe('CLI Share Results', () => {
`snyk iac test ./iac/arm/rule_test.json --report --target-reference=${testTargetRef}`,
);

expect(exitCode).toEqual(1);

const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));

expect(testRequests[0]).toMatchObject({
body: {
contributors: expect.any(Array),
Expand All @@ -221,6 +211,7 @@ describe('CLI Share Results', () => {
],
},
});
expect(exitCode).toEqual(1);
});
});
});
Expand Down
25 changes: 10 additions & 15 deletions test/jest/acceptance/iac/custom-rules.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ describe('iac test --rules', () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);
expect(exitCode).toBe(1);

expect(stdout).toContain(
'Using custom rules to generate misconfigurations.',
Expand All @@ -33,39 +32,37 @@ describe('iac test --rules', () => {
expect(stdout).toContain(
'introduced by input > resource > aws_security_group[allow_ssh] > tags',
);
expect(exitCode).toBe(1);
});

it('does not show custom rules message if it scans local custom rules and uses --json flag', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf --json`,
);
expect(exitCode).toBe(1);

expect(stdout).not.toContain(
'Using custom rules to generate misconfigurations.',
);
expect(exitCode).toBe(1);
});

it('presents an error message when the local rules cannot be found', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./not/a/real/path.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'We were unable to extract the rules provided at: ./not/a/real/path.tar.gz',
);
expect(exitCode).toBe(2);
});

it('presents an error message when the user is not entitled to custom-rules', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --org=no-custom-rules-entitlements --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
`Flag "--rules" is currently not supported for this org. To enable it, please contact snyk support.`,
);
expect(exitCode).toBe(2);
});

describe.each([
Expand All @@ -76,9 +73,8 @@ describe('iac test --rules', () => {
const { stderr, exitCode } = await run(
`snyk iac ${testedCommand} --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toEqual(1);
expect(stderr).toEqual('');
expect(exitCode).toEqual(1);
});

it('should display a message informing of the application of custom rules', async () => {
Expand Down Expand Up @@ -219,7 +215,6 @@ describe('custom rules pull from a remote OCI registry', () => {
expect(SNYK_CFG_OCI_REGISTRY_URL).toBeDefined();
expect(SNYK_CFG_OCI_REGISTRY_USERNAME).toBeDefined();
expect(SNYK_CFG_OCI_REGISTRY_PASSWORD).toBeDefined();
expect(exitCode).toBe(1);

expect(stdout).toContain(
'Using custom rules to generate misconfigurations.',
Expand All @@ -231,6 +226,7 @@ describe('custom rules pull from a remote OCI registry', () => {
expect(stdout).toContain(
'introduced by input > resource > aws_security_group[allow_ssh] > tags',
);
expect(exitCode).toBe(1);
});

describe.each([
Expand All @@ -246,9 +242,8 @@ describe('custom rules pull from a remote OCI registry', () => {
SNYK_CFG_OCI_REGISTRY_PASSWORD: SNYK_CFG_OCI_REGISTRY_PASSWORD as string,
},
);

expect(exitCode).toEqual(1);
expect(stderr).toContain('');
expect(exitCode).toEqual(1);
});

it('should display a message informing of the application of custom rules', async () => {
Expand Down Expand Up @@ -313,10 +308,10 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'There was an authentication error. Incorrect credentials provided.',
);
expect(exitCode).toBe(2);
});

it('presents an error message when there is a local and remote rules conflict', async () => {
Expand All @@ -328,10 +323,10 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'Remote and local custom rules bundle can not be used at the same time.',
);
expect(exitCode).toBe(2);
});

it('presents an error message when the user is not entitled to custom-rules', async () => {
Expand All @@ -346,9 +341,9 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
`The custom rules feature is currently not supported for this org. To enable it, please contact snyk support.`,
);
expect(exitCode).toBe(2);
});
});
3 changes: 1 addition & 2 deletions test/jest/acceptance/iac/file-output.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ describe('iac test --sarif-file-output', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/file-logging -d`,
);
expect(exitCode).toBe(1);

expect(stdout).not.toContain('PRIVATE_FILE_CONTENT_CHECK');
expect(exitCode).toBe(1);
});
});
2 changes: 1 addition & 1 deletion test/jest/acceptance/iac/iac-entitlement.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ describe('iac test with infrastructureAsCode entitlement', () => {
const { stdout, exitCode } = await run(
`snyk iac test --org=no-iac-entitlements ./iac/terraform/sg_open_ssh.tf`,
);
expect(exitCode).toBe(2);
expect(stdout).toContain(
'This feature is currently not enabled for your org. To enable it, please contact snyk support.',
);
expect(exitCode).toBe(2);
});
});
27 changes: 0 additions & 27 deletions test/jest/acceptance/iac/report.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,32 +23,24 @@ describe('iac report', () => {
afterAll(async () => teardown());

it('should return exit code 1', async () => {
// Act
const { exitCode } = await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
expect(exitCode).toEqual(1);
});

it('should include test results in the output', async () => {
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

expect(stdout).toContain('Infrastructure as code issues:');
});

it('should include a link to the projects page in the output', async () => {
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

expect(stdout).toContain(
`Your test results are available at: ${API_HOST}/org/test-org/projects under the name snyk/cli`,
);
});

it('should forward the scan results to the /iac-cli-share-results endpoint', async () => {
// Act
await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));
Expand Down Expand Up @@ -78,26 +70,16 @@ describe('iac report', () => {

describe("when called without the 'iacCliShareResults' feature flag", () => {
it('should return an error status code', async () => {
// Arrange
server.setFeatureFlag('iacCliShareResults', false);

// Act
const { exitCode } = await run(
`snyk iac report ./iac/arm/rule_test.json`,
);

// Assert
expect(exitCode).toBe(2);
});

it('should print an appropriate error message', async () => {
// Arrange
server.setFeatureFlag('iacCliShareResults', false);

// Act
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
expect(stdout).toMatch(
"Feature flag 'iacCliShareResults' is not currently enabled for your org, to enable please contact snyk support",
);
Expand All @@ -106,28 +88,19 @@ describe('iac report', () => {

describe("when called without a preceding 'iac'", () => {
it('should return an error status code', async () => {
// Act
const { exitCode } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(exitCode).toBe(2);
});

it('should print an appropriate error message', async () => {
// Act
const { stdout } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(stdout).toContain(
'"report" is not a supported command. Did you mean to use "iac report"?',
);
});

it('should return an empty stderr', async () => {
// Act
const { stderr } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(stderr).toEqual('');
});
});
Expand Down
Loading

0 comments on commit 4cd5f66

Please sign in to comment.