Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: get stdout failure text before exit code #3097

Merged
merged 1 commit into from
Apr 5, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 9 additions & 18 deletions test/jest/acceptance/iac/cli-share-results.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ describe('CLI Share Results', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);
expect(exitCode).toBe(2);

expect(stdout).toMatch(
'Flag "--report" is only supported if feature flag "iacCliShareResults" is enabled. To enable it, please contact Snyk support.',
);
expect(exitCode).toBe(2);
});
});

Expand All @@ -49,20 +49,18 @@ describe('CLI Share Results', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);
expect(exitCode).toBe(1);

expect(stdout).toContain(
`Your test results are available at: http://localhost:${server.getPort()}/org/test-org/projects under the name snyk/cli`,
);
expect(exitCode).toBe(1);
});

it('forwards value to iac-cli-share-results endpoint', async () => {
const { exitCode } = await run(
`snyk iac test ./iac/arm/rule_test.json --report`,
);

expect(exitCode).toEqual(1);

const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));
Expand All @@ -88,15 +86,14 @@ describe('CLI Share Results', () => {
],
}),
);
expect(exitCode).toEqual(1);
});

it('forwards project tags', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --tags=foo=bar',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -108,15 +105,14 @@ describe('CLI Share Results', () => {
expect(request.body).toMatchObject({
tags: [{ key: 'foo', value: 'bar' }],
});
expect(exitCode).toEqual(1);
});

it('forwards project environment', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-environment=saas',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -130,15 +126,14 @@ describe('CLI Share Results', () => {
environment: ['saas'],
},
});
expect(exitCode).toEqual(1);
});

it('forwards project lifecycle', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-lifecycle=sandbox',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));
Expand All @@ -152,28 +147,26 @@ describe('CLI Share Results', () => {
lifecycle: ['sandbox'],
},
});
expect(exitCode).toEqual(1);
});

it('forwards project business criticality', async () => {
const { exitCode } = await run(
'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high',
);

expect(exitCode).toEqual(1);

const requests = server
.getRequests()
.filter((request) => request.url.includes('/iac-cli-share-results'));

expect(requests.length).toEqual(1);

const [request] = requests;

expect(request.body).toMatchObject({
attributes: {
criticality: ['high'],
},
});
expect(exitCode).toEqual(1);
});

it('should print an error message if test usage is exceeded', async () => {
Expand All @@ -183,8 +176,8 @@ describe('CLI Share Results', () => {
'snyk iac test ./iac/arm/rule_test.json --report --project-business-criticality=high',
);

expect(exitCode).toEqual(2);
expect(stdout).toMatch(/test limit reached/i);
expect(exitCode).toEqual(2);
});

describe('with target reference', () => {
Expand All @@ -195,12 +188,9 @@ describe('CLI Share Results', () => {
`snyk iac test ./iac/arm/rule_test.json --report --target-reference=${testTargetRef}`,
);

expect(exitCode).toEqual(1);

const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));

expect(testRequests[0]).toMatchObject({
body: {
contributors: expect.any(Array),
Expand All @@ -221,6 +211,7 @@ describe('CLI Share Results', () => {
],
},
});
expect(exitCode).toEqual(1);
});
});
});
Expand Down
25 changes: 10 additions & 15 deletions test/jest/acceptance/iac/custom-rules.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ describe('iac test --rules', () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);
expect(exitCode).toBe(1);

expect(stdout).toContain(
'Using custom rules to generate misconfigurations.',
Expand All @@ -33,39 +32,37 @@ describe('iac test --rules', () => {
expect(stdout).toContain(
'introduced by input > resource > aws_security_group[allow_ssh] > tags',
);
expect(exitCode).toBe(1);
});

it('does not show custom rules message if it scans local custom rules and uses --json flag', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf --json`,
);
expect(exitCode).toBe(1);

expect(stdout).not.toContain(
'Using custom rules to generate misconfigurations.',
);
expect(exitCode).toBe(1);
});

it('presents an error message when the local rules cannot be found', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --rules=./not/a/real/path.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'We were unable to extract the rules provided at: ./not/a/real/path.tar.gz',
);
expect(exitCode).toBe(2);
});

it('presents an error message when the user is not entitled to custom-rules', async () => {
const { stdout, exitCode } = await run(
`snyk iac test --org=no-custom-rules-entitlements --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
`Flag "--rules" is currently not supported for this org. To enable it, please contact snyk support.`,
);
expect(exitCode).toBe(2);
});

describe.each([
Expand All @@ -76,9 +73,8 @@ describe('iac test --rules', () => {
const { stderr, exitCode } = await run(
`snyk iac ${testedCommand} --rules=./iac/custom-rules/custom.tar.gz ./iac/terraform/sg_open_ssh.tf`,
);

expect(exitCode).toEqual(1);
expect(stderr).toEqual('');
expect(exitCode).toEqual(1);
});

it('should display a message informing of the application of custom rules', async () => {
Expand Down Expand Up @@ -219,7 +215,6 @@ describe('custom rules pull from a remote OCI registry', () => {
expect(SNYK_CFG_OCI_REGISTRY_URL).toBeDefined();
expect(SNYK_CFG_OCI_REGISTRY_USERNAME).toBeDefined();
expect(SNYK_CFG_OCI_REGISTRY_PASSWORD).toBeDefined();
expect(exitCode).toBe(1);

expect(stdout).toContain(
'Using custom rules to generate misconfigurations.',
Expand All @@ -231,6 +226,7 @@ describe('custom rules pull from a remote OCI registry', () => {
expect(stdout).toContain(
'introduced by input > resource > aws_security_group[allow_ssh] > tags',
);
expect(exitCode).toBe(1);
});

describe.each([
Expand All @@ -246,9 +242,8 @@ describe('custom rules pull from a remote OCI registry', () => {
SNYK_CFG_OCI_REGISTRY_PASSWORD: SNYK_CFG_OCI_REGISTRY_PASSWORD as string,
},
);

expect(exitCode).toEqual(1);
expect(stderr).toContain('');
expect(exitCode).toEqual(1);
});

it('should display a message informing of the application of custom rules', async () => {
Expand Down Expand Up @@ -313,10 +308,10 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'There was an authentication error. Incorrect credentials provided.',
);
expect(exitCode).toBe(2);
});

it('presents an error message when there is a local and remote rules conflict', async () => {
Expand All @@ -328,10 +323,10 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
'Remote and local custom rules bundle can not be used at the same time.',
);
expect(exitCode).toBe(2);
});

it('presents an error message when the user is not entitled to custom-rules', async () => {
Expand All @@ -346,9 +341,9 @@ describe('custom rules pull from a remote OCI registry', () => {
},
);

expect(exitCode).toBe(2);
expect(stdout).toContain(
`The custom rules feature is currently not supported for this org. To enable it, please contact snyk support.`,
);
expect(exitCode).toBe(2);
});
});
3 changes: 1 addition & 2 deletions test/jest/acceptance/iac/file-output.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ describe('iac test --sarif-file-output', () => {
const { stdout, exitCode } = await run(
`snyk iac test ./iac/file-logging -d`,
);
expect(exitCode).toBe(1);

expect(stdout).not.toContain('PRIVATE_FILE_CONTENT_CHECK');
expect(exitCode).toBe(1);
});
});
2 changes: 1 addition & 1 deletion test/jest/acceptance/iac/iac-entitlement.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ describe('iac test with infrastructureAsCode entitlement', () => {
const { stdout, exitCode } = await run(
`snyk iac test --org=no-iac-entitlements ./iac/terraform/sg_open_ssh.tf`,
);
expect(exitCode).toBe(2);
expect(stdout).toContain(
'This feature is currently not enabled for your org. To enable it, please contact snyk support.',
);
expect(exitCode).toBe(2);
});
});
27 changes: 0 additions & 27 deletions test/jest/acceptance/iac/report.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,32 +23,24 @@ describe('iac report', () => {
afterAll(async () => teardown());

it('should return exit code 1', async () => {
// Act
const { exitCode } = await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
expect(exitCode).toEqual(1);
});

it('should include test results in the output', async () => {
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

expect(stdout).toContain('Infrastructure as code issues:');
});

it('should include a link to the projects page in the output', async () => {
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

expect(stdout).toContain(
`Your test results are available at: ${API_HOST}/org/test-org/projects under the name snyk/cli`,
);
});

it('should forward the scan results to the /iac-cli-share-results endpoint', async () => {
// Act
await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
const testRequests = server
.getRequests()
.filter((request) => request.url?.includes('/iac-cli-share-results'));
Expand Down Expand Up @@ -78,26 +70,16 @@ describe('iac report', () => {

describe("when called without the 'iacCliShareResults' feature flag", () => {
it('should return an error status code', async () => {
// Arrange
server.setFeatureFlag('iacCliShareResults', false);

// Act
const { exitCode } = await run(
`snyk iac report ./iac/arm/rule_test.json`,
);

// Assert
expect(exitCode).toBe(2);
});

it('should print an appropriate error message', async () => {
// Arrange
server.setFeatureFlag('iacCliShareResults', false);

// Act
const { stdout } = await run(`snyk iac report ./iac/arm/rule_test.json`);

// Assert
expect(stdout).toMatch(
"Feature flag 'iacCliShareResults' is not currently enabled for your org, to enable please contact snyk support",
);
Expand All @@ -106,28 +88,19 @@ describe('iac report', () => {

describe("when called without a preceding 'iac'", () => {
it('should return an error status code', async () => {
// Act
const { exitCode } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(exitCode).toBe(2);
});

it('should print an appropriate error message', async () => {
// Act
const { stdout } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(stdout).toContain(
'"report" is not a supported command. Did you mean to use "iac report"?',
);
});

it('should return an empty stderr', async () => {
// Act
const { stderr } = await run(`snyk report ./iac/arm/rule_test.json`);

// Assert
expect(stderr).toEqual('');
});
});
Expand Down
Loading