Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update unit test suite so it validates junit output #146

Merged
merged 3 commits into from
Oct 4, 2020
Merged

Update unit test suite so it validates junit output #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
26a5549
Update unit test suite so it validates junit output
palmerj3 Oct 4, 2020
a7cce22
Clear jsonResults after each test
palmerj3 Oct 4, 2020
dc1e274
Remove jest 22 and 23 testing since we need setupFilesAfterEnv
palmerj3 Oct 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ node_js:
- "13"
- "14"
env:
- JEST_VERSION=^22.0.0
- JEST_VERSION=^23.0.0
- JEST_VERSION=^24.0.0
- JEST_VERSION=^25.0.0
- JEST_VERSION=^26.0.0
Expand Down
78 changes: 49 additions & 29 deletions __tests__/buildJsonResults.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,46 @@
const buildJsonResults = require('../utils/buildJsonResults');
const constants = require('../constants/index');

let jsonResults;
let ignoreJunitErrors = false;

describe('buildJsonResults', () => {
afterEach(() => {
if (ignoreJunitErrors !== true) {
// Verify each tests JSON output results in a
// compliant junit.xml file based on __tests__/lib/junit.xsd (jenkins xsd)
expect(jsonResults).toBeCompliantJUnit();
}

// Reset ignoreJunitErrors
ignoreJunitErrors = false;
jsonResults = undefined;
});

it('should contain number of tests in testSuite', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);
jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);

expect(jsonResults.testsuites[1].testsuite[0]._attr.tests).toBe(1);
});

it('should contain number of tests in testSuites', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);
jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);

expect(jsonResults.testsuites[0]._attr.tests).toBe(1);
});

it('should return the proper name from ancestorTitles when usePathForSuiteName is "false"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);
jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);

expect(jsonResults.testsuites[1].testsuite[0]._attr.name).toBe('foo');
});

it('should return the proper filename when suiteNameTemplate is "{filename}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
suiteNameTemplate: "{filename}"
}));
Expand All @@ -36,7 +51,7 @@ describe('buildJsonResults', () => {

it('should support suiteNameTemplate as function', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
suiteNameTemplate: (vars) => {
return 'function called with vars: ' + Object.keys(vars).join(', ');
Expand All @@ -48,7 +63,7 @@ describe('buildJsonResults', () => {

it('should return the proper classname when classNameTemplate is "{classname}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{classname}"
}));
Expand All @@ -58,7 +73,7 @@ describe('buildJsonResults', () => {

it('should return the proper title when classNameTemplate is "{title}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{title}"
}));
Expand All @@ -68,7 +83,7 @@ describe('buildJsonResults', () => {

it('should return the proper filepath when classNameTemplate is "{filepath}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{filepath}"
}));
Expand All @@ -78,7 +93,7 @@ describe('buildJsonResults', () => {

it('should return the proper filename when classNameTemplate is "{filename}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{filename}"
}));
Expand All @@ -89,7 +104,7 @@ describe('buildJsonResults', () => {
it('should return the proper displayName when classNameTemplate is {displayName}', () => {
const multiProjectNoFailingTestsReport = require('../__mocks__/multi-project-no-failing-tests.json');

const jsonResults = buildJsonResults(multiProjectNoFailingTestsReport, '/',
jsonResults = buildJsonResults(multiProjectNoFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{displayName}"
}));
Expand All @@ -99,7 +114,7 @@ describe('buildJsonResults', () => {

it('should return the proper suitename when classNameTemplate is "{suitename}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: "{suitename}"
}));
Expand All @@ -109,7 +124,7 @@ describe('buildJsonResults', () => {

it('should support return the function result when classNameTemplate is a function', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
classNameTemplate: (vars) => {
return 'function called with vars: ' + Object.keys(vars).join(', ');
Expand All @@ -121,7 +136,7 @@ describe('buildJsonResults', () => {

it('should return the proper filepath when titleTemplate is "{filepath}"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
titleTemplate: "{filepath}"
}));
Expand All @@ -130,7 +145,7 @@ describe('buildJsonResults', () => {

it('should return the proper filepath when suiteNameTemplate is "{filepath}" and usePathForSuiteName is "false"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
suiteNameTemplate: "{filepath}"
}));
Expand All @@ -139,7 +154,7 @@ describe('buildJsonResults', () => {

it('should return the proper name from ancestorTitles when suiteNameTemplate is set to "{title}" and usePathForSuiteName is "true"', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
usePathForSuiteName: "true"
}));
Expand All @@ -148,7 +163,7 @@ describe('buildJsonResults', () => {

it('should return the proper name from testFilePath when usePathForSuiteName is "true"; no appDirectory set', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
usePathForSuiteName: "true"
}));
Expand All @@ -157,7 +172,7 @@ describe('buildJsonResults', () => {

it('should return the proper name from testFilePath when usePathForSuiteName is "true"; with appDirectory set', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/path/to/test',
jsonResults = buildJsonResults(noFailingTestsReport, '/path/to/test',
Object.assign({}, constants.DEFAULT_OPTIONS, {
usePathForSuiteName: "true"
}));
Expand All @@ -166,14 +181,14 @@ describe('buildJsonResults', () => {

it('should return the proper classname when ancestorSeparator is default', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS));
expect(jsonResults.testsuites[1].testsuite[2].testcase[0]._attr.classname).toBe('foo baz should bar');
});

it('should return the proper classname when ancestorSeparator is customized', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
ancestorSeparator: " › "
}));
Expand All @@ -182,7 +197,7 @@ describe('buildJsonResults', () => {

it('should parse failure messages for failing tests', () => {
const failingTestsReport = require('../__mocks__/failing-tests.json');
const jsonResults = buildJsonResults(failingTestsReport, '/path/to/test', constants.DEFAULT_OPTIONS);
jsonResults = buildJsonResults(failingTestsReport, '/path/to/test', constants.DEFAULT_OPTIONS);

const failureMsg = jsonResults.testsuites[1].testsuite[2].testcase[1].failure;

Expand All @@ -198,7 +213,7 @@ describe('buildJsonResults', () => {
const startDate = new Date(multiProjectNoFailingTestsReport.startTime);
spyOn(Date, 'now').and.returnValue(startDate.getTime() + 1234);

const jsonResults = buildJsonResults(multiProjectNoFailingTestsReport, '/',
jsonResults = buildJsonResults(multiProjectNoFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
suiteNameTemplate: "{displayName}-foo",
titleTemplate: "{displayName}-foo"
Expand All @@ -209,13 +224,18 @@ describe('buildJsonResults', () => {

it('should not return the file name by default', () => {
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);
jsonResults = buildJsonResults(noFailingTestsReport, '/', constants.DEFAULT_OPTIONS);
expect(jsonResults.testsuites[1].testsuite[2].testcase[0]._attr.file).toBe(undefined);
});

it('should return the file name when addFileAttribute is "true"', () => {
// Ignore junit errors for this attribute
// It is added for circle-ci and is known to not generate
// jenkins-compatible junit
ignoreJunitErrors = true;

const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
const jsonResults = buildJsonResults(noFailingTestsReport, '/',
jsonResults = buildJsonResults(noFailingTestsReport, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
addFileAttribute: "true"
}));
Expand All @@ -224,17 +244,17 @@ describe('buildJsonResults', () => {

it('should show output of console if includeConsoleOutput is true', () => {
const reportWithConsoleOutput = require('../__mocks__/test-with-console-output.json');
const jsonResults = buildJsonResults(reportWithConsoleOutput, '/',
jsonResults = buildJsonResults(reportWithConsoleOutput, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
includeConsoleOutput: "true"
}));

expect(jsonResults.testsuites[1].testsuite[1]['system-out']).toBeDefined();
expect(jsonResults.testsuites[1].testsuite[3]['system-out']).toBeDefined();
});

it('should NOT show output of console if includeConsoleOutput is not set or false', () => {
const reportWithConsoleOutput = require('../__mocks__/test-with-console-output.json');
const jsonResults = buildJsonResults(reportWithConsoleOutput, '/',
jsonResults = buildJsonResults(reportWithConsoleOutput, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
includeConsoleOutput: "false"
}));
Expand All @@ -244,17 +264,17 @@ describe('buildJsonResults', () => {

it('should show short console output if includeShortConsoleOutput is true', () => {
const reportWithShortConsoleOutput = require('../__mocks__/test-with-console-output.json');
const jsonResults = buildJsonResults(reportWithShortConsoleOutput, '/',
jsonResults = buildJsonResults(reportWithShortConsoleOutput, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
includeShortConsoleOutput: "true"
}));

expect(jsonResults.testsuites[1].testsuite[1]['system-out']._cdata).toEqual("[\n \"I am bar\",\n \"Some output here from a lib\"\n]");
expect(jsonResults.testsuites[1].testsuite[3]['system-out']._cdata).toEqual("[\n \"I am bar\",\n \"Some output here from a lib\"\n]");
});

it('should NOT show short console output if includeShortConsoleOutput is not set or false', () => {
const reportWithShortConsoleOutput = require('../__mocks__/test-with-console-output.json');
const jsonResults = buildJsonResults(reportWithShortConsoleOutput, '/',
jsonResults = buildJsonResults(reportWithShortConsoleOutput, '/',
Object.assign({}, constants.DEFAULT_OPTIONS, {
includeShortConsoleOutput: "false"
}));
Expand Down
8 changes: 4 additions & 4 deletions __tests__/lib/setupTests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ const schemaPath = path.join(__dirname, 'junit.xsd');
const schemaStr = fs.readFileSync(schemaPath);
const schema = libxmljs.parseXmlString(schemaStr);

expect.extend({
toConvertToXmlAndPassXsd(jsonResults) {
global.expect.extend({
toBeCompliantJUnit(jsonResults) {
const xmlStr = xml(jsonResults, { indent: ' '});

const libxmljsObj = libxmljs.parseXmlString(xmlStr);
const isValid = libxmljsObj.validate(schema);

if (!isValid) {
return {
message: () => libxmljsObj.validationErrors.join('\n'),
message: () => `${libxmljsObj.validationErrors.join('\n')}\n${xmlStr}`,
pass: false
}
} else {
Expand All @@ -26,4 +26,4 @@ expect.extend({
}
}
}
});
});
4 changes: 2 additions & 2 deletions jest.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ module.exports = {
'<rootDir>/node_modules/',
'<rootDir>/__tests__/lib'
],
testEnvironment: 'node'
testEnvironment: 'node',
setupFilesAfterEnv: ["<rootDir>/__tests__/lib/setupTests.js"]
},
"<rootDir>/integration-tests/testResultsProcessor/",
"<rootDir>/integration-tests/reporter/"
],
setupFilesAfterEnv: ["<rootDir>/__tests__/lib/setupTests.js"],
reporters: ['default', '.']
};
56 changes: 28 additions & 28 deletions utils/buildJsonResults.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,34 +105,6 @@ module.exports = function (report, appDirectory, options) {
jsonResults.testsuites[0]._attr.failures += suite.numFailingTests;
jsonResults.testsuites[0]._attr.tests += suiteNumTests;

// Write stdout console output if available
if (options.includeConsoleOutput === 'true' && suite.console && suite.console.length) {
// Stringify the entire console object
// Easier this way because formatting in a readable way is tough with XML
// And this can be parsed more easily
let testSuiteConsole = {
'system-out': {
_cdata: JSON.stringify(suite.console, null, 2)
}
};

testSuite.testsuite.push(testSuiteConsole);
}

// Write short stdout console output if available
if (options.includeShortConsoleOutput === 'true' && suite.console && suite.console.length) {
// Extract and then Stringify the console message value
// Easier this way because formatting in a readable way is tough with XML
// And this can be parsed more easily
let testSuiteConsole = {
'system-out': {
_cdata: JSON.stringify(suite.console.map(item => item.message), null, 2)
}
};

testSuite.testsuite.push(testSuiteConsole);
}

if (!ignoreSuitePropertiesCheck) {
let junitSuiteProperties = require(junitSuitePropertiesFilePath)(suite);

Expand Down Expand Up @@ -206,6 +178,34 @@ module.exports = function (report, appDirectory, options) {
testSuite.testsuite.push(testCase);
});

// Write stdout console output if available
if (options.includeConsoleOutput === 'true' && suite.console && suite.console.length) {
// Stringify the entire console object
// Easier this way because formatting in a readable way is tough with XML
// And this can be parsed more easily
let testSuiteConsole = {
'system-out': {
_cdata: JSON.stringify(suite.console, null, 2)
}
};

testSuite.testsuite.push(testSuiteConsole);
}

// Write short stdout console output if available
if (options.includeShortConsoleOutput === 'true' && suite.console && suite.console.length) {
// Extract and then Stringify the console message value
// Easier this way because formatting in a readable way is tough with XML
// And this can be parsed more easily
let testSuiteConsole = {
'system-out': {
_cdata: JSON.stringify(suite.console.map(item => item.message), null, 2)
}
};

testSuite.testsuite.push(testSuiteConsole);
}

jsonResults.testsuites.push(testSuite);
});

Expand Down