Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

include instanceId in client audit csv export #335

Merged
merged 3 commits into from
Mar 12, 2021
Merged

include instanceId in client audit csv export #335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
018fac4
bug #333: include instanceId as first column in client audit csv export
ktuite Mar 4, 2021
4887282
bug #333: updating client audit tests to reflect adding instance ID
ktuite Mar 8, 2021
dc9edc1
bug #333: added client audit test related to deprecatedIds
ktuite Mar 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions lib/data/client-audits.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ const parseClientAudits = (buffer) => {
};

// helper for streamClientAudits below.
const formatRow = (row) => {
const formatRow = (row, instanceId) => {
const out = [];
// prepend instance id (not part of ClientAudit headers, but extracted nearby)
out.push(instanceId);
for (const header of headers) out.push(row[header]);
return out;
};
Expand All @@ -73,26 +75,35 @@ const streamClientAudits = (inStream, form) => {
const csvifier = new Transform({
objectMode: true,
transform(x, _, done) {
// data here contains ClientAudit attchement info as well as associated
// submission instanceId fetched through query in
// model/query/client-audits.js
const data = x.row;

// TODO: we do not currently try/catch this block because it feels low risk.
// this may not actually be the case..
if (first === true) {
archive.append(outStream, { name: sanitize(`${form.xmlFormId} - audit.csv`) }); // eslint-disable-line no-use-before-define
archive.finalize();
this.push(headers);
// include an initial column in aggregated audit csv for instanceId called
// "instance ID" to match Briefcase export
this.push(['instance ID', ...headers]);
first = false;
}

if (data.content != null) {
// parse the individual audit events out of the blob of this ClientAudit
// and link each one its submission instanceId
parseClientAudits(data.content)
.then((rows) => {
for (const row of rows) this.push(formatRow(row));
for (const row of rows) this.push(formatRow(row, data.instanceId));
done();
})
.catch(done);
} else {
done(null, formatRow(data));
// client audit events may already be available in this table
// and not stored in blob content. handle this case, too.
done(null, formatRow(data, data.instanceId));
}
}, flush(done) {
archive.finalize(); // finalize without attaching a zip if no rows came back.
Expand Down
4 changes: 2 additions & 2 deletions lib/model/query/client-audits.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ const existsForBlob = (blobId) => ({ maybeOne }) =>
.then((x) => x.isDefined());

const streamForExport = (formId, draft, options = QueryOptions.none) => ({ stream }) => stream(sql`
select client_audits.*, blobs.content from submission_defs
select client_audits.*, blobs.content, submissions."instanceId" from submission_defs
inner join
(select id, "submitterId", "createdAt" from submissions
(select id, "submitterId", "createdAt", "instanceId" from submissions
where "formId"=${formId} and draft=${draft} and "deletedAt" is null) as submissions
on submissions.id=submission_defs."submissionId"
inner join
Expand Down
106 changes: 73 additions & 33 deletions test/integration/api/submissions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,15 +1246,15 @@ describe('api: /forms/:id/submissions', () => {
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value
a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
d,/data/d,2000-01-01T00:10,,10,11,12,gg,
e,/data/e,2000-01-01T00:11,,,,,hh,ii
f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
one,d,/data/d,2000-01-01T00:10,,10,11,12,gg,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);

done();
Expand Down Expand Up @@ -1287,21 +1287,21 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value
a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
d,/data/d,2000-01-01T00:10,,10,11,12,gg,
e,/data/e,2000-01-01T00:11,,,,,hh,ii
f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
one,d,/data/d,2000-01-01T00:10,,10,11,12,gg,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);

done();
}))))));

it('should return adhoc-processed consolidated client audit log attachments', testService((service, container) =>
it('should return consolidated client audit log filtered by user', testService((service, container) =>
service.login('alice', (asAlice) =>
service.login('bob', (asBob) =>
asAlice.post('/v1/projects/1/forms?publish=true')
Expand All @@ -1326,12 +1326,12 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value
a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
d,/data/d,2000-01-01T00:10,,10,11,12,gg,
e,/data/e,2000-01-01T00:11,,,,,hh,ii
result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb
one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd
one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
one,d,/data/d,2000-01-01T00:10,,10,11,12,gg,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
`);

done();
Expand Down Expand Up @@ -1363,10 +1363,10 @@ e,/data/e,2000-01-01T00:11,,,,,hh,ii
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value
f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);

done();
Expand Down Expand Up @@ -1397,14 +1397,54 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value
f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);

done();
})))))));

context('versioning', () => {
const withClientAuditIds = (deprecatedId, instanceId) => testData.instances.clientAudits.one
.replace('one</instance', `${instanceId}</instanceID><deprecatedID>${deprecatedId}</deprecated`);

it('should return original instanceId and latest attached audit log when instanceId deprecated with new audit log', testService((service) =>
service.login('alice', (asAlice) =>
asAlice.post('/v1/projects/1/forms?publish=true')
.set('Content-Type', 'application/xml')
.send(testData.forms.clientAudits)
.expect(200)
.then(() => asAlice.post('/v1/projects/1/submission')
.set('X-OpenRosa-Version', '1.0')
.attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' })
.attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' })
.expect(201))
.then(() => asAlice.post('/v1/projects/1/submission')
.set('X-OpenRosa-Version', '1.0')
.attach('audit.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'audit.csv' })
.attach('xml_submission_file', Buffer.from(withClientAuditIds('one', 'two')), { filename: 'data.xml' })
.expect(201))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
.then(() => new Promise((done) =>
zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
'audits - audit.csv'
]);

result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value
one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);

done();
})))))));
});
});

describe('.csv GET', () => {
Expand Down