From 018fac43fd3f09690399a21385687e32303ddd1e Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Thu, 4 Mar 2021 15:36:58 -0800 Subject: [PATCH 1/3] bug #333: include instanceId as first column in client audit csv export --- lib/data/client-audits.js | 19 +++++++++++++++---- lib/model/query/client-audits.js | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/data/client-audits.js b/lib/data/client-audits.js index 440f14e07..f0fde6c80 100644 --- a/lib/data/client-audits.js +++ b/lib/data/client-audits.js @@ -59,8 +59,10 @@ const parseClientAudits = (buffer) => { }; // helper for streamClientAudits below. -const formatRow = (row) => { +const formatRow = (row, instanceId) => { const out = []; + // prepend instance id (not part of ClientAudit headers, but extracted nearby) + out.push(instanceId); for (const header of headers) out.push(row[header]); return out; }; @@ -73,6 +75,9 @@ const streamClientAudits = (inStream, form) => { const csvifier = new Transform({ objectMode: true, transform(x, _, done) { + // data here contains ClientAudit attchement info as well as associated + // submission instanceId fetched through query in + // model/query/client-audits.js const data = x.row; // TODO: we do not currently try/catch this block because it feels low risk. @@ -80,19 +85,25 @@ const streamClientAudits = (inStream, form) => { if (first === true) { archive.append(outStream, { name: sanitize(`${form.xmlFormId} - audit.csv`) }); // eslint-disable-line no-use-before-define archive.finalize(); - this.push(headers); + // include an initial column in aggregated audit csv for instanceId called + // "instance ID" to match Briefcase export + this.push(['instance ID', ...headers]); first = false; } if (data.content != null) { + // parse the individual audit events out of the blob of this ClientAudit + // and link each one its submission instanceId parseClientAudits(data.content) .then((rows) => { - for (const row of rows) this.push(formatRow(row)); + for (const row of rows) this.push(formatRow(row, data.instanceId)); done(); }) .catch(done); } else { - done(null, formatRow(data)); + // client audit events may already be available in this table + // and not stored in blob content. handle this case, too. + done(null, formatRow(data, data.instanceId)); } }, flush(done) { archive.finalize(); // finalize without attaching a zip if no rows came back. diff --git a/lib/model/query/client-audits.js b/lib/model/query/client-audits.js index 6680a1be5..aab99dd15 100644 --- a/lib/model/query/client-audits.js +++ b/lib/model/query/client-audits.js @@ -19,9 +19,9 @@ const existsForBlob = (blobId) => ({ maybeOne }) => .then((x) => x.isDefined()); const streamForExport = (formId, draft, options = QueryOptions.none) => ({ stream }) => stream(sql` -select client_audits.*, blobs.content from submission_defs +select client_audits.*, blobs.content, submissions."instanceId" from submission_defs inner join - (select id, "submitterId", "createdAt" from submissions + (select id, "submitterId", "createdAt", "instanceId" from submissions where "formId"=${formId} and draft=${draft} and "deletedAt" is null) as submissions on submissions.id=submission_defs."submissionId" inner join From 4887282c2490781dad35704b343f4a49c49b607b Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Mon, 8 Mar 2021 10:35:06 -0800 Subject: [PATCH 2/3] bug #333: updating client audit tests to reflect adding instance ID --- test/integration/api/submissions.js | 66 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/test/integration/api/submissions.js b/test/integration/api/submissions.js index ae0d42486..cbb001a42 100644 --- a/test/integration/api/submissions.js +++ b/test/integration/api/submissions.js @@ -1246,15 +1246,15 @@ describe('api: /forms/:id/submissions', () => { 'audits - audit.csv' ]); - result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value -a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb -b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd -c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff -d,/data/d,2000-01-01T00:10,,10,11,12,gg, -e,/data/e,2000-01-01T00:11,,,,,hh,ii -f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb -g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd -h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb +one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd +one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff +one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, +one,e,/data/e,2000-01-01T00:11,,,,,hh,ii +two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb +two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd +two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); done(); @@ -1287,21 +1287,21 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff 'audits - audit.csv' ]); - result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value -a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb -b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd -c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff -d,/data/d,2000-01-01T00:10,,10,11,12,gg, -e,/data/e,2000-01-01T00:11,,,,,hh,ii -f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb -g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd -h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb +one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd +one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff +one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, +one,e,/data/e,2000-01-01T00:11,,,,,hh,ii +two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb +two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd +two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); done(); })))))); - it('should return adhoc-processed consolidated client audit log attachments', testService((service, container) => + it('should return consolidated client audit log filtered by user', testService((service, container) => service.login('alice', (asAlice) => service.login('bob', (asBob) => asAlice.post('/v1/projects/1/forms?publish=true') @@ -1326,12 +1326,12 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff 'audits - audit.csv' ]); - result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value -a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb -b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd -c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff -d,/data/d,2000-01-01T00:10,,10,11,12,gg, -e,/data/e,2000-01-01T00:11,,,,,hh,ii + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb +one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd +one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff +one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, +one,e,/data/e,2000-01-01T00:11,,,,,hh,ii `); done(); @@ -1363,10 +1363,10 @@ e,/data/e,2000-01-01T00:11,,,,,hh,ii 'audits - audit.csv' ]); - result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value -f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb -g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd -h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb +one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd +one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); done(); @@ -1397,10 +1397,10 @@ h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff 'audits - audit.csv' ]); - result['audits - audit.csv'].should.equal(`event,node,start,end,latitude,longitude,accuracy,old-value,new-value -f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb -g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd -h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb +one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd +one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); done(); From dc9edc1928cd9a47cb83532372f1834caa7c8b4c Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Thu, 11 Mar 2021 11:23:53 -0800 Subject: [PATCH 3/3] bug #333: added client audit test related to deprecatedIds --- test/integration/api/submissions.js | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/integration/api/submissions.js b/test/integration/api/submissions.js index cbb001a42..a78adb9f9 100644 --- a/test/integration/api/submissions.js +++ b/test/integration/api/submissions.js @@ -1405,6 +1405,46 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff done(); }))))))); + + context('versioning', () => { + const withClientAuditIds = (deprecatedId, instanceId) => testData.instances.clientAudits.one + .replace('one${deprecatedId} + service.login('alice', (asAlice) => + asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200) + .then(() => asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201)) + .then(() => asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(withClientAuditIds('one', 'two')), { filename: 'data.xml' }) + .expect(201)) + .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') + .expect(200) + .then(() => new Promise((done) => + zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + result.filenames.should.containDeep([ + 'audits.csv', + 'media/audit.csv', + 'audits - audit.csv' + ]); + + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb +one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd +one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff +`); + + done(); + }))))))); + }); }); describe('.csv GET', () => {