diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index eb62c4d114640..d287ec58530b9 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -164,11 +164,16 @@ export class CsvGenerator { cell = '-'; } - try { - // expected values are a string of JSON where the value(s) is in an array - cell = JSON.parse(cell); - } catch (e) { - // ignore + const isIdField = tableColumn === '_id'; // _id field can not be formatted or mutated + if (!isIdField) { + try { + // unwrap the value + // expected values are a string of JSON where the value(s) is in an array + // examples: "[""Jan 1, 2020 @ 04:00:00.000""]","[""username""]" + cell = JSON.parse(cell); + } catch (e) { + // ignore + } } // We have to strip singular array values out of their array wrapper, @@ -381,6 +386,7 @@ export class CsvGenerator { break; // empty report with just the header } + // FIXME: make tabifyDocs handle the formatting, to get the same formatting logic as Discover? const formatters = this.getFormatters(table); await this.generateRows(columns, table, builder, formatters, settings); diff --git a/x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz b/x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz new file mode 100644 index 0000000000000..c42d21903c912 Binary files /dev/null and b/x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json b/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json new file mode 100644 index 0000000000000..d2ee24696e0f1 --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json @@ -0,0 +1,25 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "test_elastic", + "mappings": { + "properties": { + "timestamp": { + "format": "yyyyMMddHHmmss||yyyyMMddHHmmssZ||strict_date_optional_time||epoch_millis", + "type": "date" + }, + "message_type": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json new file mode 100644 index 0000000000000..770758f52d0d3 --- /dev/null +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json @@ -0,0 +1,96 @@ +{ + "attributes": { + "fieldAttrs": "{}", + "fieldFormatMap": "{}", + "fields": "[]", + "name": "test_elastic*", + "runtimeFieldMap": "{}", + "sourceFilters": "[]", + "timeFieldName": "timestamp", + "title": "test_elastic*", + "typeMeta": "{}" + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T20:55:46.970Z", + "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", + "migrationVersion": { + "index-pattern": "8.0.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2022-10-25T20:55:46.970Z", + "version": "WzIxOCwxXQ==" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "testsearch" + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T20:57:39.872Z", + "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-10-25T20:57:39.872Z", + "version": "WzI2MCwxXQ==" +} + +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "optionsJSON": "{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}", + "panelsJSON": "[{\"version\":\"8.6.0\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":18,\"i\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\"},\"panelIndex\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7307be50-603d-4091-b4b9-e76a96c6a33a\"}]", + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-15y", + "timeRestore": true, + "timeTo": "2022-10-30T00:00:00.000Z", + "title": "rbbaf", + "version": 1 + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T21:01:17.780Z", + "id": "b78b1350-54a7-11ed-b3f3-41d5096a3cfd", + "migrationVersion": { + "dashboard": "8.6.0" + }, + "references": [ + { + "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", + "name": "7307be50-603d-4091-b4b9-e76a96c6a33a:panel_7307be50-603d-4091-b4b9-e76a96c6a33a", + "type": "search" + } + ], + "type": "dashboard", + "updated_at": "2022-10-25T21:01:17.780Z", + "version": "WzMzNiwxXQ==" +} \ No newline at end of file diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap index 43649d0ed7552..73c7a6ef4b542 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap @@ -1,5 +1,14 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Reporting APIs CSV Generation from SearchSource _id field is a big integer passes through the value without mutation 1`] = ` +"\\"_id\\",\\"_index\\",\\"_score\\",\\"message_type\\",timestamp +202209071000000604,\\"test_elastic\\",\\"-\\",OP,\\"Jan 1, 2020 @ 11:00:00.000\\" +202209071000000605,\\"test_elastic\\",\\"-\\",OP,\\"Jan 1, 2020 @ 11:00:00.000\\" +202209071000000606,\\"test_elastic\\",\\"-\\",OP,\\"Jan 1, 2020 @ 11:00:00.000\\" +202209071000000607,\\"test_elastic\\",\\"-\\",OP,\\"Jan 1, 2020 @ 11:00:00.000\\" +" +`; + exports[`Reporting APIs CSV Generation from SearchSource date formatting With filters and timebased data, default to UTC 1`] = ` "\\"@timestamp\\",clientip,extension \\"Sep 20, 2015 @ 10:26:48.725\\",\\"74.214.76.90\\",jpg diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts index 922ff565b4e29..3941037733c70 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts @@ -394,6 +394,85 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('_id field is a big integer', () => { + before(async () => { + await Promise.all([ + esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'), + kibanaServer.importExport.load( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ), + ]); + }); + + after(async () => { + await Promise.all([ + esArchiver.unload('x-pack/test/functional/es_archives/reporting/big_int_id_field'), + kibanaServer.importExport.unload( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ), + ]); + }); + it('passes through the value without mutation', async () => { + const { text } = (await generateAPI.getCSVFromSearchSource( + getMockJobParams({ + browserTimezone: 'UTC', + version: '8.6.0', + searchSource: { + query: { query: '', language: 'kuery' }, + fields: [{ field: '*', include_unmapped: 'true' }], + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + sort: [{ timestamp: 'desc' }], + filter: [ + { + meta: { + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + params: {}, + field: 'timestamp', + }, + query: { + range: { + timestamp: { + format: 'strict_date_optional_time', + gte: '2007-10-25T21:18:23.905Z', + lte: '2022-10-30T00:00:00.000Z', + }, + }, + }, + }, + ], + parent: { + query: { query: '', language: 'kuery' }, + filter: [], + parent: { + filter: [ + { + meta: { + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + params: {}, + field: 'timestamp', + }, + query: { + range: { + timestamp: { + format: 'strict_date_optional_time', + gte: '2007-10-25T21:18:23.905Z', + lte: '2022-10-30T00:00:00.000Z', + }, + }, + }, + }, + ], + }, + }, + }, + columns: [], + title: 'testsearch', + }) + )) as supertest.Response; + expectSnapshot(text).toMatch(); + }); + }); + describe('validation', () => { it('Return a 404', async () => { const { body } = (await generateAPI.getCSVFromSearchSource(