add unit test for createChunks

elastic · Mar 4, 2021 · 30ae3ea · 30ae3ea
1 parent b6756c8
commit 30ae3ea
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 5 deletions.
diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.test.js
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { GeoJsonImporter, toEsDoc } from './geojson_importer';
+import { GeoJsonImporter, createChunks, toEsDoc } from './geojson_importer';
 import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public';
 import '@loaders.gl/polyfills';
 
@@ -264,3 +264,36 @@ describe('toEsDoc', () => {
     });
   });
 });
+
+describe('createChunks', () => {
+  const GEOMETRY_COLLECTION_DOC_CHARS = JSON.stringify(
+    toEsDoc(GEOMETRY_COLLECTION_FEATURE, ES_FIELD_TYPES.GEO_SHAPE)
+  ).length;
+
+  const features = [
+    GEOMETRY_COLLECTION_FEATURE,
+    GEOMETRY_COLLECTION_FEATURE,
+    GEOMETRY_COLLECTION_FEATURE,
+    GEOMETRY_COLLECTION_FEATURE,
+    GEOMETRY_COLLECTION_FEATURE,
+  ];
+
+  test('should break features into chunks', () => {
+    const maxChunkCharCount = GEOMETRY_COLLECTION_DOC_CHARS * 3.5;
+    const chunks = createChunks(features, ES_FIELD_TYPES.GEO_SHAPE, maxChunkCharCount);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].length).toBe(3);
+    expect(chunks[1].length).toBe(2);
+  });
+
+  test('should break features into chunks containing only single feature when feature size is greater then maxChunkCharCount', () => {
+    const maxChunkCharCount = GEOMETRY_COLLECTION_DOC_CHARS * 0.8;
+    const chunks = createChunks(features, ES_FIELD_TYPES.GEO_SHAPE, maxChunkCharCount);
+    expect(chunks.length).toBe(5);
+    expect(chunks[0].length).toBe(1);
+    expect(chunks[1].length).toBe(1);
+    expect(chunks[2].length).toBe(1);
+    expect(chunks[3].length).toBe(1);
+    expect(chunks[4].length).toBe(1);
+  });
+});
diff --git a/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts b/x-pack/plugins/file_upload/public/importer/geojson_importer/geojson_importer.ts
@@ -120,7 +120,7 @@ export class GeoJsonImporter extends Importer {
       }
 
       // Import block in chunks to avoid sending too much data to Elasticsearch at a time.
-      const chunks = createChunks(this._features, this._geoFieldType);
+      const chunks = createChunks(this._features, this._geoFieldType, MAX_CHUNK_CHAR_COUNT);
       const blockSizeInBytes = this._blockSizeInBytes;
 
       // reset block for next read
@@ -338,9 +338,10 @@ export class GeoJsonImporter extends Importer {
   }
 }
 
-function createChunks(
+export function createChunks(
   features: Feature[],
-  geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE
+  geoFieldType: ES_FIELD_TYPES.GEO_POINT | ES_FIELD_TYPES.GEO_SHAPE,
+  maxChunkCharCount: number
 ): ImportDoc[][] {
   const chunks: ImportDoc[][] = [];
 
@@ -349,7 +350,7 @@ function createChunks(
   for (let i = 0; i < features.length; i++) {
     const doc = toEsDoc(features[i], geoFieldType);
     const docChars = JSON.stringify(doc).length + 1; // +1 adds CHAR for comma once document is in list
-    if (chunk.length === 0 || chunkChars + docChars < MAX_CHUNK_CHAR_COUNT) {
+    if (chunk.length === 0 || chunkChars + docChars < maxChunkCharCount) {
       // add ES document to current chunk
       chunk.push(doc);
       chunkChars += docChars;