[SIEM][Detection Engine][Lists] Adds version and immutability data st…

…ructures (elastic#72730)

###  Summary

The intent is to get the data structures in similar to rules so that we can have eventually immutable and versioned lists in later releases without too much hassle of upgrading the list and list item data structures.

* Adds version and immutability data structures to the exception lists and the value lists.
* Adds an optional version number to the update route of each so that you can modify the number either direction or you can omit it and it works like the detection rules where it will auto-increment the number.
* Does _not_ add a version and immutability to the exception list items and value list items.
* Does _not_ update the version number when you add a new exception list item or value list item. 

**Examples:**

❯ ./post_list.sh
```json
{
  "_version": "WzAsMV0=",
  "id": "ip_list",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "This list describes bad internet ip",
  "immutable": false,
  "name": "Simple list with an ip",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:11.679Z",
  "updated_by": "yo",
  "version": 1
}
```
❯ ./post_exception_list.sh
```json
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTgsMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "This is a sample endpoint type exception",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "detection",
  "updated_at": "2020-07-21T20:31:35.952Z",
  "updated_by": "yo",
  "version": 1
}
```

```json
❯ ./update_list.sh
{
  "_version": "WzEsMV0=",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "Some other description here for you",
  "id": "ip_list",
  "immutable": false,
  "name": "Changed the name here to something else",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:47.089Z",
  "updated_by": "yo",
  "version": 2
}
```

```json
❯ ./update_exception_list.sh
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTksMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "Different description",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "endpoint",
  "updated_at": "2020-07-21T20:31:56.628Z",
  "updated_by": "yo",
  "version": 2
}
```

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

x-pack/plugins/lists/common/constants.mock.ts

@@ -61,3 +61,5 @@ export const COMMENTS = [];
 export const FILTER = 'name:Nicolas Bourbaki';
 export const CURSOR = 'c29tZXN0cmluZ2ZvcnlvdQ==';
 export const _VERSION = 'WzI5NywxXQ==';
+export const VERSION = 1;
+export const IMMUTABLE = false;

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -311,3 +311,15 @@ export type DeserializerOrUndefined = t.TypeOf<typeof deserializerOrUndefined>;
 export const _version = t.string;
 export const _versionOrUndefined = t.union([_version, t.undefined]);
 export type _VersionOrUndefined = t.TypeOf<typeof _versionOrUndefined>;
+
+export const version = t.number;
+export type Version = t.TypeOf<typeof version>;
+
+export const versionOrUndefined = t.union([version, t.undefined]);
+export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;
+
+export const immutable = t.boolean;
+export type Immutable = t.TypeOf<typeof immutable>;
+
+export const immutableOrUndefined = t.union([immutable, t.undefined]);
+export type ImmutableOrUndefined = t.TypeOf<typeof immutableOrUndefined>;

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts

@@ -8,23 +8,27 @@ import { IndexEsListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   META,
   NAME,
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getIndexESListMock = (): IndexEsListSchema => ({
   created_at: DATE_NOW,
   created_by: USER,
   description: DESCRIPTION,
   deserializer: undefined,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
   tie_breaker_id: TIE_BREAKER,
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts

@@ -13,13 +13,15 @@ import {
   created_by,
   description,
   deserializerOrUndefined,
+  immutable,
   metaOrUndefined,
   name,
   serializerOrUndefined,
   tie_breaker_id,
   type,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const indexEsListSchema = t.exact(
@@ -28,13 +30,15 @@ export const indexEsListSchema = t.exact(
     created_by,
     description,
     deserializer: deserializerOrUndefined,
+    immutable,
     meta: metaOrUndefined,
     name,
     serializer: serializerOrUndefined,
     tie_breaker_id,
     type,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts

@@ -10,13 +10,15 @@ import { SearchEsListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   LIST_ID,
   LIST_INDEX,
   META,
   NAME,
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 import { getShardMock } from '../../get_shard.mock';
 
@@ -25,13 +27,15 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({
   created_by: USER,
   description: DESCRIPTION,
   deserializer: undefined,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
   tie_breaker_id: TIE_BREAKER,
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });
 
 export const getSearchListMock = (): SearchResponse<SearchEsListSchema> => ({

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts

@@ -13,13 +13,15 @@ import {
   created_by,
   description,
   deserializerOrUndefined,
+  immutable,
   metaOrUndefined,
   name,
   serializerOrUndefined,
   tie_breaker_id,
   type,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const searchEsListSchema = t.exact(
@@ -28,13 +30,15 @@ export const searchEsListSchema = t.exact(
     created_by,
     description,
     deserializer: deserializerOrUndefined,
+    immutable,
     meta: metaOrUndefined,
     name,
     serializer: serializerOrUndefined,
     tie_breaker_id,
     type,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.mock.ts

@@ -4,7 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { DESCRIPTION, ENDPOINT_TYPE, META, NAME, NAMESPACE_TYPE } from '../../constants.mock';
+import {
+  DESCRIPTION,
+  ENDPOINT_TYPE,
+  META,
+  NAME,
+  NAMESPACE_TYPE,
+  VERSION,
+} from '../../constants.mock';
 
 import { CreateExceptionListSchema } from './create_exception_list_schema';
 
@@ -17,4 +24,5 @@ export const getCreateExceptionListSchemaMock = (): CreateExceptionListSchema =>
   namespace_type: NAMESPACE_TYPE,
   tags: [],
   type: ENDPOINT_TYPE,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.ts

@@ -21,7 +21,11 @@ import {
   tags,
 } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
-import { DefaultUuid } from '../../siem_common_deps';
+import {
+  DefaultUuid,
+  DefaultVersionNumber,
+  DefaultVersionNumberDecoded,
+} from '../../siem_common_deps';
 import { NamespaceType } from '../types';
 
 export const createExceptionListSchema = t.intersection([
@@ -39,6 +43,7 @@ export const createExceptionListSchema = t.intersection([
       meta, // defaults to undefined if not set during decode
       namespace_type, // defaults to 'single' if not set during decode
       tags, // defaults to empty array if not set during decode
+      version: DefaultVersionNumber, // defaults to numerical 1 if not set during decode
     })
   ),
 ]);
@@ -54,4 +59,5 @@ export type CreateExceptionListSchemaDecoded = Omit<
   tags: Tags;
   list_id: ListId;
   namespace_type: NamespaceType;
+  version: DefaultVersionNumberDecoded;
 };

x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { DESCRIPTION, LIST_ID, META, NAME, TYPE } from '../../constants.mock';
+import { DESCRIPTION, LIST_ID, META, NAME, TYPE, VERSION } from '../../constants.mock';
 
 import { CreateListSchema } from './create_list_schema';
 
@@ -16,4 +16,5 @@ export const getCreateListSchemaMock = (): CreateListSchema => ({
   name: NAME,
   serializer: undefined,
   type: TYPE,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/request/create_list_schema.ts

@@ -8,6 +8,7 @@ import * as t from 'io-ts';
 
 import { description, deserializer, id, meta, name, serializer, type } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
+import { DefaultVersionNumber, DefaultVersionNumberDecoded } from '../../siem_common_deps';
 
 export const createListSchema = t.intersection([
   t.exact(
@@ -17,8 +18,18 @@ export const createListSchema = t.intersection([
       type,
     })
   ),
-  t.exact(t.partial({ deserializer, id, meta, serializer })),
+  t.exact(
+    t.partial({
+      deserializer, // defaults to undefined if not set during decode
+      id, // defaults to undefined if not set during decode
+      meta, // defaults to undefined if not set during decode
+      serializer, // defaults to undefined if not set during decode
+      version: DefaultVersionNumber, // defaults to a numerical 1 if not set during decode
+    })
+  ),
 ]);
 
 export type CreateListSchema = t.OutputOf<typeof createListSchema>;
-export type CreateListSchemaDecoded = RequiredKeepUndefined<t.TypeOf<typeof createListSchema>>;
+export type CreateListSchemaDecoded = RequiredKeepUndefined<
+  Omit<t.TypeOf<typeof createListSchema>, 'version'>
+> & { version: DefaultVersionNumberDecoded };

x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { _version, description, id, meta, name } from '../common/schemas';
+import { _version, description, id, meta, name, version } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 
 export const patchListSchema = t.intersection([
@@ -17,7 +17,15 @@ export const patchListSchema = t.intersection([
       id,
     })
   ),
-  t.exact(t.partial({ _version, description, meta, name })),
+  t.exact(
+    t.partial({
+      _version, // is undefined if not set during decode
+      description, // is undefined if not set during decode
+      meta, // is undefined if not set during decode
+      name, // is undefined if not set during decode
+      version, // is undefined if not set during decode
+    })
+  ),
 ]);
 
 export type PatchListSchema = t.OutputOf<typeof patchListSchema>;

x-pack/plugins/lists/common/schemas/request/update_exception_list_schema.ts

@@ -21,6 +21,7 @@ import {
   name,
   namespace_type,
   tags,
+  version,
 } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 import { NamespaceType } from '../types';
@@ -42,6 +43,7 @@ export const updateExceptionListSchema = t.intersection([
       meta, // defaults to undefined if not set during decode
       namespace_type, // defaults to 'single' if not set during decode
       tags, // defaults to empty array if not set during decode
+      version, // defaults to undefined if not set during decode
     })
   ),
 ]);

x-pack/plugins/lists/common/schemas/request/update_list_schema.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { _version, description, id, meta, name } from '../common/schemas';
+import { _version, description, id, meta, name, version } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 
 export const updateListSchema = t.intersection([
@@ -23,6 +23,7 @@ export const updateListSchema = t.intersection([
     t.partial({
       _version, // defaults to undefined if not set during decode
       meta, // defaults to undefined if not set during decode
+      version, // defaults to undefined if not set during decode
     })
   ),
 ]);

x-pack/plugins/lists/common/schemas/response/create_endpoint_list_schema.test.ts

@@ -41,7 +41,7 @@ describe('create_endpoint_list_schema', () => {
     const message = pipe(checked, foldLeftRight);
 
     expect(getPaths(left(message.errors))).toEqual([
-      'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by"',
+      'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,immutable,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by,version"',
     ]);
     expect(message.schema).toEqual({});
   });

x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts

@@ -8,9 +8,11 @@ import {
   DATE_NOW,
   DESCRIPTION,
   ENDPOINT_TYPE,
+  IMMUTABLE,
   META,
   TIE_BREAKER,
   USER,
+  VERSION,
   _VERSION,
 } from '../../constants.mock';
 import { ENDPOINT_LIST_ID } from '../..';
@@ -23,6 +25,7 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
   created_by: USER,
   description: DESCRIPTION,
   id: '1',
+  immutable: IMMUTABLE,
   list_id: ENDPOINT_LIST_ID,
   meta: META,
   name: 'Sample Endpoint Exception List',
@@ -32,4 +35,5 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
   type: ENDPOINT_TYPE,
   updated_at: DATE_NOW,
   updated_by: 'user_name',
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/response/exception_list_schema.ts

@@ -16,6 +16,7 @@ import {
   description,
   exceptionListType,
   id,
+  immutable,
   list_id,
   metaOrUndefined,
   name,
@@ -24,6 +25,7 @@ import {
   tie_breaker_id,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const exceptionListSchema = t.exact(
@@ -34,6 +36,7 @@ export const exceptionListSchema = t.exact(
     created_by,
     description,
     id,
+    immutable,
     list_id,
     meta: metaOrUndefined,
     name,
@@ -43,6 +46,7 @@ export const exceptionListSchema = t.exact(
     type: exceptionListType,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts

@@ -8,12 +8,14 @@ import { ListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   LIST_ID,
   META,
   NAME,
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
@@ -23,11 +25,13 @@ export const getListResponseMock = (): ListSchema => ({
   description: DESCRIPTION,
   deserializer: undefined,
   id: LIST_ID,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
   tie_breaker_id: TIE_BREAKER,
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });