[backend] add ability to validate draft and ingest it back through worker (#6577)

opencti-platform/opencti-front/lang/back/de.json

@@ -206,7 +206,10 @@
   "Display name": "Name anzeigen",
   "DLL service": "DLL-Dienst",
   "Do you want to delete this support package?": "Möchten Sie dieses Support-Paket löschen?",
+  "Draft change": "Entwurfsänderung",
   "Draft ids": "Entwurfs-Ids",
+  "Draft name": "Name des Entwurfs",
+  "Draft operation": "Entwurf einer Operation",
   "DST": "DST",
   "DST byte count": "DST-Byte-Anzahl",
   "DST packets": "DST-Pakete",

opencti-platform/opencti-front/lang/back/en.json

@@ -206,7 +206,10 @@
   "Display name": "Display name",
   "DLL service": "DLL service",
   "Do you want to delete this support package?": "Do you want to delete this support package?",
+  "Draft change": "Draft change",
   "Draft ids": "Draft ids",
+  "Draft name": "Draft name",
+  "Draft operation": "Draft operation",
   "DST": "DST",
   "DST byte count": "DST byte count",
   "DST packets": "DST packets",

opencti-platform/opencti-front/lang/back/es.json

@@ -206,7 +206,10 @@
   "Display name": "Mostrar nombre",
   "DLL service": "Servicio DLL",
   "Do you want to delete this support package?": "¿Desea eliminar este paquete de soporte?",
+  "Draft change": "Proyecto de cambio",
   "Draft ids": "Id. de borrador",
+  "Draft name": "Nombre del proyecto",
+  "Draft operation": "Operación borrador",
   "DST": "DST",
   "DST byte count": "DST byte count",
   "DST packets": "Paquetes DST",

opencti-platform/opencti-front/lang/back/fr.json

@@ -206,7 +206,10 @@
   "Display name": "Nom d'affichage",
   "DLL service": "Service DLL",
   "Do you want to delete this support package?": "Voulez-vous supprimer ce paquet de support ?",
+  "Draft change": "Changement en brouillon",
   "Draft ids": "Identifiants de draft",
+  "Draft name": "Nom du brouillon",
+  "Draft operation": "Opération de brouillon",
   "DST": "DST",
   "DST byte count": "Nombre d'octets DST",
   "DST packets": "Paquets DST",

opencti-platform/opencti-front/lang/back/ja.json

@@ -206,7 +206,10 @@
   "Display name": "表示名",
   "DLL service": "DLLサービス",
   "Do you want to delete this support package?": "このサポートパッケージを削除しますか？",
+  "Draft change": "ドラフト変更",
   "Draft ids": "ドラフトID",
+  "Draft name": "ドラフト名",
+  "Draft operation": "ドラフト操作",
   "DST": "DST",
   "DST byte count": "DST バイト数",
   "DST packets": "DSTパケット",

opencti-platform/opencti-front/lang/back/ko.json

@@ -206,7 +206,10 @@
   "Display name": "표시 이름",
   "DLL service": "DLL 서비스",
   "Do you want to delete this support package?": "이 지원 패키지를 삭제하시겠습니까?",
+  "Draft change": "초안 변경",
   "Draft ids": "초안 ID",
+  "Draft name": "초안 이름",
+  "Draft operation": "초안 작업",
   "DST": "DST",
   "DST byte count": "DST 바이트 수",
   "DST packets": "DST 패킷",

opencti-platform/opencti-front/lang/back/zh.json

@@ -206,7 +206,10 @@
   "Display name": "显示名称",
   "DLL service": "DLL 服务",
   "Do you want to delete this support package?": "您想删除此支持包吗？",
+  "Draft change": "修改草案",
   "Draft ids": "草案 ID",
+  "Draft name": "草案名称",
+  "Draft operation": "草案操作",
   "DST": "DST",
   "DST byte count": "DST 字节数",
   "DST packets": "DST 数据包",

opencti-platform/opencti-front/lang/front/de.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "Anreicherung",
   "INTERNAL_EXPORT_FILE": "Dateien exportieren",
   "INTERNAL_IMPORT_FILE": "Dateien importieren",
+  "INTERNAL_INGESTION": "Dateneingabe",
   "Interval": "Intervall",
   "Interval between injections (in minute)": "Intervall zwischen den Injektionen (in Minuten)",
   "Intrusion set": "Intrusion set",

opencti-platform/opencti-front/lang/front/en.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "Enrichment",
   "INTERNAL_EXPORT_FILE": "Files export",
   "INTERNAL_IMPORT_FILE": "Files import",
+  "INTERNAL_INGESTION": "Data ingestion",
   "Interval": "Interval",
   "Interval between injections (in minute)": "Interval between injections (in minute)",
   "Intrusion set": "Intrusion set",

opencti-platform/opencti-front/lang/front/es.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "Enriquecimiento",
   "INTERNAL_EXPORT_FILE": "Exportación de ficheros",
   "INTERNAL_IMPORT_FILE": "Importación de ficheros",
+  "INTERNAL_INGESTION": "Ingesta de datos",
   "Interval": "Intervalo",
   "Interval between injections (in minute)": "Intervalo entre inyecciones (en minutos)",
   "Intrusion set": "Set de intrusión",

opencti-platform/opencti-front/lang/front/fr.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "Enrichissement",
   "INTERNAL_EXPORT_FILE": "Export de fichiers",
   "INTERNAL_IMPORT_FILE": "Import de fichiers",
+  "INTERNAL_INGESTION": "L'ingestion de données",
   "Interval": "Intervalle",
   "Interval between injections (in minute)": "Intervalle entre les injections (en minute)",
   "Intrusion set": "Mode opératoire",

opencti-platform/opencti-front/lang/front/ja.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "データエンリッチメント",
   "INTERNAL_EXPORT_FILE": "ファイルエクスポート",
   "INTERNAL_IMPORT_FILE": "ファイルインポート",
+  "INTERNAL_INGESTION": "データの取り込み",
   "Interval": "インターバル",
   "Interval between injections (in minute)": "注入間隔（分）",
   "Intrusion set": "侵入セット",

opencti-platform/opencti-front/lang/front/ko.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "풍부화",
   "INTERNAL_EXPORT_FILE": "파일 내보내기",
   "INTERNAL_IMPORT_FILE": "파일 가져오기",
+  "INTERNAL_INGESTION": "데이터 수집",
   "Interval": "간격",
   "Interval between injections (in minute)": "주입 간격 (분 단위)",
   "Intrusion set": "침입 세트",

opencti-platform/opencti-front/lang/front/zh.json

@@ -1439,6 +1439,7 @@
   "INTERNAL_ENRICHMENT": "丰富",
   "INTERNAL_EXPORT_FILE": "导出文件",
   "INTERNAL_IMPORT_FILE": "导入文件",
+  "INTERNAL_INGESTION": "数据摄取",
   "Interval": "时间间隔",
   "Interval between injections (in minute)": "注射间隔时间（分钟）",
   "Intrusion set": "入侵集合",

opencti-platform/opencti-front/src/schema/relay.schema.graphql

@@ -8715,6 +8715,7 @@ type Mutation {
   supportPackageForceZip(input: SupportPackageForceZipInput!): SupportPackage
   supportPackageDelete(id: ID!): ID
   draftWorkspaceAdd(input: DraftWorkspaceAddInput!): DraftWorkspace
+  draftWorkspaceValidate(id: ID!): Work
   draftWorkspaceDelete(id: ID!): ID
 }
 

opencti-platform/opencti-front/src/utils/hooks/useAttributes.ts

@@ -4,6 +4,7 @@ import useAuth from './useAuth';
 const ignoredAttributes = [
   'id',
   'draft_ids',
+  'draft_change',
   'parent_types',
   'base_type',
   'internal_id',

opencti-platform/opencti-graphql/src/connector/connector-domain.ts

@@ -1,18 +1,23 @@
 import { importCsvConnector, importCsvConnectorRuntime } from './importCsv/importCsv-domain';
 import type { AuthContext, AuthUser } from '../types/user';
 import { ENABLED_IMPORT_CSV_BUILT_IN_CONNECTOR } from './importCsv/importCsv-configuration';
+import { DRAFT_VALIDATION_CONNECTOR, draftValidationConnectorRuntime } from '../modules/draftWorkspace/draftWorkspace-connector';
+import { isFeatureEnabled } from '../config/conf';
 
 export const builtInConnectorsRuntime = async (context: AuthContext, user: AuthUser) => {
   const builtInConnectors = [];
   if (ENABLED_IMPORT_CSV_BUILT_IN_CONNECTOR) {
     const csvConnector = await importCsvConnectorRuntime(context, user);
     builtInConnectors.push(csvConnector);
   }
+  if (isFeatureEnabled('DRAFT_WORKSPACE')) {
+    builtInConnectors.push(await draftValidationConnectorRuntime());
+  }
   return builtInConnectors;
 };
 
 export const builtInConnectors = () => {
-  return [importCsvConnector()];
+  return [importCsvConnector(), DRAFT_VALIDATION_CONNECTOR];
 };
 
 export const builtInConnector = (id: string) => {

opencti-platform/opencti-graphql/src/connector/importCsv/importCsv-domain.ts

@@ -9,7 +9,7 @@ export const importCsvConnector = () => {
 
 export const importCsvConnectorRuntime = async (context: AuthContext, user: AuthUser) => {
   const connector = importCsvConnector();
-  const configurations = await connector.connector_schema_runtime_fn(context, user);
+  const configurations = connector.connector_schema_runtime_fn ? await connector.connector_schema_runtime_fn(context, user) : [];
   const configurationsFiltered: BasicStoreEntityCsvMapper[] = [];
   await Promise.all(configurations.map(async (c) => {
     const mapperErrors = await getCsvMapperErrorMessage(context, user, c);

opencti-platform/opencti-graphql/src/connector/internalConnector.d.ts

@@ -24,5 +24,5 @@ export interface Connector {
   connector_type: string;
   name: string;
   built_in: boolean;
-  connector_schema_runtime_fn: <T extends BasicStoreEntityCsvMapper> (context: AuthContext, user: AuthUser) => Promise<T[]>;
+  connector_schema_runtime_fn?: <T extends BasicStoreEntityCsvMapper> (context: AuthContext, user: AuthUser) => Promise<T[]>;
 }

opencti-platform/opencti-graphql/src/database/draft-utils.js

@@ -3,7 +3,13 @@
 import { getDraftContext } from '../utils/draftContext';
 import { READ_INDEX_DRAFT_OBJECTS } from './utils';
 
-export const buildDraftFilter = (context, user) => {
+export const DRAFT_OPERATION_CREATE = 'create';
+export const DRAFT_OPERATION_UPDATE = 'update';
+export const DRAFT_OPERATION_DELETE = 'delete';
+export const DRAFT_OPERATION_DELETE_LINKED = 'delete_linked';
+
+export const buildDraftFilter = (context, user, opts = {}) => {
+  const { includeDeletedInDraft = false } = opts;
   const draftContext = getDraftContext(context, user);
   const draftMust = [];
   if (draftContext) {
@@ -30,10 +36,26 @@
       },
     };
     draftMust.push(draftBool);
+
+    if (!includeDeletedInDraft) {
+      const excludeDeletedDraft = {
+        bool: {
+          must_not: [
+            { terms: { 'draft_change.draft_operation.keyword': [DRAFT_OPERATION_DELETE, DRAFT_OPERATION_DELETE_LINKED] } },
+          ]
+        }
+      };
+      draftMust.push(excludeDeletedDraft);
+    }
   }
   return draftMust;
 };
 
 export const isDraftSupportedEntity = (element) => {
   return !isInternalObject(element.entity_type) && !isInternalRelationship(element.entity_type);
 };
+
+// TODO: once update metadata is better refined, add it to draft_change
+export const getDraftChanges = (initialInstance) => {
+  return initialInstance.draft_change ?? { draft_operation: DRAFT_OPERATION_UPDATE };
+};