First sketch for custom data threaded through item processing

Refs: #2334
eikek · Nov 13, 2023 · 83ad2c5 · 83ad2c5
1 parent fe72fbe
commit 83ad2c5
Show file tree

Hide file tree

Showing 12 changed files with 50 additions and 19 deletions.
diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala
@@ -15,10 +15,10 @@ import docspell.common.ProcessItemArgs.ProcessMeta
 import docspell.common.{CollectiveId, Ident, Language}
 import docspell.logging.Logger
 
-import io.circe.Codec
 import io.circe.generic.extras.Configuration
 import io.circe.generic.extras.semiauto.deriveConfiguredCodec
 import io.circe.generic.semiauto.deriveCodec
+import io.circe.{Codec, Json}
 
 case class NewFile(metadata: Meta = Meta.empty, file: String) {
 
@@ -41,7 +41,8 @@ object NewFile {
   case class Meta(
       language: Option[Language],
       skipDuplicate: Option[Boolean],
-      attachmentsOnly: Option[Boolean]
+      attachmentsOnly: Option[Boolean],
+      customData: Option[Json]
   ) {
 
     def toProcessMeta(
@@ -62,12 +63,13 @@ object NewFile {
         fileFilter = None,
         tags = None,
         reprocess = false,
-        attachmentsOnly = attachmentsOnly
+        attachmentsOnly = attachmentsOnly,
+        customData = customData
       )
   }
 
   object Meta {
-    val empty = Meta(None, None, None)
+    val empty = Meta(None, None, None, None)
     implicit val jsonCodec: Codec[Meta] = deriveCodec
   }
 

diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala
@@ -15,7 +15,7 @@ import docspell.common._
 import docspell.logging.Logger
 
 import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import io.circe.{Decoder, Encoder}
+import io.circe.{Decoder, Encoder, Json}
 
 case class NewItem(metadata: Option[Meta], files: List[String]) {
 
@@ -25,7 +25,7 @@ case class NewItem(metadata: Option[Meta], files: List[String]) {
       sourceAbbrev: String
   ): ProcessItemArgs.ProcessMeta =
     metadata
-      .getOrElse(Meta(None, None, None, None, None, None, None))
+      .getOrElse(Meta.empty)
       .toProcessArgs(cid, collLang, sourceAbbrev)
 
   def resolveFiles[F[_]: Files: Monad](
@@ -58,7 +58,8 @@ object NewItem {
       source: Option[String],
       skipDuplicate: Option[Boolean],
       tags: Option[List[String]],
-      attachmentsOnly: Option[Boolean]
+      attachmentsOnly: Option[Boolean],
+      customData: Option[Json]
   ) {
 
     def toProcessArgs(
@@ -78,11 +79,14 @@ object NewItem {
         fileFilter = None,
         tags = tags,
         reprocess = false,
-        attachmentsOnly = attachmentsOnly
+        attachmentsOnly = attachmentsOnly,
+        customData = customData
       )
   }
 
   object Meta {
+    val empty: Meta = Meta(None, None, None, None, None, None, None, None)
+
     implicit val jsonEncoder: Encoder[Meta] = deriveEncoder
     implicit val jsonDecoder: Decoder[Meta] = deriveDecoder
   }

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -19,6 +19,8 @@ import docspell.scheduler.{Job, JobStore}
 import docspell.store.Store
 import docspell.store.records._
 
+import io.circe.Json
+
 trait OUpload[F[_]] {
 
   def submit(
@@ -69,7 +71,8 @@ object OUpload {
       tags: List[String],
       language: Option[Language],
       attachmentsOnly: Option[Boolean],
-      flattenArchives: Option[Boolean]
+      flattenArchives: Option[Boolean],
+      customData: Option[Json]
   )
 
   case class UploadData[F[_]](
@@ -157,7 +160,8 @@ object OUpload {
             data.meta.fileFilter.some,
             data.meta.tags.some,
             false,
-            data.meta.attachmentsOnly
+            data.meta.attachmentsOnly,
+            data.meta.customData
           )
           args = ProcessItemArgs(meta, files.toList)
           jobs <- right(

diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala
@@ -54,7 +54,8 @@ object ProcessItemArgs {
       fileFilter: Option[Glob],
       tags: Option[List[String]],
       reprocess: Boolean,
-      attachmentsOnly: Option[Boolean]
+      attachmentsOnly: Option[Boolean],
+      customData: Option[Json]
   )
 
   object ProcessMeta {

diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala
@@ -75,6 +75,7 @@ object ItemAddonTask extends AddonTaskExtension {
       givenMeta = proposals,
       tags = tags.map(_.name).toList,
       classifyProposals = MetaProposalList.empty,
-      classifyTags = Nil
+      classifyTags = Nil,
+      customData = None // can't retain this information from a final item. TODO
     )
 }
diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
@@ -112,7 +112,8 @@ object CreateItem {
         MetaProposalList.empty,
         Nil,
         MetaProposalList.empty,
-        Nil
+        Nil,
+        ctx.args.meta.customData
       )
     }
 
@@ -175,7 +176,8 @@ object CreateItem {
           MetaProposalList.empty,
           Nil,
           MetaProposalList.empty,
-          Nil
+          Nil,
+          ctx.args.meta.customData
         )
       )
     }

diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala
@@ -46,7 +46,8 @@ case class ItemData(
     tags: List[String],
     // proposals obtained from the classifier
     classifyProposals: MetaProposalList,
-    classifyTags: List[String]
+    classifyTags: List[String],
+    customData: Option[Json]
 ) {
 
   /** sort by weight; order of equal weights is not important, just choose one others are
@@ -121,6 +122,7 @@ object ItemData {
             )
           )
           .asJson,
+        "customData" -> data.customData.asJson,
         "tags" -> data.tags.asJson,
         "assumedTags" -> data.classifyTags.asJson,
         "assumedCorrOrg" -> data.finalProposals

diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala
@@ -101,7 +101,8 @@ object ReProcessItem {
         MetaProposalList.empty,
         Nil,
         MetaProposalList.empty,
-        Nil
+        Nil,
+        None // cannot retain customData from an already existing item
       )).getOrElseF(
         Sync[F].raiseError(new Exception(s"Item not found: ${ctx.args.itemId.id}"))
       )
@@ -134,7 +135,8 @@ object ReProcessItem {
               None,
               None,
               true,
-              None // attachOnly (not used when reprocessing attachments)
+              None, // attachOnly (not used when reprocessing attachments)
+              None // cannot retain customData from an already existing item
             ),
             Nil
           ).pure[F]

diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -328,6 +328,7 @@ object ScanMailboxTask {
           args.tags.getOrElse(Nil),
           args.language,
           args.attachmentsOnly,
+          None,
           None
         )
         data = OUpload.UploadData(

diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -8250,6 +8250,14 @@ components:
             attachments of the e-mail are imported and the e-mail body
             is discarded. E-mails that don't have any attachments are
             skipped.
+        customData:
+          type: string
+          format: json
+          default: null
+          description: |
+            Custom user data that gets threaded through the processing. Docspell
+            ignores it completely, but will pass it to the outcome of processing
+            to be able to react on it in addons or other ways.
 
     Collective:
       description: |

diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@@ -32,6 +32,7 @@ import docspell.store.queries.{
 import docspell.store.records._
 import docspell.store.{AddResult, UpdateResult}
 
+import io.circe.Json
 import org.http4s.headers.`Content-Type`
 import org.http4s.multipart.Multipart
 import org.log4s.Logger
@@ -315,7 +316,8 @@ trait Conversions {
               m.tags.map(_.items).getOrElse(Nil),
               m.language,
               m.attachmentsOnly,
-              m.flattenArchives
+              m.flattenArchives,
+              m.customData.map(Json.fromString) // TODO fix openapi spec
             )
           )
         )
@@ -333,6 +335,7 @@ trait Conversions {
             Nil,
             None,
             None,
+            None,
             None
           )
         )

diff --git a/modules/store/src/main/scala/db/migration/common/MigrateCollectiveIdTaskArgs.scala b/modules/store/src/main/scala/db/migration/common/MigrateCollectiveIdTaskArgs.scala
@@ -348,7 +348,8 @@ object MigrateCollectiveIdTaskArgs extends TransactorSupport {
             fileFilter = oldArgs.meta.fileFilter,
             tags = oldArgs.meta.tags,
             reprocess = oldArgs.meta.reprocess,
-            attachmentsOnly = oldArgs.meta.attachmentsOnly
+            attachmentsOnly = oldArgs.meta.attachmentsOnly,
+            customData = None
           ),
           oldArgs.files.map(f =>
             ProcessItemArgs