Merge pull request #356 from nfdi4plants/datamap

Rework data nodes

src/ARCtrl/ARC.fs

@@ -15,28 +15,41 @@ module ARCAux =
         contracts 
         |> Array.choose ArcAssay.tryFromReadContract
 
+    let getAssayDataMapFromContracts (assayIdentifier) (contracts: Contract []) = 
+        contracts 
+        |> Array.tryPick (DataMap.tryFromReadContractForAssay assayIdentifier)
 
     // No idea where to move this
     let getArcStudiesFromContracts (contracts: Contract []) =
         contracts 
         |> Array.choose ArcStudy.tryFromReadContract
 
+    let getStudyDataMapFromContracts (studyIdentifier) (contracts: Contract []) = 
+        contracts 
+        |> Array.tryPick (DataMap.tryFromReadContractForStudy studyIdentifier)
+
     let getArcInvestigationFromContracts (contracts: Contract []) =
         contracts 
         |> Array.choose ArcInvestigation.tryFromReadContract
         |> Array.exactlyOne 
 
     let updateFSByISA (isa : ArcInvestigation option) (fs : FileSystem) = 
-        let (studyNames,assayNames) = 
+        let (studies,assays) = 
             match isa with
             | Some inv ->         
-                inv.StudyIdentifiers |> Seq.toArray, inv.AssayIdentifiers |> Seq.toArray
+                inv.Studies |> Seq.toArray, inv.Assays |> Seq.toArray
             | None -> ([||],[||])
-        let assays = FileSystemTree.createAssaysFolder (assayNames |> Array.map FileSystemTree.createAssayFolder)
-        let studies = FileSystemTree.createStudiesFolder (studyNames |> Array.map FileSystemTree.createStudyFolder)
+        let assaysFolder = 
+            assays
+            |> Array.map (fun a -> FileSystemTree.createAssayFolder(a.Identifier,a.DataMap.IsSome))
+            |> FileSystemTree.createAssaysFolder
+        let studiesFolder = 
+            studies
+            |> Array.map (fun s -> FileSystemTree.createStudyFolder(s.Identifier,s.DataMap.IsSome))
+            |> FileSystemTree.createStudiesFolder
         let investigation = FileSystemTree.createInvestigationFile()
         let tree = 
-            FileSystemTree.createRootFolder [|investigation;assays;studies|]
+            FileSystemTree.createRootFolder [|investigation;assaysFolder;studiesFolder|]
             |> FileSystem.create
         fs.Union(tree)
 
@@ -219,14 +232,16 @@ type ARC(?isa : ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSystem) =
                 investigation.DeleteStudy(si)
         )
 
-        studies |> Array.iter (fun study ->
+        studies |> Array.iter (fun study ->            
             /// Try find registered study in parsed READ contracts
             let registeredStudyOpt = investigation.Studies |> Seq.tryFind (fun s -> s.Identifier = study.Identifier)
             match registeredStudyOpt with
             | Some registeredStudy -> 
                 registeredStudy.UpdateReferenceByStudyFile(study,true)
             | None -> 
                 investigation.AddStudy(study)
+            let datamap = ARCAux.getAssayDataMapFromContracts study.Identifier contracts
+            study.DataMap <- datamap
             study.StaticHash <- study.GetLightHashCode()
         )
         assays |> Array.iter (fun assay ->
@@ -243,9 +258,11 @@ type ARC(?isa : ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSystem) =
                 |> Array.fold (fun tables study -> 
                     ArcTables.updateReferenceTablesBySheets(ArcTables(study.Tables),tables,false)
                 ) (ArcTables(assay.Tables))
+            let datamap = ARCAux.getAssayDataMapFromContracts assay.Identifier contracts
+            assay.DataMap <- datamap
             assay.Tables <- updatedTables.Tables
         )
-        investigation.Assays |> Seq.iter (fun a -> a.StaticHash <- a.GetHashCode())
+        investigation.Assays |> Seq.iter (fun a -> a.StaticHash <- a.GetLightHashCode())
         investigation.Studies |> Seq.iter (fun s -> s.StaticHash <- s.GetLightHashCode())
         investigation.StaticHash <- investigation.GetLightHashCode()
         this.ISA <- Some investigation
@@ -275,13 +292,28 @@ type ARC(?isa : ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSystem) =
                     Identifier.Study.fileNameFromIdentifier s.Identifier,
                     (DTOType.ISA_Study, ArcStudy.toFsWorkbook s)
                 )
+                if s.DataMap.IsSome then 
+                    let dm = s.DataMap.Value
+                    dm.StaticHash <- dm.GetHashCode()
+                    workbooks.Add (
+                        Identifier.Study.datamapFileNameFromIdentifier s.Identifier,
+                        (DTOType.ISA_Datamap, Spreadsheet.DataMap.toFsWorkbook dm)
+                    )
+
             )
             inv.Assays
             |> Seq.iter (fun a ->
-                a.StaticHash <- a.GetHashCode()
+                a.StaticHash <- a.GetLightHashCode()
                 workbooks.Add (
                     Identifier.Assay.fileNameFromIdentifier a.Identifier,
-                    (DTOType.ISA_Assay, Spreadsheet.ArcAssay.toFsWorkbook a))                
+                    (DTOType.ISA_Assay, Spreadsheet.ArcAssay.toFsWorkbook a))     
+                if a.DataMap.IsSome then 
+                    let dm = a.DataMap.Value
+                    dm.StaticHash <- dm.GetHashCode()
+                    workbooks.Add (
+                        Identifier.Assay.datamapFileNameFromIdentifier a.Identifier,
+                        (DTOType.ISA_Datamap, Spreadsheet.DataMap.toFsWorkbook dm)
+                    )
             )
 
         | None -> 
@@ -329,15 +361,33 @@ type ARC(?isa : ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSystem) =
                     elif s.StaticHash <> hash then 
                         yield s.ToUpdateContract()
                     s.StaticHash <- hash
+
+                    match s.DataMap with
+                    | Some dm when dm.StaticHash = 0 -> 
+                        yield dm.ToCreateContractForStudy(s.Identifier)
+                        dm.StaticHash <- dm.GetHashCode()
+                    | Some dm when dm.StaticHash <> dm.GetHashCode() ->
+                        yield dm.ToUpdateContractForStudy(s.Identifier)
+                        dm.StaticHash <- dm.GetHashCode()
+                    | _ -> ()
 
                 // Get Assay contracts
                 for a in inv.Assays do
-                    let hash = a.GetHashCode()
+                    let hash = a.GetLightHashCode()
                     if a.StaticHash = 0 then 
                         yield! a.ToCreateContract(WithFolder = true)
                     elif a.StaticHash <> hash then 
                         yield a.ToUpdateContract()
                     a.StaticHash <- hash
+
+                    match a.DataMap with
+                    | Some dm when dm.StaticHash = 0 -> 
+                        yield dm.ToCreateContractForAssay(a.Identifier)
+                        dm.StaticHash <- dm.GetHashCode()
+                    | Some dm when dm.StaticHash <> dm.GetHashCode() ->
+                        yield dm.ToUpdateContractForAssay(a.Identifier)
+                        dm.StaticHash <- dm.GetHashCode()
+                    | _ -> ()
             |]
 
 

src/ARCtrl/ARCtrl.fsproj

@@ -47,5 +47,7 @@
       <Package Name="requests" Version="&gt;= 2.28.1 &lt; 3.0.0" ResolutionStrategy="Max" />
     </PythonDependencies>
   </PropertyGroup>
-	<ItemGroup />
+	<ItemGroup>
+	  <PackageReference Update="FSharp.Core" Version="7.0.401" />
+	</ItemGroup>
 </Project>

src/ARCtrl/Xlsx.fs

@@ -7,6 +7,11 @@ open FsSpreadsheet
 
 module XlsxHelper =
 
+    [<AttachMembers>]
+    type DatamapXlsx() =
+        member _.fromFsWorkbook (fswb: FsWorkbook) = DataMap.fromFsWorkbook fswb
+        member _.toFsWorkbook (datamap: DataMap) = DataMap.toFsWorkbook datamap
+
     [<AttachMembers>]
     type AssayXlsx() =
         member _.fromFsWorkbook (fswb: FsWorkbook) = ArcAssay.fromFsWorkbook fswb
@@ -28,6 +33,7 @@ open XlsxHelper
 
 [<AttachMembers>]
 type XlsxController =
+    static member Datamap = DatamapXlsx()
     static member Assay = AssayXlsx()
     static member Study = StudyXlsx()
     static member Investigation = InvestigationXlsx()

src/Contract/ARC.fs

@@ -21,6 +21,8 @@ let tryISAReadContractFromPath (path: string) =
         Some <| Contract.createRead(p, DTOType.ISA_Assay) 
     | StudyPath p ->
         Some <| Contract.createRead(p, DTOType.ISA_Study) 
+    | DatamapPath p ->
+        Some <| Contract.createRead(p, DTOType.ISA_Datamap)
     | anyElse -> 
         None
 

src/Contract/ARCtrl.Contract.fsproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <None Include="README.md" />
     <Compile Include="Contract.fs" />
+    <Compile Include="Datamap.fs" />
     <Compile Include="ArcAssay.fs" />
     <Compile Include="ArcStudy.fs" />
     <Compile Include="ArcInvestigation.fs" />
@@ -19,9 +20,11 @@
     <ProjectReference Include="..\Spreadsheet\ARCtrl.Spreadsheet.fsproj" />
   </ItemGroup>
   <ItemGroup>
-    <Content Include="*.fsproj; **\*.fs; **\*.fsi" PackagePath="fable\" />
     <None Include="../../build/logo.png" Pack="true" PackagePath="\" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Update="FSharp.Core" Version="7.0.401" />
+  </ItemGroup>
   <PropertyGroup>
     <Authors>nfdi4plants, Kevin Frey, Lukas Weil, Kevin Schneider, Oliver Maus</Authors>
     <Description>ARC helper functions for contracts management.</Description>

src/Contract/Contract.fs

@@ -9,8 +9,9 @@ open Fable.Core.JsInterop
 [<RequireQualifiedAccess>]
 type DTOType = 
     | [<CompiledName("ISA_Assay")>] ISA_Assay // isa.assay.xlsx
-    | [<CompiledName("ISA_Study")>] ISA_Study
-    | [<CompiledName("ISA_Investigation")>] ISA_Investigation 
+    | [<CompiledName("ISA_Study")>] ISA_Study // isa.study.xlsx
+    | [<CompiledName("ISA_Investigation")>] ISA_Investigation // isa.investigation.xlsx
+    | [<CompiledName("ISA_Datamap")>] ISA_Datamap // isa.datamap.xlsx
     | [<CompiledName("JSON")>] JSON // arc.json
     | [<CompiledName("Markdown")>] Markdown // README.md
     | [<CompiledName("CWL")>] CWL // workflow.cwl, might be a new DTO once we 

src/Contract/Datamap.fs

@@ -0,0 +1,92 @@
+namespace ARCtrl.Contract
+
+open ARCtrl.FileSystem
+open ARCtrl.Path
+open ARCtrl.Spreadsheet
+open ARCtrl
+open ARCtrl.Helper
+open FsSpreadsheet
+
+[<AutoOpen>]
+module DatamapContractExtensions = 
+
+    let (|DatamapPath|_|) (input) =
+        match input with
+        | [|AssaysFolderName; anyAssayName; DataMapFileName|] -> 
+            let path = ARCtrl.Path.combineMany input
+            Some path
+        | [|StudiesFolderName; anyStudyName; DataMapFileName|] -> 
+            let path = ARCtrl.Path.combineMany input
+            Some path
+        | _ -> None
+
+    type DataMap with
+
+        member this.ToCreateContractForAssay (assayIdentifier : string) =
+            let path = Identifier.Assay.datamapFileNameFromIdentifier assayIdentifier
+            Contract.createCreate(path, DTOType.ISA_Datamap, DTO.Spreadsheet (this |> DataMap.toFsWorkbook))
+
+        member this.ToUpdateContractForAssay (assayIdentifier : string) =
+            let path = Identifier.Assay.datamapFileNameFromIdentifier assayIdentifier
+            let c = Contract.createUpdate(path, DTOType.ISA_Datamap, DTO.Spreadsheet (this |> DataMap.toFsWorkbook))
+            c
+
+        member this.ToDeleteContractForAssay (assayIdentifier : string) =
+            let path = Identifier.Assay.datamapFileNameFromIdentifier assayIdentifier
+            let c = Contract.createDelete(path)
+            c
+
+        static member toDeleteContractForAssay (assayIdentifier : string) =
+            fun (dataMap : DataMap) -> 
+                dataMap.ToDeleteContractForAssay(assayIdentifier)
+
+        static member toUpdateContractForAssay (assayIdentifier : string) =
+            fun (dataMap : DataMap) -> 
+                dataMap.ToUpdateContractForAssay(assayIdentifier)
+
+        static member tryFromReadContractForAssay (assayIdentifier : string) (c:Contract) =
+            let path = Identifier.Assay.datamapFileNameFromIdentifier assayIdentifier
+            match c with
+            | {Path = p; Operation = READ; DTOType = Some DTOType.ISA_Datamap; DTO = Some (DTO.Spreadsheet fsworkbook)} when p = path ->
+                let dm = 
+                    fsworkbook :?> FsWorkbook
+                    |> DataMap.fromFsWorkbook                
+                dm.StaticHash <- dm.GetHashCode()
+                dm
+                |> Some 
+            | _ -> None
+
+
+        member this.ToCreateContractForStudy (studyIdentifier : string) =
+            let path = Identifier.Study.datamapFileNameFromIdentifier studyIdentifier
+            Contract.createCreate(path, DTOType.ISA_Datamap, DTO.Spreadsheet (this |> DataMap.toFsWorkbook))
+
+        member this.ToUpdateContractForStudy (studyIdentifier : string) =
+            let path = Identifier.Study.datamapFileNameFromIdentifier studyIdentifier
+            let c = Contract.createUpdate(path, DTOType.ISA_Datamap, DTO.Spreadsheet (this |> DataMap.toFsWorkbook))
+            c
+
+        member this.ToDeleteContractForStudy (studyIdentifier : string) =
+            let path = Identifier.Study.datamapFileNameFromIdentifier studyIdentifier
+            let c = Contract.createDelete(path)
+            c
+
+        static member toDeleteContractForStudy (studyIdentifier : string) =
+            fun (dataMap : DataMap) -> 
+                dataMap.ToDeleteContractForStudy(studyIdentifier)
+
+        static member toUpdateContractForStudy (studyIdentifier : string) =
+            fun (dataMap : DataMap) -> 
+                dataMap.ToUpdateContractForStudy(studyIdentifier)
+
+
+        static member tryFromReadContractForStudy (studyIdentifier : string) (c:Contract) =
+            let path = Identifier.Study.datamapFileNameFromIdentifier studyIdentifier
+            match c with
+            | {Path = p; Operation = READ; DTOType = Some DTOType.ISA_Datamap; DTO = Some (DTO.Spreadsheet fsworkbook)} when p = path->
+                let dm = 
+                    fsworkbook :?> FsWorkbook
+                    |> DataMap.fromFsWorkbook                
+                dm.StaticHash <- dm.GetHashCode()
+                Some (dm)
+            | _ -> None

src/Core/ARCtrl.Core.fsproj

@@ -16,6 +16,8 @@
     <Compile Include="Comment.fs" />
     <Compile Include="CommentList.fs" />
     <Compile Include="OntologyAnnotation.fs" />
+    <Compile Include="DataFile.fs" />
+    <Compile Include="Data.fs" />
     <Compile Include="Person.fs" />
     <Compile Include="Publication.fs" />
     <Compile Include="OntologySourceReference.fs" />
@@ -29,8 +31,6 @@
     <Compile Include="Process\Material.fs" />
     <Compile Include="Process\Source.fs" />
     <Compile Include="Process\Sample.fs" />
-    <Compile Include="Process\DataFile.fs" />
-    <Compile Include="Process\Data.fs" />
     <Compile Include="Process\Component.fs" />
     <Compile Include="Process\ProtocolParameter.fs" />
     <Compile Include="Process\Protocol.fs" />
@@ -46,6 +46,7 @@
     <Compile Include="Table\ArcTableAux.fs" />
     <Compile Include="Table\ArcTable.fs" />
     <Compile Include="Table\ArcTables.fs" />
+    <Compile Include="Table\DataMap.fs" />
     <Compile Include="ArcTypes.fs" />
     <Compile Include="Template.fs" />
     <Compile Include="Templates.fs" />
@@ -57,9 +58,11 @@
     <ProjectReference Include="..\FileSystem\ARCtrl.FileSystem.fsproj" />
   </ItemGroup>
   <ItemGroup>
-    <Content Include="*.fsproj; **\*.fs; **\*.fsi" PackagePath="fable\" />
     <None Include="../../build/logo.png" Pack="true" PackagePath="\" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Update="FSharp.Core" Version="7.0.401" />
+  </ItemGroup>
   <PropertyGroup>
     <Authors>nfdi4plants, Lukas Weil, Kevin Frey, Kevin Schneider, Oliver Maus</Authors>
     <Description>ARC and ISA compliant experimental metadata toolkit in F#. This project is meant as an easy means to open, manipulate and save ISA (Investigation,Study,Assay) metadata files in the dotnet environment.</Description>