Add temporary changes from nfdi4plants/ARCtrl#189

src/arc-export/ARCExtension.fs

@@ -0,0 +1,107 @@
+module ARCExtension
+
+open FileSystemTreeExtension
+
+open ARCtrl
+open ARCtrl.FileSystem
+
+
+//temporarily add implementations from https://github.com/nfdi4plants/ARCtrl/pull/189
+type ARC with
+    /// <summary>
+    /// Returns the FileSystemTree of the ARC with only the registered files and folders included.
+    /// </summary>
+    /// <param name="IgnoreHidden">Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true)</param>
+    member this.GetRegisteredPayload(?IgnoreHidden:bool) =
+
+        let isaCopy = this.ISA |> Option.map (fun i -> i.Copy()) // not sure if needed, but let's be safe
+
+        let registeredStudies =     
+            isaCopy
+            |> Option.map (fun isa -> isa.Studies.ToArray()) // to-do: isa.RegisteredStudies
+            |> Option.defaultValue [||]
+
+        let registeredAssays =     
+            registeredStudies
+            |> Array.map (fun s -> s.Assays.ToArray()) // to-do: s.RegisteredAssays
+            |> Array.concat
+
+        let includeRootFiles : Set<string> = 
+            set [
+                "isa.investigation.xlsx"
+                "README.md"
+            ]
+
+        let includeStudyFiles = 
+            registeredStudies
+            |> Array.map (fun s -> 
+                let studyFoldername = $"studies/{s.Identifier}"
+
+                set [
+                    yield $"{studyFoldername}/isa.study.xlsx"
+                    yield $"{studyFoldername}/README.md"
+
+                    //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now.
+                    for (kv) in s.Tables[0].Values do
+                        yield kv.Value.AsFreeText // from arc root
+                        yield $"{studyFoldername}/resources/{kv.Value.AsFreeText}" // from study root > resources
+                        yield $"{studyFoldername}/protocols/{kv.Value.AsFreeText}" // from study root > protocols
+                ]
+            )
+            |> Set.unionMany
+
+        let includeAssayFiles = 
+            registeredAssays
+            |> Array.map (fun a -> 
+                let assayFoldername = $"assays/{a.Identifier}"
+
+                set [
+                    yield $"{assayFoldername}/isa.assay.xlsx"
+                    yield $"{assayFoldername}/README.md"
+
+                    //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now.
+                    for (kv) in a.Tables[0].Values do
+                        yield kv.Value.AsFreeText // from arc root
+                        yield $"{assayFoldername}/dataset/{kv.Value.AsFreeText}" // from assay root > dataset
+                        yield $"{assayFoldername}/protocols/{kv.Value.AsFreeText}" // from assay root > protocols
+                ]
+            )
+            |> Set.unionMany
+
+
+        let includeFiles = Set.unionMany [includeRootFiles; includeStudyFiles; includeAssayFiles]
+
+        let ignoreHidden = defaultArg IgnoreHidden true
+        let fsCopy = this.FileSystem.Copy() // not sure if needed, but let's be safe
+
+        fsCopy.Tree
+        |> FileSystemTree.toFilePaths()
+        |> Array.filter (fun p -> 
+            p.StartsWith("workflows") 
+            || p.StartsWith("runs") 
+            || includeFiles.Contains(p)
+        )
+        |> FileSystemTree.fromFilePaths
+        |> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree
+        |> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree)
+        |> Option.defaultValue (FileSystemTree.fromFilePaths [||])
+
+    /// <summary>
+    /// Returns the FileSystemTree of the ARC with only and folders included that are considered additional payload.
+    /// </summary>
+    /// <param name="IgnoreHidden">Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true)</param>
+
+    member this.GetAdditionalPayload(?IgnoreHidden:bool) =
+        let ignoreHidden = defaultArg IgnoreHidden true
+        let registeredPayload = 
+            this.GetRegisteredPayload()
+            |> FileSystemTree.toFilePaths()
+            |> set
+
+        this.FileSystem.Copy().Tree
+        |> FileSystemTree.toFilePaths()
+        |> Array.filter (fun p -> not (registeredPayload.Contains(p)))
+        |> FileSystemTree.fromFilePaths
+        |> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree
+        |> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree)
+        |> Option.defaultValue (FileSystemTree.fromFilePaths [||])

src/arc-export/ArcSummaryMarkdown.fs

@@ -1,12 +1,14 @@
 module ArcSummaryMarkdown
 
+open FileSystemTreeExtension
+
 open ARCtrl
 open ARCtrl.FileSystem
 open ARCtrl.NET
 
 let [<Literal>] MARKDOWN_TEMPLATE = """## [Data set] [[ARC_TITLE]]
 
-### File contents:
+### Registered ARC content:
 
 [[FILE_TREE]]
 """
@@ -16,25 +18,6 @@ type ARCtrl.FileSystem.FileSystemTree with
 
     static member createItemString (level:int) (item: string) =
         $"""{String.replicate level "    "}- {item}""" 
-
-    member this.FilterNodes (predicate: string -> bool) = 
-        let rec loop (parent: FileSystemTree) = 
-            match parent with 
-            | File n ->  
-                if predicate n then Some (FileSystemTree.File n) else None 
-            | Folder (n, children) -> 
-                if predicate n then 
-                    let filteredChildren = children |> Array.choose loop 
-                    if Array.isEmpty filteredChildren then  
-                        None
-                    else
-                        Some (FileSystemTree.Folder (n,filteredChildren))
-                else
-                    None
-        loop this 
-
-    static member filterNodes (predicate: string -> bool) =
-        fun (tree: FileSystemTree) -> tree.FilterNodes predicate
 
     static member toMarkdownTOC (tree: FileSystemTree) =
         let rec loop (level:int) (acc:string list) (fs: FileSystemTree) =
@@ -54,12 +37,6 @@ type ARCtrl.FileSystem.FileSystemTree with
                 finalAccum
 
         tree
-        |> FileSystemTree.filterNodes( 
-            fun item -> 
-                let predicate = not (item.StartsWith("."))
-                predicate
-        )
-        |> Option.get
         |> loop 0 []
         |> Seq.rev
         |> String.concat System.Environment.NewLine

src/arc-export/FileSystemTreeExtension.fs

@@ -0,0 +1,55 @@
+module FileSystemTreeExtension
+
+open ARCtrl
+open ARCtrl.FileSystem
+
+//temporarily add implementations from https://github.com/nfdi4plants/ARCtrl/pull/189
+type ARCtrl.FileSystem.FileSystemTree with
+
+    member this.FilterFiles (predicate: string -> bool) =
+        let rec loop (parent: FileSystemTree) =
+            match parent with
+            | File n -> 
+                if predicate n then Some (File n) else None
+            | Folder (n, children) ->
+                Folder (n, children |> Array.choose loop)
+                |> Some
+
+        loop this
+
+    static member filterFiles (predicate: string -> bool) =
+        fun (tree: FileSystemTree) -> tree.FilterFiles predicate
+
+    member this.FilterFolders (predicate: string -> bool) =
+        let rec loop (parent: FileSystemTree) =
+            match parent with
+            | File n -> Some (File n)
+            | Folder (n, children) ->
+                if predicate n then 
+                    Folder (n, children |> Array.choose loop)
+                    |> Some
+                else
+                    None
+        loop this
+
+    static member filterFolders (predicate: string -> bool) =
+        fun (tree: FileSystemTree) -> tree.FilterFolders predicate
+
+    member this.Filter (predicate: string -> bool) = 
+        let rec loop (parent: FileSystemTree) = 
+            match parent with 
+            | File n ->  
+                if predicate n then Some (FileSystemTree.File n) else None 
+            | Folder (n, children) -> 
+                if predicate n then 
+                    let filteredChildren = children |> Array.choose loop 
+                    if Array.isEmpty filteredChildren then  
+                        None
+                    else
+                        Some (FileSystemTree.Folder (n,filteredChildren))
+                else
+                    None
+        loop this 
+
+    static member filter (predicate: string -> bool) =
+        fun (tree: FileSystemTree) -> tree.Filter predicate

src/arc-export/Program.fs

@@ -4,6 +4,8 @@ open ARCtrl.FileSystem
 open ARCtrl.ISA
 open ARCtrl.NET
 open Argu
+open FileSystemTreeExtension
+open ARCExtension
 open ArcSummaryMarkdown
 open ARCtrl.NET.Contract
 
@@ -48,18 +50,20 @@ try
     let jsonFile = Path.Combine(outPath,"arc.json")
 
     let mdfile = Path.Combine(outPath,"arc-summary.md")
+
 
     let inv, mdContent = 
         try 
             let arc = loadARCCustom arcPath
+            let registeredPayload = arc.GetRegisteredPayload(IgnoreHidden = true)
             let inv = arc.ISA |> Option.get
 
             getAllFilePaths arcPath |> Seq.iter (printfn "%s")
 
             inv,
             MARKDOWN_TEMPLATE
                 .Replace("[[ARC_TITLE]]", inv.Title |> Option.defaultValue "Untitled ARC")
-                .Replace("[[FILE_TREE]]", FileSystemTree.toMarkdownTOC arc.FileSystem.Tree)
+                .Replace("[[FILE_TREE]]", FileSystemTree.toMarkdownTOC registeredPayload)
         with 
         | err ->
             printfn "Could not read investigation, writing empty arc json."

src/arc-export/arc-export.fsproj

@@ -9,7 +9,8 @@
   <ItemGroup>
     <Compile Include="Defaults.fs" />
     <Compile Include="CLIArgs.fs" />
-    <Compile Include="FileTree.fs" />
+    <Compile Include="FileSystemTreeExtension.fs" />
+    <Compile Include="ARCExtension.fs" />
     <Compile Include="ArcSummaryMarkdown.fs" />
     <Compile Include="Program.fs" />
   </ItemGroup>