Skip to content

Commit

Permalink
Merge pull request #189 from nfdi4plants/implement-#187
Browse files Browse the repository at this point in the history 
FileSystemTree additions and Payload detection
  • Loading branch information
@HLWeil
HLWeil authored Sep 14, 2023
2 parents d6c525e + affab06 commit 4e676ea
Show file tree
Hide file tree
Showing 4 changed files with 432 additions and 71 deletions.
101 changes: 101 additions & 0 deletions src/ARCtrl/ARCtrl.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,108 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste
let fsCopy = _fs.Copy()
new ARC(?isa = isaCopy, ?cwl = _cwl, fs = fsCopy)

/// <summary>
/// Returns the FileSystemTree of the ARC with only the registered files and folders included.
/// </summary>
/// <param name="IgnoreHidden">Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true)</param>
member this.GetRegisteredPayload(?IgnoreHidden:bool) =

let isaCopy = _isa |> Option.map (fun i -> i.Copy()) // not sure if needed, but let's be safe

let registeredStudies =
isaCopy
|> Option.map (fun isa -> isa.Studies.ToArray()) // to-do: isa.RegisteredStudies
|> Option.defaultValue [||]

let registeredAssays =
registeredStudies
|> Array.map (fun s -> s.Assays.ToArray()) // to-do: s.RegisteredAssays
|> Array.concat

let includeRootFiles : Set<string> =
set [
Path.InvestigationFileName
Path.READMEFileName
]

let includeStudyFiles =
registeredStudies
|> Array.map (fun s ->
let studyFoldername = $"{Path.StudiesFolderName}/{s.Identifier}"

set [
yield $"{studyFoldername}/{Path.StudyFileName}"
yield $"{studyFoldername}/{Path.READMEFileName}"

//just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now.
for table in s.Tables do
for kv in table.Values do
let textValue = kv.Value.ToFreeTextCell().AsFreeText
yield textValue // from arc root
yield $"{studyFoldername}/{Path.StudiesResourcesFolderName}/{textValue}" // from study root > resources
yield $"{studyFoldername}/{Path.StudiesProtocolsFolderName}/{textValue}" // from study root > protocols
]
)
|> Set.unionMany

let includeAssayFiles =
registeredAssays
|> Array.map (fun a ->
let assayFoldername = $"{Path.AssaysFolderName}/{a.Identifier}"

set [
yield $"{assayFoldername}/{Path.AssayFileName}"
yield $"{assayFoldername}/{Path.READMEFileName}"

//just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now.
for table in a.Tables do
for kv in table.Values do
let textValue = kv.Value.ToFreeTextCell().AsFreeText
yield textValue // from arc root
yield $"{assayFoldername}/{Path.AssayDatasetFolderName}/{textValue}" // from assay root > dataset
yield $"{assayFoldername}/{Path.AssayProtocolsFolderName}/{textValue}" // from assay root > protocols
]
)
|> Set.unionMany


let includeFiles = Set.unionMany [includeRootFiles; includeStudyFiles; includeAssayFiles]

let ignoreHidden = defaultArg IgnoreHidden true
let fsCopy = _fs.Copy() // not sure if needed, but let's be safe

fsCopy.Tree
|> FileSystemTree.toFilePaths()
|> Array.filter (fun p ->
p.StartsWith(Path.WorkflowsFolderName)
|| p.StartsWith(Path.RunsFolderName)
|| includeFiles.Contains(p)
)
|> FileSystemTree.fromFilePaths
|> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree
|> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree)
|> Option.defaultValue (FileSystemTree.fromFilePaths [||])

/// <summary>
/// Returns the FileSystemTree of the ARC with only and folders included that are considered additional payload.
/// </summary>
/// <param name="IgnoreHidden">Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true)</param>
member this.GetAdditionalPayload(?IgnoreHidden:bool) =
let ignoreHidden = defaultArg IgnoreHidden true
let registeredPayload =
this.GetRegisteredPayload()
|> FileSystemTree.toFilePaths()
|> set

_fs.Copy().Tree
|> FileSystemTree.toFilePaths()
|> Array.filter (fun p -> not (registeredPayload.Contains(p)))
|> FileSystemTree.fromFilePaths
|> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree
|> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree)
|> Option.defaultValue (FileSystemTree.fromFilePaths [||])


//-Pseudo code-//
//// Option 1
Expand Down
26 changes: 19 additions & 7 deletions src/FileSystem/FileSystemTree.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,16 +128,28 @@ type FileSystemTree =
| File n ->
if predicate n then Some (File n) else None
| Folder (n, children) ->
let filteredChildren = children |> Array.choose loop
if Array.isEmpty filteredChildren then
None
else
Folder (n, filteredChildren)
|> Some
Folder (n, children |> Array.choose loop)
|> Some

loop this

static member filterFiles (predicate: string -> bool) =
fun (tree: FileSystemTree) -> tree.Filter predicate
fun (tree: FileSystemTree) -> tree.FilterFiles predicate

member this.FilterFolders (predicate: string -> bool) =
let rec loop (parent: FileSystemTree) =
match parent with
| File n -> Some (File n)
| Folder (n, children) ->
if predicate n then
Folder (n, children |> Array.choose loop)
|> Some
else
None
loop this

static member filterFolders (predicate: string -> bool) =
fun (tree: FileSystemTree) -> tree.FilterFolders predicate

member this.Filter (predicate: string -> bool) =
let rec loop (parent: FileSystemTree) =
Expand Down
149 changes: 149 additions & 0 deletions tests/ARCtrl/ARCtrl.Tests.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,158 @@ let private test_updateFileSystem = testList "update_Filesystem" [
)
]

open ARCtrl.FileSystem

let private ``payload_file_filters`` =

let orderFST (fs : FileSystemTree) =
fs
|> FileSystemTree.toFilePaths()
|> Array.sort
|> FileSystemTree.fromFilePaths

testList "payload file filters" [
let inv = ArcInvestigation("MyInvestigation", "BestTitle")

let assay = ArcAssay("registered_assay")
let assayTable = assay.InitTable("MyAssayTable")
assayTable.AppendColumn(CompositeHeader.Input (IOType.RawDataFile), [|CompositeCell.createFreeText "registered_assay_input.txt"|])
assayTable.AppendColumn(CompositeHeader.ProtocolREF, [|CompositeCell.createFreeText "assay_protocol.rtf"|])
assayTable.AppendColumn(CompositeHeader.Output (IOType.DerivedDataFile), [|CompositeCell.createFreeText "registered_assay_output.txt"|])

let study = ArcStudy("registered_study")
let studyTable = study.InitTable("MyStudyTable")
studyTable.AppendColumn(CompositeHeader.Input (IOType.Sample), [|CompositeCell.createFreeText "some_study_input_material"|])
studyTable.AppendColumn(CompositeHeader.FreeText "Some File", [|CompositeCell.createFreeText "xd/some_file_that_lies_in_slashxd.txt"|])
studyTable.AppendColumn(CompositeHeader.ProtocolREF, [|CompositeCell.createFreeText "study_protocol.pdf"|])
studyTable.AppendColumn(CompositeHeader.Output (IOType.RawDataFile), [|CompositeCell.createFreeText "registered_study_output.txt"|])
study.AddAssay(assay)

inv.AddStudy(study)

let fs =
Folder("root",[|
File "isa.investigation.xlsx"; // this should be included
File "README.md"; // this should be included
Folder("xd", [|File "some_file_that_lies_in_slashxd.txt"|]); // this should be included
Folder(".arc", [|File ".gitkeep"|]);
Folder(".git",[|
File "config"; File "description"; File "HEAD";
Folder("hooks",[|
File "applypatch-msg.sample"; File "commit-msg.sample";
File "fsmonitor-watchman.sample"; File "post-update.sample";
File "pre-applypatch.sample"; File "pre-commit.sample";
File "pre-merge-commit.sample"; File "pre-push.sample";
File "pre-rebase.sample"; File "pre-receive.sample";
File "prepare-commit-msg.sample";
File "push-to-checkout.sample"; File "update.sample"
|]);
Folder ("info", [|File "exclude"|])
|]);
Folder("assays",[|
File ".gitkeep";
Folder("registered_assay",[|
File "isa.assay.xlsx"; // this should be included
File "README.md"; // this should be included
Folder ("dataset", [|
File "registered_assay_input.txt" // this should be included
File "registered_assay_output.txt" // this should be included
File "unregistered_file.txt"
|]; );
Folder ("protocols", [|File "assay_protocol.rtf"|]) // this should be included
|]);
Folder
("unregistered_assay",[|
File "isa.assay.xlsx"; File "README.md";
Folder ("dataset", [|File ".gitkeep"|]);
Folder ("protocols", [|File ".gitkeep"|])
|])
|]);
Folder("runs", [|File ".gitkeep"|]); // this folder should be included (empty)
Folder("studies",[|
File ".gitkeep";
Folder("registered_study",[|
File "isa.study.xlsx"; // this should be included
File "README.md"; // this should be included
Folder ("protocols", [|File "study_protocol.pdf"|]); // this should be included
Folder ("resources", [|File "registered_study_output.txt"|]) // this should be included
|]);
Folder("unregistered_study",[|
File "isa.study.xlsx"; File "README.md";
Folder ("protocols", [|File ".gitkeep"|]);
Folder ("resources", [|File ".gitkeep"|])
|]);
|]);
Folder ("workflows", [|File ".gitkeep"|]) // this folder should be included (empty)
|])

let arc = ARC(isa = inv, fs = FileSystem.create(fs))

test "GetRegisteredPayload" {
let expected =
Folder("root",[|
File "isa.investigation.xlsx"; // this should be included
File "README.md"; // this should be included
Folder("xd", [|File "some_file_that_lies_in_slashxd.txt"|]); // this should be included
Folder("assays",[|
Folder("registered_assay",[|
File "isa.assay.xlsx"; // this should be included
File "README.md"; // this should be included
Folder ("dataset", [|
File "registered_assay_input.txt" // this should be included
File "registered_assay_output.txt" // this should be included
|]; );
Folder ("protocols", [|File "assay_protocol.rtf"|]) // this should be included
|]);
|]);
Folder("runs", [||]); // this folder should be included (empty)
Folder("studies",[|
Folder("registered_study",[|
File "isa.study.xlsx"; // this should be included
File "README.md"; // this should be included
Folder ("protocols", [|File "study_protocol.pdf"|]); // this should be included
Folder ("resources", [|File "registered_study_output.txt"|]) // this should be included
|]);
|]);
Folder ("workflows", [||]) // this folder should be included (empty)
|])

let actual = arc.GetRegisteredPayload()
Expect.equal (orderFST actual) (orderFST expected) "incorrect payload."
}
test "GetAdditionalPayload" {
let expected =
Folder("root",[|
Folder("assays",[|
Folder("registered_assay",[|
Folder ("dataset", [|
File "unregistered_file.txt"
|]; );
|]);
Folder
("unregistered_assay",[|
File "isa.assay.xlsx"; File "README.md";
Folder ("dataset", [||]);
Folder ("protocols", [||])
|])
|]);
Folder("studies",[|
Folder("unregistered_study",[|
File "isa.study.xlsx"; File "README.md";
Folder ("protocols", [||]);
Folder ("resources", [||])
|]);
|]);
|])
let actual = arc.GetAdditionalPayload()
Expect.equal (orderFST actual) (orderFST expected) "incorrect payload."
}
]

let main = testList "main" [
test_model
test_updateFileSystem
test_isaFromContracts
test_writeContracts
payload_file_filters
]
Loading

0 comments on commit 4e676ea

Please sign in to comment.