Generic LDNodes and RO-Crate parsing rework

Directory.Packages.props

@@ -15,7 +15,7 @@
     <PackageVersion Include="FsSpreadsheet.Js"      Version="6.3.1" />
     <PackageVersion Include="FsSpreadsheet.Py"      Version="6.3.1" />
     <PackageVersion Include="YAMLicious"            Version="0.0.3" />
-    <PackageVersion Include="DynamicObj"            Version="4.0.3" />
+    <PackageVersion Include="DynamicObj"            Version="7.0.1" />
     <PackageVersion Include="Fable.SimpleHttp"      Version="3.5.0" />
     <PackageVersion Include="Fable.Fetch"           Version="2.6.0" />
     <PackageVersion Include="Fable.Node"            Version="1.2.0" />

build/ProjectInfo.fs

@@ -5,19 +5,22 @@ open Helpers
 
 let project = "ARCtrl"
 
+let allTestsProject = "tests/All"
+
 /// Dotnet and JS test paths
 let testProjects = 
     [
         "tests/All"
-        //"tests/Core"
-        //"tests/Json"
-        //"tests/Spreadsheet"
-        //"tests/FileSystem"
-        //"tests/ARCtrl"
-        //"tests/Yaml"
-        //"tests/ValidationPackages"
-        //"tests/Contract"
-        //"tests/ROCrate"
+        "tests/ARCtrl"
+        "tests/Contract"
+        "tests/Core"
+        "tests/CWL"
+        "tests/FileSystem"
+        "tests/Json"
+        "tests/ROCrate"
+        "tests/Spreadsheet"
+        "tests/ValidationPackages"
+        "tests/Yaml"
     ]
 
 /// Native JS test paths

build/TestTasks.fs

@@ -6,6 +6,8 @@ open Fake.DotNet
 open ProjectInfo
 open BasicTasks
 open Fake.Core
+open Fake.IO
+open Fake.IO.Globbing.Operators
 
 module RunTests = 
 
@@ -33,19 +35,26 @@ module RunTests =
             Trace.traceImportant "Skipping JavaScript tests"
     )
 
+    let prePareJsTests = BuildTask.create "PrepareJsTests" [] {
+        !! "tests/TestingUtils/TestResults"
+        |> Shell.cleanDirs
+        System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/js") |> ignore
+        //System.IO.File.Copy(jsHelperFilePath, $"{allTestsProject}/js/{jsHelperFileName}") |> ignore
+
+    }
+
+
     let runTestsJs = BuildTask.createFn "runTestsJS" [clean] (fun tp ->
         if tp.Context.Arguments |> List.exists (fun a -> a.ToLower() = skipTestsFlag.ToLower()) |> not then
             Trace.traceImportant "Start Js tests"
-            for path in ProjectInfo.testProjects do
-                // Setup test results directory after clean
-                System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/js") |> ignore
-                // transpile js files from fsharp code
-                run dotnet $"fable {path} -o {path}/js --nocache" ""
-
-                System.IO.File.Copy(jsHelperFilePath, $"{path}/js/{jsHelperFileName}") |> ignore
-                // run mocha in target path to execute tests
-                // "--timeout 20000" is used, because json schema validation takes a bit of time.
-                run node $"{path}/js/Main.js" ""
+            // Setup test results directory after clean
+            System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/js") |> ignore
+            // transpile js files from fsharp code
+            run dotnet $"fable {allTestsProject} -o {allTestsProject}/js --nocache" ""
+            System.IO.File.Copy(jsHelperFilePath, $"{allTestsProject}/js/{jsHelperFileName}") |> ignore
+            // run mocha in target path to execute tests
+            // "--timeout 20000" is used, because json schema validation takes a bit of time.
+            run node $"{allTestsProject}/js/Main.js" ""
         else
             Trace.traceImportant "Skipping Js tests"
     )
@@ -65,13 +74,12 @@ module RunTests =
     let runTestsPy = BuildTask.createFn "runTestsPy" [clean] (fun tp ->
         if tp.Context.Arguments |> List.exists (fun a -> a.ToLower() = skipTestsFlag.ToLower()) |> not then
             Trace.traceImportant "Start Python tests"
-            for path in ProjectInfo.testProjects do
-                // Setup test results directory after clean
-                System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/py") |> ignore
-                //transpile py files from fsharp code
-                run dotnet $"fable {path} -o {path}/py --lang python --nocache" ""
-                // run pyxpecto in target path to execute tests in python
-                run python $"{path}/py/main.py" ""
+            // Setup test results directory after clean
+            System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/py") |> ignore
+            //transpile py files from fsharp code
+            run dotnet $"fable {allTestsProject} -o {allTestsProject}/py --lang python --nocache" ""
+            // run pyxpecto in target path to execute tests in python
+            run python $"{allTestsProject}/py/main.py" ""
         else
             Trace.traceImportant "Skipping Python tests"
 
@@ -81,8 +89,7 @@ module RunTests =
         if tp.Context.Arguments |> List.exists (fun a -> a.ToLower() = skipTestsFlag.ToLower()) |> not then
             Trace.traceImportant "Start .NET tests"
             let dotnetRun = run dotnet "run"
-            testProjects
-            |> Seq.iter dotnetRun
+            dotnetRun allTestsProject
         else
             Trace.traceImportant "Skipping .NET tests"
     )
@@ -105,6 +112,7 @@ module RunTests =
                 run python $"{p}/py/main.py" ""
                 // transpile js files from fsharp code
                 run dotnet $"fable {p} -o {p}/js" ""
+                System.IO.Directory.CreateDirectory("./tests/TestingUtils/TestResults/js") |> ignore
                 System.IO.File.Copy(jsHelperFilePath, $"{p}/js/{jsHelperFileName}") |> ignore
                 // run mocha in target path to execute tests
                 // "--timeout 20000" is used, because json schema validation takes a bit of time.

src/ARCtrl/ARC.fs

@@ -301,6 +301,59 @@ type ARC(?isa : ArcInvestigation, ?cwl : unit, ?fs : FileSystem.FileSystem) =
     #endif
 
 
+    member this.MakeDataFilesAbsolute() =
+        let filesPaths = this.FileSystem.Tree.ToFilePaths() |> set
+        let checkExistenceFromRoot = fun p -> filesPaths |> Set.contains p
+        match this.ISA with
+        | Some inv -> 
+            inv.Studies |> Seq.iter (fun s ->
+                s.Tables |> Seq.iter (fun t ->
+                    match t.TryGetInputColumn() with
+                    | Some col when col.Header.IsDataColumn -> 
+                        col.Cells |> Array.iter (fun c ->
+                            if c.AsData.FilePath.IsSome then
+
+                                let newFilePath =
+                                    c.AsData.GetAbsolutePathForStudy(s.Identifier,checkExistenceFromRoot)
+                                c.AsData.FilePath <- Some newFilePath
+                        )
+                    | _ -> ()
+                    match t.TryGetOutputColumn() with
+                    | Some col when col.Header.IsDataColumn -> 
+                        col.Cells |> Array.iter (fun c ->
+                            if c.AsData.FilePath.IsSome then
+                                let newFilePath =
+                                    c.AsData.GetAbsolutePathForStudy(s.Identifier,checkExistenceFromRoot)
+                                c.AsData.FilePath <- Some newFilePath
+                        )
+                    | _ -> ()
+                )
+            )
+            inv.Assays |> Seq.iter (fun a ->
+                a.Tables |> Seq.iter (fun t ->
+                    match t.TryGetInputColumn() with
+                    | Some col when col.Header.IsDataColumn -> 
+                        col.Cells |> Array.iter (fun c ->
+                            if c.AsData.FilePath.IsSome then
+                                let newFilePath =
+                                    c.AsData.GetAbsolutePathForAssay (a.Identifier,checkExistenceFromRoot)
+                                c.AsData.FilePath <- Some newFilePath
+                        )
+                    | _ -> ()
+                    match t.TryGetOutputColumn() with
+                    | Some col when col.Header.IsDataColumn -> 
+                        col.Cells |> Array.iter (fun c ->
+                            if c.AsData.FilePath.IsSome then
+                                let newFilePath =
+                                    c.AsData.GetAbsolutePathForAssay (a.Identifier,checkExistenceFromRoot)
+                                c.AsData.FilePath <- Some newFilePath
+                        )
+                    | _ -> ()
+                )
+            )
+        | None -> ()
+
+
     //static member updateISA (isa : ISA.Investigation) (arc : ARC) : ARC =
     //    raise (System.NotImplementedException())
 
@@ -701,11 +754,24 @@ type ARC(?isa : ArcInvestigation, ?cwl : unit, ?fs : FileSystem.FileSystem) =
         ARCtrl.Contract.Git.gitattributesFileName, ARCtrl.Contract.Git.gitattributesContract
     |]
 
-    static member fromROCrateJsonString (s:string) = 
-        let isa = ARCtrl.Json.Decode.fromJsonString ARCtrl.Json.ARC.ROCrate.decoder s
-        ARC(?isa = isa)
+    static member fromDeprecatedROCrateJsonString (s:string) =
+        try 
+            let isa = ARCtrl.Json.Decode.fromJsonString ARCtrl.Json.ARC.ROCrate.decoderDeprecated s
+            ARC(isa = isa)
+        with
+        | ex -> 
+            failwithf "Could not parse deprecated ARC-RO-Crate metadata: \n%s" ex.Message
+
+    static member fromROCrateJsonString (s:string) =
+        try 
+            let isa = ARCtrl.Json.Decode.fromJsonString ARCtrl.Json.ARC.ROCrate.decoder s
+            ARC(isa = isa)
+        with
+        | ex -> 
+            failwithf "Could not parse ARC-RO-Crate metadata: \n%s" ex.Message
 
     member this.ToROCrateJsonString(?spaces) =
+        this.MakeDataFilesAbsolute()
         ARCtrl.Json.ARC.ROCrate.encoder (Option.get _isa)
         |> ARCtrl.Json.Encode.toJsonString (ARCtrl.Json.Encode.defaultSpaces spaces)
 

src/ARCtrl/ARCtrl.Javascript.fsproj

@@ -10,6 +10,7 @@
 
   <ItemGroup>   
     <Content Include="ARCtrl.Common.props" Pack="true" PackagePath="fable/" />
+    <Compile Include="ColumnIndex.fs" />
     <Compile Include="JsonIO\Decode.fs" />
     <Compile Include="JsonIO\Encode.fs" />
     <Compile Include="JsonIO\Comment.fs" />
@@ -48,7 +49,8 @@
     <Compile Include="JsonIO\Study.fs" />
     <Compile Include="JsonIO\Investigation.fs" />
     <Compile Include="JsonIO\LDObject.fs" />
-    <Compile Include="JsonIO\ARC.fs" />
+    <Compile Include="Conversion.fs" />
+    <Compile Include="ROCrateIO.fs" />
     <Compile Include="CrossAsync.fs" />
     <None Include="README.md" />
   </ItemGroup>

src/ARCtrl/ARCtrl.Python.fsproj

@@ -12,6 +12,7 @@
     <None Include="README.md" />
     <Compile Include="CrossAsync.fs" />
     <Content Include="ARCtrl.Common.props" Pack="true" PackagePath="fable/" />
+    <Compile Include="ColumnIndex.fs" />
     <Compile Include="JsonIO\Decode.fs" />
     <Compile Include="JsonIO\Encode.fs" />
     <Compile Include="JsonIO\Comment.fs" />
@@ -50,7 +51,8 @@
     <Compile Include="JsonIO\Study.fs" />
     <Compile Include="JsonIO\Investigation.fs" />
     <Compile Include="JsonIO\LDObject.fs" />
-    <Compile Include="JsonIO\ARC.fs" />   
+    <Compile Include="Conversion.fs" />
+    <Compile Include="ROCrateIO.fs" />
   </ItemGroup>
 
   <ItemGroup>

src/ARCtrl/ARCtrl.fsproj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <None Include="README.md" />
     <Content Include="ARCtrl.Common.props" Pack="true" PackagePath="fable/" />
+    <Compile Include="ColumnIndex.fs" />
     <Compile Include="JsonIO\Decode.fs" />
     <Compile Include="JsonIO\Encode.fs" />
     <Compile Include="JsonIO\Comment.fs" />
@@ -48,13 +49,14 @@
     <Compile Include="JsonIO\Study.fs" />
     <Compile Include="JsonIO\Investigation.fs" />
     <Compile Include="JsonIO\LDObject.fs" />
-    <Compile Include="JsonIO\ARC.fs" />
+    <Compile Include="Conversion.fs" />
+    <Compile Include="ROCrateIO.fs" />
   </ItemGroup>
 
   <ItemGroup>
-    <Compile Include ="CrossAsync.fs" />
-    <Compile Include ="ContractIO\FileSystemHelper.fs" />
-    <Compile Include ="ContractIO\ContractIO.fs" />
+    <Compile Include="CrossAsync.fs" />
+    <Compile Include="ContractIO\FileSystemHelper.fs" />
+    <Compile Include="ContractIO\ContractIO.fs" />
     <Compile Include="WebRequest\WebRequest.fs" />
     <Compile Include="Template.Web.fs" />
     <Compile Include="ARC.fs" />