From 0ca078f133d607bcf54d94da7c2a57adeec0c3f7 Mon Sep 17 00:00:00 2001 From: Dmitry Grechka Date: Mon, 5 Dec 2022 20:26:35 +0600 Subject: [PATCH 1/3] wip --- .github/dependabot.yml | 11 ++++ Alpheus.sln | 5 +- AlpheusCore/AngaraGraphCommon.fs | 46 ++++++++++++++++- AlpheusCore/DependencyGraph.fs | 2 +- AlpheusCore/StatusGraph.fs | 64 +++++++++++++++++------- AlpheusUnitTests/AlpheusUnitTests.fsproj | 1 + AlpheusUnitTests/ApiTests.Vectors.fs | 40 +++++++++++++++ AlpheusUnitTests/ApiTests.fs | 33 +----------- AlpheusUnitTests/OsCommands.fs | 30 +++++++++++ 9 files changed, 180 insertions(+), 52 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 AlpheusUnitTests/OsCommands.fs diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8c23067 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "nuget" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/Alpheus.sln b/Alpheus.sln index 7dcdca1..fa48efd 100644 --- a/Alpheus.sln +++ b/Alpheus.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29009.5 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.33122.133 MinimumVisualStudioVersion = 10.0.40219.1 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "AlpheusCore", "AlpheusCore\AlpheusCore.fsproj", "{F429F84A-B34A-4FA2-9610-6D50B921D5B1}" EndProject @@ -12,6 +12,7 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution items", "Solution items", "{11C4123F-810D-46CE-90E3-F6151573483C}" ProjectSection(SolutionItems) = preProject .drone.yml = .drone.yml + .github\dependabot.yml = .github\dependabot.yml README.md = README.md EndProjectSection EndProject diff --git a/AlpheusCore/AngaraGraphCommon.fs b/AlpheusCore/AngaraGraphCommon.fs index 8fd505e..2c0a2b0 100644 --- a/AlpheusCore/AngaraGraphCommon.fs +++ b/AlpheusCore/AngaraGraphCommon.fs @@ -40,7 +40,9 @@ let getInputTypes<'a> (v:MethodVertex) = | Source src -> List.empty | Command cmd -> cmd.Inputs |> Seq.map(fun a -> max 0 (a.Artefact.Rank - rank) |> arrayType<'a>) |> List.ofSeq -let toJaggedArrayOrValue (mapValue: (string list * 'a) -> 'c) (map: MdMap) : obj = +/// returns either jagged array of 'c or single 'c element. Thus return type obj. This brakes the type checking, +/// but unfortunately that's how Angara expects artefact to be of that type +let toJaggedArrayOrValue (mapValue: (string list * 'a) -> 'c) (map: MdMap) = let rec toJaggedArrayOrValueRec (mapValue: (string list * 'a) -> 'c) (index: string list) (map: MdMapTree) : obj = let isValue = function | MdMapTree.Value _ -> true @@ -69,6 +71,48 @@ let toJaggedArrayOrValue (mapValue: (string list * 'a) -> 'c) (map: MdMap failwith "Data is incomplete and has missing elements")) toJaggedArrayOrValueRec mapValue [] (map |> MdMap.toTree) +/// returns either jagged array of 'c or single 'c element. Thus return type obj. This brakes the type checking, +/// but unfortunately that's how Angara expects artefact to be of that type +let toJaggedArrayOrValueAsync (mapValue: (string list * 'a) -> Async<'c>) (map: MdMap) : Async = + let rec toJaggedArrayOrValueRec (mapValue: (string list * 'a) -> Async<'c>) (index: string list) (map: MdMapTree) : Async = + let isValue = function + | MdMapTree.Value _ -> true + | MdMapTree.Map _ -> false + + let mapToArray (getElement: (string * MdMapTree) -> Async<'b>) (map: Map>) : Async<'b[]> = + map |> Map.toSeq |> Seq.sortBy fst |> Seq.map getElement |> Seq.toArray |> Async.Parallel + + let append v list = list |> List.append [v] + + match map with + | MdMapTree.Value v -> + async { + let! mappedValue = mapValue (index, v) + return upcast(mappedValue) + } + | MdMapTree.Map subMap -> + match subMap |> Map.forall(fun _ -> isValue) with + | true -> // final level + async { + let! arr = subMap |> mapToArray (fun (k,t) -> + let newIndex = index |> append k + match t with + | MdMapTree.Value v -> mapValue (newIndex, v) + | MdMapTree.Map _ -> failwith "Unreachable case") + return arr + } + | false -> + async { + let! arr = subMap |> mapToArray (fun (k,t) -> + let newIndex = index |> append k + match t with + | MdMapTree.Map _ -> toJaggedArrayOrValueRec mapValue newIndex t + | MdMapTree.Value _ -> failwith "Data is incomplete and has missing elements") + return arr + } + toJaggedArrayOrValueRec mapValue [] (map |> MdMap.toTree) + + /// Truncates `index` so its length is `rank`, if rank is less or equal to the length of the index. /// Throws if the rank is greater than the length of the index. let rec internal truncateIndex (rank: int) (index: string list) = diff --git a/AlpheusCore/DependencyGraph.fs b/AlpheusCore/DependencyGraph.fs index d287916..4f24884 100644 --- a/AlpheusCore/DependencyGraph.fs +++ b/AlpheusCore/DependencyGraph.fs @@ -247,7 +247,7 @@ and LinkToArtefact(artefact: ArtefactVertex, expectedVersion: ArtefactVersion) = member s.AnalyzeStatus checkStoragePresence (index: string list) = if index.Length <> artefact.Rank then invalidArg "index" "Index doesn't correspond to the rank of the artefact" async { - let expectedVersion = MdMap.find index expected + let expectedVersion = MdMap.tryFind index expected |> Option.flatten let! artefactItemActualVersion = artefact.ActualVersion.Get index let expectation = match expectedVersion with diff --git a/AlpheusCore/StatusGraph.fs b/AlpheusCore/StatusGraph.fs index cafb53a..36f5a1c 100644 --- a/AlpheusCore/StatusGraph.fs +++ b/AlpheusCore/StatusGraph.fs @@ -28,17 +28,14 @@ type SourceMethod(source: SourceVertex, experimentRoot, checkStoragePresense) = let expectedArtefact = source.Output let artefact = expectedArtefact.Artefact - let getItemStatus (link:LinkToArtefact) index = - link.AnalyzeStatus checkStoragePresense index - - // Output of the method is an scalar or a vector of full paths to the data of the artefact. - let indices = - artefact.Id - |> PathUtils.enumerateItems experimentRoot - |> MdMap.toSeq |> Seq.map fst |> Array.ofSeq + let diskItems = artefact.Id |> PathUtils.enumerateItems experimentRoot + let expectedVersions = expectedArtefact.ExpectedVersion + let actualIndices = diskItems |> MdMap.toSeq |> Seq.map fst + let! actualVersionsStrached = actualIndices |> Seq.map artefact.ActualVersion.Get |> Async.Parallel + let actualVersions = Seq.zip actualIndices actualVersionsStrached |> Seq.fold (fun s e -> let k,v = e in MdMap.add k v s) MdMap.empty - let! itemStatuses = - indices |> Array.map (getItemStatus expectedArtefact) |> Async.Parallel + let merger _ expectedOpt actualOpt = (Option.flatten expectedOpt), (Option.flatten actualOpt) + let expAndAct = Utils.mdmapMerge merger expectedVersions actualVersions let linkStatusToCommandVertextStatus status = match status with @@ -53,12 +50,45 @@ type SourceMethod(source: SourceVertex, experimentRoot, checkStoragePresense) = failwith "The artefact data is not found neither on disk nor in any of the available storages" | Local -> UpToDate [ArtefactLocation.Local] | Remote -> UpToDate [ArtefactLocation.Remote] - let result = - Array.map2 (fun index status -> {ArtefactId = artefact.Id; Index = index; Status = linkStatusToCommandVertextStatus status}) indices itemStatuses - |> Seq.map (fun x -> x :> Artefact) - |> List.ofSeq - - return seq{ yield (result, null) } + + let getItemLinkStatus (link:LinkToArtefact) index = + link.AnalyzeStatus checkStoragePresense index + + let outputArtefactAsync : Async = + async { + // the behavior significantly differs for scalar and vector source vertices + // for scalar, we always evaluate the expected version + if expAndAct.IsScalar then + let! status = getItemLinkStatus expectedArtefact [] + let vertStatus = linkStatusToCommandVertextStatus status + return upcast { ArtefactId = artefact.Id; Index = []; Status = vertStatus} + else + // For the vector (e.g. having "*" in the path) we distinguish two cases: + // * Nothing on disk match the pattern. + // In this case we respect the expected version and the vector indices stored there + // * There is at list one item on disk that matches the pattern. + // We consider this is a new source of vector map index. We completely ignore expected version indices + if Seq.isEmpty actualIndices then + // nothing is on disk. Evaluating the expected version indices + let itemMapper (arg:string list * HashString option): Async = + async { + let idx, _ = arg // idx, expectedVer + let! status = getItemLinkStatus expectedArtefact idx + let vertItemStatus = linkStatusToCommandVertextStatus status + return { ArtefactId= artefact.Id; Index = idx; Status=vertItemStatus } + } + let! artefactItems = expectedVersions |> toJaggedArrayOrValueAsync itemMapper + return artefactItems + else + // there are something on disk that match the pattern + // we emit the indices based on what is on the disk + let itemMapper (pair:string list * string): ArtefactItem = + let idx,_ = pair + { ArtefactId= artefact.Id; Index = idx; Status=UpToDate[ArtefactLocation.Local] } + return diskItems |> toJaggedArrayOrValue itemMapper + } + let! outputArtefact = outputArtefactAsync + return Seq.singleton([outputArtefact], null) } |> Async.RunSynchronously @@ -70,7 +100,7 @@ type CommandMethod(command: CommandLineVertex, override s.Execute(inputs, _) = //ignoring checkpoint. async{ // Rules of execution - // The artefact is valid either if actual disk version matches expected version or if the disk version is absend and expected version is restorable from storage + // The artefact is valid either if actual disk version matches expected version or if the disk version is absent and expected version is restorable from storage // We can bypass the computation entirely if inputs and outputs are valid let inputItems = inputs |> List.map (fun inp -> inp :?> ArtefactItem) diff --git a/AlpheusUnitTests/AlpheusUnitTests.fsproj b/AlpheusUnitTests/AlpheusUnitTests.fsproj index 191b52b..2dd5629 100644 --- a/AlpheusUnitTests/AlpheusUnitTests.fsproj +++ b/AlpheusUnitTests/AlpheusUnitTests.fsproj @@ -35,6 +35,7 @@ PreserveNewest + diff --git a/AlpheusUnitTests/ApiTests.Vectors.fs b/AlpheusUnitTests/ApiTests.Vectors.fs index e07101f..14ab0cd 100644 --- a/AlpheusUnitTests/ApiTests.Vectors.fs +++ b/AlpheusUnitTests/ApiTests.Vectors.fs @@ -3,6 +3,7 @@ open Xunit open ItisLab.Alpheus.Tests open ItisLab.Alpheus.Tests.Utils +open ItisLab.Alpheus.Tests.OsCommands open ItisLab.Alpheus.API open System.Collections open System.Collections.Generic @@ -78,6 +79,22 @@ type ``Vector scenarios``(output) as this = | TargetPlatform.Linux -> strings |> Seq.map(fun s -> s + "\n") |> String.concat "" | _ -> failwith "Unknown platform" + let buildVectorExperiment(path) = + async { + let path = Path.GetFullPath path + let! _ = API.createExperimentDirectory path + Directory.CreateDirectory(Path.Combine(path,"vec1")) |> ignore + Directory.CreateDirectory(Path.Combine(path,"vec2")) |> ignore + + do! File.WriteAllTextAsync(Path.Combine(path,"vec1","a.txt"),"File 1\\r\\n") |> Async.AwaitTask + do! File.WriteAllTextAsync(Path.Combine(path,"vec1","b.txt"),"File 2\\r\\n") |> Async.AwaitTask + do! File.WriteAllTextAsync(Path.Combine(path,"vec1","c.txt"),"File 3\\r\\n") |> Async.AwaitTask + + let! res1 = API.buildAsync path path ["vec1\*.txt"] ["vec2\*.txt"] copyFileCommand DependencyGraph.CommandExecutionSettings.Default + assertResultOk res1 + return () + } + [] member s.``Runs same method for multiple input files``() = @@ -725,6 +742,29 @@ type ``Vector scenarios``(output) as this = //["sample2"] |> concatStrings |> assertFileContent (Path.Combine(root, "samples", "sample2.txt")) //["sample3"] |> concatStrings |> assertFileContent (Path.Combine(root, "samples", "sample3.txt")) } + + [] + member s.``Status: Uncomputed vector``() = + async { + let path = Path.GetFullPath s.RelativeExperimentRoot + do! buildVectorExperiment(path) + + let res = API.status(path,ArtefactId.Path(Path.Combine("vec2","*.txt.alph"))) + match res with + | Ok r -> + //let expectedStatuses:Map> = + // [ + // ArtefactId.Path "1.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); + // ArtefactId.Path "2.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); + // ArtefactId.Path "3.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); + // ArtefactId.Path "1_2.txt",MdMap.scalar (StatusGraph.ArtefactStatus.NeedsRecomputation OutdatedReason.InputsOutdated); + // ArtefactId.Path "1_2_3.txt",MdMap.scalar (StatusGraph.ArtefactStatus.NeedsRecomputation OutdatedReason.InputsOutdated); + // ] |> Map.ofList + //Assert.True(equalStatuses expectedStatuses r) + () + | Error e-> + Assert.True(false, sprintf "Error: %A" e) + } diff --git a/AlpheusUnitTests/ApiTests.fs b/AlpheusUnitTests/ApiTests.fs index d34df57..c32187b 100644 --- a/AlpheusUnitTests/ApiTests.fs +++ b/AlpheusUnitTests/ApiTests.fs @@ -3,6 +3,7 @@ open Xunit open ItisLab.Alpheus.Tests open ItisLab.Alpheus.Tests.Utils +open ItisLab.Alpheus.Tests.OsCommands open ItisLab.Alpheus.API open System.Collections open System.Collections.Generic @@ -380,44 +381,15 @@ let equalStatuses expected actual = let s2 = Map.toSeq actual Seq.forall2 (fun x y -> let idx1,v1 = x in let idx2,v2 = y in (idx1=idx2) && MdMap.equal (fun _ elem1 elem2 -> elem1=elem2) v1 v2) s1 s2 - - type ScalarScenarios(output) = inherit SingleUseOneTimeDirectory(output) - - let copyFileCommand = - if isTestRuntimeWindows then - "cmd /C \"copy $in1 $out1\"" - else - "/bin/sh -c \"cp $in1 $out1\"" - + let twoOutputsCommand = if isTestRuntimeWindows then "cmd /C \"cmd.exe /C copy $in1 $out1 & cmd.exe /C copy $in2 $out2\"" else "/bin/sh -c \"cat $in1 > $out1; cat $in2 >> $out2\"" - let copyDirCommand = - if isTestRuntimeWindows then - "robocopy /E $in1 $out1" - else - // ubuntu cp works differently if the dest dir exists. - // as alpheus creates output dirs before running the command we need to delete it - "/bin/sh -c \"rm -Rv $out1 ; cp -Rv $in1 $out1\"" - - let concatCommand = - if isTestRuntimeWindows then - "cmd /C \"cat.cmd $out1 $in1 $in2\"" - else - "/bin/sh -c \"cat $in1 > $out1; cat $in2 >> $out1\"" - - // first file is duplicated - let concatCommand2 = - if isTestRuntimeWindows then - "cmd /C \"cat.cmd $out1 $in1 $in1\"" - else - "/bin/sh -c \"cat $in1 > $out1; cat $in1 >> $out1\"" - let buildExperiment(path) = async { @@ -1448,4 +1420,3 @@ type ScalarScenarios(output) = | Error e-> Assert.True(false, sprintf "Error: %A" e) } - diff --git a/AlpheusUnitTests/OsCommands.fs b/AlpheusUnitTests/OsCommands.fs new file mode 100644 index 0000000..07f3d16 --- /dev/null +++ b/AlpheusUnitTests/OsCommands.fs @@ -0,0 +1,30 @@ +module ItisLab.Alpheus.Tests.OsCommands + +open ItisLab.Alpheus.Tests.Utils + +let copyFileCommand = + if isTestRuntimeWindows then + "cmd /C \"copy $in1 $out1\"" + else + "/bin/sh -c \"cp $in1 $out1\"" + +let copyDirCommand = + if isTestRuntimeWindows then + "robocopy /E $in1 $out1" + else + // ubuntu cp works differently if the dest dir exists. + // as alpheus creates output dirs before running the command we need to delete it + "/bin/sh -c \"rm -Rv $out1 ; cp -Rv $in1 $out1\"" + +let concatCommand = + if isTestRuntimeWindows then + "cmd /C \"cat.cmd $out1 $in1 $in2\"" + else + "/bin/sh -c \"cat $in1 > $out1; cat $in2 >> $out1\"" + +// first file is duplicated +let concatCommand2 = + if isTestRuntimeWindows then + "cmd /C \"cat.cmd $out1 $in1 $in1\"" + else + "/bin/sh -c \"cat $in1 > $out1; cat $in1 >> $out1\"" From a3e0e668c24e105c07197faf1bc9b0b583bb63eb Mon Sep 17 00:00:00 2001 From: Dmitry Grechka Date: Tue, 6 Dec 2022 22:42:06 +0600 Subject: [PATCH 2/3] Status of vector source works --- AlpheusCore/ComputationGraph.fs | 3 ++- AlpheusCore/StatusGraph.fs | 14 ++++++++++---- AlpheusUnitTests/ApiTests.Vectors.fs | 24 ++++++++++++------------ AlpheusUnitTests/ApiTests.fs | 5 ----- AlpheusUnitTests/TestUtils.fs | 8 +++++++- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/AlpheusCore/ComputationGraph.fs b/AlpheusCore/ComputationGraph.fs index 1dc1a37..c270e7e 100644 --- a/AlpheusCore/ComputationGraph.fs +++ b/AlpheusCore/ComputationGraph.fs @@ -276,7 +276,8 @@ type CommandMethod(command: CommandLineVertex, outputItems |> toJaggedArrayOrValue (fun (extraIndex, itemFullPath) -> { FullPath = itemFullPath; Index = index @ extraIndex; UpdateType=Process }) else upcast { FullPath = outputPath; Index = index; UpdateType=Process } - return Seq.singleton(outputPaths |> List.map outPathToArtefactItem, null) + let outputs = outputPaths |> List.map outPathToArtefactItem + return Seq.singleton(outputs, null) } |> Async.RunSynchronously diff --git a/AlpheusCore/StatusGraph.fs b/AlpheusCore/StatusGraph.fs index 36f5a1c..b8eb2d0 100644 --- a/AlpheusCore/StatusGraph.fs +++ b/AlpheusCore/StatusGraph.fs @@ -172,10 +172,16 @@ let getStatuses (g:FlowGraph>) = ArtefactStatus.UpToDate(List.item outputIdx outputs) | Outdated reason -> NeedsRecomputation reason - let outputNumToRes idx : (ArtefactId * string list* ArtefactStatus) = - let artItem: ArtefactItem = downcast x.TryGet(idx).Value - artItem.ArtefactId, artItem.Index, (methodInstanceStatusToOutputStatus artItem.Status idx) - Seq.init outputsCount outputNumToRes + let outputNumToRes idx : (ArtefactId * string list* ArtefactStatus) seq = + let genTuple artItem = + artItem.ArtefactId, artItem.Index, (methodInstanceStatusToOutputStatus artItem.Status idx) + match x.TryGet(idx).Value with + | :? ArtefactItem as artItem -> + artItem |> genTuple |> Seq.singleton + | :? (ArtefactItem[]) as vector -> + vector |> Seq.map genTuple + | _ -> failwith "Unexpected type of the artItem" + Seq.init outputsCount outputNumToRes |> Seq.collect id let itemsStatus = state |> MdMap.toSeq |> Seq.collect (fun x -> let _,v = x in v.Data.Value |> toArtefactItemStatus) itemsStatus diff --git a/AlpheusUnitTests/ApiTests.Vectors.fs b/AlpheusUnitTests/ApiTests.Vectors.fs index 14ab0cd..8809a17 100644 --- a/AlpheusUnitTests/ApiTests.Vectors.fs +++ b/AlpheusUnitTests/ApiTests.Vectors.fs @@ -90,7 +90,7 @@ type ``Vector scenarios``(output) as this = do! File.WriteAllTextAsync(Path.Combine(path,"vec1","b.txt"),"File 2\\r\\n") |> Async.AwaitTask do! File.WriteAllTextAsync(Path.Combine(path,"vec1","c.txt"),"File 3\\r\\n") |> Async.AwaitTask - let! res1 = API.buildAsync path path ["vec1\*.txt"] ["vec2\*.txt"] copyFileCommand DependencyGraph.CommandExecutionSettings.Default + let! res1 = API.buildAsync path path ["vec1/*.txt"] ["vec2/*.txt"] copyFileCommand DependencyGraph.CommandExecutionSettings.Default assertResultOk res1 return () } @@ -744,23 +744,23 @@ type ``Vector scenarios``(output) as this = } [] - member s.``Status: Uncomputed vector``() = + member s.``Status: Source vector``() = async { let path = Path.GetFullPath s.RelativeExperimentRoot do! buildVectorExperiment(path) - let res = API.status(path,ArtefactId.Path(Path.Combine("vec2","*.txt.alph"))) + let res = API.status(path,ArtefactId.Path("vec1/*.txt")) match res with | Ok r -> - //let expectedStatuses:Map> = - // [ - // ArtefactId.Path "1.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); - // ArtefactId.Path "2.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); - // ArtefactId.Path "3.txt",MdMap.scalar (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local); - // ArtefactId.Path "1_2.txt",MdMap.scalar (StatusGraph.ArtefactStatus.NeedsRecomputation OutdatedReason.InputsOutdated); - // ArtefactId.Path "1_2_3.txt",MdMap.scalar (StatusGraph.ArtefactStatus.NeedsRecomputation OutdatedReason.InputsOutdated); - // ] |> Map.ofList - //Assert.True(equalStatuses expectedStatuses r) + let expectedVector = MdMap.empty + let expectedVector = MdMap.add ["a"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + let expectedVector = MdMap.add ["b"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + let expectedVector = MdMap.add ["c"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + let expectedStatuses:Map> = + [ + ArtefactId.Path "vec1/*.txt",expectedVector; + ] |> Map.ofList + Assert.True(equalStatuses expectedStatuses r) () | Error e-> Assert.True(false, sprintf "Error: %A" e) diff --git a/AlpheusUnitTests/ApiTests.fs b/AlpheusUnitTests/ApiTests.fs index c32187b..6f1da86 100644 --- a/AlpheusUnitTests/ApiTests.fs +++ b/AlpheusUnitTests/ApiTests.fs @@ -376,11 +376,6 @@ type DepGraphSaveRestore(output) = } |> toAsyncFact -let equalStatuses expected actual = - let s1 = Map.toSeq expected - let s2 = Map.toSeq actual - Seq.forall2 (fun x y -> let idx1,v1 = x in let idx2,v2 = y in (idx1=idx2) && MdMap.equal (fun _ elem1 elem2 -> elem1=elem2) v1 v2) s1 s2 - type ScalarScenarios(output) = inherit SingleUseOneTimeDirectory(output) diff --git a/AlpheusUnitTests/TestUtils.fs b/AlpheusUnitTests/TestUtils.fs index 9d932b3..625e81a 100644 --- a/AlpheusUnitTests/TestUtils.fs +++ b/AlpheusUnitTests/TestUtils.fs @@ -4,6 +4,7 @@ open System.Threading.Tasks open System open Xunit open Xunit.Abstractions +open Angara.Data // Ensure we match the return type xUnit.net is looking for let toAsyncFact computation : Task = Async.StartAsTask computation :> _ @@ -95,4 +96,9 @@ let assertResultOk result = | Error(e) -> Assert.True(false, sprintf "Expected successful operation, but got error: %A" e) type Microsoft.FSharp.Collections.List<'T> with - member this.ToArray() = this |> Array.ofList \ No newline at end of file + member this.ToArray() = this |> Array.ofList + +let equalStatuses expected actual = + let s1 = Map.toSeq expected + let s2 = Map.toSeq actual + Seq.forall2 (fun x y -> let idx1,v1 = x in let idx2,v2 = y in (idx1=idx2) && MdMap.equal (fun _ elem1 elem2 -> elem1=elem2) v1 v2) s1 s2 \ No newline at end of file From 97ff0382c5f5c5532cdfaeabe0626183e39f3da8 Mon Sep 17 00:00:00 2001 From: Dmitry Grechka Date: Wed, 7 Dec 2022 20:58:54 +0600 Subject: [PATCH 3/3] compute status test --- AlpheusUnitTests/ApiTests.Vectors.fs | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/AlpheusUnitTests/ApiTests.Vectors.fs b/AlpheusUnitTests/ApiTests.Vectors.fs index 8809a17..8cff0ba 100644 --- a/AlpheusUnitTests/ApiTests.Vectors.fs +++ b/AlpheusUnitTests/ApiTests.Vectors.fs @@ -765,6 +765,36 @@ type ``Vector scenarios``(output) as this = | Error e-> Assert.True(false, sprintf "Error: %A" e) } + + [] + member s.``Status: vector``() = + async { + let path = Path.GetFullPath s.RelativeExperimentRoot + do! buildVectorExperiment(path) + + let res = API.status(path,ArtefactId.Path("vec2/*.txt")) + match res with + | Ok r -> + let expectedVector = MdMap.empty + let expectedVector = MdMap.add ["a"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + let expectedVector = MdMap.add ["b"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + let expectedVector = MdMap.add ["c"] (StatusGraph.ArtefactStatus.UpToDate DependencyGraph.Local) expectedVector + + let expectedVector2 = MdMap.empty + let expectedVector2 = MdMap.add ["a"] (StatusGraph.ArtefactStatus.NeedsRecomputation InputsOutdated) expectedVector2 + let expectedVector2 = MdMap.add ["b"] (StatusGraph.ArtefactStatus.NeedsRecomputation InputsOutdated) expectedVector2 + let expectedVector2 = MdMap.add ["c"] (StatusGraph.ArtefactStatus.NeedsRecomputation InputsOutdated) expectedVector2 + + let expectedStatuses:Map> = + [ + ArtefactId.Path "vec1/*.txt",expectedVector; + ArtefactId.Path "vec2/*.txt",expectedVector2; + ] |> Map.ofList + Assert.True(equalStatuses expectedStatuses r) + () + | Error e-> + Assert.True(false, sprintf "Error: %A" e) + }