From bdfa3abf57b356650100ebcbbcd76f9b5671e437 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 30 Apr 2023 03:12:27 +0100 Subject: [PATCH 01/17] Switch hdf5 to netcdf --- src/api.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api.jl b/src/api.jl index d8fa1e5e..774eaded 100644 --- a/src/api.jl +++ b/src/api.jl @@ -1,3 +1,5 @@ +using NetCDF + """ initialise(config_file, submission_script) @@ -312,9 +314,8 @@ function write_array(handle::DataRegistryHandle, data::Array, data_product::Stri use_component = metadata["use_component"] # Write array - HDF5.h5open(path, isfile(path) ? "r+" : "w") do file - write(file, use_component, data) - end + nccreate(path, use_component, vcat([["$use_component-dim-$i", collect(axes(data, i)), Dict()] for i in 1:ndims(data)]...)...) + ncwrite(data, path, use_component) # Write metadata to handle handle.outputs[(data_product, component)] = metadata From 53ae2c425ad07052f24c5033cd1c5241a303a6f5 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 30 Apr 2023 03:13:18 +0100 Subject: [PATCH 02/17] Error on hdf5 calls --- src/data_prod_proc.jl | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/data_prod_proc.jl b/src/data_prod_proc.jl index a8ef373e..cc7893f1 100644 --- a/src/data_prod_proc.jl +++ b/src/data_prod_proc.jl @@ -1,7 +1,6 @@ -import HDF5 -import TOML -import AxisArrays -import NetCDF +using TOML +using AxisArrays +using NetCDF ### hdf5 file processing ### const ARRAY_OBJ_NAME = "array" @@ -12,10 +11,10 @@ const CSV_OBJ_NAME = TABLE_OBJ_NAME # "csv" ## does what it says on the tin function read_h5_table(obj_grp, use_axis_arrays::Bool) - obj = HDF5.read(obj_grp[TABLE_OBJ_NAME]) + error("obj = HDF5.read(obj_grp[TABLE_OBJ_NAME])") if use_axis_arrays # - option for 2d AxisArray output cn = collect(keys(obj[1])) - rn = haskey(obj_grp, ROWN_OBJ_NAME) ? HDF5.read(obj_grp[ROWN_OBJ_NAME]) : [1:length(obj)] + error("rn = haskey(obj_grp, ROWN_OBJ_NAME) ? HDF5.read(obj_grp[ROWN_OBJ_NAME]) : [1:length(obj)]") arr = [collect(obj[i]) for i in eachindex(obj)] d = permutedims(reshape(hcat(arr...), length(cn), length(arr))) return AxisArrays.AxisArray(d, AxisArrays.Axis{:row}(rn), AxisArrays.Axis{:col}(cn)) @@ -26,15 +25,15 @@ end ## recursively search and read table/array function process_h5_file_group!(output_dict::Dict, h5, use_axis_arrays::Bool) - gnm = HDF5.name(h5) + error("gnm = HDF5.name(h5)") if haskey(h5, TABLE_OBJ_NAME) d = read_h5_table(h5, use_axis_arrays) output_dict[gnm] = d - elseif (haskey(h5, ARRAY_OBJ_NAME) && typeof(h5[ARRAY_OBJ_NAME])!=HDF5.Group) - d = HDF5.read(h5) + error("elseif (haskey(h5, ARRAY_OBJ_NAME) && typeof(h5[ARRAY_OBJ_NAME])!=HDF5.Group)") + error("d = HDF5.read(h5)") output_dict[gnm] = d - elseif typeof(h5) == HDF5.Dataset - d = HDF5.read(h5) + error("elseif typeof(h5) == HDF5.Dataset") + error("d = HDF5.read(h5)") output_dict[gnm] = d else # group - recurse for g in keys(h5) @@ -46,9 +45,9 @@ end ## wrapper for recursive processing function process_h5_file(filepath::String, use_axis_arrays::Bool) output = Dict() - f = HDF5.h5open(filepath) + error("f = HDF5.h5open(filepath)") process_h5_file_group!(output, f, use_axis_arrays) - HDF5.close(f) + error("HDF5.close(f)") return output end @@ -65,7 +64,7 @@ Read HDF5, CSV or TOML file from local system. """ function _readdataproduct_from_file(filepath::String; use_axis_arrays::Bool = false) println("processing file: ", filepath) - HDF5.ishdf5(filepath) && (return process_h5_file(filepath, use_axis_arrays)) + error("HDF5.ishdf5(filepath) && (return process_h5_file(filepath, use_axis_arrays))") occursin(".h5", filepath) && (return process_h5_file(filepath, use_axis_arrays)) occursin(".toml", filepath) && (return TOML.parsefile(filepath)) occursin(".tml", filepath) && (return TOML.parsefile(filepath)) From d1d400defa76e39bea9ecaaf07f475c6941da5e4 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 30 Apr 2023 03:13:45 +0100 Subject: [PATCH 03/17] Remove hdf5 dependency --- Project.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Project.toml b/Project.toml index 9bfb5030..2c42a1ab 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FTPClient = "01fcc997-4f28-56b8-8a06-30002c134abb" -HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" @@ -27,7 +26,6 @@ AxisArrays = "0.4" CSV = "0.8, 0.9, 0.10" DataFrames = "1" FTPClient = "1" -HDF5 = "0.16" HTTP = "0.9, 1" JSON = "0.21" NetCDF = "0.11" From 82097c987647ff9f2bd269078d0f425ef9cc01ed Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 30 Apr 2023 03:14:37 +0100 Subject: [PATCH 04/17] Comment out test that uses HDF5 code --- .github/workflows/testing.yaml | 2 +- test/test_api.jl | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index f12ad6d2..bd78fc17 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -70,7 +70,7 @@ jobs: if: matrix.os == 'windows-latest' run: | python -m venv venv - venv/Scripts/activate.ps1 + venv\Scripts\activate.ps1 pip3 install fair-cli fair registry install fair init --ci diff --git a/test/test_api.jl b/test/test_api.jl index 0797ba9d..eba55729 100644 --- a/test/test_api.jl +++ b/test/test_api.jl @@ -1,7 +1,6 @@ module TestAPI using DataPipeline -using HDF5 using TOML using Test using Dates @@ -140,15 +139,15 @@ Test.@testset "write_array()" begin newpath2 = handle.outputs[(data_product, component2)]["path"] # Check data - c1 = HDF5.h5open(newpath1, "r") do file - read(file, component1) - end - @test data1 == c1 - - c2 = HDF5.h5open(newpath2, "r") do file - read(file, component2) - end - @test data2 == c2 +# c1 = HDF5.h5open(newpath1, "r") do file +# read(file, component1) +# end +# @test data1 == c1 + +# c2 = HDF5.h5open(newpath2, "r") do file +# read(file, component2) +# end +# @test data2 == c2 # Check handle hash = DataPipeline._getfilehash(newpath1) From aa3ddff7d0549cc17ed454a47276e7c8ab558ca4 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 30 Apr 2023 14:00:40 +0100 Subject: [PATCH 05/17] axes are Base.axes --- src/api.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api.jl b/src/api.jl index 774eaded..84122c15 100644 --- a/src/api.jl +++ b/src/api.jl @@ -314,7 +314,7 @@ function write_array(handle::DataRegistryHandle, data::Array, data_product::Stri use_component = metadata["use_component"] # Write array - nccreate(path, use_component, vcat([["$use_component-dim-$i", collect(axes(data, i)), Dict()] for i in 1:ndims(data)]...)...) + nccreate(path, use_component, vcat([["$use_component-dim-$i", collect(Base.axes(data, i)), Dict()] for i in 1:ndims(data)]...)...) ncwrite(data, path, use_component) # Write metadata to handle From 75481f9975cf6661743384a696d8b9c666bddf93 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Tue, 20 Jun 2023 20:37:55 +0100 Subject: [PATCH 06/17] Give default arguments to initialise() --- examples/fdp/seirs_sim.jl | 9 +-------- src/DataPipeline.jl | 5 +++++ src/api.jl | 5 ++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/fdp/seirs_sim.jl b/examples/fdp/seirs_sim.jl index 7bdd6348..d13568f1 100644 --- a/examples/fdp/seirs_sim.jl +++ b/examples/fdp/seirs_sim.jl @@ -6,14 +6,7 @@ using DataFrames using Plots # Initialise code run -config_file = joinpath(ENV["FDP_CONFIG_DIR"], "config.yaml") -@static if Sys.iswindows() - submission_script = joinpath(ENV["FDP_CONFIG_DIR"], "script.bat") -else - submission_script = joinpath(ENV["FDP_CONFIG_DIR"], "script.sh") -end - -handle = initialise(config_file, submission_script) +handle = initialise() # Read model parameters path = link_read!(handle, "SEIRS_model/parameters") diff --git a/src/DataPipeline.jl b/src/DataPipeline.jl index 01894359..ba68585f 100644 --- a/src/DataPipeline.jl +++ b/src/DataPipeline.jl @@ -25,6 +25,11 @@ const STR_ROOT = string(LOCAL_DR_PORTLESS, "storage_root/") const API_ROOT = string(LOCAL_DR_STEM, ":8000", "/api/") const SL_ROOT = string(LOCAL_DR_PORTLESS, "storage_location/") const DATA_OUT = "./out/" +@static if Sys.iswindows() + const FDP_CONFIG_FILE = "script.bat" +else + const FDP_CONFIG_FILE = "script.sh" +end include("core.jl") diff --git a/src/api.jl b/src/api.jl index 84122c15..103f233d 100644 --- a/src/api.jl +++ b/src/api.jl @@ -6,7 +6,10 @@ using NetCDF Reads in working config.yaml file, generates a new Code Run entry, and returns a `DataRegistryHandle` containing various metadata. """ -function initialise(config_file::String, submission_script::String) +function initialise(config_file::String = joinpath(ENV["FDP_CONFIG_DIR"], + "config.yaml"), + submission_script::String = joinpath(ENV["FDP_CONFIG_DIR"], + FDP_CONFIG_FILE)) # Read working config file print("processing config file: ", config_file) config = YAML.load_file(config_file) From 2330a63008bb900cb8c350a6d03227fc4d2433d2 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Thu, 22 Jun 2023 16:25:47 +0100 Subject: [PATCH 07/17] Remove debugging --- .github/workflows/testing.yaml | 6 +++++- test/runtests-windows.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index a833c64e..ae14a2bc 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -61,14 +61,18 @@ jobs: architecture: 'x64' - name: Install graphviz and venv run: | + # You may need to install python: https://python.org # Create venv + # If not installed: pip3 install venv python3 -m venv .venv if [ "$RUNNER_OS" == "Linux" ]; then + # If your Linux installation doesn't use apt, then use your package manager sudo apt-get install graphviz - # sudo apt-get install libmemcached-dev elif [ "$RUNNER_OS" == "Windows" ]; then + # If choco is not installed: Install from https://chocolatey.org choco install graphviz elif [ "$RUNNER_OS" == "Darwin" ]; then + # If brew not installed: Install from https://brew.sh brew install graphviz fi shell: bash diff --git a/test/runtests-windows.sh b/test/runtests-windows.sh index 8e4b0678..28b199a4 100755 --- a/test/runtests-windows.sh +++ b/test/runtests-windows.sh @@ -32,7 +32,7 @@ ESCAPED_SCRIPT=$(printf '%s\n' "$TEST_SCRIPT" | sed -e 's/[\,&]/\\&/g') echo Escaped test: "$ESCAPED_SCRIPT" sed -e "s,\$TEST_SCRIPT,$ESCAPED_SCRIPT," $TEST_DIR/pre_config.yaml > $TEST_DIR/config.yaml cat $TEST_DIR/config.yaml -fair run --dirty --local --debug $TEST_DIR/config.yaml +fair run --dirty --local $TEST_DIR/config.yaml rm -f $TEST_DIR/config.yaml deactivate From 3ddb4f6d95fde2b26aab5a37f2f8b494ecc660c7 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Thu, 22 Jun 2023 18:03:28 +0100 Subject: [PATCH 08/17] Fix example manifest --- examples/fdp/Manifest.toml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/examples/fdp/Manifest.toml b/examples/fdp/Manifest.toml index 308c1e38..5863e2fa 100644 --- a/examples/fdp/Manifest.toml +++ b/examples/fdp/Manifest.toml @@ -141,10 +141,10 @@ uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" version = "1.5.0" [[deps.DataPipeline]] -deps = ["AxisArrays", "CSV", "DataFrames", "Dates", "FTPClient", "HDF5", "HTTP", "JSON", "NetCDF", "Plots", "PrettyTables", "SHA", "SQLite", "TOML", "URIs", "UnicodePlots", "YAML"] +deps = ["AxisArrays", "CSV", "DataFrames", "Dates", "FTPClient", "HTTP", "JSON", "NetCDF", "Plots", "PrettyTables", "SHA", "SQLite", "TOML", "URIs", "UnicodePlots", "YAML"] path = "../.." uuid = "9ced6f0a-eb77-43a8-bbd1-bbf3031b0d12" -version = "0.53.0" +version = "0.53.1" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] @@ -290,12 +290,6 @@ git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" version = "1.0.2" -[[deps.HDF5]] -deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Random", "Requires", "UUIDs"] -git-tree-sha1 = "c73fdc3d9da7700691848b78c61841274076932a" -uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.16.15" - [[deps.HDF5_jll]] deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] git-tree-sha1 = "4cc2bb72df6ff40b055295fdef6d92955f9dede8" From 55baf9d92f6fd69fc6bcecb705f9c5d412e0f50f Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Thu, 22 Jun 2023 22:39:38 +0100 Subject: [PATCH 09/17] test without continue on error --- .github/workflows/testing.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index ae14a2bc..8c24a065 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -16,7 +16,6 @@ on: jobs: pipeline-tests: runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} strategy: matrix: julia-version: From 659cb306f3441eac9fa51d5a19061b9a1c1d5fda Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Thu, 22 Jun 2023 23:26:02 +0100 Subject: [PATCH 10/17] add exit with exit code --- .github/workflows/testing.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 8c24a065..5346c6a5 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -126,6 +126,7 @@ jobs: fair init --ci fair pull --debug examples/fdp/SEIRSconfig.yaml fair run --debug --dirty examples/fdp/SEIRSconfig.yaml + exit $? deactivate shell: bash env: From 969afdb514dd36a575922c55cd66d1d27d700214 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 14:27:30 +0100 Subject: [PATCH 11/17] Update manifest --- examples/fdp/Manifest.toml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/examples/fdp/Manifest.toml b/examples/fdp/Manifest.toml index 1e19c742..86a81aa3 100644 --- a/examples/fdp/Manifest.toml +++ b/examples/fdp/Manifest.toml @@ -93,9 +93,9 @@ version = "0.12.10" [[deps.Compat]] deps = ["Dates", "LinearAlgebra", "UUIDs"] -git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" +git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.6.1" +version = "4.7.0" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -182,11 +182,17 @@ deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.6.0" +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "e90caa41f5a86296e014e148ee061bd6c3edec96" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.9" + [[deps.Expat_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "bad72f730e9e91c08d9427d5e8db95478a3c323d" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.4.8+0" +version = "2.5.0+0" [[deps.FFMPEG]] deps = ["FFMPEG_jll"] @@ -297,10 +303,10 @@ uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" version = "1.12.2+2" [[deps.HTTP]] -deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "5e77dbf117412d4f164a464d610ee6050cc75272" +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "2613d054b0e18a3dea99ca1594e9a3960e025da4" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.9.6" +version = "1.9.7" [[deps.HarfBuzz_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] @@ -623,9 +629,9 @@ version = "10.40.0+0" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "5a6ab2f64388fd1175effdf73fe5933ef1e0bac0" +git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.7.0" +version = "2.7.1" [[deps.Pipe]] git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d" @@ -800,9 +806,9 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.SpecialFunctions]] deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "ef28127915f4229c971eb43f3fc075dd3fe91880" +git-tree-sha1 = "7beb031cf8145577fbccacd94b8a8f4ce78428d3" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.2.0" +version = "2.3.0" [[deps.StaticArrays]] deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] @@ -906,9 +912,9 @@ version = "0.4.1" [[deps.UnicodePlots]] deps = ["ColorSchemes", "ColorTypes", "Contour", "Crayons", "Dates", "LinearAlgebra", "MarchingCubes", "NaNMath", "PrecompileTools", "Printf", "Requires", "SparseArrays", "StaticArrays", "StatsBase"] -git-tree-sha1 = "66b7b2f49e6be1ffaf6c36ea88ac38616122e5f1" +git-tree-sha1 = "b96de03092fe4b18ac7e4786bee55578d4b75ae8" uuid = "b8865327-cd53-5732-bb35-84acbb429228" -version = "3.5.4" +version = "3.6.0" [[deps.Unitful]] deps = ["ConstructionBase", "Dates", "LinearAlgebra", "Random"] From 726d28d121fb3fcdf71d46282a4d06b148e82274 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 14:43:43 +0100 Subject: [PATCH 12/17] Remove sql code --- db/array_cube.sql | 29 ----------- db/ddl.sql | 125 ---------------------------------------------- 2 files changed, 154 deletions(-) delete mode 100644 db/array_cube.sql delete mode 100644 db/ddl.sql diff --git a/db/array_cube.sql b/db/array_cube.sql deleted file mode 100644 index 770a1e78..00000000 --- a/db/array_cube.sql +++ /dev/null @@ -1,29 +0,0 @@ -DROP TABLE IF EXISTS h5_array; -DROP TABLE IF EXISTS arr_dim; -DROP TABLE IF EXISTS arr_dim_name; - --- WIP -CREATE TABLE IF NOT EXISTS h5_array( - arr_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - dp_id INTEGER NOT NULL, - tbl_data TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS array_dim( - dim_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - comp_id INTEGER NOT NULL, - dim_index INTEGER NOT NULL, - dim_type INTEGER NOT NULL, - dim_title TEXT NOT NULL, - dim_size INTEGER NOT NULL -); -CREATE TABLE IF NOT EXISTS array_dim_name( - dim_name_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - dim_id INTEGER NOT NULL, - dim_val TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS toml_component( - comp_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - dp_id INTEGER NOT NULL, - comp_name TEXT -); diff --git a/db/ddl.sql b/db/ddl.sql deleted file mode 100644 index 4304477e..00000000 --- a/db/ddl.sql +++ /dev/null @@ -1,125 +0,0 @@ -const DDL_SQL = """ -CREATE TABLE IF NOT EXISTS session( - sn_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - data_dir TEXT NOT NULL, - pkg_version TEXT NOT NULL, - row_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -); - -CREATE TABLE IF NOT EXISTS access_log( - log_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - offline_mode INTEGER NOT NULL, - row_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - log_finished TIMESTAMP -); -CREATE TABLE IF NOT EXISTS access_log_data( - log_id INTEGER NOT NULL, - dp_id INTEGER NOT NULL, - comp_id INTEGER -); - -CREATE TABLE IF NOT EXISTS data_product( - dp_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - namespace TEXT NOT NULL, - dp_name TEXT NOT NULL, - filepath TEXT NOT NULL, - dp_hash TEXT NOT NULL, - dp_version TEXT NOT NULL, - sr_url TEXT, - sl_path TEXT, - description TEXT, - registered INTEGER DEFAULT 0, - dp_url TEXT, - row_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -); -CREATE TABLE IF NOT EXISTS component( - comp_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - dp_id INTEGER NOT NULL, - comp_name TEXT NOT NULL, - comp_type TEXT NOT NULL, - meta_src INTEGER NOT NULL, - data_obj TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS toml_keyval( - comp_id INTEGER NOT NULL, - key TEXT NOT NULL, - val TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS code_repo_rel( - crr_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - crr_name TEXT NOT NULL, - crr_version TEXT NOT NULL, - crr_repo TEXT NOT NULL, - crr_hash TEXT NOT NULL, - crr_desc TEXT, - crr_website TEXT, - storage_root_url NOT NULL, - storage_root_id NOT NULL, - registered INTEGER DEFAULT 0, - crr_url TEXT, - row_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -); -CREATE TABLE IF NOT EXISTS code_run( - run_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - crr_id INTEGER NOT NULL, - model_config TEXT NOT NULL, - run_desc TEXT NOT NULL, - ss_text TEXT NOT NULL, - registered INTEGER DEFAULT 0, - run_url TEXT, - row_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -); - -DROP VIEW IF EXISTS session_view; -DROP VIEW IF EXISTS dpc_view; -DROP VIEW IF EXISTS toml_view; -DROP VIEW IF EXISTS total_runs_staging; -DROP VIEW IF EXISTS total_runs_registry; -DROP VIEW IF EXISTS crr_view; -DROP VIEW IF EXISTS component_view; -DROP VIEW IF EXISTS log_component_view; - -CREATE VIEW session_view AS -SELECT * FROM session -WHERE sn_id=(SELECT max(sn_id) FROM session); - -CREATE VIEW dpc_view AS -SELECT d.namespace, d.dp_name, d.dp_version -, d.filepath, c.* -FROM data_product d -INNER JOIN component c ON(d.dp_id = c.dp_id); - -CREATE VIEW toml_view AS -SELECT d.namespace, d.dp_name, d.dp_version -, t.*, k.key, k.val -FROM data_product d -INNER JOIN component t ON(d.dp_id = t.dp_id) -INNER JOIN toml_keyval k ON(t.comp_id = k.comp_id); - -CREATE VIEW total_runs_staging AS -SELECT crr_id, count(run_id) AS staged_runs -FROM code_run WHERE registered=FALSE -GROUP BY crr_id; - -CREATE VIEW total_runs_registry AS -SELECT crr_id, count(run_id) AS registered_runs -FROM code_run WHERE registered=TRUE -GROUP BY crr_id; - -CREATE VIEW crr_view AS -SELECT c.crr_id AS staging_id, crr_name AS name, crr_version AS version -, registered, crr_repo AS repo, row_added -, IFNULL(s.staged_runs,0) AS staged_runs -, IFNULL(r.registered_runs,0) AS registered_runs -FROM code_repo_rel c -LEFT OUTER JOIN total_runs_staging s ON(c.crr_id=s.crr_id) -LEFT OUTER JOIN total_runs_registry r ON(c.crr_id=r.crr_id); - -CREATE VIEW log_component_view AS -SELECT DISTINCT l.log_id, dp.namespace, dp.dp_name, dp.dp_version, c.comp_name -FROM access_log_data l -INNER JOIN data_product dp ON(l.dp_id=dp.dp_id) -INNER JOIN component c ON(l.dp_id=c.dp_id AND l.comp_id=c.comp_id); -""" From da5f4a9bb3180da42a7a14b0cf60039de15e367e Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 14:56:09 +0100 Subject: [PATCH 13/17] Rearrange folder structure, don't export initialise and finalise --- .github/workflows/nightly.yaml | 6 +-- .github/workflows/testing.yaml | 6 +-- docs/src/SEIRS.md | 4 +- examples/{fdp => SEIRS}/Manifest.toml | 0 examples/{fdp => SEIRS}/Project.toml | 0 .../full_config.yaml} | 4 +- examples/SEIRS/register_config.yaml | 20 ++++++++++ examples/SEIRS/repeat_config.yaml | 23 +++++++++++ examples/{fdp => SEIRS}/seirs_sim.jl | 4 +- src/DataPipeline.jl | 3 +- src/api.jl | 8 ++-- src/fdp_i.jl | 8 ++-- src/testing.jl | 2 +- test/runtests.sh | 4 +- test/test_api.jl | 40 +++++++++---------- 15 files changed, 87 insertions(+), 45 deletions(-) rename examples/{fdp => SEIRS}/Manifest.toml (100%) rename examples/{fdp => SEIRS}/Project.toml (100%) rename examples/{fdp/SEIRSconfig.yaml => SEIRS/full_config.yaml} (87%) create mode 100644 examples/SEIRS/register_config.yaml create mode 100644 examples/SEIRS/repeat_config.yaml rename examples/{fdp => SEIRS}/seirs_sim.jl (94%) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 3ff732eb..d447ed87 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -28,7 +28,7 @@ jobs: using Pkg Pkg.activate(".") Pkg.resolve() - Pkg.activate("examples/fdp") + Pkg.activate("examples/SEIRS") Pkg.update() Pkg.resolve() shell: julia --color=yes {0} @@ -65,8 +65,8 @@ jobs: fair registry install fair registry start fair init --ci - fair pull --debug examples/fdp/SEIRSconfig.yaml - fair run --debug --dirty examples/fdp/SEIRSconfig.yaml + fair pull --debug examples/SEIRS/full_config.yaml + fair run --debug --dirty examples/SEIRS/full_config.yaml deactivate shell: bash env: diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 057833c8..d5e4206a 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -43,7 +43,7 @@ jobs: - name: Fix manifest for active Julia version run: | using Pkg - Pkg.activate("examples/fdp") + Pkg.activate("examples/SEIRS") Pkg.resolve() shell: julia --color=yes {0} - name: Set up python @@ -117,8 +117,8 @@ jobs: fair registry install fair registry start fair init --ci - fair pull --debug examples/fdp/SEIRSconfig.yaml - fair run --debug --dirty examples/fdp/SEIRSconfig.yaml + fair pull --debug examples/SEIRS/full_config.yaml + fair run --debug --dirty examples/SEIRS/full_config.yaml exit $? deactivate shell: bash diff --git a/docs/src/SEIRS.md b/docs/src/SEIRS.md index fa560b09..72aa6faf 100644 --- a/docs/src/SEIRS.md +++ b/docs/src/SEIRS.md @@ -17,11 +17,11 @@ show(run(`fair init --ci`)) ```@example print(pwd()) -read(`fair pull /home/runner/work/DataPipeline.jl/DataPipeline.jl/examples/fdp/SEIRSconfig.yaml`, String) +read(`fair pull /home/runner/work/DataPipeline.jl/DataPipeline.jl/examples/SEIRS/full_config.yaml`, String) ``` dsf ```@example -show(run(`fair run /home/runner/work/DataPipeline.jl/DataPipeline.jl/examples/fdp/SEIRSconfig.yaml`)) +show(run(`fair run /home/runner/work/DataPipeline.jl/DataPipeline.jl/examples/SEIRS/full_config.yaml`)) ``` diff --git a/examples/fdp/Manifest.toml b/examples/SEIRS/Manifest.toml similarity index 100% rename from examples/fdp/Manifest.toml rename to examples/SEIRS/Manifest.toml diff --git a/examples/fdp/Project.toml b/examples/SEIRS/Project.toml similarity index 100% rename from examples/fdp/Project.toml rename to examples/SEIRS/Project.toml diff --git a/examples/fdp/SEIRSconfig.yaml b/examples/SEIRS/full_config.yaml similarity index 87% rename from examples/fdp/SEIRSconfig.yaml rename to examples/SEIRS/full_config.yaml index dddd6431..32becad4 100644 --- a/examples/fdp/SEIRSconfig.yaml +++ b/examples/SEIRS/full_config.yaml @@ -2,8 +2,8 @@ run_metadata: default_input_namespace: testing description: SEIRS Model julia script: | - julia --project=examples/fdp -e "using Pkg; Pkg.instantiate()" - julia --project=examples/fdp examples/fdp/seirs_sim.jl + julia --project=examples/SEIRS -e "using Pkg; Pkg.instantiate()" + julia --project=examples/SEIRS examples/SEIRS/seirs_sim.jl register: - namespace: PSU diff --git a/examples/SEIRS/register_config.yaml b/examples/SEIRS/register_config.yaml new file mode 100644 index 00000000..bde959a3 --- /dev/null +++ b/examples/SEIRS/register_config.yaml @@ -0,0 +1,20 @@ +run_metadata: + default_input_namespace: testing + description: SEIRS Model julia registration + +register: +- namespace: PSU + full_name: Pennsylvania State University + website: https://ror.org/04p491231 + +- external_object: SEIRS_model/parameters + namespace_name: PSU + root: https://raw.githubusercontent.com/ + path: FAIRDataPipeline/rSimpleModel/main/inst/extdata/static_params_SEIRS.csv + title: Static parameters of the model + description: Static parameters of the model + identifier: https://doi.org/10.1038/s41592-020-0856-2 + file_type: csv + release_date: 2020-06-01T12:00 + version: "1.0.0" + primary: False diff --git a/examples/SEIRS/repeat_config.yaml b/examples/SEIRS/repeat_config.yaml new file mode 100644 index 00000000..f6dbc3ed --- /dev/null +++ b/examples/SEIRS/repeat_config.yaml @@ -0,0 +1,23 @@ +run_metadata: + default_input_namespace: PSU + description: SEIRS Model repeat julia + script: | + julia --project=examples/SEIRS -e "using Pkg; Pkg.instantiate()" + julia --project=examples/SEIRS examples/SEIRS/seirs_sim.jl + +read: +- data_product: SEIRS_model/parameters + version: "1.0.0" + +write: +- data_product: model_output + description: SEIRS model results + file_type: csv + use: + data_product: SEIRS_model/results/model_output/julia + +- data_product: figure + description: SEIRS output plot + file_type: pdf + use: + data_product: SEIRS_model/results/figure/julia diff --git a/examples/fdp/seirs_sim.jl b/examples/SEIRS/seirs_sim.jl similarity index 94% rename from examples/fdp/seirs_sim.jl rename to examples/SEIRS/seirs_sim.jl index d13568f1..aace2895 100644 --- a/examples/fdp/seirs_sim.jl +++ b/examples/SEIRS/seirs_sim.jl @@ -1,12 +1,12 @@ ### SEIRS model example using DataPipeline -using DataPipeline.SeirsModel +using DataPipeline.SEIRSModel using CSV using DataFrames using Plots # Initialise code run -handle = initialise() +handle = DataPipeline.initialise() # Read model parameters path = link_read!(handle, "SEIRS_model/parameters") diff --git a/src/DataPipeline.jl b/src/DataPipeline.jl index 02fc7f80..e2293691 100644 --- a/src/DataPipeline.jl +++ b/src/DataPipeline.jl @@ -38,7 +38,6 @@ FDP_LOCAL_TOKEN() = get(ENV, "FDP_LOCAL_TOKEN", "fake_token") include("core.jl") include("api.jl") -export initialise, finalise export link_read!, link_write! export read_array, read_table, read_distribution, read_estimate export write_array, write_table, write_distribution, write_estimate @@ -52,7 +51,7 @@ include("api_audit.jl") # DR audits include("testing.jl") # ---- SEIRS model ---- -module SeirsModel +module SEIRSModel include("model.jl") export modelseirs, plotseirs, getparameter diff --git a/src/api.jl b/src/api.jl index d5c92e8b..967ed4c5 100644 --- a/src/api.jl +++ b/src/api.jl @@ -143,7 +143,7 @@ end read_array(handle, data_product[, component]) Read [array] data product. -- note that it must already have been downloaded from the remote data store using `fdp pull`. +- note that it must already have been downloaded from the remote data store using `fair pull`. - the latest version of the data is read unless otherwise specified. """ function read_array(handle::DataRegistryHandle, data_product::String, component=nothing) @@ -174,7 +174,7 @@ end Read [table] data product. - note that it must already have been downloaded from the remote data store using - `fdp pull`. + `fair pull`. - the latest version of the data is read unless otherwise specified. """ function read_table(handle::DataRegistryHandle, data_product::String, component=nothing) @@ -190,7 +190,7 @@ end Read TOML-based data product. - note that it must already have been downloaded from the remote data store using - `fdp pull`. + `fair pull`. - the specific version can be specified in the config file (else the latest version is used.) """ @@ -216,7 +216,7 @@ end Read TOML-based data product. - note that it must already have been downloaded from the remote data store using - `fdp pull`. + `fair pull`. - the specific version can be specified in the config file (else the latest version is used.) """ diff --git a/src/fdp_i.jl b/src/fdp_i.jl index b5819815..8402ee35 100644 --- a/src/fdp_i.jl +++ b/src/fdp_i.jl @@ -2,10 +2,10 @@ # - implements: https://fairdatapipeline.github.io/docs/interface/example0/ ## BASELINE FDP FUNCTIONALITY: -# fdp pull config.yaml -# fdp run config.yaml -# fdp push config.yaml -## NB. 'fdp' -> FAIR +# fair pull config.yaml +# fair run config.yaml +# fair push config.yaml +## NB. 'fair' -> FAIR ## LOCAL DR INSTRUCTIONS: # - start using: ~/.fair/registry/scripts/start_fair_registry diff --git a/src/testing.jl b/src/testing.jl index 9a314064..a4a2228a 100644 --- a/src/testing.jl +++ b/src/testing.jl @@ -28,7 +28,7 @@ function _createconfig(path) "write_data_store" => write_data_store, "script_path" => expanduser("~/.fair/registry/datastore/script.sh"), "description" => "A description", - "script" => "julia examples/fdp/seirs_sim.jl", + "script" => "julia examples/SEIRS/seirs_sim.jl", "remote_repo" => "https://github.com/FAIRDataPipeline/DataPipeline.jl.git", "local_data_registry_url" => "http://localhost:8000/api/") data = Dict("run_metadata" => run_metadata) diff --git a/test/runtests.sh b/test/runtests.sh index 64263b64..664f1db8 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -23,8 +23,8 @@ source $WORKSPACE/.venv/$ACTIVATE_DIR/activate fair registry install fair registry start fair init --ci -if ! fair pull --local $WORKSPACE/examples/fdp/SEIRSconfig.yaml; then exit 1; fi -if ! fair run --dirty --local $WORKSPACE/examples/fdp/SEIRSconfig.yaml; then exit 1; fi +if ! fair pull --local $WORKSPACE/examples/SEIRS/full_config.yaml; then exit 1; fi +if ! fair run --dirty --local $WORKSPACE/examples/SEIRS/full_config.yaml; then exit 1; fi TEST_SCRIPT="$(printf ' %q' "$@")" echo Test: "$TEST_SCRIPT" diff --git a/test/test_api.jl b/test/test_api.jl index eba55729..c7ed9f61 100644 --- a/test/test_api.jl +++ b/test/test_api.jl @@ -10,7 +10,7 @@ datetime = Dates.format(Dates.now(), "yyyymmdd-HHMMSS") cpath = joinpath("coderun", datetime, "config.yaml") config = DataPipeline._createconfig(cpath) -handle = initialise(config, config) +handle = DataPipeline.initialise(config, config) datastore = handle.config["run_metadata"]["write_data_store"] namespace = handle.config["run_metadata"]["default_output_namespace"] version = "0.0.1" @@ -39,7 +39,7 @@ Test.@testset "link_write()" begin use_version = version) DataPipeline._addwrite(config, data_product2, "description", file_type = file_type, use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Check function output @@ -60,7 +60,7 @@ Test.@testset "link_write()" begin end # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check path test_path = joinpath("$(datastore)$(namespace)", "$data_product", @@ -84,7 +84,7 @@ Test.@testset "link_read()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addread(config, data_product, use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.inputs == Dict() # Check function output @@ -95,7 +95,7 @@ Test.@testset "link_read()" begin @test length(handle.inputs) == 1 # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check data dat = open(path1) do file @@ -115,7 +115,7 @@ Test.@testset "write_array()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Write components @@ -133,7 +133,7 @@ Test.@testset "write_array()" begin isfile(path1) # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) newpath1 = handle.outputs[(data_product, component1)]["path"] newpath2 = handle.outputs[(data_product, component2)]["path"] @@ -164,7 +164,7 @@ Test.@testset "read_array()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addread(config, data_product, use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Read components @@ -174,7 +174,7 @@ Test.@testset "read_array()" begin @test dat2 == data2 # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check handle @test handle.inputs[(data_product, component1)]["use_dp"] == data_product @@ -187,7 +187,7 @@ Test.@testset "write_estimate()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Write components @@ -210,7 +210,7 @@ Test.@testset "write_estimate()" begin @test c2 == estimate2 # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check handle newpath1 = handle.outputs[(data_product, component1)]["path"] @@ -229,7 +229,7 @@ Test.@testset "read_estimate()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addread(config, data_product, use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Read components @@ -239,7 +239,7 @@ Test.@testset "read_estimate()" begin @test dat2 == estimate2 # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check handle @test handle.inputs[(data_product, component1)]["use_dp"] == data_product @@ -252,7 +252,7 @@ Test.@testset "write_distribution()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Write components @@ -278,7 +278,7 @@ Test.@testset "write_distribution()" begin @test c2 == distribution # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check handle newpath1 = handle.outputs[(data_product, component1)]["path"] @@ -296,7 +296,7 @@ Test.@testset "read_distribution()" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addread(config, data_product, use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) @test handle.outputs == Dict() # Read components @@ -306,7 +306,7 @@ Test.@testset "read_distribution()" begin @test dat2 == distribution # Finalise Code Run - finalise(handle) + DataPipeline.finalise(handle) # Check handle @test handle.inputs[(data_product, component1)]["use_dp"] == data_product @@ -324,7 +324,7 @@ Test.@testset "new components aren't added to existing data products" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) # Write component msg = string("data product already exists in registry: ", data_product, " :-(ns: ", @@ -339,7 +339,7 @@ Test.@testset "new components aren't added to existing data products" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) # Write component msg = string("data product already exists in registry: ", data_product, " :-(ns: ", @@ -354,7 +354,7 @@ Test.@testset "new components aren't added to existing data products" begin # Create working config.yaml config = DataPipeline._createconfig(cpath) DataPipeline._addwrite(config, data_product, "description", use_version = version) - handle = initialise(config, config) + handle = DataPipeline.initialise(config, config) # Write component msg = string("data product already exists in registry: ", data_product, " :-(ns: ", From 8863bb5fbfbdb0aece035670b7392bc333d7c795 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 15:21:38 +0100 Subject: [PATCH 14/17] Minor fixes --- examples/SEIRS/seirs_sim.jl | 2 +- src/DataPipeline.jl | 1 - src/core.jl | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/SEIRS/seirs_sim.jl b/examples/SEIRS/seirs_sim.jl index aace2895..7dd7b284 100644 --- a/examples/SEIRS/seirs_sim.jl +++ b/examples/SEIRS/seirs_sim.jl @@ -39,4 +39,4 @@ path = link_write!(handle, "figure") savefig(g, path) # Register code run in local registry -finalise(handle) +DataPipeline.finalise(handle) diff --git a/src/DataPipeline.jl b/src/DataPipeline.jl index e2293691..21f83315 100644 --- a/src/DataPipeline.jl +++ b/src/DataPipeline.jl @@ -33,7 +33,6 @@ else end FDP_PATH_CONFIG() = joinpath(FDP_CONFIG_DIR(), "config.yaml") FDP_PATH_SUBMISSION() = joinpath(FDP_CONFIG_DIR(), FDP_SUBMISSION_SCRIPT) -FDP_LOCAL_TOKEN() = get(ENV, "FDP_LOCAL_TOKEN", "fake_token") include("core.jl") diff --git a/src/core.jl b/src/core.jl index a934cd8c..8b17443b 100644 --- a/src/core.jl +++ b/src/core.jl @@ -190,5 +190,5 @@ end Get local repository access token. """ function _gettoken() - return string("token ", FDP_LOCAL_TOKEN()) + return string("token ", ENV["FDP_LOCAL_TOKEN"]) end From fa309a43abc751259a01d027637a013f2846b0b9 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 16:13:23 +0100 Subject: [PATCH 15/17] Fix paths --- src/DataPipeline.jl | 8 ++------ src/api.jl | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/DataPipeline.jl b/src/DataPipeline.jl index 21f83315..07f9277f 100644 --- a/src/DataPipeline.jl +++ b/src/DataPipeline.jl @@ -20,19 +20,15 @@ using URIs const C_DEBUG_MODE = false const LOCAL_DR_STEM = "http://localhost" -const LOCAL_DR_PORTLESS = string(LOCAL_DR_STEM, "/api/") -const STR_ROOT = string(LOCAL_DR_PORTLESS, "storage_root/") const API_ROOT = string(LOCAL_DR_STEM, ":8000", "/api/") -const SL_ROOT = string(LOCAL_DR_PORTLESS, "storage_location/") -const DATA_OUT = "./out/" +const SL_ROOT = string(API_ROOT, "storage_location/") FDP_CONFIG_DIR() = get(ENV, "FDP_CONFIG_DIR", ".") @static if Sys.iswindows() const FDP_SUBMISSION_SCRIPT = "script.bat" else const FDP_SUBMISSION_SCRIPT = "script.sh" end -FDP_PATH_CONFIG() = joinpath(FDP_CONFIG_DIR(), "config.yaml") -FDP_PATH_SUBMISSION() = joinpath(FDP_CONFIG_DIR(), FDP_SUBMISSION_SCRIPT) +const FDP_CONFIG_FILE = "config.yaml" include("core.jl") diff --git a/src/api.jl b/src/api.jl index 967ed4c5..1033451a 100644 --- a/src/api.jl +++ b/src/api.jl @@ -6,8 +6,8 @@ using NetCDF Reads in working config.yaml file, generates a new Code Run entry, and returns a `DataRegistryHandle` containing various metadata. """ -function initialise(config_file::String = FDP_PATH_CONFIG(), - submission_script::String = FDP_PATH_SUBMISSION()) +function initialise(config_file::String = joinpath(FDP_CONFIG_DIR(), FDP_CONFIG_FILE), + submission_script::String = joinpath(FDP_CONFIG_DIR(), FDP_SUBMISSION_SCRIPT)) # Read working config file print("processing config file: ", config_file) config = YAML.load_file(config_file) From 5dba19db2b0b6806a8377d6e8ddbe0d9a771faf7 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 16:40:14 +0100 Subject: [PATCH 16/17] Rearrange using commands --- Project.toml | 2 -- src/DataPipeline.jl | 11 ----------- src/api.jl | 2 ++ src/api_audit.jl | 2 ++ src/core.jl | 5 +++++ src/data_prod_proc.jl | 1 + src/fdp_i.jl | 10 ++++++++-- src/model.jl | 9 ++++----- src/testing.jl | 4 ++++ 9 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Project.toml b/Project.toml index 2919e984..f723e9fc 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -FTPClient = "01fcc997-4f28-56b8-8a06-30002c134abb" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" @@ -25,7 +24,6 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" AxisArrays = "0.4" CSV = "0.8, 0.9, 0.10" DataFrames = "1" -FTPClient = "1" HTTP = "0.9, 1" JSON = "0.21" NetCDF = "0.11" diff --git a/src/DataPipeline.jl b/src/DataPipeline.jl index 07f9277f..f5a15139 100644 --- a/src/DataPipeline.jl +++ b/src/DataPipeline.jl @@ -7,17 +7,6 @@ language-agnostic RESTful API that is used to interact with the Data Registry. module DataPipeline -using CSV -using DataFrames -using Dates -using FTPClient -using HTTP -using JSON -using Plots -using SHA -using YAML -using URIs - const C_DEBUG_MODE = false const LOCAL_DR_STEM = "http://localhost" const API_ROOT = string(LOCAL_DR_STEM, ":8000", "/api/") diff --git a/src/api.jl b/src/api.jl index 1033451a..a24f53c3 100644 --- a/src/api.jl +++ b/src/api.jl @@ -1,3 +1,5 @@ +using YAML +using Dates using NetCDF """ diff --git a/src/api_audit.jl b/src/api_audit.jl index 04f0946f..26033f54 100644 --- a/src/api_audit.jl +++ b/src/api_audit.jl @@ -1,3 +1,5 @@ +using URIs + ### what's my file # NB. THIS IS NOW BROKEN DUE TO CHANGES TO THE DR SCHEMA *********** function whats_my_hash(fh::String) diff --git a/src/core.jl b/src/core.jl index 8b17443b..73e88b15 100644 --- a/src/core.jl +++ b/src/core.jl @@ -1,3 +1,8 @@ +using JSON +using HTTP +using URIs +using SHA + """ DataRegistryHandle diff --git a/src/data_prod_proc.jl b/src/data_prod_proc.jl index cc7893f1..a0032b14 100644 --- a/src/data_prod_proc.jl +++ b/src/data_prod_proc.jl @@ -1,6 +1,7 @@ using TOML using AxisArrays using NetCDF +using CSV ### hdf5 file processing ### const ARRAY_OBJ_NAME = "array" diff --git a/src/fdp_i.jl b/src/fdp_i.jl index 8402ee35..10054f5c 100644 --- a/src/fdp_i.jl +++ b/src/fdp_i.jl @@ -1,3 +1,7 @@ +using JSON +using URIs +using TOML + ### new interface for FAIR data pipeline ### # - implements: https://fairdatapipeline.github.io/docs/interface/example0/ @@ -71,8 +75,10 @@ function _registerobject(path::String, root::String, description::String; end # Register object - object_query = Dict("description" => description, "storage_location" => storage_loc_uri, - "authors" => [authors_url], "file_type" => file_type_url) + object_query = Dict("description" => description, + "storage_location" => storage_loc_uri, + "authors" => [authors_url], + "file_type" => file_type_url) object_url = _postentry("object", object_query) return object_url diff --git a/src/model.jl b/src/model.jl index 43767a0c..277cb707 100644 --- a/src/model.jl +++ b/src/model.jl @@ -25,7 +25,7 @@ function modelseirs(initial_state::Dict, timesteps::Int64, years::Int64, mu = time_unit_days / (inv_mu * 365.25) sigma = time_unit_days / inv_sigma - results = DataFrames.DataFrame(time=0, S=S, E=E, I=I, R=R) + results = DataFrame(time=0, S=S, E=E, I=I, R=R) for t = 1:timesteps N = S + E + I + R @@ -49,9 +49,8 @@ function modelseirs(initial_state::Dict, timesteps::Int64, years::Int64, new_I = results.I[t] + I_rate new_R = results.R[t] + R_rate - new = DataFrames.DataFrame(time=t * time_unit_days, - S=new_S, E=new_E, - I=new_I, R=new_R) + new = DataFrame(time=t * time_unit_days, S=new_S, + E=new_E, I=new_I, R=new_R) results = vcat(results, new) end @@ -92,4 +91,4 @@ Get parameter from DataFrame function getparameter(data::DataFrames.DataFrame, parameter::String) output = filter(row -> row.param == parameter, data).value[1] return output -end \ No newline at end of file +end diff --git a/src/testing.jl b/src/testing.jl index a4a2228a..2202c8e5 100644 --- a/src/testing.jl +++ b/src/testing.jl @@ -1,3 +1,7 @@ +using YAML +using Dates +using SHA + """ _startregistry() From 2cc515b029c14be9c05295cd515194dc13e1bdb8 Mon Sep 17 00:00:00 2001 From: richardreeve Date: Sun, 25 Jun 2023 16:40:33 +0100 Subject: [PATCH 17/17] Fix example manifest --- examples/SEIRS/Manifest.toml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/examples/SEIRS/Manifest.toml b/examples/SEIRS/Manifest.toml index 86a81aa3..8d54a420 100644 --- a/examples/SEIRS/Manifest.toml +++ b/examples/SEIRS/Manifest.toml @@ -141,7 +141,7 @@ uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" version = "1.5.0" [[deps.DataPipeline]] -deps = ["AxisArrays", "CSV", "DataFrames", "Dates", "FTPClient", "HTTP", "JSON", "NetCDF", "Plots", "PrettyTables", "SHA", "SQLite", "TOML", "URIs", "UnicodePlots", "YAML"] +deps = ["AxisArrays", "CSV", "DataFrames", "Dates", "HTTP", "JSON", "NetCDF", "Plots", "PrettyTables", "SHA", "SQLite", "TOML", "URIs", "UnicodePlots", "YAML"] path = "../.." uuid = "9ced6f0a-eb77-43a8-bbd1-bbf3031b0d12" version = "0.53.2" @@ -206,12 +206,6 @@ git-tree-sha1 = "74faea50c1d007c85837327f6775bea60b5492dd" uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" version = "4.4.2+2" -[[deps.FTPClient]] -deps = ["LibCURL", "URIParser"] -git-tree-sha1 = "fcdcac297167852b23bae6d7bb3bf82a726ff70c" -uuid = "01fcc997-4f28-56b8-8a06-30002c134abb" -version = "1.2.1" - [[deps.FilePathsBase]] deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12" @@ -886,12 +880,6 @@ git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.9.13" -[[deps.URIParser]] -deps = ["Unicode"] -git-tree-sha1 = "53a9f49546b8d2dd2e688d216421d050c9a31d0d" -uuid = "30578b45-9adc-5946-b283-645ec420af67" -version = "0.4.1" - [[deps.URIs]] git-tree-sha1 = "074f993b0ca030848b897beff716d93aca60f06a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"