From 179fd4fc80d26b56d2a9d0fdef66c55216be50d0 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:22:01 +0200 Subject: [PATCH 1/6] add formatter boilerplate --- .JuliaFormatter.toml | 1 + .github/workflows/format-check.yml | 37 +++++ .gitignore | 1 + format/Manifest.toml | 221 +++++++++++++++++++++++++++++ format/Project.toml | 2 + format/run.jl | 14 ++ 6 files changed, 276 insertions(+) create mode 100644 .JuliaFormatter.toml create mode 100644 .github/workflows/format-check.yml create mode 100644 format/Manifest.toml create mode 100644 format/Project.toml create mode 100644 format/run.jl diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 00000000..857c3ae3 --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "yas" diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml new file mode 100644 index 00000000..3cb68a38 --- /dev/null +++ b/.github/workflows/format-check.yml @@ -0,0 +1,37 @@ +name: format-check +on: + push: + branches: + - 'main' + - /^release-.*$/ + tags: ['*'] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + format-check: + name: Format check (Julia ${{ matrix.julia-version }} - ${{ github.event_name }}) + # Run on push's or non-draft PRs + if: (github.event_name == 'push') || (github.event.pull_request.draft == false) + runs-on: ubuntu-latest + strategy: + matrix: + julia-version: [1.9.3] + steps: + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - uses: actions/checkout@v1 + - name: Instantiate `format` environment and format + run: | + julia --project=format -e 'using Pkg; Pkg.instantiate()' + julia --project=format 'format/run.jl' + - uses: reviewdog/action-suggester@v1 + if: github.event_name == 'pull_request' + with: + tool_name: JuliaFormatter + fail_on_error: true diff --git a/.gitignore b/.gitignore index e03551c7..872239aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ docs/build/ docs/site/ .DS_Store tmpdir/* +!format/Manifest.toml diff --git a/format/Manifest.toml b/format/Manifest.toml new file mode 100644 index 00000000..ba35806a --- /dev/null +++ b/format/Manifest.toml @@ -0,0 +1,221 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.3" +manifest_format = "2.0" +project_hash = "30b405be1c677184b7703a9bfb3d2100029ccad0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.CSTParser]] +deps = ["Tokenize"] +git-tree-sha1 = "3ddd48d200eb8ddf9cb3e0189fc059fd49b97c1f" +uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" +version = "3.3.6" + +[[deps.CommonMark]] +deps = ["Crayons", "JSON", "PrecompileTools", "URIs"] +git-tree-sha1 = "532c4185d3c9037c0237546d817858b23cf9e071" +uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" +version = "0.8.12" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "8a62af3e248a8c4bad6b32cbbe663ae02275e32c" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.10.0" + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + + [deps.Compat.weakdeps] + Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.15" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.Glob]] +git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" +uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" +version = "1.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.JuliaFormatter]] +deps = ["CSTParser", "CommonMark", "DataStructures", "Glob", "Pkg", "PrecompileTools", "Tokenize"] +git-tree-sha1 = "80031f6e58b09b0de4553bf63d9a36ec5db57967" +uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899" +version = "1.0.39" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.2" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.7.2" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.2" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.0" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.1" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Tokenize]] +git-tree-sha1 = "90538bf898832b6ebd900fa40f223e695970e3a5" +uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" +version = "0.5.25" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/format/Project.toml b/format/Project.toml new file mode 100644 index 00000000..f3aab8b8 --- /dev/null +++ b/format/Project.toml @@ -0,0 +1,2 @@ +[deps] +JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" diff --git a/format/run.jl b/format/run.jl new file mode 100644 index 00000000..3499f038 --- /dev/null +++ b/format/run.jl @@ -0,0 +1,14 @@ +using JuliaFormatter + +function main() + perfect = format("."; style=YASStyle(), verbose=true) + if perfect + @info "Linting complete - no files altered" + else + @info "Linting complete - files altered" + run(`git status`) + end + return nothing +end + +main() From 301b1d1073d6162215006b8539cb32af4c9a88b4 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:22:49 +0200 Subject: [PATCH 2/6] format --- OndaEDFSchemas.jl/src/OndaEDFSchemas.jl | 27 ++-- OndaEDFSchemas.jl/test/runtests.jl | 19 +-- docs/make.jl | 4 +- src/OndaEDF.jl | 3 +- src/export_edf.jl | 43 ++++-- src/import_edf.jl | 118 ++++++++++------- src/standards.jl | 97 ++++++++++---- test/export.jl | 38 ++++-- test/import.jl | 84 +++++++----- test/runtests.jl | 168 +++++++++++++++++------- test/signal_labels.jl | 24 ++-- 11 files changed, 408 insertions(+), 217 deletions(-) diff --git a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl index c5a6e221..d5833f77 100644 --- a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl +++ b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl @@ -29,10 +29,14 @@ export PlanV1, PlanV2, FilePlanV1, FilePlanV2, EDFAnnotationV1 kind::Union{Missing,AbstractString} = lift(String, kind) channel::Union{Missing,AbstractString} = lift(String, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) - sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_resolution_in_unit) - sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_offset_in_unit) - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) - sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_rate) + sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_resolution_in_unit) + sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_offset_in_unit) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, + sample_type) + sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_rate) # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end @@ -52,21 +56,21 @@ end seconds_per_record::Float64 # Onda.SignalV2 fields (channels -> channel), may be missing recording::Union{UUID,Missing} = lift(UUID, recording) - sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, sensor_type) + sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, + sensor_type) sensor_label::Union{Missing,AbstractString} = lift(_validate_signal_sensor_label, coalesce(sensor_label, sensor_type)) channel::Union{Missing,AbstractString} = lift(_validate_signal_channel, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) sample_resolution_in_unit::Union{Missing,Float64} sample_offset_in_unit::Union{Missing,Float64} - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, + sample_type) sample_rate::Union{Missing,Float64} # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end - - const PLAN_DOC_TEMPLATE = """ @version PlanV{{ VERSION }} begin # EDF.SignalHeader fields @@ -159,11 +163,14 @@ end @doc _file_plan_doc(1) FilePlanV1 @doc _file_plan_doc(2) FilePlanV2 -const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion,FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} +const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion, + FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{String}) = AbstractString # we need this because Arrow write can introduce a Missing for the error column # (I think because of how missing/nothing sentinels are handled?) -Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) = Union{Nothing,Missing,AbstractString} +function Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) + return Union{Nothing,Missing,AbstractString} +end @schema "edf.annotation" EDFAnnotation diff --git a/OndaEDFSchemas.jl/test/runtests.jl b/OndaEDFSchemas.jl/test/runtests.jl index c9354eaf..2bf12d62 100644 --- a/OndaEDFSchemas.jl/test/runtests.jl +++ b/OndaEDFSchemas.jl/test/runtests.jl @@ -32,19 +32,19 @@ function mock_plan(; v, rng=GLOBAL_RNG) physical_dimension="uV", physical_minimum=0.0, physical_maximum=2.0, - digital_minimum=-1f4, - digital_maximum=1f4, + digital_minimum=-1.0f4, + digital_maximum=1.0f4, prefilter="HP 0.1Hz; LP 80Hz; N 60Hz", samples_per_record=128, seconds_per_record=1.0, channel=ingested ? "cz-m1" : missing, sample_unit=ingested ? "microvolt" : missing, - sample_resolution_in_unit=ingested ? 1f-4 : missing, + sample_resolution_in_unit=ingested ? 1.0f-4 : missing, sample_offset_in_unit=ingested ? 1.0 : missing, sample_type=ingested ? "float32" : missing, - sample_rate=ingested ? 1/128 : missing, + sample_rate=ingested ? 1 / 128 : missing, error=errored ? "Error blah blah" : nothing, - recording= (ingested && rand(rng, Bool)) ? uuid4() : missing, + recording=(ingested && rand(rng, Bool)) ? uuid4() : missing, specific_kwargs...) end @@ -63,7 +63,7 @@ end @testset "Schema version $v" for v in (1, 2) SamplesInfo = v == 1 ? Onda.SamplesInfoV1 : SamplesInfoV2 - + @testset "ondaedf.plan@$v" begin rng = StableRNG(10) plans = mock_plan(30; v, rng) @@ -75,21 +75,22 @@ end # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in plans if !ismissing(p.channel)) + for p in plans if !ismissing(p.channel)) end @testset "ondaedf.file-plan@$v" begin rng = StableRNG(11) file_plans = mock_file_plan(50; v, rng) schema = Tables.schema(file_plans) - @test nothing === Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) + @test nothing === + Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) tbl = Arrow.Table(Arrow.tobuffer(file_plans; maxdepth=9)) @test isequal(Tables.columntable(tbl), Tables.columntable(file_plans)) # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in file_plans if !ismissing(p.channel)) + for p in file_plans if !ismissing(p.channel)) end end diff --git a/docs/make.jl b/docs/make.jl index 37eefe3f..3bbb4af3 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,12 +1,12 @@ using OndaEDF using Documenter -makedocs(modules=[OndaEDF, OndaEDF.OndaEDFSchemas], +makedocs(; modules=[OndaEDF, OndaEDF.OndaEDFSchemas], sitename="OndaEDF", authors="Beacon Biosignals and other contributors", pages=["OndaEDF" => "index.md", "Converting from EDF" => "convert-to-onda.md", "API Documentation" => "api.md"]) -deploydocs(repo="github.com/beacon-biosignals/OndaEDF.jl.git", +deploydocs(; repo="github.com/beacon-biosignals/OndaEDF.jl.git", push_preview=true) diff --git a/src/OndaEDF.jl b/src/OndaEDF.jl index 2da65f6f..85c86c74 100644 --- a/src/OndaEDF.jl +++ b/src/OndaEDF.jl @@ -16,7 +16,8 @@ using Legolas: lift using Tables: rowmerge export write_plan -export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, plan_edf_to_onda_samples_groups, store_edf_as_onda +export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, + plan_edf_to_onda_samples_groups, store_edf_as_onda export onda_to_edf include("standards.jl") diff --git a/src/export_edf.jl b/src/export_edf.jl index 0685d204..6bf34d21 100644 --- a/src/export_edf.jl +++ b/src/export_edf.jl @@ -12,7 +12,8 @@ end SignalExtrema(samples::Samples) = SignalExtrema(samples.info) function SignalExtrema(info::SamplesInfoV2) digital_extrema = (typemin(sample_type(info)), typemax(sample_type(info))) - physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + info.sample_offset_in_unit + physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + + info.sample_offset_in_unit return SignalExtrema(physical_extrema..., digital_extrema...) end @@ -23,7 +24,9 @@ end const DATA_RECORD_SIZE_LIMIT = 30720 const EDF_BYTE_LIMIT = 8 -edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) = Int16(samples.info.sample_rate * seconds_per_record) +function edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) + return Int16(samples.info.sample_rate * seconds_per_record) +end _rationalize(x) = rationalize(x) _rationalize(x::Int) = x // 1 @@ -40,10 +43,12 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) else scale = gcd(numerator.(sample_rates) .* seconds_per_record) samples_per_record ./= scale - sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && throw(RecordSizeException(all_samples)) + sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && + throw(RecordSizeException(all_samples)) seconds_per_record /= scale end - sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && throw(EDFPrecisionError(seconds_per_record)) + sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && + throw(EDFPrecisionError(seconds_per_record)) end record_duration_in_nanoseconds = Nanosecond(seconds_per_record * 1_000_000_000) signal_duration = maximum(Onda.duration, all_samples) @@ -53,7 +58,7 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) end struct RecordSizeException <: Exception - samples + samples::Any end struct EDFPrecisionError <: Exception @@ -64,13 +69,13 @@ function Base.showerror(io::IO, exception::RecordSizeException) print(io, "RecordSizeException: sample rates ") print(io, [s.info.sample_rate for s in exception.samples]) print(io, " cannot be resolved to a data record size smaller than ") - print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") + return print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") end function Base.showerror(io::IO, exception::EDFPrecisionError) print(io, "EDFPrecisionError: String representation of value ") print(io, exception.value) - print(io, " is longer than 8 ASCII characters") + return print(io, " is longer than 8 ASCII characters") end ##### @@ -88,8 +93,10 @@ end function onda_samples_to_edf_header(samples::AbstractVector{<:Samples}; version::AbstractString="0", - patient_metadata=EDF.PatientID(missing, missing, missing, missing), - recording_metadata=EDF.RecordingID(missing, missing, missing, missing), + patient_metadata=EDF.PatientID(missing, missing, + missing, missing), + recording_metadata=EDF.RecordingID(missing, missing, + missing, missing), is_contiguous::Bool=true, start::DateTime=DateTime(Year(1985))) return EDF.FileHeader(version, patient_metadata, recording_metadata, start, @@ -177,7 +184,8 @@ function reencode_samples(samples::Samples, sample_type::Type{<:Integer}=Int16) return encode(new_samples) end -function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, seconds_per_record::Float64) +function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, + seconds_per_record::Float64) edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}}[] for samples in onda_samples # encode samples, rescaling if necessary @@ -187,7 +195,8 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, se for channel_name in samples.info.channels sample_count = edf_sample_count_per_record(samples, seconds_per_record) physical_dimension = onda_to_edf_unit(samples.info.sample_unit) - edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, channel_name), + edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, + channel_name), "", physical_dimension, extrema.physical_min, extrema.physical_max, extrema.digital_min, extrema.digital_max, @@ -195,7 +204,10 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, se # manually convert here in case we have input samples whose encoded # values are convertible losslessly to Int16: sample_data = Int16.(vec(samples[channel_name, :].data)) - padding = Iterators.repeated(zero(Int16), (sample_count - (length(sample_data) % sample_count)) % sample_count) + padding = Iterators.repeated(zero(Int16), + (sample_count - + (length(sample_data) % sample_count)) % + sample_count) edf_signal_samples = append!(sample_data, padding) push!(edf_signals, EDF.Signal(edf_signal_header, edf_signal_samples)) end @@ -243,13 +255,16 @@ function onda_to_edf(samples::AbstractVector{<:Samples}, annotations=[]; kwargs. edf_header = onda_samples_to_edf_header(samples; kwargs...) edf_signals = onda_samples_to_edf_signals(samples, edf_header.seconds_per_record) if !isempty(annotations) - records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, nothing, String[""])] + records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, + nothing, String[""])] for i in 0:(edf_header.record_count - 1)] for annotation in sort(Tables.rowtable(annotations); by=row -> start(row.span)) annotation_onset_in_seconds = start(annotation.span).value / 1e9 annotation_duration_in_seconds = duration(annotation.span).value / 1e9 matching_record = records[Int(fld(annotation_onset_in_seconds, edf_header.seconds_per_record)) + 1] - tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, annotation_duration_in_seconds, [annotation.value]) + tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, + annotation_duration_in_seconds, + [annotation.value]) push!(matching_record, tal) end push!(edf_signals, EDF.AnnotationsSignal(records)) diff --git a/src/import_edf.jl b/src/import_edf.jl index 2601f971..c0ccd0a8 100644 --- a/src/import_edf.jl +++ b/src/import_edf.jl @@ -40,12 +40,12 @@ end # the initial reference name should match the canonical channel name, # otherwise the channel extraction will be rejected. function _normalize_references(original_label, canonical_names) - label = replace(_safe_lowercase(original_label), r"\s"=>"") - label = replace(replace(label, '('=>""), ')'=>"") - label = replace(label, r"\*$"=>"") - label = replace(label, '-'=>'…') - label = replace(label, '+'=>"…+…") - label = replace(label, '/'=>"…/…") + label = replace(_safe_lowercase(original_label), r"\s" => "") + label = replace(replace(label, '(' => ""), ')' => "") + label = replace(label, r"\*$" => "") + label = replace(label, '-' => '…') + label = replace(label, '+' => "…+…") + label = replace(label, '/' => "…/…") m = match(r"^\[(.*)\]$", label) if m !== nothing label = only(m.captures) @@ -68,8 +68,8 @@ function _normalize_references(original_label, canonical_names) end end recombined = '-'^startswith(original_label, '-') * join(parts, '-') - recombined = replace(recombined, "-+-"=>"_plus_") - recombined = replace(recombined, "-/-"=>"_over_") + recombined = replace(recombined, "-+-" => "_plus_") + recombined = replace(recombined, "-/-" => "_over_") return first(parts), recombined end @@ -162,17 +162,18 @@ end ##### struct MismatchedSampleRateError <: Exception - sample_rates + sample_rates::Any end function Base.showerror(io::IO, err::MismatchedSampleRateError) - print(io, """ - found mismatched sample rate between channel encodings: $(err.sample_rates) - - OndaEDF does not currently automatically resolve mismatched sample rates; - please preprocess your data before attempting `import_edf` so that channels - of the same signal share a common sample rate. - """) + return print(io, + """ + found mismatched sample rate between channel encodings: $(err.sample_rates) + + OndaEDF does not currently automatically resolve mismatched sample rates; + please preprocess your data before attempting `import_edf` so that channels + of the same signal share a common sample rate. + """) end # I wasn't super confident that the `sample_offset_in_unit` calculation I derived @@ -231,7 +232,7 @@ function promote_encodings(encodings; pick_offset=(_ -> 0.0), pick_resolution=mi sample_resolution_in_unit=missing, sample_rate=missing) end - + sample_type = mapreduce(Onda.sample_type, promote_type, encodings) sample_rates = [e.sample_rate for e in encodings] @@ -281,7 +282,7 @@ end function Base.showerror(io::IO, e::SamplesInfoError) print(io, "SamplesInfoError: ", e.msg, " caused by: ") - Base.showerror(io, e.cause) + return Base.showerror(io, e.cause) end function groupby(f, list) @@ -298,8 +299,12 @@ canonical_channel_name(channel_name) = channel_name # "channel" => ["alt1", "alt2", ...] canonical_channel_name(channel_alternates::Pair) = first(channel_alternates) -plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) = plan_edf_to_onda_samples(signal.header, s; kwargs...) -plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) = plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) +function plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) + return plan_edf_to_onda_samples(signal.header, s; kwargs...) +end +function plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) + return plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) +end """ plan_edf_to_onda_samples(header, seconds_per_record; labels=STANDARD_LABELS, @@ -355,7 +360,8 @@ function plan_edf_to_onda_samples(header, preprocess_labels=nothing) # we don't check this inside the try/catch because it's a user/method error # rather than a data/ingest error - ismissing(seconds_per_record) && throw(ArgumentError(":seconds_per_record not found in header, or missing")) + ismissing(seconds_per_record) && + throw(ArgumentError(":seconds_per_record not found in header, or missing")) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing @@ -363,7 +369,7 @@ function plan_edf_to_onda_samples(header, "Instead, preprocess signal header rows to before calling " * "`plan_edf_to_onda_samples`")) end - + row = (; header..., seconds_per_record, error=nothing) try @@ -383,11 +389,12 @@ function plan_edf_to_onda_samples(header, for canonical in channel_names channel_name = canonical_channel_name(canonical) - matched = match_edf_label(edf_label, signal_names, channel_name, channel_names) - + matched = match_edf_label(edf_label, signal_names, channel_name, + channel_names) + if matched !== nothing # create SamplesInfo and return - row = rowmerge(row; + row = rowmerge(row; channel=matched, sensor_type=first(signal_names), sensor_label=first(signal_names)) @@ -436,7 +443,8 @@ function plan_edf_to_onda_samples(edf::EDF.File; labels=STANDARD_LABELS, units=STANDARD_UNITS, preprocess_labels=nothing, - onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, + :sample_rate)) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing throw(ArgumentError("the `preprocess_labels` argument has been removed. " * @@ -444,7 +452,6 @@ function plan_edf_to_onda_samples(edf::EDF.File; "`plan_edf_to_onda_samples`. See the OndaEDF README.")) end - true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) plan_rows = map(true_signals) do s return plan_edf_to_onda_samples(s.header, edf.header.seconds_per_record; @@ -472,14 +479,15 @@ The updated rows are returned, sorted first by the columns named in `onda_signal_groupby` and second by order of occurrence within the input rows. """ function plan_edf_to_onda_samples_groups(plan_rows; - onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, + :sample_rate)) plan_rows = Tables.rows(plan_rows) # if `edf_signal_index` is not present, create it before we re-order things plan_rows = map(enumerate(plan_rows)) do (i, row) edf_signal_index = coalesce(_get(row, :edf_signal_index), i) return rowmerge(row; edf_signal_index) end - + grouped_rows = groupby(grouper(onda_signal_groupby), plan_rows) sorted_keys = sort!(collect(keys(grouped_rows))) plan_rows = mapreduce(vcat, enumerate(sorted_keys)) do (onda_signal_index, key) @@ -494,8 +502,8 @@ _get(x, property) = hasproperty(x, property) ? getproperty(x, property) : missin function grouper(vars=(:sensor_type, :sample_unit, :sample_rate)) return x -> NamedTuple{vars}(_get.(Ref(x), vars)) end -grouper(vars::AbstractVector{Symbol}) = grouper((vars..., )) -grouper(var::Symbol) = grouper((var, )) +grouper(vars::AbstractVector{Symbol}) = grouper((vars...,)) +grouper(var::Symbol) = grouper((var,)) # return Samples for each :onda_signal_index """ @@ -524,10 +532,10 @@ as specified in the docstring for `Onda.encode`. `dither_storage=nothing` disabl $SAMPLES_ENCODED_WARNING """ -function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_storage=missing) - +function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, + dither_storage=missing) true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) - + if validate Legolas.validate(Tables.schema(Tables.columns(plan_table)), Legolas.SchemaVersion("ondaedf.file-plan", 2)) @@ -540,7 +548,7 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_st EDF.read!(edf) plan_rows = Tables.rows(plan_table) - grouped_plan_rows = groupby(grouper((:onda_signal_index, )), plan_rows) + grouped_plan_rows = groupby(grouper((:onda_signal_index,)), plan_rows) exec_rows = map(collect(grouped_plan_rows)) do (idx, rows) try info = merge_samples_info(rows) @@ -555,7 +563,8 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_st else signals = [true_signals[row.edf_signal_index] for row in rows] samples = onda_samples_from_edf_signals(SamplesInfoV2(info), signals, - edf.header.seconds_per_record; dither_storage) + edf.header.seconds_per_record; + dither_storage) end return (; idx, samples, plan_rows=rows) catch e @@ -644,7 +653,8 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, edf_seconds_per_record; dither_storage=missing) sample_count = length(first(edf_signals).samples) if !all(length(s.samples) == sample_count for s in edf_signals) - error("mismatched sample counts between `EDF.Signal`s: ", [length(s.samples) for s in edf_signals]) + error("mismatched sample counts between `EDF.Signal`s: ", + [length(s.samples) for s in edf_signals]) end sample_data = Matrix{sample_type(target)}(undef, length(target.channels), sample_count) for (i, edf_signal) in enumerate(edf_signals) @@ -662,12 +672,12 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, Onda.encode(sample_type(target), target.sample_resolution_in_unit, target.sample_offset_in_unit, decoded_samples, dither_storage) - catch e - if e isa DomainError - @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." - end - rethrow() - end + catch e + if e isa DomainError + @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." + end + rethrow() + end else encoded_samples = edf_signal.samples end @@ -726,8 +736,10 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() kwargs...) # Validate input argument early on - signals_path = joinpath(onda_dir, "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") - annotations_path = joinpath(onda_dir, "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") + signals_path = joinpath(onda_dir, + "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") + annotations_path = joinpath(onda_dir, + "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") EDF.read!(edf) file_format = "lpcm.zst" @@ -737,7 +749,7 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() signals = Onda.SignalV2[] edf_samples, plan = edf_to_onda_samples(edf; kwargs...) - + errors = _get(Tables.columns(plan), :error) if !ismissing(errors) # why unique? because errors that occur during execution get inserted @@ -749,10 +761,11 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end end end - + edf_samples = postprocess_samples(edf_samples) for samples in edf_samples - sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", file_format) + sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", + file_format) file_path = joinpath(onda_dir, "samples", sample_filename) signal = store(file_path, file_format, samples, recording_uuid, Second(0)) push!(signals, signal) @@ -776,7 +789,8 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end function validate_arrow_prefix(prefix) - prefix == basename(prefix) || throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) + prefix == basename(prefix) || + throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) pm = match(r"(.*)\.onda\.(signals|annotations)\.arrow", prefix) if pm !== nothing @warn "Extracting prefix \"$(pm.captures[1])\" from provided prefix \"$prefix\"" @@ -843,12 +857,14 @@ function edf_to_onda_annotations(edf::EDF.File, uuid::UUID) if tal.duration_in_seconds === nothing stop_nanosecond = start_nanosecond else - stop_nanosecond = start_nanosecond + Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) + stop_nanosecond = start_nanosecond + + Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) end for annotation_string in tal.annotations isempty(annotation_string) && continue annotation = EDFAnnotationV1(; recording=uuid, id=uuid4(), - span=TimeSpan(start_nanosecond, stop_nanosecond), + span=TimeSpan(start_nanosecond, + stop_nanosecond), value=annotation_string) push!(annotations, annotation) end diff --git a/src/standards.jl b/src/standards.jl index de1ed562..e71fe4cc 100644 --- a/src/standards.jl +++ b/src/standards.jl @@ -17,7 +17,8 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], "degrees_fahrenheit" => ["degF", "degf"], "kelvin" => ["K"], "percent" => ["%"], - "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", "L/min", "l/min"], + "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", + "L/min", "l/min"], "millimeter_of_mercury" => ["mmHg", "mmhg", "MMHG"], "beat_per_minute" => ["B/m", "b/m", "bpm", "BPM", "BpM", "Bpm"], "centimeter_of_water" => ["cmH2O", "cmh2o", "cmH20"], @@ -29,8 +30,9 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], # The case-sensitivity of EDF physical dimension names means you can't/shouldn't # naively convert/lowercase them to compliant Onda unit names, so we have to be # very conservative here and error if we don't recognize the input. -function edf_to_onda_unit(edf_physical_dimension::AbstractString, unit_alternatives=STANDARD_UNITS) - edf_physical_dimension = replace(edf_physical_dimension, r"\s"=>"") +function edf_to_onda_unit(edf_physical_dimension::AbstractString, + unit_alternatives=STANDARD_UNITS) + edf_physical_dimension = replace(edf_physical_dimension, r"\s" => "") for (onda_unit, potential_edf_matches) in unit_alternatives any(==(edf_physical_dimension), potential_edf_matches) && return onda_unit end @@ -56,11 +58,17 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an ["eeg"] => ["pg1", "nz", "pg2", "fp1", "fpz", "fp2", "af7", "af3", "afz", "af4", "af8", - "f9", "f7", "f5", "f3", "f1", "fz", "f2", "f4", "f6", "f8", "f10", - "ft9", "ft7", "fc5", "fc3", "fc1", "fcz", "fc2", "fc4", "fc6", "ft8", "ft10", - "a1", "m1", "t9", "t7", "t3", "c5", "c3", "c1", "cz", "c2", "c4", "c6", "t4", "t8", "t10", "a2", "m2", - "tp9", "tp7", "cp5", "cp3", "cp1", "cpz", "cp2", "cp4", "cp6", "tp8", "tp10", - "t5", "p9", "p7", "p5", "p3", "p1", "pz", "p2", "p4", "p6", "p8", "p10", "t6", + "f9", "f7", "f5", "f3", "f1", "fz", "f2", "f4", + "f6", "f8", "f10", + "ft9", "ft7", "fc5", "fc3", "fc1", "fcz", "fc2", + "fc4", "fc6", "ft8", "ft10", + "a1", "m1", "t9", "t7", "t3", "c5", "c3", "c1", + "cz", "c2", "c4", "c6", "t4", "t8", "t10", "a2", + "m2", + "tp9", "tp7", "cp5", "cp3", "cp1", "cpz", "cp2", + "cp4", "cp6", "tp8", "tp10", + "t5", "p9", "p7", "p5", "p3", "p1", "pz", "p2", + "p4", "p6", "p8", "p10", "t6", "po7", "po3", "poz", "po4", "po8", "o1" => ["01"], "oz", "o2" => ["02"], "iz"], @@ -68,19 +76,47 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an # by label alone to tell whether such channels refer to I, II, etc. or aVL, aVR, # etc., so there's a burden on users to preprocess their EKG labels ["ecg", "ekg"] => ["i" => ["1"], "ii" => ["2"], "iii" => ["3"], - "avl"=> ["ecgl", "ekgl", "ecg", "ekg", "l"], "avr"=> ["ekgr", "ecgr", "r"], "avf", - "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v1r", "v2r", "v3r", "v4r", "v5r", "v6r", "v7r", "v8r", "v9r", + "avl" => ["ecgl", "ekgl", "ecg", "ekg", "l"], + "avr" => ["ekgr", "ecgr", "r"], "avf", + "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", + "v1r", "v2r", "v3r", "v4r", "v5r", "v6r", + "v7r", "v8r", "v9r", "x", "y", "z"], # EOG should not have any channel names overlapping with EEG channel names - ["eog", "eeg"] => ["left"=> ["eogl", "loc", "lefteye", "leye", "e1", "eog1", "l", "left eye", "leog", "log", "li", "lue"], - "right"=> ["eogr", "roc", "righteye", "reye", "e2", "eog2", "r", "right eye", "reog", "rog", "re", "rae"]], - ["emg"] => ["chin1" => ["chn", "chin_1", "chn1", "kinn", "menton", "submental", "submentalis", "submental1", "subm1", "chin", "mentalis", "chinl", "chinli", "chinleft", "subm_1", "subment"], - "chin2" => ["chn2", "chin_2", "submental2", "subm2", "chinr", "chinre", "chinright", "subm_2"], - "chin3" => ["chn3", "submental3", "subm3", "chincenter"], - "intercostal"=> ["ic"], - "left_anterior_tibialis"=> ["lat", "lat1", "l", "left", "leftlimb", "tibl", "tibli", "plml", "leg1", "lleg", "lleg1", "legl", "jambe_l", "leftleg"], - "right_anterior_tibialis"=> ["rat", "rat1", "r", "right", "rightlimb", "tibr", "tibre", "plmr", "leg2", "leg3", "rleg", "rleg1", "legr", "jambe_r", "rightleg"]], + ["eog", "eeg"] => ["left" => ["eogl", "loc", "lefteye", "leye", + "e1", "eog1", "l", "left eye", + "leog", "log", "li", "lue"], + "right" => ["eogr", "roc", "righteye", + "reye", "e2", "eog2", "r", + "right eye", "reog", "rog", + "re", "rae"]], + ["emg"] => ["chin1" => ["chn", "chin_1", "chn1", "kinn", + "menton", "submental", "submentalis", + "submental1", "subm1", "chin", + "mentalis", "chinl", "chinli", + "chinleft", "subm_1", "subment"], + "chin2" => ["chn2", "chin_2", "submental2", + "subm2", "chinr", "chinre", + "chinright", "subm_2"], + "chin3" => ["chn3", "submental3", "subm3", + "chincenter"], + "intercostal" => ["ic"], + "left_anterior_tibialis" => ["lat", "lat1", "l", + "left", "leftlimb", + "tibl", "tibli", + "plml", "leg1", + "lleg", "lleg1", + "legl", "jambe_l", + "leftleg"], + "right_anterior_tibialis" => ["rat", "rat1", "r", + "right", "rightlimb", + "tibr", "tibre", + "plmr", "leg2", + "leg3", "rleg", + "rleg1", "legr", + "jambe_r", + "rightleg"]], # it is common to see ambiguous channels, which could be leg or face EMG # if leg EMG is present in separate channels, # post-processing might map "emg_ambiguous" @@ -88,15 +124,24 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an ["emg_ambiguous", "emg"] => ["1" => ["aux1", "l"], "2" => ["aux2", "r"], "3" => ["aux3"]], - ["heart_rate"] => ["heart_rate"=> ["hr", "pulse", "pulso", "pr", "pulserate"]], - ["snore"] => ["snore" => ["ronquido", "ronquido derivad", "schnarchen", "ronfl", "schnarchmikro"]], + ["heart_rate"] => ["heart_rate" => ["hr", "pulse", "pulso", + "pr", "pulserate"]], + ["snore"] => ["snore" => ["ronquido", "ronquido derivad", + "schnarchen", "ronfl", + "schnarchmikro"]], ["positive_airway_pressure", "pap"] => ["ipap", "epap", "cpap"], - ["pap_device_cflow"] => ["pap_device_cflow"=> ["cflow", "airflow", "flow"]], - ["pap_device_cpres"] => ["pap_device_cpres"=> ["cpres"]], - ["pap_device_leak"] => ["pap_device_leak"=> ["leak", "airleak"]], + ["pap_device_cflow"] => ["pap_device_cflow" => ["cflow", + "airflow", + "flow"]], + ["pap_device_cpres"] => ["pap_device_cpres" => ["cpres"]], + ["pap_device_leak"] => ["pap_device_leak" => ["leak", + "airleak"]], ["ptaf"] => ["ptaf"], - ["respiratory_effort"] => ["chest" => ["thorax", "torax", "brust", "thor"], "abdomen"=> ["abd", "abdo", "bauch"]], - ["tidal_volume"] => ["tidal_volume"=> ["tvol", "tidal"]], + ["respiratory_effort"] => ["chest" => ["thorax", "torax", + "brust", "thor"], + "abdomen" => ["abd", "abdo", + "bauch"]], + ["tidal_volume"] => ["tidal_volume" => ["tvol", "tidal"]], ["spo2"] => ["spo2"], ["sao2"] => ["sao2", "osat"], ["etco2"] => ["etco2" => ["capno"]]) diff --git a/test/export.jl b/test/export.jl index 840a60ce..97827afc 100644 --- a/test/export.jl +++ b/test/export.jl @@ -1,5 +1,4 @@ @testset "EDF Export" begin - n_records = 100 edf, edf_channel_indices = make_test_data(MersenneTwister(42), 256, 512, n_records) uuid = uuid4() @@ -10,7 +9,9 @@ signal_names = ["eeg", "eog", "ecg", "emg", "heart_rate", "tidal_volume", "respiratory_effort", "snore", "positive_airway_pressure", "pap_device_leak", "pap_device_cflow", "sao2", "ptaf"] - samples_to_export = onda_samples[indexin(signal_names, getproperty.(getproperty.(onda_samples, :info), :sensor_type))] + samples_to_export = onda_samples[indexin(signal_names, + getproperty.(getproperty.(onda_samples, :info), + :sensor_type))] exported_edf = onda_to_edf(samples_to_export, annotations) @test exported_edf.header.record_count == 200 offset = 0 @@ -20,30 +21,35 @@ edf_indices = (1:length(channel_names)) .+ offset offset += length(channel_names) samples_data = Onda.decode(samples).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, exported_edf.signals[edf_indices]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, + exported_edf.signals[edf_indices]) @test isapprox(samples_data, edf_samples; rtol=0.02) for (i, channel_name) in zip(edf_indices, channel_names) s = exported_edf.signals[i] @test s.header.label == OndaEDF.export_edf_label(signal_name, channel_name) - @test s.header.physical_dimension == OndaEDF.onda_to_edf_unit(samples.info.sample_unit) + @test s.header.physical_dimension == + OndaEDF.onda_to_edf_unit(samples.info.sample_unit) end end @testset "Record metadata" begin function change_sample_rate(samples; sample_rate) info = SamplesInfoV2(Tables.rowmerge(samples.info; sample_rate=sample_rate)) - new_data = similar(samples.data, 0, Onda.index_from_time(sample_rate, Onda.duration(samples)) - 1) + new_data = similar(samples.data, 0, + Onda.index_from_time(sample_rate, Onda.duration(samples)) - + 1) return Samples(new_data, info, samples.encoded; validate=false) end eeg_samples = only(filter(row -> row.info.sensor_type == "eeg", onda_samples)) ecg_samples = only(filter(row -> row.info.sensor_type == "ecg", onda_samples)) - massive_eeg = change_sample_rate(eeg_samples, sample_rate=5000.0) + massive_eeg = change_sample_rate(eeg_samples; sample_rate=5000.0) @test OndaEDF.edf_record_metadata([massive_eeg]) == (1000000, 1 / 5000) chunky_eeg = change_sample_rate(eeg_samples; sample_rate=9999.0) chunky_ecg = change_sample_rate(ecg_samples; sample_rate=425.0) - @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, chunky_ecg]) + @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, + chunky_ecg]) e_notation_eeg = change_sample_rate(eeg_samples; sample_rate=20_000_000.0) @test OndaEDF.edf_record_metadata([e_notation_eeg]) == (4.0e9, 1 / 20_000_000) @@ -62,7 +68,8 @@ @testset "Exception and Error handling" begin messages = ("RecordSizeException: sample rates [9999.0, 425.0] cannot be resolved to a data record size smaller than 61440 bytes", "EDFPrecisionError: String representation of value 2.0576999e7 is longer than 8 ASCII characters") - exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), OndaEDF.EDFPrecisionError(20576999.0)) + exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), + OndaEDF.EDFPrecisionError(20576999.0)) for (message, exception) in zip(messages, exceptions) buffer = IOBuffer() showerror(buffer, exception) @@ -79,7 +86,8 @@ @test getproperty.(round_tripped, :span) == getproperty.(ann_sorted, :span) @test getproperty.(round_tripped, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(round_tripped, :recording) == getproperty.(ann_sorted, :recording) + @test getproperty.(round_tripped, :recording) == + getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(round_tripped, :id) .!= getproperty.(ann_sorted, :id)) end @@ -93,11 +101,14 @@ @test getproperty.(nt.annotations, :span) == getproperty.(ann_sorted, :span) @test getproperty.(nt.annotations, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(nt.annotations, :recording) == getproperty.(ann_sorted, :recording) + @test getproperty.(nt.annotations, :recording) == + getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(nt.annotations, :id) .!= getproperty.(ann_sorted, :id)) - @testset "$(samples_orig.info.sensor_type)" for (samples_orig, signal_round_tripped) in zip(onda_samples, nt.signals) + @testset "$(samples_orig.info.sensor_type)" for (samples_orig, + signal_round_tripped) in + zip(onda_samples, nt.signals) info_orig = samples_orig.info info_round_tripped = SamplesInfoV2(signal_round_tripped) @@ -118,7 +129,9 @@ # import empty annotations exported_edf2 = onda_to_edf(samples_to_export) - @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, mktempdir(), uuid; import_annotations=true) + @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, + mktempdir(), uuid; + import_annotations=true) end @testset "re-encoding" begin @@ -247,5 +260,4 @@ @test EDF.decode(signal) == vec(decode(samples).data) end end - end diff --git a/test/import.jl b/test/import.jl index 6c61c668..24d127ea 100644 --- a/test/import.jl +++ b/test/import.jl @@ -5,7 +5,6 @@ using Legolas: validate, SchemaVersion, read using StableRNGs @testset "Import EDF" begin - @testset "edf_to_onda_samples" begin n_records = 100 for T in (Int16, EDF.Int24) @@ -25,7 +24,8 @@ using StableRNGs @test_throws(ArgumentError(":seconds_per_record not found in header, or missing"), plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals))) - signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals), + signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), + edf.signals), edf.header.seconds_per_record) @testset "signal-wise plan" begin @@ -34,17 +34,19 @@ using StableRNGs validate_extracted_signals(s.info for s in returned_samples) end - + @testset "custom grouping" begin - signal_plans = [rowmerge(plan; grp=string(plan.sensor_type, plan.sample_unit, plan.sample_rate)) + signal_plans = [rowmerge(plan; + grp=string(plan.sensor_type, plan.sample_unit, + plan.sample_rate)) for plan in signal_plans] - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; onda_signal_groupby=:grp) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) validate_extracted_signals(s.info for s in returned_samples) # one channel per signal, group by label - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; onda_signal_groupby=:label) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) @test all(==(1), channel_count.(returned_samples)) @@ -55,7 +57,8 @@ using StableRNGs # orders before grouping. plans_numbered = [rowmerge(plan; edf_signal_index) for (edf_signal_index, plan) - in enumerate(signal_plans)] + in + enumerate(signal_plans)] plans_rev = reverse!(plans_numbered) @test last(plans_rev).edf_signal_index == 1 @@ -74,10 +77,9 @@ using StableRNGs grouped_plans_rev_bad = plan_edf_to_onda_samples_groups(plans_rev_bad) @test_throws(ArgumentError("Plan's label EcG EKGL does not match EDF label EEG C3-M2!"), edf_to_onda_samples(edf, grouped_plans_rev_bad)) - end end - + @testset "store_edf_as_onda" begin n_records = 100 edf, edf_channel_indices = make_test_data(StableRNG(42), 256, 512, n_records) @@ -105,11 +107,12 @@ using StableRNGs signals = Dict(s.sensor_type => s for s in nt.signals) - @testset "Signal roundtrip" begin + @testset "Signal roundtrip" begin for (signal_name, edf_indices) in edf_channel_indices @testset "$signal_name" begin onda_samples = load(signals[string(signal_name)]).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, edf.signals[sort(edf_indices)]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, + edf.signals[sort(edf_indices)]) @test isapprox(onda_samples, edf_samples; rtol=0.02) end end @@ -122,11 +125,15 @@ using StableRNGs start = Nanosecond(Second(i)) stop = start + Nanosecond(Second(i + 1)) # two annotations with same 1s span and different values: - @test any(a -> a.value == "$i a" && a.span.start == start && a.span.stop == stop, nt.annotations) - @test any(a -> a.value == "$i b" && a.span.start == start && a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i a" && a.span.start == start && + a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i b" && a.span.start == start && + a.span.stop == stop, nt.annotations) # two annotations with instantaneous (1ns) span and different values - @test any(a -> a.value == "$i c" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) - @test any(a -> a.value == "$i d" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i c" && a.span.start == start && + a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i d" && a.span.start == start && + a.span.stop == start + Nanosecond(1), nt.annotations) end end @@ -161,21 +168,26 @@ using StableRNGs end mktempdir() do root - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", annotations_prefix="edff") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", + annotations_prefix="edff") @test nt.signals_path == joinpath(root, "edfff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edff.onda.annotations.arrow") end mktempdir() do root @test_logs (:warn, r"Extracting prefix") begin - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edff.onda.signals.arrow", annotations_prefix="edf") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; + signals_prefix="edff.onda.signals.arrow", + annotations_prefix="edf") end @test nt.signals_path == joinpath(root, "edff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edf.onda.annotations.arrow") end mktempdir() do root - @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="stuff/edf", annotations_prefix="edf") + @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; + signals_prefix="stuff/edf", + annotations_prefix="edf") end end @@ -196,13 +208,13 @@ using StableRNGs @testset "duplicate sensor_type" begin rng = StableRNG(1234) - _signal = function(label, transducer, unit, lo, hi) + _signal = function (label, transducer, unit, lo, hi) return test_edf_signal(rng, label, transducer, unit, lo, hi, Float32(typemin(Int16)), Float32(typemax(Int16)), 128, 10, Int16) end - T = Union{EDF.AnnotationsSignal, EDF.Signal{Int16}} + T = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}} edf_signals = T[_signal("EMG Chin1", "E", "mV", -100, 100), _signal("EMG Chin2", "E", "mV", -120, 90), _signal("EMG LAT", "E", "uV", 0, 1000)] @@ -212,7 +224,8 @@ using StableRNGs edf_header, edf_signals) plan = plan_edf_to_onda_samples(test_edf) - sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, p.onda_signal_index) for p in plan)) + sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, + p.onda_signal_index) for p in plan)) @test length(sensors.sensor_type) == 2 @test all(==("emg"), sensors.sensor_type) # TODO: uniquify this in the grouping... @@ -228,28 +241,35 @@ using StableRNGs one_plan = plan_edf_to_onda_samples(one_signal, edf.header.seconds_per_record) @test one_plan.label == one_signal.header.label - @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; preprocess_labels=identity) + @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; + preprocess_labels=identity) - err_plan = @test_logs (:error, ) plan_edf_to_onda_samples(one_signal, 1.0; units=[1, 2, 3]) + err_plan = @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + units=[1, 2, 3]) @test err_plan.error isa String # malformed units arg: elements should be de-structurable @test contains(err_plan.error, "BoundsError") # malformed labels/units - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; labels=[["signal"] => nothing]) - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + labels=[["signal"] => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + units=["millivolt" => nothing]) # unit not found does not error but does create a missing - unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => ["mV"]]) + unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; + units=["millivolt" => ["mV"]]) @test unitless_plan.error === nothing @test ismissing(unitless_plan.sample_unit) - + # error on execution plans = plan_edf_to_onda_samples(edf) # intentionally combine signals of different sensor_types - different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), plans) + different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), + plans) bad_plans = rowmerge.(plans[[1, different]]; onda_signal_index=1) - bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, bad_plans) + bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, + bad_plans) @test all(row.error isa String for row in bad_plans_exec) @test all(occursin("ArgumentError", row.error) for row in bad_plans_exec) @test isempty(bad_samples) @@ -299,7 +319,7 @@ using StableRNGs @test validate(Tables.schema(plan_exec), SchemaVersion("ondaedf.file-plan", 2)) === nothing - plan_rt = let io=IOBuffer() + plan_rt = let io = IOBuffer() OndaEDF.write_plan(io, plan) seekstart(io) Legolas.read(io; validate=true) @@ -308,8 +328,8 @@ using StableRNGs plan_exec_cols = Tables.columns(plan_exec) plan_rt_cols = Tables.columns(plan_rt) for col in Tables.columnnames(plan_exec_cols) - @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), Tables.getcolumn(plan_exec_cols, col))) + @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), + Tables.getcolumn(plan_exec_cols, col))) end end - end diff --git a/test/runtests.jl b/test/runtests.jl index c17e48eb..edbceba1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,61 +14,126 @@ function test_edf_signal(rng, label, transducer, physical_units, return EDF.Signal(header, samples) end -function make_test_data(rng, sample_rate, samples_per_record, n_records, ::Type{T}=Int16) where {T} +function make_test_data(rng, sample_rate, samples_per_record, n_records, + ::Type{T}=Int16) where {T} imin16, imax16 = Float32(typemin(T)), Float32(typemax(T)) anns_1 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] for i in 1:n_records] + EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] + for i in 1:n_records] anns_2 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] for i in 1:n_records] - _edf_signal = (label, transducer, unit, lo, hi) -> test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, - imax16, samples_per_record, n_records, T) - edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{T}}[ - _edf_signal("EEG F3-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG F4-M1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG C3-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG O1-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("C4-M1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("O2-A1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("E1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("E2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("Fpz", "E", "uV", -32768.0f0, 32767.0f0), - EDF.AnnotationsSignal(samples_per_record, anns_1), - _edf_signal("EMG LAT", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EMG RAT", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("SNORE", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("IPAP", "", "cmH2O", -74.0465f0, 74.19587f0), - _edf_signal("EPAP", "", "cmH2O", -73.5019f0, 74.01962f0), - _edf_signal("CFLOW", "", "LPM", -309.153f0, 308.8513f0), - _edf_signal("PTAF", "", "v", -125.009f0, 125.009f0), - _edf_signal("Leak", "", "LPM", -147.951f0, 148.4674f0), - _edf_signal("CHEST", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("ABD", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("Tidal", "", "mL", -4928.18f0, 4906.871f0), - _edf_signal("SaO2", "", "%", 0.0f0, 100.0f0), - EDF.AnnotationsSignal(samples_per_record, anns_2), - _edf_signal("EKG EKGR- REF", "E", "uV", -9324.0f0, 2034.0f0), - _edf_signal("IC", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("HR", "", "BpM", -32768.0f0, 32768.0f0), - _edf_signal("EcG EKGL", "E", "uV", -10932.0f0, 1123.0f0), - _edf_signal("- REF", "E", "uV", -10932.0f0, 1123.0f0), - _edf_signal("REF1", "E", "uV", -10932.0f0, 1123.0f0), - ] + EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] + for i in 1:n_records] + _edf_signal = (label, transducer, unit, lo, hi) -> test_edf_signal(rng, label, + transducer, unit, lo, + hi, imin16, + imax16, + samples_per_record, + n_records, T) + edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{T}}[_edf_signal("EEG F3-M2", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("EEG F4-M1", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("EEG C3-M2", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("EEG O1-M2", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("C4-M1", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("O2-A1", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("E1", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("E2", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("Fpz", "E", "uV", + -32768.0f0, + 32767.0f0), + EDF.AnnotationsSignal(samples_per_record, + anns_1), + _edf_signal("EMG LAT", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("EMG RAT", "E", + "uV", -32768.0f0, + 32767.0f0), + _edf_signal("SNORE", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("IPAP", "", + "cmH2O", + -74.0465f0, + 74.19587f0), + _edf_signal("EPAP", "", + "cmH2O", + -73.5019f0, + 74.01962f0), + _edf_signal("CFLOW", "", "LPM", + -309.153f0, + 308.8513f0), + _edf_signal("PTAF", "", "v", + -125.009f0, + 125.009f0), + _edf_signal("Leak", "", "LPM", + -147.951f0, + 148.4674f0), + _edf_signal("CHEST", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("ABD", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("Tidal", "", "mL", + -4928.18f0, + 4906.871f0), + _edf_signal("SaO2", "", "%", + 0.0f0, 100.0f0), + EDF.AnnotationsSignal(samples_per_record, + anns_2), + _edf_signal("EKG EKGR- REF", + "E", "uV", + -9324.0f0, + 2034.0f0), + _edf_signal("IC", "E", "uV", + -32768.0f0, + 32767.0f0), + _edf_signal("HR", "", "BpM", + -32768.0f0, + 32768.0f0), + _edf_signal("EcG EKGL", "E", + "uV", -10932.0f0, + 1123.0f0), + _edf_signal("- REF", "E", "uV", + -10932.0f0, + 1123.0f0), + _edf_signal("REF1", "E", "uV", + -10932.0f0, + 1123.0f0)] seconds_per_record = samples_per_record / sample_rate - edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, n_records, seconds_per_record) + edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, + n_records, seconds_per_record) edf = EDF.File((io = IOBuffer(); close(io); io), edf_header, edf_signals) - return edf, Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], - :eog => [7, 8], - :ecg => [27, 24], - :emg => [25, 11, 12], - :heart_rate => [26], - :tidal_volume => [21], - :respiratory_effort => [19, 20], - :snore => [13], - :positive_airway_pressure => [14, 15], - :pap_device_leak => [18], - :pap_device_cflow => [16], - :sao2 => [22], - :ptaf => [17]) + return edf, + Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], + :eog => [7, 8], + :ecg => [27, 24], + :emg => [25, 11, 12], + :heart_rate => [26], + :tidal_volume => [21], + :respiratory_effort => [19, 20], + :snore => [13], + :positive_airway_pressure => [14, 15], + :pap_device_leak => [18], + :pap_device_cflow => [16], + :sao2 => [22], + :ptaf => [17]) end function validate_extracted_signals(signals) @@ -85,7 +150,8 @@ function validate_extracted_signals(signals) @test signals["positive_airway_pressure"].sample_unit == "centimeter_of_water" @test signals["heart_rate"].channels == ["heart_rate"] @test signals["heart_rate"].sample_unit == "beat_per_minute" - @test signals["emg"].channels == ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] + @test signals["emg"].channels == + ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] @test signals["emg"].sample_unit == "microvolt" @test signals["eog"].channels == ["left", "right"] @test signals["eog"].sample_unit == "microvolt" diff --git a/test/signal_labels.jl b/test/signal_labels.jl index 7cbc6334..2d7a8a6c 100644 --- a/test/signal_labels.jl +++ b/test/signal_labels.jl @@ -1,14 +1,22 @@ @testset "EDF.Signal label handling" begin signal_names = ["eeg", "eog", "test"] canonical_names = OndaEDF.STANDARD_LABELS[["eeg"]] - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", canonical_names) == "c3-m1_plus_a2_over_2" - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", canonical_names) == nothing - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", canonical_names) == nothing - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", canonical_names) == nothing - @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label("EOG L", signal_names, "left", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" - @test OndaEDF.match_edf_label("EOG R", signal_names, "right", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", + canonical_names) == "c3-m1_plus_a2_over_2" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", + canonical_names) == nothing + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", + canonical_names) == nothing + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", + canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", + canonical_names) == nothing + @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", + canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label("EOG L", signal_names, "left", + OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" + @test OndaEDF.match_edf_label("EOG R", signal_names, "right", + OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" for (signal_names, channel_names) in OndaEDF.STANDARD_LABELS for channel_name in channel_names name = channel_name isa Pair ? first(channel_name) : channel_name From 5b2505a9ca3abd9b458b7c330074b6e59241727e Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:50:58 +0200 Subject: [PATCH 3/6] Revert "format" This reverts commit 301b1d1073d6162215006b8539cb32af4c9a88b4. --- OndaEDFSchemas.jl/src/OndaEDFSchemas.jl | 27 ++-- OndaEDFSchemas.jl/test/runtests.jl | 19 ++- docs/make.jl | 4 +- src/OndaEDF.jl | 3 +- src/export_edf.jl | 43 ++---- src/import_edf.jl | 118 +++++++---------- src/standards.jl | 97 ++++---------- test/export.jl | 38 ++---- test/import.jl | 84 +++++------- test/runtests.jl | 168 +++++++----------------- test/signal_labels.jl | 24 ++-- 11 files changed, 217 insertions(+), 408 deletions(-) diff --git a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl index d5833f77..c5a6e221 100644 --- a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl +++ b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl @@ -29,14 +29,10 @@ export PlanV1, PlanV2, FilePlanV1, FilePlanV2, EDFAnnotationV1 kind::Union{Missing,AbstractString} = lift(String, kind) channel::Union{Missing,AbstractString} = lift(String, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) - sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, - sample_resolution_in_unit) - sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, - sample_offset_in_unit) - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, - sample_type) - sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, - sample_rate) + sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_resolution_in_unit) + sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_offset_in_unit) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) + sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_rate) # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end @@ -56,21 +52,21 @@ end seconds_per_record::Float64 # Onda.SignalV2 fields (channels -> channel), may be missing recording::Union{UUID,Missing} = lift(UUID, recording) - sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, - sensor_type) + sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, sensor_type) sensor_label::Union{Missing,AbstractString} = lift(_validate_signal_sensor_label, coalesce(sensor_label, sensor_type)) channel::Union{Missing,AbstractString} = lift(_validate_signal_channel, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) sample_resolution_in_unit::Union{Missing,Float64} sample_offset_in_unit::Union{Missing,Float64} - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, - sample_type) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) sample_rate::Union{Missing,Float64} # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end + + const PLAN_DOC_TEMPLATE = """ @version PlanV{{ VERSION }} begin # EDF.SignalHeader fields @@ -163,14 +159,11 @@ end @doc _file_plan_doc(1) FilePlanV1 @doc _file_plan_doc(2) FilePlanV2 -const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion, - FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} +const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion,FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{String}) = AbstractString # we need this because Arrow write can introduce a Missing for the error column # (I think because of how missing/nothing sentinels are handled?) -function Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) - return Union{Nothing,Missing,AbstractString} -end +Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) = Union{Nothing,Missing,AbstractString} @schema "edf.annotation" EDFAnnotation diff --git a/OndaEDFSchemas.jl/test/runtests.jl b/OndaEDFSchemas.jl/test/runtests.jl index 2bf12d62..c9354eaf 100644 --- a/OndaEDFSchemas.jl/test/runtests.jl +++ b/OndaEDFSchemas.jl/test/runtests.jl @@ -32,19 +32,19 @@ function mock_plan(; v, rng=GLOBAL_RNG) physical_dimension="uV", physical_minimum=0.0, physical_maximum=2.0, - digital_minimum=-1.0f4, - digital_maximum=1.0f4, + digital_minimum=-1f4, + digital_maximum=1f4, prefilter="HP 0.1Hz; LP 80Hz; N 60Hz", samples_per_record=128, seconds_per_record=1.0, channel=ingested ? "cz-m1" : missing, sample_unit=ingested ? "microvolt" : missing, - sample_resolution_in_unit=ingested ? 1.0f-4 : missing, + sample_resolution_in_unit=ingested ? 1f-4 : missing, sample_offset_in_unit=ingested ? 1.0 : missing, sample_type=ingested ? "float32" : missing, - sample_rate=ingested ? 1 / 128 : missing, + sample_rate=ingested ? 1/128 : missing, error=errored ? "Error blah blah" : nothing, - recording=(ingested && rand(rng, Bool)) ? uuid4() : missing, + recording= (ingested && rand(rng, Bool)) ? uuid4() : missing, specific_kwargs...) end @@ -63,7 +63,7 @@ end @testset "Schema version $v" for v in (1, 2) SamplesInfo = v == 1 ? Onda.SamplesInfoV1 : SamplesInfoV2 - + @testset "ondaedf.plan@$v" begin rng = StableRNG(10) plans = mock_plan(30; v, rng) @@ -75,22 +75,21 @@ end # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in plans if !ismissing(p.channel)) + for p in plans if !ismissing(p.channel)) end @testset "ondaedf.file-plan@$v" begin rng = StableRNG(11) file_plans = mock_file_plan(50; v, rng) schema = Tables.schema(file_plans) - @test nothing === - Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) + @test nothing === Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) tbl = Arrow.Table(Arrow.tobuffer(file_plans; maxdepth=9)) @test isequal(Tables.columntable(tbl), Tables.columntable(file_plans)) # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in file_plans if !ismissing(p.channel)) + for p in file_plans if !ismissing(p.channel)) end end diff --git a/docs/make.jl b/docs/make.jl index 3bbb4af3..37eefe3f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,12 +1,12 @@ using OndaEDF using Documenter -makedocs(; modules=[OndaEDF, OndaEDF.OndaEDFSchemas], +makedocs(modules=[OndaEDF, OndaEDF.OndaEDFSchemas], sitename="OndaEDF", authors="Beacon Biosignals and other contributors", pages=["OndaEDF" => "index.md", "Converting from EDF" => "convert-to-onda.md", "API Documentation" => "api.md"]) -deploydocs(; repo="github.com/beacon-biosignals/OndaEDF.jl.git", +deploydocs(repo="github.com/beacon-biosignals/OndaEDF.jl.git", push_preview=true) diff --git a/src/OndaEDF.jl b/src/OndaEDF.jl index 85c86c74..2da65f6f 100644 --- a/src/OndaEDF.jl +++ b/src/OndaEDF.jl @@ -16,8 +16,7 @@ using Legolas: lift using Tables: rowmerge export write_plan -export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, - plan_edf_to_onda_samples_groups, store_edf_as_onda +export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, plan_edf_to_onda_samples_groups, store_edf_as_onda export onda_to_edf include("standards.jl") diff --git a/src/export_edf.jl b/src/export_edf.jl index 6bf34d21..0685d204 100644 --- a/src/export_edf.jl +++ b/src/export_edf.jl @@ -12,8 +12,7 @@ end SignalExtrema(samples::Samples) = SignalExtrema(samples.info) function SignalExtrema(info::SamplesInfoV2) digital_extrema = (typemin(sample_type(info)), typemax(sample_type(info))) - physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + - info.sample_offset_in_unit + physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + info.sample_offset_in_unit return SignalExtrema(physical_extrema..., digital_extrema...) end @@ -24,9 +23,7 @@ end const DATA_RECORD_SIZE_LIMIT = 30720 const EDF_BYTE_LIMIT = 8 -function edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) - return Int16(samples.info.sample_rate * seconds_per_record) -end +edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) = Int16(samples.info.sample_rate * seconds_per_record) _rationalize(x) = rationalize(x) _rationalize(x::Int) = x // 1 @@ -43,12 +40,10 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) else scale = gcd(numerator.(sample_rates) .* seconds_per_record) samples_per_record ./= scale - sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && - throw(RecordSizeException(all_samples)) + sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && throw(RecordSizeException(all_samples)) seconds_per_record /= scale end - sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && - throw(EDFPrecisionError(seconds_per_record)) + sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && throw(EDFPrecisionError(seconds_per_record)) end record_duration_in_nanoseconds = Nanosecond(seconds_per_record * 1_000_000_000) signal_duration = maximum(Onda.duration, all_samples) @@ -58,7 +53,7 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) end struct RecordSizeException <: Exception - samples::Any + samples end struct EDFPrecisionError <: Exception @@ -69,13 +64,13 @@ function Base.showerror(io::IO, exception::RecordSizeException) print(io, "RecordSizeException: sample rates ") print(io, [s.info.sample_rate for s in exception.samples]) print(io, " cannot be resolved to a data record size smaller than ") - return print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") + print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") end function Base.showerror(io::IO, exception::EDFPrecisionError) print(io, "EDFPrecisionError: String representation of value ") print(io, exception.value) - return print(io, " is longer than 8 ASCII characters") + print(io, " is longer than 8 ASCII characters") end ##### @@ -93,10 +88,8 @@ end function onda_samples_to_edf_header(samples::AbstractVector{<:Samples}; version::AbstractString="0", - patient_metadata=EDF.PatientID(missing, missing, - missing, missing), - recording_metadata=EDF.RecordingID(missing, missing, - missing, missing), + patient_metadata=EDF.PatientID(missing, missing, missing, missing), + recording_metadata=EDF.RecordingID(missing, missing, missing, missing), is_contiguous::Bool=true, start::DateTime=DateTime(Year(1985))) return EDF.FileHeader(version, patient_metadata, recording_metadata, start, @@ -184,8 +177,7 @@ function reencode_samples(samples::Samples, sample_type::Type{<:Integer}=Int16) return encode(new_samples) end -function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, - seconds_per_record::Float64) +function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, seconds_per_record::Float64) edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}}[] for samples in onda_samples # encode samples, rescaling if necessary @@ -195,8 +187,7 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, for channel_name in samples.info.channels sample_count = edf_sample_count_per_record(samples, seconds_per_record) physical_dimension = onda_to_edf_unit(samples.info.sample_unit) - edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, - channel_name), + edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, channel_name), "", physical_dimension, extrema.physical_min, extrema.physical_max, extrema.digital_min, extrema.digital_max, @@ -204,10 +195,7 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, # manually convert here in case we have input samples whose encoded # values are convertible losslessly to Int16: sample_data = Int16.(vec(samples[channel_name, :].data)) - padding = Iterators.repeated(zero(Int16), - (sample_count - - (length(sample_data) % sample_count)) % - sample_count) + padding = Iterators.repeated(zero(Int16), (sample_count - (length(sample_data) % sample_count)) % sample_count) edf_signal_samples = append!(sample_data, padding) push!(edf_signals, EDF.Signal(edf_signal_header, edf_signal_samples)) end @@ -255,16 +243,13 @@ function onda_to_edf(samples::AbstractVector{<:Samples}, annotations=[]; kwargs. edf_header = onda_samples_to_edf_header(samples; kwargs...) edf_signals = onda_samples_to_edf_signals(samples, edf_header.seconds_per_record) if !isempty(annotations) - records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, - nothing, String[""])] + records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, nothing, String[""])] for i in 0:(edf_header.record_count - 1)] for annotation in sort(Tables.rowtable(annotations); by=row -> start(row.span)) annotation_onset_in_seconds = start(annotation.span).value / 1e9 annotation_duration_in_seconds = duration(annotation.span).value / 1e9 matching_record = records[Int(fld(annotation_onset_in_seconds, edf_header.seconds_per_record)) + 1] - tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, - annotation_duration_in_seconds, - [annotation.value]) + tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, annotation_duration_in_seconds, [annotation.value]) push!(matching_record, tal) end push!(edf_signals, EDF.AnnotationsSignal(records)) diff --git a/src/import_edf.jl b/src/import_edf.jl index c0ccd0a8..2601f971 100644 --- a/src/import_edf.jl +++ b/src/import_edf.jl @@ -40,12 +40,12 @@ end # the initial reference name should match the canonical channel name, # otherwise the channel extraction will be rejected. function _normalize_references(original_label, canonical_names) - label = replace(_safe_lowercase(original_label), r"\s" => "") - label = replace(replace(label, '(' => ""), ')' => "") - label = replace(label, r"\*$" => "") - label = replace(label, '-' => '…') - label = replace(label, '+' => "…+…") - label = replace(label, '/' => "…/…") + label = replace(_safe_lowercase(original_label), r"\s"=>"") + label = replace(replace(label, '('=>""), ')'=>"") + label = replace(label, r"\*$"=>"") + label = replace(label, '-'=>'…') + label = replace(label, '+'=>"…+…") + label = replace(label, '/'=>"…/…") m = match(r"^\[(.*)\]$", label) if m !== nothing label = only(m.captures) @@ -68,8 +68,8 @@ function _normalize_references(original_label, canonical_names) end end recombined = '-'^startswith(original_label, '-') * join(parts, '-') - recombined = replace(recombined, "-+-" => "_plus_") - recombined = replace(recombined, "-/-" => "_over_") + recombined = replace(recombined, "-+-"=>"_plus_") + recombined = replace(recombined, "-/-"=>"_over_") return first(parts), recombined end @@ -162,18 +162,17 @@ end ##### struct MismatchedSampleRateError <: Exception - sample_rates::Any + sample_rates end function Base.showerror(io::IO, err::MismatchedSampleRateError) - return print(io, - """ - found mismatched sample rate between channel encodings: $(err.sample_rates) - - OndaEDF does not currently automatically resolve mismatched sample rates; - please preprocess your data before attempting `import_edf` so that channels - of the same signal share a common sample rate. - """) + print(io, """ + found mismatched sample rate between channel encodings: $(err.sample_rates) + + OndaEDF does not currently automatically resolve mismatched sample rates; + please preprocess your data before attempting `import_edf` so that channels + of the same signal share a common sample rate. + """) end # I wasn't super confident that the `sample_offset_in_unit` calculation I derived @@ -232,7 +231,7 @@ function promote_encodings(encodings; pick_offset=(_ -> 0.0), pick_resolution=mi sample_resolution_in_unit=missing, sample_rate=missing) end - + sample_type = mapreduce(Onda.sample_type, promote_type, encodings) sample_rates = [e.sample_rate for e in encodings] @@ -282,7 +281,7 @@ end function Base.showerror(io::IO, e::SamplesInfoError) print(io, "SamplesInfoError: ", e.msg, " caused by: ") - return Base.showerror(io, e.cause) + Base.showerror(io, e.cause) end function groupby(f, list) @@ -299,12 +298,8 @@ canonical_channel_name(channel_name) = channel_name # "channel" => ["alt1", "alt2", ...] canonical_channel_name(channel_alternates::Pair) = first(channel_alternates) -function plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) - return plan_edf_to_onda_samples(signal.header, s; kwargs...) -end -function plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) - return plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) -end +plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) = plan_edf_to_onda_samples(signal.header, s; kwargs...) +plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) = plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) """ plan_edf_to_onda_samples(header, seconds_per_record; labels=STANDARD_LABELS, @@ -360,8 +355,7 @@ function plan_edf_to_onda_samples(header, preprocess_labels=nothing) # we don't check this inside the try/catch because it's a user/method error # rather than a data/ingest error - ismissing(seconds_per_record) && - throw(ArgumentError(":seconds_per_record not found in header, or missing")) + ismissing(seconds_per_record) && throw(ArgumentError(":seconds_per_record not found in header, or missing")) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing @@ -369,7 +363,7 @@ function plan_edf_to_onda_samples(header, "Instead, preprocess signal header rows to before calling " * "`plan_edf_to_onda_samples`")) end - + row = (; header..., seconds_per_record, error=nothing) try @@ -389,12 +383,11 @@ function plan_edf_to_onda_samples(header, for canonical in channel_names channel_name = canonical_channel_name(canonical) - matched = match_edf_label(edf_label, signal_names, channel_name, - channel_names) - + matched = match_edf_label(edf_label, signal_names, channel_name, channel_names) + if matched !== nothing # create SamplesInfo and return - row = rowmerge(row; + row = rowmerge(row; channel=matched, sensor_type=first(signal_names), sensor_label=first(signal_names)) @@ -443,8 +436,7 @@ function plan_edf_to_onda_samples(edf::EDF.File; labels=STANDARD_LABELS, units=STANDARD_UNITS, preprocess_labels=nothing, - onda_signal_groupby=(:sensor_type, :sample_unit, - :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing throw(ArgumentError("the `preprocess_labels` argument has been removed. " * @@ -452,6 +444,7 @@ function plan_edf_to_onda_samples(edf::EDF.File; "`plan_edf_to_onda_samples`. See the OndaEDF README.")) end + true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) plan_rows = map(true_signals) do s return plan_edf_to_onda_samples(s.header, edf.header.seconds_per_record; @@ -479,15 +472,14 @@ The updated rows are returned, sorted first by the columns named in `onda_signal_groupby` and second by order of occurrence within the input rows. """ function plan_edf_to_onda_samples_groups(plan_rows; - onda_signal_groupby=(:sensor_type, :sample_unit, - :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) plan_rows = Tables.rows(plan_rows) # if `edf_signal_index` is not present, create it before we re-order things plan_rows = map(enumerate(plan_rows)) do (i, row) edf_signal_index = coalesce(_get(row, :edf_signal_index), i) return rowmerge(row; edf_signal_index) end - + grouped_rows = groupby(grouper(onda_signal_groupby), plan_rows) sorted_keys = sort!(collect(keys(grouped_rows))) plan_rows = mapreduce(vcat, enumerate(sorted_keys)) do (onda_signal_index, key) @@ -502,8 +494,8 @@ _get(x, property) = hasproperty(x, property) ? getproperty(x, property) : missin function grouper(vars=(:sensor_type, :sample_unit, :sample_rate)) return x -> NamedTuple{vars}(_get.(Ref(x), vars)) end -grouper(vars::AbstractVector{Symbol}) = grouper((vars...,)) -grouper(var::Symbol) = grouper((var,)) +grouper(vars::AbstractVector{Symbol}) = grouper((vars..., )) +grouper(var::Symbol) = grouper((var, )) # return Samples for each :onda_signal_index """ @@ -532,10 +524,10 @@ as specified in the docstring for `Onda.encode`. `dither_storage=nothing` disabl $SAMPLES_ENCODED_WARNING """ -function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, - dither_storage=missing) +function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_storage=missing) + true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) - + if validate Legolas.validate(Tables.schema(Tables.columns(plan_table)), Legolas.SchemaVersion("ondaedf.file-plan", 2)) @@ -548,7 +540,7 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, EDF.read!(edf) plan_rows = Tables.rows(plan_table) - grouped_plan_rows = groupby(grouper((:onda_signal_index,)), plan_rows) + grouped_plan_rows = groupby(grouper((:onda_signal_index, )), plan_rows) exec_rows = map(collect(grouped_plan_rows)) do (idx, rows) try info = merge_samples_info(rows) @@ -563,8 +555,7 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, else signals = [true_signals[row.edf_signal_index] for row in rows] samples = onda_samples_from_edf_signals(SamplesInfoV2(info), signals, - edf.header.seconds_per_record; - dither_storage) + edf.header.seconds_per_record; dither_storage) end return (; idx, samples, plan_rows=rows) catch e @@ -653,8 +644,7 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, edf_seconds_per_record; dither_storage=missing) sample_count = length(first(edf_signals).samples) if !all(length(s.samples) == sample_count for s in edf_signals) - error("mismatched sample counts between `EDF.Signal`s: ", - [length(s.samples) for s in edf_signals]) + error("mismatched sample counts between `EDF.Signal`s: ", [length(s.samples) for s in edf_signals]) end sample_data = Matrix{sample_type(target)}(undef, length(target.channels), sample_count) for (i, edf_signal) in enumerate(edf_signals) @@ -672,12 +662,12 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, Onda.encode(sample_type(target), target.sample_resolution_in_unit, target.sample_offset_in_unit, decoded_samples, dither_storage) - catch e - if e isa DomainError - @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." - end - rethrow() - end + catch e + if e isa DomainError + @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." + end + rethrow() + end else encoded_samples = edf_signal.samples end @@ -736,10 +726,8 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() kwargs...) # Validate input argument early on - signals_path = joinpath(onda_dir, - "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") - annotations_path = joinpath(onda_dir, - "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") + signals_path = joinpath(onda_dir, "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") + annotations_path = joinpath(onda_dir, "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") EDF.read!(edf) file_format = "lpcm.zst" @@ -749,7 +737,7 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() signals = Onda.SignalV2[] edf_samples, plan = edf_to_onda_samples(edf; kwargs...) - + errors = _get(Tables.columns(plan), :error) if !ismissing(errors) # why unique? because errors that occur during execution get inserted @@ -761,11 +749,10 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end end end - + edf_samples = postprocess_samples(edf_samples) for samples in edf_samples - sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", - file_format) + sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", file_format) file_path = joinpath(onda_dir, "samples", sample_filename) signal = store(file_path, file_format, samples, recording_uuid, Second(0)) push!(signals, signal) @@ -789,8 +776,7 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end function validate_arrow_prefix(prefix) - prefix == basename(prefix) || - throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) + prefix == basename(prefix) || throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) pm = match(r"(.*)\.onda\.(signals|annotations)\.arrow", prefix) if pm !== nothing @warn "Extracting prefix \"$(pm.captures[1])\" from provided prefix \"$prefix\"" @@ -857,14 +843,12 @@ function edf_to_onda_annotations(edf::EDF.File, uuid::UUID) if tal.duration_in_seconds === nothing stop_nanosecond = start_nanosecond else - stop_nanosecond = start_nanosecond + - Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) + stop_nanosecond = start_nanosecond + Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) end for annotation_string in tal.annotations isempty(annotation_string) && continue annotation = EDFAnnotationV1(; recording=uuid, id=uuid4(), - span=TimeSpan(start_nanosecond, - stop_nanosecond), + span=TimeSpan(start_nanosecond, stop_nanosecond), value=annotation_string) push!(annotations, annotation) end diff --git a/src/standards.jl b/src/standards.jl index e71fe4cc..de1ed562 100644 --- a/src/standards.jl +++ b/src/standards.jl @@ -17,8 +17,7 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], "degrees_fahrenheit" => ["degF", "degf"], "kelvin" => ["K"], "percent" => ["%"], - "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", - "L/min", "l/min"], + "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", "L/min", "l/min"], "millimeter_of_mercury" => ["mmHg", "mmhg", "MMHG"], "beat_per_minute" => ["B/m", "b/m", "bpm", "BPM", "BpM", "Bpm"], "centimeter_of_water" => ["cmH2O", "cmh2o", "cmH20"], @@ -30,9 +29,8 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], # The case-sensitivity of EDF physical dimension names means you can't/shouldn't # naively convert/lowercase them to compliant Onda unit names, so we have to be # very conservative here and error if we don't recognize the input. -function edf_to_onda_unit(edf_physical_dimension::AbstractString, - unit_alternatives=STANDARD_UNITS) - edf_physical_dimension = replace(edf_physical_dimension, r"\s" => "") +function edf_to_onda_unit(edf_physical_dimension::AbstractString, unit_alternatives=STANDARD_UNITS) + edf_physical_dimension = replace(edf_physical_dimension, r"\s"=>"") for (onda_unit, potential_edf_matches) in unit_alternatives any(==(edf_physical_dimension), potential_edf_matches) && return onda_unit end @@ -58,17 +56,11 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an ["eeg"] => ["pg1", "nz", "pg2", "fp1", "fpz", "fp2", "af7", "af3", "afz", "af4", "af8", - "f9", "f7", "f5", "f3", "f1", "fz", "f2", "f4", - "f6", "f8", "f10", - "ft9", "ft7", "fc5", "fc3", "fc1", "fcz", "fc2", - "fc4", "fc6", "ft8", "ft10", - "a1", "m1", "t9", "t7", "t3", "c5", "c3", "c1", - "cz", "c2", "c4", "c6", "t4", "t8", "t10", "a2", - "m2", - "tp9", "tp7", "cp5", "cp3", "cp1", "cpz", "cp2", - "cp4", "cp6", "tp8", "tp10", - "t5", "p9", "p7", "p5", "p3", "p1", "pz", "p2", - "p4", "p6", "p8", "p10", "t6", + "f9", "f7", "f5", "f3", "f1", "fz", "f2", "f4", "f6", "f8", "f10", + "ft9", "ft7", "fc5", "fc3", "fc1", "fcz", "fc2", "fc4", "fc6", "ft8", "ft10", + "a1", "m1", "t9", "t7", "t3", "c5", "c3", "c1", "cz", "c2", "c4", "c6", "t4", "t8", "t10", "a2", "m2", + "tp9", "tp7", "cp5", "cp3", "cp1", "cpz", "cp2", "cp4", "cp6", "tp8", "tp10", + "t5", "p9", "p7", "p5", "p3", "p1", "pz", "p2", "p4", "p6", "p8", "p10", "t6", "po7", "po3", "poz", "po4", "po8", "o1" => ["01"], "oz", "o2" => ["02"], "iz"], @@ -76,47 +68,19 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an # by label alone to tell whether such channels refer to I, II, etc. or aVL, aVR, # etc., so there's a burden on users to preprocess their EKG labels ["ecg", "ekg"] => ["i" => ["1"], "ii" => ["2"], "iii" => ["3"], - "avl" => ["ecgl", "ekgl", "ecg", "ekg", "l"], - "avr" => ["ekgr", "ecgr", "r"], "avf", - "v1", "v2", "v3", "v4", "v5", "v6", "v7", - "v8", "v9", - "v1r", "v2r", "v3r", "v4r", "v5r", "v6r", - "v7r", "v8r", "v9r", + "avl"=> ["ecgl", "ekgl", "ecg", "ekg", "l"], "avr"=> ["ekgr", "ecgr", "r"], "avf", + "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", + "v1r", "v2r", "v3r", "v4r", "v5r", "v6r", "v7r", "v8r", "v9r", "x", "y", "z"], # EOG should not have any channel names overlapping with EEG channel names - ["eog", "eeg"] => ["left" => ["eogl", "loc", "lefteye", "leye", - "e1", "eog1", "l", "left eye", - "leog", "log", "li", "lue"], - "right" => ["eogr", "roc", "righteye", - "reye", "e2", "eog2", "r", - "right eye", "reog", "rog", - "re", "rae"]], - ["emg"] => ["chin1" => ["chn", "chin_1", "chn1", "kinn", - "menton", "submental", "submentalis", - "submental1", "subm1", "chin", - "mentalis", "chinl", "chinli", - "chinleft", "subm_1", "subment"], - "chin2" => ["chn2", "chin_2", "submental2", - "subm2", "chinr", "chinre", - "chinright", "subm_2"], - "chin3" => ["chn3", "submental3", "subm3", - "chincenter"], - "intercostal" => ["ic"], - "left_anterior_tibialis" => ["lat", "lat1", "l", - "left", "leftlimb", - "tibl", "tibli", - "plml", "leg1", - "lleg", "lleg1", - "legl", "jambe_l", - "leftleg"], - "right_anterior_tibialis" => ["rat", "rat1", "r", - "right", "rightlimb", - "tibr", "tibre", - "plmr", "leg2", - "leg3", "rleg", - "rleg1", "legr", - "jambe_r", - "rightleg"]], + ["eog", "eeg"] => ["left"=> ["eogl", "loc", "lefteye", "leye", "e1", "eog1", "l", "left eye", "leog", "log", "li", "lue"], + "right"=> ["eogr", "roc", "righteye", "reye", "e2", "eog2", "r", "right eye", "reog", "rog", "re", "rae"]], + ["emg"] => ["chin1" => ["chn", "chin_1", "chn1", "kinn", "menton", "submental", "submentalis", "submental1", "subm1", "chin", "mentalis", "chinl", "chinli", "chinleft", "subm_1", "subment"], + "chin2" => ["chn2", "chin_2", "submental2", "subm2", "chinr", "chinre", "chinright", "subm_2"], + "chin3" => ["chn3", "submental3", "subm3", "chincenter"], + "intercostal"=> ["ic"], + "left_anterior_tibialis"=> ["lat", "lat1", "l", "left", "leftlimb", "tibl", "tibli", "plml", "leg1", "lleg", "lleg1", "legl", "jambe_l", "leftleg"], + "right_anterior_tibialis"=> ["rat", "rat1", "r", "right", "rightlimb", "tibr", "tibre", "plmr", "leg2", "leg3", "rleg", "rleg1", "legr", "jambe_r", "rightleg"]], # it is common to see ambiguous channels, which could be leg or face EMG # if leg EMG is present in separate channels, # post-processing might map "emg_ambiguous" @@ -124,24 +88,15 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an ["emg_ambiguous", "emg"] => ["1" => ["aux1", "l"], "2" => ["aux2", "r"], "3" => ["aux3"]], - ["heart_rate"] => ["heart_rate" => ["hr", "pulse", "pulso", - "pr", "pulserate"]], - ["snore"] => ["snore" => ["ronquido", "ronquido derivad", - "schnarchen", "ronfl", - "schnarchmikro"]], + ["heart_rate"] => ["heart_rate"=> ["hr", "pulse", "pulso", "pr", "pulserate"]], + ["snore"] => ["snore" => ["ronquido", "ronquido derivad", "schnarchen", "ronfl", "schnarchmikro"]], ["positive_airway_pressure", "pap"] => ["ipap", "epap", "cpap"], - ["pap_device_cflow"] => ["pap_device_cflow" => ["cflow", - "airflow", - "flow"]], - ["pap_device_cpres"] => ["pap_device_cpres" => ["cpres"]], - ["pap_device_leak"] => ["pap_device_leak" => ["leak", - "airleak"]], + ["pap_device_cflow"] => ["pap_device_cflow"=> ["cflow", "airflow", "flow"]], + ["pap_device_cpres"] => ["pap_device_cpres"=> ["cpres"]], + ["pap_device_leak"] => ["pap_device_leak"=> ["leak", "airleak"]], ["ptaf"] => ["ptaf"], - ["respiratory_effort"] => ["chest" => ["thorax", "torax", - "brust", "thor"], - "abdomen" => ["abd", "abdo", - "bauch"]], - ["tidal_volume"] => ["tidal_volume" => ["tvol", "tidal"]], + ["respiratory_effort"] => ["chest" => ["thorax", "torax", "brust", "thor"], "abdomen"=> ["abd", "abdo", "bauch"]], + ["tidal_volume"] => ["tidal_volume"=> ["tvol", "tidal"]], ["spo2"] => ["spo2"], ["sao2"] => ["sao2", "osat"], ["etco2"] => ["etco2" => ["capno"]]) diff --git a/test/export.jl b/test/export.jl index 97827afc..840a60ce 100644 --- a/test/export.jl +++ b/test/export.jl @@ -1,4 +1,5 @@ @testset "EDF Export" begin + n_records = 100 edf, edf_channel_indices = make_test_data(MersenneTwister(42), 256, 512, n_records) uuid = uuid4() @@ -9,9 +10,7 @@ signal_names = ["eeg", "eog", "ecg", "emg", "heart_rate", "tidal_volume", "respiratory_effort", "snore", "positive_airway_pressure", "pap_device_leak", "pap_device_cflow", "sao2", "ptaf"] - samples_to_export = onda_samples[indexin(signal_names, - getproperty.(getproperty.(onda_samples, :info), - :sensor_type))] + samples_to_export = onda_samples[indexin(signal_names, getproperty.(getproperty.(onda_samples, :info), :sensor_type))] exported_edf = onda_to_edf(samples_to_export, annotations) @test exported_edf.header.record_count == 200 offset = 0 @@ -21,35 +20,30 @@ edf_indices = (1:length(channel_names)) .+ offset offset += length(channel_names) samples_data = Onda.decode(samples).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, - exported_edf.signals[edf_indices]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, exported_edf.signals[edf_indices]) @test isapprox(samples_data, edf_samples; rtol=0.02) for (i, channel_name) in zip(edf_indices, channel_names) s = exported_edf.signals[i] @test s.header.label == OndaEDF.export_edf_label(signal_name, channel_name) - @test s.header.physical_dimension == - OndaEDF.onda_to_edf_unit(samples.info.sample_unit) + @test s.header.physical_dimension == OndaEDF.onda_to_edf_unit(samples.info.sample_unit) end end @testset "Record metadata" begin function change_sample_rate(samples; sample_rate) info = SamplesInfoV2(Tables.rowmerge(samples.info; sample_rate=sample_rate)) - new_data = similar(samples.data, 0, - Onda.index_from_time(sample_rate, Onda.duration(samples)) - - 1) + new_data = similar(samples.data, 0, Onda.index_from_time(sample_rate, Onda.duration(samples)) - 1) return Samples(new_data, info, samples.encoded; validate=false) end eeg_samples = only(filter(row -> row.info.sensor_type == "eeg", onda_samples)) ecg_samples = only(filter(row -> row.info.sensor_type == "ecg", onda_samples)) - massive_eeg = change_sample_rate(eeg_samples; sample_rate=5000.0) + massive_eeg = change_sample_rate(eeg_samples, sample_rate=5000.0) @test OndaEDF.edf_record_metadata([massive_eeg]) == (1000000, 1 / 5000) chunky_eeg = change_sample_rate(eeg_samples; sample_rate=9999.0) chunky_ecg = change_sample_rate(ecg_samples; sample_rate=425.0) - @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, - chunky_ecg]) + @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, chunky_ecg]) e_notation_eeg = change_sample_rate(eeg_samples; sample_rate=20_000_000.0) @test OndaEDF.edf_record_metadata([e_notation_eeg]) == (4.0e9, 1 / 20_000_000) @@ -68,8 +62,7 @@ @testset "Exception and Error handling" begin messages = ("RecordSizeException: sample rates [9999.0, 425.0] cannot be resolved to a data record size smaller than 61440 bytes", "EDFPrecisionError: String representation of value 2.0576999e7 is longer than 8 ASCII characters") - exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), - OndaEDF.EDFPrecisionError(20576999.0)) + exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), OndaEDF.EDFPrecisionError(20576999.0)) for (message, exception) in zip(messages, exceptions) buffer = IOBuffer() showerror(buffer, exception) @@ -86,8 +79,7 @@ @test getproperty.(round_tripped, :span) == getproperty.(ann_sorted, :span) @test getproperty.(round_tripped, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(round_tripped, :recording) == - getproperty.(ann_sorted, :recording) + @test getproperty.(round_tripped, :recording) == getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(round_tripped, :id) .!= getproperty.(ann_sorted, :id)) end @@ -101,14 +93,11 @@ @test getproperty.(nt.annotations, :span) == getproperty.(ann_sorted, :span) @test getproperty.(nt.annotations, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(nt.annotations, :recording) == - getproperty.(ann_sorted, :recording) + @test getproperty.(nt.annotations, :recording) == getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(nt.annotations, :id) .!= getproperty.(ann_sorted, :id)) - @testset "$(samples_orig.info.sensor_type)" for (samples_orig, - signal_round_tripped) in - zip(onda_samples, nt.signals) + @testset "$(samples_orig.info.sensor_type)" for (samples_orig, signal_round_tripped) in zip(onda_samples, nt.signals) info_orig = samples_orig.info info_round_tripped = SamplesInfoV2(signal_round_tripped) @@ -129,9 +118,7 @@ # import empty annotations exported_edf2 = onda_to_edf(samples_to_export) - @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, - mktempdir(), uuid; - import_annotations=true) + @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, mktempdir(), uuid; import_annotations=true) end @testset "re-encoding" begin @@ -260,4 +247,5 @@ @test EDF.decode(signal) == vec(decode(samples).data) end end + end diff --git a/test/import.jl b/test/import.jl index 24d127ea..6c61c668 100644 --- a/test/import.jl +++ b/test/import.jl @@ -5,6 +5,7 @@ using Legolas: validate, SchemaVersion, read using StableRNGs @testset "Import EDF" begin + @testset "edf_to_onda_samples" begin n_records = 100 for T in (Int16, EDF.Int24) @@ -24,8 +25,7 @@ using StableRNGs @test_throws(ArgumentError(":seconds_per_record not found in header, or missing"), plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals))) - signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), - edf.signals), + signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals), edf.header.seconds_per_record) @testset "signal-wise plan" begin @@ -34,19 +34,17 @@ using StableRNGs validate_extracted_signals(s.info for s in returned_samples) end - + @testset "custom grouping" begin - signal_plans = [rowmerge(plan; - grp=string(plan.sensor_type, plan.sample_unit, - plan.sample_rate)) + signal_plans = [rowmerge(plan; grp=string(plan.sensor_type, plan.sample_unit, plan.sample_rate)) for plan in signal_plans] - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, onda_signal_groupby=:grp) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) validate_extracted_signals(s.info for s in returned_samples) # one channel per signal, group by label - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, onda_signal_groupby=:label) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) @test all(==(1), channel_count.(returned_samples)) @@ -57,8 +55,7 @@ using StableRNGs # orders before grouping. plans_numbered = [rowmerge(plan; edf_signal_index) for (edf_signal_index, plan) - in - enumerate(signal_plans)] + in enumerate(signal_plans)] plans_rev = reverse!(plans_numbered) @test last(plans_rev).edf_signal_index == 1 @@ -77,9 +74,10 @@ using StableRNGs grouped_plans_rev_bad = plan_edf_to_onda_samples_groups(plans_rev_bad) @test_throws(ArgumentError("Plan's label EcG EKGL does not match EDF label EEG C3-M2!"), edf_to_onda_samples(edf, grouped_plans_rev_bad)) + end end - + @testset "store_edf_as_onda" begin n_records = 100 edf, edf_channel_indices = make_test_data(StableRNG(42), 256, 512, n_records) @@ -107,12 +105,11 @@ using StableRNGs signals = Dict(s.sensor_type => s for s in nt.signals) - @testset "Signal roundtrip" begin + @testset "Signal roundtrip" begin for (signal_name, edf_indices) in edf_channel_indices @testset "$signal_name" begin onda_samples = load(signals[string(signal_name)]).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, - edf.signals[sort(edf_indices)]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, edf.signals[sort(edf_indices)]) @test isapprox(onda_samples, edf_samples; rtol=0.02) end end @@ -125,15 +122,11 @@ using StableRNGs start = Nanosecond(Second(i)) stop = start + Nanosecond(Second(i + 1)) # two annotations with same 1s span and different values: - @test any(a -> a.value == "$i a" && a.span.start == start && - a.span.stop == stop, nt.annotations) - @test any(a -> a.value == "$i b" && a.span.start == start && - a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i a" && a.span.start == start && a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i b" && a.span.start == start && a.span.stop == stop, nt.annotations) # two annotations with instantaneous (1ns) span and different values - @test any(a -> a.value == "$i c" && a.span.start == start && - a.span.stop == start + Nanosecond(1), nt.annotations) - @test any(a -> a.value == "$i d" && a.span.start == start && - a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i c" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i d" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) end end @@ -168,26 +161,21 @@ using StableRNGs end mktempdir() do root - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", - annotations_prefix="edff") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", annotations_prefix="edff") @test nt.signals_path == joinpath(root, "edfff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edff.onda.annotations.arrow") end mktempdir() do root @test_logs (:warn, r"Extracting prefix") begin - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; - signals_prefix="edff.onda.signals.arrow", - annotations_prefix="edf") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edff.onda.signals.arrow", annotations_prefix="edf") end @test nt.signals_path == joinpath(root, "edff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edf.onda.annotations.arrow") end mktempdir() do root - @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; - signals_prefix="stuff/edf", - annotations_prefix="edf") + @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="stuff/edf", annotations_prefix="edf") end end @@ -208,13 +196,13 @@ using StableRNGs @testset "duplicate sensor_type" begin rng = StableRNG(1234) - _signal = function (label, transducer, unit, lo, hi) + _signal = function(label, transducer, unit, lo, hi) return test_edf_signal(rng, label, transducer, unit, lo, hi, Float32(typemin(Int16)), Float32(typemax(Int16)), 128, 10, Int16) end - T = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}} + T = Union{EDF.AnnotationsSignal, EDF.Signal{Int16}} edf_signals = T[_signal("EMG Chin1", "E", "mV", -100, 100), _signal("EMG Chin2", "E", "mV", -120, 90), _signal("EMG LAT", "E", "uV", 0, 1000)] @@ -224,8 +212,7 @@ using StableRNGs edf_header, edf_signals) plan = plan_edf_to_onda_samples(test_edf) - sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, - p.onda_signal_index) for p in plan)) + sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, p.onda_signal_index) for p in plan)) @test length(sensors.sensor_type) == 2 @test all(==("emg"), sensors.sensor_type) # TODO: uniquify this in the grouping... @@ -241,35 +228,28 @@ using StableRNGs one_plan = plan_edf_to_onda_samples(one_signal, edf.header.seconds_per_record) @test one_plan.label == one_signal.header.label - @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; - preprocess_labels=identity) + @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; preprocess_labels=identity) - err_plan = @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; - units=[1, 2, 3]) + err_plan = @test_logs (:error, ) plan_edf_to_onda_samples(one_signal, 1.0; units=[1, 2, 3]) @test err_plan.error isa String # malformed units arg: elements should be de-structurable @test contains(err_plan.error, "BoundsError") # malformed labels/units - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; - labels=[["signal"] => nothing]) - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; - units=["millivolt" => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; labels=[["signal"] => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => nothing]) # unit not found does not error but does create a missing - unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; - units=["millivolt" => ["mV"]]) + unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => ["mV"]]) @test unitless_plan.error === nothing @test ismissing(unitless_plan.sample_unit) - + # error on execution plans = plan_edf_to_onda_samples(edf) # intentionally combine signals of different sensor_types - different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), - plans) + different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), plans) bad_plans = rowmerge.(plans[[1, different]]; onda_signal_index=1) - bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, - bad_plans) + bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, bad_plans) @test all(row.error isa String for row in bad_plans_exec) @test all(occursin("ArgumentError", row.error) for row in bad_plans_exec) @test isempty(bad_samples) @@ -319,7 +299,7 @@ using StableRNGs @test validate(Tables.schema(plan_exec), SchemaVersion("ondaedf.file-plan", 2)) === nothing - plan_rt = let io = IOBuffer() + plan_rt = let io=IOBuffer() OndaEDF.write_plan(io, plan) seekstart(io) Legolas.read(io; validate=true) @@ -328,8 +308,8 @@ using StableRNGs plan_exec_cols = Tables.columns(plan_exec) plan_rt_cols = Tables.columns(plan_rt) for col in Tables.columnnames(plan_exec_cols) - @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), - Tables.getcolumn(plan_exec_cols, col))) + @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), Tables.getcolumn(plan_exec_cols, col))) end end + end diff --git a/test/runtests.jl b/test/runtests.jl index edbceba1..c17e48eb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,126 +14,61 @@ function test_edf_signal(rng, label, transducer, physical_units, return EDF.Signal(header, samples) end -function make_test_data(rng, sample_rate, samples_per_record, n_records, - ::Type{T}=Int16) where {T} +function make_test_data(rng, sample_rate, samples_per_record, n_records, ::Type{T}=Int16) where {T} imin16, imax16 = Float32(typemin(T)), Float32(typemax(T)) anns_1 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] - for i in 1:n_records] + EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] for i in 1:n_records] anns_2 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] - for i in 1:n_records] - _edf_signal = (label, transducer, unit, lo, hi) -> test_edf_signal(rng, label, - transducer, unit, lo, - hi, imin16, - imax16, - samples_per_record, - n_records, T) - edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{T}}[_edf_signal("EEG F3-M2", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("EEG F4-M1", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("EEG C3-M2", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("EEG O1-M2", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("C4-M1", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("O2-A1", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("E1", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("E2", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("Fpz", "E", "uV", - -32768.0f0, - 32767.0f0), - EDF.AnnotationsSignal(samples_per_record, - anns_1), - _edf_signal("EMG LAT", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("EMG RAT", "E", - "uV", -32768.0f0, - 32767.0f0), - _edf_signal("SNORE", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("IPAP", "", - "cmH2O", - -74.0465f0, - 74.19587f0), - _edf_signal("EPAP", "", - "cmH2O", - -73.5019f0, - 74.01962f0), - _edf_signal("CFLOW", "", "LPM", - -309.153f0, - 308.8513f0), - _edf_signal("PTAF", "", "v", - -125.009f0, - 125.009f0), - _edf_signal("Leak", "", "LPM", - -147.951f0, - 148.4674f0), - _edf_signal("CHEST", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("ABD", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("Tidal", "", "mL", - -4928.18f0, - 4906.871f0), - _edf_signal("SaO2", "", "%", - 0.0f0, 100.0f0), - EDF.AnnotationsSignal(samples_per_record, - anns_2), - _edf_signal("EKG EKGR- REF", - "E", "uV", - -9324.0f0, - 2034.0f0), - _edf_signal("IC", "E", "uV", - -32768.0f0, - 32767.0f0), - _edf_signal("HR", "", "BpM", - -32768.0f0, - 32768.0f0), - _edf_signal("EcG EKGL", "E", - "uV", -10932.0f0, - 1123.0f0), - _edf_signal("- REF", "E", "uV", - -10932.0f0, - 1123.0f0), - _edf_signal("REF1", "E", "uV", - -10932.0f0, - 1123.0f0)] + EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] for i in 1:n_records] + _edf_signal = (label, transducer, unit, lo, hi) -> test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, + imax16, samples_per_record, n_records, T) + edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{T}}[ + _edf_signal("EEG F3-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG F4-M1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG C3-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG O1-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("C4-M1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("O2-A1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("E1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("E2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("Fpz", "E", "uV", -32768.0f0, 32767.0f0), + EDF.AnnotationsSignal(samples_per_record, anns_1), + _edf_signal("EMG LAT", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EMG RAT", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("SNORE", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("IPAP", "", "cmH2O", -74.0465f0, 74.19587f0), + _edf_signal("EPAP", "", "cmH2O", -73.5019f0, 74.01962f0), + _edf_signal("CFLOW", "", "LPM", -309.153f0, 308.8513f0), + _edf_signal("PTAF", "", "v", -125.009f0, 125.009f0), + _edf_signal("Leak", "", "LPM", -147.951f0, 148.4674f0), + _edf_signal("CHEST", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("ABD", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("Tidal", "", "mL", -4928.18f0, 4906.871f0), + _edf_signal("SaO2", "", "%", 0.0f0, 100.0f0), + EDF.AnnotationsSignal(samples_per_record, anns_2), + _edf_signal("EKG EKGR- REF", "E", "uV", -9324.0f0, 2034.0f0), + _edf_signal("IC", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("HR", "", "BpM", -32768.0f0, 32768.0f0), + _edf_signal("EcG EKGL", "E", "uV", -10932.0f0, 1123.0f0), + _edf_signal("- REF", "E", "uV", -10932.0f0, 1123.0f0), + _edf_signal("REF1", "E", "uV", -10932.0f0, 1123.0f0), + ] seconds_per_record = samples_per_record / sample_rate - edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, - n_records, seconds_per_record) + edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, n_records, seconds_per_record) edf = EDF.File((io = IOBuffer(); close(io); io), edf_header, edf_signals) - return edf, - Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], - :eog => [7, 8], - :ecg => [27, 24], - :emg => [25, 11, 12], - :heart_rate => [26], - :tidal_volume => [21], - :respiratory_effort => [19, 20], - :snore => [13], - :positive_airway_pressure => [14, 15], - :pap_device_leak => [18], - :pap_device_cflow => [16], - :sao2 => [22], - :ptaf => [17]) + return edf, Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], + :eog => [7, 8], + :ecg => [27, 24], + :emg => [25, 11, 12], + :heart_rate => [26], + :tidal_volume => [21], + :respiratory_effort => [19, 20], + :snore => [13], + :positive_airway_pressure => [14, 15], + :pap_device_leak => [18], + :pap_device_cflow => [16], + :sao2 => [22], + :ptaf => [17]) end function validate_extracted_signals(signals) @@ -150,8 +85,7 @@ function validate_extracted_signals(signals) @test signals["positive_airway_pressure"].sample_unit == "centimeter_of_water" @test signals["heart_rate"].channels == ["heart_rate"] @test signals["heart_rate"].sample_unit == "beat_per_minute" - @test signals["emg"].channels == - ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] + @test signals["emg"].channels == ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] @test signals["emg"].sample_unit == "microvolt" @test signals["eog"].channels == ["left", "right"] @test signals["eog"].sample_unit == "microvolt" diff --git a/test/signal_labels.jl b/test/signal_labels.jl index 2d7a8a6c..7cbc6334 100644 --- a/test/signal_labels.jl +++ b/test/signal_labels.jl @@ -1,22 +1,14 @@ @testset "EDF.Signal label handling" begin signal_names = ["eeg", "eog", "test"] canonical_names = OndaEDF.STANDARD_LABELS[["eeg"]] - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", - canonical_names) == "c3-m1_plus_a2_over_2" - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", - canonical_names) == nothing - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", - canonical_names) == nothing - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", - canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", - canonical_names) == nothing - @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", - canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label("EOG L", signal_names, "left", - OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" - @test OndaEDF.match_edf_label("EOG R", signal_names, "right", - OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", canonical_names) == "c3-m1_plus_a2_over_2" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", canonical_names) == nothing + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", canonical_names) == nothing + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", canonical_names) == nothing + @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label("EOG L", signal_names, "left", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" + @test OndaEDF.match_edf_label("EOG R", signal_names, "right", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" for (signal_names, channel_names) in OndaEDF.STANDARD_LABELS for channel_name in channel_names name = channel_name isa Pair ? first(channel_name) : channel_name From 9101d5eaf01f341be31bcf3e3d68f8cbf6cf649d Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:56:26 +0200 Subject: [PATCH 4/6] make formatting less gross --- src/standards.jl | 2 ++ test/export.jl | 3 ++- test/runtests.jl | 10 +++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/standards.jl b/src/standards.jl index de1ed562..f73b9366 100644 --- a/src/standards.jl +++ b/src/standards.jl @@ -42,6 +42,7 @@ function onda_to_edf_unit(onda_sample_unit::String, unit_alternatives=STANDARD_U return lift(first, units) end +#! format: off const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 and 10/10 # physical montage; channels are ordered from left-to-right, # front-to-back w.r.t a top-down, nose-up view of the head. @@ -100,3 +101,4 @@ const STANDARD_LABELS = Dict(# This EEG channel name list is a combined 10/20 an ["spo2"] => ["spo2"], ["sao2"] => ["sao2", "osat"], ["etco2"] => ["etco2" => ["capno"]]) +#! format: on diff --git a/test/export.jl b/test/export.jl index 840a60ce..c816e8fb 100644 --- a/test/export.jl +++ b/test/export.jl @@ -97,7 +97,8 @@ # new UUID for each annotation created during import @test all(getproperty.(nt.annotations, :id) .!= getproperty.(ann_sorted, :id)) - @testset "$(samples_orig.info.sensor_type)" for (samples_orig, signal_round_tripped) in zip(onda_samples, nt.signals) + @testset "$(samples_orig.info.sensor_type)" for + (samples_orig, signal_round_tripped) in zip(onda_samples, nt.signals) info_orig = samples_orig.info info_round_tripped = SamplesInfoV2(signal_round_tripped) diff --git a/test/runtests.jl b/test/runtests.jl index c17e48eb..2e49f4ab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,9 +20,13 @@ function make_test_data(rng, sample_rate, samples_per_record, n_records, ::Type{ EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] for i in 1:n_records] anns_2 = [[EDF.TimestampedAnnotationList(i, nothing, []), EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] for i in 1:n_records] - _edf_signal = (label, transducer, unit, lo, hi) -> test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, - imax16, samples_per_record, n_records, T) - edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{T}}[ + _edf_signal = (label, transducer, unit, lo, hi) -> begin + return test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, + imax16, samples_per_record, n_records, T) + end + # Shorthand for the eltype here + E = Union{EDF.AnnotationsSignal,EDF.Signal{T}} + edf_signals = E[ _edf_signal("EEG F3-M2", "E", "uV", -32768.0f0, 32767.0f0), _edf_signal("EEG F4-M1", "E", "uV", -32768.0f0, 32767.0f0), _edf_signal("EEG C3-M2", "E", "uV", -32768.0f0, 32767.0f0), From a66f1c9e80278bdaddd47364052394dc6e1c8483 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:56:35 +0200 Subject: [PATCH 5/6] format --- OndaEDFSchemas.jl/src/OndaEDFSchemas.jl | 27 ++++-- OndaEDFSchemas.jl/test/runtests.jl | 19 ++-- docs/make.jl | 4 +- src/OndaEDF.jl | 3 +- src/export_edf.jl | 43 ++++++--- src/import_edf.jl | 118 ++++++++++++++---------- src/standards.jl | 8 +- test/export.jl | 39 +++++--- test/import.jl | 84 ++++++++++------- test/runtests.jl | 106 +++++++++++---------- test/signal_labels.jl | 24 +++-- 11 files changed, 280 insertions(+), 195 deletions(-) diff --git a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl index c5a6e221..d5833f77 100644 --- a/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl +++ b/OndaEDFSchemas.jl/src/OndaEDFSchemas.jl @@ -29,10 +29,14 @@ export PlanV1, PlanV2, FilePlanV1, FilePlanV2, EDFAnnotationV1 kind::Union{Missing,AbstractString} = lift(String, kind) channel::Union{Missing,AbstractString} = lift(String, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) - sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_resolution_in_unit) - sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_offset_in_unit) - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) - sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, sample_rate) + sample_resolution_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_resolution_in_unit) + sample_offset_in_unit::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_offset_in_unit) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, + sample_type) + sample_rate::Union{Missing,LPCM_SAMPLE_TYPE_UNION} = lift(convert_number_to_lpcm_sample_type, + sample_rate) # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end @@ -52,21 +56,21 @@ end seconds_per_record::Float64 # Onda.SignalV2 fields (channels -> channel), may be missing recording::Union{UUID,Missing} = lift(UUID, recording) - sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, sensor_type) + sensor_type::Union{Missing,AbstractString} = lift(_validate_signal_sensor_type, + sensor_type) sensor_label::Union{Missing,AbstractString} = lift(_validate_signal_sensor_label, coalesce(sensor_label, sensor_type)) channel::Union{Missing,AbstractString} = lift(_validate_signal_channel, channel) sample_unit::Union{Missing,AbstractString} = lift(String, sample_unit) sample_resolution_in_unit::Union{Missing,Float64} sample_offset_in_unit::Union{Missing,Float64} - sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, sample_type) + sample_type::Union{Missing,AbstractString} = lift(onda_sample_type_from_julia_type, + sample_type) sample_rate::Union{Missing,Float64} # errors, use `nothing` to indicate no error error::Union{Nothing,String} = coalesce(error, nothing) end - - const PLAN_DOC_TEMPLATE = """ @version PlanV{{ VERSION }} begin # EDF.SignalHeader fields @@ -159,11 +163,14 @@ end @doc _file_plan_doc(1) FilePlanV1 @doc _file_plan_doc(2) FilePlanV2 -const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion,FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} +const OndaEDFSchemaVersions = Union{PlanV1SchemaVersion,PlanV2SchemaVersion, + FilePlanV1SchemaVersion,FilePlanV2SchemaVersion} Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{String}) = AbstractString # we need this because Arrow write can introduce a Missing for the error column # (I think because of how missing/nothing sentinels are handled?) -Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) = Union{Nothing,Missing,AbstractString} +function Legolas.accepted_field_type(::OndaEDFSchemaVersions, ::Type{Union{Nothing,String}}) + return Union{Nothing,Missing,AbstractString} +end @schema "edf.annotation" EDFAnnotation diff --git a/OndaEDFSchemas.jl/test/runtests.jl b/OndaEDFSchemas.jl/test/runtests.jl index c9354eaf..2bf12d62 100644 --- a/OndaEDFSchemas.jl/test/runtests.jl +++ b/OndaEDFSchemas.jl/test/runtests.jl @@ -32,19 +32,19 @@ function mock_plan(; v, rng=GLOBAL_RNG) physical_dimension="uV", physical_minimum=0.0, physical_maximum=2.0, - digital_minimum=-1f4, - digital_maximum=1f4, + digital_minimum=-1.0f4, + digital_maximum=1.0f4, prefilter="HP 0.1Hz; LP 80Hz; N 60Hz", samples_per_record=128, seconds_per_record=1.0, channel=ingested ? "cz-m1" : missing, sample_unit=ingested ? "microvolt" : missing, - sample_resolution_in_unit=ingested ? 1f-4 : missing, + sample_resolution_in_unit=ingested ? 1.0f-4 : missing, sample_offset_in_unit=ingested ? 1.0 : missing, sample_type=ingested ? "float32" : missing, - sample_rate=ingested ? 1/128 : missing, + sample_rate=ingested ? 1 / 128 : missing, error=errored ? "Error blah blah" : nothing, - recording= (ingested && rand(rng, Bool)) ? uuid4() : missing, + recording=(ingested && rand(rng, Bool)) ? uuid4() : missing, specific_kwargs...) end @@ -63,7 +63,7 @@ end @testset "Schema version $v" for v in (1, 2) SamplesInfo = v == 1 ? Onda.SamplesInfoV1 : SamplesInfoV2 - + @testset "ondaedf.plan@$v" begin rng = StableRNG(10) plans = mock_plan(30; v, rng) @@ -75,21 +75,22 @@ end # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in plans if !ismissing(p.channel)) + for p in plans if !ismissing(p.channel)) end @testset "ondaedf.file-plan@$v" begin rng = StableRNG(11) file_plans = mock_file_plan(50; v, rng) schema = Tables.schema(file_plans) - @test nothing === Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) + @test nothing === + Legolas.validate(schema, Legolas.SchemaVersion("ondaedf.file-plan", v)) tbl = Arrow.Table(Arrow.tobuffer(file_plans; maxdepth=9)) @test isequal(Tables.columntable(tbl), Tables.columntable(file_plans)) # conversion to samples info with channel -> channels @test all(x -> isa(x, SamplesInfo), SamplesInfo(Tables.rowmerge(p; channels=[p.channel])) - for p in file_plans if !ismissing(p.channel)) + for p in file_plans if !ismissing(p.channel)) end end diff --git a/docs/make.jl b/docs/make.jl index 37eefe3f..3bbb4af3 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,12 +1,12 @@ using OndaEDF using Documenter -makedocs(modules=[OndaEDF, OndaEDF.OndaEDFSchemas], +makedocs(; modules=[OndaEDF, OndaEDF.OndaEDFSchemas], sitename="OndaEDF", authors="Beacon Biosignals and other contributors", pages=["OndaEDF" => "index.md", "Converting from EDF" => "convert-to-onda.md", "API Documentation" => "api.md"]) -deploydocs(repo="github.com/beacon-biosignals/OndaEDF.jl.git", +deploydocs(; repo="github.com/beacon-biosignals/OndaEDF.jl.git", push_preview=true) diff --git a/src/OndaEDF.jl b/src/OndaEDF.jl index 2da65f6f..85c86c74 100644 --- a/src/OndaEDF.jl +++ b/src/OndaEDF.jl @@ -16,7 +16,8 @@ using Legolas: lift using Tables: rowmerge export write_plan -export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, plan_edf_to_onda_samples_groups, store_edf_as_onda +export edf_to_onda_samples, edf_to_onda_annotations, plan_edf_to_onda_samples, + plan_edf_to_onda_samples_groups, store_edf_as_onda export onda_to_edf include("standards.jl") diff --git a/src/export_edf.jl b/src/export_edf.jl index 0685d204..6bf34d21 100644 --- a/src/export_edf.jl +++ b/src/export_edf.jl @@ -12,7 +12,8 @@ end SignalExtrema(samples::Samples) = SignalExtrema(samples.info) function SignalExtrema(info::SamplesInfoV2) digital_extrema = (typemin(sample_type(info)), typemax(sample_type(info))) - physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + info.sample_offset_in_unit + physical_extrema = @. (info.sample_resolution_in_unit * digital_extrema) + + info.sample_offset_in_unit return SignalExtrema(physical_extrema..., digital_extrema...) end @@ -23,7 +24,9 @@ end const DATA_RECORD_SIZE_LIMIT = 30720 const EDF_BYTE_LIMIT = 8 -edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) = Int16(samples.info.sample_rate * seconds_per_record) +function edf_sample_count_per_record(samples::Samples, seconds_per_record::Float64) + return Int16(samples.info.sample_rate * seconds_per_record) +end _rationalize(x) = rationalize(x) _rationalize(x::Int) = x // 1 @@ -40,10 +43,12 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) else scale = gcd(numerator.(sample_rates) .* seconds_per_record) samples_per_record ./= scale - sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && throw(RecordSizeException(all_samples)) + sum(samples_per_record) > DATA_RECORD_SIZE_LIMIT && + throw(RecordSizeException(all_samples)) seconds_per_record /= scale end - sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && throw(EDFPrecisionError(seconds_per_record)) + sizeof(string(seconds_per_record)) > EDF_BYTE_LIMIT && + throw(EDFPrecisionError(seconds_per_record)) end record_duration_in_nanoseconds = Nanosecond(seconds_per_record * 1_000_000_000) signal_duration = maximum(Onda.duration, all_samples) @@ -53,7 +58,7 @@ function edf_record_metadata(all_samples::AbstractVector{<:Onda.Samples}) end struct RecordSizeException <: Exception - samples + samples::Any end struct EDFPrecisionError <: Exception @@ -64,13 +69,13 @@ function Base.showerror(io::IO, exception::RecordSizeException) print(io, "RecordSizeException: sample rates ") print(io, [s.info.sample_rate for s in exception.samples]) print(io, " cannot be resolved to a data record size smaller than ") - print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") + return print(io, DATA_RECORD_SIZE_LIMIT * 2, " bytes") end function Base.showerror(io::IO, exception::EDFPrecisionError) print(io, "EDFPrecisionError: String representation of value ") print(io, exception.value) - print(io, " is longer than 8 ASCII characters") + return print(io, " is longer than 8 ASCII characters") end ##### @@ -88,8 +93,10 @@ end function onda_samples_to_edf_header(samples::AbstractVector{<:Samples}; version::AbstractString="0", - patient_metadata=EDF.PatientID(missing, missing, missing, missing), - recording_metadata=EDF.RecordingID(missing, missing, missing, missing), + patient_metadata=EDF.PatientID(missing, missing, + missing, missing), + recording_metadata=EDF.RecordingID(missing, missing, + missing, missing), is_contiguous::Bool=true, start::DateTime=DateTime(Year(1985))) return EDF.FileHeader(version, patient_metadata, recording_metadata, start, @@ -177,7 +184,8 @@ function reencode_samples(samples::Samples, sample_type::Type{<:Integer}=Int16) return encode(new_samples) end -function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, seconds_per_record::Float64) +function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, + seconds_per_record::Float64) edf_signals = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}}[] for samples in onda_samples # encode samples, rescaling if necessary @@ -187,7 +195,8 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, se for channel_name in samples.info.channels sample_count = edf_sample_count_per_record(samples, seconds_per_record) physical_dimension = onda_to_edf_unit(samples.info.sample_unit) - edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, channel_name), + edf_signal_header = EDF.SignalHeader(export_edf_label(signal_name, + channel_name), "", physical_dimension, extrema.physical_min, extrema.physical_max, extrema.digital_min, extrema.digital_max, @@ -195,7 +204,10 @@ function onda_samples_to_edf_signals(onda_samples::AbstractVector{<:Samples}, se # manually convert here in case we have input samples whose encoded # values are convertible losslessly to Int16: sample_data = Int16.(vec(samples[channel_name, :].data)) - padding = Iterators.repeated(zero(Int16), (sample_count - (length(sample_data) % sample_count)) % sample_count) + padding = Iterators.repeated(zero(Int16), + (sample_count - + (length(sample_data) % sample_count)) % + sample_count) edf_signal_samples = append!(sample_data, padding) push!(edf_signals, EDF.Signal(edf_signal_header, edf_signal_samples)) end @@ -243,13 +255,16 @@ function onda_to_edf(samples::AbstractVector{<:Samples}, annotations=[]; kwargs. edf_header = onda_samples_to_edf_header(samples; kwargs...) edf_signals = onda_samples_to_edf_signals(samples, edf_header.seconds_per_record) if !isempty(annotations) - records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, nothing, String[""])] + records = [[EDF.TimestampedAnnotationList(edf_header.seconds_per_record * i, + nothing, String[""])] for i in 0:(edf_header.record_count - 1)] for annotation in sort(Tables.rowtable(annotations); by=row -> start(row.span)) annotation_onset_in_seconds = start(annotation.span).value / 1e9 annotation_duration_in_seconds = duration(annotation.span).value / 1e9 matching_record = records[Int(fld(annotation_onset_in_seconds, edf_header.seconds_per_record)) + 1] - tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, annotation_duration_in_seconds, [annotation.value]) + tal = EDF.TimestampedAnnotationList(annotation_onset_in_seconds, + annotation_duration_in_seconds, + [annotation.value]) push!(matching_record, tal) end push!(edf_signals, EDF.AnnotationsSignal(records)) diff --git a/src/import_edf.jl b/src/import_edf.jl index 2601f971..c0ccd0a8 100644 --- a/src/import_edf.jl +++ b/src/import_edf.jl @@ -40,12 +40,12 @@ end # the initial reference name should match the canonical channel name, # otherwise the channel extraction will be rejected. function _normalize_references(original_label, canonical_names) - label = replace(_safe_lowercase(original_label), r"\s"=>"") - label = replace(replace(label, '('=>""), ')'=>"") - label = replace(label, r"\*$"=>"") - label = replace(label, '-'=>'…') - label = replace(label, '+'=>"…+…") - label = replace(label, '/'=>"…/…") + label = replace(_safe_lowercase(original_label), r"\s" => "") + label = replace(replace(label, '(' => ""), ')' => "") + label = replace(label, r"\*$" => "") + label = replace(label, '-' => '…') + label = replace(label, '+' => "…+…") + label = replace(label, '/' => "…/…") m = match(r"^\[(.*)\]$", label) if m !== nothing label = only(m.captures) @@ -68,8 +68,8 @@ function _normalize_references(original_label, canonical_names) end end recombined = '-'^startswith(original_label, '-') * join(parts, '-') - recombined = replace(recombined, "-+-"=>"_plus_") - recombined = replace(recombined, "-/-"=>"_over_") + recombined = replace(recombined, "-+-" => "_plus_") + recombined = replace(recombined, "-/-" => "_over_") return first(parts), recombined end @@ -162,17 +162,18 @@ end ##### struct MismatchedSampleRateError <: Exception - sample_rates + sample_rates::Any end function Base.showerror(io::IO, err::MismatchedSampleRateError) - print(io, """ - found mismatched sample rate between channel encodings: $(err.sample_rates) - - OndaEDF does not currently automatically resolve mismatched sample rates; - please preprocess your data before attempting `import_edf` so that channels - of the same signal share a common sample rate. - """) + return print(io, + """ + found mismatched sample rate between channel encodings: $(err.sample_rates) + + OndaEDF does not currently automatically resolve mismatched sample rates; + please preprocess your data before attempting `import_edf` so that channels + of the same signal share a common sample rate. + """) end # I wasn't super confident that the `sample_offset_in_unit` calculation I derived @@ -231,7 +232,7 @@ function promote_encodings(encodings; pick_offset=(_ -> 0.0), pick_resolution=mi sample_resolution_in_unit=missing, sample_rate=missing) end - + sample_type = mapreduce(Onda.sample_type, promote_type, encodings) sample_rates = [e.sample_rate for e in encodings] @@ -281,7 +282,7 @@ end function Base.showerror(io::IO, e::SamplesInfoError) print(io, "SamplesInfoError: ", e.msg, " caused by: ") - Base.showerror(io, e.cause) + return Base.showerror(io, e.cause) end function groupby(f, list) @@ -298,8 +299,12 @@ canonical_channel_name(channel_name) = channel_name # "channel" => ["alt1", "alt2", ...] canonical_channel_name(channel_alternates::Pair) = first(channel_alternates) -plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) = plan_edf_to_onda_samples(signal.header, s; kwargs...) -plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) = plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) +function plan_edf_to_onda_samples(signal::EDF.Signal, s; kwargs...) + return plan_edf_to_onda_samples(signal.header, s; kwargs...) +end +function plan_edf_to_onda_samples(header::EDF.SignalHeader, s; kwargs...) + return plan_edf_to_onda_samples(_named_tuple(header), s; kwargs...) +end """ plan_edf_to_onda_samples(header, seconds_per_record; labels=STANDARD_LABELS, @@ -355,7 +360,8 @@ function plan_edf_to_onda_samples(header, preprocess_labels=nothing) # we don't check this inside the try/catch because it's a user/method error # rather than a data/ingest error - ismissing(seconds_per_record) && throw(ArgumentError(":seconds_per_record not found in header, or missing")) + ismissing(seconds_per_record) && + throw(ArgumentError(":seconds_per_record not found in header, or missing")) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing @@ -363,7 +369,7 @@ function plan_edf_to_onda_samples(header, "Instead, preprocess signal header rows to before calling " * "`plan_edf_to_onda_samples`")) end - + row = (; header..., seconds_per_record, error=nothing) try @@ -383,11 +389,12 @@ function plan_edf_to_onda_samples(header, for canonical in channel_names channel_name = canonical_channel_name(canonical) - matched = match_edf_label(edf_label, signal_names, channel_name, channel_names) - + matched = match_edf_label(edf_label, signal_names, channel_name, + channel_names) + if matched !== nothing # create SamplesInfo and return - row = rowmerge(row; + row = rowmerge(row; channel=matched, sensor_type=first(signal_names), sensor_label=first(signal_names)) @@ -436,7 +443,8 @@ function plan_edf_to_onda_samples(edf::EDF.File; labels=STANDARD_LABELS, units=STANDARD_UNITS, preprocess_labels=nothing, - onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, + :sample_rate)) # keep the kwarg so we can throw a more informative error if preprocess_labels !== nothing throw(ArgumentError("the `preprocess_labels` argument has been removed. " * @@ -444,7 +452,6 @@ function plan_edf_to_onda_samples(edf::EDF.File; "`plan_edf_to_onda_samples`. See the OndaEDF README.")) end - true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) plan_rows = map(true_signals) do s return plan_edf_to_onda_samples(s.header, edf.header.seconds_per_record; @@ -472,14 +479,15 @@ The updated rows are returned, sorted first by the columns named in `onda_signal_groupby` and second by order of occurrence within the input rows. """ function plan_edf_to_onda_samples_groups(plan_rows; - onda_signal_groupby=(:sensor_type, :sample_unit, :sample_rate)) + onda_signal_groupby=(:sensor_type, :sample_unit, + :sample_rate)) plan_rows = Tables.rows(plan_rows) # if `edf_signal_index` is not present, create it before we re-order things plan_rows = map(enumerate(plan_rows)) do (i, row) edf_signal_index = coalesce(_get(row, :edf_signal_index), i) return rowmerge(row; edf_signal_index) end - + grouped_rows = groupby(grouper(onda_signal_groupby), plan_rows) sorted_keys = sort!(collect(keys(grouped_rows))) plan_rows = mapreduce(vcat, enumerate(sorted_keys)) do (onda_signal_index, key) @@ -494,8 +502,8 @@ _get(x, property) = hasproperty(x, property) ? getproperty(x, property) : missin function grouper(vars=(:sensor_type, :sample_unit, :sample_rate)) return x -> NamedTuple{vars}(_get.(Ref(x), vars)) end -grouper(vars::AbstractVector{Symbol}) = grouper((vars..., )) -grouper(var::Symbol) = grouper((var, )) +grouper(vars::AbstractVector{Symbol}) = grouper((vars...,)) +grouper(var::Symbol) = grouper((var,)) # return Samples for each :onda_signal_index """ @@ -524,10 +532,10 @@ as specified in the docstring for `Onda.encode`. `dither_storage=nothing` disabl $SAMPLES_ENCODED_WARNING """ -function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_storage=missing) - +function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, + dither_storage=missing) true_signals = filter(x -> isa(x, EDF.Signal), edf.signals) - + if validate Legolas.validate(Tables.schema(Tables.columns(plan_table)), Legolas.SchemaVersion("ondaedf.file-plan", 2)) @@ -540,7 +548,7 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_st EDF.read!(edf) plan_rows = Tables.rows(plan_table) - grouped_plan_rows = groupby(grouper((:onda_signal_index, )), plan_rows) + grouped_plan_rows = groupby(grouper((:onda_signal_index,)), plan_rows) exec_rows = map(collect(grouped_plan_rows)) do (idx, rows) try info = merge_samples_info(rows) @@ -555,7 +563,8 @@ function edf_to_onda_samples(edf::EDF.File, plan_table; validate=true, dither_st else signals = [true_signals[row.edf_signal_index] for row in rows] samples = onda_samples_from_edf_signals(SamplesInfoV2(info), signals, - edf.header.seconds_per_record; dither_storage) + edf.header.seconds_per_record; + dither_storage) end return (; idx, samples, plan_rows=rows) catch e @@ -644,7 +653,8 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, edf_seconds_per_record; dither_storage=missing) sample_count = length(first(edf_signals).samples) if !all(length(s.samples) == sample_count for s in edf_signals) - error("mismatched sample counts between `EDF.Signal`s: ", [length(s.samples) for s in edf_signals]) + error("mismatched sample counts between `EDF.Signal`s: ", + [length(s.samples) for s in edf_signals]) end sample_data = Matrix{sample_type(target)}(undef, length(target.channels), sample_count) for (i, edf_signal) in enumerate(edf_signals) @@ -662,12 +672,12 @@ function onda_samples_from_edf_signals(target::SamplesInfoV2, edf_signals, Onda.encode(sample_type(target), target.sample_resolution_in_unit, target.sample_offset_in_unit, decoded_samples, dither_storage) - catch e - if e isa DomainError - @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." - end - rethrow() - end + catch e + if e isa DomainError + @warn "DomainError during `Onda.encode` can be due to a dithering bug; try calling with `dither_storage=nothing` to disable dithering." + end + rethrow() + end else encoded_samples = edf_signal.samples end @@ -726,8 +736,10 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() kwargs...) # Validate input argument early on - signals_path = joinpath(onda_dir, "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") - annotations_path = joinpath(onda_dir, "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") + signals_path = joinpath(onda_dir, + "$(validate_arrow_prefix(signals_prefix)).onda.signals.arrow") + annotations_path = joinpath(onda_dir, + "$(validate_arrow_prefix(annotations_prefix)).onda.annotations.arrow") EDF.read!(edf) file_format = "lpcm.zst" @@ -737,7 +749,7 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() signals = Onda.SignalV2[] edf_samples, plan = edf_to_onda_samples(edf; kwargs...) - + errors = _get(Tables.columns(plan), :error) if !ismissing(errors) # why unique? because errors that occur during execution get inserted @@ -749,10 +761,11 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end end end - + edf_samples = postprocess_samples(edf_samples) for samples in edf_samples - sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", file_format) + sample_filename = string(recording_uuid, "_", samples.info.sensor_type, ".", + file_format) file_path = joinpath(onda_dir, "samples", sample_filename) signal = store(file_path, file_format, samples, recording_uuid, Second(0)) push!(signals, signal) @@ -776,7 +789,8 @@ function store_edf_as_onda(edf::EDF.File, onda_dir, recording_uuid::UUID=uuid4() end function validate_arrow_prefix(prefix) - prefix == basename(prefix) || throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) + prefix == basename(prefix) || + throw(ArgumentError("prefix \"$prefix\" is invalid: cannot contain directory separator")) pm = match(r"(.*)\.onda\.(signals|annotations)\.arrow", prefix) if pm !== nothing @warn "Extracting prefix \"$(pm.captures[1])\" from provided prefix \"$prefix\"" @@ -843,12 +857,14 @@ function edf_to_onda_annotations(edf::EDF.File, uuid::UUID) if tal.duration_in_seconds === nothing stop_nanosecond = start_nanosecond else - stop_nanosecond = start_nanosecond + Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) + stop_nanosecond = start_nanosecond + + Nanosecond(round(Int, 1e9 * tal.duration_in_seconds)) end for annotation_string in tal.annotations isempty(annotation_string) && continue annotation = EDFAnnotationV1(; recording=uuid, id=uuid4(), - span=TimeSpan(start_nanosecond, stop_nanosecond), + span=TimeSpan(start_nanosecond, + stop_nanosecond), value=annotation_string) push!(annotations, annotation) end diff --git a/src/standards.jl b/src/standards.jl index f73b9366..f8cea9d6 100644 --- a/src/standards.jl +++ b/src/standards.jl @@ -17,7 +17,8 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], "degrees_fahrenheit" => ["degF", "degf"], "kelvin" => ["K"], "percent" => ["%"], - "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", "L/min", "l/min"], + "liter_per_minute" => ["L/m", "l/m", "LPM", "Lpm", "lpm", "LpM", + "L/min", "l/min"], "millimeter_of_mercury" => ["mmHg", "mmhg", "MMHG"], "beat_per_minute" => ["B/m", "b/m", "bpm", "BPM", "BpM", "Bpm"], "centimeter_of_water" => ["cmH2O", "cmh2o", "cmH20"], @@ -29,8 +30,9 @@ const STANDARD_UNITS = Dict("nanovolt" => ["nV"], # The case-sensitivity of EDF physical dimension names means you can't/shouldn't # naively convert/lowercase them to compliant Onda unit names, so we have to be # very conservative here and error if we don't recognize the input. -function edf_to_onda_unit(edf_physical_dimension::AbstractString, unit_alternatives=STANDARD_UNITS) - edf_physical_dimension = replace(edf_physical_dimension, r"\s"=>"") +function edf_to_onda_unit(edf_physical_dimension::AbstractString, + unit_alternatives=STANDARD_UNITS) + edf_physical_dimension = replace(edf_physical_dimension, r"\s" => "") for (onda_unit, potential_edf_matches) in unit_alternatives any(==(edf_physical_dimension), potential_edf_matches) && return onda_unit end diff --git a/test/export.jl b/test/export.jl index c816e8fb..97827afc 100644 --- a/test/export.jl +++ b/test/export.jl @@ -1,5 +1,4 @@ @testset "EDF Export" begin - n_records = 100 edf, edf_channel_indices = make_test_data(MersenneTwister(42), 256, 512, n_records) uuid = uuid4() @@ -10,7 +9,9 @@ signal_names = ["eeg", "eog", "ecg", "emg", "heart_rate", "tidal_volume", "respiratory_effort", "snore", "positive_airway_pressure", "pap_device_leak", "pap_device_cflow", "sao2", "ptaf"] - samples_to_export = onda_samples[indexin(signal_names, getproperty.(getproperty.(onda_samples, :info), :sensor_type))] + samples_to_export = onda_samples[indexin(signal_names, + getproperty.(getproperty.(onda_samples, :info), + :sensor_type))] exported_edf = onda_to_edf(samples_to_export, annotations) @test exported_edf.header.record_count == 200 offset = 0 @@ -20,30 +21,35 @@ edf_indices = (1:length(channel_names)) .+ offset offset += length(channel_names) samples_data = Onda.decode(samples).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, exported_edf.signals[edf_indices]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, + exported_edf.signals[edf_indices]) @test isapprox(samples_data, edf_samples; rtol=0.02) for (i, channel_name) in zip(edf_indices, channel_names) s = exported_edf.signals[i] @test s.header.label == OndaEDF.export_edf_label(signal_name, channel_name) - @test s.header.physical_dimension == OndaEDF.onda_to_edf_unit(samples.info.sample_unit) + @test s.header.physical_dimension == + OndaEDF.onda_to_edf_unit(samples.info.sample_unit) end end @testset "Record metadata" begin function change_sample_rate(samples; sample_rate) info = SamplesInfoV2(Tables.rowmerge(samples.info; sample_rate=sample_rate)) - new_data = similar(samples.data, 0, Onda.index_from_time(sample_rate, Onda.duration(samples)) - 1) + new_data = similar(samples.data, 0, + Onda.index_from_time(sample_rate, Onda.duration(samples)) - + 1) return Samples(new_data, info, samples.encoded; validate=false) end eeg_samples = only(filter(row -> row.info.sensor_type == "eeg", onda_samples)) ecg_samples = only(filter(row -> row.info.sensor_type == "ecg", onda_samples)) - massive_eeg = change_sample_rate(eeg_samples, sample_rate=5000.0) + massive_eeg = change_sample_rate(eeg_samples; sample_rate=5000.0) @test OndaEDF.edf_record_metadata([massive_eeg]) == (1000000, 1 / 5000) chunky_eeg = change_sample_rate(eeg_samples; sample_rate=9999.0) chunky_ecg = change_sample_rate(ecg_samples; sample_rate=425.0) - @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, chunky_ecg]) + @test_throws OndaEDF.RecordSizeException OndaEDF.edf_record_metadata([chunky_eeg, + chunky_ecg]) e_notation_eeg = change_sample_rate(eeg_samples; sample_rate=20_000_000.0) @test OndaEDF.edf_record_metadata([e_notation_eeg]) == (4.0e9, 1 / 20_000_000) @@ -62,7 +68,8 @@ @testset "Exception and Error handling" begin messages = ("RecordSizeException: sample rates [9999.0, 425.0] cannot be resolved to a data record size smaller than 61440 bytes", "EDFPrecisionError: String representation of value 2.0576999e7 is longer than 8 ASCII characters") - exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), OndaEDF.EDFPrecisionError(20576999.0)) + exceptions = (OndaEDF.RecordSizeException([chunky_eeg, chunky_ecg]), + OndaEDF.EDFPrecisionError(20576999.0)) for (message, exception) in zip(messages, exceptions) buffer = IOBuffer() showerror(buffer, exception) @@ -79,7 +86,8 @@ @test getproperty.(round_tripped, :span) == getproperty.(ann_sorted, :span) @test getproperty.(round_tripped, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(round_tripped, :recording) == getproperty.(ann_sorted, :recording) + @test getproperty.(round_tripped, :recording) == + getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(round_tripped, :id) .!= getproperty.(ann_sorted, :id)) end @@ -93,12 +101,14 @@ @test getproperty.(nt.annotations, :span) == getproperty.(ann_sorted, :span) @test getproperty.(nt.annotations, :value) == getproperty.(ann_sorted, :value) # same recording UUID passed as original: - @test getproperty.(nt.annotations, :recording) == getproperty.(ann_sorted, :recording) + @test getproperty.(nt.annotations, :recording) == + getproperty.(ann_sorted, :recording) # new UUID for each annotation created during import @test all(getproperty.(nt.annotations, :id) .!= getproperty.(ann_sorted, :id)) - @testset "$(samples_orig.info.sensor_type)" for - (samples_orig, signal_round_tripped) in zip(onda_samples, nt.signals) + @testset "$(samples_orig.info.sensor_type)" for (samples_orig, + signal_round_tripped) in + zip(onda_samples, nt.signals) info_orig = samples_orig.info info_round_tripped = SamplesInfoV2(signal_round_tripped) @@ -119,7 +129,9 @@ # import empty annotations exported_edf2 = onda_to_edf(samples_to_export) - @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, mktempdir(), uuid; import_annotations=true) + @test_logs (:warn, r"No annotations found in") store_edf_as_onda(exported_edf2, + mktempdir(), uuid; + import_annotations=true) end @testset "re-encoding" begin @@ -248,5 +260,4 @@ @test EDF.decode(signal) == vec(decode(samples).data) end end - end diff --git a/test/import.jl b/test/import.jl index 6c61c668..24d127ea 100644 --- a/test/import.jl +++ b/test/import.jl @@ -5,7 +5,6 @@ using Legolas: validate, SchemaVersion, read using StableRNGs @testset "Import EDF" begin - @testset "edf_to_onda_samples" begin n_records = 100 for T in (Int16, EDF.Int24) @@ -25,7 +24,8 @@ using StableRNGs @test_throws(ArgumentError(":seconds_per_record not found in header, or missing"), plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals))) - signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), edf.signals), + signal_plans = plan_edf_to_onda_samples.(filter(x -> isa(x, EDF.Signal), + edf.signals), edf.header.seconds_per_record) @testset "signal-wise plan" begin @@ -34,17 +34,19 @@ using StableRNGs validate_extracted_signals(s.info for s in returned_samples) end - + @testset "custom grouping" begin - signal_plans = [rowmerge(plan; grp=string(plan.sensor_type, plan.sample_unit, plan.sample_rate)) + signal_plans = [rowmerge(plan; + grp=string(plan.sensor_type, plan.sample_unit, + plan.sample_rate)) for plan in signal_plans] - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; onda_signal_groupby=:grp) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) validate_extracted_signals(s.info for s in returned_samples) # one channel per signal, group by label - grouped_plans = plan_edf_to_onda_samples_groups(signal_plans, + grouped_plans = plan_edf_to_onda_samples_groups(signal_plans; onda_signal_groupby=:label) returned_samples, plan = edf_to_onda_samples(edf, grouped_plans) @test all(==(1), channel_count.(returned_samples)) @@ -55,7 +57,8 @@ using StableRNGs # orders before grouping. plans_numbered = [rowmerge(plan; edf_signal_index) for (edf_signal_index, plan) - in enumerate(signal_plans)] + in + enumerate(signal_plans)] plans_rev = reverse!(plans_numbered) @test last(plans_rev).edf_signal_index == 1 @@ -74,10 +77,9 @@ using StableRNGs grouped_plans_rev_bad = plan_edf_to_onda_samples_groups(plans_rev_bad) @test_throws(ArgumentError("Plan's label EcG EKGL does not match EDF label EEG C3-M2!"), edf_to_onda_samples(edf, grouped_plans_rev_bad)) - end end - + @testset "store_edf_as_onda" begin n_records = 100 edf, edf_channel_indices = make_test_data(StableRNG(42), 256, 512, n_records) @@ -105,11 +107,12 @@ using StableRNGs signals = Dict(s.sensor_type => s for s in nt.signals) - @testset "Signal roundtrip" begin + @testset "Signal roundtrip" begin for (signal_name, edf_indices) in edf_channel_indices @testset "$signal_name" begin onda_samples = load(signals[string(signal_name)]).data - edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, edf.signals[sort(edf_indices)]) + edf_samples = mapreduce(transpose ∘ EDF.decode, vcat, + edf.signals[sort(edf_indices)]) @test isapprox(onda_samples, edf_samples; rtol=0.02) end end @@ -122,11 +125,15 @@ using StableRNGs start = Nanosecond(Second(i)) stop = start + Nanosecond(Second(i + 1)) # two annotations with same 1s span and different values: - @test any(a -> a.value == "$i a" && a.span.start == start && a.span.stop == stop, nt.annotations) - @test any(a -> a.value == "$i b" && a.span.start == start && a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i a" && a.span.start == start && + a.span.stop == stop, nt.annotations) + @test any(a -> a.value == "$i b" && a.span.start == start && + a.span.stop == stop, nt.annotations) # two annotations with instantaneous (1ns) span and different values - @test any(a -> a.value == "$i c" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) - @test any(a -> a.value == "$i d" && a.span.start == start && a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i c" && a.span.start == start && + a.span.stop == start + Nanosecond(1), nt.annotations) + @test any(a -> a.value == "$i d" && a.span.start == start && + a.span.stop == start + Nanosecond(1), nt.annotations) end end @@ -161,21 +168,26 @@ using StableRNGs end mktempdir() do root - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", annotations_prefix="edff") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edfff", + annotations_prefix="edff") @test nt.signals_path == joinpath(root, "edfff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edff.onda.annotations.arrow") end mktempdir() do root @test_logs (:warn, r"Extracting prefix") begin - nt = OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="edff.onda.signals.arrow", annotations_prefix="edf") + nt = OndaEDF.store_edf_as_onda(edf, root, uuid; + signals_prefix="edff.onda.signals.arrow", + annotations_prefix="edf") end @test nt.signals_path == joinpath(root, "edff.onda.signals.arrow") @test nt.annotations_path == joinpath(root, "edf.onda.annotations.arrow") end mktempdir() do root - @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; signals_prefix="stuff/edf", annotations_prefix="edf") + @test_throws ArgumentError OndaEDF.store_edf_as_onda(edf, root, uuid; + signals_prefix="stuff/edf", + annotations_prefix="edf") end end @@ -196,13 +208,13 @@ using StableRNGs @testset "duplicate sensor_type" begin rng = StableRNG(1234) - _signal = function(label, transducer, unit, lo, hi) + _signal = function (label, transducer, unit, lo, hi) return test_edf_signal(rng, label, transducer, unit, lo, hi, Float32(typemin(Int16)), Float32(typemax(Int16)), 128, 10, Int16) end - T = Union{EDF.AnnotationsSignal, EDF.Signal{Int16}} + T = Union{EDF.AnnotationsSignal,EDF.Signal{Int16}} edf_signals = T[_signal("EMG Chin1", "E", "mV", -100, 100), _signal("EMG Chin2", "E", "mV", -120, 90), _signal("EMG LAT", "E", "uV", 0, 1000)] @@ -212,7 +224,8 @@ using StableRNGs edf_header, edf_signals) plan = plan_edf_to_onda_samples(test_edf) - sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, p.onda_signal_index) for p in plan)) + sensors = Tables.columntable(unique((; p.sensor_type, p.sensor_label, + p.onda_signal_index) for p in plan)) @test length(sensors.sensor_type) == 2 @test all(==("emg"), sensors.sensor_type) # TODO: uniquify this in the grouping... @@ -228,28 +241,35 @@ using StableRNGs one_plan = plan_edf_to_onda_samples(one_signal, edf.header.seconds_per_record) @test one_plan.label == one_signal.header.label - @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; preprocess_labels=identity) + @test_throws ArgumentError plan_edf_to_onda_samples(one_signal, 1.0; + preprocess_labels=identity) - err_plan = @test_logs (:error, ) plan_edf_to_onda_samples(one_signal, 1.0; units=[1, 2, 3]) + err_plan = @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + units=[1, 2, 3]) @test err_plan.error isa String # malformed units arg: elements should be de-structurable @test contains(err_plan.error, "BoundsError") # malformed labels/units - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; labels=[["signal"] => nothing]) - @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + labels=[["signal"] => nothing]) + @test_logs (:error,) plan_edf_to_onda_samples(one_signal, 1.0; + units=["millivolt" => nothing]) # unit not found does not error but does create a missing - unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; units=["millivolt" => ["mV"]]) + unitless_plan = plan_edf_to_onda_samples(one_signal, 1.0; + units=["millivolt" => ["mV"]]) @test unitless_plan.error === nothing @test ismissing(unitless_plan.sample_unit) - + # error on execution plans = plan_edf_to_onda_samples(edf) # intentionally combine signals of different sensor_types - different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), plans) + different = findfirst(row -> !isequal(row.sensor_type, first(plans).sensor_type), + plans) bad_plans = rowmerge.(plans[[1, different]]; onda_signal_index=1) - bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, bad_plans) + bad_samples, bad_plans_exec = @test_logs (:error,) OndaEDF.edf_to_onda_samples(edf, + bad_plans) @test all(row.error isa String for row in bad_plans_exec) @test all(occursin("ArgumentError", row.error) for row in bad_plans_exec) @test isempty(bad_samples) @@ -299,7 +319,7 @@ using StableRNGs @test validate(Tables.schema(plan_exec), SchemaVersion("ondaedf.file-plan", 2)) === nothing - plan_rt = let io=IOBuffer() + plan_rt = let io = IOBuffer() OndaEDF.write_plan(io, plan) seekstart(io) Legolas.read(io; validate=true) @@ -308,8 +328,8 @@ using StableRNGs plan_exec_cols = Tables.columns(plan_exec) plan_rt_cols = Tables.columns(plan_rt) for col in Tables.columnnames(plan_exec_cols) - @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), Tables.getcolumn(plan_exec_cols, col))) + @test all(isequal.(Tables.getcolumn(plan_rt_cols, col), + Tables.getcolumn(plan_exec_cols, col))) end end - end diff --git a/test/runtests.jl b/test/runtests.jl index 2e49f4ab..3f7681b6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,65 +14,68 @@ function test_edf_signal(rng, label, transducer, physical_units, return EDF.Signal(header, samples) end -function make_test_data(rng, sample_rate, samples_per_record, n_records, ::Type{T}=Int16) where {T} +function make_test_data(rng, sample_rate, samples_per_record, n_records, + ::Type{T}=Int16) where {T} imin16, imax16 = Float32(typemin(T)), Float32(typemax(T)) anns_1 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] for i in 1:n_records] + EDF.TimestampedAnnotationList(i, i + 1, ["", "$i a", "$i b"])] + for i in 1:n_records] anns_2 = [[EDF.TimestampedAnnotationList(i, nothing, []), - EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] for i in 1:n_records] + EDF.TimestampedAnnotationList(i, 0, ["", "$i c", "$i d"])] + for i in 1:n_records] _edf_signal = (label, transducer, unit, lo, hi) -> begin - return test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, - imax16, samples_per_record, n_records, T) + return test_edf_signal(rng, label, transducer, unit, lo, hi, imin16, + imax16, samples_per_record, n_records, T) end # Shorthand for the eltype here E = Union{EDF.AnnotationsSignal,EDF.Signal{T}} - edf_signals = E[ - _edf_signal("EEG F3-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG F4-M1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG C3-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EEG O1-M2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("C4-M1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("O2-A1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("E1", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("E2", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("Fpz", "E", "uV", -32768.0f0, 32767.0f0), - EDF.AnnotationsSignal(samples_per_record, anns_1), - _edf_signal("EMG LAT", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("EMG RAT", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("SNORE", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("IPAP", "", "cmH2O", -74.0465f0, 74.19587f0), - _edf_signal("EPAP", "", "cmH2O", -73.5019f0, 74.01962f0), - _edf_signal("CFLOW", "", "LPM", -309.153f0, 308.8513f0), - _edf_signal("PTAF", "", "v", -125.009f0, 125.009f0), - _edf_signal("Leak", "", "LPM", -147.951f0, 148.4674f0), - _edf_signal("CHEST", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("ABD", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("Tidal", "", "mL", -4928.18f0, 4906.871f0), - _edf_signal("SaO2", "", "%", 0.0f0, 100.0f0), - EDF.AnnotationsSignal(samples_per_record, anns_2), - _edf_signal("EKG EKGR- REF", "E", "uV", -9324.0f0, 2034.0f0), - _edf_signal("IC", "E", "uV", -32768.0f0, 32767.0f0), - _edf_signal("HR", "", "BpM", -32768.0f0, 32768.0f0), - _edf_signal("EcG EKGL", "E", "uV", -10932.0f0, 1123.0f0), - _edf_signal("- REF", "E", "uV", -10932.0f0, 1123.0f0), - _edf_signal("REF1", "E", "uV", -10932.0f0, 1123.0f0), - ] + edf_signals = E[_edf_signal("EEG F3-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG F4-M1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG C3-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EEG O1-M2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("C4-M1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("O2-A1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("E1", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("E2", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("Fpz", "E", "uV", -32768.0f0, 32767.0f0), + EDF.AnnotationsSignal(samples_per_record, anns_1), + _edf_signal("EMG LAT", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("EMG RAT", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("SNORE", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("IPAP", "", "cmH2O", -74.0465f0, 74.19587f0), + _edf_signal("EPAP", "", "cmH2O", -73.5019f0, 74.01962f0), + _edf_signal("CFLOW", "", "LPM", -309.153f0, 308.8513f0), + _edf_signal("PTAF", "", "v", -125.009f0, 125.009f0), + _edf_signal("Leak", "", "LPM", -147.951f0, 148.4674f0), + _edf_signal("CHEST", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("ABD", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("Tidal", "", "mL", -4928.18f0, 4906.871f0), + _edf_signal("SaO2", "", "%", 0.0f0, 100.0f0), + EDF.AnnotationsSignal(samples_per_record, anns_2), + _edf_signal("EKG EKGR- REF", "E", "uV", -9324.0f0, 2034.0f0), + _edf_signal("IC", "E", "uV", -32768.0f0, 32767.0f0), + _edf_signal("HR", "", "BpM", -32768.0f0, 32768.0f0), + _edf_signal("EcG EKGL", "E", "uV", -10932.0f0, 1123.0f0), + _edf_signal("- REF", "E", "uV", -10932.0f0, 1123.0f0), + _edf_signal("REF1", "E", "uV", -10932.0f0, 1123.0f0)] seconds_per_record = samples_per_record / sample_rate - edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, n_records, seconds_per_record) + edf_header = EDF.FileHeader("0", "", "", DateTime("2014-10-27T22:24:28"), true, + n_records, seconds_per_record) edf = EDF.File((io = IOBuffer(); close(io); io), edf_header, edf_signals) - return edf, Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], - :eog => [7, 8], - :ecg => [27, 24], - :emg => [25, 11, 12], - :heart_rate => [26], - :tidal_volume => [21], - :respiratory_effort => [19, 20], - :snore => [13], - :positive_airway_pressure => [14, 15], - :pap_device_leak => [18], - :pap_device_cflow => [16], - :sao2 => [22], - :ptaf => [17]) + return edf, + Dict(:eeg => [9, 1, 2, 3, 5, 4, 6], + :eog => [7, 8], + :ecg => [27, 24], + :emg => [25, 11, 12], + :heart_rate => [26], + :tidal_volume => [21], + :respiratory_effort => [19, 20], + :snore => [13], + :positive_airway_pressure => [14, 15], + :pap_device_leak => [18], + :pap_device_cflow => [16], + :sao2 => [22], + :ptaf => [17]) end function validate_extracted_signals(signals) @@ -89,7 +92,8 @@ function validate_extracted_signals(signals) @test signals["positive_airway_pressure"].sample_unit == "centimeter_of_water" @test signals["heart_rate"].channels == ["heart_rate"] @test signals["heart_rate"].sample_unit == "beat_per_minute" - @test signals["emg"].channels == ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] + @test signals["emg"].channels == + ["left_anterior_tibialis", "right_anterior_tibialis", "intercostal"] @test signals["emg"].sample_unit == "microvolt" @test signals["eog"].channels == ["left", "right"] @test signals["eog"].sample_unit == "microvolt" diff --git a/test/signal_labels.jl b/test/signal_labels.jl index 7cbc6334..2d7a8a6c 100644 --- a/test/signal_labels.jl +++ b/test/signal_labels.jl @@ -1,14 +1,22 @@ @testset "EDF.Signal label handling" begin signal_names = ["eeg", "eog", "test"] canonical_names = OndaEDF.STANDARD_LABELS[["eeg"]] - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", canonical_names) == "c3-m1_plus_a2_over_2" - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", canonical_names) == nothing - @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", canonical_names) == nothing - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", canonical_names) == nothing - @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", canonical_names) == "-fpz-ref-cpz" - @test OndaEDF.match_edf_label("EOG L", signal_names, "left", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" - @test OndaEDF.match_edf_label("EOG R", signal_names, "right", OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c3", + canonical_names) == "c3-m1_plus_a2_over_2" + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", ["ecg"], "c3", + canonical_names) == nothing + @test OndaEDF.match_edf_label("EEG C3-(M1 +A2)/2 - rEf3", signal_names, "c4", + canonical_names) == nothing + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fpz", + canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label(" TEsT -Fpz -REF-cpz", signal_names, "fp", + canonical_names) == nothing + @test OndaEDF.match_edf_label(" -Fpz -REF-cpz", signal_names, "fpz", + canonical_names) == "-fpz-ref-cpz" + @test OndaEDF.match_edf_label("EOG L", signal_names, "left", + OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "left" + @test OndaEDF.match_edf_label("EOG R", signal_names, "right", + OndaEDF.STANDARD_LABELS[["eog", "eeg"]]) == "right" for (signal_names, channel_names) in OndaEDF.STANDARD_LABELS for channel_name in channel_names name = channel_name isa Pair ? first(channel_name) : channel_name From 8d9467b72e13aa6af85afa767be5f153dc51ddc1 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:52:06 +0200 Subject: [PATCH 6/6] Update .github/workflows/format-check.yml --- .github/workflows/format-check.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index 3cb68a38..40f048b2 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -1,10 +1,5 @@ name: format-check on: - push: - branches: - - 'main' - - /^release-.*$/ - tags: ['*'] pull_request: types: [opened, synchronize, reopened, ready_for_review] concurrency: