From 514559ed39b193b2f27a83d32ebdf1840c4f2403 Mon Sep 17 00:00:00 2001 From: Alexander Barth Date: Wed, 10 Jan 2024 12:20:48 +0100 Subject: [PATCH] implement alternative to missing values --- docs/make.jl | 1 + docs/src/index.md | 2 +- src/NCDatasets.jl | 3 ++- src/dataset.jl | 18 +++++++++++++----- src/types.jl | 3 ++- test/test_fillvalue.jl | 28 ++++++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 847ec579..335f607d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -16,6 +16,7 @@ makedocs( "Dimensions" => "dimensions.md", "Variables" => "variables.md", "Attributes" => "attributes.md", + "Fill values" => "fillvalue.md", "Performance tips" => "performance.md", "Known issues" => "issues.md", "Experimental features" => "experimental.md", diff --git a/docs/src/index.md b/docs/src/index.md index 026d4abe..d132669e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -36,7 +36,7 @@ To get started quickly see the [Quickstart](@ref) section. Otherwise see the fol * [Dimensions](@ref) : accessing/creating NetCDF dimensions * [Variables](@ref) : accessing/examining the variables (or dimensions) stored within a NetCDF dataset. * [Attributes](@ref) : accessing/creating NetCDF attributes -* See [Performance tips](@ref performance_tips), [Known issues](@ref), [Experimental features](@ref) for more information. +* See [Fill values and missing values](@ref), [Performance tips](@ref performance_tips), [Known issues](@ref), [Experimental features](@ref) for more information. ## Quick start diff --git a/src/NCDatasets.jl b/src/NCDatasets.jl index a5f69475..f222a546 100644 --- a/src/NCDatasets.jl +++ b/src/NCDatasets.jl @@ -46,7 +46,8 @@ import CommonDataModel: AbstractDataset, AbstractVariable, iswritable, sync, CatArrays, SubDataset, @select, select, Near, coordinate_value, coordinate_names, split_by_and, - chunking, deflate, checksum + chunking, deflate, checksum, + _experimental_missing_value import DiskArrays diff --git a/src/dataset.jl b/src/dataset.jl index 3900f9e8..a68ea920 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -28,6 +28,7 @@ const jlType = Dict( const ncType = Dict(value => key for (key, value) in jlType) iswritable(ds::NCDataset) = ds.iswritable +_experimental_missing_value(ds::NCDataset) = ds._experimental_missing_value function isopen(ds::NCDataset) try @@ -64,7 +65,9 @@ end function NCDataset(ncid::Integer, iswritable::Bool, isdefmode::Ref{Bool}; - parentdataset = nothing) + parentdataset = nothing, + _experimental_missing_value = missing, + ) function _finalize(ds) # only close open root group @@ -72,13 +75,15 @@ function NCDataset(ncid::Integer, close(ds) end end - - ds = NCDataset{typeof(parentdataset)}( + @debug "_experimental_missing_value" _experimental_missing_value + ds = NCDataset{typeof(parentdataset),typeof(_experimental_missing_value)}( parentdataset, ncid, iswritable, isdefmode, - Dict{String,String}()) + Dict{String,String}(), + _experimental_missing_value, + ) if !iswritable initboundsmap!(ds) @@ -169,6 +174,7 @@ function NCDataset(filename::AbstractString, diskless::Bool = false, persist::Bool = false, memory::Union{Vector{UInt8},Nothing} = nothing, + _experimental_missing_value = missing, attrib = []) ncid = -1 @@ -226,7 +232,9 @@ function NCDataset(filename::AbstractString, end iswritable = mode != "r" - ds = NCDataset(ncid,iswritable,isdefmode) + ds = NCDataset( + ncid,iswritable,isdefmode, + _experimental_missing_value = _experimental_missing_value) # set global attributes for (attname,attval) in attrib diff --git a/src/types.jl b/src/types.jl index 1985e5d4..1afb05bf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -24,7 +24,7 @@ struct Variable{NetCDFType,N,TDS} <: AbstractNCVariable{NetCDFType, N} end # must be mutable to register a finalizer -mutable struct NCDataset{TDS} <: AbstractNCDataset where TDS <: Union{AbstractNCDataset,Nothing} +mutable struct NCDataset{TDS,T_experimental_missing_value} <: AbstractNCDataset where TDS <: Union{AbstractNCDataset,Nothing} # parent_dataset is nothing for the root dataset parentdataset::TDS ncid::Cint @@ -35,6 +35,7 @@ mutable struct NCDataset{TDS} <: AbstractNCDataset where TDS <: Union{AbstractNC # mapping between variables related via the bounds attribute # It is only used for read-only datasets to improve performance _boundsmap::Dict{String,String} + _experimental_missing_value::T_experimental_missing_value end "Alias to `NCDataset`" diff --git a/test/test_fillvalue.jl b/test/test_fillvalue.jl index b9038d13..00f83804 100644 --- a/test/test_fillvalue.jl +++ b/test/test_fillvalue.jl @@ -166,3 +166,31 @@ v = defVar(ds,"instrument_mode",String,("mode_items",),fillvalue = "UNDEFINED MO @test fillvalue(v) == "UNDEFINED MODE" close(ds) rm(filename) + + + +# Alternative to Missing for NetCDF fillvalue + +fname = tempname() +data = randn(3,4) +fv = 9999 +data[2,2] = fv + +ds = NCDataset(fname,"c") +defDim(ds,"lon",size(data,1)) +defDim(ds,"lat",size(data,2)) +ncv = defVar(ds,"data",Float64,("lon","lat"),fillvalue = fv) +ncv.var[:,:] = data + +ncv = cfvariable(ds,"data",_experimental_missing_value = NaN) +@test eltype(ncv) == Float64 +@test ncv[1,1] == data[1,1] +@test isnan(ncv[2,2]) +close(ds) + + +ds = NCDataset(fname,"r",_experimental_missing_value = NaN) +ncv = ds["data"] + +@test ncv[1,1] == data[1,1] +@test isnan(ncv[2,2])