Configs

Central constants and utilities for MARIS NetCDF encoding: variable and group mappings, lookup-table loading, enumeration types, and path helpers.

from nbdev.showdoc import show_doc

Exported source

AVOGADRO = 6.02214076e23

Exported source

NA = 'Not available'

Exported source

NC_DIM = 'id'

NC_CSV maps handler column names (ALLCAPS keys) to their output names in both NetCDF and CSV formats. Each entry is KEY: (nc_name, csv_name). A None value means that key isn’t used in that format: if it lacks an NC name, it’s CSV-only; if it lacks a CSV name, it’s NC-only.

NC_VARS and CSV_VARS are derived automatically from this single source.

Exported source

NC_CSV = {
    'AREA':            ('area',          'area'),
    'BIO_GROUP':       ('bio_group',     None),
    'BODY_PART':       ('body_part',     'bodypar_id'),
    'BOTTOM':          ('bottom',        'slicedown'),
    'COUNT_MET':       ('count_met',     'counmet_id'),
    'DL':              ('dl',            'detection'),
    'DLV':             ('dlv',           'detection_lim'),
    'DRYWT':           ('drywt',         'drywt'),
    'FILT':            ('filt',          'filtered'),
    'LAB':             ('lab',           'lab_id'),
    'LAT':             ('lat',           'latitude'),
    'LON':             ('lon',           'longitude'),
    'NUCLIDE':         ('nuclide',       'nuclide_id'),
    'PERCENTWT':       ('percentwt',     'percentwt'),
    'PH':              ('ph',            None),
    'PREP_MET':        ('prep_met',      'prepmet_id'),
    'PROFILE_ID':      ('profile_id',    'profile_id'),
    'REF_ID':          (None,            'ref_id'),
    'SAL':             ('sal',           'salinity'),
    'SAMP_MET':        ('samp_met',      'sampmet_id'),
    'SAMPLE_TYPE':     (None,            'samptype_id'),
    'SED_TYPE':        ('sed_type',      'sedtype_id'),
    'SMP_DEPTH':       ('smp_depth',     'sampdepth'),
    'SMP_ID':          ('id',            None),
    'SMP_ID_PROVIDER': ('id_provider',   'samplabcode'),
    'SPECIES':         ('species',       'species_id'),
    'STATION':         ('station',       'station'),
    'TAXONDB':         (None,            'taxondb'),
    'TAXONDBID':       (None,            'taxondb_id'),
    'TAXONDBURL':      (None,            'taxondb_url'),
    'TAXONNAME':       (None,            'taxonname'),
    'TAXONRANK':       (None,            'taxonrank'),
    'TAXONREPNAME':    (None,            'taxonrepname'),
    'TEMP':            ('temp',          'temperatur'),
    'TIME':            ('time',          'begperiod'),
    'TOP':             ('top',           'sliceup'),
    'TOT_DEPTH':       ('tot_depth',     'totdepth'),
    'UNC':             ('unc',           'uncertaint'),
    'UNIT':            ('unit',          'unit_id'),
    'VALUE':           ('value',         'activity'),
    'VOL':             ('vol',           'volume'),
    'WETWT':           ('wetwt',         'wetwt'),
}

NC_VARS  = {k: v[0] for k,v in NC_CSV.items() if v[0] is not None}
CSV_VARS = {k: v[1] for k,v in NC_CSV.items() if v[1] is not None}

TODO: Fields documented in the OpenRefine CSV format but not yet integrated into the pipeline: - SAMPLE_LAB_CODE: sample lab code (relationship with SMP_ID/SMP_ID_PROVIDER TBD) - SED_REPNAME: sediment replicate name - MEASURE_NOTE: measurement notes - REFERENCE_NOTE: reference notes - SAMPLE_NOTE: sample notes - PH: documented in NC_VARS but missing from field-definition.ipynb

Not yet in NC_CSV: need to decide if/how they map to NC variables before adding.

Exported source

NC_GROUPS = {'BIOTA': 'biota',
             'SEAWATER': 'seawater',
             'SEDIMENT': 'sediment',
             'SUSPENDED_MATTER': 'suspended_matter'}

Two parallel group mappings: NC_GROUPS maps each sample-type group to its lowercase NetCDF group name (used when writing to the file). SMP_TYPE_LUT maps the same groups to the MARIS database integer IDs (used when encoding samptype_id in CSV export). Both are needed because the NetCDF schema and the legacy CSV schema identify groups differently.

Exported source

SMP_TYPE_LUT = {
    'SEAWATER': 1,
    'BIOTA': 2,
    'SEDIMENT': 3,
    'SUSPENDED_MATTER': 4
}

NC_DTYPES defines every variable that uses a NetCDF enumeration type. Each entry maps a handler column key to {name, fname, key, value}: - name — the NetCDF4 enumeration type name (e.g. 'nuclide_t') - fname — the Excel file bundled in marisco/files/lut/ - key — the Excel column used as dictionary keys - value — the Excel column used as integer IDs

Exported source

NC_DTYPES = {
    'AREA': {
        'name': 'area_t', 
        'fname': 'dbo_area.xlsx',
        'key': 'displayName', 
        'value':'areaId'
    },
    'BIO_GROUP': {
        'name': 'bio_group_t', 
        'fname': 'dbo_biogroup.xlsx',
        'key': 'biogroup', 
        'value':'biogroup_id'
    },
    'BODY_PART': {
        'name': 'body_part_t', 
        'fname': 'dbo_bodypar.xlsx',
        'key': 'bodypar', 
        'value':'bodypar_id'
    },
    'COUNT_MET': {
        'name': 'count_met_t', 
        'fname': 'dbo_counmet.xlsx',
        'key': 'counmet',
        'value':'counmet_id'
    },
    'DL': {
        'name': 'dl_t', 
        'fname': 'dbo_detectlimit.xlsx',
        'key': 'name_sanitized', 
        'value':'id'
    },
    'FILT': {
        'name': 'filt_t', 
        'fname': 'dbo_filtered.xlsx',
        'key': 'name',
        'value':'id'
    },
    'NUCLIDE': {
        'name': 'nuclide_t', 
        'fname': 'dbo_nuclide.xlsx',
        'key': 'nc_name',
        'value':'nuclide_id'
    },
    'PREP_MET': {
        'name': 'prep_met_t', 
        'fname': 'dbo_prepmet.xlsx', 
        'key': 'prepmet',
        'value':'prepmet_id'
    },
    'SAMP_MET': {
        'name': 'samp_met_t', 
        'fname': 'dbo_sampmet.xlsx', 
        'key': 'sampmet',
        'value':'sampmet_id'
    },
    'SED_TYPE': {
        'name': 'sed_type_t', 
        'fname': 'dbo_sedtype.xlsx', 
        'key': 'sedtype', 
        'value':'sedtype_id'
    },
    'SPECIES': {
        'name': 'species_t', 
        # 'fname': 'dbo_species_cleaned.xlsx',
        'fname': 'dbo_species_2024_11_19.xlsx',
        'key': 'species', 
        'value':'species_id'
    },
    'UNIT': {
        'name': 'unit_t', 
        'fname': 'dbo_unit.xlsx', 
        'key': 'unit_sanitized', 
        'value':'unit_id'
    },
    'LAB': {
        'name': 'lab_t', 
        #'fname': 'dbo_lab.xlsx', 
        'fname': 'dbo_lab_cleaned.xlsx', 
        'key': 'lab', 
        'value':'lab_id'
    }
}

Exported source

CSV_DTYPES = {
    'AREA': {'state': 'decoded'},
    'NUCLIDE': {'state': 'encoded'},  # encoded nuclide_id
    'UNIT': {'state': 'encoded'},  # encoded unit_id
    'DL': {'state': 'decoded'},
    'FILT': {'state': 'decoded'},
    'COUNT_MET': {'state': 'encoded'},  # encoded counmet_id
    'SAMP_MET': {'state': 'encoded'},  # encoded sampmet_id
    'PREP_MET': {'state': 'encoded'},  # encoded prepmet_id
    'SPECIES': {'state': 'encoded'},  # encoded species_id
    'BODY_PART': {'state': 'encoded'},  # encoded bodypar_id
    'SED_TYPE': {'state': 'encoded'},  # encoded sedtype_id
    'LAB': {'state': 'encoded'},  # encoded lab_id
}

CSV_DTYPES controls whether a column is written as human-readable names ('decoded') or as integer IDs ('encoded') in CSV export. For example, 'AREA' uses 'decoded' so the CSV contains area names, while 'NUCLIDE' uses 'encoded' so it contains nuclide_id values matching the MARIS database.

Exported source

ZOTERO_LIB_ID = '2432820'

NetCDF global attributes

The set of valid global attribute names, extracted from the template CDL. Any key written to the NetCDF global attrs dict during encoding must belong to this set; otherwise a misspelled name would be silently injected into the output file.

Exported source

NC_GLOBAL_ATTRS = {
    'id', 'title', 'summary', 'keywords', 'history',
    'keywords_vocabulary', 'keywords_vocabulary_url',
    'record', 'featureType', 'cdm_data_type', 'Conventions',
    'publisher_name', 'publisher_email', 'publisher_url', 'publisher_institution',
    'creator_name', 'institution', 'metadata_link', 'creator_email', 'creator_url',
    'references', 'license', 'comment',
    'geospatial_lat_min', 'geospatial_lat_max', 'geospatial_lon_min', 'geospatial_lon_max',
    'geospatial_vertical_min', 'geospatial_vertical_max', 'geospatial_bounds', 'geospatial_bounds_crs',
    'time_coverage_start', 'time_coverage_end', 'local_time_zone',
    'date_created', 'date_modified', 'publisher_postprocess_logs',
}

Path helpers

source

lut_path


def lut_path(
    
)->Path: # Path to LUTs directory

Return the path to the lookup tables directory.

source

lut_fname


def lut_fname(
    key:str, # NC_DTYPES key, e.g. 'SPECIES', 'UNIT', 'DL'
)->Path: # Full path to the lookup table Excel file

Return the full path to a lookup table file by its NC_DTYPES key.

source

nc_tpl_path


def nc_tpl_path(
    
)->Path: # Path to MARIS NetCDF template

Return the path to the MARIS NetCDF template file.

_pkg_files('marisco')

Path('/app/data/marisco/marisco')

nc_tpl_path()

Path('/app/data/marisco/marisco/files/nc/maris-template.nc')

source

cache_path


def cache_path(
    
)->Path: # Path to cache directory

Return the path to the cache directory, creating it if needed.

Utilities function

source

get_time_units


def get_time_units(
    
)->str:

Get the units attribute of the time variable from a NetCDF file.

Exported source

NETCDF_TO_PYTHON_TYPE = {
    'u8': int,
    'f4': float
    }

Usage example:

time_units = get_time_units(); time_units

'seconds since 1970-01-01 00:00:00.0'

Enumeration types

Enumeration types are used to avoid using strings as NetCDF4 variable values. Instead, enumeration types (lookup tables) such as {'Crustaceans': 2, 'Echinoderms': 3, ...} are prepended to the NetCDF file template and associated ids (integers) are used as values.

source

sanitize


def sanitize(
    s:str | float, # String or float to sanitize
)->str | float: # Sanitized string or original float

Sanitize dictionary key to comply with NetCDF enumeration type:

Remove (, ), ., /, -
Strip the string
Return original value if it’s not a string (e.g., NaN)

For example:

test_eq(sanitize('key (sanitized)'), 'key sanitized')
test_eq(sanitize('key san.itized'), 'key sanitized')
test_eq(sanitize('key-sanitized'), 'key sanitized')
test_eq(sanitize('key/sanitized'), 'key sanitized')

NetCDF4 enumeration type seems to not accept keys containing non alphanumeric characters like parentheses, dots, slash, … As a result, MARIS lookup table needs to be sanitized.

source

try_int


def try_int(
    x:Any, # Value to attempt integer conversion on
)->int | typing.Any: # Integer if successful, or the original value

Try to convert x to an integer.

Sanitised keys are then coerced to integers where possible — some lookup-table keys (e.g. area codes) come as numeric strings like "1", which NetCDF4 enums interpret more reliably as integers. The try_int helper handles this conversion without breaking on genuinely non-numeric string keys.

source

get_lut


def get_lut(
    key_or_fname:str, # NC_DTYPES key (e.g. 'NUCLIDE') or Excel filename
    key:Optional=None, # Column for dict keys; inferred from NC_DTYPES if applicable
    value:Optional=None, # Column for dict values; inferred from NC_DTYPES if applicable
    src_dir:Optional=None, # Directory containing lookup tables (default: lut_path())
    do_sanitize:bool=True, # Sanitization required?
    reverse:bool=False, # Reverse lookup table (value, key)
    check_duplicates:bool=False, # Check for duplicates in lookup table
    as_df:bool=False, # Return DataFrame instead of dict (for fuzzy_merge etc.)
)->Union: # MARIS lookup table (key, value) or (key, value) DataFrame

Convert MARIS db lookup table excel file to dictionary or DataFrame.

For example:

get_lut('BIO_GROUP', reverse=False)

{'Not applicable': -1,
 'Not available': 0,
 'Birds': 1,
 'Crustaceans': 2,
 'Echinoderms': 3,
 'Fish': 4,
 'Mammals': 5,
 'Molluscs': 6,
 'Others': 7,
 'Plankton': 8,
 'Polychaete worms': 9,
 'Reptile': 10,
 'Seaweeds and plants': 11,
 'Cephalopods': 12,
 'Gastropods': 13,
 'Bivalves': 14}

source

Enums


def Enums(
    lut_src_dir:str, # Directory containing lookup tables
    dtypes:Dict={'AREA': {'name': 'area_t', 'fname': 'dbo_area.xlsx', 'key': 'displayName', 'value': 'areaId'}, 'BIO_GROUP': {'name': 'bio_group_t', 'fname': 'dbo_biogroup.xlsx', 'key': 'biogroup', 'value': 'biogroup_id'}, 'BODY_PART': {'name': 'body_part_t', 'fname': 'dbo_bodypar.xlsx', 'key': 'bodypar', 'value': 'bodypar_id'}, 'COUNT_MET': {'name': 'count_met_t', 'fname': 'dbo_counmet.xlsx', 'key': 'counmet', 'value': 'counmet_id'}, 'DL': {'name': 'dl_t', 'fname': 'dbo_detectlimit.xlsx', 'key': 'name_sanitized', 'value': 'id'}, 'FILT': {'name': 'filt_t', 'fname': 'dbo_filtered.xlsx', 'key': 'name', 'value': 'id'}, 'NUCLIDE': {'name': 'nuclide_t', 'fname': 'dbo_nuclide.xlsx', 'key': 'nc_name', 'value': 'nuclide_id'}, 'PREP_MET': {'name': 'prep_met_t', 'fname': 'dbo_prepmet.xlsx', 'key': 'prepmet', 'value': 'prepmet_id'}, 'SAMP_MET': {'name': 'samp_met_t', 'fname': 'dbo_sampmet.xlsx', 'key': 'sampmet', 'value': 'sampmet_id'}, 'SED_TYPE': {'name': 'sed_type_t', 'fname': 'dbo_sedtype.xlsx', 'key': 'sedtype', 'value': 'sedtype_id'}, 'SPECIES': {'name': 'species_t', 'fname': 'dbo_species_2024_11_19.xlsx', 'key': 'species', 'value': 'species_id'}, 'UNIT': {'name': 'unit_t', 'fname': 'dbo_unit.xlsx', 'key': 'unit_sanitized', 'value': 'unit_id'}, 'LAB': {'name': 'lab_t', 'fname': 'dbo_lab_cleaned.xlsx', 'key': 'lab', 'value': 'lab_id'}}, # Dict keyed by NC_DTYPES key, each is {name, fname, key, value}
):

Hold and filter MARIS NetCDF enumeration types loaded from lookup tables.

source

Enums.lookup


def lookup(
    
)->Dict:

Load all enumeration types defined in NC_DTYPES as {name: id} dictionaries, available via self.types[var_name].

source

Enums.lookup


def lookup(
    
)->Dict:

Load all enumeration types defined in NC_DTYPES as {name: id} dictionaries, available via self.types[var_name].

source

Enums.filter


def filter(
    var_name:str, # NC_DTYPES key for the enumeration, e.g. 'SPECIES'
    values:list, # Enumeration IDs to keep
)->Dict:

Return a subset of an enumeration keeping only entries whose id is in values.

source

Enums.filter


def filter(
    var_name:str, # NC_DTYPES key for the enumeration, e.g. 'SPECIES'
    values:list, # Enumeration IDs to keep
)->Dict:

Return a subset of an enumeration keeping only entries whose id is in values.

lut_src_dir_test = lut_path()
enums = Enums(lut_src_dir=lut_src_dir_test)

# Keep only 'Detected value' (id=1) and 'Not detected' (id=3) from the DL enumeration
enums.filter('DL', values=[1, 3])

{'Detected value': 1, 'Not detected': 3}

enums.types['DL']

{'Not applicable': -1,
 'Not available': 0,
 'Detected value': 1,
 'Detection limit': 2,
 'Not detected': 3,
 'Derived': 4}