Skip to content
Snippets Groups Projects
Commit 4fedb02a authored by Florian Obersteiner's avatar Florian Obersteiner :octopus:
Browse files

Merge branch 'dev-0014' into 'main' 

skip-no-acn-datafiles

See merge request !5
parents 96e25e08 d8393f08
No related branches found
No related tags found
1 merge request!5skip-no-acn-datafiles
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 7
− 0
View file @ 4fedb02a
......@@ -29,6 +29,13 @@ Types of changes
## [Unreleased]
### Changed
- skip files that have no ACN data
- tweak layout so that flights dropdown list doesn't get mangled up
## v0.0.13 - 2025-03-26
### Added
- load data from THREDDS TDS
......
main.py
+ 3
− 2
View file @ 4fedb02a
......@@ -4,6 +4,7 @@
# -*- coding: utf-8 -*-
import logging as log
from pathlib import Path
import dash_bootstrap_components as dbc
......@@ -41,8 +42,8 @@ data, ap_info = (
if appconfig.USE_THREDDS
else load_from_disk(Path("./testdata").glob("*.nc"), appconfig, airportsconfig, True) # type: ignore
)
print(f"Data loaded. Used THREDDS: {appconfig.USE_THREDDS}")
print(f"Data size: {data.info()}")
log.debug(f"Data loaded. Used THREDDS: {appconfig.USE_THREDDS}")
log.debug(f"Data size: {data.info()}")
app.layout = create_layout(data, appconfig)
server = app.server # for deployment via gunicorn / WSGI server
......
requirements.txt 0 → 100644
+ 105
− 0
View file @ 4fedb02a
# This file was autogenerated by uv via the following command:
# uv pip compile pyproject.toml -o requirements.txt
beautifulsoup4==4.13.3
# via siphon
blinker==1.9.0
# via flask
certifi==2025.1.31
# via
# netcdf4
# requests
cftime==1.6.4.post1
# via netcdf4
charset-normalizer==3.4.1
# via requests
click==8.1.8
# via flask
dash==3.0.1
# via
# caribic-dash (pyproject.toml)
# dash-bootstrap-components
dash-bootstrap-components==2.0.0
# via caribic-dash (pyproject.toml)
flask==3.0.3
# via dash
gunicorn==23.0.0
# via caribic-dash (pyproject.toml)
idna==3.10
# via requests
importlib-metadata==8.6.1
# via dash
itsdangerous==2.2.0
# via flask
jinja2==3.1.6
# via flask
markupsafe==3.0.2
# via
# jinja2
# werkzeug
narwhals==1.32.0
# via plotly
nest-asyncio==1.6.0
# via dash
netcdf4==1.7.2
# via caribic-dash (pyproject.toml)
numpy==2.2.4
# via
# cftime
# netcdf4
# pandas
# siphon
# xarray
packaging==24.2
# via
# gunicorn
# plotly
# xarray
pandas==2.2.3
# via
# caribic-dash (pyproject.toml)
# siphon
# xarray
plotly==6.0.1
# via
# caribic-dash (pyproject.toml)
# dash
protobuf==6.30.1
# via siphon
python-dateutil==2.9.0.post0
# via pandas
python-dotenv==1.1.0
# via caribic-dash (pyproject.toml)
pytz==2025.2
# via pandas
requests==2.32.3
# via
# dash
# siphon
retrying==1.3.4
# via dash
setuptools==78.1.0
# via dash
siphon==0.10.0
# via caribic-dash (pyproject.toml)
six==1.17.0
# via
# python-dateutil
# retrying
soupsieve==2.6
# via beautifulsoup4
typing-extensions==4.13.0
# via
# beautifulsoup4
# dash
tzdata==2025.2
# via pandas
urllib3==2.3.0
# via requests
werkzeug==3.0.6
# via
# dash
# flask
xarray==2025.3.0
# via caribic-dash (pyproject.toml)
zipp==3.21.0
# via importlib-metadata
src/data.py
+ 10
− 4
View file @ 4fedb02a
......@@ -20,12 +20,16 @@ def load_from_thredds(
"""
Load multiple netCDF files from a THREDS TDS into a single DataFrame.
"""
print("begin load data from THREDDS...")
log.debug("begin load data from THREDDS...")
dataframes = []
airports = {}
for i, ds in enumerate(datasets):
fullset = datasets[i].remote_access(use_xarray=True).to_dataframe()
if "ACN" in fullset.columns and fullset["ACN"].isna().all():
log.warning(f"'ACN' column is all NaN values! Skip {ds.name}")
continue
if drop:
df = pd.DataFrame()
for v in config.VARIABLES:
......@@ -42,8 +46,6 @@ def load_from_thredds(
df["date"] = datetime.strptime(ds.name.split("_")[1], "%Y%m%d").strftime("%Y-%m-%d")
# make a linear interpolation of the acetonitrile column for BB flaggin
if "ACN" in df.columns:
if df["ACN"].isna().all():
log.warning("'ACN' column is all NaN values!")
df["BB_flag"] = df["ACN"].interpolate() > (
config.BB_THRESH + config.ACN_PRC_NSIGMA * df["ACN_prc"].interpolate()
)
......@@ -52,7 +54,7 @@ def load_from_thredds(
dataframes.append(df)
print("Data loaded from THREDDS successfully")
log.debug("Data loaded from THREDDS successfully")
return (pd.concat(dataframes), airports)
......@@ -67,6 +69,10 @@ def load_from_disk(
for path in sorted(paths):
fullset = xr.open_dataset(path).to_dataframe()
if "ACN" in fullset.columns and fullset["ACN"].isna().all():
log.warning(f"'ACN' column is all NaN values! Skip {path.name}")
continue
if drop:
df = pd.DataFrame()
for v in config.VARIABLES:
......
src/layout.py
+ 2
− 2
View file @ 4fedb02a
......@@ -100,7 +100,7 @@ def create_layout(df: pd.DataFrame, config: DotDict) -> list:
searchable=True,
style={
"display": "inline-block",
"width": "180px",
"width": "190px",
},
),
html.Label(
......@@ -131,7 +131,7 @@ def create_layout(df: pd.DataFrame, config: DotDict) -> list:
searchable=True,
style={
"display": "inline-block",
"width": "180px",
"width": "190px",
},
),
html.Label(
......
src/thredds.py
+ 4
− 2
View file @ 4fedb02a
import logging as log
from siphon.catalog import Dataset, TDSCatalog
from .config import DotDict
def get_datasets(cfg: DotDict) -> list[Dataset]:
print("begin query THREDDS...")
log.debug("begin query THREDDS...")
catalog = TDSCatalog(cfg.THREDDS_URL)
flights = (
list(range(cfg.FLIGHTNO_MIN, cfg.FLIGHTNO_MAX + 1))
......@@ -23,5 +25,5 @@ def get_datasets(cfg: DotDict) -> list[Dataset]:
continue
datasets.append(catalog.datasets[i])
print("THREDDS query completed")
log.debug("THREDDS query completed")
return datasets
uv.lock
+ 6
− 6
View file @ 4fedb02a
......@@ -696,11 +696,11 @@ wheels = [
[[package]]
name = "setuptools"
version = "78.0.2"
version = "78.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/4c/f4/aa8d364f0dc1f33b2718938648c31202e2db5cd6479a73f0a9ca5a88372d/setuptools-78.0.2.tar.gz", hash = "sha256:137525e6afb9022f019d6e884a319017f9bf879a0d8783985d32cbc8683cab93", size = 1367747 }
sdist = { url = "https://files.pythonhosted.org/packages/a9/5a/0db4da3bc908df06e5efae42b44e75c81dd52716e10192ff36d0c1c8e379/setuptools-78.1.0.tar.gz", hash = "sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54", size = 1367827 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/aa/db/2fd473dfe436ad19fda190f4079162d400402aedfcc41e048d38c0a375c6/setuptools-78.0.2-py3-none-any.whl", hash = "sha256:4a612c80e1f1d71b80e4906ce730152e8dec23df439f82731d9d0b608d7b700d", size = 1255965 },
{ url = "https://files.pythonhosted.org/packages/54/21/f43f0a1fa8b06b32812e0975981f4677d28e0f3271601dc88ac5a5b83220/setuptools-78.1.0-py3-none-any.whl", hash = "sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8", size = 1256108 },
]
[[package]]
......@@ -748,11 +748,11 @@ wheels = [
[[package]]
name = "typing-extensions"
version = "4.12.2"
version = "4.13.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 }
sdist = { url = "https://files.pythonhosted.org/packages/0e/3e/b00a62db91a83fff600de219b6ea9908e6918664899a2d85db222f4fbf19/typing_extensions-4.13.0.tar.gz", hash = "sha256:0a4ac55a5820789d87e297727d229866c9650f6521b64206413c4fbada24d95b", size = 106520 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 },
{ url = "https://files.pythonhosted.org/packages/e0/86/39b65d676ec5732de17b7e3c476e45bb80ec64eb50737a8dce1a4178aba1/typing_extensions-4.13.0-py3-none-any.whl", hash = "sha256:c8dd92cc0d6425a97c18fbb9d1954e5ff92c1ca881a309c45f06ebc0b79058e5", size = 45683 },
]
[[package]]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Help | Imprint | Privacy policy | Accessibility | Contact