From e0478cd93514a6d624c0586b102521fda9be71b3 Mon Sep 17 00:00:00 2001 From: akrherz Date: Mon, 20 Nov 2023 08:35:45 -0600 Subject: [PATCH 1/3] mnt: quell numpy cast RuntimeWarning --- scripts/iemre/db_to_netcdf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/iemre/db_to_netcdf.py b/scripts/iemre/db_to_netcdf.py index d439384df3..2be43e4a3a 100644 --- a/scripts/iemre/db_to_netcdf.py +++ b/scripts/iemre/db_to_netcdf.py @@ -8,11 +8,14 @@ """ import datetime import sys +import warnings import numpy as np from pyiem import iemre from pyiem.util import logger, ncopen, utc +# We are going from float64 to uint16, so this appears to be unavoidable +warnings.simplefilter("ignore", RuntimeWarning) LOG = logger() @@ -32,7 +35,7 @@ def main(argv): if vname not in nc.variables: continue # Careful here, ds could contain NaN values - nc.variables[vname][idx, :, :] = np.ma.array( + nc.variables[vname][idx] = np.ma.array( ds[vname].values, mask=np.isnan(ds[vname].values) ) From 32076b67d3784e6992f671580a8ea9f026cc89aa Mon Sep 17 00:00:00 2001 From: akrherz Date: Mon, 20 Nov 2023 11:42:25 -0600 Subject: [PATCH 2/3] fix: use uint16 storage for power_swdn --- scripts/iemre/init_daily.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/iemre/init_daily.py b/scripts/iemre/init_daily.py index 5273fe4d5b..d3e2eea18f 100644 --- a/scripts/iemre/init_daily.py +++ b/scripts/iemre/init_daily.py @@ -178,9 +178,11 @@ def init_year(ts): v2.coordinates = "lon lat" v2.description = "Daily averaged wind speed magnitude" + # 0 to 65535 so 0 to 65.535 v1 = nc.createVariable( - "power_swdn", float, ("time", "lat", "lon"), fill_value=1.0e20 + "power_swdn", np.uint16, ("time", "lat", "lon"), fill_value=65535 ) + v1.scale_factor = 0.001 v1.units = "MJ d-1" v1.long_name = "All Sky Insolation Incident on a Horizontal Surface" v1.standard_name = "All Sky Insolation Incident on a Horizontal Surface" From f70c8e79283f0738cea39c915ffab6d6365f002b Mon Sep 17 00:00:00 2001 From: akrherz Date: Mon, 20 Nov 2023 12:18:16 -0600 Subject: [PATCH 3/3] mnt: migrate AHPS stage metadata to NWPS #578 --- scripts/hads/process_ahps_xml.py | 99 ------------------------- scripts/hads/process_nwps_stages.py | 79 ++++++++++++++++++++ scripts/windrose/daily_drive_network.py | 2 +- 3 files changed, 80 insertions(+), 100 deletions(-) delete mode 100644 scripts/hads/process_ahps_xml.py create mode 100644 scripts/hads/process_nwps_stages.py diff --git a/scripts/hads/process_ahps_xml.py b/scripts/hads/process_ahps_xml.py deleted file mode 100644 index d6c7087506..0000000000 --- a/scripts/hads/process_ahps_xml.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Ingest the rich metadata found within the AHPS2 website!""" -import sys - -import requests -from pyiem.network import Table as NetworkTable -from pyiem.util import get_dbconn, logger -from twisted.words.xish import domish, xpath - -LOG = logger() - - -def process_site(mcursor, nwsli, meta): - """Do our processing work""" - - url = ( - "http://water.weather.gov/ahps2/hydrograph_to_xml.php?" - f"gage={nwsli}&output=xml" - ) - - elementStream = domish.elementStream() - roots = [] - results = [] - elementStream.DocumentStartEvent = roots.append - elementStream.ElementEvent = lambda elem: roots[0].addChild(elem) - elementStream.DocumentEndEvent = lambda: results.append(roots[0]) - try: - req = requests.get(url, timeout=30) - xml = req.content - if xml.strip() == "No results found for this gage.": - LOG.info("No results for %s", nwsli) - return - except Exception as exp: - LOG.exception(exp) - return - try: - elementStream.parse(xml) - except Exception as exp: - LOG.exception(exp) - return - - elem = results[0] - - nodes = xpath.queryForNodes("/site/sigstages", elem) - if nodes is None: - LOG.info("No sigstages data found for %s", nwsli) - return - - sigstages = nodes[0] - diction = "low action bankfull flood moderate major record".split() - is_new = False - msg = "" - for s in sigstages.children: - if s.name not in diction: - continue - val = str(s) - if val == "": - continue - key = f"sigstage_{s.name}" - val = float(val) - if val < 0: - continue - # is this updated info? - current = meta.get(key) - if current is None or abs(current - val) > 0.01: - is_new = True - msg += f"{s.name}: {current}->{val} " - meta[key] = val - if not is_new: - return - - LOG.debug("updating %s with %s", nwsli, msg) - mcursor.execute( - """ - UPDATE stations SET sigstage_low = %(sigstage_low)s, - sigstage_action = %(sigstage_action)s, - sigstage_bankfull = %(sigstage_bankfull)s, - sigstage_flood = %(sigstage_flood)s, - sigstage_moderate = %(sigstage_moderate)s, - sigstage_major = %(sigstage_major)s, - sigstage_record = %(sigstage_record)s - WHERE iemid = %(iemid)s - """, - meta, - ) - - -def main(argv): - """Go Main Go""" - nt = NetworkTable(argv[1]) - with get_dbconn("mesosite") as dbconn: - for sid, meta in nt.sts.items(): - mcursor = dbconn.cursor() - process_site(mcursor, sid, meta) - mcursor.close() - dbconn.commit() - - -if __name__ == "__main__": - main(sys.argv) diff --git a/scripts/hads/process_nwps_stages.py b/scripts/hads/process_nwps_stages.py new file mode 100644 index 0000000000..ee2e326a17 --- /dev/null +++ b/scripts/hads/process_nwps_stages.py @@ -0,0 +1,79 @@ +"""Sync NWPS entries for flood stages! + +Called from windrose/daily_drive_network.py +""" +import sys + +import requests +from pyiem.network import Table as NetworkTable +from pyiem.util import get_dbconn, logger + +LOG = logger() + + +def process_site(mcursor, nwsli, meta): + """Do our processing work""" + + url = f"https://preview-api.water.noaa.gov/v1/gauges/{nwsli}" + + try: + res = requests.get(url, timeout=30).json() + if res.get("code") == 5: + LOG.info("No data found for %s", nwsli) + return + except Exception as exp: + LOG.exception(exp) + return + + msg = "" + for name, entry in res.get("flood", {}).get("categories", {}).items(): + key = f"sigstage_{name}" + val = entry["stage"] + if val < 0: + continue + # is this updated info? + current = meta.get(key) + if current is None or abs(current - val) > 0.01: + msg += f"{name}: {current}->{val} " + meta[key] = val + if msg == "": + return + + LOG.info("updating %s with %s", nwsli, msg) + mcursor.execute( + """ + UPDATE stations SET sigstage_low = %s, + sigstage_action = %s, + sigstage_bankfull = %s, + sigstage_flood = %s, + sigstage_moderate = %s, + sigstage_major = %s, + sigstage_record = %s + WHERE iemid = %s + """, + ( + meta.get("sigstage_low"), + meta.get("sigstage_action"), + meta.get("sigstage_bankfull"), + meta.get("sigstage_flood"), + meta.get("sigstage_moderate"), + meta.get("sigstage_major"), + meta.get("sigstage_record"), + meta["iemid"], + ), + ) + + +def main(argv): + """Go Main Go""" + nt = NetworkTable(argv[1]) + with get_dbconn("mesosite") as dbconn: + for sid, meta in nt.sts.items(): + mcursor = dbconn.cursor() + process_site(mcursor, sid, meta) + mcursor.close() + dbconn.commit() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/windrose/daily_drive_network.py b/scripts/windrose/daily_drive_network.py index ee53284b7a..0b4f9455d2 100644 --- a/scripts/windrose/daily_drive_network.py +++ b/scripts/windrose/daily_drive_network.py @@ -22,7 +22,7 @@ def do_network(network): # Special Logic to compute archive period. subprocess.call(["python", "../dbutil/compute_hads_sts.py", network]) # Update stage details. - subprocess.call(["python", "../hads/process_ahps_xml.py", network]) + subprocess.call(["python", "../hads/process_nwps_stages.py", network]) elif network.find("_RWIS") > 0: subprocess.call( ["python", "../dbutil/compute_alldata_sts.py", "rwis", network],