Merge pull request #591 from akrherz/231119

Omnibus

scripts/hads/process_nwps_stages.py

@@ -0,0 +1,79 @@
+"""Sync NWPS entries for flood stages!
+
+Called from windrose/daily_drive_network.py
+"""
+import sys
+
+import requests
+from pyiem.network import Table as NetworkTable
+from pyiem.util import get_dbconn, logger
+
+LOG = logger()
+
+
+def process_site(mcursor, nwsli, meta):
+    """Do our processing work"""
+
+    url = f"https://preview-api.water.noaa.gov/v1/gauges/{nwsli}"
+
+    try:
+        res = requests.get(url, timeout=30).json()
+        if res.get("code") == 5:
+            LOG.info("No data found for %s", nwsli)
+            return
+    except Exception as exp:
+        LOG.exception(exp)
+        return
+
+    msg = ""
+    for name, entry in res.get("flood", {}).get("categories", {}).items():
+        key = f"sigstage_{name}"
+        val = entry["stage"]
+        if val < 0:
+            continue
+        # is this updated info?
+        current = meta.get(key)
+        if current is None or abs(current - val) > 0.01:
+            msg += f"{name}: {current}->{val} "
+        meta[key] = val
+    if msg == "":
+        return
+
+    LOG.info("updating %s with %s", nwsli, msg)
+    mcursor.execute(
+        """
+        UPDATE stations SET sigstage_low = %s,
+        sigstage_action = %s,
+        sigstage_bankfull = %s,
+        sigstage_flood = %s,
+        sigstage_moderate = %s,
+        sigstage_major = %s,
+        sigstage_record = %s
+        WHERE iemid = %s
+    """,
+        (
+            meta.get("sigstage_low"),
+            meta.get("sigstage_action"),
+            meta.get("sigstage_bankfull"),
+            meta.get("sigstage_flood"),
+            meta.get("sigstage_moderate"),
+            meta.get("sigstage_major"),
+            meta.get("sigstage_record"),
+            meta["iemid"],
+        ),
+    )
+
+
+def main(argv):
+    """Go Main Go"""
+    nt = NetworkTable(argv[1])
+    with get_dbconn("mesosite") as dbconn:
+        for sid, meta in nt.sts.items():
+            mcursor = dbconn.cursor()
+            process_site(mcursor, sid, meta)
+            mcursor.close()
+            dbconn.commit()
+
+
+if __name__ == "__main__":
+    main(sys.argv)

scripts/iemre/db_to_netcdf.py

@@ -8,11 +8,14 @@
 """
 import datetime
 import sys
+import warnings
 
 import numpy as np
 from pyiem import iemre
 from pyiem.util import logger, ncopen, utc
 
+# We are going from float64 to uint16, so this appears to be unavoidable
+warnings.simplefilter("ignore", RuntimeWarning)
 LOG = logger()
 
 
@@ -32,7 +35,7 @@ def main(argv):
             if vname not in nc.variables:
                 continue
             # Careful here, ds could contain NaN values
-            nc.variables[vname][idx, :, :] = np.ma.array(
+            nc.variables[vname][idx] = np.ma.array(
                 ds[vname].values, mask=np.isnan(ds[vname].values)
             )
 

scripts/iemre/init_daily.py

@@ -178,9 +178,11 @@ def init_year(ts):
     v2.coordinates = "lon lat"
     v2.description = "Daily averaged wind speed magnitude"
 
+    # 0 to 65535 so 0 to 65.535
     v1 = nc.createVariable(
-        "power_swdn", float, ("time", "lat", "lon"), fill_value=1.0e20
+        "power_swdn", np.uint16, ("time", "lat", "lon"), fill_value=65535
     )
+    v1.scale_factor = 0.001
     v1.units = "MJ d-1"
     v1.long_name = "All Sky Insolation Incident on a Horizontal Surface"
     v1.standard_name = "All Sky Insolation Incident on a Horizontal Surface"

scripts/windrose/daily_drive_network.py

@@ -22,7 +22,7 @@ def do_network(network):
         # Special Logic to compute archive period.
         subprocess.call(["python", "../dbutil/compute_hads_sts.py", network])
         # Update stage details.
-        subprocess.call(["python", "../hads/process_ahps_xml.py", network])
+        subprocess.call(["python", "../hads/process_nwps_stages.py", network])
     elif network.find("_RWIS") > 0:
         subprocess.call(
             ["python", "../dbutil/compute_alldata_sts.py", "rwis", network],