mnt: sundry updates per log review

cgi-bin/request/gis/watch_by_county.py

@@ -5,15 +5,15 @@
 from io import BytesIO
 
 from osgeo import ogr
+from pyiem.exceptions import IncompleteWebRequest
 from pyiem.webutil import iemapp
 
 ogr.UseExceptions()
 PROJFILE = "/opt/iem/data/gis/meta/4326.prj"
 
 
-@iemapp()
-def application(environ, start_response):
-    """Go Main Go"""
+def get_ts_fn(environ):
+    """Figure out what is requested."""
     # Get CGI vars
     if "year" in environ:
         year = int(environ.get("year"))
@@ -26,7 +26,18 @@ def application(environ, start_response):
     else:
         ts = datetime.datetime.utcnow()
         fn = "watch_by_county"
+    if "etn" in environ:
+        int(environ.get("etn"))
+    return ts, fn
 
+
+@iemapp()
+def application(environ, start_response):
+    """Go Main Go"""
+    try:
+        ts, fn = get_ts_fn(environ)
+    except Exception:
+        raise IncompleteWebRequest("bad input provided")
     if "etn" in environ:
         etnLimiter = f"and eventid = {int(environ.get('etn'))}"
         fn = f"watch_by_county_{ts:Y%m%d%H%M}_{int(environ.get('etn'))}"

htdocs/GIS/tiff/index.py

@@ -7,6 +7,7 @@
 from calendar import month_abbr
 from datetime import datetime, timedelta
 
+from pyiem.exceptions import IncompleteWebRequest
 from pyiem.htmlgen import make_select
 from pyiem.templates.iem import TEMPLATE
 from pyiem.util import utc
@@ -85,7 +86,10 @@ def workflow(key, tmpdir, ts):
             raise FileNotFoundError("Failed to find recent file for service")
 
     else:
-        valid = datetime.strptime(ts, "%Y%m%d%H%M")
+        try:
+            valid = datetime.strptime(ts, "%Y%m%d%H%M")
+        except Exception:
+            raise IncompleteWebRequest("Invalid ts provided")
         testfn = (
             f"/mesonet/ARCHIVE/data/{valid:%Y/%m/%d}/model/ffg/5kmffg_"
             f"{valid:%Y%m%d%H}.grib2"

htdocs/iemre/hourly.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 from pyiem import iemre
+from pyiem.exceptions import BadWebRequest
 from pyiem.util import convert_value, ncopen, utc
 from pyiem.webutil import iemapp
 from pymemcache.client import Client
@@ -94,7 +95,10 @@ def workflow(sts, ets, i, j):
 @iemapp()
 def application(environ, start_response):
     """Do Something Fun!"""
-    sts, ets = get_timerange(environ)
+    try:
+        sts, ets = get_timerange(environ)
+    except ValueError:
+        raise BadWebRequest("Invalid date provided")
     lat = float(environ.get("lat", 41.99))
     lon = float(environ.get("lon", -95.1))
     # fmt = form.get("format", "json")

htdocs/plotting/auto/index.py

@@ -15,6 +15,7 @@
 import requests
 from paste.request import get_cookie_dict
 from pyiem.database import get_dbconnc, get_sqlalchemy_conn
+from pyiem.exceptions import BadWebRequest
 from pyiem.htmlgen import make_select, station_select
 from pyiem.nws.vtec import VTEC_PHENOMENA, VTEC_SIGNIFICANCE
 from pyiem.reference import SECTORS_NAME, state_names
@@ -533,7 +534,10 @@ def generate_form(apid, fdict, headers, cookies):
                 f'value="{value}">'
             )
         elif arg["type"] in ["month", "zhour", "hour", "day", "year"]:
-            form = datetypes_handler(arg, int(value))
+            try:
+                form = datetypes_handler(arg, int(value))
+            except ValueError:
+                raise BadWebRequest("Invalid value provided")
         elif arg["type"] == "select":
             form = make_select(
                 arg["name"],

htdocs/plotting/auto/scripts/p37.py

@@ -20,9 +20,10 @@
 import numpy as np
 import pandas as pd
 from matplotlib.ticker import AutoMinorLocator, MaxNLocator
+from pyiem.database import get_dbconn
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
-from pyiem.util import get_autoplot_context, get_dbconn
+from pyiem.util import get_autoplot_context
 
 PDICT = {
     "NAM": "NAM (9 Dec 2008 - current)",

scripts/coop/cfs_extract.py

@@ -7,66 +7,67 @@
 Run at 5 AM local from RUN_10_AFTER.sh
 """
 import datetime
-import os
 
 import numpy as np
 import pygrib
+from pyiem.database import get_dbconn
 from pyiem.network import Table as NetworkTable
-from pyiem.util import convert_value, get_dbconn, logger, utc
+from pyiem.util import archive_fetch, convert_value, logger, utc
 
 LOG = logger()
 
 
 def do_agg(dkey, fname, ts, data):
     """Do aggregate"""
-    fn = ts.strftime(
-        f"/mesonet/ARCHIVE/data/%Y/%m/%d/model/cfs/%H/{fname}.01."
-        "%Y%m%d%H.daily.grib2"
+    ppath = ts.strftime(
+        f"%Y/%m/%d/model/cfs/%H/{fname}.01.%Y%m%d%H.daily.grib2"
     )
-    if not os.path.isfile(fn):
-        LOG.info("missing %s", fn)
-        return
-    # Precip
-    gribs = pygrib.open(fn)
-    for grib in gribs:
-        if data["x"] is None:
-            lat, lon = grib.latlons()
-            data["y"] = lat[:, 0]
-            data["x"] = lon[0, :]
-        ftime = ts + datetime.timedelta(hours=grib.forecastTime)
-        cst = ftime - datetime.timedelta(hours=7)
-        key = cst.strftime("%Y-%m-%d")
-        d = data["fx"].setdefault(
-            key, dict(precip=None, high=None, low=None, srad=None)
-        )
-        LOG.info("Writting %s %s from ftime: %s", dkey, key, ftime)
-        if d[dkey] is None:
-            d[dkey] = grib.values * 6 * 3600.0
-        else:
-            d[dkey] += grib.values * 6 * 3600.0
+    with archive_fetch(ppath) as fn:
+        if fn is None:
+            LOG.info("missing %s", ppath)
+            return
+        # Precip
+        gribs = pygrib.open(fn)
+        for grib in gribs:
+            if data["x"] is None:
+                lat, lon = grib.latlons()
+                data["y"] = lat[:, 0]
+                data["x"] = lon[0, :]
+            ftime = ts + datetime.timedelta(hours=grib.forecastTime)
+            cst = ftime - datetime.timedelta(hours=7)
+            key = cst.strftime("%Y-%m-%d")
+            d = data["fx"].setdefault(
+                key, dict(precip=None, high=None, low=None, srad=None)
+            )
+            LOG.info("Writting %s %s from ftime: %s", dkey, key, ftime)
+            if d[dkey] is None:
+                d[dkey] = grib.values * 6 * 3600.0
+            else:
+                d[dkey] += grib.values * 6 * 3600.0
 
 
 def do_temp(dkey, fname, func, ts, data):
     """Do Temperatures"""
-    fn = ts.strftime(
-        f"/mesonet/ARCHIVE/data/%Y/%m/%d/model/cfs/%H/{fname}"
-        ".01.%Y%m%d%H.daily.grib2"
+    ppath = ts.strftime(
+        f"%Y/%m/%d/model/cfs/%H/{fname}.01.%Y%m%d%H.daily.grib2"
     )
-    if not os.path.isfile(fn):
-        return
-    gribs = pygrib.open(fn)
-    for grib in gribs:
-        ftime = ts + datetime.timedelta(hours=grib.forecastTime)
-        cst = ftime - datetime.timedelta(hours=7)
-        key = cst.strftime("%Y-%m-%d")
-        if key not in data["fx"]:
-            continue
-        d = data["fx"][key]
-        LOG.info("Writting %s %s from ftime: %s", dkey, key, ftime)
-        if d[dkey] is None:
-            d[dkey] = grib.values
-        else:
-            d[dkey] = func(d[dkey], grib.values)
+    with archive_fetch(ppath) as fn:
+        if fn is None:
+            LOG.info("missing %s", ppath)
+            return
+        gribs = pygrib.open(fn)
+        for grib in gribs:
+            ftime = ts + datetime.timedelta(hours=grib.forecastTime)
+            cst = ftime - datetime.timedelta(hours=7)
+            key = cst.strftime("%Y-%m-%d")
+            if key not in data["fx"]:
+                continue
+            d = data["fx"][key]
+            LOG.info("Writting %s %s from ftime: %s", dkey, key, ftime)
+            if d[dkey] is None:
+                d[dkey] = grib.values
+            else:
+                d[dkey] = func(d[dkey], grib.values)
 
 
 def process(ts):

scripts/coop/ndfd_extract.py

@@ -10,14 +10,14 @@
 Run from RUN_10_AFTER.sh at 1z
 """
 import datetime
-import os
-import sys
 
+import click
 import numpy as np
 import pygrib
 import pyproj
+from pyiem.database import get_dbconn
 from pyiem.network import Table as NetworkTable
-from pyiem.util import convert_value, get_dbconn, logger, utc
+from pyiem.util import archive_fetch, convert_value, logger, utc
 
 LOG = logger()
 nt = NetworkTable("IACLIMATE")
@@ -76,18 +76,19 @@ def process(ts):
         if fhour > 72 and fhour % 6 != 0:
             continue
         ftime = ts + datetime.timedelta(hours=fhour)
-        fn = (
-            f"/mesonet/ARCHIVE/data/{ts:%Y/%m/%d}/model/ndfd/{ts:%H}/"
+        ppath = (
+            f"{ts:%Y/%m/%d}/model/ndfd/{ts:%H}/"
             f"ndfd.t{ts:%H}z.awp2p5f{fhour:03.0f}.grib2"
         )
-        if not os.path.isfile(fn):
-            LOG.warning("missing: %s", fn)
-            continue
-        LOG.info("-> %s", fn)
-        gribs = pygrib.open(fn)
-        do_precip(gribs, ftime, data)
-        do_temp("Maximum temperature", "high", gribs, ftime, data)
-        do_temp("Minimum temperature", "low", gribs, ftime, data)
+        with archive_fetch(ppath) as fn:
+            if fn is None:
+                LOG.warning("missing: %s", ppath)
+                continue
+            LOG.info("-> %s", fn)
+            gribs = pygrib.open(fn)
+            do_precip(gribs, ftime, data)
+            do_temp("Maximum temperature", "high", gribs, ftime, data)
+            do_temp("Minimum temperature", "low", gribs, ftime, data)
 
     return data
 
@@ -159,11 +160,13 @@ def dbsave(ts, data):
         pgconn.commit()
 
 
-def main(argv):
+@click.command()
+@click.option("--date", "ts", help="Date to process")
+def main(ts):
     """Go!"""
     # Extract 00 UTC Data
-    if len(argv) == 4:
-        ts = utc(int(argv[1]), int(argv[2]), int(argv[3]))
+    if ts is not None:
+        ts = ts.replace(tzinfo=datetime.timezone.utc)
     else:
         ts = utc().replace(hour=0, minute=0, second=0, microsecond=0)
     data = process(ts)
@@ -174,4 +177,4 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main()

scripts/current/stage4_hourly.py

@@ -2,13 +2,12 @@
     Plot the hourly stage IV precip data
 """
 import datetime
-import os
-import sys
 from zoneinfo import ZoneInfo
 
+import click
 import pygrib
 from pyiem.plot import MapPlot, get_cmap
-from pyiem.util import logger, mm2inch, utc
+from pyiem.util import archive_fetch, logger, mm2inch, utc
 
 LOG = logger()
 
@@ -22,22 +21,19 @@ def doit(ts):
     if ((gmtnow - ts).days * 86400.0 + (gmtnow - ts).seconds) < 7200:
         routes = "ac"
 
-    fn = "/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" % (
-        ts.strftime("%Y/%m/%d"),
-        ts.strftime("%Y%m%d%H"),
-    )
-    if not os.path.isfile(fn):
-        LOG.info("Missing stage4 %s", fn)
-        return
-
-    try:
-        grbs = pygrib.open(fn)
-        grib = grbs[1]
-    except Exception:
-        LOG.warning("Read %s failure", fn)
-        return
-    lats, lons = grib.latlons()
-    vals = mm2inch(grib.values)
+    ppath = f"{ts:%Y/%m/%d}/stage4/ST4.{ts:%Y%m%d%H}.01h.grib"
+    with archive_fetch(ppath) as fn:
+        if fn is None:
+            LOG.info("Missing stage4 %s", ppath)
+            return
+        try:
+            grbs = pygrib.open(fn)
+            grib = grbs[1]
+        except Exception:
+            LOG.warning("Read %s failure", fn)
+            return
+        lats, lons = grib.latlons()
+        vals = mm2inch(grib.values)
 
     cmap = get_cmap("jet")
     cmap.set_under("white")
@@ -82,10 +78,12 @@ def doit(ts):
         mp.close()
 
 
-def main(argv):
+@click.command()
+@click.option("--valid", "ts", type=click.DateTime(), help="UTC Timestamp")
+def main(ts):
     """Go main Go"""
-    if len(argv) == 5:
-        ts = utc(int(argv[1]), int(argv[2]), int(argv[3]), int(argv[4]))
+    if ts is not None:
+        ts = ts.replace(tzinfo=ZoneInfo("UTC"))
         doit(ts)
     else:
         ts = utc()
@@ -95,4 +93,4 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main()