Merge pull request #673 from akrherz/240209-2

mnt: sundry updates per review

htdocs/geojson/sbw.py

@@ -77,6 +77,8 @@ def validate_ts(val):
     year = int(val[:4])
     if year < 1986 or year > (utc().year + 1):
         raise ValueError("year invalid")
+    if len(val) == 10:
+        val += "T00:00:00Z"
     # YYYY-mm-ddTHH:MI
     if len(val) == 16:
         val += ":00Z"

htdocs/json/climodat_dd.py

@@ -4,10 +4,11 @@
 
 import numpy as np
 from metpy.units import units
+from pyiem.database import get_dbconn
 from pyiem.exceptions import IncompleteWebRequest
 from pyiem.iemre import find_ij
 from pyiem.meteorology import gdd as calc_gdd
-from pyiem.util import c2f, get_dbconn, ncopen
+from pyiem.util import c2f, ncopen
 from pyiem.webutil import iemapp
 from pymemcache.client import Client
 
@@ -64,16 +65,17 @@ def run(station, sdate, edate, gddbase, gddceil):
         "accum": accum,
     }
     idx, jdx = find_ij(lon, lat)
-    for model in ["gfs", "ndfd"]:
-        with ncopen(f"/mesonet/data/iemre/{model}_current.nc") as nc:
-            highs = c2f(nc.variables["high_tmpk"][:, jdx, idx] - 273.15)
-            lows = c2f(nc.variables["low_tmpk"][:, jdx, idx] - 273.15)
-            taxis = compute_taxis(nc.variables["time"])
-            gdds, total = compute(taxis, highs, lows, gddbase, gddceil)
-            res[model] = gdds
-            res[f"{model}_accum"] = total
-            res[f"{model}_sdate"] = f"{gdds[0]['date']}"
-            res[f"{model}_edate"] = f"{gdds[-1]['date']}"
+    if idx is not None:
+        for model in ["gfs", "ndfd"]:
+            with ncopen(f"/mesonet/data/iemre/{model}_current.nc") as nc:
+                highs = c2f(nc.variables["high_tmpk"][:, jdx, idx] - 273.15)
+                lows = c2f(nc.variables["low_tmpk"][:, jdx, idx] - 273.15)
+                taxis = compute_taxis(nc.variables["time"])
+                gdds, total = compute(taxis, highs, lows, gddbase, gddceil)
+                res[model] = gdds
+                res[f"{model}_accum"] = total
+                res[f"{model}_sdate"] = f"{gdds[0]['date']}"
+                res[f"{model}_edate"] = f"{gdds[-1]['date']}"
 
     return json.dumps(res)
 

htdocs/json/dcp_vars.php

@@ -17,7 +17,7 @@
     "SELECT distinct key from $table WHERE station = $1"
 );
 
-$station = isset($_REQUEST["station"]) ? strtoupper($_REQUEST["station"]) :
+$station = isset($_REQUEST["station"]) ? strtoupper(xssafe($_REQUEST["station"])) :
     die(json_encode('Please provide a station variable (NWSLI)'));
 
 $rs = pg_execute($hads, "SELECT", array($station));

htdocs/json/network.php

@@ -8,7 +8,7 @@
 require_once "../../include/network.php";
 require_once "../../include/forms.php";
 
-$network = isset($_REQUEST["network"]) ? $_REQUEST["network"] : "KCCI";
+$network = isset($_REQUEST["network"]) ? xssafe($_REQUEST["network"]) : "KCCI";
 $nt = new NetworkTable($network);
 
 $ar = array("stations" => array());

htdocs/json/nwstext_center_date.py

@@ -7,15 +7,13 @@
 
 # extras
 from pyiem.exceptions import IncompleteWebRequest
-from pyiem.util import get_dbconn, html_escape, utc
+from pyiem.util import html_escape, utc
 from pyiem.webutil import iemapp
 
 
-@iemapp(default_tz="UTC")
+@iemapp(default_tz="UTC", iemdb="afos")
 def application(environ, start_response):
     """Answer request."""
-    pgconn = get_dbconn("afos")
-    acursor = pgconn.cursor()
     center = environ.get("center", "KOKX")[:4]
     cb = environ.get("callback")
     if environ.get("date") is not None:
@@ -38,15 +36,15 @@ def application(environ, start_response):
             'SRF', 'SQW', 'SVR', 'SVS', 'TCV', 'TOR', 'TSU', 'WCN', 'WSW')
         """
 
-    acursor.execute(
+    environ["iemdb.afos.cursor"].execute(
         "SELECT data, to_char(entered at time zone 'UTC', "
-        "'YYYY-MM-DDThh24:MI:00Z') from products "
+        "'YYYY-MM-DDThh24:MI:00Z') as ee from products "
         "where source = %s and entered >= %s and "
         f"entered < %s {pil_limiter} ORDER by entered ASC",
         (center, environ["sts"], environ["ets"]),
     )
-    for row in acursor:
-        root["products"].append({"data": row[0], "entered": row[1]})
+    for row in environ["iemdb.afos.cursor"]:
+        root["products"].append({"data": row["data"], "entered": row["ee"]})
 
     data = json.dumps(root)
     if cb is not None:

htdocs/json/stage4.py

@@ -32,7 +32,7 @@ def dowork(environ):
     sidx = iemre.hourly_offset(sts)
     eidx = iemre.hourly_offset(ets)
 
-    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year,)
+    ncfn = f"/mesonet/data/stage4/{date.year}_stage4_hourly.nc"
     res = {"gridi": -1, "gridj": -1, "data": []}
     if not os.path.isfile(ncfn):
         return json.dumps(res)
@@ -69,7 +69,7 @@ def application(environ, start_response):
     valid = environ.get("valid")
     cb = environ.get("callback", None)
 
-    mckey = "/json/stage4/%.2f/%.2f/%s?callback=%s" % (lon, lat, valid, cb)
+    mckey = f"/json/stage4/{lon:.2f}/{lat:.2f}/{valid}?callback={cb}"
     mc = Client("iem-memcached:11211")
     res = mc.get(mckey)
     if not res:
@@ -80,7 +80,7 @@ def application(environ, start_response):
     mc.close()
 
     if cb is not None:
-        res = "%s(%s)" % (html_escape(cb), res)
+        res = f"{html_escape(cb)}({res})"
 
     headers = [("Content-type", "application/json")]
     start_response("200 OK", headers)

htdocs/json/state_ugc.php

@@ -16,7 +16,7 @@
         "ORDER by name ASC"
 );
 
-$st = isset($_REQUEST["state"]) ? $_REQUEST["state"] : 'IA';
+$st = isset($_REQUEST["state"]) ? xssafe($_REQUEST["state"]) : 'IA';
 
 $rs = pg_execute($dbconn, "SELECT", array($st));
 

htdocs/json/webcams.php

@@ -8,8 +8,8 @@
 require_once "../../include/forms.php";
 
 // This should be a UTC timestamp, gasp!
-$ts = isset($_REQUEST["ts"]) ? strtotime($_REQUEST["ts"]) : 0;
-$network = isset($_REQUEST["network"]) ? substr($_REQUEST["network"], 0, 4) : "KCCI";
+$ts = isset($_REQUEST["ts"]) ? strtotime(xssafe($_REQUEST["ts"])) : 0;
+$network = isset($_REQUEST["network"]) ? substr(xssafe($_REQUEST["network"]), 0, 4) : "KCCI";
 
 $connect = iemdb("mesosite");
 pg_exec($connect, "SET TIME ZONE 'UTC'");

htdocs/plotting/auto/scripts100/p102.py

@@ -190,4 +190,4 @@ def plotter(fdict):
 
 
 if __name__ == "__main__":
-    plotter(dict(ltype=["SNOW"], station="PBZ"))
+    plotter({})

htdocs/plotting/auto/scripts100/p162.py

@@ -9,9 +9,10 @@
 import numpy as np
 import numpy.ma as ma
 import pandas as pd
+from pyiem.database import get_sqlalchemy_conn
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes, get_cmap
-from pyiem.util import get_autoplot_context, get_sqlalchemy_conn
+from pyiem.util import get_autoplot_context
 
 
 def get_description():

htdocs/plotting/auto/scripts100/p166.py

@@ -1,4 +1,8 @@
-"""Watches"""
+"""
+This plot presents a summary of the number of year
+to date watches issued by the Storm Prediction Center and the percentage
+of those watches that at least touched the given state.
+"""
 import datetime
 
 import matplotlib.ticker as ticker
@@ -16,12 +20,7 @@
 
 def get_description():
     """Return a dict describing how to call this plotter"""
-    desc = {}
-    desc["data"] = True
-    desc["description"] = """This plot presents a summary of the number of year
-    to date watches issued by the Storm Prediction Center and the percentage
-    of those watches that at least touched the given state.
-    """
+    desc = {"data": True, "description": __doc__}
     desc["arguments"] = [
         dict(type="state", name="state", default="IA", label="Select State:"),
         dict(

htdocs/plotting/auto/scripts100/p168.py

@@ -1,4 +1,13 @@
-"""First fall or last spring"""
+"""
+This chart presents the last date of fall or first
+date of spring that a given temperature threshold was last or first
+reached.  Note that leap day creates some ambiguity with an analysis like
+this, so for example, the 15th of November is considered equal for each
+year.  The plot truncates once you reach the 20th of December.  If you use
+the downloaded file, please note that you need to consider the levels
+above the given threshold as the latest date.  The downloaded file simply
+provides the latest date at a given temperature.
+"""
 import calendar
 import datetime
 
@@ -11,17 +20,7 @@
 
 def get_description():
     """Return a dict describing how to call this plotter"""
-    desc = {}
-    desc["data"] = True
-    desc["description"] = """This chart presents the last date of fall or first
-    date of spring that a given temperature threshold was last or first
-    reached.  Note that leap day creates some ambiguity with an analysis like
-    this, so for example, the 15th of November is considered equal for each
-    year.  The plot truncates once you reach the 20th of December.  If you use
-    the downloaded file, please note that you need to consider the levels
-    above the given threshold as the latest date.  The downloaded file simply
-    provides the latest date at a given temperature.
-    """
+    desc = {"data": True, "description": __doc__}
     desc["arguments"] = [
         dict(
             type="station",
@@ -106,4 +105,4 @@ def plotter(fdict):
 
 
 if __name__ == "__main__":
-    plotter(dict(station="IA7594", network="IACLIMATE"))
+    plotter({})

htdocs/plotting/auto/scripts100/p170.py

@@ -24,9 +24,10 @@
 import datetime
 
 import pandas as pd
+from pyiem.database import get_sqlalchemy_conn
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
-from pyiem.util import get_autoplot_context, get_sqlalchemy_conn
+from pyiem.util import get_autoplot_context
 
 PDICT = {
     "TS": "All Thunder Reports (TS)",
@@ -173,4 +174,4 @@ def plotter(fdict):
 
 
 if __name__ == "__main__":
-    plotter(dict(zstation="ALO", year=2017, var="TSFZRA", network="IA_ASOS"))
+    plotter({})

htdocs/plotting/auto/scripts100/p173.py

@@ -1,4 +1,13 @@
-"""autoplot ya'll"""
+"""
+This chart presents the hourly average wind speeds
+by month of the year or by custom periods.
+The hours presented are valid in the local time zone
+of the reporting station.  For example in Iowa, 3 PM would represent
+3 PM CDT in the summer and 3 PM CST in the winter.  Please complain to us
+if this logic causes you heartburn!  The format of the date periods is
+two digit month followed by two digit day for both the start and end
+date.
+"""
 import calendar
 import datetime
 
@@ -13,17 +22,7 @@
 
 def get_description():
     """Return a dict describing how to call this plotter"""
-    desc = {}
-    desc["data"] = True
-    desc["cache"] = 86400
-    desc["description"] = """This chart presents the hourly average wind speeds
-    by month of the year or by custom periods.
-    The hours presented are valid in the local time zone
-    of the reporting station.  For example in Iowa, 3 PM would represent
-    3 PM CDT in the summer and 3 PM CST in the winter.  Please complain to us
-    if this logic causes you heartburn!  The format of the date periods is
-    two digit month followed by two digit day for both the start and end
-    date."""
+    desc = {"description": __doc__, "data": True, "cache": 86400}
     today = datetime.date.today()
     desc["arguments"] = [
         dict(

htdocs/plotting/auto/scripts100/p175.py

@@ -1,4 +1,9 @@
-"""snow cover coverage."""
+"""
+This chart displays estimated areal coverage of
+snow cover for a single state.  This estimate is based on a 0.125x0.125
+degree analysis of NWS COOP observations.  The date shown would represent
+snow depth reported approximately at 7 AM.
+"""
 import datetime
 import os
 
@@ -7,28 +12,18 @@
 import numpy as np
 import pandas as pd
 from pyiem import iemre, reference
+from pyiem.database import get_sqlalchemy_conn
 from pyiem.exceptions import NoDataFound
 from pyiem.grid.zs import CachingZonalStats
 from pyiem.plot import figure_axes
-from pyiem.util import (
-    convert_value,
-    get_autoplot_context,
-    get_sqlalchemy_conn,
-    ncopen,
-)
+from pyiem.util import convert_value, get_autoplot_context, ncopen
 
 
 def get_description():
     """Return a dict describing how to call this plotter"""
-    desc = {}
+    desc = {"description": __doc__, "data": True, "cache": 86400}
     today = datetime.date.today()
     year = today.year if today.month > 9 else today.year - 1
-    desc["description"] = """This chart displays estimated areal coverage of
-    snow cover for a single state.  This estimate is based on a 0.125x0.125
-    degree analysis of NWS COOP observations.  The date shown would represent
-    snow depth reported approximately at 7 AM.
-    """
-    desc["data"] = True
     desc["arguments"] = [
         dict(
             type="year",

htdocs/plotting/auto/scripts100/p180.py

@@ -1,17 +1,17 @@
-"""This application plots daily climatology for
-    a location or two of your choice.  You can pick which climatology to use
-    and effectively build a difference plot when picking the same station,
-    but using a different climatology."""
-# stdlib
+"""
+This application plots daily climatology for
+a location or two of your choice.  You can pick which climatology to use
+and effectively build a difference plot when picking the same station,
+but using a different climatology.
+"""
 import calendar
 
 import matplotlib.dates as mdates
-
-# third party
 import pandas as pd
+from pyiem.database import get_sqlalchemy_conn
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure, fitbox
-from pyiem.util import get_autoplot_context, get_sqlalchemy_conn
+from pyiem.util import get_autoplot_context
 
 PDICT = {
     "por": "Period of Record (por) Climatology",
@@ -347,19 +347,4 @@ def plotter(fdict):
 
 
 if __name__ == "__main__":
-    plotter(
-        dict(
-            v="snow",
-            network1="IACLIMATE",
-            station1="IA0112",
-            c1="1951",
-            network2="IACLIMATE",
-            station2="IA0112",
-            c2="ncei81",
-            s="0",
-            sy1=1981,
-            ey1=2010,
-            sy2=1991,
-            ey2=2020,
-        )
-    )
+    plotter({})

htdocs/plotting/auto/scripts100/p187.py

@@ -1,13 +1,15 @@
-"""This chart presents the rank a station's yearly
-    summary value has against an unweighted population of available
-    observations in the state.  The green line is a simple average of the
-    plot.
+"""
+This chart presents the rank a station's yearly
+summary value has against an unweighted population of available
+observations in the state.  The green line is a simple average of the
+plot.
 """
 
 import pandas as pd
+from pyiem.database import get_sqlalchemy_conn
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
-from pyiem.util import get_autoplot_context, get_sqlalchemy_conn
+from pyiem.util import get_autoplot_context
 
 PDICT = {
     "precip": "Total Precipitation",