Merge pull request #588 from akrherz/231115

Omnibus

htdocs/plotting/auto/scripts200/p238.py

@@ -5,14 +5,19 @@
 
 import pandas as pd
 from pyiem.exceptions import NoDataFound
-from pyiem.plot import MapPlot
+from pyiem.plot import MapPlot, centered_bins, pretty_bins
 from pyiem.util import get_autoplot_context, get_sqlalchemy_conn, utc
 from sqlalchemy import text
 
 PDICT = {
     "corn": "Corn Grain",
+    "soybeans": "Soybeans",
 }
 PDICT2 = {"yes": "Label counties with values", "no": "Don't show values "}
+PDICT3 = {
+    "departure": "Yield Departure from previous 10 year average",
+    "value": "Yield",
+}
 
 
 def get_description():
@@ -32,6 +37,20 @@ def get_description():
             "default": "IA",
             "label": "Select state/sector",
         },
+        {
+            "type": "select",
+            "options": PDICT,
+            "default": "corn",
+            "name": "crop",
+            "label": "Select Crop",
+        },
+        {
+            "type": "select",
+            "options": PDICT3,
+            "default": "departure",
+            "name": "var",
+            "label": "Select variable to plot",
+        },
         {
             "type": "select",
             "label": "Label Values?",
@@ -48,6 +67,10 @@ def plotter(fdict):
     """Go"""
     ctx = get_autoplot_context(fdict, get_description())
     year1 = max(ctx["year"] - 10, 1981)
+    params = {"y1": year1, "year": ctx["year"], "crop": ctx["crop"].upper()}
+    params["util"] = "GRAIN"
+    if ctx["crop"] == "soybeans":
+        params["util"] = "ALL UTILIZATION PRACTICES"
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
             text(
@@ -56,8 +79,8 @@ def plotter(fdict):
                 select year, state_alpha || 'C' || county_ansi as ugc,
                 avg(num_value) as num_value from
                 nass_quickstats where county_ansi is not null and
-                statisticcat_desc = 'YIELD' and commodity_desc = 'CORN' and
-                util_practice_desc = 'GRAIN' and year >= :y1 and year <= :year
+                statisticcat_desc = 'YIELD' and commodity_desc = :crop and
+                util_practice_desc = :util and year >= :y1 and year <= :year
                 GROUP by year, ugc),
             agg as (
                 select ugc, count(*), avg(num_value) from data
@@ -70,23 +93,27 @@ def plotter(fdict):
             """
             ),
             conn,
-            params={"y1": year1, "year": ctx["year"]},
+            params=params,
             index_col="ugc",
         )
     if df.empty:
         raise NoDataFound("Could not find any data, sorry.")
+    title = "Yield"
+    col = "num_value"
+    bins = pretty_bins(df[col].min(), df[col].max())
+    if ctx["var"] == "departure":
+        title = f"Yield Departure from {year1}-{ctx['year'] - 1} Average"
+        col = "delta"
+        bins = centered_bins(max(df[col].abs().max(), 50))
     mp = MapPlot(
         apctx=ctx,
-        title=(
-            f"USDA NASS {ctx['year']} Corn Yield Deparature from "
-            f"{year1}-{ctx['year'] - 1} Average"
-        ),
+        title=f"USDA NASS {ctx['crop'].capitalize()} {ctx['year']} {title}",
         stateborderwidth=3,
         nocaption=True,
     )
     mp.fill_ugcs(
-        df["delta"].to_dict(),
-        bins=range(-50, 51, 10),
+        df[col].to_dict(),
+        bins=bins,
         cmap=ctx["cmap"],
         units="bu/ac",
         ilabel=ctx["ilabel"] == "yes",

htdocs/sites/obhistory.php

@@ -157,6 +157,35 @@ function hads_formatter($i, $row, $shefcols)
         $html
     );
 }
+function scan_formatter($i, $row){
+    $ts = strtotime(substr($row["local_valid"], 0, 16));
+    return sprintf(
+        "<tr style=\"background: %s;\">" .
+            "<td>%s</td><td>%s</td><td>%s</td>".
+    "<td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>".
+    "<td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>".
+    "<td>%s</td><td>%s</td></tr>",
+        ($i % 2 == 0) ? "#FFF" : "#EEE",
+        date("g:i A", $ts),
+        wind_formatter($row),
+        temp_formatter($row["tmpf"]),
+        temp_formatter($row["dwpf"]),
+        $row["relh"],
+        $row["srad"],
+        precip_formatter($row["phour"]),
+        $row["soilt2"],
+        $row["soilm2"],
+        $row["soilt4"],
+        $row["soilm4"],
+        $row["soilt8"],
+        $row["soilm8"],
+        $row["soilt20"],
+        $row["soilm20"],
+        $row["soilt40"],
+        $row["soilm40"],
+    );
+
+}
 $year = get_int404("year", date("Y"));
 $month = get_int404("month", date("m"));
 $day = get_int404("day", date("d"));
@@ -450,6 +479,32 @@ function toggleMADIS(){
     {$shefextra}
     </tr>
 EOM;
+} else if ($network == "SCAN") {
+    $header = <<<EOM
+    <tr align="center" bgcolor="#b0c4de">
+    <th rowspan="2">Time</th>
+    <th rowspan="2">Wind<br>(mph)</th>
+    <th colspan="2">Temp (&deg;F)</th>
+    <th rowspan="2">RH%</td>
+    <th rowspan="2">Solar (W/m2)</th>
+    <th rowspan="2">Precipitation (in.)</th>
+    <th colspan="2">2 Inch</th>
+    <th colspan="2">4 Inch</th>
+    <th colspan="2">8 Inch</th>
+    <th colspan="2">20 Inch</th>
+    <th colspan="2">40 Inch</th>
+    </tr>
+
+    <tr align="center" bgcolor="#b0c4de">
+    <th>Air</th>
+    <th>Dwpt</th>
+    <th>Temp</th><th>VWC</th>
+    <th>Temp</th><th>VWC</th>
+    <th>Temp</th><th>VWC</th>
+    <th>Temp</th><th>VWC</th>
+    <th>Temp</th><th>VWC</th>
+    </tr>
+EOM;
 } else {
     $header = <<<EOM
     <tr align="center" bgcolor="#b0c4de">
@@ -480,6 +535,8 @@ function toggleMADIS(){
         $table .= asos_formatter($i, $row);
     } else if (preg_match("/DCP|COOP/", $network)) {
         $table .= hads_formatter($i, $row, $shefcols);
+    } else if ($network == "SCAN") {
+        $table .= scan_formatter($i, $row);
     } else {
         $table .= formatter($i, $row);
     }

scripts/conus404/init_hourly.py

@@ -13,6 +13,8 @@ def init_year(ts):
     """
     Create a new NetCDF file for a year of our specification!
     """
+    #  wget -O /tmp/wrfconstants_usgs404.nc
+    # https://data.rda.ucar.edu/ds559.0/INVARIANT/wrfconstants_usgs404.nc
     tplnc = ncopen("/tmp/wrfconstants_usgs404.nc")
 
     ncfn = f"/mesonet/data/conus404/{ts.year}_hourly.nc"
@@ -33,8 +35,8 @@ def init_year(ts):
     nc.comment = "No Comment at this time"
 
     # Setup Dimensions
-    nc.createDimension("lat", tplnc.dimensions["south_north"].size)
-    nc.createDimension("lon", tplnc.dimensions["west_east"].size)
+    nc.createDimension("south_north", tplnc.dimensions["south_north"].size)
+    nc.createDimension("west_east", tplnc.dimensions["west_east"].size)
     ts2 = datetime.datetime(ts.year + 1, 1, 1)
     days = (ts2 - ts).days
     LOG.info("Year %s has %s days", ts.year, days)
@@ -48,14 +50,14 @@ def init_year(ts):
     ncv[:] = tplnc.variables["ZS"][:]
 
     # Setup Coordinate Variables
-    lat = nc.createVariable("lat", float, ("lat", "lon"))
+    lat = nc.createVariable("lat", float, ("south_north", "west_east"))
     lat.units = "degrees_north"
     lat.long_name = "Latitude"
     lat.standard_name = "latitude"
     lat.axis = "Y"
     lat[:] = tplnc.variables["XLAT"][:]
 
-    lon = nc.createVariable("lon", float, ("lat", "lon"))
+    lon = nc.createVariable("lon", float, ("south_north", "west_east"))
     lon.units = "degrees_east"
     lon.long_name = "Longitude"
     lon.standard_name = "longitude"
@@ -72,7 +74,10 @@ def init_year(ts):
 
     # 0->65535 `T2`
     tmpk = nc.createVariable(
-        "tmpk", np.uint16, ("time", "lat", "lon"), fill_value=65535
+        "tmpk",
+        np.uint16,
+        ("time", "south_north", "west_east"),
+        fill_value=65535,
     )
     tmpk.units = "K"
     tmpk.scale_factor = 0.01
@@ -82,7 +87,10 @@ def init_year(ts):
 
     # 0->65535  0 to 655.35 `TD2`
     dwpk = nc.createVariable(
-        "dwpk", np.uint16, ("time", "lat", "lon"), fill_value=65335
+        "dwpk",
+        np.uint16,
+        ("time", "south_north", "west_east"),
+        fill_value=65335,
     )
     dwpk.units = "K"
     dwpk.scale_factor = 0.01
@@ -93,7 +101,10 @@ def init_year(ts):
     # NOTE: we need to store negative numbers here, gasp
     # -32768 to 32767 so -98 to 98 mps `U10`
     uwnd = nc.createVariable(
-        "uwnd", np.int16, ("time", "lat", "lon"), fill_value=32767
+        "uwnd",
+        np.int16,
+        ("time", "south_north", "west_east"),
+        fill_value=32767,
     )
     uwnd.scale_factor = 0.003
     uwnd.units = "meters per second"
@@ -104,7 +115,10 @@ def init_year(ts):
     # NOTE: we need to store negative numbers here, gasp
     # -32768 to 32767 so -98 to 98 mps `V10`
     vwnd = nc.createVariable(
-        "vwnd", np.int16, ("time", "lat", "lon"), fill_value=32767
+        "vwnd",
+        np.int16,
+        ("time", "south_north", "west_east"),
+        fill_value=32767,
     )
     vwnd.scale_factor = 0.003
     vwnd.units = "meters per second"
@@ -114,7 +128,10 @@ def init_year(ts):
 
     # 0->65535  0 to 327.675 `PREC_ACC_NC`
     p01m = nc.createVariable(
-        "p01m", np.uint16, ("time", "lat", "lon"), fill_value=65535
+        "p01m",
+        np.uint16,
+        ("time", "south_north", "west_east"),
+        fill_value=65535,
     )
     p01m.units = "mm"
     p01m.scale_factor = 0.005
@@ -126,7 +143,7 @@ def init_year(ts):
     # NOTE: Condensation is + and Evapration is -
     # -128 to 127 for -25 to 25 `ACETLSM`
     ncv = nc.createVariable(
-        "evap", np.int8, ("time", "lat", "lon"), fill_value=127
+        "evap", np.int8, ("time", "south_north", "west_east"), fill_value=127
     )
     ncv.units = "mm"
     ncv.scale_factor = 0.4
@@ -137,7 +154,10 @@ def init_year(ts):
 
     # 0 -> 65535 so 0 to 1966 `ACSWDNLSM`
     ncv = nc.createVariable(
-        "rsds", np.uint16, ("time", "lat", "lon"), fill_value=65535
+        "rsds",
+        np.uint16,
+        ("time", "south_north", "west_east"),
+        fill_value=65535,
     )
     ncv.units = "W m-2"
     ncv.scale_factor = 0.03
@@ -150,7 +170,7 @@ def init_year(ts):
     ncv = nc.createVariable(
         "soilt",
         np.uint8,
-        ("time", "soil_level", "lat", "lon"),
+        ("time", "soil_level", "south_north", "west_east"),
         fill_value=255,
     )
     ncv.units = "K"
@@ -164,7 +184,7 @@ def init_year(ts):
     ncv = nc.createVariable(
         "soilm",
         np.uint8,
-        ("time", "soil_level", "lat", "lon"),
+        ("time", "soil_level", "south_north", "west_east"),
         fill_value=255,
     )
     ncv.units = "m^3 m^-3"

scripts/conus404/merge_hourly.py

@@ -34,7 +34,7 @@ def process(valid):
     lat = nc.variables["lat"][:]
     lon = nc.variables["lon"][:]
     dist = ((lat - 42.03) ** 2 + (lon - -93.65) ** 2) ** 0.5
-    j, i = np.unravel_index(np.argmin(dist), dist.shape)
+    j, i = np.unravel_index(np.argmin(dist), dist.shape)  # (589, 769)
 
     # Radiation is a delta, so we need the previous hour too
     prev = valid - datetime.timedelta(hours=1)