Merge pull request #1049 from akrherz/250101

Omnibus

pylib/iemweb/autoplot/scripts/p5.py

@@ -131,7 +131,11 @@ def plotter(ctx: dict):
             if i == 1:
                 ax.text(d, 1 - 0.1, f"{d}", color="tan", ha="center", va="top")
         ax.plot([1, 31], [i, i], lw=2, color="tan", zorder=3)
-        for _, row in df[df["month"] == i].iterrows():
+        records = df[df["month"] == i]
+        # likely zeros
+        if len(records) > 30:
+            continue
+        for _, row in records.iterrows():
             ax.scatter(
                 row["day"].day,
                 i + 0.06,

scripts/mrms/merge_mrms_q3.py

@@ -3,16 +3,18 @@
 called from RUN_10_AFTER.sh
 """
 
-import datetime
 import gzip
 import os
 import tempfile
+from datetime import date, datetime, timedelta
 from zoneinfo import ZoneInfo
 
 import click
+import netCDF4
 import numpy as np
 import pygrib
-from pyiem import iemre, mrms
+from pyiem.iemre import daily_offset, get_daily_mrms_ncname
+from pyiem.mrms import fetch as mrms_fetch
 from pyiem.util import logger, ncopen, utc
 
 LOG = logger()
@@ -22,40 +24,35 @@
 PRODFLIP = utc(2020, 10, 13, 19)
 
 
-def run(dt: datetime.date, for_dep: bool):
+def run(nc: netCDF4.Dataset, dt: date, for_dep: bool):
     """Update netcdf file with the MRMS data
 
     Args:
-      dt (datetime.date): date to process
+      dt (date): date to process
       for_dep (bool): for Daily Erosion Project
     """
-    ncfn = iemre.get_daily_mrms_ncname(dt.year)
+    offset = daily_offset(dt)
     if for_dep:
-        ncfn = ncfn.replace("daily", "dep")
-    nc = ncopen(ncfn, "a", timeout=300)
-    offset = iemre.daily_offset(dt)
-    if for_dep:
-        offset = offset - iemre.daily_offset(datetime.date(dt.year, 4, 11))
+        offset = offset - daily_offset(date(dt.year, 4, 11))
     ncprecip = nc.variables["p01d"]
 
-    midnight = datetime.datetime(
+    midnight = datetime(
         dt.year, dt.month, dt.day, tzinfo=ZoneInfo("America/Chicago")
     )
     utcnow = utc()
     total = None
-    lats = None
     UTC = ZoneInfo("UTC")
     # Stop at 6 PM if we are using DEP
     for hr in range(1, 18 if for_dep else 25):
-        utcdt = (midnight + datetime.timedelta(hours=hr)).astimezone(UTC)
+        utcdt = (midnight + timedelta(hours=hr)).astimezone(UTC)
         if utcdt > utcnow:
             continue
         gribfn = None
         qcprod = "MultiSensor_QPE_01H_Pass2"
         if utcdt < PRODFLIP:
             qcprod = "GaugeCorr_QPE_01H"
         for prefix in [qcprod, "RadarOnly_QPE_01H"]:
-            fn = mrms.fetch(prefix, utcdt)
+            fn = mrms_fetch(prefix, utcdt)
             if fn is None:
                 continue
             LOG.info("Processing %s %s", utcdt, prefix)
@@ -70,9 +67,7 @@ def run(dt: datetime.date, for_dep: bool):
             continue
         with pygrib.open(gribfn) as grbs:
             grb = grbs[1]
-            if lats is None:
-                lats, _ = grb.latlons()
-            val = grb["values"]
+            val: np.ndarray = grb["values"]
         os.unlink(gribfn)
 
         # Anything less than zero, we set to zero
@@ -82,31 +77,45 @@ def run(dt: datetime.date, for_dep: bool):
         else:
             total += val
 
-    if lats is None:
+    if total is None:
         LOG.warning("nodata for %s, using zeros", dt)
-        ncprecip[offset, :, :] = 0
-        nc.close()
+        ncprecip[offset] = 0
         return
+    # To save some space, we store a tigher CONUS domain
+    # total.shape is 3500 x 7000, our 2700 x 6100
+    # The upper left corner grib is -129.995, 54.995
+    # The upper left corner netcdf is -125.995, 49.995
     # CAREFUL HERE!  The MRMS grid is North to South
-    # set top (smallest y)
-    y0 = int((lats[0, 0] - nc.variables["lat"][-1]) * 100.0)
-    y1 = int((lats[0, 0] - nc.variables["lat"][0]) * 100.0)
-    x0 = int((nc.variables["lon"][0] - mrms.WEST) * 100.0)
-    x1 = int((nc.variables["lon"][-1] - mrms.WEST) * 100.0) + 1  # Stinks
+    if total.shape != (3500, 7000):
+        LOG.warning("total shape is %s, skipping", total.shape)
+        return
+    x0 = 400
+    x1 = 400 + 6100
+    y0 = 500
+    y1 = 500 + 2700
     LOG.info("Writing precip at offset: %s [for_dep:%s]", offset, for_dep)
     ncprecip[offset, :, :] = np.flipud(total[y0:y1, x0:x1])
-    nc.close()
 
 
 @click.command()
-@click.option("--date", "dt", type=click.DateTime(), help="Date to process")
+@click.option(
+    "--date",
+    "dt",
+    type=click.DateTime(),
+    help="Date to process",
+    required=True,
+)
 @click.option("--for-dep", is_flag=True, help="For Daily Erosion Project")
-def main(dt, for_dep):
+def main(dt: datetime, for_dep):
     """go main go"""
     if for_dep and (f"{dt:%m%d}" < "0411" or f"{dt:%m%d}" > "0615"):
         LOG.info("DEP not needed for %s", dt)
         return
-    run(dt.date(), for_dep)
+    ncfn = get_daily_mrms_ncname(dt.year)
+    if for_dep:
+        ncfn = ncfn.replace("daily", "dep")
+    with ncopen(ncfn, "a", timeout=300) as nc:
+        run(nc, dt.date(), for_dep)
 
 
 if __name__ == "__main__":

tests/iemweb/autoplot/urllist.txt

@@ -1,10 +1,14 @@
 /plotting/auto/plot/1/network:IACLIMATE::station:IATAME::threshold:44::month1:6::num1:2::var1:hdd65::month2:6::num2:2::var2:gdd41::year:2007::_r:96::dpi:100.png
 /plotting/auto/plot/2/network:IACLIMATE::station:IATAME::month:2::year:2024::gddbase:50::gddceil:86::_r:t::dpi:100.png
 /plotting/auto/plot/3/network:IACLIMATE::station:IATAME::month:summer::type:sum-precip::threshold:-99::syear:1850::eyear:2024::decadal:yes::_r:43::dpi:100.png
+/plotting/auto/plot/4/year:2024::threshold:1.0::period:7::state:IA::_r:t::dpi:100.png
+/plotting/auto/plot/5/network:IACLIMATE::station:IATAME::var:max_snow::_r:43::dpi:100.png
 /plotting/auto/plot/6/state:IA::year:2024::month:3::type:sum-precip::_r:t::dpi:100.png
+/plotting/auto/plot/7/network:IACLIMATE::station:IATAME::year:2024::gdd1:1135::gdd2:1660::gddbase:50::gddceil:86::cmap:jet::_r:t::dpi:100.png
 /plotting/auto/plot/11/network:IA_ASOS::zstation:AMW::year:2024::emphasis:-99::var:dwpf::opt:below::_r:43::dpi:100.png
 /plotting/auto/plot/11/network:IA_ASOS::zstation:AMW::year:2024::emphasis:20::var:feel::opt:touches::_r:43::dpi:100.png
 /plotting/auto/plot/14/network:IACLIMATE::station:IATAME::year:2024::_r:t::dpi:100.png
+/plotting/auto/plot/15/network:IACLIMATE::station:IATAME::varname:high::year:2024::_r:t::dpi:100.png
 /plotting/auto/plot/17/network:IA_ASOS::station:AMW::month:7::year:2024::p:temps::_r:t::dpi:100.png
 /plotting/auto/plot/17/network:IA_ASOS::station:AMW::month:7::year:2024::p:precip::_r:t::dpi:100.png
 /plotting/auto/plot/17/network:IA_ASOS::station:BNW::month:7::year:2024::p:temps::_r:t::dpi:100.png
@@ -31,6 +35,7 @@
 /plotting/auto/plot/34/network:IACLIMATE::station:IATAME::var:high_over::which:average::climo:1951::threshold:32::_r:43::dpi:100.png
 /plotting/auto/plot/37/network:IA_ASOS::zstation:AMW::month:8::year:2024::model:NBE::var:wsp::_r:t::dpi:100::_cb:1.png
 /plotting/auto/plot/37/network:IA_ASOS::zstation:AMW::month:8::year:2024::model:GFS::var:t::_r:t::dpi:100.png
+/plotting/auto/plot/38/network:IACLIMATE::station:IATAME::var:best::climo:era5land_srad::year:2024::_r:t::dpi:100.png
 /plotting/auto/plot/39/network:IACLIMATE::station:IATAME::compare:manual::year:2024::month:1::date:2024-01-03::_r:t::dpi:100.png
 /plotting/auto/plot/40/network:IA_ASOS::zstation:DSM::month:1::ptype:vsby::year:2020::_r:t::dpi:100.png
 /plotting/auto/plot/40/network:IA_ASOS::zstation:DSM::month:1::ptype:sky::year:2020::_r:t::dpi:100.png
@@ -199,8 +204,9 @@
 /plotting/auto/plot/165/w:air::sector:IA::var:high_above::popt:values::year:2020::threshold:32::p:50::sday:0824::syear:1893::eyear:2024::cmap:BrBG::_r:t::dpi:100.png
 /plotting/auto/plot/165/w:air::sector:IA::var:spring_below::popt:contour::year:2024::threshold:32::sday:1026::cmap:BrBG::_r:t::dpi:100.png
 /plotting/auto/plot/165/w:air::sector:AZ::var:spring_below::popt:contour::year:2024::threshold:32::sday:1026::cmap:BrBG::_r:t::dpi:100.png
-/plotting/auto/plot/166/w:cwa::state:IA::network:WFO::cwa:DMX::fema:7::limit:ytd::_r:t::dpi:100::_cb:1.png
-/plotting/auto/plot/166/w:fema::state:IA::network:WFO::cwa:DMX::fema:7::limit:ytd::_r:t::dpi:100::_cb:1.png
+/plotting/auto/plot/166/w:state::state:IA::network:WFO::cwa:GRB::fema:7::limit:year::_r:t::dpi:100.png
+/plotting/auto/plot/166/w:cwa::state:IA::network:WFO::cwa:DMX::fema:7::limit:year::_r:t::dpi:100::_cb:1.png
+/plotting/auto/plot/166/w:fema::state:IA::network:WFO::cwa:DMX::fema:7::limit:year::_r:t::dpi:100::_cb:1.png
 /plotting/auto/plot/167/network:IA_ASOS::zstation:DSM::month:9::year:2020::_r:t::dpi:100.png
 /plotting/auto/plot/169/network:IA_ASOS::zstation:AMW::v:tmpf::hours:24::month:winter::dir:warm::how:over::_r:t::dpi:100.png
 /plotting/auto/plot/169/network:IA_ASOS::zstation:AMW::v:alti::hours:24::month:feb::dir:cool::how:over::_r:43::dpi:100.png
@@ -251,6 +257,7 @@
 /plotting/auto/plot/185/sector:IA::date:2024-01-01::threshold:2.0::cmap:terrain::_r:t::dpi:100.png
 /plotting/auto/plot/191/sdate:2024-01-01::edate:2024-09-12::w:fema::network:WFO::wfo:DMX::state:IA::fema:7::ugc:IAC169::heatmap:yes::phenomenav1:TO::significancev1:W::_r:t::dpi:100.png
 /plotting/auto/plot/192/t:state::network:WFO::wfo:STO::state:IA::v:vsby::valid:2020-06-15%201400::cmap:gray::_r:t::dpi:100.png
+/plotting/auto/plot/193/csector:conus::date:2024-07-01::z:0::f:120::opt:both::scale:auto::cmap:gist_ncar::_r:t::dpi:100.png
 /plotting/auto/plot/194/csector:midwest::sdate:2020-05-01::edate:2024-07-17::d:0::w:percent::cmap:plasma::_r:t::dpi:100.png
 /plotting/auto/plot/196/network:IA_ASOS::station:AMW::opt:no::var:chill::_r:43::dpi:100.png
 /plotting/auto/plot/197/csector:IA::var:corn_planting::w:avg::weeks:1::date:2021-05-09::cmap:BrBG::_r:43::dpi:100.png