Merge pull request #672 from akrherz/240209

IEMRE Updates

htdocs/iemre/index.phtml

@@ -115,38 +115,77 @@ degree.</li>
      </ul>
      </div>
 </div>
-         
+
 <br clear="all" />
-<h3>Data Availability</h3>
-
-<table class="table">
-         <thead>
-<tr><th>Hourly Interval</th><th>Daily Interval</th></tr>
-</thead>
-         <tr>
-<td>
-<ul>
- <li>2 Meter Air and Dew Point Temperature</li>
- <li>10 Meter Wind Speed and Direction</li>
- <li>Precipitation</li>
- <li>Approx 4inch Soil Temperature</li>
- <li>Sky Coverage Percentage</li>
-</ul>
-</td>
-<td>
-<ul>
- <li>High and Low 2 Meter Air Temperature</li>
- <li>High and Low ~4 inch soil temperature</li>
- <li>Average wind speed magnitude</li>
- <li>Precipitation</li>
- <li>Snowfall at 12 UTC</li>
- <li>Snow Depth at 12 UTC</li>
- <li>Solar Radiation</li>
- <li>Solar Radiation via NASA POWER</li>
- <li>Average Dew Point Temperature</li>
-</ul>
-</td>
+<h3>Hourly Data Sources</h3>
+
+<p>Yearly <a href="/onsite/iemre/">netcdf files</a> since 1950, with updates
+at about 10 after the hour.  Since we want real-time data, some of these
+model sources are not available immediately, so the denoted intermediate source
+fills the gap until the final source is available.</p>
+
+<table class="table-striped">
+<thead><tr><th>Variable</th><th>Intermediate Source</th><th>Final Source</th></tr></thead>
+<tbody>
+
+<tr>
+<th>Precipitation</th>
+<td>Stage IV</td>
+<td>1997+ Stage IV adjusted by PRISM
+<br />1950-1997 ERA5-Land</td>
+</tr>
+
+<tr>
+<th>2m Air Temperature
+<br />2 m Dew Point Temperature
+<br />10m Wind Speed
+</th>
+<td>Crude ASOS grid analysis</td>
+<td>2010+: RTMA, once available
+<br />1950-2010 ERA5-Land</td>
+</tr>
+
+<tr>
+<th>4 inch (cough 0-10 or 0-7cm) Soil Temperature</th>
+<td>HRRR</td>
+<td>1950+: ERA5-Land</td>
+</tr>
+
+<tr>
+<th>Sky Coverage</th>
+<td>Crude ASOS grid analysis</td>
+<td>Crude ASOS grid analysis</td>
 </tr>
+
+</tbody>
+</table>
+
+<!-- ......................................................... -->
+<h3>Daily Data Sources</h3>
+
+<p>Yearly <a href="/onsite/iemre/">netcdf files</a> since 1893. Some of these
+variables are directly sourced from the hourly grids above.  The concept of
+a daily variable is a bit nebulous, but is intended to be a 6 to 6 UTC period.
+</p>
+
+<table class="table-striped">
+<thead><tr><th>Variable</th><th>Intermediate Source</th><th>Final Source</th></tr></thead>
+<tbody>
+
+<tr>
+<th>High/Low 4 inch (cough 0-10 or 0-7cm) Soil Temperature</th>
+<td>HRRR</td>
+<td>1950+: ERA5-Land</td>
+</tr>
+
+<tr>
+<th>Average 2m Dew Point Temperature</th>
+<td>Simple average of hourly RTMA</td>
+<td>2010+: RTMA, once available
+<br />1950-2010 ERA5-Land</td>
+</tr>
+
+</tbody>
 </table>
 
 <h3>Data Access</h3>
@@ -177,7 +216,7 @@ Example: https://mesonet.agron.iastate.edu/iemre/hourly/2010-05-01/42.54/-96.40/
 
 
 <h4>Raw Data Access</h4>
-                 
+
 <p>The IEM makes extensive use of the <a href="https://www.unidata.ucar.edu/software/netcdf/">NetCDF</a>
 file format to store these gridded datasets.  You can find individual web directories for various
 IEMRE related datasets: 

htdocs/plotting/auto/scripts/p86.py

@@ -28,6 +28,8 @@
     "high_tmpk_12z": "Maximum Temperature at 12 UTC",
     "high_soil4t": "Maximum 4 Inch Soil Temperature",
     "low_soil4t": "Minimum 4 Inch Soil Temperature",
+    "min_rh": "Minimum Relative Humidity",
+    "max_rh": "Maximum Relative Humidity",
     "power_swdn": "NASA POWER :: Incident Shortwave Down",
     "rsds": "Solar Radiation",
     "avg_dwpk": "Average Dew Point",
@@ -80,7 +82,9 @@ def unit_convert(nc, varname, idx0, jslice, islice):
     data = None
     if not varname.startswith("range"):
         data = nc.variables[varname][idx0, jslice, islice]
-    if varname in ["rsds", "power_swdn"]:
+    if varname in ["min_rh", "max_rh"]:
+        pass
+    elif varname in ["rsds", "power_swdn"]:
         # Value is in W m**-2, we want MJ
         multi = (86400.0 / 1000000.0) if varname == "rsds" else 1
         data = data * multi
@@ -105,7 +109,6 @@ def unit_convert(nc, varname, idx0, jslice, islice):
         "high_soil4t",
         "low_soil4t",
     ]:
-        # Value is in W m**-2, we want MJ
         data = masked_array(data, units("degK")).to(units("degF")).m
     else:  # range_tmpk range_tmpk_12z
         vname2 = f"low_tmpk{'_12z' if varname == 'range_tmpk_12z' else ''}"
@@ -161,6 +164,9 @@ def plotter(fdict):
             plot_units = "mph"
             clevs = pretty_bins(0, ptiles[1])
             clevs[0] = 0.01
+        elif varname in ["min_rh", "max_rh"]:
+            plot_units = "%"
+            clevs = pretty_bins(0, 100)
         elif varname in ["p01d", "p01d_12z", "snow_12z", "snowd_12z"]:
             # Value is in W m**-2, we want MJ
             plot_units = "inch"

scripts/RUN_0Z.sh

@@ -13,7 +13,7 @@ python ingest_climdiv.py &
 
 cd ../iemre
 # need to run daily analysis for climodat estimator to then work
-python daily_analysis.py $(date +'%Y %m %d')
+python daily_analysis.py --date=$(date +'%Y-%m-%d')
 
 cd ../climodat
 python sync_coop_updates.py

scripts/RUN_10_AFTER.sh

@@ -35,7 +35,7 @@ fi
 # we run at 11 PM for today as well
 if [ $LHH -eq "23" ]
 then
-    python daily_analysis.py $TODAY
+    python daily_analysis.py --date=$(date +'%Y-%m-%d')
     python grid_rsds.py	$TODAY
 fi
 

scripts/RUN_MIDNIGHT.sh

@@ -12,7 +12,7 @@ python update_daily_srad.py $(date --date '1 day ago' +'%Y %m %d')
 
 # Need this done so that IEMRE daily grids are there for DEP
 cd ../iemre
-python daily_analysis.py $(date --date '1 day ago' +'%Y %m %d')
+python daily_analysis.py --date=$(date --date '1 day ago' +'%Y-%m-%d')
 
 cd ../asos
 python adjust_report_type.py $(date -u --date '1 day ago' +'%Y %m %d')

scripts/RUN_NOON.sh

@@ -9,7 +9,7 @@ cd ../iemre
 python stage4_12z_adjust.py $(date +'%Y %m %d')
 # Copy the data to IEMRE hourly
 python precip_ingest.py $(date +'%Y %m %d')
-python daily_analysis.py $(date +'%Y %m %d')
+python daily_analysis.py --date=$(date +'%Y-%m-%d')
 
 cd ../prism
 python ingest_prism.py $(date --date '1 days ago' +'%Y %m %d')
@@ -34,9 +34,9 @@ python sync_coop_updates.py
 cd ../iemre
 # Since we have now adjusted the 12z precip 1 day ago, we should rerun
 # iemre for two days ago
-python daily_analysis.py $(date --date '2 days ago' +'%Y %m %d')
+python daily_analysis.py --date=$(date --date '2 days ago' +'%Y-%m-%d')
 # Updated soil temperature data from ERA5
-python daily_analysis.py $(date --date '10 days ago' +'%Y %m %d')
+python daily_analysis.py --date=$(date --date '10 days ago' +'%Y-%m-%d')
 
 # and now recompute climodat statewide/climate from two days ago
 cd ../climodat

scripts/dbutil/add_iem_data_entry.py

@@ -4,7 +4,8 @@
 """
 import datetime
 
-from pyiem.util import get_dbconnc, logger
+from pyiem.database import get_dbconnc
+from pyiem.util import logger
 
 LOG = logger()
 

scripts/dbutil/add_summary_entries.py

@@ -2,7 +2,8 @@
 import datetime
 import sys
 
-from pyiem.util import get_dbconn, logger
+from pyiem.database import get_dbconn
+from pyiem.util import logger
 
 LOG = logger()
 

scripts/dbutil/clean_afos.py

@@ -3,8 +3,9 @@
 called from RUN_2AM.sh
 """
 
+from pyiem.database import get_dbconn
 from pyiem.reference import state_names
-from pyiem.util import get_dbconn, logger
+from pyiem.util import logger
 
 LOG = logger()
 

scripts/dbutil/clean_mos.py

@@ -3,7 +3,8 @@
 Run from `RUN_2AM.sh` script.
 """
 
-from pyiem.util import get_dbconn, logger
+from pyiem.database import get_dbconn
+from pyiem.util import logger
 
 LOG = logger()
 

scripts/dbutil/clean_telemetry.py

@@ -4,7 +4,8 @@
 """
 from datetime import datetime, timedelta
 
-from pyiem.util import get_dbconnc, logger
+from pyiem.database import get_dbconnc
+from pyiem.util import logger
 
 LOG = logger()
 

scripts/dbutil/clean_unknown_hads.py

@@ -5,7 +5,8 @@
 Run from RUN_2AM.sh
 """
 
-from pyiem.util import get_dbconn, logger
+from pyiem.database import get_dbconn
+from pyiem.util import logger
 
 LOG = logger()