mnt: sundry rollups

htdocs/robots.txt

@@ -9,6 +9,4 @@ Disallow: /data/NIDS/
 Disallow: /data/nexrd2/
 Disallow: /data/model/
 Disallow: /archive/nexrad/
-Disallow: /nwnwebsite/
-Disallow: /archive/data/
 Disallow: /archive/raw/snet/

htdocs/topics/first_freeze.phtml

@@ -86,15 +86,5 @@ $t->content = <<<EOM
 </div>
 </div>
 
-
-<!--
-<h3></h3>
-
-<p><img src="" class="img img-responsive" alt="autoplot"></p>
-
-<p><a class="btn btn-sm btn-primary" href="">Generate This Map</a></p>
--->
-
-
 EOM;
 $t->render("single.phtml");

htdocs/wx/afos/bottom.php

@@ -78,7 +78,6 @@
 </pre>
     </div>
 
-
 </body>
 
 </html>

htdocs/wx/afos/p.php

@@ -1,12 +1,12 @@
 <?php
+define("IEM_APPID", 47);
+
 require_once "../../../config/settings.inc.php";
 require_once "../../../include/database.inc.php";
 require_once "../../../include/myview.php";
 require_once "../../../include/forms.php";
 $t = new MyView();
 
-define("IEM_APPID", 47);
-
 $e = get_int404("e", null);
 $pil = isset($_GET['pil']) ? strtoupper(substr(xssafe($_GET['pil']), 0, 6)) : null;
 $bbb = isset($_GET["bbb"]) ? strtoupper(substr(xssafe($_GET["bbb"]), 0, 3)) : null;
@@ -164,10 +164,21 @@ function exact_product($conn, $e, $pil, $bbb){
     $content .= '<div class="alert alert-danger"><i class="fa fa-file"></i> ' .
         "Found {$rows} products at the given pil and timestamp. Scroll down to see them all.</div>";
 }
+$extratools = "";
 $img = "";
 for ($i = 0; $row = pg_fetch_assoc($rs); $i++) {
     if ($i == 0) {
         $basets = strtotime($row["mytime"]);
+        if (substr($pil, 0, 3) == "CLI" || substr($pil, 0, 3) == "CF6") {
+            $station = sprintf("%s%s", substr($row["source"], 0, 1), substr($pil, 3, 3));
+            $year = date("Y", $basets);
+            $ccc = substr($pil, 0, 3);
+            $cc = strtolower($ccc);
+            $extratools = <<<EOM
+            <p><a class="btn btn-success" href="/nws/{$cc}table.php?station={$station}&opt=bystation&year={$year}">
+            <i class="fa fa-list"></i> Daily {$ccc} Table for {$station}</a></p>
+EOM;
+        }
         $newe = date("YmdHi", $basets);
         $rawtext = sprintf(
             "/api/1/nwstext/%s-%s-%s-%s",
@@ -252,28 +263,30 @@ function exact_product($conn, $e, $pil, $bbb){
         $year2 = intval($year) + 1;
         $content .= <<<EOF
 <div class="row">
-<div class="col-sm-6 col-md-6">
+<div class="col-sm-7 col-md-7">
 <p>Displaying AFOS PIL: <strong>$pil</strong> 
 Received: <strong>{$dstamp} UTC</strong>
 
-<a rel="nofollow" class="btn btn-primary" 
+<p><a class="btn btn-primary" 
  href="p.php?dir=prev&pil=$pil&e=$newe"><i class="fa fa-arrow-left"></i> 
  Previous in Time</a>
-<a rel="nofollow" class="btn btn-primary" 
+<a class="btn btn-primary" 
  href="p.php?pil=$pil">Latest Product</a>
-<a rel="nofollow" class="btn btn-primary" 
+<a class="btn btn-primary" 
  href="p.php?dir=next&pil=$pil&e=$newe">Next in Time <i class="fa fa-arrow-right"></i></a>
 
-<p><a rel="nofollow" class="btn btn-primary" 
+<p><a class="btn btn-primary" 
  href="{$listlink}">View All {$row["source"]} Products for {$date2}</a>
- <a rel="nofollow" class="btn btn-primary" 
+ <a class="btn btn-primary" 
  href="{$pil_listlink}">View All {$pil3} Products for {$date2}</a>
-<a rel="nofollow" class="btn btn-primary"
+<a class="btn btn-primary"
  href="{$t->twitter_image}">View As Image</a>
 <a class="btn btn-primary" href="{$rawtext}">Download As Text</a></p>
 
+{$extratools}
+
 </div>
-<div class="col-sm-6 col-md-6 well">
+<div class="col-sm-5 col-md-5 well">
 <form method="GET" action="/cgi-bin/afos/retrieve.py" name="bulk">
 <input type="hidden" name="dl" value="1">
 <input type="hidden" name="limit" value="9999">

htdocs/wx/afos/retreive.php

@@ -1,5 +1,5 @@
 <?php
-// 1 Aug 2022 still being used...
+// 27 Dec 2023 still being used...
 $pil = isset($_REQUEST["pil"]) ? strtoupper($_REQUEST["pil"]) : 'AFDDMX';
 $cnt = isset($_REQUEST["cnt"]) ? intval($_REQUEST["cnt"]) : 1;
 $center = isset($_REQUEST["center"]) ? substr($_REQUEST["center"], 0, 4) : "";

htdocs/wx/afos/static.css

@@ -1,64 +1,77 @@
 /* Used by index.phtml */
 .tab-content {
-  height: 500px;
-  overflow-y: scroll;
+    height: 500px;
+    overflow-y: scroll;
 }
-.ui-datepicker{
+
+.ui-datepicker {
     z-index: 99999 !important;
 }
-.ui-datepicker-month{
+
+.ui-datepicker-month {
     color: #000 !important;
 }
-.ui-datepicker-year{
+
+.ui-datepicker-year {
     color: #000 !important;
 }
-.panel.with-nav-tabs .panel-heading{
+
+.panel.with-nav-tabs .panel-heading {
     padding: 5px 5px 0 5px;
 }
-.panel.with-nav-tabs .nav-tabs{
+
+.panel.with-nav-tabs .nav-tabs {
     border-bottom: none;
 }
-.panel.with-nav-tabs .nav-justified{
+
+.panel.with-nav-tabs .nav-justified {
     margin-bottom: -1px;
 }
+
 /********************************************************************/
 /*** PANEL DEFAULT ***/
-.with-nav-tabs.panel-default .nav-tabs > li > a,
-.with-nav-tabs.panel-default .nav-tabs > li > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > li > a:focus {
+.with-nav-tabs.panel-default .nav-tabs>li>a,
+.with-nav-tabs.panel-default .nav-tabs>li>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>li>a:focus {
     color: #777;
 }
-.with-nav-tabs.panel-default .nav-tabs > .open > a,
-.with-nav-tabs.panel-default .nav-tabs > .open > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > .open > a:focus,
-.with-nav-tabs.panel-default .nav-tabs > li > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > li > a:focus {
+
+.with-nav-tabs.panel-default .nav-tabs>.open>a,
+.with-nav-tabs.panel-default .nav-tabs>.open>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>.open>a:focus,
+.with-nav-tabs.panel-default .nav-tabs>li>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>li>a:focus {
     color: #777;
     background-color: #ddd;
     border-color: transparent;
 }
-.with-nav-tabs.panel-default .nav-tabs > li.active > a,
-.with-nav-tabs.panel-default .nav-tabs > li.active > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > li.active > a:focus {
+
+.with-nav-tabs.panel-default .nav-tabs>li.active>a,
+.with-nav-tabs.panel-default .nav-tabs>li.active>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>li.active>a:focus {
     color: #555;
     background-color: #fff;
     border-color: #ddd;
     border-bottom-color: transparent;
 }
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu {
+
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu {
     background-color: #f5f5f5;
     border-color: #ddd;
 }
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > li > a {
-    color: #777;   
+
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>li>a {
+    color: #777;
 }
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > li > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > li > a:focus {
+
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>li>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>li>a:focus {
     background-color: #ddd;
 }
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > .active > a,
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > .active > a:hover,
-.with-nav-tabs.panel-default .nav-tabs > li.dropdown .dropdown-menu > .active > a:focus {
+
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>.active>a,
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>.active>a:hover,
+.with-nav-tabs.panel-default .nav-tabs>li.dropdown .dropdown-menu>.active>a:focus {
     color: #fff;
     background-color: #555;
 }

htdocs/wx/afos/top.php

@@ -4,7 +4,7 @@
     <title>Top</title>
 </head>
 
-<body bgcolor="white">
+<body>
 
     <form method="GET" action="/cgi-bin/afos/retrieve.py" target="display">
 

scripts/RUN_0Z.sh

@@ -17,7 +17,7 @@ python daily_analysis.py $(date +'%Y %m %d')
 
 cd ../climodat
 python sync_coop_updates.py
-python daily_estimator.py $(date +'%Y %m %d')
+python daily_estimator.py --date=$(date +'%Y-%m-%d')
 
 # Wait a bit, so that more obs can come in
 sleep 300

scripts/RUN_2AM.sh

@@ -15,7 +15,7 @@ python swat_realtime.py &
 
 # Run the climodat estimator to get sites that are valid at midnight
 cd ../climodat
-python daily_estimator.py $(date --date '1 days ago' +'%Y %m %d')
+python daily_estimator.py --date=$(date --date '1 days ago' +'%Y-%m-%d')
 python compute_climate.py $(date --date '1 days ago' +'%Y %m %d')
 
 # Look for stuff we missed with noaaport ingest

scripts/RUN_NOON.sh

@@ -21,10 +21,10 @@ python prism_adjust_stage4.py $(date --date '1 days ago' +'%Y %m %d')
 python precip_ingest.py $(date --date '1 days ago' +'%Y %m %d')
 
 cd ../climodat
-python daily_estimator.py $(date +'%Y %m %d')
-python daily_estimator.py $(date --date '1 days ago'  +'%Y %m %d')
+python daily_estimator.py --date=$(date +'%Y-%m-%d')
+python daily_estimator.py --date=$(date --date '1 days ago'  +'%Y-%m-%d')
 # Perhaps some more QC happened, that we now need to pick up
-python daily_estimator.py $(date --date '7 days ago'  +'%Y %m %d')
+python daily_estimator.py --date=$(date --date '7 days ago'  +'%Y-%m-%d')
 python compute4regions.py $(date +'%Y %m %d')
 python compute4regions.py $(date --date '1 days ago'  +'%Y %m %d')
 python hrrr_solarrad.py $(date --date '1 days ago'  +'%Y %m %d')

scripts/climodat/daily_estimator.py

@@ -1,15 +1,15 @@
 """Climodat Daily Data Estimator.
 
-   python daily_estimator.py YYYY MM DD
+python daily_estimator.py --date=YYYY-MM-DD
 
 RUN_NOON.sh - processes the current date, this skips any calendar day sites
 RUN_NOON.sh - processes yesterday, running all sites
 RUN_0Z.sh - processes the current date and gets the prelim calday sites data.
 RUN_2AM.sh - processes yesterday, which should run all sites
 """
 import datetime
-import sys
 
+import click
 import numpy as np
 import pandas as pd
 from metpy.units import units
@@ -138,7 +138,7 @@ def estimate_hilo(df, ds):
     lowgrid00 = k2f(ds["low_tmpk"].values)
 
     for sid, row in df[pd.isna(df["high"])].iterrows():
-        if row["temp24_hour"] in [0, 22, 23]:
+        if row["temp24_hour"] in [0, 22, 23, 24]:
             val = highgrid00[row["gridj"], row["gridi"]]
             temp_hour = 0
         else:
@@ -150,7 +150,7 @@ def estimate_hilo(df, ds):
             df.at[sid, "high"] = val
             df.at[sid, "dirty"] = True
     for sid, row in df[pd.isna(df["low"])].iterrows():
-        if row["temp24_hour"] in [0, 22, 23]:
+        if row["temp24_hour"] in [0, 22, 23, 24]:
             val = lowgrid00[row["gridj"], row["gridi"]]
             temp_hour = 0
         else:
@@ -180,11 +180,15 @@ def commit(cursor, table, df, ts):
     allowed_failures = 10
     for sid, row in df[df["dirty"]].iterrows():
         LOG.info(
-            "sid: %s high: %s low: %s precip: %s snow: %s snowd: %s",
+            "%s high(%s): %s[%s] low: %s prec(%s): %s[%s] snow: %s snowd: %s",
             sid,
+            row["temp_estimated"],
             row["high"],
+            row["temp_hour"],
             row["low"],
+            row["precip_estimated"],
             row["precip"],
+            row["precip_hour"],
             row["snow"],
             row["snowd"],
         )
@@ -243,8 +247,8 @@ def merge_obs(df, state, ts):
             round(max_tmpf::numeric, 0) as high,
             round(min_tmpf::numeric, 0) as low,
             pday as precip, snow, snowd,
-            coalesce(extract(hour from (coop_valid + '1 minute'::interval)
-            at time zone tzname), 24) as temp_hour
+            extract(hour from (coop_valid + '1 minute'::interval)
+            at time zone tzname) as temp_hour
             from summary s JOIN stations t
             on (t.iemid = s.iemid) WHERE t.network = ANY(:networks)
             and s.day = :dt
@@ -314,18 +318,15 @@ def merge_threaded(df, threaded):
             df.loc[sid] = df.loc[copysid]
 
 
-def main(argv):
+@click.command()
+@click.option("--date", type=click.DateTime(), help="Date to process")
+@click.option("--state", default=None, help="State to process")
+def main(date, state):
     """Go Main Go."""
-    date = datetime.date(int(argv[1]), int(argv[2]), int(argv[3]))
+    date = date.date()
     ds = iemre.get_grids(date)
     pgconn = get_dbconn("coop")
-    states = (
-        state_names.keys()
-        if len(argv) < 5
-        else [
-            argv[4],
-        ]
-    )
+    states = state_names.keys() if state is None else [state]
     for state in states:
         cursor = pgconn.cursor()
         df, threaded = load_table(state, date)
@@ -347,5 +348,4 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    # See how we are called
-    main(sys.argv)
+    main()