fix: station sts/ets computations

akrherz · Nov 4, 2023 · 5b09b2d · 5b09b2d
1 parent 9108773
commit 5b09b2d
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 54 deletions.
diff --git a/scripts/dbutil/compute_climate_sts.py b/scripts/dbutil/compute_climate_sts.py
@@ -1,4 +1,7 @@
-"""Determine when a CLIMATE track site started..."""
+"""Determine when a CLIMATE track site started...
+
+Called from RUN_CLIMODAT_STATE.sh
+"""
 import datetime
 import sys
 
@@ -29,44 +32,40 @@ def main(argv):
     )
     for row in acursor:
         station = row[0]
-        sts = row[1]
-        ets = row[2]
         if station not in nt.sts:
             LOG.info("%s is unknown in mesosite table", station)
             continue
-        meta = nt.sts[station]
-        if meta["archive_begin"] is None or meta["archive_begin"] != sts:
-            LOG.warning(
-                "Updated %s STS WAS: %s NOW: %s" "",
-                station,
-                nt.sts[station]["archive_begin"],
-                sts,
-            )
-
-            mcursor.execute(
-                "UPDATE stations SET archive_begin = %s "
-                "WHERE id = %s and network = %s",
-                (sts, station, net),
-            )
-        archive_end = ets
-        if ets > floor:
-            archive_end = None
-        if (
-            archive_end is None and meta["archive_end"] is not None
-        ) or archive_end != meta["archive_end"]:
-            LOG.warning(
-                "Updated %s ETS WAS: %s NOW: %s" "",
-                station,
-                nt.sts[station]["archive_end"],
-                archive_end,
-            )
-
-            mcursor.execute(
-                "UPDATE stations SET archive_end = %s "
-                "WHERE id = %s and network = %s",
-                (archive_end, station, net),
-            )
-
+        sts = row[1]
+        ets = row[2]
+        if ets >= floor:
+            ets = None
+        osts = nt.sts[station]["archive_begin"]
+        oets = nt.sts[station]["archive_end"]
+        oonline = nt.sts[station]["online"]
+        online = ets is None
+        noop = osts == sts and oets == ets and oonline == online
+        loglvl = LOG.info if noop else LOG.warning
+        loglvl(
+            "%s%s [%s] sts:%s->%s ets:%s->%s OL:%s->%s",
+            "  --> " if not noop else "",
+            station,
+            net,
+            osts,
+            sts,
+            oets,
+            ets,
+            oonline,
+            online,
+        )
+        if noop:
+            continue
+        mcursor.execute(
+            """
+            UPDATE stations SET archive_begin = %s, archive_end = %s,
+            online = %s WHERE iemid = %s
+            """,
+            (sts, ets, ets is None, nt.sts[station]["iemid"]),
+        )
     mcursor.close()
     mesosite.commit()
     mesosite.close()

diff --git a/scripts/dbutil/compute_coop_sts.py b/scripts/dbutil/compute_coop_sts.py
@@ -14,7 +14,7 @@
 
 def do_network(network):
     """Do network"""
-    nt = NetworkTable(network)
+    nt = NetworkTable(network, only_online=False)
     iem, icursor = get_dbconnc("iem")
     for sid in nt.sts:
         icursor.execute(
@@ -33,17 +33,21 @@ def do_network(network):
             ets = None
         osts = nt.sts[sid]["archive_begin"]
         oets = nt.sts[sid]["archive_end"]
-        noop = osts == sts and oets == ets
+        oonline = nt.sts[sid]["online"]
+        online = ets is None
+        noop = osts == sts and oets == ets and oonline == online
         loglvl = LOG.info if noop else LOG.warning
         loglvl(
-            "%s%s [%s] sts:%s->%s ets:%s->%s",
+            "%s%s [%s] sts:%s->%s ets:%s->%s OL:%s->%s",
             "  --> " if not noop else "",
             sid,
             network,
             osts,
             sts,
             oets,
             ets,
+            oonline,
+            online,
         )
         if noop:
             continue

diff --git a/scripts/dbutil/compute_hads_sts.py b/scripts/dbutil/compute_hads_sts.py
@@ -42,34 +42,32 @@ def get_maxvalid(sid):
 
 def do_network(network):
     """Do network"""
-    nt = NetworkTable(network)
+    nt = NetworkTable(network, only_online=False)
     for sid in nt.sts:
-        sts = get_minvalid(sid)
         osts = nt.sts[sid]["archive_begin"]
+        oets = nt.sts[sid]["archive_end"]
+        sts = get_minvalid(sid)
         ets = get_maxvalid(sid)
         if ets is not None and ets.year >= (TODAY.year - 1):
             ets = None
-        oets = nt.sts[sid]["archive_end"]
-        LOG.info(
-            "%s [%s] sts:%s|%s ets:%s|%s",
+        oonline = nt.sts[sid]["online"]
+        online = ets is None
+        noop = osts == sts and oets == ets and oonline == online
+        loglvl = LOG.info if noop else LOG.warning
+        loglvl(
+            "%s%s [%s] sts:%s->%s ets:%s->%s OL:%s->%s",
+            "  --> " if not noop else "",
             sid,
             network,
             osts,
             sts,
             oets,
             ets,
+            oonline,
+            online,
         )
-        if osts == sts and oets == ets:
+        if noop:
             continue
-        LOG.warning(
-            "%s [%s] sts:%s->%s ets:%s->%s",
-            sid,
-            network,
-            osts,
-            sts,
-            oets,
-            ets,
-        )
         cursor = MESOSITEDB.cursor()
         cursor.execute(
             """