Omnibus

.github/copilot-instructions.md

@@ -0,0 +1,24 @@
+# IEM Repo
+
+Every time you choose to apply a rule(s), explicitly state the rule(s) in the output. You can abbreviate the rule description to a single word or phrase.
+
+## Project Context
+
+This repo does a lot of different things with weather data modification.
+
+## Code Style and Structure
+
+```text
+cgi-bin/     # One line front end references to pylib application code
+config/      # PHP configuration
+data/        # Stuff used by PHP and python scripts
+deployment/  # Stuff associated with deployment of this code
+docs/        # centralized docs
+htdocs/      # The apache webroot with mostly PHP stuff and python pointers
+             # to things within pylib
+├── agclimate/ # ISU Soil Moisture Network
+include/     # PHP include scripts
+pylib/       # python library stuff used within this repo only
+scripts/     # python cron jobs that process data
+tests/       # Python testing code mostly for pylib and for integration tests 
+```

config/navbar.json

@@ -162,6 +162,10 @@
                     "title": "Presentations",
                     "url": "/present/"
                 },
+                {
+                    "title": "Referenced By",
+                    "url": "/info/refs.php"
+                },
                 {
                     "title": "Station Data and Metadata",
                     "url": "/sites/locate.php"

htdocs/info/refs.php

@@ -0,0 +1,71 @@
+<?php
+require_once "../../config/settings.inc.php";
+require_once "../../include/myview.php";
+require_once "../../include/database.inc.php";
+$t = new MyView();
+$t->title = "Referenced By";
+
+$table = "";
+$pgconn = iemdb("mesosite");
+$rs = pg_query(
+    $pgconn,
+    "SELECT * from website_citations ORDER by publication_date DESC");
+
+$XREF = Array(
+    "ARCHIVE" => "/archive/",
+    "ASOS" => "/request/download.phtml",
+    "ASOS1MIN" => "/request/asos/1min.phtml",
+    "CLIMODAT" => "/climodat/",
+    "COW" => "/cow/",
+    "HADS" => "/request/dcp/fe.phtml",
+    "IEMRE" => "/iemre",
+    "ISUSM" => "/agclimate/",
+    "MOSAIC" => "/docs/nexrad_mosaic",
+    "NWSTEXT" => "/nws/",
+    "VTEC" => "/request/gis/watchwarn.phtml",
+    "WINDROSE" => "/archive/",
+);
+
+$year = "";
+$total = pg_num_rows($rs);
+while ($row = pg_fetch_assoc($rs)) {
+    if ($year != substr($row["publication_date"], 0, 4)) {
+        $year = substr($row["publication_date"], 0, 4);
+        $table .= sprintf("<tr><td colspan=\"4\"><h3>%s</h3></td></tr>\n", $year);
+    }
+    $table .= sprintf(
+        "<tr><td><a href=\"%s\" class=\"btn btn-primary\">%s</a></td>".
+        "<td><a href=\"%s\">%s</a></td><td>%s</td></tr>\n",
+        $XREF[$row["iem_resource"]], $row["iem_resource"],
+        $row["link"], $row["publication_date"], $row["title"],
+    );
+}
+$t->content = <<<EOM
+<ul class="breadcrumb">
+<li><a href="/info/">IEM Information</a></li>
+<li class="active">Referenced By</li>
+</ul>
+
+<h3>Scholar Work Referencing IEM</h3>
+
+<p>Beginning in late 2024, the IEM has attempted to curate a list of scholarly
+products referencing datasets found on this website.  The backfilling of
+this list remains a work in progress!</p>
+
+<p>There are currently <strong>{$total}</strong> references in this list.</p>
+
+<table class="table table-striped">
+<thead class="sticky">
+<tr>
+  <th>IEM Resource</th>
+  <th>Publication Date/Link</th>
+  <th>Title</th>
+</tr>
+</thead>
+<tbody>
+{$table}
+</tbody>
+</table>
+
+EOM;
+$t->render('single.phtml');

include/generators.php

@@ -10,11 +10,12 @@ function get_website_citations($label){
         $conn,
         "CITSELECT",
         "SELECT * from website_citations WHERE iem_resource = $1 ".
-        "ORDER by publication_date DESC");
+        "ORDER by publication_date DESC LIMIT 10");
     $rs = pg_execute($conn, "CITSELECT", array($label));
     $s = <<<EOM
-<h3>Publications Citing IEM Data</h3>
-<p>These are publications that have cited the usage of data from this page. This
+<h3>Publications Citing IEM Data (<a href="/info/refs.php">View All</a>)</h3>
+<p>These are the most recent 10 publications that have cited the usage of data
+from this page. This
 list is not exhaustive, so please <a href="/info/contacts.php">let us know</a>
 if you have a publication that should be added.</p>
 <ul>

pylib/iemweb/afos/retrieve.py

@@ -13,6 +13,7 @@
 Changelog
 ~~~~~~~~~
 
+- 2025-01-08: Added some caching due to incessant requests for the same data.
 - 2024-08-25: Add ``order`` parameter to allow for order of the returned
   products.
 - 2024-03-29: Initial documentation release and migrate to a pydantic schema
@@ -21,6 +22,12 @@
 Examples
 ~~~~~~~~
 
+Return all of the Daily Summary Messages for Des Moines from 1 Jan 2025 to 9
+Jan 2025 in text format.
+
+https://mesonet.agron.iastate.edu/cgi-bin/afos/retrieve.py?\
+limit=9999&pil=DSMDSM&fmt=text&sdate=2025-01-01&edate=2025-01-09
+
 Return all TORnado warnings issued between 20 and 21 UTC on 27 Apr 2011 as
 a zip file.
 
@@ -204,7 +211,40 @@ def special_metar_logic(pils, limit, fmt, sio, order):
     return [sio.getvalue().encode("ascii", "ignore")]
 
 
-@iemapp(help=__doc__, schema=MyModel, parse_times=False)
+def get_mckey(environ: dict) -> Optional[str]:
+    """Cache a specific request."""
+    # limit=9999&pil=DSMDEN&fmt=text&sdate=2025-01-07&edate=2025-01-09
+    if (
+        environ["pil"][0].startswith("DSM")
+        and environ["fmt"] == "text"
+        and environ["sdate"] is not None
+        and environ["edate"] is not None
+    ):
+        return (
+            f"afos_retrieve.py_{environ['pil'][0]}_{environ['sdate']:%Y%m%d}_"
+            f"{environ['edate']:%Y%m%d}_{environ['limit']}"
+        )
+    return None
+
+
+def get_ct(environ: dict) -> str:
+    """Figure out the content type."""
+    fmt = environ["fmt"]
+    if fmt == "zip" or environ["dl"]:
+        return "application/octet-stream"
+    if fmt == "html":
+        return "text/html"
+    return "text/plain"
+
+
+@iemapp(
+    help=__doc__,
+    schema=MyModel,
+    memcacheexpire=600,
+    memcachekey=get_mckey,
+    content_type=get_ct,
+    parse_times=False,
+)
 def application(environ, start_response):
     """Process the request"""
     order = environ["order"]
@@ -223,17 +263,12 @@ def application(environ, start_response):
         environ["sdate"], environ["edate"] = environ["edate"], environ["sdate"]
     fmt = environ["fmt"]
     headers = [("X-Content-Type-Options", "nosniff")]
+    headers.append(("Content-type", get_ct(environ)))
     if environ["dl"] or fmt == "zip":
         suffix = "zip" if fmt == "zip" else "txt"
-        headers.append(("Content-type", "application/octet-stream"))
         headers.append(
             ("Content-disposition", f"attachment; filename=afos.{suffix}")
         )
-    else:
-        if fmt == "text":
-            headers.append(("Content-type", "text/plain"))
-        elif fmt == "html":
-            headers.append(("Content-type", "text/html"))
     start_response("200 OK", headers)
 
     sio = StringIO()
@@ -312,4 +347,4 @@ def application(environ, start_response):
 
         if cursor.rowcount == 0:
             sio.write(f"ERROR: Could not Find: {','.join(pils)}")
-    return [sio.getvalue().encode("ascii", "ignore")]
+    return sio.getvalue()