From 62bf77ab203efcee4e58394e73ac997002f4cdaa Mon Sep 17 00:00:00 2001 From: akrherz Date: Wed, 8 Jan 2025 11:11:25 -0600 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20Build=20out=20IEM=20Referenced?= =?UTF-8?q?=20By=20plumbing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://mesonet.agron.iastate.edu/info/refs.php --- config/navbar.json | 4 +++ htdocs/info/refs.php | 71 ++++++++++++++++++++++++++++++++++++++++++ include/generators.php | 7 +++-- 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 htdocs/info/refs.php diff --git a/config/navbar.json b/config/navbar.json index c05b84caa8..e235a53b65 100644 --- a/config/navbar.json +++ b/config/navbar.json @@ -162,6 +162,10 @@ "title": "Presentations", "url": "/present/" }, + { + "title": "Referenced By", + "url": "/info/refs.php" + }, { "title": "Station Data and Metadata", "url": "/sites/locate.php" diff --git a/htdocs/info/refs.php b/htdocs/info/refs.php new file mode 100644 index 0000000000..d6e139e16c --- /dev/null +++ b/htdocs/info/refs.php @@ -0,0 +1,71 @@ +title = "Referenced By"; + +$table = ""; +$pgconn = iemdb("mesosite"); +$rs = pg_query( + $pgconn, + "SELECT * from website_citations ORDER by publication_date DESC"); + +$XREF = Array( + "ARCHIVE" => "/archive/", + "ASOS" => "/request/download.phtml", + "ASOS1MIN" => "/request/asos/1min.phtml", + "CLIMODAT" => "/climodat/", + "COW" => "/cow/", + "HADS" => "/request/dcp/fe.phtml", + "IEMRE" => "/iemre", + "ISUSM" => "/agclimate/", + "MOSAIC" => "/docs/nexrad_mosaic", + "NWSTEXT" => "/nws/", + "VTEC" => "/request/gis/watchwarn.phtml", + "WINDROSE" => "/archive/", +); + +$year = ""; +$total = pg_num_rows($rs); +while ($row = pg_fetch_assoc($rs)) { + if ($year != substr($row["publication_date"], 0, 4)) { + $year = substr($row["publication_date"], 0, 4); + $table .= sprintf("

%s

\n", $year); + } + $table .= sprintf( + "%s". + "%s%s\n", + $XREF[$row["iem_resource"]], $row["iem_resource"], + $row["link"], $row["publication_date"], $row["title"], + ); +} +$t->content = << +
  • IEM Information
    • +
  • Referenced By
    • + + +

    Scholar Work Referencing IEM

    + +

    Beginning in late 2024, the IEM has attempted to curate a list of scholarly +products referencing datasets found on this website. The backfilling of +this list remains a work in progress!

    + +

    There are currently {$total} references in this list.

    + + + + + + + + + + +{$table} + +
    IEM ResourcePublication Date/LinkTitle
    + +EOM; +$t->render('single.phtml'); diff --git a/include/generators.php b/include/generators.php index 78654ab288..036d9a3718 100644 --- a/include/generators.php +++ b/include/generators.php @@ -10,11 +10,12 @@ function get_website_citations($label){ $conn, "CITSELECT", "SELECT * from website_citations WHERE iem_resource = $1 ". - "ORDER by publication_date DESC"); + "ORDER by publication_date DESC LIMIT 10"); $rs = pg_execute($conn, "CITSELECT", array($label)); $s = <<Publications Citing IEM Data -

    These are publications that have cited the usage of data from this page. This +

    Publications Citing IEM Data (View All)

    +

    These are the most recent 10 publications that have cited the usage of data +from this page. This list is not exhaustive, so please let us know if you have a publication that should be added.

      From 9cba8804e9eabba9d6d28419b8f96e6352b0b86b Mon Sep 17 00:00:00 2001 From: akrherz Date: Wed, 8 Jan 2025 12:14:22 -0600 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=9D=20Add=20copilot=20instructions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/copilot-instructions.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..b7ce44a940 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,24 @@ +# IEM Repo + +Every time you choose to apply a rule(s), explicitly state the rule(s) in the output. You can abbreviate the rule description to a single word or phrase. + +## Project Context + +This repo does a lot of different things with weather data modification. + +## Code Style and Structure + +```text +cgi-bin/ # One line front end references to pylib application code +config/ # PHP configuration +data/ # Stuff used by PHP and python scripts +deployment/ # Stuff associated with deployment of this code +docs/ # centralized docs +htdocs/ # The apache webroot with mostly PHP stuff and python pointers + # to things within pylib +├── agclimate/ # ISU Soil Moisture Network +include/ # PHP include scripts +pylib/ # python library stuff used within this repo only +scripts/ # python cron jobs that process data +tests/ # Python testing code mostly for pylib and for integration tests +``` From 749008bc133660121febe9aadcc3a95c32041918 Mon Sep 17 00:00:00 2001 From: akrherz Date: Wed, 8 Jan 2025 13:02:00 -0600 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Add=20some=20caching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylib/iemweb/afos/retrieve.py | 51 +++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/pylib/iemweb/afos/retrieve.py b/pylib/iemweb/afos/retrieve.py index 418e4e3e78..0e219e2025 100644 --- a/pylib/iemweb/afos/retrieve.py +++ b/pylib/iemweb/afos/retrieve.py @@ -13,6 +13,7 @@ Changelog ~~~~~~~~~ +- 2025-01-08: Added some caching due to incessant requests for the same data. - 2024-08-25: Add ``order`` parameter to allow for order of the returned products. - 2024-03-29: Initial documentation release and migrate to a pydantic schema @@ -21,6 +22,12 @@ Examples ~~~~~~~~ +Return all of the Daily Summary Messages for Des Moines from 1 Jan 2025 to 9 +Jan 2025 in text format. + +https://mesonet.agron.iastate.edu/cgi-bin/afos/retrieve.py?\ +limit=9999&pil=DSMDSM&fmt=text&sdate=2025-01-01&edate=2025-01-09 + Return all TORnado warnings issued between 20 and 21 UTC on 27 Apr 2011 as a zip file. @@ -204,7 +211,40 @@ def special_metar_logic(pils, limit, fmt, sio, order): return [sio.getvalue().encode("ascii", "ignore")] -@iemapp(help=__doc__, schema=MyModel, parse_times=False) +def get_mckey(environ: dict) -> Optional[str]: + """Cache a specific request.""" + # limit=9999&pil=DSMDEN&fmt=text&sdate=2025-01-07&edate=2025-01-09 + if ( + environ["pil"][0].startswith("DSM") + and environ["fmt"] == "text" + and environ["sdate"] is not None + and environ["edate"] is not None + ): + return ( + f"afos_retrieve.py_{environ['pil'][0]}_{environ['sdate']:%Y%m%d}_" + f"{environ['edate']:%Y%m%d}_{environ['limit']}" + ) + return None + + +def get_ct(environ: dict) -> str: + """Figure out the content type.""" + fmt = environ["fmt"] + if fmt == "zip" or environ["dl"]: + return "application/octet-stream" + if fmt == "html": + return "text/html" + return "text/plain" + + +@iemapp( + help=__doc__, + schema=MyModel, + memcacheexpire=600, + memcachekey=get_mckey, + content_type=get_ct, + parse_times=False, +) def application(environ, start_response): """Process the request""" order = environ["order"] @@ -223,17 +263,12 @@ def application(environ, start_response): environ["sdate"], environ["edate"] = environ["edate"], environ["sdate"] fmt = environ["fmt"] headers = [("X-Content-Type-Options", "nosniff")] + headers.append(("Content-type", get_ct(environ))) if environ["dl"] or fmt == "zip": suffix = "zip" if fmt == "zip" else "txt" - headers.append(("Content-type", "application/octet-stream")) headers.append( ("Content-disposition", f"attachment; filename=afos.{suffix}") ) - else: - if fmt == "text": - headers.append(("Content-type", "text/plain")) - elif fmt == "html": - headers.append(("Content-type", "text/html")) start_response("200 OK", headers) sio = StringIO() @@ -312,4 +347,4 @@ def application(environ, start_response): if cursor.rowcount == 0: sio.write(f"ERROR: Could not Find: {','.join(pils)}") - return [sio.getvalue().encode("ascii", "ignore")] + return sio.getvalue()