From 52e1b4b93c5046bed7dc157604b5d839dd36994a Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Mon, 25 May 2026 12:38:01 +1000 Subject: [PATCH] Rename downloader.download_xl to download_xlsx and drop dead dict keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small related cleanups around the AEMO registration list download, completing the xls→xlsx work begun by #60 (defaults filename + URL pieces already done by PR #72's ebf27ea): 1. Rename `download_xl` → `download_xlsx` in downloader.py. The function downloads the NEM Registration and Exemption List, which AEMO ships as a .xlsx file (since the #60 fix). The old "xl" name was ambiguous; "xlsx" matches what's actually on the wire. PR #67's e5cac94 (bundled inside the Session refactor) renamed it to `download_xml` — that's wrong, the file isn't XML. This PR takes the rename Matt intended but uses the correct name. 2. Fix the docstring, which claimed the function "downloads a zipped csv using a url, extracts the csv and saves it a specified location". The function body is just `requests.get` + `open.write` — no zip, no CSV, no extraction. Replaced with what it actually does. 3. Remove two dead entries keyed by the literal string "_downloader.download_xl": - `defaults.static_table_url["_downloader.download_xl"]` - `data_fetch_methods.static_downloader_map["_downloader.download_xl"]` Both dicts are looked up by table-name (e.g. "Generators and Scheduled Loads", "VARIABLES_FCAS_4_SECOND") — nothing ever queries them with a Python identifier as the key. `git grep` confirms no consumers. PR #72's xlsx fix just updated the URL value of the defaults entry; this drops the entry entirely. Existing static-table tests in tests/end_to_end_table_tests/test_static_tables.py exercise the renamed function path via the offline mock server — all 8 pass after the rename. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/nemosis/data_fetch_methods.py | 3 +-- src/nemosis/defaults.py | 1 - src/nemosis/downloader.py | 6 +++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nemosis/data_fetch_methods.py b/src/nemosis/data_fetch_methods.py index 0aeb6f9..5ed1c62 100644 --- a/src/nemosis/data_fetch_methods.py +++ b/src/nemosis/data_fetch_methods.py @@ -446,8 +446,7 @@ def _finalise_csv_data(data, table_name): static_downloader_map = { "VARIABLES_FCAS_4_SECOND": _downloader.download_csv, "ELEMENTS_FCAS_4_SECOND": _downloader.download_elements_file, - "Generators and Scheduled Loads": _downloader.download_xl, - "_downloader.download_xl": _downloader.download_xl, + "Generators and Scheduled Loads": _downloader.download_xlsx, } static_file_reader_map = { diff --git a/src/nemosis/defaults.py b/src/nemosis/defaults.py index 9c199b2..1027ae1 100644 --- a/src/nemosis/defaults.py +++ b/src/nemosis/defaults.py @@ -134,7 +134,6 @@ "ELEMENTS_FCAS_4_SECOND": "https://www.nemweb.com.au/Reports/Current/Causer_Pays_Elements/", "VARIABLES_FCAS_4_SECOND": "https://aemo.com.au/-/media/files/electricity/nem/settlements_and_payments/settlements/auction-reports/archive/ancillary-services-market-causer-pays-variables-file.csv", "Generators and Scheduled Loads": "https://www.aemo.com.au/-/media/files/electricity/nem/Participant_Information/NEM-Registration-and-Exemption-List.xlsx", - "_downloader.download_xl": "https://www.aemo.com.au/-/media/files/electricity/nem/Participant_Information/NEM-Registration-and-Exemption-List.xlsx", } aemo_mms_url = "http://www.nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/{}/MMSDM_{}_{}/MMSDM_Historical_Data_SQLLoader/DATA/{}.zip" diff --git a/src/nemosis/downloader.py b/src/nemosis/downloader.py index dac391c..4137cc3 100644 --- a/src/nemosis/downloader.py +++ b/src/nemosis/downloader.py @@ -280,10 +280,10 @@ def download_elements_file(url, path_and_name): f.write(r.content) -def download_xl(url, path_and_name): +def download_xlsx(url, path_and_name): """ - This function downloads a zipped csv using a url, extracts the csv and - saves it a specified location + Download an Excel (.xlsx) file from a URL and save it to the + specified path. Used for AEMO's NEM Registration and Exemption List. """ r = requests.get(url, headers=USR_AGENT_HEADER) with open(path_and_name, "wb") as f: