From d412c1a378d81b49bff406b179fdd073d0fff74a Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Fri, 24 Apr 2026 10:23:18 -0400 Subject: [PATCH 1/4] hash: updated hash to use URL. also added DRY_RUN option --- discorss.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/discorss.py b/discorss.py index c4dc604..7e4f279 100755 --- a/discorss.py +++ b/discorss.py @@ -26,6 +26,7 @@ import re class Discorss: FEED_TIMEOUT_SECONDS = 15 + DRY_RUN = True def __init__(self): self.config_dir = os.environ.get("XDG_CONFIG_HOME") @@ -66,7 +67,6 @@ class Discorss: self.logger.debug("Parsing feed %s...", hook["name"]) feeds = await self._fetch_feed(hook) latest_post = None - latest_post_time = None prev_best = 0 bad_time = False self.logger.debug("About to sort through entries for feed %s ...", hook["name"]) @@ -79,7 +79,6 @@ class Discorss: bad_time = True if published_time > prev_best: latest_post = feed - latest_post_time = published_time prev_best = published_time if latest_post is None: @@ -91,15 +90,16 @@ class Discorss: "Feed %s doesn't supply a published time, using updated time instead", hook["name"], ) - # Hash the title of the latest post and use that to determine if it's been posted + self.logger.debug("Feed url is %s", latest_post["url"]) + # Hash the url of the latest post and use that to determine if it's been posted # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 - self.logger.debug("About to hash %s ...", latest_post["title"]) + self.logger.debug("About to hash %s ...", latest_post["url"]) try: new_hash = hashlib.sha3_512( - bytes(latest_post["title"], "utf-8") # Removed time from hash + bytes(latest_post["url"], "utf-8") # Removed time from hash ).hexdigest() except TypeError: - self.logger.error("Title of %s isn't hashing correctly", hook["name"]) + self.logger.error("URL of %s isn't hashing correctly", hook["name"]) return None if hook.get("lasthash") == new_hash: @@ -144,7 +144,13 @@ class Discorss: webhook_string = json.dumps(webhook) self.logger.debug("About to run POST for %s", hook["name"]) - response = await self._post_webhook(hook, webhook_string, custom_header) + if not self.DRY_RUN: + response = await self._post_webhook(hook, webhook_string, custom_header) + else: + self.logger.debug( + "Dry run, not actually posting to webhook, faking return code 200" + ) + response.status_code = 200 if response.status_code not in self.success_codes: self.logger.error( "Error %d while trying to post %s", response.status_code, hook["name"] @@ -267,9 +273,9 @@ class Discorss: logging.basicConfig( filename=str(self.log_dir + self.log_file_path), encoding="utf-8", - level=logging.ERROR, + level=logging.DEBUG, datefmt="%m/%d/%Y %H:%M:%S", - format="%(asctime)s: %(levelname)s: %(message)s", + format="%(asctime)s -> %(levelname)s: %(message)s", ) return From 5a3e3333b369b2c3ad1e785d1e924f26de62cec3 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Fri, 24 Apr 2026 10:26:26 -0400 Subject: [PATCH 2/4] hash: added FIFO for hashes, should reduce duplicates --- discorss.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/discorss.py b/discorss.py index 7e4f279..c880a98 100755 --- a/discorss.py +++ b/discorss.py @@ -27,6 +27,7 @@ import re class Discorss: FEED_TIMEOUT_SECONDS = 15 DRY_RUN = True + HASH_HISTORY_LIMIT = 10 def __init__(self): self.config_dir = os.environ.get("XDG_CONFIG_HOME") @@ -63,6 +64,20 @@ class Discorss: timeout=self.FEED_TIMEOUT_SECONDS, ) + def _get_hash_history(self, hook): + # now we store a list of hashes 10 long + # this function checks if it's the old format and updates it if needed + existing_hashes = hook.get("lasthash", []) + if isinstance(existing_hashes, str): + return [existing_hashes] + if isinstance(existing_hashes, list): + return [ + saved_hash + for saved_hash in existing_hashes + if isinstance(saved_hash, str) + ] + return [] + async def _process_feed(self, hook, last_check): self.logger.debug("Parsing feed %s...", hook["name"]) feeds = await self._fetch_feed(hook) @@ -102,7 +117,7 @@ class Discorss: self.logger.error("URL of %s isn't hashing correctly", hook["name"]) return None - if hook.get("lasthash") == new_hash: + if new_hash in self._get_hash_history(hook): return None # Generate the webhook @@ -200,7 +215,11 @@ class Discorss: self.logger.info( "Feed %s has no existing hash, likely a new feed!", hook["name"] ) - self.app_config["feeds"][i]["lasthash"] = result + hash_history = self._get_hash_history(hook) + hash_history.append(result) + if len(hash_history) > self.HASH_HISTORY_LIMIT: + hash_history = hash_history[-self.HASH_HISTORY_LIMIT :] + self.app_config["feeds"][i]["lasthash"] = hash_history # This function gets and formats the brief excerpt that goes in the embed # Different feeds put summaries in different fields, so we pick the best From 98d1a3ba450bd781a4c0bca7fd3d10bd9d89ed8f Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Fri, 24 Apr 2026 10:31:07 -0400 Subject: [PATCH 3/4] args: implemented argparse for config file, log, and dry run --- discorss.py | 54 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/discorss.py b/discorss.py index c880a98..438a799 100755 --- a/discorss.py +++ b/discorss.py @@ -22,23 +22,34 @@ import os import sys import argparse import re +from types import SimpleNamespace class Discorss: FEED_TIMEOUT_SECONDS = 15 - DRY_RUN = True HASH_HISTORY_LIMIT = 10 - def __init__(self): + def __init__(self, args=None): + if args is None: + args = SimpleNamespace( + dry_run=False, + config_file=None, + log_file=None, + ) + + self.DRY_RUN = args.dry_run self.config_dir = os.environ.get("XDG_CONFIG_HOME") home_dir = Path.home() if self.config_dir is None: - self.config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" - self.config_dir = str(home_dir) + "/.config/discorss" + default_config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" else: - self.config_file_path = self.config_dir + r"/discorss/discorss.conf" - self.log_dir = r"/var/log/discorss" - self.log_file_path = r"/app.log" + default_config_file_path = self.config_dir + r"/discorss/discorss.conf" + self.config_file_path = args.config_file or default_config_file_path + self.config_dir = str(Path(self.config_file_path).parent) + + default_log_file_path = "/var/log/discorss/app.log" + self.log_file_path = args.log_file or default_log_file_path + self.log_dir = str(Path(self.log_file_path).parent) # Yes, I know you "can't parse HTML with regex", but # just watch me. self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>") @@ -165,7 +176,7 @@ class Discorss: self.logger.debug( "Dry run, not actually posting to webhook, faking return code 200" ) - response.status_code = 200 + response = SimpleNamespace(status_code=200) if response.status_code not in self.success_codes: self.logger.error( "Error %d while trying to post %s", response.status_code, hook["name"] @@ -290,7 +301,7 @@ class Discorss: # Set up logging self.logger = logging.getLogger(__name__) logging.basicConfig( - filename=str(self.log_dir + self.log_file_path), + filename=self.log_file_path, encoding="utf-8", level=logging.DEBUG, datefmt="%m/%d/%Y %H:%M:%S", @@ -321,7 +332,30 @@ class Discorss: def main(): - app = Discorss() + parser = argparse.ArgumentParser( + description="DiscoRSS: publish feed updates to Discord webhooks." + ) + parser.add_argument( + "-d", + "--dry-run", + action="store_true", + help="Parse feeds and update state without posting to Discord.", + ) + parser.add_argument( + "-c", + "--config-file", + default=None, + help="Alternate config file path. Defaults to the existing config location.", + ) + parser.add_argument( + "-l", + "--log-file", + default=None, + help="Alternate log file path. Defaults to the existing log location.", + ) + args = parser.parse_args() + + app = Discorss(args) app.process() From 63300e6012983e271f7006f3aa761eb6ced4ea14 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Fri, 24 Apr 2026 10:51:59 -0400 Subject: [PATCH 4/4] logging: update logging levels and added new messages --- discorss.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/discorss.py b/discorss.py index 438a799..a77dd48 100755 --- a/discorss.py +++ b/discorss.py @@ -116,16 +116,15 @@ class Discorss: "Feed %s doesn't supply a published time, using updated time instead", hook["name"], ) - self.logger.debug("Feed url is %s", latest_post["url"]) # Hash the url of the latest post and use that to determine if it's been posted # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 - self.logger.debug("About to hash %s ...", latest_post["url"]) + self.logger.debug("About to hash %s ...", latest_post["link"]) try: new_hash = hashlib.sha3_512( - bytes(latest_post["url"], "utf-8") # Removed time from hash + bytes(latest_post["link"], "utf-8") # Removed time from hash ).hexdigest() except TypeError: - self.logger.error("URL of %s isn't hashing correctly", hook["name"]) + self.logger.error("URL %s isn't hashing correctly", hook["link"]) return None if new_hash in self._get_hash_history(hook): @@ -173,7 +172,7 @@ class Discorss: if not self.DRY_RUN: response = await self._post_webhook(hook, webhook_string, custom_header) else: - self.logger.debug( + self.logger.info( "Dry run, not actually posting to webhook, faking return code 200" ) response = SimpleNamespace(status_code=200) @@ -200,14 +199,14 @@ class Discorss: for i, result in enumerate(results): hook = self.app_config["feeds"][i] if isinstance(result, asyncio.TimeoutError): - self.logger.error( + self.logger.critical( "Timed out processing feed %s after %d seconds", hook["name"], self.FEED_TIMEOUT_SECONDS, ) continue if isinstance(result, requests.RequestException): - self.logger.error( + self.logger.critical( "Network error while processing feed %s: %s", hook["name"], result, @@ -223,7 +222,7 @@ class Discorss: if result is None: continue if "lasthash" not in hook: - self.logger.info( + self.logger.debug( "Feed %s has no existing hash, likely a new feed!", hook["name"] ) hash_history = self._get_hash_history(hook) @@ -303,10 +302,11 @@ class Discorss: logging.basicConfig( filename=self.log_file_path, encoding="utf-8", - level=logging.DEBUG, + level=logging.INFO, datefmt="%m/%d/%Y %H:%M:%S", - format="%(asctime)s -> %(levelname)s: %(message)s", + format="%(asctime)s [%(threadName)s] -> %(levelname)s: %(message)s", ) + self.logger.info("========= Started discorss.py ==========") return def process(self): @@ -324,7 +324,7 @@ class Discorss: self.app_config["lastupdate"] = self.now with open(self.config_file_path, "w") as config_file: json.dump(self.app_config, config_file, indent=4) - + self.logger.info("========= Ended discord.py =========") return