diff --git a/discorss.py b/discorss.py index c4dc604..a77dd48 100755 --- a/discorss.py +++ b/discorss.py @@ -22,21 +22,34 @@ import os import sys import argparse import re +from types import SimpleNamespace class Discorss: FEED_TIMEOUT_SECONDS = 15 + HASH_HISTORY_LIMIT = 10 - def __init__(self): + def __init__(self, args=None): + if args is None: + args = SimpleNamespace( + dry_run=False, + config_file=None, + log_file=None, + ) + + self.DRY_RUN = args.dry_run self.config_dir = os.environ.get("XDG_CONFIG_HOME") home_dir = Path.home() if self.config_dir is None: - self.config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" - self.config_dir = str(home_dir) + "/.config/discorss" + default_config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" else: - self.config_file_path = self.config_dir + r"/discorss/discorss.conf" - self.log_dir = r"/var/log/discorss" - self.log_file_path = r"/app.log" + default_config_file_path = self.config_dir + r"/discorss/discorss.conf" + self.config_file_path = args.config_file or default_config_file_path + self.config_dir = str(Path(self.config_file_path).parent) + + default_log_file_path = "/var/log/discorss/app.log" + self.log_file_path = args.log_file or default_log_file_path + self.log_dir = str(Path(self.log_file_path).parent) # Yes, I know you "can't parse HTML with regex", but # just watch me. self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>") @@ -62,11 +75,24 @@ class Discorss: timeout=self.FEED_TIMEOUT_SECONDS, ) + def _get_hash_history(self, hook): + # now we store a list of hashes 10 long + # this function checks if it's the old format and updates it if needed + existing_hashes = hook.get("lasthash", []) + if isinstance(existing_hashes, str): + return [existing_hashes] + if isinstance(existing_hashes, list): + return [ + saved_hash + for saved_hash in existing_hashes + if isinstance(saved_hash, str) + ] + return [] + async def _process_feed(self, hook, last_check): self.logger.debug("Parsing feed %s...", hook["name"]) feeds = await self._fetch_feed(hook) latest_post = None - latest_post_time = None prev_best = 0 bad_time = False self.logger.debug("About to sort through entries for feed %s ...", hook["name"]) @@ -79,7 +105,6 @@ class Discorss: bad_time = True if published_time > prev_best: latest_post = feed - latest_post_time = published_time prev_best = published_time if latest_post is None: @@ -91,18 +116,18 @@ class Discorss: "Feed %s doesn't supply a published time, using updated time instead", hook["name"], ) - # Hash the title of the latest post and use that to determine if it's been posted + # Hash the url of the latest post and use that to determine if it's been posted # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 - self.logger.debug("About to hash %s ...", latest_post["title"]) + self.logger.debug("About to hash %s ...", latest_post["link"]) try: new_hash = hashlib.sha3_512( - bytes(latest_post["title"], "utf-8") # Removed time from hash + bytes(latest_post["link"], "utf-8") # Removed time from hash ).hexdigest() except TypeError: - self.logger.error("Title of %s isn't hashing correctly", hook["name"]) + self.logger.error("URL %s isn't hashing correctly", hook["link"]) return None - if hook.get("lasthash") == new_hash: + if new_hash in self._get_hash_history(hook): return None # Generate the webhook @@ -144,7 +169,13 @@ class Discorss: webhook_string = json.dumps(webhook) self.logger.debug("About to run POST for %s", hook["name"]) - response = await self._post_webhook(hook, webhook_string, custom_header) + if not self.DRY_RUN: + response = await self._post_webhook(hook, webhook_string, custom_header) + else: + self.logger.info( + "Dry run, not actually posting to webhook, faking return code 200" + ) + response = SimpleNamespace(status_code=200) if response.status_code not in self.success_codes: self.logger.error( "Error %d while trying to post %s", response.status_code, hook["name"] @@ -168,14 +199,14 @@ class Discorss: for i, result in enumerate(results): hook = self.app_config["feeds"][i] if isinstance(result, asyncio.TimeoutError): - self.logger.error( + self.logger.critical( "Timed out processing feed %s after %d seconds", hook["name"], self.FEED_TIMEOUT_SECONDS, ) continue if isinstance(result, requests.RequestException): - self.logger.error( + self.logger.critical( "Network error while processing feed %s: %s", hook["name"], result, @@ -191,10 +222,14 @@ class Discorss: if result is None: continue if "lasthash" not in hook: - self.logger.info( + self.logger.debug( "Feed %s has no existing hash, likely a new feed!", hook["name"] ) - self.app_config["feeds"][i]["lasthash"] = result + hash_history = self._get_hash_history(hook) + hash_history.append(result) + if len(hash_history) > self.HASH_HISTORY_LIMIT: + hash_history = hash_history[-self.HASH_HISTORY_LIMIT :] + self.app_config["feeds"][i]["lasthash"] = hash_history # This function gets and formats the brief excerpt that goes in the embed # Different feeds put summaries in different fields, so we pick the best @@ -265,12 +300,13 @@ class Discorss: # Set up logging self.logger = logging.getLogger(__name__) logging.basicConfig( - filename=str(self.log_dir + self.log_file_path), + filename=self.log_file_path, encoding="utf-8", - level=logging.ERROR, + level=logging.INFO, datefmt="%m/%d/%Y %H:%M:%S", - format="%(asctime)s: %(levelname)s: %(message)s", + format="%(asctime)s [%(threadName)s] -> %(levelname)s: %(message)s", ) + self.logger.info("========= Started discorss.py ==========") return def process(self): @@ -288,7 +324,7 @@ class Discorss: self.app_config["lastupdate"] = self.now with open(self.config_file_path, "w") as config_file: json.dump(self.app_config, config_file, indent=4) - + self.logger.info("========= Ended discord.py =========") return @@ -296,7 +332,30 @@ class Discorss: def main(): - app = Discorss() + parser = argparse.ArgumentParser( + description="DiscoRSS: publish feed updates to Discord webhooks." + ) + parser.add_argument( + "-d", + "--dry-run", + action="store_true", + help="Parse feeds and update state without posting to Discord.", + ) + parser.add_argument( + "-c", + "--config-file", + default=None, + help="Alternate config file path. Defaults to the existing config location.", + ) + parser.add_argument( + "-l", + "--log-file", + default=None, + help="Alternate log file path. Defaults to the existing log location.", + ) + args = parser.parse_args() + + app = Discorss(args) app.process()