logging: update logging levels and added new messages

discorss.py

@@ -22,21 +22,34 @@ import os
 import sys
 import argparse
 import re
+from types import SimpleNamespace
 
 
 class Discorss:
     FEED_TIMEOUT_SECONDS = 15
+    HASH_HISTORY_LIMIT = 10
 
-    def __init__(self):
+    def __init__(self, args=None):
+        if args is None:
+            args = SimpleNamespace(
+                dry_run=False,
+                config_file=None,
+                log_file=None,
+            )
+
+        self.DRY_RUN = args.dry_run
         self.config_dir = os.environ.get("XDG_CONFIG_HOME")
         home_dir = Path.home()
         if self.config_dir is None:
-            self.config_file_path = str(home_dir) + "/.config/discorss/discorss.conf"
-            self.config_dir = str(home_dir) + "/.config/discorss"
+            default_config_file_path = str(home_dir) + "/.config/discorss/discorss.conf"
         else:
-            self.config_file_path = self.config_dir + r"/discorss/discorss.conf"
-        self.log_dir = r"/var/log/discorss"
-        self.log_file_path = r"/app.log"
+            default_config_file_path = self.config_dir + r"/discorss/discorss.conf"
+        self.config_file_path = args.config_file or default_config_file_path
+        self.config_dir = str(Path(self.config_file_path).parent)
+
+        default_log_file_path = "/var/log/discorss/app.log"
+        self.log_file_path = args.log_file or default_log_file_path
+        self.log_dir = str(Path(self.log_file_path).parent)
         # Yes, I know you "can't parse HTML with regex", but
         # just watch me.
         self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>")
@@ -62,11 +75,24 @@ class Discorss:
             timeout=self.FEED_TIMEOUT_SECONDS,
         )
 
+    def _get_hash_history(self, hook):
+        # now we store a list of hashes 10 long
+        # this function checks if it's the old format and updates it if needed
+        existing_hashes = hook.get("lasthash", [])
+        if isinstance(existing_hashes, str):
+            return [existing_hashes]
+        if isinstance(existing_hashes, list):
+            return [
+                saved_hash
+                for saved_hash in existing_hashes
+                if isinstance(saved_hash, str)
+            ]
+        return []
+
     async def _process_feed(self, hook, last_check):
         self.logger.debug("Parsing feed %s...", hook["name"])
         feeds = await self._fetch_feed(hook)
         latest_post = None
-        latest_post_time = None
         prev_best = 0
         bad_time = False
         self.logger.debug("About to sort through entries for feed %s ...", hook["name"])
@@ -79,7 +105,6 @@ class Discorss:
                 bad_time = True
             if published_time > prev_best:
                 latest_post = feed
-                latest_post_time = published_time
                 prev_best = published_time
 
         if latest_post is None:
@@ -91,18 +116,18 @@ class Discorss:
                 "Feed %s doesn't supply a published time, using updated time instead",
                 hook["name"],
             )
-        # Hash the title of the latest post and use that to determine if it's been posted
+        # Hash the url of the latest post and use that to determine if it's been posted
         # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3
-        self.logger.debug("About to hash %s ...", latest_post["title"])
+        self.logger.debug("About to hash %s ...", latest_post["link"])
         try:
             new_hash = hashlib.sha3_512(
-                bytes(latest_post["title"], "utf-8")  # Removed time from hash
+                bytes(latest_post["link"], "utf-8")  # Removed time from hash
             ).hexdigest()
         except TypeError:
-            self.logger.error("Title of %s isn't hashing correctly", hook["name"])
+            self.logger.error("URL %s isn't hashing correctly", hook["link"])
             return None
 
-        if hook.get("lasthash") == new_hash:
+        if new_hash in self._get_hash_history(hook):
             return None
 
         # Generate the webhook
@@ -144,7 +169,13 @@ class Discorss:
         webhook_string = json.dumps(webhook)
 
         self.logger.debug("About to run POST for %s", hook["name"])
-        response = await self._post_webhook(hook, webhook_string, custom_header)
+        if not self.DRY_RUN:
+            response = await self._post_webhook(hook, webhook_string, custom_header)
+        else:
+            self.logger.info(
+                "Dry run, not actually posting to webhook, faking return code 200"
+            )
+            response = SimpleNamespace(status_code=200)
         if response.status_code not in self.success_codes:
             self.logger.error(
                 "Error %d while trying to post %s", response.status_code, hook["name"]
@@ -168,14 +199,14 @@ class Discorss:
         for i, result in enumerate(results):
             hook = self.app_config["feeds"][i]
             if isinstance(result, asyncio.TimeoutError):
-                self.logger.error(
+                self.logger.critical(
                     "Timed out processing feed %s after %d seconds",
                     hook["name"],
                     self.FEED_TIMEOUT_SECONDS,
                 )
                 continue
             if isinstance(result, requests.RequestException):
-                self.logger.error(
+                self.logger.critical(
                     "Network error while processing feed %s: %s",
                     hook["name"],
                     result,
@@ -191,10 +222,14 @@ class Discorss:
             if result is None:
                 continue
             if "lasthash" not in hook:
-                self.logger.info(
+                self.logger.debug(
                     "Feed %s has no existing hash, likely a new feed!", hook["name"]
                 )
-            self.app_config["feeds"][i]["lasthash"] = result
+            hash_history = self._get_hash_history(hook)
+            hash_history.append(result)
+            if len(hash_history) > self.HASH_HISTORY_LIMIT:
+                hash_history = hash_history[-self.HASH_HISTORY_LIMIT :]
+            self.app_config["feeds"][i]["lasthash"] = hash_history
 
     # This function gets and formats the brief excerpt that goes in the embed
     # Different feeds put summaries in different fields, so we pick the best
@@ -265,12 +300,13 @@ class Discorss:
         # Set up logging
         self.logger = logging.getLogger(__name__)
         logging.basicConfig(
-            filename=str(self.log_dir + self.log_file_path),
+            filename=self.log_file_path,
             encoding="utf-8",
-            level=logging.ERROR,
+            level=logging.INFO,
             datefmt="%m/%d/%Y %H:%M:%S",
-            format="%(asctime)s: %(levelname)s: %(message)s",
+            format="%(asctime)s [%(threadName)s] -> %(levelname)s: %(message)s",
         )
+        self.logger.info("========= Started discorss.py ==========")
         return
 
     def process(self):
@@ -288,7 +324,7 @@ class Discorss:
         self.app_config["lastupdate"] = self.now
         with open(self.config_file_path, "w") as config_file:
             json.dump(self.app_config, config_file, indent=4)
-
+        self.logger.info("========= Ended discord.py =========")
         return
 
 
@@ -296,7 +332,30 @@ class Discorss:
 
 
 def main():
-    app = Discorss()
+    parser = argparse.ArgumentParser(
+        description="DiscoRSS: publish feed updates to Discord webhooks."
+    )
+    parser.add_argument(
+        "-d",
+        "--dry-run",
+        action="store_true",
+        help="Parse feeds and update state without posting to Discord.",
+    )
+    parser.add_argument(
+        "-c",
+        "--config-file",
+        default=None,
+        help="Alternate config file path. Defaults to the existing config location.",
+    )
+    parser.add_argument(
+        "-l",
+        "--log-file",
+        default=None,
+        help="Alternate log file path. Defaults to the existing log location.",
+    )
+    args = parser.parse_args()
+
+    app = Discorss(args)
     app.process()