logging: add stop message, change default level

discorss.py

@@ -28,7 +28,8 @@ from types import SimpleNamespace
 class Discorss:
     FEED_TIMEOUT_SECONDS = 15
     HASH_HISTORY_LIMIT = 10
-    APP_VERSION = "0.3rc1"
+    APP_VERSION = "0.3rc2"
+    IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".gif", ".webp")
 
     def __init__(self, args=None):
         if args is None:
@@ -54,8 +55,10 @@ class Discorss:
         # Yes, I know you "can't parse HTML with regex", but
         # just watch me.
         self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>")
+        self.img_src_filter = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.I)
         self.success_codes = [200, 201, 202, 203, 204, 205, 206]
         self.app_config = {}
+        print(f"Logging to {self.log_file_path}")
 
     async def _fetch_feed(self, hook):
         response = await asyncio.to_thread(
@@ -96,20 +99,20 @@ class Discorss:
 
     async def _process_feed(self, hook, last_check):
         self.logger.debug("Parsing feed %s...", hook["name"])
-        feeds = await self._fetch_feed(hook)
+        feed = await self._fetch_feed(hook)
         latest_post = None
         prev_best = 0
         bad_time = False
         self.logger.debug("About to sort through entries for feed %s ...", hook["name"])
-        for feed in feeds["entries"]:
+        for post in feed["entries"]:
             try:
-                published_time = time.mktime(feed["published_parsed"])
+                published_time = time.mktime(post["published_parsed"])
                 published_time = published_time + hook["offset"]
             except KeyError:
-                published_time = time.mktime(feed["updated_parsed"])
+                published_time = time.mktime(post["updated_parsed"])
                 bad_time = True
             if published_time > prev_best:
-                latest_post = feed
+                latest_post = post
                 prev_best = published_time
 
         if latest_post is None:
@@ -142,29 +145,37 @@ class Discorss:
             last_check,
             self.now,
         )
-        webhook = {
+        embed = {
-            "embeds": [
+            "title": str(latest_post["title"]),
+            "url": str(latest_post["link"]),
+            "color": 2123412,
+            "footer": {
+                "text": "DiscoRSS",
+                "icon_url": "https://frzn.dev/~amr/images/discorss.png",
+            },
+            "author": {
+                "name": str(hook["name"]),
+                "url": str(hook["siteurl"]),
+            },
+            "fields": [
                 {
-                    "title": str(latest_post["title"]),
+                    "name": "Excerpt from post:",
-                    "url": str(latest_post["link"]),
+                    "value": self.get_description(latest_post),
-                    "color": 2123412,
-                    "footer": {
-                        "text": "DiscoRSS",
-                        "icon_url": "https://frzn.dev/~amr/images/discorss.png",
-                    },
-                    "author": {
-                        "name": str(hook["name"]),
-                        "url": str(hook["siteurl"]),
-                    },
-                    "fields": [
-                        {
-                            "name": "Excerpt from post:",
-                            "value": self.get_description(latest_post),
-                        }
-                    ],
-                    # "timestamp": str(self.now),
                 }
             ],
+            # "timestamp": str(self.now),
+        }
+        self.logger.debug(
+            "Checking for images in post %s from %s...",
+            latest_post["title"],
+            hook["name"],
+        )
+        image_url = self.get_image_url(latest_post)
+        if image_url is not None:
+            embed["thumbnail"] = {"url": image_url}
+
+        webhook = {
+            "embeds": [embed],
             "attachments": [],
         }
         custom_header = {
@@ -241,9 +252,9 @@ class Discorss:
     # This function gets and formats the brief excerpt that goes in the embed
     # Different feeds put summaries in different fields, so we pick the best
     # one and limit it to 250 characters.
-    def get_description(self, feed, length=250, min_length=150, addons=None):
+    def get_description(self, post, length=250, min_length=150, addons=None):
         try:
-            temporary_string = str(feed["summary_detail"]["value"])
+            temporary_string = str(post["summary_detail"]["value"])
             temporary_string = self.html_filter.sub("", temporary_string)
             while length > min_length:
                 if temporary_string[length - 1 : length] == " ":
@@ -251,7 +262,7 @@ class Discorss:
                 else:
                     length -= 1
         except KeyError:
-            temporary_string = str(feed["description"])
+            temporary_string = str(post["description"])
             temporary_string = self.html_filter.sub("", temporary_string)
             while length > min_length:
                 if temporary_string[length - 1 : length] == " ":
@@ -264,6 +275,58 @@ class Discorss:
             desc = desc + str(addons)
         return desc
 
+    # attempting to extract image previews from feeds which primarily feature
+    # images, like NASA's Picture of the Day feed
+    def get_image_url(self, post):
+        image_candidates = []
+        # check the most common fields, this should catch the majority of image
+        # feeds' embedded urls
+        for media in post.get("media_content", []):
+            if self.is_image_url(media.get("url"), media.get("type")):
+                image_candidates.append(media["url"])
+
+        for enclosure in post.get("enclosures", []):
+            if self.is_image_url(enclosure.get("href"), enclosure.get("type")):
+                image_candidates.append(enclosure["href"])
+
+        for link in post.get("links", []):
+            if self.is_image_url(link.get("href"), link.get("type")):
+                image_candidates.append(link["href"])
+
+        for media in post.get("media_thumbnail", []):
+            if self.is_image_url(media.get("url"), media.get("type")):
+                image_candidates.append(media["url"])
+
+        for field in ["summary_detail", "content"]:
+            value = post.get(field)
+            if isinstance(value, list):
+                values = [item.get("value", "") for item in value]
+            elif isinstance(value, dict):
+                values = [value.get("value", "")]
+            else:
+                values = []
+            for text in values:
+                match = self.img_src_filter.search(str(text))
+                if match and self.is_image_url(match.group(1)):
+                    image_candidates.append(match.group(1))
+        self.logger.debug("Found the following image candidates in %s...", post["title"])
+        for i in image_candidates:
+            self.logger.debug("%s", i)
+        if len(image_candidates) > 0:
+            return image_candidates[0]
+        return None
+
+    # a silly little helper just to validate image links
+    # this isn't 100% foolproof but it should work for the
+    # vast majority of feeds out there, unless they use some
+    # really weird image type like image/bpg
+    def is_image_url(self, url, mime_type=None):
+        if not url:
+            return False
+        if mime_type and str(mime_type).lower().startswith("image/"):
+            return True
+        return str(url).lower().split("?", 1)[0].endswith(self.IMAGE_EXTENSIONS)
+
     # Some of this could go in __init__
     def setup(self):
         os.environ["TZ"] = "America/Toronto"
@@ -274,7 +337,7 @@ class Discorss:
         logging.basicConfig(
             filename=self.log_file_path,
             encoding="utf-8",
-            level=logging.WARNING,
+            level=logging.DEBUG,
             datefmt="%m/%d/%Y %H:%M:%S",
             format="%(asctime)s [%(threadName)s] -> %(levelname)s: %(message)s",
         )
@@ -329,6 +392,8 @@ class Discorss:
         self.app_config["lastupdate"] = self.now
         with open(self.config_file_path, "w") as config_file:
             json.dump(self.app_config, config_file, indent=4)
+
+        self.logger.info("Stopping DiscoRSS version {}...".format(self.APP_VERSION))
         return