diff --git a/discorss.py b/discorss.py index bf431b9..be0ce2c 100755 --- a/discorss.py +++ b/discorss.py @@ -28,7 +28,8 @@ from types import SimpleNamespace class Discorss: FEED_TIMEOUT_SECONDS = 15 HASH_HISTORY_LIMIT = 10 - APP_VERSION = "0.3rc1" + APP_VERSION = "0.3rc2" + IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".gif", ".webp") def __init__(self, args=None): if args is None: @@ -54,8 +55,10 @@ class Discorss: # Yes, I know you "can't parse HTML with regex", but # just watch me. self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>") + self.img_src_filter = re.compile(r']+src=["\']([^"\']+)["\']', re.I) self.success_codes = [200, 201, 202, 203, 204, 205, 206] self.app_config = {} + print(f"Logging to {self.log_file_path}") async def _fetch_feed(self, hook): response = await asyncio.to_thread( @@ -96,20 +99,20 @@ class Discorss: async def _process_feed(self, hook, last_check): self.logger.debug("Parsing feed %s...", hook["name"]) - feeds = await self._fetch_feed(hook) + feed = await self._fetch_feed(hook) latest_post = None prev_best = 0 bad_time = False self.logger.debug("About to sort through entries for feed %s ...", hook["name"]) - for feed in feeds["entries"]: + for post in feed["entries"]: try: - published_time = time.mktime(feed["published_parsed"]) + published_time = time.mktime(post["published_parsed"]) published_time = published_time + hook["offset"] except KeyError: - published_time = time.mktime(feed["updated_parsed"]) + published_time = time.mktime(post["updated_parsed"]) bad_time = True if published_time > prev_best: - latest_post = feed + latest_post = post prev_best = published_time if latest_post is None: @@ -142,29 +145,37 @@ class Discorss: last_check, self.now, ) - webhook = { - "embeds": [ + embed = { + "title": str(latest_post["title"]), + "url": str(latest_post["link"]), + "color": 2123412, + "footer": { + "text": "DiscoRSS", + "icon_url": "https://frzn.dev/~amr/images/discorss.png", + }, + "author": { + "name": str(hook["name"]), + "url": str(hook["siteurl"]), + }, + "fields": [ { - "title": str(latest_post["title"]), - "url": str(latest_post["link"]), - "color": 2123412, - "footer": { - "text": "DiscoRSS", - "icon_url": "https://frzn.dev/~amr/images/discorss.png", - }, - "author": { - "name": str(hook["name"]), - "url": str(hook["siteurl"]), - }, - "fields": [ - { - "name": "Excerpt from post:", - "value": self.get_description(latest_post), - } - ], - # "timestamp": str(self.now), + "name": "Excerpt from post:", + "value": self.get_description(latest_post), } ], + # "timestamp": str(self.now), + } + self.logger.debug( + "Checking for images in post %s from %s...", + latest_post["title"], + hook["name"], + ) + image_url = self.get_image_url(latest_post) + if image_url is not None: + embed["thumbnail"] = {"url": image_url} + + webhook = { + "embeds": [embed], "attachments": [], } custom_header = { @@ -241,9 +252,9 @@ class Discorss: # This function gets and formats the brief excerpt that goes in the embed # Different feeds put summaries in different fields, so we pick the best # one and limit it to 250 characters. - def get_description(self, feed, length=250, min_length=150, addons=None): + def get_description(self, post, length=250, min_length=150, addons=None): try: - temporary_string = str(feed["summary_detail"]["value"]) + temporary_string = str(post["summary_detail"]["value"]) temporary_string = self.html_filter.sub("", temporary_string) while length > min_length: if temporary_string[length - 1 : length] == " ": @@ -251,7 +262,7 @@ class Discorss: else: length -= 1 except KeyError: - temporary_string = str(feed["description"]) + temporary_string = str(post["description"]) temporary_string = self.html_filter.sub("", temporary_string) while length > min_length: if temporary_string[length - 1 : length] == " ": @@ -264,6 +275,58 @@ class Discorss: desc = desc + str(addons) return desc + # attempting to extract image previews from feeds which primarily feature + # images, like NASA's Picture of the Day feed + def get_image_url(self, post): + image_candidates = [] + # check the most common fields, this should catch the majority of image + # feeds' embedded urls + for media in post.get("media_content", []): + if self.is_image_url(media.get("url"), media.get("type")): + image_candidates.append(media["url"]) + + for enclosure in post.get("enclosures", []): + if self.is_image_url(enclosure.get("href"), enclosure.get("type")): + image_candidates.append(enclosure["href"]) + + for link in post.get("links", []): + if self.is_image_url(link.get("href"), link.get("type")): + image_candidates.append(link["href"]) + + for media in post.get("media_thumbnail", []): + if self.is_image_url(media.get("url"), media.get("type")): + image_candidates.append(media["url"]) + + for field in ["summary_detail", "content"]: + value = post.get(field) + if isinstance(value, list): + values = [item.get("value", "") for item in value] + elif isinstance(value, dict): + values = [value.get("value", "")] + else: + values = [] + for text in values: + match = self.img_src_filter.search(str(text)) + if match and self.is_image_url(match.group(1)): + image_candidates.append(match.group(1)) + self.logger.debug("Found the following image candidates in %s...", post["title"]) + for i in image_candidates: + self.logger.debug("%s", i) + if len(image_candidates) > 0: + return image_candidates[0] + return None + + # a silly little helper just to validate image links + # this isn't 100% foolproof but it should work for the + # vast majority of feeds out there, unless they use some + # really weird image type like image/bpg + def is_image_url(self, url, mime_type=None): + if not url: + return False + if mime_type and str(mime_type).lower().startswith("image/"): + return True + return str(url).lower().split("?", 1)[0].endswith(self.IMAGE_EXTENSIONS) + # Some of this could go in __init__ def setup(self): os.environ["TZ"] = "America/Toronto" @@ -274,7 +337,7 @@ class Discorss: logging.basicConfig( filename=self.log_file_path, encoding="utf-8", - level=logging.WARNING, + level=logging.DEBUG, datefmt="%m/%d/%Y %H:%M:%S", format="%(asctime)s [%(threadName)s] -> %(levelname)s: %(message)s", ) @@ -329,6 +392,8 @@ class Discorss: self.app_config["lastupdate"] = self.now with open(self.config_file_path, "w") as config_file: json.dump(self.app_config, config_file, indent=4) + + self.logger.info("Stopping DiscoRSS version {}...".format(self.APP_VERSION)) return