From 8683d08d1c6006be6503205991c73fb27e4dd179 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Wed, 5 Feb 2025 15:29:04 -0500 Subject: [PATCH 1/6] First attempt at hash-based feed tracking --- discorss.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/discorss.py b/discorss.py index 46224b3..bd1f182 100755 --- a/discorss.py +++ b/discorss.py @@ -12,6 +12,7 @@ import requests import feedparser +import hashlib from pathlib import Path import json import time @@ -59,12 +60,20 @@ def main(): app_config = json.load(config_file) now = time.mktime(time.localtime()) last_check = app_config["lastupdate"] - for hook in app_config["feeds"]: + for i, hook in enumerate(app_config["feeds"]): # Get the feed feed = feedparser.parse(hook["url"]) published_time = time.mktime(feed.entries[0]["published_parsed"]) published_time = published_time + hook["offset"] print("Parsing feed {}...".format(hook["name"])) + new_hash = hashlib.sha3_512(feed.entries[0]["title"]) + try: + if hook["lasthash"] != new_hash: + app_config["feeds"][i]["lasthash"] = new_hash + else: + continue + except KeyError: + app_config["feeds"][i]["lasthash"] = new_hash # Generate the webhook webhook = { "embeds": [ From bd693f6f42a11b38c23d82731486b5fb53107c2e Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Wed, 5 Feb 2025 23:27:49 -0500 Subject: [PATCH 2/6] Added check for non-existant lastupdate key --- .gitignore | 3 ++- discorss.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f8ec04c..d48e085 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.conf *.txt -log/ \ No newline at end of file +log/ +*.bak \ No newline at end of file diff --git a/discorss.py b/discorss.py index bd1f182..b869e7c 100755 --- a/discorss.py +++ b/discorss.py @@ -59,7 +59,10 @@ def main(): with open(config_file_path, "r") as config_file: app_config = json.load(config_file) now = time.mktime(time.localtime()) - last_check = app_config["lastupdate"] + try: + last_check = app_config["lastupdate"] + except KeyError: + last_check = now - 21600 # first run, no lastupdate, check up to 6 hours ago for i, hook in enumerate(app_config["feeds"]): # Get the feed feed = feedparser.parse(hook["url"]) From 87193d0f9402a3578e4b6e18952338d45dcc48da Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Wed, 5 Feb 2025 23:28:14 -0500 Subject: [PATCH 3/6] Added sha3_512 hash of post title, to migrate from using time --- discorss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/discorss.py b/discorss.py index b869e7c..7eb752e 100755 --- a/discorss.py +++ b/discorss.py @@ -69,7 +69,8 @@ def main(): published_time = time.mktime(feed.entries[0]["published_parsed"]) published_time = published_time + hook["offset"] print("Parsing feed {}...".format(hook["name"])) - new_hash = hashlib.sha3_512(feed.entries[0]["title"]) + # Hash the title of the latest post and use that to determine if it's been posted + new_hash = hashlib.sha3_512(bytes(feed.entries[0]["title"], 'utf-8')).hexdigest() try: if hook["lasthash"] != new_hash: app_config["feeds"][i]["lasthash"] = new_hash From a188f8ee5d129842842f52297548753b36fcd536 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Wed, 5 Feb 2025 23:40:27 -0500 Subject: [PATCH 4/6] Removed part of time check --- discorss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discorss.py b/discorss.py index 7eb752e..6cff9c7 100755 --- a/discorss.py +++ b/discorss.py @@ -105,8 +105,8 @@ def main(): "content-type": "application/json", } webhook_string = json.dumps(webhook) - # print(webhook_string) - if published_time > last_check and published_time < now: + + if published_time > last_check: r = requests.post( hook["webhook"], data=webhook_string, headers=custom_header ) From 630560405624b9716c04c0484a8f64379ceccd9f Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Thu, 6 Feb 2025 04:53:55 -0500 Subject: [PATCH 5/6] Updated gitignore to ignore pyvenv files --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d48e085..fed12ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ *.conf *.txt log/ -*.bak \ No newline at end of file +*.bak +bin/ +lib/ +*.cfg \ No newline at end of file From d15578f7b2aef7220337fb06f0f827c131d6c129 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Fri, 7 Feb 2025 20:24:14 -0500 Subject: [PATCH 6/6] Changed formatting, added some comments, moved config file location --- discorss.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/discorss.py b/discorss.py index 3cd9ce6..be76e69 100755 --- a/discorss.py +++ b/discorss.py @@ -19,29 +19,41 @@ import time import os import re -config_file_path = r"/etc/discorss.conf" -# config_file_path = r"discorss.conf" +config_dir = os.environ.get('XDG_CONFIG_HOME') +if config_dir is None: + config_file_path = r"~/.config/discorss/discorss.conf" + config_dir = r"~/.config/discorss" +else: + config_file_path = config_dir + r"/discorss/discorss.conf" log_file_path = r"/var/log/discorss" # log_file_path = r"./log" log_file_name = r"/app.log" # Yes, I know you "can't parse HTML with regex", but # just watch me. html_filter = re.compile(r"\<\/?([A-Za-z \"\=])*\>") -success_codes = ['200', '201', '202', '203', '204', '205', '206'] +success_codes = ["200", "201", "202", "203", "204", "205", "206"] +# This function gets and formats the brief excerpt that goes in the embed +# Different feeds put summaries in different fields, so we pick the best +# one and limit it to 150 characters. +# TODO: make the character limit smarter, as to split at a natural point def get_description(feed): try: temporary_string = str(feed.entries[0]["summary_detail"]["value"]) temporary_string = html_filter.sub("", temporary_string) desc = ( - temporary_string[:150] if len(temporary_string) > 150 else temporary_string + temporary_string[:150] + if len(temporary_string) > 150 + else temporary_string ) except KeyError: temporary_string = str(feed.entries[0]["description"]) temporary_string = html_filter.sub("", temporary_string) desc = ( - temporary_string[:150] if len(temporary_string) > 150 else temporary_string + temporary_string[:150] + if len(temporary_string) > 150 + else temporary_string ) return desc @@ -49,13 +61,14 @@ def get_description(feed): def main(): os.environ["TZ"] = "America/Toronto" time.tzset() + # Check for log and config files/paths, create empty directories if needed try: Path(log_file_path).mkdir(parents=True, exist_ok=True) except FileExistsError: - print("This path already exists and is not a directory!") - # Load and read the config file + print("The logfile path {} already exists and is not a directory!".format(log_file_path)) if not Path(config_file_path).exists(): - print("No config file! Snarf. Directories were created for you.") + print("No config file at {}! Snarf.\n{} was created for you.".format(config_file_path, config_dir)) + Path(config_file_path).mkdir(parents=True, exist_ok=True) return with open(config_file_path, "r") as config_file: app_config = json.load(config_file) @@ -63,7 +76,9 @@ def main(): try: last_check = app_config["lastupdate"] except KeyError: - last_check = now - 21600 # first run, no lastupdate, check up to 6 hours ago + last_check = ( + now - 21600 + ) # first run, no lastupdate, check up to 6 hours ago for i, hook in enumerate(app_config["feeds"]): # Get the feed feed = feedparser.parse(hook["url"]) @@ -71,7 +86,9 @@ def main(): published_time = published_time + hook["offset"] print("Parsing feed {}...".format(hook["name"])) # Hash the title of the latest post and use that to determine if it's been posted - new_hash = hashlib.sha3_512(bytes(feed.entries[0]["title"], 'utf-8')).hexdigest() + new_hash = hashlib.sha3_512( + bytes(feed.entries[0]["title"], "utf-8") + ).hexdigest() try: if hook["lasthash"] != new_hash: app_config["feeds"][i]["lasthash"] = new_hash @@ -90,7 +107,10 @@ def main(): "name": "DiscoRSS", # "url": "https://git.frzn.dev/amr/discorss", }, - "author": {"name": str(hook["name"]), "url": str(hook["siteurl"])}, + "author": { + "name": str(hook["name"]), + "url": str(hook["siteurl"]), + }, "fields": [ { "name": "Excerpt from post:",