From b72f1d72910e0d0c2310445a7f14bf379df4ef1d Mon Sep 17 00:00:00 2001 From: RoscoeDaWah Date: Fri, 14 Mar 2025 10:50:45 +0000 Subject: [PATCH 01/13] docs: add syntax highlighting to README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8fa07b9..06b28fe 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Logging was recently enabled. Make sure that the user running the script (especi To configure the script, create ~/.config/discorss/discorss.conf with the following structure: -``` +```json { "feeds": [ { @@ -54,7 +54,7 @@ To automate feed posting, create a systemd service and timer to execute the scri Use the command `systemctl --user edit --full --force discorss.service` and then paste in something like this: -``` +```systemd [Unit] Description=Discord RSS feeder Wants=discorss.timer @@ -68,7 +68,7 @@ WantedBy=default.target ``` Make sure to edit the ExecStart to point to the correct location. Then we need a systemd timer to automatically fire the script. Run `systemctl --user edit --full --force discorss.timer` and then paste in this: -``` +```systemd [Unit] Description=Timer for DiscoRSS Requires=discorss.service From ce71ef1e81b5dc391c180a9155a39a35bb13b60a Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sun, 16 Mar 2025 02:20:10 -0400 Subject: [PATCH 02/13] chore: Changed warning text, and some logging values --- discorss.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/discorss.py b/discorss.py index 555c3e2..56d5d16 100755 --- a/discorss.py +++ b/discorss.py @@ -89,7 +89,7 @@ def setupPaths(): Path(config_dir).mkdir(parents=True, exist_ok=True) except FileExistsError: print( - "The config dir {} already exists and is not a directory! Please fix manually.".format( + "The config dir {} already exists and is not a directory! Please fix manually. Quitting!".format( config_dir ) ) @@ -138,11 +138,12 @@ def main(): else: continue if bad_time is True: - logger.warning( + logger.debug( "Feed %s doesn't supply a published time, using updated time instead", hook["name"], ) # Hash the title and time of the latest post and use that to determine if it's been posted + # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 new_hash = hashlib.sha3_512( bytes(latest_post["title"] + str(published_time), "utf-8") ).hexdigest() From 0de8e237a0351e7e999611e8b27de652033e2bf7 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Wed, 16 Apr 2025 16:55:50 -0400 Subject: [PATCH 03/13] feat: created installation helper script This script will create a basic systemd user service and timer, and then optionally activate them. It will also create the config directory and optionally put in an example config using the RSS feed from Phoronix (the Linux news site). This will likely need some tweaking in the future. --- install.sh | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 install.sh diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..59b3ea1 --- /dev/null +++ b/install.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script will set up a basic systemd service and timer for DiscoRSS +# You can optionally edit the entries here before running it, or you can +# use systemctl --user edit --full discorss.service or discorss.timer +# after installing them. + +cat << EOF > discorss.service +[Unit] +Description=Discord RSS feeder +Wants=discorss.timer + +[Service] +Type=oneshot +ExecStart=/home/amr/workspace/python/discorss/discorss.py + +[Install] +WantedBy=default.target + +EOF + +cat << EOF > discorss.timer +[Unit] +Description=Timer for DiscoRSS +Requires=discorss.service + +[Timer] +Unit=discorss.service +OnCalendar=*:0/5:00 +AccuracySec=1s + +[Install] +WantedBy=timers.target + +EOF + +cp discorss.service ~/.config/systemd/user/ +cp discorss.timer ~/.config/systemd/user/ + +systemctl --user daemon-reload + +printf "Would you like a basic example config created for you? [y/n]" +read answer1 +if [ "$answer1" =~ ^[yYnN]$ ]; then + mkdir -p -v ~/.config/discorss + cat << EOF > ~/.config/discorss/discorss.conf +{ + "feeds": [ + { + "name": "Phoronix", + "siteurl": "https://www.phoronix.com/", + "url": "http://www.phoronix.com/rss.php", + "webhook": "PASTE WEBHOOK URL HERE", + "offset": 0, + } + ], +} +EOF + printf "Make sure to edit ~/.config/discorss/discorss.conf and add in your custom feeds and webhook URLS! The script will just error out if you don't do this." +else + printf "Make sure to create a config at ~/.config/discorss/discorss.conf and follow the pattern shown in the README." +fi + +printf "Would you like to have the timer enabled and started now? [y/n]" +read answer +if [ "$answer" =~ ^[yYnN]$ ]; then + systemctl --user enable --now discorss.timer + printf "discorss.timer enabled and started. Don't enable or start discorss.service -- the timer does this automatically." +else + printf "Don't forget to run systemctl --user enable --now discorss.timer when you are ready! Don't enable or start discorss.service -- the timer does this automatically." +fi + +printf "You should be almost ready to go! Double-check your config files, and check systemctl --user list-timers once the discorss.timer is enabled to see when it will fire next. The default is every 5 minutes." + +printf "Remember, if you need help or encounter any bugs, contact me via the issues tracker on the git repository where you got this from!" From f70b18040a07fa69b3487840f31eecb465610512 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sat, 19 Apr 2025 09:04:42 -0400 Subject: [PATCH 04/13] fix: wrapped hash in try/except to detect empty feeds Also changed file mode of install.sh to +x --- discorss.py | 18 ++++++++++++------ install.sh | 0 2 files changed, 12 insertions(+), 6 deletions(-) mode change 100644 => 100755 install.sh diff --git a/discorss.py b/discorss.py index 56d5d16..403b864 100755 --- a/discorss.py +++ b/discorss.py @@ -19,6 +19,7 @@ import json import time import os import sys +import argparse import re config_dir = os.environ.get("XDG_CONFIG_HOME") @@ -144,9 +145,13 @@ def main(): ) # Hash the title and time of the latest post and use that to determine if it's been posted # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 - new_hash = hashlib.sha3_512( - bytes(latest_post["title"] + str(published_time), "utf-8") - ).hexdigest() + try: + new_hash = hashlib.sha3_512( + bytes(latest_post["title"] + str(published_time), "utf-8") + ).hexdigest() + except TypeError: + logger.error("Title of %s isn't hashing correctly", hook["name"]) + continue try: if hook["lasthash"] != new_hash: app_config["feeds"][i]["lasthash"] = new_hash @@ -169,10 +174,10 @@ def main(): { "title": str(latest_post["title"]), "url": str(latest_post["link"]), - "color": 216128, + "color": 2123412, "footer": { - "name": "DiscoRSS", - # "url": "https://git.frzn.dev/amr/discorss", + "text": "DiscoRSS", + "icon_url": "https://frzn.dev/~amr/images/discorss.png", }, "author": { "name": str(hook["name"]), @@ -184,6 +189,7 @@ def main(): "value": get_description(latest_post), } ], + # "timestamp": str(now), } ], "attachments": [], diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 From af51c317e28ee46d3707f6ad86a959dbe29f14cd Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sat, 19 Apr 2025 12:45:07 -0400 Subject: [PATCH 05/13] bug: trying to track down lock-up bug For some reason the script seems to be occasionally locking up, and then because the systemd service state is stuck in "starting" it never finishes which means the timer never gets reset. Adding some debug statements to try and figure out the cause. Also changed logging to DEBUG level. I'd much rather fix the bug but a timeout would also solve the issue. --- discorss.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/discorss.py b/discorss.py index 403b864..ab49694 100755 --- a/discorss.py +++ b/discorss.py @@ -104,7 +104,7 @@ def setupPaths(): logging.basicConfig( filename=str(log_dir + log_file_path), encoding="utf-8", - level=logging.INFO, + level=logging.DEBUG, datefmt="%m/%d/%Y %H:%M:%S", format="%(asctime)s: %(levelname)s: %(message)s", ) @@ -200,6 +200,7 @@ def main(): } webhook_string = json.dumps(webhook) + logger.debug("About to run POST for %s", hook["name"]) r = requests.post(hook["webhook"], data=webhook_string, headers=custom_header) if r.status_code not in success_codes: logger.error( @@ -211,6 +212,7 @@ def main(): # End of feed loop # Dump updated config back to json file + logger.debug("Dumping config back to %s", str(config_file_path)) app_config["lastupdate"] = now with open(config_file_path, "w") as config_file: json.dump(app_config, config_file, indent=4) From a62a0fcdc310747172895766e10ea19eca673700 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sun, 20 Apr 2025 13:30:43 -0400 Subject: [PATCH 06/13] debug: added more logging to narrow down issue --- discorss.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/discorss.py b/discorss.py index ab49694..76f243d 100755 --- a/discorss.py +++ b/discorss.py @@ -125,6 +125,7 @@ def main(): feeds = feedparser.parse(hook["url"]) latest_post = [] prev_best = 0 + logger.debug("About to sort through entries for feed %s ...", hook["name"]) for feed in feeds["entries"]: try: bad_time = False @@ -145,6 +146,7 @@ def main(): ) # Hash the title and time of the latest post and use that to determine if it's been posted # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 + logger.debug("About to hash %s ...", latest_post["title"]) try: new_hash = hashlib.sha3_512( bytes(latest_post["title"] + str(published_time), "utf-8") From 087a6339c8adf4df3c23d298fe1cbddcbdf23eef Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sun, 20 Apr 2025 16:06:16 -0400 Subject: [PATCH 07/13] class refactor: Squashed commit of the following: commit 597f3837244cce6f501cf16f96e2cac3a81c8ab2 Author: A.M. Rowsell Date: Sun Apr 20 16:04:00 2025 -0400 fix: typo in log setup, error when replacing w/self commit 810cbd6f3d32ca0c7f74ec90f17c0011661d4785 Author: A.M. Rowsell Date: Sun Apr 20 13:51:41 2025 -0400 refactor: transformed into class-based app --- discorss.py | 392 +++++++++++++++++++++++++++------------------------- 1 file changed, 204 insertions(+), 188 deletions(-) diff --git a/discorss.py b/discorss.py index 76f243d..d90c7fb 100755 --- a/discorss.py +++ b/discorss.py @@ -22,204 +22,220 @@ import sys import argparse import re -config_dir = os.environ.get("XDG_CONFIG_HOME") -home_dir = Path.home() -if config_dir is None: - config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" - config_dir = str(home_dir) + "/.config/discorss" -else: - config_file_path = config_dir + r"/discorss/discorss.conf" -log_dir = r"/var/log/discorss" -log_file_path = r"/app.log" -# Yes, I know you "can't parse HTML with regex", but -# just watch me. -html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>") -success_codes = [200, 201, 202, 203, 204, 205, 206] -app_config = {} -# IDEA: Consider making this into a class-based program -# This would solve a couple issues around global variables and generally -# make things a bit neater +class Discorss: + def __init__(self): + self.config_dir = os.environ.get("XDG_CONFIG_HOME") + home_dir = Path.home() + if self.config_dir is None: + self.config_file_path = str(home_dir) + "/.config/discorss/discorss.conf" + self.config_dir = str(home_dir) + "/.config/discorss" + else: + self.config_file_path = self.config_dir + r"/discorss/discorss.conf" + self.log_dir = r"/var/log/discorss" + self.log_file_path = r"/app.log" + # Yes, I know you "can't parse HTML with regex", but + # just watch me. + self.html_filter = re.compile(r"\<\/?([A-Za-z0-9 \:\.\-\/\"\=])*\>") + self.success_codes = [200, 201, 202, 203, 204, 205, 206] + self.app_config = {} - -# This function gets and formats the brief excerpt that goes in the embed -# Different feeds put summaries in different fields, so we pick the best -# one and limit it to 250 characters. -def get_description(feed, length=250, min_length=150, addons=None): - try: - temporary_string = str(feed["summary_detail"]["value"]) - temporary_string = html_filter.sub("", temporary_string) - while length > min_length: - if temporary_string[length - 1 : length] == " ": - break - else: - length -= 1 - except KeyError: - temporary_string = str(feed["description"]) - temporary_string = html_filter.sub("", temporary_string) - while length > min_length: - if temporary_string[length - 1 : length] == " ": - break - else: - length -= 1 - - desc = temporary_string[:length] - if addons is not None: - desc = desc + str(addons) - return desc - - -def setupPaths(): - global app_config - global logger - # Check for log and config files/paths, create empty directories if needed - # TODO: make this cleaner - if not Path(log_dir).exists(): - print("No log file path exists. Yark! We'll try and make {}...".format(log_dir)) + # This function gets and formats the brief excerpt that goes in the embed + # Different feeds put summaries in different fields, so we pick the best + # one and limit it to 250 characters. + def get_description(self, feed, length=250, min_length=150, addons=None): try: - Path(log_dir).mkdir(parents=True, exist_ok=True) - except FileExistsError: - print("The path {} already exists and is not a directory!".format(log_dir)) - if not Path(config_file_path).exists(): - print( - "No config file at {}! Snarf. We'll try and make {}...".format( - config_file_path, config_dir - ) - ) - try: - Path(config_dir).mkdir(parents=True, exist_ok=True) - except FileExistsError: + temporary_string = str(feed["summary_detail"]["value"]) + temporary_string = self.html_filter.sub("", temporary_string) + while length > min_length: + if temporary_string[length - 1 : length] == " ": + break + else: + length -= 1 + except KeyError: + temporary_string = str(feed["description"]) + temporary_string = self.html_filter.sub("", temporary_string) + while length > min_length: + if temporary_string[length - 1 : length] == " ": + break + else: + length -= 1 + + desc = temporary_string[:length] + if addons is not None: + desc = desc + str(addons) + return desc + + def setupPaths(self): + # Check for log and config files/paths, create empty directories if needed + # TODO: make this cleaner + if not Path(self.log_dir).exists(): print( - "The config dir {} already exists and is not a directory! Please fix manually. Quitting!".format( - config_dir + "No log file path exists. Yark! We'll try and make {}...".format( + self.log_dir ) ) - sys.exit(255) + try: + Path(self.log_dir).mkdir(parents=True, exist_ok=True) + except FileExistsError: + print( + "The path {} already exists and is not a directory!".format( + self.log_dir + ) + ) + if not Path(self.config_file_path).exists(): + print( + "No config file at {}! Snarf. We'll try and make {}...".format( + self.config_file_path, self.config_dir + ) + ) + try: + Path(self.config_dir).mkdir(parents=True, exist_ok=True) + except FileExistsError: + print( + "The config dir {} already exists and is not a directory! Please fix manually. Quitting!".format( + self.config_dir + ) + ) + sys.exit(255) + return + # Loading the config file + with open(self.config_file_path, "r") as config_file: + self.app_config = json.load(config_file) + # Set up logging + self.logger = logging.getLogger(__name__) + logging.basicConfig( + filename=str(self.log_dir + self.log_file_path), + encoding="utf-8", + level=logging.DEBUG, + datefmt="%m/%d/%Y %H:%M:%S", + format="%(asctime)s: %(levelname)s: %(message)s", + ) return - # Loading the config file - with open(config_file_path, "r") as config_file: - app_config = json.load(config_file) - # Set up logging - logger = logging.getLogger(__name__) - logging.basicConfig( - filename=str(log_dir + log_file_path), - encoding="utf-8", - level=logging.DEBUG, - datefmt="%m/%d/%Y %H:%M:%S", - format="%(asctime)s: %(levelname)s: %(message)s", - ) - return + + def process(self): + os.environ["TZ"] = "America/Toronto" + time.tzset() + now = time.mktime(time.localtime()) + self.setupPaths() # Handle the config and log paths + try: + last_check = self.app_config["lastupdate"] + except KeyError: + last_check = ( + now - 21600 + ) # first run, no lastupdate, check up to 6 hours ago + for i, hook in enumerate(self.app_config["feeds"]): # Feed loop start + self.logger.debug("Parsing feed %s...", hook["name"]) + self.feeds = feedparser.parse(hook["url"]) + self.latest_post = [] + prev_best = 0 + self.logger.debug( + "About to sort through entries for feed %s ...", hook["name"] + ) + for feed in self.feeds["entries"]: + try: + bad_time = False + published_time = time.mktime(feed["published_parsed"]) + published_time = published_time + hook["offset"] + except KeyError: + published_time = time.mktime(feed["updated_parsed"]) + bad_time = True + if published_time > prev_best: + latest_post = feed + prev_best = published_time + else: + continue + if bad_time is True: + self.logger.debug( + "Feed %s doesn't supply a published time, using updated time instead", + hook["name"], + ) + # Hash the title and time of the latest post and use that to determine if it's been posted + # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 + self.logger.debug("About to hash %s ...", latest_post["title"]) + try: + new_hash = hashlib.sha3_512( + bytes(latest_post["title"] + str(published_time), "utf-8") + ).hexdigest() + except TypeError: + self.logger.error("Title of %s isn't hashing correctly", hook["name"]) + continue + try: + if hook["lasthash"] != new_hash: + self.app_config["feeds"][i]["lasthash"] = new_hash + else: + continue + except KeyError: + self.app_config["feeds"][i]["lasthash"] = new_hash + self.logger.info( + "Feed %s has no existing hash, likely a new feed!", hook["name"] + ) + # Generate the webhook + self.logger.info( + "Publishing webhook for %s. Last check was %d, now is %d", + hook["name"], + last_check, + now, + ) + webhook = { + "embeds": [ + { + "title": str(latest_post["title"]), + "url": str(latest_post["link"]), + "color": 2123412, + "footer": { + "text": "DiscoRSS", + "icon_url": "https://frzn.dev/~amr/images/discorss.png", + }, + "author": { + "name": str(hook["name"]), + "url": str(hook["siteurl"]), + }, + "fields": [ + { + "name": "Excerpt from post:", + "value": self.get_description(latest_post), + } + ], + # "timestamp": str(now), + } + ], + "attachments": [], + } + custom_header = { + "user-agent": "DiscoRSS (https://git.frzn.dev/amr/discorss, 0.2rc3)", + "content-type": "application/json", + } + webhook_string = json.dumps(webhook) + + self.logger.debug("About to run POST for %s", hook["name"]) + r = requests.post( + hook["webhook"], data=webhook_string, headers=custom_header + ) + if r.status_code not in self.success_codes: + self.logger.error( + "Error %d while trying to post %s", r.status_code, hook["name"] + ) + else: + self.logger.debug("Got %d when posting %s", r.status_code, hook["name"]) + + # End of feed loop + + # Dump updated config back to json file + self.logger.debug("Dumping config back to %s", str(self.config_file_path)) + self.app_config["lastupdate"] = now + with open(self.config_file_path, "w") as config_file: + json.dump(self.app_config, config_file, indent=4) + + return + + +# end of Discorss class def main(): - os.environ["TZ"] = "America/Toronto" - time.tzset() - now = time.mktime(time.localtime()) - setupPaths() # Handle the config and log paths - try: - last_check = app_config["lastupdate"] - except KeyError: - last_check = now - 21600 # first run, no lastupdate, check up to 6 hours ago - for i, hook in enumerate(app_config["feeds"]): # Feed loop start - logger.debug("Parsing feed %s...", hook["name"]) - feeds = feedparser.parse(hook["url"]) - latest_post = [] - prev_best = 0 - logger.debug("About to sort through entries for feed %s ...", hook["name"]) - for feed in feeds["entries"]: - try: - bad_time = False - published_time = time.mktime(feed["published_parsed"]) - published_time = published_time + hook["offset"] - except KeyError: - published_time = time.mktime(feed["updated_parsed"]) - bad_time = True - if published_time > prev_best: - latest_post = feed - prev_best = published_time - else: - continue - if bad_time is True: - logger.debug( - "Feed %s doesn't supply a published time, using updated time instead", - hook["name"], - ) - # Hash the title and time of the latest post and use that to determine if it's been posted - # Yes, SHA3-512 is totally unnecessary for this purpose, but I love SHA3 - logger.debug("About to hash %s ...", latest_post["title"]) - try: - new_hash = hashlib.sha3_512( - bytes(latest_post["title"] + str(published_time), "utf-8") - ).hexdigest() - except TypeError: - logger.error("Title of %s isn't hashing correctly", hook["name"]) - continue - try: - if hook["lasthash"] != new_hash: - app_config["feeds"][i]["lasthash"] = new_hash - else: - continue - except KeyError: - app_config["feeds"][i]["lasthash"] = new_hash - logger.info( - "Feed %s has no existing hash, likely a new feed!", hook["name"] - ) - # Generate the webhook - logger.info( - "Publishing webhook for %s. Last check was %d, now is %d", - hook["name"], - last_check, - now, - ) - webhook = { - "embeds": [ - { - "title": str(latest_post["title"]), - "url": str(latest_post["link"]), - "color": 2123412, - "footer": { - "text": "DiscoRSS", - "icon_url": "https://frzn.dev/~amr/images/discorss.png", - }, - "author": { - "name": str(hook["name"]), - "url": str(hook["siteurl"]), - }, - "fields": [ - { - "name": "Excerpt from post:", - "value": get_description(latest_post), - } - ], - # "timestamp": str(now), - } - ], - "attachments": [], - } - custom_header = { - "user-agent": "DiscoRSS (https://git.frzn.dev/amr/discorss, 0.2rc3)", - "content-type": "application/json", - } - webhook_string = json.dumps(webhook) - - logger.debug("About to run POST for %s", hook["name"]) - r = requests.post(hook["webhook"], data=webhook_string, headers=custom_header) - if r.status_code not in success_codes: - logger.error( - "Error %d while trying to post %s", r.status_code, hook["name"] - ) - else: - logger.debug("Got %d when posting %s", r.status_code, hook["name"]) - - # End of feed loop - - # Dump updated config back to json file - logger.debug("Dumping config back to %s", str(config_file_path)) - app_config["lastupdate"] = now - with open(config_file_path, "w") as config_file: - json.dump(app_config, config_file, indent=4) - - return + app = Discorss() + app.process() if __name__ == "__main__": From 2b4e4216f4c285fdb3cf467e6238339f78846e10 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Sun, 20 Apr 2025 16:10:08 -0400 Subject: [PATCH 08/13] release: bump version number to 0.2 --- discorss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discorss.py b/discorss.py index d90c7fb..99b4398 100755 --- a/discorss.py +++ b/discorss.py @@ -203,7 +203,7 @@ class Discorss: "attachments": [], } custom_header = { - "user-agent": "DiscoRSS (https://git.frzn.dev/amr/discorss, 0.2rc3)", + "user-agent": "DiscoRSS (https://git.frzn.dev/amr/discorss, 0.2)", "content-type": "application/json", } webhook_string = json.dumps(webhook) From 2e18ede6a84c18a6b895d6314fba16e7c5af8b26 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Mon, 21 Apr 2025 19:29:56 -0400 Subject: [PATCH 09/13] docs/fix: Updated README.md and install.sh --- README.md | 20 +++++++++++++++----- install.sh | 1 + 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 06b28fe..48148d4 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,15 @@ The remaining imports should all be part of the standard Python install. ## Important Notes -As it currently is written, the script uses the hash of the post title to prevent sending duplicates. However, a recent change to check for the publish time was added, only because some feeds are not in reverse chronological order (latest post at top of feed, ie, entry index 0). Because of this, we do actually need to check the publish times. This still needs some testing and things might be a bit broken because of it. If you see any issues please let me know. - -Logging was recently enabled. Make sure that the user running the script (especially when using systemd timers) has write access to the /var/log/discorss directory. The app will try and create the directory for you, but if your user doesn't have permissions to create directories in /var/log this will fail and this will probably crash the script as is. I will try and remember to catch that exception and exit gracefully with an error message to stdout. If you want the logs to go somewhere else, just edit the log_dir variable near the top of discorss.py. Choose a directory that makes sense. Unfortunately, as far as I know, the XDG standards don't have an equivalent to the /var/log directory in the user directory, so I wasn't sure what the best default was. In the future, we may switch to logging using systemd and journald directly, though it is nice to have a separate file. +The logger will try and put the logs in `/var/log/discorss`. Make sure to create this directory and give the user running the script write permissions there. If you want the logs to go somewhere else, just edit the log_dir variable near the top of discorss.py. Choose a directory that makes sense. Unfortunately, as far as I know, the XDG standards don't have an equivalent to the /var/log directory in the user directory, so I wasn't sure what the best default was. In the future, we may switch to logging using systemd and journald directly, though it is nice to have a separate file. ## How to setup -To configure the script, create ~/.config/discorss/discorss.conf with the following structure: +Note: see the Automation section below for info about using the `install.sh` script to help get all the files in the right places. + +### Config file format + +To configure the script, create `~/.config/discorss/discorss.conf` using JSON formatting like this: ```json { @@ -50,6 +52,13 @@ The offset should only be required if feeds aren't showing up. This is because f ## Automation +**New**: There is now `install.sh` in the repo which will automatically help you set up both the config file and the systemd unit files for the service and timer, using essentially the exact text below. It will copy them to the user systemd unit folder, `~/.config/systemd/user` and optionally enable the timer. It's a good idea to edit the configuration file at `~/.config/discorss/discorss.conf` and paste in your webhook URLs and add any other feeds you want before starting the timer, unless you can do it really quickly before the next 5 minute spot on the clock :) +Of course, if it fires with an invalid config, the script will just crash, and you'll probably just have to manually start the timer once the config is fixed, so not a big deal. + +_Remember to create `/var/log/discorss` and change it to be writeable by the user running the service!_ + +### Manual method + To automate feed posting, create a systemd service and timer to execute the script. Use the command `systemctl --user edit --full --force discorss.service` and then paste in something like this: @@ -61,13 +70,14 @@ Wants=discorss.timer [Service] Type=oneshot +TimeoutStartSec=120 ExecStart=/path/to/discorss.py [Install] WantedBy=default.target ``` -Make sure to edit the ExecStart to point to the correct location. Then we need a systemd timer to automatically fire the script. Run `systemctl --user edit --full --force discorss.timer` and then paste in this: +The TimeoutStartSec will catch any issues with the script locking up due to, e.g., DNS failures or RSS feeds being slow/unavailable. 2 minutes should be more than enough time unless you are running hundreds of feeds. Also make sure to edit the ExecStart to point to the correct location. Then we need a systemd timer to automatically fire the script. Run `systemctl --user edit --full --force discorss.timer` and then paste in this: ```systemd [Unit] Description=Timer for DiscoRSS diff --git a/install.sh b/install.sh index 59b3ea1..f69c1a2 100755 --- a/install.sh +++ b/install.sh @@ -16,6 +16,7 @@ Wants=discorss.timer [Service] Type=oneshot +TimeoutStartSec=120 ExecStart=/home/amr/workspace/python/discorss/discorss.py [Install] From e4539b5733c8bdc499398b4388b544587c989912 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Tue, 22 Apr 2025 02:11:01 -0400 Subject: [PATCH 10/13] chore: rename function, move some init code Also switching logging back to ERROR from DEBUG. The solution to the lockups for now is to just use systemd timer timeouts. --- discorss.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/discorss.py b/discorss.py index 99b4398..9d001ce 100755 --- a/discorss.py +++ b/discorss.py @@ -66,7 +66,10 @@ class Discorss: desc = desc + str(addons) return desc - def setupPaths(self): + def setup(self): + os.environ["TZ"] = "America/Toronto" + time.tzset() + self.now = time.mktime(time.localtime()) # Check for log and config files/paths, create empty directories if needed # TODO: make this cleaner if not Path(self.log_dir).exists(): @@ -107,22 +110,19 @@ class Discorss: logging.basicConfig( filename=str(self.log_dir + self.log_file_path), encoding="utf-8", - level=logging.DEBUG, + level=logging.ERROR, datefmt="%m/%d/%Y %H:%M:%S", format="%(asctime)s: %(levelname)s: %(message)s", ) return def process(self): - os.environ["TZ"] = "America/Toronto" - time.tzset() - now = time.mktime(time.localtime()) - self.setupPaths() # Handle the config and log paths + self.setup() # Handle the config and log paths try: last_check = self.app_config["lastupdate"] except KeyError: last_check = ( - now - 21600 + self.now - 21600 ) # first run, no lastupdate, check up to 6 hours ago for i, hook in enumerate(self.app_config["feeds"]): # Feed loop start self.logger.debug("Parsing feed %s...", hook["name"]) @@ -172,10 +172,10 @@ class Discorss: ) # Generate the webhook self.logger.info( - "Publishing webhook for %s. Last check was %d, now is %d", + "Publishing webhook for %s. Last check was %d, self.now is %d", hook["name"], last_check, - now, + self.now, ) webhook = { "embeds": [ @@ -197,7 +197,7 @@ class Discorss: "value": self.get_description(latest_post), } ], - # "timestamp": str(now), + # "timestamp": str(self.now), } ], "attachments": [], @@ -223,7 +223,7 @@ class Discorss: # Dump updated config back to json file self.logger.debug("Dumping config back to %s", str(self.config_file_path)) - self.app_config["lastupdate"] = now + self.app_config["lastupdate"] = self.now with open(self.config_file_path, "w") as config_file: json.dump(self.app_config, config_file, indent=4) From b0f08c405bc78d77028bc5f3c9a5240f513adeba Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Tue, 22 Apr 2025 03:17:16 -0400 Subject: [PATCH 11/13] fix: install.sh works properly now (with colour!) --- install.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/install.sh b/install.sh index f69c1a2..3c32200 100755 --- a/install.sh +++ b/install.sh @@ -9,6 +9,8 @@ # use systemctl --user edit --full discorss.service or discorss.timer # after installing them. +printf "\e[1;34mDisco\e[1;38;5;208mRSS\e[0m Install Helper Script\n\n" + cat << EOF > discorss.service [Unit] Description=Discord RSS feeder @@ -39,14 +41,16 @@ WantedBy=timers.target EOF +mkdir -p ~/.config/systemd/user/ + cp discorss.service ~/.config/systemd/user/ cp discorss.timer ~/.config/systemd/user/ systemctl --user daemon-reload -printf "Would you like a basic example config created for you? [y/n]" +printf "Would you like a basic example config created for you? [y/n]: " read answer1 -if [ "$answer1" =~ ^[yYnN]$ ]; then +if [[ "$answer1" =~ ^([yY])$ ]]; then mkdir -p -v ~/.config/discorss cat << EOF > ~/.config/discorss/discorss.conf { @@ -61,20 +65,20 @@ if [ "$answer1" =~ ^[yYnN]$ ]; then ], } EOF - printf "Make sure to edit ~/.config/discorss/discorss.conf and add in your custom feeds and webhook URLS! The script will just error out if you don't do this." + printf "\nMake sure to edit \e[1;34m~/.config/discorss/discorss.conf\e[0m and add in your custom feeds and webhook URLS! The script will just error out if you don't do this." else - printf "Make sure to create a config at ~/.config/discorss/discorss.conf and follow the pattern shown in the README." + printf "\nMake sure to create a config at \e[1;34m~/.config/discorss/discorss.conf\e[0m and follow the pattern shown in the README." fi -printf "Would you like to have the timer enabled and started now? [y/n]" +printf "\nWould you like to have the timer enabled and started now? [y/n]: " read answer -if [ "$answer" =~ ^[yYnN]$ ]; then +if [[ "$answer" =~ ^([yY])$ ]]; then systemctl --user enable --now discorss.timer - printf "discorss.timer enabled and started. Don't enable or start discorss.service -- the timer does this automatically." + printf "\ndiscorss.timer enabled and started. \e[1;31mDon't enable or start discorss.service\e[0m -- the timer does this automatically." else - printf "Don't forget to run systemctl --user enable --now discorss.timer when you are ready! Don't enable or start discorss.service -- the timer does this automatically." + printf "\nDon't forget to run \e[1;32msystemctl --user enable --now discorss.timer\e[0m when you are ready! \e[1;31mDon't enable or start discorss.service\e[0m -- the timer does this automatically." fi -printf "You should be almost ready to go! Double-check your config files, and check systemctl --user list-timers once the discorss.timer is enabled to see when it will fire next. The default is every 5 minutes." +printf "\n\nYou should be almost ready to go! Double-check your config files, and check \e[1;32msystemctl --user list-timers\e[0m once the discorss.timer is enabled to see when it will fire next. The default is every 5 minutes." -printf "Remember, if you need help or encounter any bugs, contact me via the issues tracker on the git repository where you got this from!" +printf "\nRemember, if you need help or encounter any bugs, contact me via the issues tracker on the git repository where you got this from!" From 9d2530ab02803d27be53dc55111f78fbaed0f192 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Tue, 22 Apr 2025 04:26:49 -0400 Subject: [PATCH 12/13] fix: corrected errors in install.sh Also improved the script to actually use the script location in the discorss.service file... yeah I should have done that from the start, d'oh! --- install.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 3c32200..d3ed185 100755 --- a/install.sh +++ b/install.sh @@ -11,6 +11,8 @@ printf "\e[1;34mDisco\e[1;38;5;208mRSS\e[0m Install Helper Script\n\n" +workingDir=$(pwd) + cat << EOF > discorss.service [Unit] Description=Discord RSS feeder @@ -19,7 +21,7 @@ Wants=discorss.timer [Service] Type=oneshot TimeoutStartSec=120 -ExecStart=/home/amr/workspace/python/discorss/discorss.py +ExecStart=$workingDir/discorss.py [Install] WantedBy=default.target @@ -45,7 +47,8 @@ mkdir -p ~/.config/systemd/user/ cp discorss.service ~/.config/systemd/user/ cp discorss.timer ~/.config/systemd/user/ - +rm -f discorss.service +rm -f discorss.timer systemctl --user daemon-reload printf "Would you like a basic example config created for you? [y/n]: " @@ -60,9 +63,9 @@ if [[ "$answer1" =~ ^([yY])$ ]]; then "siteurl": "https://www.phoronix.com/", "url": "http://www.phoronix.com/rss.php", "webhook": "PASTE WEBHOOK URL HERE", - "offset": 0, + "offset": 0 } - ], + ] } EOF printf "\nMake sure to edit \e[1;34m~/.config/discorss/discorss.conf\e[0m and add in your custom feeds and webhook URLS! The script will just error out if you don't do this." From b243bc7bb4991f85eab236fd7e9c6486bba39051 Mon Sep 17 00:00:00 2001 From: "A.M. Rowsell" Date: Tue, 22 Apr 2025 17:29:45 -0400 Subject: [PATCH 13/13] dev: Added more prompts to install script --- install.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/install.sh b/install.sh index d3ed185..7a89398 100755 --- a/install.sh +++ b/install.sh @@ -13,7 +13,12 @@ printf "\e[1;34mDisco\e[1;38;5;208mRSS\e[0m Install Helper Script\n\n" workingDir=$(pwd) -cat << EOF > discorss.service +printf "Would you like the systemd service and timer files created for you? [y/n]: " +read answer +if [[ "$answer" =~ ^([yY])$ ]]; then + + cat << EOF > discorss.service +# Autogenerated by install.sh [Unit] Description=Discord RSS feeder Wants=discorss.timer @@ -28,7 +33,8 @@ WantedBy=default.target EOF -cat << EOF > discorss.timer + cat << EOF > discorss.timer +# Autogenerated by install.sh [Unit] Description=Timer for DiscoRSS Requires=discorss.service @@ -43,13 +49,18 @@ WantedBy=timers.target EOF -mkdir -p ~/.config/systemd/user/ - -cp discorss.service ~/.config/systemd/user/ -cp discorss.timer ~/.config/systemd/user/ -rm -f discorss.service -rm -f discorss.timer -systemctl --user daemon-reload + printf "Making ~/.config/systemd/user in case it doesn't exist ...\n" + mkdir -p -v ~/.config/systemd/user/ + printf "Copying service and timer files there ... \n" + cp discorss.service ~/.config/systemd/user/ + cp discorss.timer ~/.config/systemd/user/ + rm -f discorss.service + rm -f discorss.timer + printf "Reloading systemd daemon ... \n\n" + systemctl --user daemon-reload +else + printf "This script is intended to be automatically run. It's designed with systemd in mind, but you are free to use any automation tools. You can look at this script for examples of how to structure systemd user services and timers.\nOf course, you could always run it by hand, if you really want to :)\n\n" +fi printf "Would you like a basic example config created for you? [y/n]: " read answer1 @@ -84,4 +95,4 @@ fi printf "\n\nYou should be almost ready to go! Double-check your config files, and check \e[1;32msystemctl --user list-timers\e[0m once the discorss.timer is enabled to see when it will fire next. The default is every 5 minutes." -printf "\nRemember, if you need help or encounter any bugs, contact me via the issues tracker on the git repository where you got this from!" +printf "\nRemember, if you need help or encounter any bugs, contact me via the issues tracker on the git repository where you got this from!\n"