From 53ce9dfe9379b2a3dc3f4c6ec69821981eab138f Mon Sep 17 00:00:00 2001 From: Fundor333 Date: Tue, 21 Jan 2025 02:58:15 +0100 Subject: [PATCH] Fix synd --- .flake8 | 6 + .pre-commit-config.yaml | 54 +++++ action_script/syndication-collector.py | 26 +-- action_script/webmention.py | 14 +- ... => 3a557bcb598aca7db31183f7d38af03a.json} | 0 ... => 6ba02a68744476c5f716ce4b31cf2486.json} | 0 ... => b0928a709a21ad14f4208bf432ec4da3.json} | 0 ... => b84affb99f8173b482f1760ec8cc9756.json} | 0 ... => e0d08bec318fd88cf4b1d8ee080b3aa9.json} | 0 ... => e17c9a6d4800b07eef1ae8844f26ec6c.json} | 0 ... => e27e929dea17d2d392d28376ed3bc87e.json} | 0 ... => f34a45a9944d9aa8679b789a8348b733.json} | 0 ... => f60fa429587a342d58545ddcaba90029.json} | 0 .../10/this-is-why-you-need-a-domain.json | 41 +++- layouts/_default/single.html | 3 + layouts/partials/syndication.html | 16 +- makefile | 16 +- package-lock.json | 12 +- poetry.lock | 190 +++++++++++++++++- pyproject.toml | 3 + requirements.txt | 3 - sorting_characters.py | 20 -- temp.json | 0 23 files changed, 343 insertions(+), 61 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml rename data/syndication/{micro/2025/10/reworking-of-the-site.json => 3a557bcb598aca7db31183f7d38af03a.json} (100%) rename data/syndication/{photos/2025/square-square-square-and-square.json => 6ba02a68744476c5f716ce4b31cf2486.json} (100%) rename data/syndication/{micro/2025/10/scraping-the-web-for-the-syndication-link.json => b0928a709a21ad14f4208bf432ec4da3.json} (100%) rename data/syndication/{micro/2025/10/this-is-why-you-need-a-domain.json => b84affb99f8173b482f1760ec8cc9756.json} (100%) rename data/syndication/{micro/2025/10/testing-the-hashtags.json => e0d08bec318fd88cf4b1d8ee080b3aa9.json} (100%) rename data/syndication/{post/2025/my-zsh-command-history.json => e17c9a6d4800b07eef1ae8844f26ec6c.json} (100%) rename data/syndication/{micro/2025/10/the-little-thing.json => e27e929dea17d2d392d28376ed3bc87e.json} (100%) rename data/syndication/{photos/2025/square-tile.json => f34a45a9944d9aa8679b789a8348b733.json} (100%) rename data/syndication/{micro/2025/10/and-another-blog-is-powered-by-webmention.json => f60fa429587a342d58545ddcaba90029.json} (100%) delete mode 100644 sorting_characters.py delete mode 100644 temp.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..b77e5868 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +ignore = E501, W503, F403, C901, B904, B902 +max-line-length = 119 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 +exclude = .git,*migrations*,.tox, .venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8372b8d4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +fail_fast: true +repos: + - repo: https://github.com/adamchainz/djade-pre-commit + rev: "1.3.2" # Replace with the latest tag on GitHub + hooks: + - id: djade + args: [ --target-version, "4.2" ] # Replace with Django version + - repo: https://github.com/pycqa/bandit + rev: 1.8.2 + hooks: + - id: bandit + args: [ "-iii", "-ll" ] + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + language_version: python3.9 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-merge-conflict + - id: detect-private-key + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: forbid-new-submodules + - id: mixed-line-ending + - id: trailing-whitespace + language: python + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 # Use the ref you want to point at + hooks: + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + - repo: https://github.com/asottile/pyupgrade + rev: v3.19.1 + hooks: + - id: pyupgrade + args: [ --py311-plus ] + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + additional_dependencies: [ + 'flake8-blind-except', + 'flake8-docstrings', + 'flake8-bugbear', + 'flake8-comprehensions', + 'flake8-docstrings', + 'flake8-implicit-str-concat', + 'pydocstyle>=5.0.0', + ] diff --git a/action_script/syndication-collector.py b/action_script/syndication-collector.py index bc5555be..6b1955c9 100644 --- a/action_script/syndication-collector.py +++ b/action_script/syndication-collector.py @@ -2,10 +2,16 @@ import feedparser from pathlib import Path import os import json +import hashlib + +domain = "https://fundor333.com" +rss_url_mastodon = "https://mastodon.social/@fundor333.rss" -domain = 'https://fundor333.com' -rss_url_mastodon = 'https://mastodon.social/@fundor333.rss' +def clean_slug(slug: str): + return hashlib.md5( + (slug.split("?")[0]).encode("utf-8"), usedforsecurity=False + ).hexdigest() class MastodonFinder: @@ -21,9 +27,10 @@ class MastodonFinder: feed = feedparser.parse(rss_url) if feed.status == 200: for entry in feed.entries: - link = entry.get('link') - for e in self.find_urls(entry.get('description')): + link = entry.get("link") + for e in self.find_urls(entry.get("description")): if domain in e: + e = clean_slug(e) if output.get(e, False): output[e].append(link) else: @@ -44,18 +51,13 @@ class WriterSyndication: def write(self): for key in self.output.keys(): - original_path = key.split(self.domain)[1] - path_list = original_path.split('/') - path_list = [x for x in path_list if x.strip()] - filename = path_list.pop() - - path_folder = os.path.join('data', "syndication", *path_list) + path_folder = os.path.join("data", "syndication") Path(path_folder).mkdir(parents=True, exist_ok=True) - path_file = os.path.join(path_folder, filename + ".json") + path_file = os.path.join(path_folder, key) - with open(path_file, 'w') as fp: + with open(path_file + ".json", "w") as fp: json.dump({"syndication": self.output[key]}, fp) def run(self): diff --git a/action_script/webmention.py b/action_script/webmention.py index 38962474..b27f6d97 100644 --- a/action_script/webmention.py +++ b/action_script/webmention.py @@ -7,7 +7,7 @@ from pathlib import Path http_domain = "https://fundor333.com" domain = "fundor333.com" -token = os.getenv('WEBMENTIONS_TOKEN') +token = os.getenv("WEBMENTIONS_TOKEN") since_days = 30 @@ -21,16 +21,14 @@ def clean_slug(slug: str): r = requests.get(url) - data = r.json() - output = {} for webmention in data["children"]: - with open('temp.json', 'w') as fp: + with open("temp.json", "w") as fp: - label = clean_slug(webmention['wm-target']) + label = clean_slug(webmention["wm-target"]) if output.get(label, False): output[label].append(webmention) @@ -39,16 +37,16 @@ for webmention in data["children"]: for key in output.keys(): original_path = key - path_list = original_path.split('/') + path_list = original_path.split("/") path_list = [x for x in path_list if x.strip()] if path_list != []: filename = path_list.pop() - path_folder = os.path.join('data', "webmentions", *path_list) + path_folder = os.path.join("data", "webmentions", *path_list) Path(path_folder).mkdir(parents=True, exist_ok=True) path_file = os.path.join(path_folder, filename + ".json") - with open(path_file, 'w') as fp: + with open(path_file, "w") as fp: json.dump(output[key], fp) diff --git a/data/syndication/micro/2025/10/reworking-of-the-site.json b/data/syndication/3a557bcb598aca7db31183f7d38af03a.json similarity index 100% rename from data/syndication/micro/2025/10/reworking-of-the-site.json rename to data/syndication/3a557bcb598aca7db31183f7d38af03a.json diff --git a/data/syndication/photos/2025/square-square-square-and-square.json b/data/syndication/6ba02a68744476c5f716ce4b31cf2486.json similarity index 100% rename from data/syndication/photos/2025/square-square-square-and-square.json rename to data/syndication/6ba02a68744476c5f716ce4b31cf2486.json diff --git a/data/syndication/micro/2025/10/scraping-the-web-for-the-syndication-link.json b/data/syndication/b0928a709a21ad14f4208bf432ec4da3.json similarity index 100% rename from data/syndication/micro/2025/10/scraping-the-web-for-the-syndication-link.json rename to data/syndication/b0928a709a21ad14f4208bf432ec4da3.json diff --git a/data/syndication/micro/2025/10/this-is-why-you-need-a-domain.json b/data/syndication/b84affb99f8173b482f1760ec8cc9756.json similarity index 100% rename from data/syndication/micro/2025/10/this-is-why-you-need-a-domain.json rename to data/syndication/b84affb99f8173b482f1760ec8cc9756.json diff --git a/data/syndication/micro/2025/10/testing-the-hashtags.json b/data/syndication/e0d08bec318fd88cf4b1d8ee080b3aa9.json similarity index 100% rename from data/syndication/micro/2025/10/testing-the-hashtags.json rename to data/syndication/e0d08bec318fd88cf4b1d8ee080b3aa9.json diff --git a/data/syndication/post/2025/my-zsh-command-history.json b/data/syndication/e17c9a6d4800b07eef1ae8844f26ec6c.json similarity index 100% rename from data/syndication/post/2025/my-zsh-command-history.json rename to data/syndication/e17c9a6d4800b07eef1ae8844f26ec6c.json diff --git a/data/syndication/micro/2025/10/the-little-thing.json b/data/syndication/e27e929dea17d2d392d28376ed3bc87e.json similarity index 100% rename from data/syndication/micro/2025/10/the-little-thing.json rename to data/syndication/e27e929dea17d2d392d28376ed3bc87e.json diff --git a/data/syndication/photos/2025/square-tile.json b/data/syndication/f34a45a9944d9aa8679b789a8348b733.json similarity index 100% rename from data/syndication/photos/2025/square-tile.json rename to data/syndication/f34a45a9944d9aa8679b789a8348b733.json diff --git a/data/syndication/micro/2025/10/and-another-blog-is-powered-by-webmention.json b/data/syndication/f60fa429587a342d58545ddcaba90029.json similarity index 100% rename from data/syndication/micro/2025/10/and-another-blog-is-powered-by-webmention.json rename to data/syndication/f60fa429587a342d58545ddcaba90029.json diff --git a/data/webmentions/micro/2025/10/this-is-why-you-need-a-domain.json b/data/webmentions/micro/2025/10/this-is-why-you-need-a-domain.json index 635e4128..2ebab403 100644 --- a/data/webmentions/micro/2025/10/this-is-why-you-need-a-domain.json +++ b/data/webmentions/micro/2025/10/this-is-why-you-need-a-domain.json @@ -1 +1,40 @@ -[{"type": "entry", "author": {"type": "card", "name": "LaemenPang", "photo": "https://webmention.io/avatar/files.mastodon.social/f05260061d91af3ebaa31c17256d6b75b697c03d670e1e295704ec78ca10502e.jpg", "url": "https://mastodon.social/@focusedontheberry"}, "url": "https://mastodon.social/@fundor333/113856768273797658#favorited-by-112349892458455305", "published": null, "wm-received": "2025-01-20T20:03:08Z", "wm-id": 1877825, "wm-source": "https://brid.gy/like/mastodon/@fundor333@mastodon.social/113856768273797658/112349892458455305", "wm-target": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", "wm-protocol": "webmention", "like-of": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", "wm-property": "like-of", "wm-private": false}, {"type": "entry", "author": {"type": "card", "name": "LaemenPang", "photo": "https://webmention.io/avatar/files.mastodon.social/f05260061d91af3ebaa31c17256d6b75b697c03d670e1e295704ec78ca10502e.jpg", "url": "https://mastodon.social/@focusedontheberry"}, "url": "https://mastodon.social/@fundor333/113856768273797658#reblogged-by-112349892458455305", "published": null, "wm-received": "2025-01-20T19:52:47Z", "wm-id": 1877823, "wm-source": "https://brid.gy/repost/mastodon/@fundor333@mastodon.social/113856768273797658/112349892458455305", "wm-target": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", "wm-protocol": "webmention", "repost-of": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", "wm-property": "repost-of", "wm-private": false}] \ No newline at end of file +[ + { + "author": { + "name": "LaemenPang", + "photo": "https://webmention.io/avatar/files.mastodon.social/f05260061d91af3ebaa31c17256d6b75b697c03d670e1e295704ec78ca10502e.jpg", + "type": "card", + "url": "https://mastodon.social/@focusedontheberry" + }, + "like-of": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", + "published": null, + "type": "entry", + "url": "https://mastodon.social/@fundor333/113856768273797658#favorited-by-112349892458455305", + "wm-id": 1877825, + "wm-private": false, + "wm-property": "like-of", + "wm-protocol": "webmention", + "wm-received": "2025-01-20T20:03:08Z", + "wm-source": "https://brid.gy/like/mastodon/@fundor333@mastodon.social/113856768273797658/112349892458455305", + "wm-target": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/" + }, + { + "author": { + "name": "LaemenPang", + "photo": "https://webmention.io/avatar/files.mastodon.social/f05260061d91af3ebaa31c17256d6b75b697c03d670e1e295704ec78ca10502e.jpg", + "type": "card", + "url": "https://mastodon.social/@focusedontheberry" + }, + "published": null, + "repost-of": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/", + "type": "entry", + "url": "https://mastodon.social/@fundor333/113856768273797658#reblogged-by-112349892458455305", + "wm-id": 1877823, + "wm-private": false, + "wm-property": "repost-of", + "wm-protocol": "webmention", + "wm-received": "2025-01-20T19:52:47Z", + "wm-source": "https://brid.gy/repost/mastodon/@fundor333@mastodon.social/113856768273797658/112349892458455305", + "wm-target": "https://fundor333.com/micro/2025/10/this-is-why-you-need-a-domain/" + } +] diff --git a/layouts/_default/single.html b/layouts/_default/single.html index a79f00df..594994d8 100644 --- a/layouts/_default/single.html +++ b/layouts/_default/single.html @@ -5,6 +5,9 @@

{{ .Title }}

+ {{ $urlized := .Page.Permalink | md5 }} + {{ $urlized}} +