From d28dcdb069ae70524dc53de22eb0d24132d67ce0 Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sun, 7 Jan 2024 16:58:56 +1100 Subject: [PATCH] fixes --- .gitignore | 3 +- README.md | 19 ++ ausglamr/settings.py | 23 +- ausglamr/urls.py | 2 +- blogs/admin.py | 33 ++- blogs/forms.py | 10 +- blogs/management/commands/check_feeds.py | 65 ++++-- .../commands/queue_announcements.py | 6 +- .../management/commands/send_weekly_email.py | 7 +- blogs/migrations/0001_initial.py | 19 +- blogs/models/blog.py | 31 ++- blogs/models/event.py | 2 +- blogs/models/group.py | 2 +- blogs/models/newsletter.py | 2 +- blogs/templates/blogs/confirm-register.html | 6 +- blogs/templates/browse/blogs.html | 7 +- blogs/templates/contribute.html | 5 +- blogs/templates/search.html | 2 +- blogs/templates/subscribe.html | 2 +- blogs/tests/test_commands.py | 221 ++++++++++++++++++ blogs/tests/test_management.py | 16 +- blogs/tests/test_models.py | 22 +- blogs/tests/test_utilities.py | 185 ++++++++++++++- blogs/tests/test_views.py | 33 ++- blogs/utilities.py | 45 +++- blogs/views/public.py | 24 +- 26 files changed, 665 insertions(+), 127 deletions(-) create mode 100644 blogs/tests/test_commands.py diff --git a/.gitignore b/.gitignore index 4c66948..1254248 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ data -static \ No newline at end of file +static +fixtures \ No newline at end of file diff --git a/README.md b/README.md index ec8ed27..2577cc0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,25 @@ A django app running on Docker. Replaces _Aus GLAM Blogs_. +## Deploy + +* `docker compose build` +* `./glamr-dev createsuperuser` +* `./glamr-dev makemigrations` +* `./glamr-dev migrate` +* `docker compose up` + +* set up database backups (as cron jobs): + + * `/usr/bin/docker exec -u root ausglamr-db-1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump` + * `/usr/bin/docker cp ausglamr-db-1:/tmp/ausglamr_backup.dump /home/hugh/backups/` + +* set up cron jobs for management commands as below + +## Migrating + +See `data/help.txt` + ## Admin Don't forget to add some Content Warnings for use by the Mastodon bot. diff --git a/ausglamr/settings.py b/ausglamr/settings.py index 9337553..20d6a44 100644 --- a/ausglamr/settings.py +++ b/ausglamr/settings.py @@ -28,11 +28,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = env("SECRET_KEY") +# DEPLOYMENT + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env("DEBUG") - ALLOWED_HOSTS = [env("DOMAIN")] +CSRF_COOKIE_SECURE=True +SESSION_COOKIE_SECURE=True CSRF_TRUSTED_ORIGINS = [f'https://{env("DOMAIN")}'] +CONN_MAX_AGE=None # persistent DB connection # Application definition @@ -149,6 +153,7 @@ ADMIN_EMAIL = env("ADMIN_EMAIL") # custom # django email settings DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL") +SERVER_EMAIL = env("SERVER_EMAIL") EMAIL_HOST = env("EMAIL_HOST") EMAIL_HOST_USER = env("EMAIL_HOST_USER") EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD") @@ -158,3 +163,19 @@ EMAIL_USE_SSL = env("EMAIL_USE_SSL") # mastodon MASTODON_ACCESS_TOKEN = env("MASTODON_ACCESS_TOKEN") MASTODON_DOMAIN = env("MASTODON_DOMAIN") + +# LOGGING + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + }, + "root": { + "handlers": ["console"], + "level": "INFO", + }, +} \ No newline at end of file diff --git a/ausglamr/urls.py b/ausglamr/urls.py index c067fa6..3f392a2 100644 --- a/ausglamr/urls.py +++ b/ausglamr/urls.py @@ -41,7 +41,7 @@ urlpatterns = [ path("subscribe", views.Subscribe.as_view(), name="subscribe"), path("subscribe-email", views.SubscribeEmail.as_view(), name="subscribe-email"), path( - "confirm-subscribe-email//", + "confirm-subscribe-email//", views.ConfirmEmail.as_view(), name="confirm-subscribe-email", ), diff --git a/blogs/admin.py b/blogs/admin.py index 9c150ef..acc0fba 100644 --- a/blogs/admin.py +++ b/blogs/admin.py @@ -19,9 +19,11 @@ def approve(modeladmin, request, queryset): instance.announce() if hasattr(instance, "event"): # CFP - recipient = instance.event.owner_email - if hasattr(instance, "owner_email"): # overrides above in case needed in future - recipient = instance.owner_email + recipient = instance.event.contact_email + if hasattr( + instance, "contact_email" + ): # overrides above in case needed in future + recipient = instance.contact_email if recipient: if hasattr(instance, "name"): @@ -47,7 +49,7 @@ def suspend(modeladmin, request, queryset): queryset.update(suspended=True) for instance in queryset: - if hasattr(instance, "owner_email"): + if hasattr(instance, "contact_email"): if hasattr(instance, "name"): title = instance.name else: @@ -58,7 +60,7 @@ def suspend(modeladmin, request, queryset):

Your blog {title} has been suspended from AusGLAMR. It may be unsuspended in future once the issue is resolved. If you would like more information, please reply to this email.

\ " - utilities.send_email(subject, message, instance.owner_email) + utilities.send_email(subject, message, instance.contact_email) @admin.action(description="Unsuspend selected blogs") @@ -67,7 +69,7 @@ def unsuspend(modeladmin, request, queryset): queryset.update(suspended=False, suspension_lifted=timezone.now()) for instance in queryset: - if hasattr(instance, "owner_email"): + if hasattr(instance, "contact_email"): if hasattr(instance, "name"): title = instance.name else: @@ -78,7 +80,7 @@ def unsuspend(modeladmin, request, queryset):

The suspension on your blog {title} has been removed on AusGLAMR. Please note that articles published whilst it was suspended will not be added to AusGLAMR retrospectively. If you would like more information, please reply to this email.

\ " - utilities.send_email(subject, message, instance.owner_email) + utilities.send_email(subject, message, instance.contact_email) @admin.action(description="Confirm selected subscribers") @@ -93,6 +95,18 @@ def unconfirm(modeladmin, request, queryset): queryset.update(confirmed=False) +@admin.action(description="Deactivate selected blogs") +def disable(modeladmin, request, queryset): + """disable selected""" + queryset.update(active=False) + + +@admin.action(description="Activate selected blogs") +def activate(modeladmin, request, queryset): + """un-disable selected""" + queryset.update(active=True) + + @admin.action(description="Send confirmation request to selected subscribers") def send_conf_request(modeladmin, request, queryset): """send to selected""" @@ -106,16 +120,17 @@ class Blog(admin.ModelAdmin): """display settings for blogs""" list_display = ( - "title", "url", + "title", "author_name", "approved", "announced", "suspended", "failing", + "active", ) ordering = ["approved", "-suspended", "-failing"] - actions = [approve, unapprove, suspend, unsuspend] + actions = [approve, unapprove, suspend, unsuspend, activate, disable] @admin.register(models.Article) diff --git a/blogs/forms.py b/blogs/forms.py index b42e6e3..89d492b 100644 --- a/blogs/forms.py +++ b/blogs/forms.py @@ -19,7 +19,7 @@ class RegisterBlogForm(forms.ModelForm): """set fields and model""" model = Blog - fields = ["url", "category", "activitypub_account_name", "owner_email"] + fields = ["url", "category", "activitypub_account_name", "contact_email"] class ConfirmBlogForm(forms.ModelForm): @@ -37,7 +37,7 @@ class ConfirmBlogForm(forms.ModelForm): "description", "category", "activitypub_account_name", - "owner_email", + "contact_email", ] @@ -55,7 +55,7 @@ class RegisterConferenceForm(forms.ModelForm): "description", "start_date", "activitypub_account_name", - "owner_email", + "contact_email", ] widgets = { "start_date": DateInput(), @@ -97,7 +97,7 @@ class RegisterGroupForm(forms.ModelForm): "url", "registration_url", "description", - "owner_email", + "contact_email", ] @@ -115,7 +115,7 @@ class RegisterNewsletterForm(forms.ModelForm): "url", "description", "activitypub_account_name", - "owner_email", + "contact_email", ] diff --git a/blogs/management/commands/check_feeds.py b/blogs/management/commands/check_feeds.py index 84f5978..b2b3c0c 100644 --- a/blogs/management/commands/check_feeds.py +++ b/blogs/management/commands/check_feeds.py @@ -1,5 +1,8 @@ """call this from cron to run through all the feeds to find new posts""" +import logging +import traceback + from datetime import datetime, timedelta, timezone import feedparser @@ -22,11 +25,12 @@ def get_tags(dictionary): """parse out tags from blog and upsert as tag instances""" tags = [] for tag_obj in dictionary: - tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first() - if not tag: - tag = models.Tag.objects.create(name=tag_obj.term.lower()) + if tag_obj.term.lower() != "uncategorized": + tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first() + if not tag: + tag = models.Tag.objects.create(name=tag_obj.term.lower()) - tags.append(tag) + tags.append(tag) return tags @@ -34,14 +38,24 @@ def get_tags(dictionary): class Command(BaseCommand): """the check_feeds command""" - # we could add arguments but we don't really need any + def add_arguments(self, parser): + parser.add_argument( + "-q", + action="store_true", + help="Suppress non-error messages", + ) def handle(self, *args, **options): """check feeds and update database""" - print(f"checking feeds at {django_timezone.localtime(django_timezone.now())}") + if not options["q"]: + logging.info( + f"checking feeds at {django_timezone.localtime(django_timezone.now())}" + ) - blogs = models.Blog.objects.filter(approved=True, suspended=False).all() + blogs = models.Blog.objects.filter( + approved=True, suspended=False, active=True + ).all() for blog in blogs: try: data = feedparser.parse(blog.feed) @@ -50,24 +64,24 @@ class Command(BaseCommand): if not models.Article.objects.filter( Q(url=article.link) | Q(guid=article.id) ).exists(): - if ( + if blog.suspension_lifted and ( blog.suspension_lifted - and blog.suspension_lifted - < date_to_tz_aware(article.updated_parsed) + > date_to_tz_aware(article.updated_parsed) ): - continue # don't ingest posts published during a suspension + continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension) - tags = get_tags( - getattr(article, "tags", None) - or getattr(article, "categories", []) + taglist = getattr(article, "tags", None) or getattr( + article, "categories", [] ) + tags = [tag.term.lower() for tag in taglist] + opt_out = False # don't include posts with opt out tags for tag in tags: if ( len( - {tag.name} + {tag} & { "notglam", "notglamr", @@ -99,8 +113,14 @@ class Command(BaseCommand): guid=article.id, ) - for tag in tags: + tags_to_add = get_tags( + getattr(article, "tags", None) + or getattr(article, "categories", []) + ) + + for tag in tags_to_add: instance.tags.add(tag) + instance.save() cutoff = django_timezone.now() - timedelta(days=3) @@ -108,11 +128,16 @@ class Command(BaseCommand): if newish: instance.announce() - blog.set_success() + blog.set_success( + updateddate=date_to_tz_aware(article.updated_parsed) + ) except Exception as e: blog.set_failing() - print(f"ERROR WITH BLOG {blog.title} - {blog.url}") - print(e) + logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}") + logging.error(e) - print(f"completed run at {django_timezone.localtime(django_timezone.now())}") + if not options["q"]: + logging.info( + f"completed run at {django_timezone.localtime(django_timezone.now())}" + ) diff --git a/blogs/management/commands/queue_announcements.py b/blogs/management/commands/queue_announcements.py index 6f43dd6..ae609e3 100644 --- a/blogs/management/commands/queue_announcements.py +++ b/blogs/management/commands/queue_announcements.py @@ -16,7 +16,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """check whether we need to queue announcements and queue them""" - conferences = models.Event.objects.filter( + events = models.Event.objects.filter( approved=True, announcements__lt=3, start_date__gte=timezone.now(), @@ -24,10 +24,10 @@ class Command(BaseCommand): calls = models.CallForPapers.objects.filter( announcements__lt=3, closing_date__gte=timezone.now(), - conference__approved=True, + event__approved=True, ) - for conf in conferences: + for conf in events: delta = conf.start_date - timezone.now().date() if ( diff --git a/blogs/management/commands/send_weekly_email.py b/blogs/management/commands/send_weekly_email.py index ee15256..7fb5cc8 100644 --- a/blogs/management/commands/send_weekly_email.py +++ b/blogs/management/commands/send_weekly_email.py @@ -1,6 +1,7 @@ """send the weekly email""" from datetime import timedelta +import logging import random from django.conf import settings @@ -22,7 +23,7 @@ class Command(BaseCommand): subscribers = models.Subscriber.objects.filter(confirmed=True) - print( + logging.info( f"Sending weekly emails to {len(subscribers)} subscribers at {timezone.now()}" ) @@ -31,7 +32,7 @@ class Command(BaseCommand): articles = models.Article.objects.filter(pubdate__gte=cutoff) events = models.Event.objects.filter(approved=True, pub_date__gte=cutoff) cfps = models.CallForPapers.objects.filter( - conference__approved=True, closing_date__gte=timezone.now().date() + event__approved=True, closing_date__gte=timezone.now().date() ) newsletters = models.Newsletter.objects.filter( approved=True, pub_date__gte=cutoff @@ -210,4 +211,4 @@ class Command(BaseCommand): send_email(subject, message, subscriber.email) - print(f"Weekly emails completed {timezone.now()}") + logging.info(f"Weekly emails completed {timezone.now()}") diff --git a/blogs/migrations/0001_initial.py b/blogs/migrations/0001_initial.py index bd25004..6fb129b 100644 --- a/blogs/migrations/0001_initial.py +++ b/blogs/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 4.2.7 on 2024-01-03 03:35 +# Generated by Django 4.2.7 on 2024-01-07 04:55 +import blogs.models.blog from django.db import migrations, models import django.db.models.deletion import django.utils.timezone @@ -70,12 +71,18 @@ class Migration(migrations.Migration): models.BooleanField(blank=True, default=False, null=True), ), ("suspension_lifted", models.DateTimeField(blank=True, null=True)), + ("active", models.BooleanField(default=True, null=True)), ( "activitypub_account_name", - models.CharField(blank=True, max_length=200, null=True), + models.CharField( + blank=True, + max_length=200, + null=True, + validators=[blogs.models.blog.validate_ap_address], + ), ), ( - "owner_email", + "contact_email", models.EmailField(blank=True, max_length=254, null=True), ), ], @@ -140,7 +147,7 @@ class Migration(migrations.Migration): models.CharField(blank=True, max_length=200, null=True), ), ( - "owner_email", + "contact_email", models.EmailField(blank=True, max_length=254, null=True), ), ("approved", models.BooleanField(default=False)), @@ -197,7 +204,7 @@ class Migration(migrations.Migration): ("registration_url", models.URLField(max_length=400, unique=True)), ("description", models.TextField(blank=True, null=True)), ( - "owner_email", + "contact_email", models.EmailField(blank=True, max_length=254, null=True), ), ("announced", models.BooleanField(default=False)), @@ -241,7 +248,7 @@ class Migration(migrations.Migration): models.CharField(blank=True, max_length=200, null=True), ), ( - "owner_email", + "contact_email", models.EmailField(blank=True, max_length=254, null=True), ), ("announced", models.BooleanField(default=False)), diff --git a/blogs/models/blog.py b/blogs/models/blog.py index 471babd..ea282e6 100644 --- a/blogs/models/blog.py +++ b/blogs/models/blog.py @@ -1,11 +1,26 @@ """blog models""" +import re + from django.db import models from django.utils import timezone +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ from .utils import Announcement, Category, ContentWarning +def validate_ap_address(value): + """is the activitypub address valid?""" + p = re.compile("@(.*)@((\w|-)*)\.((\w)*)") + m = p.match(value) + if not m: + raise ValidationError( + _("%(value)s should be in the form '@user@domain.tld'"), + params={"value": value}, + ) + + class BlogData(models.Model): """Base bloggy data""" @@ -45,8 +60,11 @@ class Blog(BlogData): failing = models.BooleanField(default=False, blank=True, null=True) suspended = models.BooleanField(default=False, blank=True, null=True) suspension_lifted = models.DateTimeField(blank=True, null=True) - activitypub_account_name = models.CharField(max_length=200, blank=True, null=True) - owner_email = models.EmailField(blank=True, null=True) + active = models.BooleanField(null=True, default=True) + activitypub_account_name = models.CharField( + max_length=200, blank=True, null=True, validators=[validate_ap_address] + ) + contact_email = models.EmailField(blank=True, null=True) def announce(self): """queue announcement""" @@ -72,10 +90,15 @@ class Blog(BlogData): self.failing = True super().save() - def set_success(self): - """set failing to false""" + def set_success(self, updateddate): + """ + set failing to false + set the updateddate to a datetime + + """ self.failing = False + self.updateddate = updateddate super().save() diff --git a/blogs/models/event.py b/blogs/models/event.py index ebb8c69..b7a4951 100644 --- a/blogs/models/event.py +++ b/blogs/models/event.py @@ -17,7 +17,7 @@ class Event(models.Model): start_date = models.DateField() announcements = models.IntegerField(null=True, blank=True, default=0) activitypub_account_name = models.CharField(max_length=200, blank=True, null=True) - owner_email = models.EmailField(blank=True, null=True) + contact_email = models.EmailField(blank=True, null=True) approved = models.BooleanField(default=False) def save(self, *args, **kwargs): diff --git a/blogs/models/group.py b/blogs/models/group.py index daaabd1..50f713a 100644 --- a/blogs/models/group.py +++ b/blogs/models/group.py @@ -15,7 +15,7 @@ class Group(models.Model): url = models.URLField(max_length=400, unique=True) registration_url = models.URLField(max_length=400, unique=True) description = models.TextField(null=True, blank=True) - owner_email = models.EmailField(blank=True, null=True) + contact_email = models.EmailField(blank=True, null=True) announced = models.BooleanField(default=False) approved = models.BooleanField(default=False) pub_date = models.DateTimeField(null=True, default=None) diff --git a/blogs/models/newsletter.py b/blogs/models/newsletter.py index a7c388e..3346a91 100644 --- a/blogs/models/newsletter.py +++ b/blogs/models/newsletter.py @@ -16,7 +16,7 @@ class Newsletter(models.Model): description = models.TextField(null=True, blank=True) activitypub_account_name = models.CharField(max_length=200, blank=True, null=True) - owner_email = models.EmailField(blank=True, null=True) + contact_email = models.EmailField(blank=True, null=True) announced = models.BooleanField(default=False) approved = models.BooleanField(default=False) pub_date = models.DateTimeField(null=True, default=None) diff --git a/blogs/templates/blogs/confirm-register.html b/blogs/templates/blogs/confirm-register.html index 9e2ab82..15f2de4 100644 --- a/blogs/templates/blogs/confirm-register.html +++ b/blogs/templates/blogs/confirm-register.html @@ -42,9 +42,9 @@ {{ form.activitypub_account_name }} - {{ form.owner_email.errors }} - - {{ form.owner_email }} + {{ form.contact_email.errors }} + + {{ form.contact_email }} diff --git a/blogs/templates/browse/blogs.html b/blogs/templates/browse/blogs.html index d82a933..0c7fd96 100644 --- a/blogs/templates/browse/blogs.html +++ b/blogs/templates/browse/blogs.html @@ -3,15 +3,16 @@ {% block content %}
Title
-
Author
Category
+
Last updated
{% for blog in blogs %}
- -
{{blog.author_name}}
+
{{blog.title}}

{{blog.description}}

{{blog.category_name}}
+
{{ blog.updateddate|date:"D d M Y" }} +
{% endfor %} diff --git a/blogs/templates/contribute.html b/blogs/templates/contribute.html index 9b1e51e..3080b36 100644 --- a/blogs/templates/contribute.html +++ b/blogs/templates/contribute.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} +{% block container %}
{% block intro %}{% endblock %} {% block title %} @@ -7,9 +8,6 @@ {% endblock %}
{% block content %} - - -

You can contribute to Aus GLAMR by registering a blog, newsletter, event, or discussion group. Creators or content should be Australasian-based.

@@ -52,3 +50,4 @@
{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/blogs/templates/search.html b/blogs/templates/search.html index 38d19a7..9d4fd4e 100644 --- a/blogs/templates/search.html +++ b/blogs/templates/search.html @@ -3,7 +3,7 @@ {% block content %}
- +
diff --git a/blogs/templates/subscribe.html b/blogs/templates/subscribe.html index b4d9223..fdd4563 100644 --- a/blogs/templates/subscribe.html +++ b/blogs/templates/subscribe.html @@ -13,7 +13,7 @@

Subscribe via email

-

Get a weekly update listing the latest blog posts, open calls for papers, and upcoming conferences.

+

Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.

diff --git a/blogs/tests/test_commands.py b/blogs/tests/test_commands.py new file mode 100644 index 0000000..775d2fd --- /dev/null +++ b/blogs/tests/test_commands.py @@ -0,0 +1,221 @@ +"""test utility functions""" + +from datetime import datetime, timezone + +from unittest.mock import patch + +from django.core.management import call_command +from django.test import TestCase + +from blogs import models + + +class FeedParserItemMock(object): + title = "" + tags = [] + author = "" + link = "" + summary = "" + updated_parsed = ((),) + published_parsed = ((),) + id = "" + + def __init__( + self, title, tags, author, link, summary, updated_parsed, published_parsed, id + ): + self.title = title + self.tags = tags + self.author = author + self.link = link + self.summary = summary + self.updated_parsed = updated_parsed + self.published_parsed = published_parsed + self.id = id + + +class FeedParserTagMock(object): + term = "" + + def __init__(self, term): + self.term = term + + +class FeedParserMock(object): + entries = [] + + def __init__(self, entries): + self.entries = entries + + +class CommandsTestCase(TestCase): + """test management command functions""" + + def setUp(self): + """set up test conf""" + + self.blog = models.Blog.objects.create( + title="My awesome blog", + url="https://test.com", + feed="https://test.com/feed.xml", + category="LIB", + approved=True, + suspended=False, + ) + + tag_one = FeedParserTagMock(term="testing") + tag_two = FeedParserTagMock(term="python") + tag_three = FeedParserTagMock(term="notglam") + + article = FeedParserItemMock( + title="My amazing blog post", + tags=[tag_one, tag_two], + author="Hugh Rundle", + link="https://test.com/1", + summary="A short summary of my post", + updated_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0), + published_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0), + id="1", + ) + + article_two = FeedParserItemMock( + title="My really amazing blog post", + tags=[tag_one, tag_two, tag_three], + author="Hugh Rundle", + link="https://test.com/2", + summary="A short summary of my next post", + updated_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0), + published_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0), + id="999", + ) + + self.feedparser = FeedParserMock(entries=[article]) + + self.feedparser_exclude = FeedParserMock(entries=[article_two]) + + def test_check_feeds(self): + """test parse a feed for basic blog info""" + + args = {"-q": True} + opts = {} + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser): + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) + article = models.Article.objects.all().first() + self.assertEqual(article.title, "My amazing blog post") + + def test_check_feeds_exclude_tag(self): + """test parse a feed with exclude tag""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser_exclude): + args = {"-q": True} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + def test_check_feeds_unapproved(self): + """test check unapproved blog feed""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + self.blog.approved = False + self.blog.save() + + with patch("feedparser.parse", return_value=self.feedparser): + args = {"-q": True} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + def test_check_feeds_inactive(self): + """test ignore inactive blog feed""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + self.blog.active = False + self.blog.save() + + with patch("feedparser.parse", return_value=self.feedparser): + args = {"-q": True} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + def test_check_feeds_suspended(self): + """test check suspended blog feed""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + self.blog.suspended = True + self.blog.save() + + with patch("feedparser.parse", return_value=self.feedparser): + args = {"-q": True} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + def test_check_feeds_previously_suspended(self): + """test blog published prior to suspension lifted is not ingested""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + self.blog.suspended = False + self.blog.suspension_lifted = datetime( + 2024, 1, 2, 21, 0, 0, 0, tzinfo=timezone.utc + ) + self.blog.save() + + with patch("feedparser.parse", return_value=self.feedparser): + args = {"-q": False} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + def test_check_feeds_previously_suspended_post_after(self): + """test blog published after suspension lifted is ingested""" + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + self.blog.suspended = False + self.blog.suspension_lifted = datetime( + 2023, 12, 31, 0, 0, 0, 0, tzinfo=timezone.utc + ) + self.blog.save() + + with patch("feedparser.parse", return_value=self.feedparser): + args = {"-q": False} + opts = {} + + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) diff --git a/blogs/tests/test_management.py b/blogs/tests/test_management.py index 1957cb2..3df3800 100644 --- a/blogs/tests/test_management.py +++ b/blogs/tests/test_management.py @@ -19,7 +19,7 @@ class ManagementTestCase(TestCase): name="Amazing Conf", url="https://test.com", category="LIB", - start_date=timezone.now() + timedelta(days=3), + start_date=timezone.localtime(timezone.now()) + timedelta(days=3), activitypub_account_name="@conf@conf.conf", approved=True, announcements=1, @@ -28,15 +28,15 @@ class ManagementTestCase(TestCase): self.cfp = models.CallForPapers.objects.create( event=self.conf, name="Call for Papers for Amazing Conf", - opening_date=timezone.now() + timedelta(days=30), - closing_date=timezone.now() + timedelta(days=1), + opening_date=timezone.localtime(timezone.now()) + timedelta(days=30), + closing_date=timezone.localtime(timezone.now()) + timedelta(days=1), ) self.cfp = models.CallForPapers.objects.create( event=self.conf, name="Call for posters", - opening_date=timezone.now() - timedelta(days=30), - closing_date=timezone.now() - timedelta(days=1), + opening_date=timezone.localtime(timezone.now()) - timedelta(days=30), + closing_date=timezone.localtime(timezone.now()) - timedelta(days=1), ) def test_queue_announcements(self): @@ -49,7 +49,7 @@ class ManagementTestCase(TestCase): # event announcement = models.Announcement.objects.first() - start_date = timezone.now() + timedelta(days=3) + start_date = timezone.localtime(timezone.now()) + timedelta(days=3) date = start_date.strftime("%a %d %b %Y") status = f"Amazing Conf (@conf@conf.conf) is a event about Libraries, starting on {date}!\n\nhttps://test.com" @@ -57,8 +57,8 @@ class ManagementTestCase(TestCase): # cfp announcement = models.Announcement.objects.last() - opening_date = timezone.now() + timedelta(days=30) - closing_date = timezone.now() + timedelta(days=1) + opening_date = timezone.localtime(timezone.now()) + timedelta(days=30) + closing_date = timezone.localtime(timezone.now()) + timedelta(days=1) opening_date_str = opening_date.strftime("%a %d %b %Y") closing_date_str = closing_date.strftime("%a %d %b %Y") status = f"Amazing Conf Call for Papers for Amazing Conf is open from {opening_date_str}, closing on {closing_date_str}!\n\nMore info at https://test.com" diff --git a/blogs/tests/test_models.py b/blogs/tests/test_models.py index bc8418f..5171685 100644 --- a/blogs/tests/test_models.py +++ b/blogs/tests/test_models.py @@ -1,6 +1,7 @@ """model tests""" -from datetime import date +from datetime import date, datetime +from datetime import timezone as dt_tz from django.test import TestCase from django.utils import timezone @@ -30,9 +31,12 @@ class BlogTestCase(TestCase): self.blog.failing = True self.blog.save() - self.blog.set_success() + self.blog.set_success( + updateddate=datetime(2020, 1, 1, 12, 59, 0, tzinfo=dt_tz.utc) + ) self.assertEqual(self.blog.failing, False) + self.assertEqual(self.blog.updateddate.isoformat(), "2020-01-01T12:59:00+00:00") def test_set_failing(self): """set_failing class function""" @@ -155,20 +159,6 @@ class NewsletterTestCase(TestCase): class UtilsTestCase(TestCase): """test utility functions""" - def test_announcement(self): - """test announcing""" - - # TODO: mock mastodon somehow - - # announcement = models.Announcement.objects.create( - # status="I have something to say!", - # summary="boasting", - # ) - - # announcement.announce() - - # self.assertEqual(models.Announcement.objects.count(), 0) - def test_content_warning(self): """test CWs""" diff --git a/blogs/tests/test_utilities.py b/blogs/tests/test_utilities.py index 9836cef..9a36451 100644 --- a/blogs/tests/test_utilities.py +++ b/blogs/tests/test_utilities.py @@ -1,8 +1,181 @@ """test utility functions""" -# TODO: -""" -get_feed_info -get_blog_info -get_webfinger_subscribe_uri -""" +import pathlib + +from django.test import TestCase +from unittest.mock import patch + +from blogs import models, utilities + + +class FeedParserFeedMock(object): + title = "" + author = "" + summary = "" + + def __init__(self, title, author, summary): + self.title = title + self.author = author + self.summary = summary + + +class FeedParserMock(object): + feed = FeedParserFeedMock + + def __init__(self, feed): + self.feed = feed + + +def request_error(): + return None + + +class RequestsMock(object): + text = "" + raise_for_status = request_error + + def __init__(self, text, raise_for_status): + self.text = text + self.raise_for_status = raise_for_status + + +class UtilityTests(TestCase): + """utility test cases""" + + def setUp(self): + """set up shared data""" + + feed = FeedParserFeedMock( + title="My amazing blog", + author="Hugh Rundle", + summary="A short summary of my blog", + ) + + feed_partial = FeedParserFeedMock( + title="My amazing blog", author=None, summary=None + ) + + self.feedparser = FeedParserMock(feed=feed) + self.feedparser_partial = FeedParserMock(feed=feed_partial) + + def test_get_feed_info(self): + """test get feed info""" + + with patch("feedparser.parse", return_value=self.feedparser): + data = utilities.get_feed_info("https://test.test") + + self.assertEqual(data["feed"], "https://test.test") + self.assertEqual(data["title"], "My amazing blog") + self.assertEqual(data["author_name"], "Hugh Rundle") + self.assertEqual(data["description"], "A short summary of my blog") + + def test_get_blog_info_no_feed(self): + """test get blog info""" + + with open( + pathlib.Path(__file__).parent.joinpath("data/example.html"), + "r", + encoding="utf-8", + ) as webfile: + website = RequestsMock(text=webfile, raise_for_status=request_error) + + with patch("feedparser.parse", return_value=self.feedparser), patch( + "requests.get", return_value=website + ): + data = utilities.get_blog_info("http://test.test") + + self.assertEqual(data, False) + + def test_get_blog_info_with_good_feed(self): + """test get blog info""" + + with open( + pathlib.Path(__file__).parent.joinpath("data/good-example.html"), + "r", + encoding="utf-8", + ) as webfile: + website = RequestsMock(text=webfile, raise_for_status=request_error) + + with patch("feedparser.parse", return_value=self.feedparser), patch( + "requests.get", return_value=website + ): + data = utilities.get_blog_info("http://test.test") + + self.assertEqual( + data, + { + "feed": "https://test.test/rss.xml", + "title": "My test website with an RSS feed", + "author_name": "Testy McTestface", + "description": "My cool website", + }, + ) + + def test_get_blog_info_with_incomplete_feed(self): + """test get blog info where the feed is incomplete""" + + with open( + pathlib.Path(__file__).parent.joinpath("data/good-example.html"), + "r", + encoding="utf-8", + ) as webfile: + website = RequestsMock(text=webfile, raise_for_status=request_error) + + with patch("feedparser.parse", return_value=self.feedparser_partial), patch( + "requests.get", return_value=website + ): + data = utilities.get_blog_info("http://test.test") + + self.assertEqual( + data, + { + "feed": "https://test.test/rss.xml", + "title": "My test website with an RSS feed", + "author_name": "Testy McTestface", + "description": "My cool website", + }, + ) + + def test_get_blog_info_with_incomplete_head(self): + """test get blog info where the head info is incomplete""" + + with open( + pathlib.Path(__file__).parent.joinpath("data/partial-example.html"), + "r", + encoding="utf-8", + ) as webfile: + website = RequestsMock(text=webfile, raise_for_status=request_error) + + with patch("feedparser.parse", return_value=self.feedparser), patch( + "requests.get", return_value=website + ): + data = utilities.get_blog_info("http://test.test") + + self.assertEqual(data["title"], "My test website with an RSS feed") + self.assertEqual(data["author_name"], "Hugh Rundle") + self.assertEqual(data["description"], "A short summary of my blog") + + def test_get_blog_info_with_incomplete_head_and_partial_feed(self): + """test get blog info where both the feed and website head are incomplete""" + + with open( + pathlib.Path(__file__).parent.joinpath("data/partial-example.html"), + "r", + encoding="utf-8", + ) as webfile: + website = RequestsMock(text=webfile, raise_for_status=request_error) + + with patch("feedparser.parse", return_value=self.feedparser_partial), patch( + "requests.get", return_value=website + ): + data = utilities.get_blog_info("http://test.test") + + self.assertEqual(data["title"], "My test website with an RSS feed") + self.assertEqual(data["author_name"], None) + self.assertEqual(data["description"], None) + + def test_get_webfinger_subscribe_uri(self): + """test get webfinger data""" + + # TODO + pass diff --git a/blogs/tests/test_views.py b/blogs/tests/test_views.py index 9054534..814d505 100644 --- a/blogs/tests/test_views.py +++ b/blogs/tests/test_views.py @@ -1,5 +1,7 @@ """test views""" +from datetime import timedelta + from unittest.mock import patch from django.contrib.auth.models import AnonymousUser @@ -16,7 +18,7 @@ class PublicTests(TestCase): """Public views test cases""" def setUp(self): - start_date = timezone.now() + start_date = timezone.now() + timedelta(days=5) self.factory = RequestFactory() self.glam_conf = models.Event.objects.create( @@ -53,7 +55,7 @@ class PublicTests(TestCase): blogs_response = self.client.get(reverse("blogs")) self.assertEqual(blogs_response.status_code, 200) - confs_response = self.client.get(reverse("conferences")) + confs_response = self.client.get(reverse("events")) self.assertEqual(confs_response.status_code, 200) groups_response = self.client.get(reverse("groups")) @@ -133,15 +135,36 @@ class PublicTests(TestCase): exists = models.Event.objects.filter(name="My event").exists() self.assertTrue(exists) - def test_register_cfp(self): + def test_register_cfp_unapproved_event(self): """post CFP registration form""" view = views.RegisterCallForPapers.as_view() form = forms.RegisterCallForPapersForm() form.data["event"] = self.glam_conf.id form.data["name"] = "Call for Papers" - form.data["url"] = "https://www.example.com" - form.data["category"] = "GLAM" + form.data["details"] = "Here are some details" + form.data["opening_date"] = "01/01/2024" + form.data["closing_date"] = "28/01/2024" + + request = self.factory.post("register-cfp/", form.data) + request.user = AnonymousUser() + + view(request) + + exists = models.CallForPapers.objects.filter(name="Call for Papers").exists() + self.assertFalse(exists) + + def test_register_cfp_approved_event(self): + """post CFP registration form""" + + self.glam_conf.approved = True + self.glam_conf.save() + + view = views.RegisterCallForPapers.as_view() + form = forms.RegisterCallForPapersForm() + form.data["event"] = self.glam_conf.id + form.data["name"] = "Call for Papers" + form.data["details"] = "Here are some details" form.data["opening_date"] = "01/01/2024" form.data["closing_date"] = "28/01/2024" diff --git a/blogs/utilities.py b/blogs/utilities.py index caceaa9..2d7344e 100644 --- a/blogs/utilities.py +++ b/blogs/utilities.py @@ -22,8 +22,9 @@ def get_feed_info(feed): blog["feed"] = feed blog["title"] = getattr(b.feed, "title", "") blog["author_name"] = getattr(b.feed, "author", None) - blog["description"] = getattr(b.feed, "subtitle", "") - blog["last_updated"] = getattr(b.feed, "updated_parsed", "") + blog["description"] = getattr(b.feed, "summary", None) or getattr( + b.feed, "subtitle", None + ) return blog @@ -46,24 +47,46 @@ def get_blog_info(url): try: author = soup.select_one('meta[name="creator"]').get("content") except AttributeError: - author = "" + author = None + + try: + description = soup.select_one('meta[name="description"]').get("content") + except AttributeError: + description = None if len(links) > 0: blog_info = get_feed_info(links[0].get("href")) - blog_info["title"] = blog_info["title"] or soup.title.string - blog_info["author_name"] = blog_info["author_name"] or author + if hasattr(soup, "title") and soup.title: + blog_info["title"] = soup.title.string # use the scraped title + else: + blog_info["title"] = blog_info.get( + "title" + ) # use the title from the feed + + blog_info["description"] = description or blog_info.get("description", "") + + else: + return False # if there is no feed info we need to put the onus back on the user to fill in the data + + normalised_author = "" + if author: + normalised_author = author.replace("(noreply@blogger.com)", "") + if normalised_author.strip() == "Unknown": + normalised_author = "" + + blog_info["author_name"] = normalised_author return blog_info except requests.Timeout: - print(f"TIMEOUT error registering {url}, trying longer timeout") + logging.warning(f"TIMEOUT error registering {url}, trying longer timeout") r = requests.get(url, headers=headers, timeout=(31, 31)) r.raise_for_status() # let it flow through, a timeout here means the site is unreasonably slow except Exception as e: - print(f"CONNECTION ERROR when registering {url}") - print(e) + logging.error(f"CONNECTION ERROR when registering {url}") + logging.error(e) return False @@ -83,13 +106,13 @@ def get_webfinger_subscribe_uri(username): r.raise_for_status() except requests.Timeout: - print(f"TIMEOUT error finding {username}, trying longer timeout") + logging.warning(f"TIMEOUT error finding {username}, trying longer timeout") r = requests.get(webfinger_url, headers=headers, timeout=(31, 31)) r.raise_for_status() # let it flow through, a timeout here means the site is unreasonably slow except Exception as e: - print(f"CONNECTION ERROR when subscribing via {username}") - print(e) + logging.error(f"CONNECTION ERROR when subscribing via {username}") + logging.error(e) return None data = r.json() diff --git a/blogs/views/public.py b/blogs/views/public.py index df4061d..323fb67 100644 --- a/blogs/views/public.py +++ b/blogs/views/public.py @@ -1,8 +1,6 @@ """public views (no need to log in)""" # pylint: disable=R6301 -import os - from itertools import chain from operator import attrgetter @@ -20,16 +18,13 @@ from django.views import View from blogs import forms, models from blogs.utilities import get_blog_info, get_webfinger_subscribe_uri + class HomeFeed(View): """the home feed when someone visits the site""" def get(self, request): """display home page""" - path = os.path.join(settings.BASE_DIR, "static/") - - print(path) - latest = models.Article.objects.order_by("-pubdate")[:10] data = {"title": "Latest blog posts", "latest": latest} @@ -42,7 +37,9 @@ class Blogs(View): def get(self, request): """here they are""" - blogs = models.Blog.objects.filter(approved=True) + blogs = models.Blog.objects.filter(approved=True, active=True).order_by( + "-updateddate" + ) for blog in blogs: blog.category_name = models.Category(blog.category).label data = {"title": "Blogs and websites", "blogs": blogs} @@ -140,14 +137,13 @@ class RegisterBlog(View): data["blog_info"] = blog_info else: - data["error"] = "Could not auto-discover your feed info, please enter manually" + data[ + "error" + ] = "Could not auto-discover your feed info, please enter manually" return render(request, "blogs/confirm-register.html", data) - data = { - "title": "Register your blog", - "form": form - } + data = {"title": "Register your blog", "form": form} return render(request, "blogs/register.html", data) @@ -311,7 +307,7 @@ class Search(View): "description", weight="C" ) - conferences = ( + events = ( models.Event.objects.annotate(rank=SearchRank(conference_vector, query)) .filter(rank__gte=0.1) .order_by("-rank") @@ -348,7 +344,7 @@ class Search(View): ) combined = sorted( - chain(articles, conferences, cfps, newsletters, groups), + chain(articles, events, cfps, newsletters, groups), key=attrgetter("rank"), reverse=True, )