fixes

.gitignore

@@ -1,2 +1,3 @@
 data
 static
+fixtures

README.md

@@ -2,6 +2,25 @@
 
 A django app running on Docker. Replaces _Aus GLAM Blogs_.
 
+## Deploy
+
+* `docker compose build`
+* `./glamr-dev createsuperuser`
+* `./glamr-dev makemigrations`
+* `./glamr-dev migrate`
+* `docker compose up`
+
+* set up database backups (as cron jobs):
+
+    * `/usr/bin/docker exec -u root ausglamr-db-1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump`
+    * `/usr/bin/docker cp ausglamr-db-1:/tmp/ausglamr_backup.dump /home/hugh/backups/`
+
+* set up cron jobs for management commands as below
+
+## Migrating
+
+See `data/help.txt`
+
 ## Admin
 
 Don't forget to add some Content Warnings for use by the Mastodon bot.

ausglamr/settings.py

@@ -28,11 +28,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = env("SECRET_KEY")
 
+# DEPLOYMENT
+
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env("DEBUG")
-
 ALLOWED_HOSTS = [env("DOMAIN")]
+CSRF_COOKIE_SECURE=True
+SESSION_COOKIE_SECURE=True
 CSRF_TRUSTED_ORIGINS = [f'https://{env("DOMAIN")}']
+CONN_MAX_AGE=None # persistent DB connection
 
 # Application definition
 
@@ -149,6 +153,7 @@ ADMIN_EMAIL = env("ADMIN_EMAIL")  # custom
 
 # django email settings
 DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")
+SERVER_EMAIL = env("SERVER_EMAIL")
 EMAIL_HOST = env("EMAIL_HOST")
 EMAIL_HOST_USER = env("EMAIL_HOST_USER")
 EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD")
@@ -158,3 +163,19 @@ EMAIL_USE_SSL = env("EMAIL_USE_SSL")
 # mastodon
 MASTODON_ACCESS_TOKEN = env("MASTODON_ACCESS_TOKEN")
 MASTODON_DOMAIN = env("MASTODON_DOMAIN")
+
+# LOGGING
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+        },
+    },
+    "root": {
+        "handlers": ["console"],
+        "level": "INFO",
+    },
+}

ausglamr/urls.py

@@ -41,7 +41,7 @@ urlpatterns = [
     path("subscribe", views.Subscribe.as_view(), name="subscribe"),
     path("subscribe-email", views.SubscribeEmail.as_view(), name="subscribe-email"),
     path(
-        "confirm-subscribe-email/<token>/<id>",
+        "confirm-subscribe-email/<token>/<user_id>",
         views.ConfirmEmail.as_view(),
         name="confirm-subscribe-email",
     ),

blogs/admin.py

@@ -19,9 +19,11 @@ def approve(modeladmin, request, queryset):
         instance.announce()
 
         if hasattr(instance, "event"):  # CFP
-            recipient = instance.event.owner_email
+            recipient = instance.event.contact_email
-        if hasattr(instance, "owner_email"):  # overrides above in case needed in future
+        if hasattr(
-            recipient = instance.owner_email
+            instance, "contact_email"
+        ):  # overrides above in case needed in future
+            recipient = instance.contact_email
 
         if recipient:
             if hasattr(instance, "name"):
@@ -47,7 +49,7 @@ def suspend(modeladmin, request, queryset):
     queryset.update(suspended=True)
 
     for instance in queryset:
-        if hasattr(instance, "owner_email"):
+        if hasattr(instance, "contact_email"):
             if hasattr(instance, "name"):
                 title = instance.name
             else:
@@ -58,7 +60,7 @@ def suspend(modeladmin, request, queryset):
             <p>Your blog {title} has been suspended from AusGLAMR. It may be unsuspended in future once the issue is resolved. If you would like more information, please reply to this email.</p> \
             </body></html>"
 
-            utilities.send_email(subject, message, instance.owner_email)
+            utilities.send_email(subject, message, instance.contact_email)
 
 
 @admin.action(description="Unsuspend selected blogs")
@@ -67,7 +69,7 @@ def unsuspend(modeladmin, request, queryset):
     queryset.update(suspended=False, suspension_lifted=timezone.now())
 
     for instance in queryset:
-        if hasattr(instance, "owner_email"):
+        if hasattr(instance, "contact_email"):
             if hasattr(instance, "name"):
                 title = instance.name
             else:
@@ -78,7 +80,7 @@ def unsuspend(modeladmin, request, queryset):
             <p>The suspension on your blog {title} has been removed on AusGLAMR. Please note that articles published whilst it was suspended will not be added to AusGLAMR retrospectively. If you would like more information, please reply to this email.</p> \
             </body></html>"
 
-            utilities.send_email(subject, message, instance.owner_email)
+            utilities.send_email(subject, message, instance.contact_email)
 
 
 @admin.action(description="Confirm selected subscribers")
@@ -93,6 +95,18 @@ def unconfirm(modeladmin, request, queryset):
     queryset.update(confirmed=False)
 
 
+@admin.action(description="Deactivate selected blogs")
+def disable(modeladmin, request, queryset):
+    """disable selected"""
+    queryset.update(active=False)
+
+
+@admin.action(description="Activate selected blogs")
+def activate(modeladmin, request, queryset):
+    """un-disable selected"""
+    queryset.update(active=True)
+
+
 @admin.action(description="Send confirmation request to selected subscribers")
 def send_conf_request(modeladmin, request, queryset):
     """send to selected"""
@@ -106,16 +120,17 @@ class Blog(admin.ModelAdmin):
     """display settings for blogs"""
 
     list_display = (
-        "title",
         "url",
+        "title",
         "author_name",
         "approved",
         "announced",
         "suspended",
         "failing",
+        "active",
     )
     ordering = ["approved", "-suspended", "-failing"]
-    actions = [approve, unapprove, suspend, unsuspend]
+    actions = [approve, unapprove, suspend, unsuspend, activate, disable]
 
 
 @admin.register(models.Article)

blogs/forms.py

@@ -19,7 +19,7 @@ class RegisterBlogForm(forms.ModelForm):
         """set fields and model"""
 
         model = Blog
-        fields = ["url", "category", "activitypub_account_name", "owner_email"]
+        fields = ["url", "category", "activitypub_account_name", "contact_email"]
 
 
 class ConfirmBlogForm(forms.ModelForm):
@@ -37,7 +37,7 @@ class ConfirmBlogForm(forms.ModelForm):
             "description",
             "category",
             "activitypub_account_name",
-            "owner_email",
+            "contact_email",
         ]
 
 
@@ -55,7 +55,7 @@ class RegisterConferenceForm(forms.ModelForm):
             "description",
             "start_date",
             "activitypub_account_name",
-            "owner_email",
+            "contact_email",
         ]
         widgets = {
             "start_date": DateInput(),
@@ -97,7 +97,7 @@ class RegisterGroupForm(forms.ModelForm):
             "url",
             "registration_url",
             "description",
-            "owner_email",
+            "contact_email",
         ]
 
 
@@ -115,7 +115,7 @@ class RegisterNewsletterForm(forms.ModelForm):
             "url",
             "description",
             "activitypub_account_name",
-            "owner_email",
+            "contact_email",
         ]
 
 

blogs/management/commands/check_feeds.py

@@ -1,5 +1,8 @@
 """call this from cron to run through all the feeds to find new posts"""
 
+import logging
+import traceback
+
 from datetime import datetime, timedelta, timezone
 
 import feedparser
@@ -22,11 +25,12 @@ def get_tags(dictionary):
     """parse out tags from blog and upsert as tag instances"""
     tags = []
     for tag_obj in dictionary:
-        tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first()
+        if tag_obj.term.lower() != "uncategorized":
-        if not tag:
+            tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first()
-            tag = models.Tag.objects.create(name=tag_obj.term.lower())
+            if not tag:
+                tag = models.Tag.objects.create(name=tag_obj.term.lower())
 
-        tags.append(tag)
+            tags.append(tag)
 
     return tags
 
@@ -34,14 +38,24 @@ def get_tags(dictionary):
 class Command(BaseCommand):
     """the check_feeds command"""
 
-    # we could add arguments but we don't really need any
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-q",
+            action="store_true",
+            help="Suppress non-error messages",
+        )
 
     def handle(self, *args, **options):
         """check feeds and update database"""
 
-        print(f"checking feeds at {django_timezone.localtime(django_timezone.now())}")
+        if not options["q"]:
+            logging.info(
+                f"checking feeds at {django_timezone.localtime(django_timezone.now())}"
+            )
 
-        blogs = models.Blog.objects.filter(approved=True, suspended=False).all()
+        blogs = models.Blog.objects.filter(
+            approved=True, suspended=False, active=True
+        ).all()
         for blog in blogs:
             try:
                 data = feedparser.parse(blog.feed)
@@ -50,24 +64,24 @@ class Command(BaseCommand):
                     if not models.Article.objects.filter(
                         Q(url=article.link) | Q(guid=article.id)
                     ).exists():
-                        if (
+                        if blog.suspension_lifted and (
                             blog.suspension_lifted
-                            and blog.suspension_lifted
+                            > date_to_tz_aware(article.updated_parsed)
-                            < date_to_tz_aware(article.updated_parsed)
                         ):
-                            continue  # don't ingest posts published during a suspension
+                            continue  # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension)
 
-                        tags = get_tags(
+                        taglist = getattr(article, "tags", None) or getattr(
-                            getattr(article, "tags", None)
+                            article, "categories", []
-                            or getattr(article, "categories", [])
                         )
 
+                        tags = [tag.term.lower() for tag in taglist]
+
                         opt_out = False
                         # don't include posts with opt out tags
                         for tag in tags:
                             if (
                                 len(
-                                    {tag.name}
+                                    {tag}
                                     & {
                                         "notglam",
                                         "notglamr",
@@ -99,8 +113,14 @@ class Command(BaseCommand):
                                 guid=article.id,
                             )
 
-                            for tag in tags:
+                            tags_to_add = get_tags(
+                                getattr(article, "tags", None)
+                                or getattr(article, "categories", [])
+                            )
+
+                            for tag in tags_to_add:
                                 instance.tags.add(tag)
+
                             instance.save()
 
                             cutoff = django_timezone.now() - timedelta(days=3)
@@ -108,11 +128,16 @@ class Command(BaseCommand):
                             if newish:
                                 instance.announce()
 
-                        blog.set_success()
+                        blog.set_success(
+                            updateddate=date_to_tz_aware(article.updated_parsed)
+                        )
 
             except Exception as e:
                 blog.set_failing()
-                print(f"ERROR WITH BLOG {blog.title} - {blog.url}")
+                logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}")
-                print(e)
+                logging.error(e)
 
-        print(f"completed run at {django_timezone.localtime(django_timezone.now())}")
+        if not options["q"]:
+            logging.info(
+                f"completed run at {django_timezone.localtime(django_timezone.now())}"
+            )

blogs/management/commands/queue_announcements.py

@@ -16,7 +16,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         """check whether we need to queue announcements and queue them"""
 
-        conferences = models.Event.objects.filter(
+        events = models.Event.objects.filter(
             approved=True,
             announcements__lt=3,
             start_date__gte=timezone.now(),
@@ -24,10 +24,10 @@ class Command(BaseCommand):
         calls = models.CallForPapers.objects.filter(
             announcements__lt=3,
             closing_date__gte=timezone.now(),
-            conference__approved=True,
+            event__approved=True,
         )
 
-        for conf in conferences:
+        for conf in events:
             delta = conf.start_date - timezone.now().date()
 
             if (

blogs/management/commands/send_weekly_email.py

@@ -1,6 +1,7 @@
 """send the weekly email"""
 
 from datetime import timedelta
+import logging
 import random
 
 from django.conf import settings
@@ -22,7 +23,7 @@ class Command(BaseCommand):
 
         subscribers = models.Subscriber.objects.filter(confirmed=True)
 
-        print(
+        logging.info(
             f"Sending weekly emails to {len(subscribers)} subscribers at {timezone.now()}"
         )
 
@@ -31,7 +32,7 @@ class Command(BaseCommand):
         articles = models.Article.objects.filter(pubdate__gte=cutoff)
         events = models.Event.objects.filter(approved=True, pub_date__gte=cutoff)
         cfps = models.CallForPapers.objects.filter(
-            conference__approved=True, closing_date__gte=timezone.now().date()
+            event__approved=True, closing_date__gte=timezone.now().date()
         )
         newsletters = models.Newsletter.objects.filter(
             approved=True, pub_date__gte=cutoff
@@ -210,4 +211,4 @@ class Command(BaseCommand):
 
             send_email(subject, message, subscriber.email)
 
-        print(f"Weekly emails completed {timezone.now()}")
+        logging.info(f"Weekly emails completed {timezone.now()}")

blogs/migrations/0001_initial.py

@@ -1,5 +1,6 @@
-# Generated by Django 4.2.7 on 2024-01-03 03:35
+# Generated by Django 4.2.7 on 2024-01-07 04:55
 
+import blogs.models.blog
 from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
@@ -70,12 +71,18 @@ class Migration(migrations.Migration):
                     models.BooleanField(blank=True, default=False, null=True),
                 ),
                 ("suspension_lifted", models.DateTimeField(blank=True, null=True)),
+                ("active", models.BooleanField(default=True, null=True)),
                 (
                     "activitypub_account_name",
-                    models.CharField(blank=True, max_length=200, null=True),
+                    models.CharField(
+                        blank=True,
+                        max_length=200,
+                        null=True,
+                        validators=[blogs.models.blog.validate_ap_address],
+                    ),
                 ),
                 (
-                    "owner_email",
+                    "contact_email",
                     models.EmailField(blank=True, max_length=254, null=True),
                 ),
             ],
@@ -140,7 +147,7 @@ class Migration(migrations.Migration):
                     models.CharField(blank=True, max_length=200, null=True),
                 ),
                 (
-                    "owner_email",
+                    "contact_email",
                     models.EmailField(blank=True, max_length=254, null=True),
                 ),
                 ("approved", models.BooleanField(default=False)),
@@ -197,7 +204,7 @@ class Migration(migrations.Migration):
                 ("registration_url", models.URLField(max_length=400, unique=True)),
                 ("description", models.TextField(blank=True, null=True)),
                 (
-                    "owner_email",
+                    "contact_email",
                     models.EmailField(blank=True, max_length=254, null=True),
                 ),
                 ("announced", models.BooleanField(default=False)),
@@ -241,7 +248,7 @@ class Migration(migrations.Migration):
                     models.CharField(blank=True, max_length=200, null=True),
                 ),
                 (
-                    "owner_email",
+                    "contact_email",
                     models.EmailField(blank=True, max_length=254, null=True),
                 ),
                 ("announced", models.BooleanField(default=False)),

blogs/models/blog.py

@@ -1,11 +1,26 @@
 """blog models"""
 
+import re
+
 from django.db import models
 from django.utils import timezone
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
 
 from .utils import Announcement, Category, ContentWarning
 
 
+def validate_ap_address(value):
+    """is the activitypub address valid?"""
+    p = re.compile("@(.*)@((\w|-)*)\.((\w)*)")
+    m = p.match(value)
+    if not m:
+        raise ValidationError(
+            _("%(value)s should be in the form '@user@domain.tld'"),
+            params={"value": value},
+        )
+
+
 class BlogData(models.Model):
     """Base bloggy data"""
 
@@ -45,8 +60,11 @@ class Blog(BlogData):
     failing = models.BooleanField(default=False, blank=True, null=True)
     suspended = models.BooleanField(default=False, blank=True, null=True)
     suspension_lifted = models.DateTimeField(blank=True, null=True)
-    activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
+    active = models.BooleanField(null=True, default=True)
-    owner_email = models.EmailField(blank=True, null=True)
+    activitypub_account_name = models.CharField(
+        max_length=200, blank=True, null=True, validators=[validate_ap_address]
+    )
+    contact_email = models.EmailField(blank=True, null=True)
 
     def announce(self):
         """queue announcement"""
@@ -72,10 +90,15 @@ class Blog(BlogData):
         self.failing = True
         super().save()
 
-    def set_success(self):
+    def set_success(self, updateddate):
-        """set failing to false"""
+        """
+        set failing to false
+        set the updateddate to a datetime
+
+        """
 
         self.failing = False
+        self.updateddate = updateddate
         super().save()
 
 

blogs/models/event.py

@@ -17,7 +17,7 @@ class Event(models.Model):
     start_date = models.DateField()
     announcements = models.IntegerField(null=True, blank=True, default=0)
     activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
-    owner_email = models.EmailField(blank=True, null=True)
+    contact_email = models.EmailField(blank=True, null=True)
     approved = models.BooleanField(default=False)
 
     def save(self, *args, **kwargs):

blogs/models/group.py

@@ -15,7 +15,7 @@ class Group(models.Model):
     url = models.URLField(max_length=400, unique=True)
     registration_url = models.URLField(max_length=400, unique=True)
     description = models.TextField(null=True, blank=True)
-    owner_email = models.EmailField(blank=True, null=True)
+    contact_email = models.EmailField(blank=True, null=True)
     announced = models.BooleanField(default=False)
     approved = models.BooleanField(default=False)
     pub_date = models.DateTimeField(null=True, default=None)

blogs/models/newsletter.py

@@ -16,7 +16,7 @@ class Newsletter(models.Model):
 
     description = models.TextField(null=True, blank=True)
     activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
-    owner_email = models.EmailField(blank=True, null=True)
+    contact_email = models.EmailField(blank=True, null=True)
     announced = models.BooleanField(default=False)
     approved = models.BooleanField(default=False)
     pub_date = models.DateTimeField(null=True, default=None)

blogs/templates/blogs/confirm-register.html

@@ -42,9 +42,9 @@
             <label for="{{ form.activitypub_account_name.id_for_label }}">Activitypub account name:</label>
             {{ form.activitypub_account_name }}
 
-            {{ form.owner_email.errors }}
+            {{ form.contact_email.errors }}
-            <label for="{{ form.owner_email.id_for_label }}">Owner email:</label>
+            <label for="{{ form.contact_email.id_for_label }}">Contact email:</label>
-            {{ form.owner_email }}
+            {{ form.contact_email }}
 
             <input class="button-primary u-pull-right" type="submit" value="Register!">
         </div>

blogs/templates/browse/blogs.html

@@ -3,15 +3,16 @@
 {% block content %}
   <div class="listing l-three header">
     <div>Title</div>
-    <div>Author</div>
     <div>Category</div>
+    <div>Last updated</div>
   </div>
   <hr/>
   {% for blog in blogs %}
   <div class="listing l-three">
-    <div><a href="{{blog.url}}">{{blog.title}}</a></div>
+    <div><a href="{{blog.url}}">{{blog.title}}</a><p>{{blog.description}}</p></div>
-    <div>{{blog.author_name}}</div>
     <div><div class="end badge badge_{{blog.category}}">{{blog.category_name}}</div></div>
+    <div>{{ blog.updateddate|date:"D d M Y" }}
+    </div>
   </div>
   <hr/>
   {% endfor %}

blogs/templates/contribute.html

@@ -1,5 +1,6 @@
 {% extends "layout.html" %}
 
+{% block container %}
 <main class="container pop-boxes">
     {% block intro %}{% endblock %}
     {% block title %}
@@ -7,9 +8,6 @@
     {% endblock %}
     <section>
         {% block content %}
-
-
-
         <section>
             <p>You can contribute to <em>Aus GLAMR</em> by registering a blog, newsletter, event, or discussion group. Creators or content should be Australasian-based.</p>
         </section>
@@ -52,3 +50,4 @@
         </section>
         {% endblock %}
 </section>
+{% endblock %}

blogs/templates/search.html

@@ -3,7 +3,7 @@
 {% block content %}
 
 <form action="{% url 'search' %}" method="get">
-    <label for="keywords">Search for keywords across blog posts, groups, newsletters, and conferences</label>
+    <label for="keywords">Search for keywords across blog posts, groups, newsletters, and events</label>
     <input type="text" name="q" value="{% if query %}{{ query }}{% endif %}">
     <button class="button" type="submit">Search!</button>
 </form>

blogs/templates/subscribe.html

@@ -13,7 +13,7 @@
                 <hgroup>
                     <h3>Subscribe via email</h3>
                 </hgroup>
-                <p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming conferences.</p>
+                <p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.</p>
                 <div>
                     <a class="button" href="{% url 'subscribe-email' %}">Subscribe via email</a>
                 </div>

blogs/tests/test_commands.py

@@ -0,0 +1,221 @@
+"""test utility functions"""
+
+from datetime import datetime, timezone
+
+from unittest.mock import patch
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from blogs import models
+
+
+class FeedParserItemMock(object):
+    title = ""
+    tags = []
+    author = ""
+    link = ""
+    summary = ""
+    updated_parsed = ((),)
+    published_parsed = ((),)
+    id = ""
+
+    def __init__(
+        self, title, tags, author, link, summary, updated_parsed, published_parsed, id
+    ):
+        self.title = title
+        self.tags = tags
+        self.author = author
+        self.link = link
+        self.summary = summary
+        self.updated_parsed = updated_parsed
+        self.published_parsed = published_parsed
+        self.id = id
+
+
+class FeedParserTagMock(object):
+    term = ""
+
+    def __init__(self, term):
+        self.term = term
+
+
+class FeedParserMock(object):
+    entries = []
+
+    def __init__(self, entries):
+        self.entries = entries
+
+
+class CommandsTestCase(TestCase):
+    """test management command functions"""
+
+    def setUp(self):
+        """set up test conf"""
+
+        self.blog = models.Blog.objects.create(
+            title="My awesome blog",
+            url="https://test.com",
+            feed="https://test.com/feed.xml",
+            category="LIB",
+            approved=True,
+            suspended=False,
+        )
+
+        tag_one = FeedParserTagMock(term="testing")
+        tag_two = FeedParserTagMock(term="python")
+        tag_three = FeedParserTagMock(term="notglam")
+
+        article = FeedParserItemMock(
+            title="My amazing blog post",
+            tags=[tag_one, tag_two],
+            author="Hugh Rundle",
+            link="https://test.com/1",
+            summary="A short summary of my post",
+            updated_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
+            published_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
+            id="1",
+        )
+
+        article_two = FeedParserItemMock(
+            title="My really amazing blog post",
+            tags=[tag_one, tag_two, tag_three],
+            author="Hugh Rundle",
+            link="https://test.com/2",
+            summary="A short summary of my next post",
+            updated_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
+            published_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
+            id="999",
+        )
+
+        self.feedparser = FeedParserMock(entries=[article])
+
+        self.feedparser_exclude = FeedParserMock(entries=[article_two])
+
+    def test_check_feeds(self):
+        """test parse a feed for basic blog info"""
+
+        args = {"-q": True}
+        opts = {}
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)
+            article = models.Article.objects.all().first()
+            self.assertEqual(article.title, "My amazing blog post")
+
+    def test_check_feeds_exclude_tag(self):
+        """test parse a feed with exclude tag"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        with patch("feedparser.parse", return_value=self.feedparser_exclude):
+            args = {"-q": True}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 0)
+            self.assertEqual(models.Tag.objects.count(), 0)
+
+    def test_check_feeds_unapproved(self):
+        """test check unapproved blog feed"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        self.blog.approved = False
+        self.blog.save()
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            args = {"-q": True}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 0)
+            self.assertEqual(models.Tag.objects.count(), 0)
+
+    def test_check_feeds_inactive(self):
+        """test ignore inactive blog feed"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        self.blog.active = False
+        self.blog.save()
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            args = {"-q": True}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 0)
+            self.assertEqual(models.Tag.objects.count(), 0)
+
+    def test_check_feeds_suspended(self):
+        """test check suspended blog feed"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        self.blog.suspended = True
+        self.blog.save()
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            args = {"-q": True}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 0)
+            self.assertEqual(models.Tag.objects.count(), 0)
+
+    def test_check_feeds_previously_suspended(self):
+        """test blog published prior to suspension lifted is not ingested"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        self.blog.suspended = False
+        self.blog.suspension_lifted = datetime(
+            2024, 1, 2, 21, 0, 0, 0, tzinfo=timezone.utc
+        )
+        self.blog.save()
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            args = {"-q": False}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 0)
+            self.assertEqual(models.Tag.objects.count(), 0)
+
+    def test_check_feeds_previously_suspended_post_after(self):
+        """test blog published after suspension lifted is ingested"""
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        self.blog.suspended = False
+        self.blog.suspension_lifted = datetime(
+            2023, 12, 31, 0, 0, 0, 0, tzinfo=timezone.utc
+        )
+        self.blog.save()
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            args = {"-q": False}
+            opts = {}
+
+            value = call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)

blogs/tests/test_management.py

@@ -19,7 +19,7 @@ class ManagementTestCase(TestCase):
             name="Amazing Conf",
             url="https://test.com",
             category="LIB",
-            start_date=timezone.now() + timedelta(days=3),
+            start_date=timezone.localtime(timezone.now()) + timedelta(days=3),
             activitypub_account_name="@conf@conf.conf",
             approved=True,
             announcements=1,
@@ -28,15 +28,15 @@ class ManagementTestCase(TestCase):
         self.cfp = models.CallForPapers.objects.create(
             event=self.conf,
             name="Call for Papers for Amazing Conf",
-            opening_date=timezone.now() + timedelta(days=30),
+            opening_date=timezone.localtime(timezone.now()) + timedelta(days=30),
-            closing_date=timezone.now() + timedelta(days=1),
+            closing_date=timezone.localtime(timezone.now()) + timedelta(days=1),
         )
 
         self.cfp = models.CallForPapers.objects.create(
             event=self.conf,
             name="Call for posters",
-            opening_date=timezone.now() - timedelta(days=30),
+            opening_date=timezone.localtime(timezone.now()) - timedelta(days=30),
-            closing_date=timezone.now() - timedelta(days=1),
+            closing_date=timezone.localtime(timezone.now()) - timedelta(days=1),
         )
 
     def test_queue_announcements(self):
@@ -49,7 +49,7 @@ class ManagementTestCase(TestCase):
 
         # event
         announcement = models.Announcement.objects.first()
-        start_date = timezone.now() + timedelta(days=3)
+        start_date = timezone.localtime(timezone.now()) + timedelta(days=3)
         date = start_date.strftime("%a %d %b %Y")
         status = f"Amazing Conf (@conf@conf.conf) is a event about Libraries, starting on {date}!\n\nhttps://test.com"
 
@@ -57,8 +57,8 @@ class ManagementTestCase(TestCase):
 
         # cfp
         announcement = models.Announcement.objects.last()
-        opening_date = timezone.now() + timedelta(days=30)
+        opening_date = timezone.localtime(timezone.now()) + timedelta(days=30)
-        closing_date = timezone.now() + timedelta(days=1)
+        closing_date = timezone.localtime(timezone.now()) + timedelta(days=1)
         opening_date_str = opening_date.strftime("%a %d %b %Y")
         closing_date_str = closing_date.strftime("%a %d %b %Y")
         status = f"Amazing Conf Call for Papers for Amazing Conf is open from {opening_date_str}, closing on {closing_date_str}!\n\nMore info at https://test.com"

blogs/tests/test_models.py

@@ -1,6 +1,7 @@
 """model tests"""
 
-from datetime import date
+from datetime import date, datetime
+from datetime import timezone as dt_tz
 
 from django.test import TestCase
 from django.utils import timezone
@@ -30,9 +31,12 @@ class BlogTestCase(TestCase):
 
         self.blog.failing = True
         self.blog.save()
-        self.blog.set_success()
+        self.blog.set_success(
+            updateddate=datetime(2020, 1, 1, 12, 59, 0, tzinfo=dt_tz.utc)
+        )
 
         self.assertEqual(self.blog.failing, False)
+        self.assertEqual(self.blog.updateddate.isoformat(), "2020-01-01T12:59:00+00:00")
 
     def test_set_failing(self):
         """set_failing class function"""
@@ -155,20 +159,6 @@ class NewsletterTestCase(TestCase):
 class UtilsTestCase(TestCase):
     """test utility functions"""
 
-    def test_announcement(self):
-        """test announcing"""
-
-        # TODO: mock mastodon somehow
-
-        # announcement = models.Announcement.objects.create(
-        #     status="I have something to say!",
-        #     summary="boasting",
-        # )
-
-        # announcement.announce()
-
-        # self.assertEqual(models.Announcement.objects.count(), 0)
-
     def test_content_warning(self):
         """test CWs"""
 

blogs/tests/test_utilities.py

@@ -1,8 +1,181 @@
 """test utility functions"""
 
-# TODO:
+import pathlib
-"""
+
-get_feed_info
+from django.test import TestCase
-get_blog_info
+from unittest.mock import patch
-get_webfinger_subscribe_uri
+
-"""
+from blogs import models, utilities
+
+
+class FeedParserFeedMock(object):
+    title = ""
+    author = ""
+    summary = ""
+
+    def __init__(self, title, author, summary):
+        self.title = title
+        self.author = author
+        self.summary = summary
+
+
+class FeedParserMock(object):
+    feed = FeedParserFeedMock
+
+    def __init__(self, feed):
+        self.feed = feed
+
+
+def request_error():
+    return None
+
+
+class RequestsMock(object):
+    text = ""
+    raise_for_status = request_error
+
+    def __init__(self, text, raise_for_status):
+        self.text = text
+        self.raise_for_status = raise_for_status
+
+
+class UtilityTests(TestCase):
+    """utility test cases"""
+
+    def setUp(self):
+        """set up shared data"""
+
+        feed = FeedParserFeedMock(
+            title="My amazing blog",
+            author="Hugh Rundle",
+            summary="A short summary of my blog",
+        )
+
+        feed_partial = FeedParserFeedMock(
+            title="My amazing blog", author=None, summary=None
+        )
+
+        self.feedparser = FeedParserMock(feed=feed)
+        self.feedparser_partial = FeedParserMock(feed=feed_partial)
+
+    def test_get_feed_info(self):
+        """test get feed info"""
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            data = utilities.get_feed_info("https://test.test")
+
+            self.assertEqual(data["feed"], "https://test.test")
+            self.assertEqual(data["title"], "My amazing blog")
+            self.assertEqual(data["author_name"], "Hugh Rundle")
+            self.assertEqual(data["description"], "A short summary of my blog")
+
+    def test_get_blog_info_no_feed(self):
+        """test get blog info"""
+
+        with open(
+            pathlib.Path(__file__).parent.joinpath("data/example.html"),
+            "r",
+            encoding="utf-8",
+        ) as webfile:
+            website = RequestsMock(text=webfile, raise_for_status=request_error)
+
+            with patch("feedparser.parse", return_value=self.feedparser), patch(
+                "requests.get", return_value=website
+            ):
+                data = utilities.get_blog_info("http://test.test")
+
+                self.assertEqual(data, False)
+
+    def test_get_blog_info_with_good_feed(self):
+        """test get blog info"""
+
+        with open(
+            pathlib.Path(__file__).parent.joinpath("data/good-example.html"),
+            "r",
+            encoding="utf-8",
+        ) as webfile:
+            website = RequestsMock(text=webfile, raise_for_status=request_error)
+
+            with patch("feedparser.parse", return_value=self.feedparser), patch(
+                "requests.get", return_value=website
+            ):
+                data = utilities.get_blog_info("http://test.test")
+
+                self.assertEqual(
+                    data,
+                    {
+                        "feed": "https://test.test/rss.xml",
+                        "title": "My test website with an RSS feed",
+                        "author_name": "Testy McTestface",
+                        "description": "My cool website",
+                    },
+                )
+
+    def test_get_blog_info_with_incomplete_feed(self):
+        """test get blog info where the feed is incomplete"""
+
+        with open(
+            pathlib.Path(__file__).parent.joinpath("data/good-example.html"),
+            "r",
+            encoding="utf-8",
+        ) as webfile:
+            website = RequestsMock(text=webfile, raise_for_status=request_error)
+
+            with patch("feedparser.parse", return_value=self.feedparser_partial), patch(
+                "requests.get", return_value=website
+            ):
+                data = utilities.get_blog_info("http://test.test")
+
+                self.assertEqual(
+                    data,
+                    {
+                        "feed": "https://test.test/rss.xml",
+                        "title": "My test website with an RSS feed",
+                        "author_name": "Testy McTestface",
+                        "description": "My cool website",
+                    },
+                )
+
+    def test_get_blog_info_with_incomplete_head(self):
+        """test get blog info where the head info is incomplete"""
+
+        with open(
+            pathlib.Path(__file__).parent.joinpath("data/partial-example.html"),
+            "r",
+            encoding="utf-8",
+        ) as webfile:
+            website = RequestsMock(text=webfile, raise_for_status=request_error)
+
+            with patch("feedparser.parse", return_value=self.feedparser), patch(
+                "requests.get", return_value=website
+            ):
+                data = utilities.get_blog_info("http://test.test")
+
+                self.assertEqual(data["title"], "My test website with an RSS feed")
+                self.assertEqual(data["author_name"], "Hugh Rundle")
+                self.assertEqual(data["description"], "A short summary of my blog")
+
+    def test_get_blog_info_with_incomplete_head_and_partial_feed(self):
+        """test get blog info where both the feed and website head are incomplete"""
+
+        with open(
+            pathlib.Path(__file__).parent.joinpath("data/partial-example.html"),
+            "r",
+            encoding="utf-8",
+        ) as webfile:
+            website = RequestsMock(text=webfile, raise_for_status=request_error)
+
+            with patch("feedparser.parse", return_value=self.feedparser_partial), patch(
+                "requests.get", return_value=website
+            ):
+                data = utilities.get_blog_info("http://test.test")
+
+                self.assertEqual(data["title"], "My test website with an RSS feed")
+                self.assertEqual(data["author_name"], None)
+                self.assertEqual(data["description"], None)
+
+    def test_get_webfinger_subscribe_uri(self):
+        """test get webfinger data"""
+
+        # TODO
+        pass

blogs/tests/test_views.py

@@ -1,5 +1,7 @@
 """test views"""
 
+from datetime import timedelta
+
 from unittest.mock import patch
 
 from django.contrib.auth.models import AnonymousUser
@@ -16,7 +18,7 @@ class PublicTests(TestCase):
     """Public views test cases"""
 
     def setUp(self):
-        start_date = timezone.now()
+        start_date = timezone.now() + timedelta(days=5)
 
         self.factory = RequestFactory()
         self.glam_conf = models.Event.objects.create(
@@ -53,7 +55,7 @@ class PublicTests(TestCase):
         blogs_response = self.client.get(reverse("blogs"))
         self.assertEqual(blogs_response.status_code, 200)
 
-        confs_response = self.client.get(reverse("conferences"))
+        confs_response = self.client.get(reverse("events"))
         self.assertEqual(confs_response.status_code, 200)
 
         groups_response = self.client.get(reverse("groups"))
@@ -133,15 +135,36 @@ class PublicTests(TestCase):
         exists = models.Event.objects.filter(name="My event").exists()
         self.assertTrue(exists)
 
-    def test_register_cfp(self):
+    def test_register_cfp_unapproved_event(self):
         """post CFP registration form"""
 
         view = views.RegisterCallForPapers.as_view()
         form = forms.RegisterCallForPapersForm()
         form.data["event"] = self.glam_conf.id
         form.data["name"] = "Call for Papers"
-        form.data["url"] = "https://www.example.com"
+        form.data["details"] = "Here are some details"
-        form.data["category"] = "GLAM"
+        form.data["opening_date"] = "01/01/2024"
+        form.data["closing_date"] = "28/01/2024"
+
+        request = self.factory.post("register-cfp/", form.data)
+        request.user = AnonymousUser()
+
+        view(request)
+
+        exists = models.CallForPapers.objects.filter(name="Call for Papers").exists()
+        self.assertFalse(exists)
+
+    def test_register_cfp_approved_event(self):
+        """post CFP registration form"""
+
+        self.glam_conf.approved = True
+        self.glam_conf.save()
+
+        view = views.RegisterCallForPapers.as_view()
+        form = forms.RegisterCallForPapersForm()
+        form.data["event"] = self.glam_conf.id
+        form.data["name"] = "Call for Papers"
+        form.data["details"] = "Here are some details"
         form.data["opening_date"] = "01/01/2024"
         form.data["closing_date"] = "28/01/2024"
 

blogs/utilities.py

@@ -22,8 +22,9 @@ def get_feed_info(feed):
     blog["feed"] = feed
     blog["title"] = getattr(b.feed, "title", "")
     blog["author_name"] = getattr(b.feed, "author", None)
-    blog["description"] = getattr(b.feed, "subtitle", "")
+    blog["description"] = getattr(b.feed, "summary", None) or getattr(
-    blog["last_updated"] = getattr(b.feed, "updated_parsed", "")
+        b.feed, "subtitle", None
+    )
 
     return blog
 
@@ -46,24 +47,46 @@ def get_blog_info(url):
             try:
                 author = soup.select_one('meta[name="creator"]').get("content")
             except AttributeError:
-                author = ""
+                author = None
+
+        try:
+            description = soup.select_one('meta[name="description"]').get("content")
+        except AttributeError:
+            description = None
 
         if len(links) > 0:
             blog_info = get_feed_info(links[0].get("href"))
 
-            blog_info["title"] = blog_info["title"] or soup.title.string
+            if hasattr(soup, "title") and soup.title:
-            blog_info["author_name"] = blog_info["author_name"] or author
+                blog_info["title"] = soup.title.string  # use the scraped title
+            else:
+                blog_info["title"] = blog_info.get(
+                    "title"
+                )  # use the title from the feed
+
+            blog_info["description"] = description or blog_info.get("description", "")
+
+        else:
+            return False  # if there is no feed info we need to put the onus back on the user to fill in the data
+
+        normalised_author = ""
+        if author:
+            normalised_author = author.replace("(noreply@blogger.com)", "")
+            if normalised_author.strip() == "Unknown":
+                normalised_author = ""
+
+            blog_info["author_name"] = normalised_author
 
         return blog_info
 
     except requests.Timeout:
-        print(f"TIMEOUT error registering {url}, trying longer timeout")
+        logging.warning(f"TIMEOUT error registering {url}, trying longer timeout")
         r = requests.get(url, headers=headers, timeout=(31, 31))
         r.raise_for_status()  # let it flow through, a timeout here means the site is unreasonably slow
 
     except Exception as e:
-        print(f"CONNECTION ERROR when registering {url}")
+        logging.error(f"CONNECTION ERROR when registering {url}")
-        print(e)
+        logging.error(e)
         return False
 
 
@@ -83,13 +106,13 @@ def get_webfinger_subscribe_uri(username):
         r.raise_for_status()
 
     except requests.Timeout:
-        print(f"TIMEOUT error finding {username}, trying longer timeout")
+        logging.warning(f"TIMEOUT error finding {username}, trying longer timeout")
         r = requests.get(webfinger_url, headers=headers, timeout=(31, 31))
         r.raise_for_status()  # let it flow through, a timeout here means the site is unreasonably slow
 
     except Exception as e:
-        print(f"CONNECTION ERROR when subscribing via {username}")
+        logging.error(f"CONNECTION ERROR when subscribing via {username}")
-        print(e)
+        logging.error(e)
         return None
 
     data = r.json()

blogs/views/public.py

@@ -1,8 +1,6 @@
 """public views (no need to log in)"""
 
 # pylint: disable=R6301
-import os
-
 from itertools import chain
 from operator import attrgetter
 
@@ -20,16 +18,13 @@ from django.views import View
 from blogs import forms, models
 from blogs.utilities import get_blog_info, get_webfinger_subscribe_uri
 
+
 class HomeFeed(View):
     """the home feed when someone visits the site"""
 
     def get(self, request):
         """display home page"""
 
-        path = os.path.join(settings.BASE_DIR, "static/")
-
-        print(path)
-
         latest = models.Article.objects.order_by("-pubdate")[:10]
 
         data = {"title": "Latest blog posts", "latest": latest}
@@ -42,7 +37,9 @@ class Blogs(View):
     def get(self, request):
         """here they are"""
 
-        blogs = models.Blog.objects.filter(approved=True)
+        blogs = models.Blog.objects.filter(approved=True, active=True).order_by(
+            "-updateddate"
+        )
         for blog in blogs:
             blog.category_name = models.Category(blog.category).label
         data = {"title": "Blogs and websites", "blogs": blogs}
@@ -140,14 +137,13 @@ class RegisterBlog(View):
                 data["blog_info"] = blog_info
 
             else:
-                data["error"] = "Could not auto-discover your feed info, please enter manually"
+                data[
+                    "error"
+                ] = "Could not auto-discover your feed info, please enter manually"
 
             return render(request, "blogs/confirm-register.html", data)
 
-        data = {
+        data = {"title": "Register your blog", "form": form}
-            "title": "Register your blog",
-            "form": form
-        }
         return render(request, "blogs/register.html", data)
 
 
@@ -311,7 +307,7 @@ class Search(View):
             "description", weight="C"
         )
 
-        conferences = (
+        events = (
             models.Event.objects.annotate(rank=SearchRank(conference_vector, query))
             .filter(rank__gte=0.1)
             .order_by("-rank")
@@ -348,7 +344,7 @@ class Search(View):
         )
 
         combined = sorted(
-            chain(articles, conferences, cfps, newsletters, groups),
+            chain(articles, events, cfps, newsletters, groups),
             key=attrgetter("rank"),
             reverse=True,
         )