hugh/ausglamr
1
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fixes

Browse source
This commit is contained in:
Hugh Rundle 2024-01-07 16:58:56 +11:00
parent 53df5c5035
commit d28dcdb069
Signed by: hugh
GPG key ID: A7E35779918253F9
26 changed files with 665 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -1,2 +1,3 @@
data
static
static
fixtures

19
README.md
View file

@ -2,6 +2,25 @@
A django app running on Docker. Replaces _Aus GLAM Blogs_.
## Deploy
* `docker compose build`
* `./glamr-dev createsuperuser`
* `./glamr-dev makemigrations`
* `./glamr-dev migrate`
* `docker compose up`
* set up database backups (as cron jobs):
* `/usr/bin/docker exec -u root ausglamr-db-1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump`
* `/usr/bin/docker cp ausglamr-db-1:/tmp/ausglamr_backup.dump /home/hugh/backups/`
* set up cron jobs for management commands as below
## Migrating
See `data/help.txt`
## Admin
Don't forget to add some Content Warnings for use by the Mastodon bot.

23
ausglamr/settings.py
View file

@ -28,11 +28,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env("SECRET_KEY")
# DEPLOYMENT
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env("DEBUG")
ALLOWED_HOSTS = [env("DOMAIN")]
CSRF_COOKIE_SECURE=True
SESSION_COOKIE_SECURE=True
CSRF_TRUSTED_ORIGINS = [f'https://{env("DOMAIN")}']
CONN_MAX_AGE=None # persistent DB connection
# Application definition
@ -149,6 +153,7 @@ ADMIN_EMAIL = env("ADMIN_EMAIL") # custom
# django email settings
DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")
SERVER_EMAIL = env("SERVER_EMAIL")
EMAIL_HOST = env("EMAIL_HOST")
EMAIL_HOST_USER = env("EMAIL_HOST_USER")
EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD")
@ -158,3 +163,19 @@ EMAIL_USE_SSL = env("EMAIL_USE_SSL")
# mastodon
MASTODON_ACCESS_TOKEN = env("MASTODON_ACCESS_TOKEN")
MASTODON_DOMAIN = env("MASTODON_DOMAIN")
# LOGGING
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"handlers": {
"console": {
"class": "logging.StreamHandler",
},
},
"root": {
"handlers": ["console"],
"level": "INFO",
},
}

2
ausglamr/urls.py
View file

@ -41,7 +41,7 @@ urlpatterns = [
path("subscribe", views.Subscribe.as_view(), name="subscribe"),
path("subscribe-email", views.SubscribeEmail.as_view(), name="subscribe-email"),
path(
"confirm-subscribe-email/<token>/<id>",
"confirm-subscribe-email/<token>/<user_id>",
views.ConfirmEmail.as_view(),
name="confirm-subscribe-email",
),

33
blogs/admin.py
View file

@ -19,9 +19,11 @@ def approve(modeladmin, request, queryset):
instance.announce()
if hasattr(instance, "event"): # CFP
recipient = instance.event.owner_email
if hasattr(instance, "owner_email"): # overrides above in case needed in future
recipient = instance.owner_email
recipient = instance.event.contact_email
if hasattr(
instance, "contact_email"
): # overrides above in case needed in future
recipient = instance.contact_email
if recipient:
if hasattr(instance, "name"):
@ -47,7 +49,7 @@ def suspend(modeladmin, request, queryset):
queryset.update(suspended=True)
for instance in queryset:
if hasattr(instance, "owner_email"):
if hasattr(instance, "contact_email"):
if hasattr(instance, "name"):
title = instance.name
else:
@ -58,7 +60,7 @@ def suspend(modeladmin, request, queryset):
<p>Your blog {title} has been suspended from AusGLAMR. It may be unsuspended in future once the issue is resolved. If you would like more information, please reply to this email.</p> \
</body></html>"
utilities.send_email(subject, message, instance.owner_email)
utilities.send_email(subject, message, instance.contact_email)
@admin.action(description="Unsuspend selected blogs")
@ -67,7 +69,7 @@ def unsuspend(modeladmin, request, queryset):
queryset.update(suspended=False, suspension_lifted=timezone.now())
for instance in queryset:
if hasattr(instance, "owner_email"):
if hasattr(instance, "contact_email"):
if hasattr(instance, "name"):
title = instance.name
else:
@ -78,7 +80,7 @@ def unsuspend(modeladmin, request, queryset):
<p>The suspension on your blog {title} has been removed on AusGLAMR. Please note that articles published whilst it was suspended will not be added to AusGLAMR retrospectively. If you would like more information, please reply to this email.</p> \
</body></html>"
utilities.send_email(subject, message, instance.owner_email)
utilities.send_email(subject, message, instance.contact_email)
@admin.action(description="Confirm selected subscribers")
@ -93,6 +95,18 @@ def unconfirm(modeladmin, request, queryset):
queryset.update(confirmed=False)
@admin.action(description="Deactivate selected blogs")
def disable(modeladmin, request, queryset):
"""disable selected"""
queryset.update(active=False)
@admin.action(description="Activate selected blogs")
def activate(modeladmin, request, queryset):
"""un-disable selected"""
queryset.update(active=True)
@admin.action(description="Send confirmation request to selected subscribers")
def send_conf_request(modeladmin, request, queryset):
"""send to selected"""
@ -106,16 +120,17 @@ class Blog(admin.ModelAdmin):
"""display settings for blogs"""
list_display = (
"title",
"url",
"title",
"author_name",
"approved",
"announced",
"suspended",
"failing",
"active",
)
ordering = ["approved", "-suspended", "-failing"]
actions = [approve, unapprove, suspend, unsuspend]
actions = [approve, unapprove, suspend, unsuspend, activate, disable]
@admin.register(models.Article)

10
blogs/forms.py
View file

@ -19,7 +19,7 @@ class RegisterBlogForm(forms.ModelForm):
"""set fields and model"""
model = Blog
fields = ["url", "category", "activitypub_account_name", "owner_email"]
fields = ["url", "category", "activitypub_account_name", "contact_email"]
class ConfirmBlogForm(forms.ModelForm):
@ -37,7 +37,7 @@ class ConfirmBlogForm(forms.ModelForm):
"description",
"category",
"activitypub_account_name",
"owner_email",
"contact_email",
]
@ -55,7 +55,7 @@ class RegisterConferenceForm(forms.ModelForm):
"description",
"start_date",
"activitypub_account_name",
"owner_email",
"contact_email",
]
widgets = {
"start_date": DateInput(),
@ -97,7 +97,7 @@ class RegisterGroupForm(forms.ModelForm):
"url",
"registration_url",
"description",
"owner_email",
"contact_email",
]
@ -115,7 +115,7 @@ class RegisterNewsletterForm(forms.ModelForm):
"url",
"description",
"activitypub_account_name",
"owner_email",
"contact_email",
]

65
blogs/management/commands/check_feeds.py
View file

@ -1,5 +1,8 @@
"""call this from cron to run through all the feeds to find new posts"""
import logging
import traceback
from datetime import datetime, timedelta, timezone
import feedparser
@ -22,11 +25,12 @@ def get_tags(dictionary):
"""parse out tags from blog and upsert as tag instances"""
tags = []
for tag_obj in dictionary:
tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first()
if not tag:
tag = models.Tag.objects.create(name=tag_obj.term.lower())
if tag_obj.term.lower() != "uncategorized":
tag = models.Tag.objects.filter(name=tag_obj.term.lower()).first()
if not tag:
tag = models.Tag.objects.create(name=tag_obj.term.lower())
tags.append(tag)
tags.append(tag)
return tags
@ -34,14 +38,24 @@ def get_tags(dictionary):
class Command(BaseCommand):
"""the check_feeds command"""
# we could add arguments but we don't really need any
def add_arguments(self, parser):
parser.add_argument(
"-q",
action="store_true",
help="Suppress non-error messages",
)
def handle(self, *args, **options):
"""check feeds and update database"""
print(f"checking feeds at {django_timezone.localtime(django_timezone.now())}")
if not options["q"]:
logging.info(
f"checking feeds at {django_timezone.localtime(django_timezone.now())}"
)
blogs = models.Blog.objects.filter(approved=True, suspended=False).all()
blogs = models.Blog.objects.filter(
approved=True, suspended=False, active=True
).all()
for blog in blogs:
try:
data = feedparser.parse(blog.feed)
@ -50,24 +64,24 @@ class Command(BaseCommand):
if not models.Article.objects.filter(
Q(url=article.link) | Q(guid=article.id)
).exists():
if (
if blog.suspension_lifted and (
blog.suspension_lifted
and blog.suspension_lifted
< date_to_tz_aware(article.updated_parsed)
> date_to_tz_aware(article.updated_parsed)
):
continue # don't ingest posts published during a suspension
continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension)
tags = get_tags(
getattr(article, "tags", None)
or getattr(article, "categories", [])
taglist = getattr(article, "tags", None) or getattr(
article, "categories", []
)
tags = [tag.term.lower() for tag in taglist]
opt_out = False
# don't include posts with opt out tags
for tag in tags:
if (
len(
{tag.name}
{tag}
& {
"notglam",
"notglamr",
@ -99,8 +113,14 @@ class Command(BaseCommand):
guid=article.id,
)
for tag in tags:
tags_to_add = get_tags(
getattr(article, "tags", None)
or getattr(article, "categories", [])
)
for tag in tags_to_add:
instance.tags.add(tag)
instance.save()
cutoff = django_timezone.now() - timedelta(days=3)
@ -108,11 +128,16 @@ class Command(BaseCommand):
if newish:
instance.announce()
blog.set_success()
blog.set_success(
updateddate=date_to_tz_aware(article.updated_parsed)
)
except Exception as e:
blog.set_failing()
print(f"ERROR WITH BLOG {blog.title} - {blog.url}")
print(e)
logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}")
logging.error(e)
print(f"completed run at {django_timezone.localtime(django_timezone.now())}")
if not options["q"]:
logging.info(
f"completed run at {django_timezone.localtime(django_timezone.now())}"
)

6
blogs/management/commands/queue_announcements.py
View file

@ -16,7 +16,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
"""check whether we need to queue announcements and queue them"""
conferences = models.Event.objects.filter(
events = models.Event.objects.filter(
approved=True,
announcements__lt=3,
start_date__gte=timezone.now(),
@ -24,10 +24,10 @@ class Command(BaseCommand):
calls = models.CallForPapers.objects.filter(
announcements__lt=3,
closing_date__gte=timezone.now(),
conference__approved=True,
event__approved=True,
)
for conf in conferences:
for conf in events:
delta = conf.start_date - timezone.now().date()
if (

7
blogs/management/commands/send_weekly_email.py
View file

@ -1,6 +1,7 @@
"""send the weekly email"""
from datetime import timedelta
import logging
import random
from django.conf import settings
@ -22,7 +23,7 @@ class Command(BaseCommand):
subscribers = models.Subscriber.objects.filter(confirmed=True)
print(
logging.info(
f"Sending weekly emails to {len(subscribers)} subscribers at {timezone.now()}"
)
@ -31,7 +32,7 @@ class Command(BaseCommand):
articles = models.Article.objects.filter(pubdate__gte=cutoff)
events = models.Event.objects.filter(approved=True, pub_date__gte=cutoff)
cfps = models.CallForPapers.objects.filter(
conference__approved=True, closing_date__gte=timezone.now().date()
event__approved=True, closing_date__gte=timezone.now().date()
)
newsletters = models.Newsletter.objects.filter(
approved=True, pub_date__gte=cutoff
@ -210,4 +211,4 @@ class Command(BaseCommand):
send_email(subject, message, subscriber.email)
print(f"Weekly emails completed {timezone.now()}")
logging.info(f"Weekly emails completed {timezone.now()}")

19
blogs/migrations/0001_initial.py
View file

@ -1,5 +1,6 @@
# Generated by Django 4.2.7 on 2024-01-03 03:35
# Generated by Django 4.2.7 on 2024-01-07 04:55
import blogs.models.blog
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
@ -70,12 +71,18 @@ class Migration(migrations.Migration):
models.BooleanField(blank=True, default=False, null=True),
),
("suspension_lifted", models.DateTimeField(blank=True, null=True)),
("active", models.BooleanField(default=True, null=True)),
(
"activitypub_account_name",
models.CharField(blank=True, max_length=200, null=True),
models.CharField(
blank=True,
max_length=200,
null=True,
validators=[blogs.models.blog.validate_ap_address],
),
),
(
"owner_email",
"contact_email",
models.EmailField(blank=True, max_length=254, null=True),
),
],
@ -140,7 +147,7 @@ class Migration(migrations.Migration):
models.CharField(blank=True, max_length=200, null=True),
),
(
"owner_email",
"contact_email",
models.EmailField(blank=True, max_length=254, null=True),
),
("approved", models.BooleanField(default=False)),
@ -197,7 +204,7 @@ class Migration(migrations.Migration):
("registration_url", models.URLField(max_length=400, unique=True)),
("description", models.TextField(blank=True, null=True)),
(
"owner_email",
"contact_email",
models.EmailField(blank=True, max_length=254, null=True),
),
("announced", models.BooleanField(default=False)),
@ -241,7 +248,7 @@ class Migration(migrations.Migration):
models.CharField(blank=True, max_length=200, null=True),
),
(
"owner_email",
"contact_email",
models.EmailField(blank=True, max_length=254, null=True),
),
("announced", models.BooleanField(default=False)),

31
blogs/models/blog.py
View file

@ -1,11 +1,26 @@
"""blog models"""
import re
from django.db import models
from django.utils import timezone
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from .utils import Announcement, Category, ContentWarning
def validate_ap_address(value):
"""is the activitypub address valid?"""
p = re.compile("@(.*)@((\w|-)*)\.((\w)*)")
m = p.match(value)
if not m:
raise ValidationError(
_("%(value)s should be in the form '@user@domain.tld'"),
params={"value": value},
)
class BlogData(models.Model):
"""Base bloggy data"""
@ -45,8 +60,11 @@ class Blog(BlogData):
failing = models.BooleanField(default=False, blank=True, null=True)
suspended = models.BooleanField(default=False, blank=True, null=True)
suspension_lifted = models.DateTimeField(blank=True, null=True)
activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
owner_email = models.EmailField(blank=True, null=True)
active = models.BooleanField(null=True, default=True)
activitypub_account_name = models.CharField(
max_length=200, blank=True, null=True, validators=[validate_ap_address]
)
contact_email = models.EmailField(blank=True, null=True)
def announce(self):
"""queue announcement"""
@ -72,10 +90,15 @@ class Blog(BlogData):
self.failing = True
super().save()
def set_success(self):
"""set failing to false"""
def set_success(self, updateddate):
"""
set failing to false
set the updateddate to a datetime
"""
self.failing = False
self.updateddate = updateddate
super().save()

2
blogs/models/event.py
View file

@ -17,7 +17,7 @@ class Event(models.Model):
start_date = models.DateField()
announcements = models.IntegerField(null=True, blank=True, default=0)
activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
owner_email = models.EmailField(blank=True, null=True)
contact_email = models.EmailField(blank=True, null=True)
approved = models.BooleanField(default=False)
def save(self, *args, **kwargs):

2
blogs/models/group.py
View file

@ -15,7 +15,7 @@ class Group(models.Model):
url = models.URLField(max_length=400, unique=True)
registration_url = models.URLField(max_length=400, unique=True)
description = models.TextField(null=True, blank=True)
owner_email = models.EmailField(blank=True, null=True)
contact_email = models.EmailField(blank=True, null=True)
announced = models.BooleanField(default=False)
approved = models.BooleanField(default=False)
pub_date = models.DateTimeField(null=True, default=None)

2
blogs/models/newsletter.py
View file

@ -16,7 +16,7 @@ class Newsletter(models.Model):
description = models.TextField(null=True, blank=True)
activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
owner_email = models.EmailField(blank=True, null=True)
contact_email = models.EmailField(blank=True, null=True)
announced = models.BooleanField(default=False)
approved = models.BooleanField(default=False)
pub_date = models.DateTimeField(null=True, default=None)

6
blogs/templates/blogs/confirm-register.html
View file

@ -42,9 +42,9 @@
<label for="{{ form.activitypub_account_name.id_for_label }}">Activitypub account name:</label>
{{ form.activitypub_account_name }}
{{ form.owner_email.errors }}
<label for="{{ form.owner_email.id_for_label }}">Owner email:</label>
{{ form.owner_email }}
{{ form.contact_email.errors }}
<label for="{{ form.contact_email.id_for_label }}">Contact email:</label>
{{ form.contact_email }}
<input class="button-primary u-pull-right" type="submit" value="Register!">
</div>

7
blogs/templates/browse/blogs.html
View file

@ -3,15 +3,16 @@
{% block content %}
<div class="listing l-three header">
<div>Title</div>
<div>Author</div>
<div>Category</div>
<div>Last updated</div>
</div>
<hr/>
{% for blog in blogs %}
<div class="listing l-three">
<div><a href="{{blog.url}}">{{blog.title}}</a></div>
<div>{{blog.author_name}}</div>
<div><a href="{{blog.url}}">{{blog.title}}</a><p>{{blog.description}}</p></div>
<div><div class="end badge badge_{{blog.category}}">{{blog.category_name}}</div></div>
<div>{{ blog.updateddate|date:"D d M Y" }}
</div>
</div>
<hr/>
{% endfor %}

5
blogs/templates/contribute.html
View file

@ -1,5 +1,6 @@
{% extends "layout.html" %}
{% block container %}
<main class="container pop-boxes">
{% block intro %}{% endblock %}
{% block title %}
@ -7,9 +8,6 @@
{% endblock %}
<section>
{% block content %}
<section>
<p>You can contribute to <em>Aus GLAMR</em> by registering a blog, newsletter, event, or discussion group. Creators or content should be Australasian-based.</p>
</section>
@ -52,3 +50,4 @@
</section>
{% endblock %}
</section>
{% endblock %}

2
blogs/templates/search.html
View file

@ -3,7 +3,7 @@
{% block content %}
<form action="{% url 'search' %}" method="get">
<label for="keywords">Search for keywords across blog posts, groups, newsletters, and conferences</label>
<label for="keywords">Search for keywords across blog posts, groups, newsletters, and events</label>
<input type="text" name="q" value="{% if query %}{{ query }}{% endif %}">
<button class="button" type="submit">Search!</button>
</form>

2
blogs/templates/subscribe.html
View file

@ -13,7 +13,7 @@
<hgroup>
<h3>Subscribe via email</h3>
</hgroup>
<p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming conferences.</p>
<p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.</p>
<div>
<a class="button" href="{% url 'subscribe-email' %}">Subscribe via email</a>
</div>

221
blogs/tests/test_commands.py Normal file
View file

@ -0,0 +1,221 @@
"""test utility functions"""
from datetime import datetime, timezone
from unittest.mock import patch
from django.core.management import call_command
from django.test import TestCase
from blogs import models
class FeedParserItemMock(object):
title = ""
tags = []
author = ""
link = ""
summary = ""
updated_parsed = ((),)
published_parsed = ((),)
id = ""
def __init__(
self, title, tags, author, link, summary, updated_parsed, published_parsed, id
):
self.title = title
self.tags = tags
self.author = author
self.link = link
self.summary = summary
self.updated_parsed = updated_parsed
self.published_parsed = published_parsed
self.id = id
class FeedParserTagMock(object):
term = ""
def __init__(self, term):
self.term = term
class FeedParserMock(object):
entries = []
def __init__(self, entries):
self.entries = entries
class CommandsTestCase(TestCase):
"""test management command functions"""
def setUp(self):
"""set up test conf"""
self.blog = models.Blog.objects.create(
title="My awesome blog",
url="https://test.com",
feed="https://test.com/feed.xml",
category="LIB",
approved=True,
suspended=False,
)
tag_one = FeedParserTagMock(term="testing")
tag_two = FeedParserTagMock(term="python")
tag_three = FeedParserTagMock(term="notglam")
article = FeedParserItemMock(
title="My amazing blog post",
tags=[tag_one, tag_two],
author="Hugh Rundle",
link="https://test.com/1",
summary="A short summary of my post",
updated_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
published_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
id="1",
)
article_two = FeedParserItemMock(
title="My really amazing blog post",
tags=[tag_one, tag_two, tag_three],
author="Hugh Rundle",
link="https://test.com/2",
summary="A short summary of my next post",
updated_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
published_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
id="999",
)
self.feedparser = FeedParserMock(entries=[article])
self.feedparser_exclude = FeedParserMock(entries=[article_two])
def test_check_feeds(self):
"""test parse a feed for basic blog info"""
args = {"-q": True}
opts = {}
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
with patch("feedparser.parse", return_value=self.feedparser):
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.Tag.objects.count(), 2)
article = models.Article.objects.all().first()
self.assertEqual(article.title, "My amazing blog post")
def test_check_feeds_exclude_tag(self):
"""test parse a feed with exclude tag"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
with patch("feedparser.parse", return_value=self.feedparser_exclude):
args = {"-q": True}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
def test_check_feeds_unapproved(self):
"""test check unapproved blog feed"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
self.blog.approved = False
self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
def test_check_feeds_inactive(self):
"""test ignore inactive blog feed"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
self.blog.active = False
self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
def test_check_feeds_suspended(self):
"""test check suspended blog feed"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
self.blog.suspended = True
self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
def test_check_feeds_previously_suspended(self):
"""test blog published prior to suspension lifted is not ingested"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
self.blog.suspended = False
self.blog.suspension_lifted = datetime(
2024, 1, 2, 21, 0, 0, 0, tzinfo=timezone.utc
)
self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": False}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
def test_check_feeds_previously_suspended_post_after(self):
"""test blog published after suspension lifted is ingested"""
self.assertEqual(models.Article.objects.count(), 0)
self.assertEqual(models.Tag.objects.count(), 0)
self.blog.suspended = False
self.blog.suspension_lifted = datetime(
2023, 12, 31, 0, 0, 0, 0, tzinfo=timezone.utc
)
self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": False}
opts = {}
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Article.objects.count(), 1)
self.assertEqual(models.Tag.objects.count(), 2)

16
blogs/tests/test_management.py
View file

@ -19,7 +19,7 @@ class ManagementTestCase(TestCase):
name="Amazing Conf",
url="https://test.com",
category="LIB",
start_date=timezone.now() + timedelta(days=3),
start_date=timezone.localtime(timezone.now()) + timedelta(days=3),
activitypub_account_name="@conf@conf.conf",
approved=True,
announcements=1,
@ -28,15 +28,15 @@ class ManagementTestCase(TestCase):
self.cfp = models.CallForPapers.objects.create(
event=self.conf,
name="Call for Papers for Amazing Conf",
opening_date=timezone.now() + timedelta(days=30),
closing_date=timezone.now() + timedelta(days=1),
opening_date=timezone.localtime(timezone.now()) + timedelta(days=30),
closing_date=timezone.localtime(timezone.now()) + timedelta(days=1),
)
self.cfp = models.CallForPapers.objects.create(
event=self.conf,
name="Call for posters",
opening_date=timezone.now() - timedelta(days=30),
closing_date=timezone.now() - timedelta(days=1),
opening_date=timezone.localtime(timezone.now()) - timedelta(days=30),
closing_date=timezone.localtime(timezone.now()) - timedelta(days=1),
)
def test_queue_announcements(self):
@ -49,7 +49,7 @@ class ManagementTestCase(TestCase):
# event
announcement = models.Announcement.objects.first()
start_date = timezone.now() + timedelta(days=3)
start_date = timezone.localtime(timezone.now()) + timedelta(days=3)
date = start_date.strftime("%a %d %b %Y")
status = f"Amazing Conf (@conf@conf.conf) is a event about Libraries, starting on {date}!\n\nhttps://test.com"
@ -57,8 +57,8 @@ class ManagementTestCase(TestCase):
# cfp
announcement = models.Announcement.objects.last()
opening_date = timezone.now() + timedelta(days=30)
closing_date = timezone.now() + timedelta(days=1)
opening_date = timezone.localtime(timezone.now()) + timedelta(days=30)
closing_date = timezone.localtime(timezone.now()) + timedelta(days=1)
opening_date_str = opening_date.strftime("%a %d %b %Y")
closing_date_str = closing_date.strftime("%a %d %b %Y")
status = f"Amazing Conf Call for Papers for Amazing Conf is open from {opening_date_str}, closing on {closing_date_str}!\n\nMore info at https://test.com"

22
blogs/tests/test_models.py
View file

@ -1,6 +1,7 @@
"""model tests"""
from datetime import date
from datetime import date, datetime
from datetime import timezone as dt_tz
from django.test import TestCase
from django.utils import timezone
@ -30,9 +31,12 @@ class BlogTestCase(TestCase):
self.blog.failing = True
self.blog.save()
self.blog.set_success()
self.blog.set_success(
updateddate=datetime(2020, 1, 1, 12, 59, 0, tzinfo=dt_tz.utc)
)
self.assertEqual(self.blog.failing, False)
self.assertEqual(self.blog.updateddate.isoformat(), "2020-01-01T12:59:00+00:00")
def test_set_failing(self):
"""set_failing class function"""
@ -155,20 +159,6 @@ class NewsletterTestCase(TestCase):
class UtilsTestCase(TestCase):
"""test utility functions"""
def test_announcement(self):
"""test announcing"""
# TODO: mock mastodon somehow
# announcement = models.Announcement.objects.create(
# status="I have something to say!",
# summary="boasting",
# )
# announcement.announce()
# self.assertEqual(models.Announcement.objects.count(), 0)
def test_content_warning(self):
"""test CWs"""

185
blogs/tests/test_utilities.py
View file

@ -1,8 +1,181 @@
"""test utility functions"""
# TODO:
"""
get_feed_info
get_blog_info
get_webfinger_subscribe_uri
"""
import pathlib
from django.test import TestCase
from unittest.mock import patch
from blogs import models, utilities
class FeedParserFeedMock(object):
title = ""
author = ""
summary = ""
def __init__(self, title, author, summary):
self.title = title
self.author = author
self.summary = summary
class FeedParserMock(object):
feed = FeedParserFeedMock
def __init__(self, feed):
self.feed = feed
def request_error():
return None
class RequestsMock(object):
text = ""
raise_for_status = request_error
def __init__(self, text, raise_for_status):
self.text = text
self.raise_for_status = raise_for_status
class UtilityTests(TestCase):
"""utility test cases"""
def setUp(self):
"""set up shared data"""
feed = FeedParserFeedMock(
title="My amazing blog",
author="Hugh Rundle",
summary="A short summary of my blog",
)
feed_partial = FeedParserFeedMock(
title="My amazing blog", author=None, summary=None
)
self.feedparser = FeedParserMock(feed=feed)
self.feedparser_partial = FeedParserMock(feed=feed_partial)
def test_get_feed_info(self):
"""test get feed info"""
with patch("feedparser.parse", return_value=self.feedparser):
data = utilities.get_feed_info("https://test.test")
self.assertEqual(data["feed"], "https://test.test")
self.assertEqual(data["title"], "My amazing blog")
self.assertEqual(data["author_name"], "Hugh Rundle")
self.assertEqual(data["description"], "A short summary of my blog")
def test_get_blog_info_no_feed(self):
"""test get blog info"""
with open(
pathlib.Path(__file__).parent.joinpath("data/example.html"),
"r",
encoding="utf-8",
) as webfile:
website = RequestsMock(text=webfile, raise_for_status=request_error)
with patch("feedparser.parse", return_value=self.feedparser), patch(
"requests.get", return_value=website
):
data = utilities.get_blog_info("http://test.test")
self.assertEqual(data, False)
def test_get_blog_info_with_good_feed(self):
"""test get blog info"""
with open(
pathlib.Path(__file__).parent.joinpath("data/good-example.html"),
"r",
encoding="utf-8",
) as webfile:
website = RequestsMock(text=webfile, raise_for_status=request_error)
with patch("feedparser.parse", return_value=self.feedparser), patch(
"requests.get", return_value=website
):
data = utilities.get_blog_info("http://test.test")
self.assertEqual(
data,
{
"feed": "https://test.test/rss.xml",
"title": "My test website with an RSS feed",
"author_name": "Testy McTestface",
"description": "My cool website",
},
)
def test_get_blog_info_with_incomplete_feed(self):
"""test get blog info where the feed is incomplete"""
with open(
pathlib.Path(__file__).parent.joinpath("data/good-example.html"),
"r",
encoding="utf-8",
) as webfile:
website = RequestsMock(text=webfile, raise_for_status=request_error)
with patch("feedparser.parse", return_value=self.feedparser_partial), patch(
"requests.get", return_value=website
):
data = utilities.get_blog_info("http://test.test")
self.assertEqual(
data,
{
"feed": "https://test.test/rss.xml",
"title": "My test website with an RSS feed",
"author_name": "Testy McTestface",
"description": "My cool website",
},
)
def test_get_blog_info_with_incomplete_head(self):
"""test get blog info where the head info is incomplete"""
with open(
pathlib.Path(__file__).parent.joinpath("data/partial-example.html"),
"r",
encoding="utf-8",
) as webfile:
website = RequestsMock(text=webfile, raise_for_status=request_error)
with patch("feedparser.parse", return_value=self.feedparser), patch(
"requests.get", return_value=website
):
data = utilities.get_blog_info("http://test.test")
self.assertEqual(data["title"], "My test website with an RSS feed")
self.assertEqual(data["author_name"], "Hugh Rundle")
self.assertEqual(data["description"], "A short summary of my blog")
def test_get_blog_info_with_incomplete_head_and_partial_feed(self):
"""test get blog info where both the feed and website head are incomplete"""
with open(
pathlib.Path(__file__).parent.joinpath("data/partial-example.html"),
"r",
encoding="utf-8",
) as webfile:
website = RequestsMock(text=webfile, raise_for_status=request_error)
with patch("feedparser.parse", return_value=self.feedparser_partial), patch(
"requests.get", return_value=website
):
data = utilities.get_blog_info("http://test.test")
self.assertEqual(data["title"], "My test website with an RSS feed")
self.assertEqual(data["author_name"], None)
self.assertEqual(data["description"], None)
def test_get_webfinger_subscribe_uri(self):
"""test get webfinger data"""
# TODO
pass

33
blogs/tests/test_views.py
View file

@ -1,5 +1,7 @@
"""test views"""
from datetime import timedelta
from unittest.mock import patch
from django.contrib.auth.models import AnonymousUser
@ -16,7 +18,7 @@ class PublicTests(TestCase):
"""Public views test cases"""
def setUp(self):
start_date = timezone.now()
start_date = timezone.now() + timedelta(days=5)
self.factory = RequestFactory()
self.glam_conf = models.Event.objects.create(
@ -53,7 +55,7 @@ class PublicTests(TestCase):
blogs_response = self.client.get(reverse("blogs"))
self.assertEqual(blogs_response.status_code, 200)
confs_response = self.client.get(reverse("conferences"))
confs_response = self.client.get(reverse("events"))
self.assertEqual(confs_response.status_code, 200)
groups_response = self.client.get(reverse("groups"))
@ -133,15 +135,36 @@ class PublicTests(TestCase):
exists = models.Event.objects.filter(name="My event").exists()
self.assertTrue(exists)
def test_register_cfp(self):
def test_register_cfp_unapproved_event(self):
"""post CFP registration form"""
view = views.RegisterCallForPapers.as_view()
form = forms.RegisterCallForPapersForm()
form.data["event"] = self.glam_conf.id
form.data["name"] = "Call for Papers"
form.data["url"] = "https://www.example.com"
form.data["category"] = "GLAM"
form.data["details"] = "Here are some details"
form.data["opening_date"] = "01/01/2024"
form.data["closing_date"] = "28/01/2024"
request = self.factory.post("register-cfp/", form.data)
request.user = AnonymousUser()
view(request)
exists = models.CallForPapers.objects.filter(name="Call for Papers").exists()
self.assertFalse(exists)
def test_register_cfp_approved_event(self):
"""post CFP registration form"""
self.glam_conf.approved = True
self.glam_conf.save()
view = views.RegisterCallForPapers.as_view()
form = forms.RegisterCallForPapersForm()
form.data["event"] = self.glam_conf.id
form.data["name"] = "Call for Papers"
form.data["details"] = "Here are some details"
form.data["opening_date"] = "01/01/2024"
form.data["closing_date"] = "28/01/2024"

45
blogs/utilities.py
View file

@ -22,8 +22,9 @@ def get_feed_info(feed):
blog["feed"] = feed
blog["title"] = getattr(b.feed, "title", "")
blog["author_name"] = getattr(b.feed, "author", None)
blog["description"] = getattr(b.feed, "subtitle", "")
blog["last_updated"] = getattr(b.feed, "updated_parsed", "")
blog["description"] = getattr(b.feed, "summary", None) or getattr(
b.feed, "subtitle", None
)
return blog
@ -46,24 +47,46 @@ def get_blog_info(url):
try:
author = soup.select_one('meta[name="creator"]').get("content")
except AttributeError:
author = ""
author = None
try:
description = soup.select_one('meta[name="description"]').get("content")
except AttributeError:
description = None
if len(links) > 0:
blog_info = get_feed_info(links[0].get("href"))
blog_info["title"] = blog_info["title"] or soup.title.string
blog_info["author_name"] = blog_info["author_name"] or author
if hasattr(soup, "title") and soup.title:
blog_info["title"] = soup.title.string # use the scraped title
else:
blog_info["title"] = blog_info.get(
"title"
) # use the title from the feed
blog_info["description"] = description or blog_info.get("description", "")
else:
return False # if there is no feed info we need to put the onus back on the user to fill in the data
normalised_author = ""
if author:
normalised_author = author.replace("(noreply@blogger.com)", "")
if normalised_author.strip() == "Unknown":
normalised_author = ""
blog_info["author_name"] = normalised_author
return blog_info
except requests.Timeout:
print(f"TIMEOUT error registering {url}, trying longer timeout")
logging.warning(f"TIMEOUT error registering {url}, trying longer timeout")
r = requests.get(url, headers=headers, timeout=(31, 31))
r.raise_for_status() # let it flow through, a timeout here means the site is unreasonably slow
except Exception as e:
print(f"CONNECTION ERROR when registering {url}")
print(e)
logging.error(f"CONNECTION ERROR when registering {url}")
logging.error(e)
return False
@ -83,13 +106,13 @@ def get_webfinger_subscribe_uri(username):
r.raise_for_status()
except requests.Timeout:
print(f"TIMEOUT error finding {username}, trying longer timeout")
logging.warning(f"TIMEOUT error finding {username}, trying longer timeout")
r = requests.get(webfinger_url, headers=headers, timeout=(31, 31))
r.raise_for_status() # let it flow through, a timeout here means the site is unreasonably slow
except Exception as e:
print(f"CONNECTION ERROR when subscribing via {username}")
print(e)
logging.error(f"CONNECTION ERROR when subscribing via {username}")
logging.error(e)
return None
data = r.json()

24
blogs/views/public.py
View file

@ -1,8 +1,6 @@
"""public views (no need to log in)"""
# pylint: disable=R6301
import os
from itertools import chain
from operator import attrgetter
@ -20,16 +18,13 @@ from django.views import View
from blogs import forms, models
from blogs.utilities import get_blog_info, get_webfinger_subscribe_uri
class HomeFeed(View):
"""the home feed when someone visits the site"""
def get(self, request):
"""display home page"""
path = os.path.join(settings.BASE_DIR, "static/")
print(path)
latest = models.Article.objects.order_by("-pubdate")[:10]
data = {"title": "Latest blog posts", "latest": latest}
@ -42,7 +37,9 @@ class Blogs(View):
def get(self, request):
"""here they are"""
blogs = models.Blog.objects.filter(approved=True)
blogs = models.Blog.objects.filter(approved=True, active=True).order_by(
"-updateddate"
)
for blog in blogs:
blog.category_name = models.Category(blog.category).label
data = {"title": "Blogs and websites", "blogs": blogs}
@ -140,14 +137,13 @@ class RegisterBlog(View):
data["blog_info"] = blog_info
else:
data["error"] = "Could not auto-discover your feed info, please enter manually"
data[
"error"
] = "Could not auto-discover your feed info, please enter manually"
return render(request, "blogs/confirm-register.html", data)
data = {
"title": "Register your blog",
"form": form
}
data = {"title": "Register your blog", "form": form}
return render(request, "blogs/register.html", data)
@ -311,7 +307,7 @@ class Search(View):
"description", weight="C"
)
conferences = (
events = (
models.Event.objects.annotate(rank=SearchRank(conference_vector, query))
.filter(rank__gte=0.1)
.order_by("-rank")
@ -348,7 +344,7 @@ class Search(View):
)
combined = sorted(
chain(articles, conferences, cfps, newsletters, groups),
chain(articles, events, cfps, newsletters, groups),
key=attrgetter("rank"),
reverse=True,
)