hugh/ausglamr
1
Fork 0
Code Issues 1 Pull requests Actions Packages Projects Releases Wiki Activity

another ridiculously large commit

Browse source
This commit is contained in:
Hugh Rundle 2024-01-26 11:00:26 +11:00
parent 8cc4bda231
commit ff1d837eb6
Signed by: hugh
GPG key ID: A7E35779918253F9
39 changed files with 708 additions and 266 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.env.example
View file

@ -25,7 +25,12 @@ POSTGRES_DB="ausglamr"
PGPORT=5432 PGPORT=5432
POSTGRES_HOST="db" POSTGRES_HOST="db"
# mastodon # mastodon bot
MASTODON_ACCESS_TOKEN="" MASTODON_ACCESS_TOKEN=""
MASTODON_DOMAIN="https://example.com" MASTODON_DOMAIN="https://example.com"
# filepaths for backups cron - include a leading slash
DOCKER_PATH="/usr/bin/docker" # example, check your path
BACKUPS_DIR="/home/my_user/backups" # where you want your backups to be stored

7
.gitignore vendored
View file

@ -1,2 +1,5 @@
data /data
fixtures fixtures
/static
.env.dev
z_README_Hugh.md

17
README.md
View file

@ -12,7 +12,7 @@ A django app running on Docker. Replaces _Aus GLAM Blogs_.
* set up database backups (as cron jobs): * set up database backups (as cron jobs):
* `./glamr-dev backup` * `./glamr-dev backup`:
* `/snap/bin/docker exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump` * `/snap/bin/docker exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump`
* `/snap/bin/docker cp ausglamr_db_1:/tmp/ausglamr_backup.dump /home/hugh/backups/` * `/snap/bin/docker cp ausglamr_db_1:/tmp/ausglamr_backup.dump /home/hugh/backups/`
@ -20,10 +20,6 @@ A django app running on Docker. Replaces _Aus GLAM Blogs_.
* set up cron jobs for management commands as below * set up cron jobs for management commands as below
## Migrating
See `data/help.txt`
## Admin ## Admin
Don't forget to add some Content Warnings for use by the Mastodon bot. Don't forget to add some Content Warnings for use by the Mastodon bot.
@ -33,6 +29,7 @@ Don't forget to add some Content Warnings for use by the Mastodon bot.
Use `glamr-dev` to make your life easier (thanks to Mouse Reeve for the inspiration): Use `glamr-dev` to make your life easier (thanks to Mouse Reeve for the inspiration):
* announce * announce
* backup
* check_feeds * check_feeds
* manage [django management command] * manage [django management command]
* makemigrations * makemigrations
@ -77,6 +74,8 @@ Run every 21 mins.
This checks all blog feeds for any new posts, and adds them to the database as long as they don't have an exclusion tag and were not published during a time the blog was suspended. This checks all blog feeds for any new posts, and adds them to the database as long as they don't have an exclusion tag and were not published during a time the blog was suspended.
Also checks newsletter articles if there is a feed.
Run every hour. Run every hour.
### queue_announcements ### queue_announcements
@ -89,4 +88,10 @@ Run daily.
Does what you think. Creates a weekly email of the latest stuff, and send to everyone in Subscribers. Does what you think. Creates a weekly email of the latest stuff, and send to everyone in Subscribers.
Run weekly. Run weekly.
### Backups
There is a `backup` command in `glamr-dev`. You can adjust the filepaths in your `.env` file.
Run daily

10
ausglamr/urls.py
View file

@ -12,13 +12,17 @@ urlpatterns = [
re_path(r"^browse/?$", views.Browse.as_view(), name="browse"), re_path(r"^browse/?$", views.Browse.as_view(), name="browse"),
re_path(r"^search/?$", views.Search.as_view(), name="search"), re_path(r"^search/?$", views.Search.as_view(), name="search"),
re_path(r"^help/?$", views.Help.as_view(), name="help"), re_path(r"^help/?$", views.Help.as_view(), name="help"),
path("feed", views.CombinedFeed(), name="feed"),
path("blog-articles/feed", views.ArticleFeed(), name="article-feed"),
path("events/feed", views.EventFeed(), name="event-feed"),
path("newsletter-editions/feed", views.EditionFeed(), name="edition-feed"),
path("contribute", views.Contribute.as_view(), name="contribute"), path("contribute", views.Contribute.as_view(), name="contribute"),
path("contact", views.Contact.as_view(), name="contact"), path("contact", views.Contact.as_view(), name="contact"),
path("blogs", views.Blogs.as_view(), name="blogs"), path("blogs", views.Blogs.as_view(), name="blogs"),
path("blogs/<category>", views.Blogs.as_view(), name="blog-category "), path("blogs/<category>", views.Blogs.as_view(), name="blog-category "),
path("blog-articles", views.Articles.as_view(), name="blog-articles"), path("blog-articles", views.Articles.as_view(), name="blog-articles"),
path("events", views.Conferences.as_view(), name="events"), path("events", views.Events.as_view(), name="events"),
path("events/<category>", views.Conferences.as_view(), name="events-category"), path("events/<category>", views.Events.as_view(), name="events-category"),
re_path(r"^cfps/?$", views.CallsForPapers.as_view(), name="cfps"), re_path(r"^cfps/?$", views.CallsForPapers.as_view(), name="cfps"),
path("groups", views.Groups.as_view(), name="groups"), path("groups", views.Groups.as_view(), name="groups"),
path("groups/<category>", views.Groups.as_view(), name="group-category"), path("groups/<category>", views.Groups.as_view(), name="group-category"),
@ -60,6 +64,4 @@ urlpatterns = [
views.UnsubscribeEmail.as_view(), views.UnsubscribeEmail.as_view(),
name="unsubscribe-email", name="unsubscribe-email",
), ),
path("feeds/blogs", views.ArticleFeed(), name="article-feed"),
path("feeds/events", views.EventFeed(), name="event-feed"),
] ]

2
blogs/forms.py
View file

@ -115,7 +115,7 @@ class RegisterNewsletterForm(forms.ModelForm):
model = Newsletter model = Newsletter
fields = [ fields = [
"name", "name",
"author", "author_name",
"category", "category",
"url", "url",
"feed", "feed",

276
blogs/management/commands/check_feeds.py
View file

@ -47,6 +47,16 @@ class Command(BaseCommand):
action="store_true", action="store_true",
help="Suppress non-error messages", help="Suppress non-error messages",
) )
parser.add_argument(
"-blogs",
action="store_true",
help="Only check blog posts",
)
parser.add_argument(
"-newsletters",
action="store_true",
help="Only check editions",
)
def handle(self, *args, **options): def handle(self, *args, **options):
"""check feeds and update database""" """check feeds and update database"""
@ -56,89 +66,153 @@ class Command(BaseCommand):
f"checking feeds at {django_timezone.localtime(django_timezone.now())}" f"checking feeds at {django_timezone.localtime(django_timezone.now())}"
) )
blogs = models.Blog.objects.filter( if not options["newsletters"]:
approved=True, suspended=False, active=True blogs = models.Blog.objects.filter(
).all() approved=True, suspended=False, active=True
for blog in blogs: ).all()
try: for blog in blogs:
data = feedparser.parse(blog.feed, agent=agent) try:
data = feedparser.parse(blog.feed, agent=agent)
for article in data.entries: for article in data.entries:
if not models.Article.objects.filter( if not models.Article.objects.filter(
Q(url=article.link) | Q(guid=getattr(article, "id", article.link)) Q(url=article.link)
).exists(): | Q(guid=getattr(article, "id", article.link))
if blog.suspension_lifted and ( ).exists():
blog.suspension_lifted if blog.suspension_lifted and (
> date_to_tz_aware(article.updated_parsed) blog.suspension_lifted
): > date_to_tz_aware(article.updated_parsed)
continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension)
taglist = getattr(article, "tags", None) or getattr(
article, "categories", []
)
tags = [tag.term.lower() for tag in taglist]
opt_out = False
# don't include posts with opt out tags
for tag in tags:
if (
len(
{tag}
& {
"notglam",
"notglamr",
"notausglamblogs",
"notausglamr",
"notglamblogs",
"#notglam",
}
)
> 0
): ):
opt_out = True continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension)
else:
continue
if not opt_out: taglist = getattr(article, "tags", None) or getattr(
author_name = getattr(article, "author", None) or getattr( article, "categories", []
blog, "author", None )
tags = [tag.term.lower() for tag in taglist]
opt_out = False
# don't include posts with opt out tags
for tag in tags:
if (
len(
{tag}
& {
"notglam",
"notglamr",
"notausglamblogs",
"notausglamr",
"notglamblogs",
"#notglam",
}
)
> 0
):
opt_out = True
else:
continue
if not opt_out:
author_name = getattr(
article, "author", None
) or getattr(blog, "author", None)
description = (
html.strip_tags(article.summary)
if (
hasattr(article, "summary")
and len(article.summary) > 0
)
else html.strip_tags(article.description)
if (
hasattr(article, "description")
and len(article.summary)
)
else html.strip_tags(article.content[0].value)[:200]
)
description += "..."
instance = models.Article.objects.create(
title=article.title,
author_name=author_name,
url=article.link,
description=description,
updateddate=date_to_tz_aware(
article.updated_parsed
),
blog=blog,
pubdate=date_to_tz_aware(article.published_parsed),
guid=getattr(article, "id", article.link),
)
tags_to_add = get_tags(
getattr(article, "tags", None)
or getattr(article, "categories", [])
)
for tag in tags_to_add:
instance.tags.add(tag)
instance.save()
cutoff = django_timezone.now() - timedelta(days=3)
newish = instance.pubdate > cutoff
if newish:
instance.announce()
blog.set_success(
updateddate=date_to_tz_aware(article.updated_parsed)
)
except Exception as e:
blog.set_failing()
logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}")
logging.info(article)
logging.error(e)
if not options["blogs"]:
newsletters = models.Newsletter.objects.filter(
approved=True, active=True, feed__isnull=False
).all()
for newsletter in newsletters:
try:
data = feedparser.parse(newsletter.feed, agent=agent)
for edition in data.entries:
if not models.Edition.objects.filter(
Q(url=edition.link)
| Q(guid=getattr(edition, "id", edition.link))
).exists():
author_name = getattr(edition, "author", None) or getattr(
edition, "author", None
) )
description = ( description = (
html.strip_tags(article.summary) html.strip_tags(edition.summary)
if ( if (
hasattr(article, "summary") hasattr(edition, "summary") and len(edition.summary)
and len(article.summary) > 0
) )
else html.strip_tags(article.description) else html.strip_tags(edition.description)
if ( if (
hasattr(article, "description") hasattr(edition, "description")
and len(article.summary) and len(edition.summary)
) )
else html.strip_tags(article.content[0].value)[:200] else html.strip_tags(edition.content[0].value)[:200]
+ "..."
) )
description += "..." description += "..."
instance = models.Article.objects.create( instance = models.Edition.objects.create(
title=article.title, title=edition.title,
author_name=author_name, author_name=author_name,
url=article.link, url=edition.link,
description=description, description=description,
updateddate=date_to_tz_aware(article.updated_parsed), updateddate=date_to_tz_aware(edition.updated_parsed),
blog=blog, newsletter=newsletter,
pubdate=date_to_tz_aware(article.published_parsed), pubdate=date_to_tz_aware(edition.published_parsed),
guid=getattr(article, "id", article.link), guid=getattr(edition, "id", edition.link),
) )
tags_to_add = get_tags(
getattr(article, "tags", None)
or getattr(article, "categories", [])
)
for tag in tags_to_add:
instance.tags.add(tag)
instance.save() instance.save()
cutoff = django_timezone.now() - timedelta(days=3) cutoff = django_timezone.now() - timedelta(days=3)
@ -146,72 +220,16 @@ class Command(BaseCommand):
if newish: if newish:
instance.announce() instance.announce()
blog.set_success( newsletter.set_success(
updateddate=date_to_tz_aware(article.updated_parsed) updateddate=date_to_tz_aware(edition.updated_parsed)
) )
except Exception as e: except Exception as e:
blog.set_failing() newsletter.set_failing()
logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}") logging.error(
logging.info(article) f"ERROR WITH NEWSLETTER {newsletter.name} - {newsletter.url}"
logging.error(e)
newsletters = models.Newsletter.objects.filter(
approved=True, active=True, feed__isnull=False
).all()
for newsletter in newsletters:
try:
data = feedparser.parse(newsletter.feed, agent=agent)
for edition in data.entries:
if not models.Edition.objects.filter(
Q(url=edition.link) | Q(guid=getattr(edition, "id", edition.link))
).exists():
author_name = getattr(edition, "author", None) or getattr(
blog, "author", None
)
description = (
html.strip_tags(edition.summary)
if (
hasattr(edition, "summary") and len(edition.summary)
)
else html.strip_tags(edition.description)
if (
hasattr(edition, "description") and len(edition.summary)
)
else html.strip_tags(edition.content[0].value)[:200] + "..."
)
description += "..."
instance = models.Edition.objects.create(
title=edition.title,
author_name=author_name,
url=edition.link,
description=description,
updateddate=date_to_tz_aware(edition.updated_parsed),
newsletter=newsletter,
pubdate=date_to_tz_aware(edition.published_parsed),
guid=getattr(edition, "id", edition.link),
)
instance.save()
cutoff = django_timezone.now() - timedelta(days=3)
newish = instance.pubdate > cutoff
if newish:
instance.announce()
newsletter.set_success(
updateddate=date_to_tz_aware(edition.updated_parsed)
) )
logging.error(e)
except Exception as e:
newsletter.set_failing()
logging.error(
f"ERROR WITH NEWSLETTER {newsletter.name} - {newsletter.url}"
)
logging.error(e)
if not options["q"]: if not options["q"]:
logging.info( logging.info(

34
blogs/management/commands/send_weekly_email.py
View file

@ -32,14 +32,15 @@ class Command(BaseCommand):
approved=True, active=True, added__gte=cutoff approved=True, active=True, added__gte=cutoff
) )
articles = models.Article.objects.filter(pubdate__gte=cutoff) articles = models.Article.objects.filter(pubdate__gte=cutoff)
events = models.Event.objects.filter(approved=True, pub_date__gte=cutoff) events = models.Event.objects.filter(approved=True, pubdate__gte=cutoff)
cfps = models.CallForPapers.objects.filter( cfps = models.CallForPapers.objects.filter(
event__approved=True, closing_date__gte=timezone.now().date() event__approved=True, closing_date__gte=timezone.now().date()
) )
newsletters = models.Newsletter.objects.filter( newsletters = models.Newsletter.objects.filter(
approved=True, pub_date__gte=cutoff approved=True, pubdate__gte=cutoff
) )
groups = models.Group.objects.filter(approved=True, pub_date__gte=cutoff) editions = models.Edition.objects.filter(pubdate__gte=cutoff)
groups = models.Group.objects.filter(approved=True, pubdate__gte=cutoff)
new_blogs = "" new_blogs = ""
for blog in blogs: for blog in blogs:
@ -78,11 +79,33 @@ class Command(BaseCommand):
if new_articles != "": if new_articles != "":
new_articles = ( new_articles = (
"<h3 style='margin-top:20px;'>New Articles</h3>" "<h3 style='margin-top:20px;'>New Blog Posts</h3>"
+ new_articles + new_articles
+ "<hr/>" + "<hr/>"
) )
new_editions = ""
for edition in editions:
title_string = f"<h4><a href='{edition.url}'>{edition.title}</a></h4>"
author_string = (
f"<p><em>{edition.author_name}</em></p>" if edition.author_name else ""
)
description_string = (
f"<p style='margin-bottom:24px;'>{edition.description}</p>"
)
string_list = [title_string, author_string, description_string]
string = "".join(string_list)
new_editions = new_editions + string
if new_editions != "":
new_editions = (
"<h3 style='margin-top:20px;'>New Newsletter Editions</h3>"
+ new_editions
+ "<hr/>"
)
coming_events = "" coming_events = ""
for event in events: for event in events:
s_date = event.start_date s_date = event.start_date
@ -195,6 +218,7 @@ class Command(BaseCommand):
subject = f"{emoji} Fresh Aus GLAMR updates for the week of {dt.day} {dt:%B} {dt.year}" subject = f"{emoji} Fresh Aus GLAMR updates for the week of {dt.day} {dt:%B} {dt.year}"
sections = [ sections = [
new_articles, new_articles,
new_editions,
new_blogs, new_blogs,
new_newsletters, new_newsletters,
new_groups, new_groups,
@ -203,7 +227,7 @@ class Command(BaseCommand):
] ]
body = "".join(sections) body = "".join(sections)
if body == "": if body == "":
body = "<p>No new updates this week.</p><p>Why not spend the time you would have been reading, publishing your own blog post instead?</p>" body = "<p>No new updates this week.</p><p>Why not spend some time publishing your own blog post instead?</p>"
for subscriber in subscribers: for subscriber in subscribers:
opt_out = f"https://{settings.DOMAIN}/unsubscribe-email/{subscriber.token}/{subscriber.id}" opt_out = f"https://{settings.DOMAIN}/unsubscribe-email/{subscriber.token}/{subscriber.id}"

32
blogs/migrations/0001_initial.py
View file

@ -1,4 +1,4 @@
# Generated by Django 4.2.7 on 2024-01-21 04:31 # Generated by Django 4.2.7 on 2024-01-22 08:13
import blogs.models.blog import blogs.models.blog
from django.db import migrations, models from django.db import migrations, models
@ -50,6 +50,10 @@ class Migration(migrations.Migration):
("url", models.URLField(max_length=2000, unique=True)), ("url", models.URLField(max_length=2000, unique=True)),
("description", models.TextField(blank=True, null=True)), ("description", models.TextField(blank=True, null=True)),
("updateddate", models.DateTimeField()), ("updateddate", models.DateTimeField()),
(
"pubdate",
models.DateTimeField(default=django.utils.timezone.now, null=True),
),
("feed", models.URLField(max_length=2000)), ("feed", models.URLField(max_length=2000)),
( (
"category", "category",
@ -143,7 +147,7 @@ class Migration(migrations.Migration):
"description", "description",
models.TextField(blank=True, max_length=250, null=True), models.TextField(blank=True, max_length=250, null=True),
), ),
("pub_date", models.DateTimeField()), ("pubdate", models.DateTimeField()),
("start_date", models.DateField()), ("start_date", models.DateField()),
( (
"announcements", "announcements",
@ -219,7 +223,10 @@ class Migration(migrations.Migration):
), ),
("announced", models.BooleanField(default=False)), ("announced", models.BooleanField(default=False)),
("approved", models.BooleanField(default=False)), ("approved", models.BooleanField(default=False)),
("pub_date", models.DateTimeField(default=None, null=True)), (
"pubdate",
models.DateTimeField(default=django.utils.timezone.now, null=True),
),
], ],
), ),
migrations.CreateModel( migrations.CreateModel(
@ -235,7 +242,7 @@ class Migration(migrations.Migration):
), ),
), ),
("name", models.CharField(max_length=100)), ("name", models.CharField(max_length=100)),
("author", models.CharField(max_length=100)), ("author_name", models.CharField(max_length=100)),
( (
"category", "category",
models.CharField( models.CharField(
@ -275,7 +282,7 @@ class Migration(migrations.Migration):
("active", models.BooleanField(default=True, null=True)), ("active", models.BooleanField(default=True, null=True)),
("failing", models.BooleanField(blank=True, default=False, null=True)), ("failing", models.BooleanField(blank=True, default=False, null=True)),
("updateddate", models.DateTimeField()), ("updateddate", models.DateTimeField()),
("pub_date", models.DateTimeField(default=None, null=True)), ("pubdate", models.DateTimeField(default=None, null=True)),
], ],
), ),
migrations.CreateModel( migrations.CreateModel(
@ -350,7 +357,10 @@ class Migration(migrations.Migration):
("url", models.URLField(max_length=2000, unique=True)), ("url", models.URLField(max_length=2000, unique=True)),
("description", models.TextField(blank=True, null=True)), ("description", models.TextField(blank=True, null=True)),
("updateddate", models.DateTimeField()), ("updateddate", models.DateTimeField()),
("pubdate", models.DateTimeField()), (
"pubdate",
models.DateTimeField(default=django.utils.timezone.now, null=True),
),
("guid", models.CharField(max_length=2000)), ("guid", models.CharField(max_length=2000)),
( (
"newsletter", "newsletter",
@ -376,7 +386,10 @@ class Migration(migrations.Migration):
), ),
("name", models.CharField(max_length=100)), ("name", models.CharField(max_length=100)),
("details", models.TextField(blank=True, max_length=250, null=True)), ("details", models.TextField(blank=True, max_length=250, null=True)),
("pub_date", models.DateTimeField(default=None, null=True)), (
"pubdate",
models.DateTimeField(default=django.utils.timezone.now, null=True),
),
("opening_date", models.DateField()), ("opening_date", models.DateField()),
("closing_date", models.DateField()), ("closing_date", models.DateField()),
("announcements", models.IntegerField(default=0, null=True)), ("announcements", models.IntegerField(default=0, null=True)),
@ -411,7 +424,10 @@ class Migration(migrations.Migration):
("url", models.URLField(max_length=2000, unique=True)), ("url", models.URLField(max_length=2000, unique=True)),
("description", models.TextField(blank=True, null=True)), ("description", models.TextField(blank=True, null=True)),
("updateddate", models.DateTimeField()), ("updateddate", models.DateTimeField()),
("pubdate", models.DateTimeField()), (
"pubdate",
models.DateTimeField(default=django.utils.timezone.now, null=True),
),
("guid", models.CharField(max_length=2000)), ("guid", models.CharField(max_length=2000)),
( (
"blog", "blog",

2
blogs/models/blog.py
View file

@ -29,6 +29,7 @@ class BlogData(models.Model):
url = models.URLField(max_length=2000, unique=True) url = models.URLField(max_length=2000, unique=True)
description = models.TextField(null=True, blank=True) description = models.TextField(null=True, blank=True)
updateddate = models.DateTimeField() updateddate = models.DateTimeField()
pubdate = models.DateTimeField(null=True, default=timezone.now)
class Meta: class Meta:
"""This is an abstract model for common data""" """This is an abstract model for common data"""
@ -107,7 +108,6 @@ class Article(BlogData):
"""A blog post""" """A blog post"""
blog = models.ForeignKey(Blog, on_delete=models.CASCADE, related_name="articles") blog = models.ForeignKey(Blog, on_delete=models.CASCADE, related_name="articles")
pubdate = models.DateTimeField()
guid = models.CharField(max_length=2000) guid = models.CharField(max_length=2000)
tags = models.ManyToManyField("Tag", related_name="articles") tags = models.ManyToManyField("Tag", related_name="articles")

14
blogs/models/event.py
View file

@ -13,7 +13,7 @@ class Event(models.Model):
category = models.CharField(choices=Category.choices, max_length=4) category = models.CharField(choices=Category.choices, max_length=4)
url = models.URLField(max_length=400, unique=True) url = models.URLField(max_length=400, unique=True)
description = models.TextField(null=True, blank=True, max_length=250) description = models.TextField(null=True, blank=True, max_length=250)
pub_date = models.DateTimeField() # for RSS feed pubdate = models.DateTimeField() # for RSS feed
start_date = models.DateField() start_date = models.DateField()
announcements = models.IntegerField(null=True, blank=True, default=0) announcements = models.IntegerField(null=True, blank=True, default=0)
activitypub_account_name = models.CharField(max_length=200, blank=True, null=True) activitypub_account_name = models.CharField(max_length=200, blank=True, null=True)
@ -21,8 +21,8 @@ class Event(models.Model):
approved = models.BooleanField(default=False) approved = models.BooleanField(default=False)
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
if not self.pub_date: if not self.pubdate:
self.pub_date = timezone.now() self.pubdate = timezone.now()
super().save(*args, **kwargs) super().save(*args, **kwargs)
def __str__(self): def __str__(self):
@ -58,7 +58,7 @@ class CallForPapers(models.Model):
max_length=100 max_length=100
) # "Call for papers", "call for participation" etc ) # "Call for papers", "call for participation" etc
details = models.TextField(null=True, blank=True, max_length=250) details = models.TextField(null=True, blank=True, max_length=250)
pub_date = models.DateTimeField(null=True, default=None) pubdate = models.DateTimeField(null=True, default=timezone.now)
opening_date = models.DateField() opening_date = models.DateField()
closing_date = models.DateField() closing_date = models.DateField()
announcements = models.IntegerField(null=True, default=0) announcements = models.IntegerField(null=True, default=0)
@ -77,9 +77,3 @@ class CallForPapers(models.Model):
Announcement.objects.create(status=status) Announcement.objects.create(status=status)
self.announcements = self.announcements + 1 self.announcements = self.announcements + 1
super().save() super().save()
def save(self, *args, **kwargs):
"""save a CFP with pub_date"""
if not self.pub_date:
self.pub_date = timezone.now()
super().save(*args, **kwargs)

8
blogs/models/group.py
View file

@ -16,9 +16,10 @@ class Group(models.Model):
registration_url = models.URLField(max_length=400, unique=True) registration_url = models.URLField(max_length=400, unique=True)
description = models.TextField(null=True, blank=True, max_length=250) description = models.TextField(null=True, blank=True, max_length=250)
contact_email = models.EmailField(blank=True, null=True) contact_email = models.EmailField(blank=True, null=True)
announced = models.BooleanField(default=False) announced = models.BooleanField(default=False)
approved = models.BooleanField(default=False) approved = models.BooleanField(default=False)
pub_date = models.DateTimeField(null=True, default=None) pubdate = models.DateTimeField(null=True, default=timezone.now)
def announce(self): def announce(self):
"""create a group announcement""" """create a group announcement"""
@ -30,8 +31,3 @@ class Group(models.Model):
Announcement.objects.create(status=status) Announcement.objects.create(status=status)
self.announced = True self.announced = True
super().save() super().save()
def save(self, *args, **kwargs):
if not self.pub_date:
self.pub_date = timezone.now()
super().save(*args, **kwargs)

32
blogs/models/newsletter.py
View file

@ -10,7 +10,7 @@ class Newsletter(models.Model):
"""a newsletter""" """a newsletter"""
name = models.CharField(max_length=100) name = models.CharField(max_length=100)
author = models.CharField(max_length=100) author_name = models.CharField(max_length=100)
category = models.CharField(choices=Category.choices, max_length=4) category = models.CharField(choices=Category.choices, max_length=4)
url = models.URLField(max_length=400, unique=True) url = models.URLField(max_length=400, unique=True)
feed = models.URLField(max_length=1000, unique=True, blank=True, null=True) feed = models.URLField(max_length=1000, unique=True, blank=True, null=True)
@ -24,17 +24,21 @@ class Newsletter(models.Model):
failing = models.BooleanField(default=False, blank=True, null=True) failing = models.BooleanField(default=False, blank=True, null=True)
updateddate = models.DateTimeField() updateddate = models.DateTimeField()
pub_date = models.DateTimeField(null=True, default=None) pubdate = models.DateTimeField(null=True, default=None)
def __str__(self):
"""display for admin dropdowns"""
return self.name
def announce(self): def announce(self):
"""create a event announcement""" """create a event announcement"""
category = Category(self.category).label category = Category(self.category).label
name = self.name name = self.author_name
if self.activitypub_account_name: if self.activitypub_account_name:
name = f"{self.name} ({self.activitypub_account_name})" name = f"{self.author_name} ({self.activitypub_account_name})"
status = f"{name} is a newsletter about {category} from {self.author}. Check it out:\n\n{self.url}" status = f"{self.name} is a newsletter about {category} from {name}. Check it out:\n\n{self.url}"
Announcement.objects.create(status=status) Announcement.objects.create(status=status)
self.announced = True self.announced = True
@ -74,5 +78,21 @@ class Edition(models.Model):
newsletter = models.ForeignKey( newsletter = models.ForeignKey(
Newsletter, on_delete=models.CASCADE, related_name="editions" Newsletter, on_delete=models.CASCADE, related_name="editions"
) )
pubdate = models.DateTimeField() pubdate = models.DateTimeField(null=True, default=timezone.now)
guid = models.CharField(max_length=2000) guid = models.CharField(max_length=2000)
def announce(self):
"""queue an edition announcement"""
author = self.newsletter.activitypub_account_name or self.author_name
if self.newsletter.activitypub_account_name:
author = f"{self.newsletter.activitypub_account_name} - "
elif self.author_name:
author = f"{self.author_name} - "
else:
author = ""
status = f"📬 {self.title} ({author}{self.newsletter.name})\n\n{self.url}"
Announcement.objects.create(status=status)

13
blogs/static/css/custom.css
View file

@ -199,7 +199,7 @@ footer .left p {
.subscribe .pop, .subscribe .pop,
.contribute .pop { .contribute .pop {
min-height: 48rem; min-height: 44rem;
} }
.contribute { .contribute {
@ -209,9 +209,6 @@ footer .left p {
grid-template-columns: 33% 33% 33%; grid-template-columns: 33% 33% 33%;
} }
/* .contribute .pop {
min-height: 28rem;
} */
.pop { .pop {
margin-top: 2em; margin-top: 2em;
@ -228,12 +225,9 @@ footer .left p {
grid-template-columns: 50% 50%; grid-template-columns: 50% 50%;
} }
.subscribe .pop { .subscribe .pop,
min-height: 34rem;
}
.contribute .pop { .contribute .pop {
min-height: 30rem; min-height: 33rem;
} }
} }
@ -336,7 +330,6 @@ footer .left p {
background-color: transparent; background-color: transparent;
border-radius: 4px; border-radius: 4px;
border: 1px solid #aaa; border: 1px solid #aaa;
cursor: pointer;
box-sizing: border-box; box-sizing: border-box;
} }

15
blogs/static/favicon.svg Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg width="100%" height="100%" viewBox="0 0 1000 1000" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
<g id="Layer1">
<rect x="-1.793" y="0.753" width="1002.14" height="1000.3"/>
</g>
<g transform="matrix(0.923477,0,0,0.923477,82.0351,29.5455)">
<g transform="matrix(0.909938,0,0,0.909938,48.7165,46.9138)">
<text x="1.567px" y="999.474px" style="font-family:'Helsinki';font-size:1395.51px;fill:white;">A</text>
</g>
<g transform="matrix(0.909938,0,0,0.909938,21.5737,36.7307)">
<text x="1.567px" y="999.474px" style="font-family:'Helsinki';font-size:1395.51px;fill:rgb(255,182,193);">A</text>
</g>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 1 KiB

6
blogs/templates/browse/articles.html
View file

@ -1,5 +1,9 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="blog-articles/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Blogs" />
{% endblock %}
{% block content %} {% block content %}
{% for post in latest %} {% for post in latest %}
<div class="card"> <div class="card">
@ -25,5 +29,7 @@
</div> </div>
{% endif %} {% endif %}
</div> </div>
{% empty %}
<p>Oh no! There are no articles available. Try checking out <a href="{% url 'newsletter-editions' %}">some newsletter editions</a>.</p>
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

6
blogs/templates/browse/blogs.html
View file

@ -1,5 +1,9 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="blog-articles/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Blogs" />
{% endblock %}
{% block content %} {% block content %}
<div class="buttons"> <div class="buttons">
<a href="{% url 'blog-articles' %}"><button class="button button-primary button-first">Latest posts</button></a> <a href="{% url 'blog-articles' %}"><button class="button button-primary button-first">Latest posts</button></a>
@ -23,7 +27,7 @@
</div> </div>
<hr/> <hr/>
{% empty %} {% empty %}
<p>Oh no! There are no blogs currently registered. Try checking out <a href="{% url 'newsletters' %}">some newsletters</a>.</p> <p>Oh no! There are no blogs currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out <a href="{% url 'newsletters' %}">some newsletters</a>.</p>
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

8
blogs/templates/browse/cfp.html
View file

@ -1,7 +1,13 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block content %} {% block feed %}
<link href="events/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Events" />
{% endblock %}
{% block content %}
<div class="buttons">
<a href="{% url 'register-cfp' %}"><button class="button">Add a CFP</button></a>
</div>
<div class="listing l-three header"> <div class="listing l-three header">
<span>Details</span> <span>Details</span>
<span>Event</span> <span>Event</span>

8
blogs/templates/browse/editions.html
View file

@ -1,5 +1,9 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="newsletter-editions/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Newsletters" />
{% endblock %}
{% block content %} {% block content %}
{% for edition in latest %} {% for edition in latest %}
<div class="card"> <div class="card">
@ -9,7 +13,7 @@
{% if edition.author_name %} {% if edition.author_name %}
<span class="author_name">{{ edition.author_name }}</span> | <span class="author_name">{{ edition.author_name }}</span> |
{% endif %} {% endif %}
<span class="blog_title">{{ edition.newsletter.title }}</span> <span class="blog_title">{{ edition.newsletter.name }}</span>
</p> </p>
</div> </div>
{% if edition.description %} {% if edition.description %}
@ -20,5 +24,7 @@
</div> </div>
{% endif %} {% endif %}
</div> </div>
{% empty %}
<p>Oh no! There are no editions available. Try checking out <a href="{% url 'blog-articles' %}">some blog posts</a>.</p>
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

8
blogs/templates/browse/events.html
View file

@ -1,9 +1,13 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="events/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Events" />
{% endblock %}
{% block content %} {% block content %}
<div class="buttons"> <div class="buttons">
<a href="{% url 'cfps' %}"><button class="button button-primary button-first">Open CFPs</button></a> <a href="{% url 'cfps' %}"><button class="button button-primary button-first">Current CFPs</button></a>
<a href="{% url 'register-event' %}"><button class="button">Add an Event</button></a> <a href="{% url 'register-event' %}"><button class="button">Add an Event</button></a>
<a href="{% url 'register-cfp' %}"><button class="button">Add a CFP</button></a> <a href="{% url 'register-cfp' %}"><button class="button">Add a CFP</button></a>
</div> </div>
@ -29,7 +33,7 @@
</div> </div>
<hr/> <hr/>
{% empty %} {% empty %}
<p>Oh no! There are no events currently registered. Try checking out <a href="{% url 'blogs' %}">some blogs</a>.</p> <p>Oh no! There are no events currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out <a href="{% url 'blogs' %}">some blogs</a>.</p>
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

8
blogs/templates/browse/newsletters.html
View file

@ -1,5 +1,9 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="newsletter-editions/feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR Newsletters" />
{% endblock %}
{% block content %} {% block content %}
<div class="buttons"> <div class="buttons">
<a href="{% url 'newsletter-editions' %}"><button class="button button-primary button-first">Latest editions</button></a> <a href="{% url 'newsletter-editions' %}"><button class="button button-primary button-first">Latest editions</button></a>
@ -19,11 +23,11 @@
<a href="{{pub.feed}}">{% include 'utils/rss-img.html' %}</a> <a href="{{pub.feed}}">{% include 'utils/rss-img.html' %}</a>
{% endif %} {% endif %}
</span> </span>
<span>{{pub.author}}</span> <span>{{pub.author_name}}</span>
<span><span class="badge badge_{{pub.category}}"><a href="/newsletters/{{pub.category}}">{{pub.category_name}}</a></span></span> <span><span class="badge badge_{{pub.category}}"><a href="/newsletters/{{pub.category}}">{{pub.category_name}}</a></span></span>
</div> </div>
<hr/> <hr/>
{% empty %} {% empty %}
<p>Oh no! There are no newsletter currently registered. Try checking out <a href="{% url 'blogs' %}">some blogs</a>.</p> <p>Oh no! There are no newsletter currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out <a href="{% url 'blogs' %}">some blogs</a>.</p>
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

14
blogs/templates/contribute.html
View file

@ -26,20 +26,18 @@
<a class="button" href="{% url 'register-group' %}">Register group</a> <a class="button" href="{% url 'register-group' %}">Register group</a>
</section> </section>
<section class="pop">
<h2>Register an event</h2>
<p>Conference, convention, seminar, workshop, talk, meet-up...
</br>Whatever you call it, you know what we mean.</p>
<a class="button" href="{% url 'register-event' %}">Register event</a>
</section>
<section class="pop"> <section class="pop">
<h2>Register a newsletter</h2> <h2>Register a newsletter</h2>
<p>Whether it uses Ghost, Write.as, Mailchimp or something else, if it's an email newsletter with some kind of GLAMR focus you can register it here.</p> <p>Whether it uses Ghost, Write.as, Mailchimp or something else, if it's an email newsletter with some kind of GLAMR focus you can register it here.</p>
<a class="button" href="{% url 'register-newsletter' %}">Register newsletter</a> <a class="button" href="{% url 'register-newsletter' %}">Register newsletter</a>
</section> </section>
<section class="pop">
<h2>Register an event</h2>
<p>Conference, convention, seminar, workshop, talk, meet-up...
</br>Whatever you call it, you know what we mean.</p>
<a class="button" href="{% url 'register-event' %}">Register event</a>
</section>
<section class="pop"> <section class="pop">
<h2>Register or update a Call for Papers</h2> <h2>Register or update a Call for Papers</h2>

10
blogs/templates/help.html
View file

@ -30,7 +30,7 @@
{% with 'aus-glam-blogs' as anchor %} {% with 'aus-glam-blogs' as anchor %}
{% url 'help' as the_url %} {% url 'help' as the_url %}
<h4 id="{{anchor}}">What happened to Aus GLAM Blogs?</h4><a href="{{ the_url }}/#{{anchor}}">🔗</a> <h4 id="{{anchor}}">What happened to Aus GLAM Blogs?</h4><a href="{{ the_url }}/#{{anchor}}">🔗</a>
<p><em>Aus GLAM Blogs</em> was focussed on blogs specifically. <em>Aus GLAMR</em> is the successor, now including more GLAMR content. All the blog articles from the original site have been migrated and will be inluded in your search results.</p> <p><em>Aus GLAM Blogs</em> was focussed on blogs specifically. <em>Aus GLAMR</em> is the successor, now including more GLAMR content. All the blog articles from the original site have been migrated and will be included in your search results.</p>
{% endwith %} {% endwith %}
{% with 'pocket' as anchor %} {% with 'pocket' as anchor %}
@ -44,6 +44,14 @@
<h4 id="{{anchor}}">Activitypub account name!!???</h4><a href="{{ the_url }}/#{{anchor}}">🔗</a> <h4 id="{{anchor}}">Activitypub account name!!???</h4><a href="{{ the_url }}/#{{anchor}}">🔗</a>
<p>ActivityPub is the protocol used by Mastodon, BlueSky, BookWyrm, Threads, and other "fediverse" social media. If you include an account name, it will be mentioned when your registered thing is announced by the AusGLAMR Mastodon bot. If you'd like to join the "fediverse", try <a href="https://ausglam.space">Aus GLAM Space</a>.</p> <p>ActivityPub is the protocol used by Mastodon, BlueSky, BookWyrm, Threads, and other "fediverse" social media. If you include an account name, it will be mentioned when your registered thing is announced by the AusGLAMR Mastodon bot. If you'd like to join the "fediverse", try <a href="https://ausglam.space">Aus GLAM Space</a>.</p>
{% endwith %} {% endwith %}
{% with 'aus' as anchor %}
{% url 'help' as the_url %}
<h4 id="{{anchor}}">Why 'Aus' GLAMR?</h4><a href="{{ the_url }}/#{{anchor}}">🔗</a>
<p>The "Aus" is short for "Australasia". This is kind of a cheat - GLAMR content from Australia, Aotearoa/New Zealand, and our neighbours is welcome. I considered calling it "ANZ GLAMR" but I didn't want to do the Australian thing of pretending that Aotearoa/New Zealand is practically just another state of Australia with no cultural or institutional differences.</p>
<p>Opening it up to the whole world would likely result in the same thing that happens on every other English-language website: an assumed default of United States of America context, and a drowning out of anything else.</p>
<p>In short this aggregator is made by an Australian in Australia with that context in mind. Kiwis and other South-West Pacific neighbours are super welcome to contribute, but they also have their own context and may prefer their own thing. I can't speak for them.</p>
{% endwith %}
</section> </section>
{% endblock %} {% endblock %}

6
blogs/templates/index.html
View file

@ -1,12 +1,16 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block feed %}
<link href="feed" type="application/atom+xml" rel="alternate" title="Aus GLAMR" />
{% endblock %}
{% block intro %}{% endblock %} {% block intro %}{% endblock %}
{% block content %} {% block content %}
<section> <section>
<p><em>Aus GLAMR</em> is the place to find out what's happening in the Australasian cultural memory professions. Whether you work in galleries, libraries, archives, museums, or records, you'll find the latest blog posts, conferences and events, newsletters, and discussion groups here!</p> <p><em>Aus GLAMR</em> is the place to find out what's happening in the Australasian cultural memory professions. Whether you work in galleries, libraries, archives, museums, or records, you'll find the latest blog posts, conferences and events, newsletters, and discussion groups here!</p>
<p>Browse one of the lists, <a href="{% url 'search' %}">search by keywords</a>, or <a href="{% url 'contribute' %}">contribute by registering</a> your blog, event, group or newsletter.</p> <p>Browse one of the lists, <a href="{% url 'search' %}">search by keywords</a>, or <a href="{% url 'contribute' %}">contribute by registering</a> your blog, event, group or newsletter.</p>
<p>You can stay up to date by following the Mastodon bot or subscribing to email updates or to one of the RSS feeds - check out the <a href="{% url 'subscribe' %}">subscribe page</a>.</p> <p>You can stay up to date by subscribing via the Mastodon bot, email updates or one of the RSS feeds - check out the <a href="{% url 'subscribe' %}">subscribe page</a>.</p>
<p>Please note that whilst off-topic or egregiously offensive content may be refused or removed, inclusion on this site does not imply endorsement by Hugh or newCardigan of the content of any particular listed publication or event.</p> <p>Please note that whilst off-topic or egregiously offensive content may be refused or removed, inclusion on this site does not imply endorsement by Hugh or newCardigan of the content of any particular listed publication or event.</p>
</section> </section>
{% endblock %} {% endblock %}

1
blogs/templates/layout.html
View file

@ -11,6 +11,7 @@
<link rel="stylesheet" href="{% static 'css/normalize.css' %}" type="text/css" /> <link rel="stylesheet" href="{% static 'css/normalize.css' %}" type="text/css" />
<link rel="stylesheet" href="{% static 'css/skeleton.css' %}" type="text/css" /> <link rel="stylesheet" href="{% static 'css/skeleton.css' %}" type="text/css" />
<link rel="stylesheet" href="{% static 'css/custom.css' %}" type="text/css" /> <link rel="stylesheet" href="{% static 'css/custom.css' %}" type="text/css" />
{% block feed %}{% endblock %}
</head> </head>
<body> <body>
<header> <header>

15
blogs/templates/subscribe.html
View file

@ -15,7 +15,7 @@
</hgroup> </hgroup>
<p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.</p> <p>Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.</p>
<div> <div>
<a class="button" href="{% url 'subscribe-email' %}">Subscribe via email</a> <a class="button button-primary" href="{% url 'subscribe-email' %}">Subscribe</a>
</div> </div>
</section> </section>
@ -23,13 +23,13 @@
<hgroup> <hgroup>
<h3>Subscribe via ActivityPub</h3> <h3>Subscribe via ActivityPub</h3>
</hgroup> </hgroup>
<p>Follow the <code class="code">@blogs@ausglam.space</code> bot to see an announcement every time a new blog post , event, or call for papers is published.</p> <p>Follow <code class="code">@ausglamr@ausglam.space</code> to see an announcement every time something is added.</p>
<form method="post" action="{% url 'subscribe' %}"> <form method="post" action="{% url 'subscribe' %}">
{% csrf_token %} {% csrf_token %}
<label for="username">Enter address you want to follow from</label> <label for="username">Enter address you want to follow from</label>
{{ form.errors.username }} {{ form.errors.username }}
<input type="text" name="username" placeholder="@user@example.social"> <input type="text" name="username" placeholder="@user@example.social">
<button class="button u-pull-right" type="submit">Follow</button> <button class="button button-primary u-pull-right" type="submit">Follow</button>
</form> </form>
</section> </section>
@ -37,11 +37,12 @@
<hgroup> <hgroup>
<h3>Subscribe via RSS</h3> <h3>Subscribe via RSS</h3>
</hgroup> </hgroup>
<p>Subscribe to the blogs feed in your favourite RSS reader to get a combined feed with every new article.</p> <p>Ok technically it's Atom. Subscribe to the feed to stay up to date using your favourite feed reader.</p>
<p>Or subscribe to the events feed to be the first to know about upcoming events.</p>
<div> <div>
<a class="button" href="{% url 'article-feed' %}">blog posts</a> <a class="button" href="{% url 'article-feed' %}">blog posts</a><br/>
<a class="button" href="{% url 'event-feed' %}">events</a> <a class="button" href="{% url 'event-feed' %}">events & cfps</a><br/>
<a class="button" href="{% url 'edition-feed' %}">newsletter editions</a><br/>
<a class="button button-primary" href="{% url 'feed' %}">everything</a>
</div> </div>
</section> </section>

11
blogs/tests/data/example.html Normal file
View file

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>My test website with no RSS feed</title>
</head>
<body>
<p>Hello</p>
</body>
</html>

14
blogs/tests/data/good-example.html Normal file
View file

@ -0,0 +1,14 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="alternate" type="application/rss+xml" title="RSS" href="https://test.test/rss.xml">
<title>My test website with an RSS feed</title>
<meta name="author" content="Testy McTestface">
<meta name="description" content="My cool website">
</head>
<body>
<p>Hello</p>
</body>
</html>

12
blogs/tests/data/partial-example.html Normal file
View file

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="alternate" type="application/rss+xml" title="RSS" href="https://test.test/rss.xml">
<title>My test website with an RSS feed</title>
</head>
<body>
<p>Hello</p>
</body>
</html>

76
blogs/tests/test_commands.py
View file

@ -33,6 +33,27 @@ class FeedParserItemMock(object):
self.id = id self.id = id
class FeedParserEditionMock(object):
title = ""
author = ""
link = ""
summary = ""
updated_parsed = ((),)
published_parsed = ((),)
id = ""
def __init__(
self, title, author, link, summary, updated_parsed, published_parsed, id
):
self.title = title
self.author
self.link = link
self.summary = summary
self.updated_parsed = updated_parsed
self.published_parsed = published_parsed
self.id = id
class FeedParserTagMock(object): class FeedParserTagMock(object):
term = "" term = ""
@ -62,6 +83,14 @@ class CommandsTestCase(TestCase):
suspended=False, suspended=False,
) )
models.Newsletter.objects.create(
name="My awesome newsletter",
url="https://test.news",
feed="https://test.news/feed",
category="ARC",
approved=True,
)
tag_one = FeedParserTagMock(term="testing") tag_one = FeedParserTagMock(term="testing")
tag_two = FeedParserTagMock(term="python") tag_two = FeedParserTagMock(term="python")
tag_three = FeedParserTagMock(term="notglam") tag_three = FeedParserTagMock(term="notglam")
@ -113,15 +142,26 @@ class CommandsTestCase(TestCase):
id="333", id="333",
) )
edition = FeedParserEditionMock(
title="My amazing newsletter edition",
author="Hugh Rundle",
link="https://news.letter/1",
summary="A summary of my edition",
updated_parsed=updated_parsed,
published_parsed=published_parsed,
id="1",
)
self.feedparser = FeedParserMock(entries=[article]) self.feedparser = FeedParserMock(entries=[article])
self.feedparser_old = FeedParserMock(entries=[article_three]) self.feedparser_old = FeedParserMock(entries=[article_three])
self.feedparser_exclude = FeedParserMock(entries=[article_two]) self.feedparser_exclude = FeedParserMock(entries=[article_two])
self.feedparser_new = FeedParserMock(entries=[article_four]) self.feedparser_new = FeedParserMock(entries=[article_four])
self.feedparser_edition = FeedParserMock(entries=[edition])
def test_check_feeds(self): def test_check_feeds(self):
"""test parse a feed for basic blog info""" """test parse a feed for basic blog info"""
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
self.assertEqual(models.Article.objects.count(), 0) self.assertEqual(models.Article.objects.count(), 0)
@ -135,13 +175,31 @@ class CommandsTestCase(TestCase):
article = models.Article.objects.all().first() article = models.Article.objects.all().first()
self.assertEqual(article.title, "My amazing blog post") self.assertEqual(article.title, "My amazing blog post")
# should be announced # should be set to be announced
self.assertEqual(models.Announcement.objects.count(), 1)
def test_check_edition_feeds(self):
"""test parse a feed for newsletter edition info"""
args = {"-q": True, "-newsletters": True}
opts = {}
self.assertEqual(models.Edition.objects.count(), 0)
with patch("feedparser.parse", return_value=self.feedparser_edition):
value = call_command("check_feeds", *args, **opts)
self.assertEqual(models.Edition.objects.count(), 1)
edition = models.Edition.objects.all().first()
self.assertEqual(edition.title, "My amazing newsletter edition")
# should be set to be announced
self.assertEqual(models.Announcement.objects.count(), 1) self.assertEqual(models.Announcement.objects.count(), 1)
def test_check_feeds_duplicate(self): def test_check_feeds_duplicate(self):
"""test we do not ingest the same post twice""" """test we do not ingest the same post twice"""
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
self.assertEqual(models.Article.objects.count(), 0) self.assertEqual(models.Article.objects.count(), 0)
@ -164,7 +222,7 @@ class CommandsTestCase(TestCase):
def test_check_feeds_new(self): def test_check_feeds_new(self):
"""test we ingest new post if id is different""" """test we ingest new post if id is different"""
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
self.assertEqual(models.Article.objects.count(), 0) self.assertEqual(models.Article.objects.count(), 0)
@ -189,7 +247,7 @@ class CommandsTestCase(TestCase):
def test_check_feeds_old_post(self): def test_check_feeds_old_post(self):
"""test parse a feed with a post older than a week""" """test parse a feed with a post older than a week"""
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
self.assertEqual(models.Article.objects.count(), 0) self.assertEqual(models.Article.objects.count(), 0)
@ -212,7 +270,7 @@ class CommandsTestCase(TestCase):
self.assertEqual(models.Tag.objects.count(), 0) self.assertEqual(models.Tag.objects.count(), 0)
with patch("feedparser.parse", return_value=self.feedparser_exclude): with patch("feedparser.parse", return_value=self.feedparser_exclude):
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
value = call_command("check_feeds", *args, **opts) value = call_command("check_feeds", *args, **opts)
@ -230,7 +288,7 @@ class CommandsTestCase(TestCase):
self.blog.save() self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser): with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
value = call_command("check_feeds", *args, **opts) value = call_command("check_feeds", *args, **opts)
@ -248,7 +306,7 @@ class CommandsTestCase(TestCase):
self.blog.save() self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser): with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
value = call_command("check_feeds", *args, **opts) value = call_command("check_feeds", *args, **opts)
@ -266,7 +324,7 @@ class CommandsTestCase(TestCase):
self.blog.save() self.blog.save()
with patch("feedparser.parse", return_value=self.feedparser): with patch("feedparser.parse", return_value=self.feedparser):
args = {"-q": True} args = {"-q": True, "-blogs": True}
opts = {} opts = {}
value = call_command("check_feeds", *args, **opts) value = call_command("check_feeds", *args, **opts)

2
blogs/tests/test_models.py
View file

@ -139,7 +139,7 @@ class NewsletterTestCase(TestCase):
"""set up test newsletter""" """set up test newsletter"""
self.news = models.Newsletter.objects.create( self.news = models.Newsletter.objects.create(
name="Awesome news", name="Awesome news",
author="Hugh", author_name="Hugh",
url="https://test.com", url="https://test.com",
category="ARC", category="ARC",
) )

10
blogs/tests/test_utilities.py
View file

@ -11,12 +11,12 @@ from blogs import models, utilities
class FeedParserFeedMock(object): class FeedParserFeedMock(object):
title = "" title = ""
author = "" author = ""
summary = "" subtitle = ""
def __init__(self, title, author, summary): def __init__(self, title, author, subtitle):
self.title = title self.title = title
self.author = author self.author = author
self.summary = summary self.subtitle = subtitle
class FeedParserMock(object): class FeedParserMock(object):
@ -48,11 +48,11 @@ class UtilityTests(TestCase):
feed = FeedParserFeedMock( feed = FeedParserFeedMock(
title="My amazing blog", title="My amazing blog",
author="Hugh Rundle", author="Hugh Rundle",
summary="A short summary of my blog", subtitle="A short summary of my blog",
) )
feed_partial = FeedParserFeedMock( feed_partial = FeedParserFeedMock(
title="My amazing blog", author=None, summary=None title="My amazing blog", author=None, subtitle=None
) )
self.feedparser = FeedParserMock(feed=feed) self.feedparser = FeedParserMock(feed=feed)

2
blogs/tests/test_views.py
View file

@ -202,7 +202,7 @@ class PublicTests(TestCase):
view = views.RegisterNewsletter.as_view() view = views.RegisterNewsletter.as_view()
form = forms.RegisterNewsletterForm() form = forms.RegisterNewsletterForm()
form.data["name"] = "My newsletter" form.data["name"] = "My newsletter"
form.data["author"] = "Bob Bobson" form.data["author_name"] = "Bob Bobson"
form.data["url"] = "https://www.example.com" form.data["url"] = "https://www.example.com"
form.data["category"] = "LIB" form.data["category"] = "LIB"

4
blogs/utilities.py
View file

@ -22,7 +22,9 @@ def get_feed_info(feed):
blog["feed"] = feed blog["feed"] = feed
blog["title"] = getattr(b.feed, "title", "") blog["title"] = getattr(b.feed, "title", "")
blog["author_name"] = getattr(b.feed, "author", None) blog["author_name"] = getattr(b.feed, "author", None)
blog["description"] = getattr(b.feed, "subtitle", None) # summary for a FEED is "subtitle" blog["description"] = getattr(
b.feed, "subtitle", None
) # summary for a FEED is "subtitle"
return blog return blog

4
blogs/views/__init__.py
View file

@ -8,7 +8,7 @@ from .public import (
Browse, Browse,
Contribute, Contribute,
Contact, Contact,
Conferences, Events,
ConfirmEmail, ConfirmEmail,
CallsForPapers, CallsForPapers,
Editions, Editions,
@ -27,4 +27,4 @@ from .public import (
UnsubscribeEmail, UnsubscribeEmail,
) )
from .feeds import ArticleFeed, EventFeed from .feeds import ArticleFeed, EditionFeed, EventFeed, CombinedFeed

228
blogs/views/feeds.py
View file

@ -1,10 +1,15 @@
"""rss feeds""" """rss feeds"""
from django.contrib.syndication.views import Feed from itertools import chain
from django.utils.translation import gettext_lazy as _ from operator import attrgetter
from blogs.models.blog import Article from django.conf import settings
from blogs.models.event import Event
from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed
from blogs import models
# pylint: disable=R6301 # pylint: disable=R6301
@ -12,13 +17,17 @@ from blogs.models.event import Event
class ArticleFeed(Feed): class ArticleFeed(Feed):
"""Combined RSS feed for all the articles""" """Combined RSS feed for all the articles"""
title = "Aus GLAMR Blogs" feed_type = Atom1Feed
link = "/feeds/blogs" link = "/blog-articles"
feed_url = "/blog-articles/feed"
feed_guid = f"https://{settings.DOMAIN}/blog-articles/feed"
title = "Aus GLAMR Blog articles"
description = "Posts from Australasian Galleries, Libraries, Archives, Museums, Records and associated blogs" description = "Posts from Australasian Galleries, Libraries, Archives, Museums, Records and associated blogs"
def items(self): def items(self):
"""each article in the feed""" """each article in the feed"""
return Article.objects.order_by("-pubdate")[:20] return models.Article.objects.order_by("-pubdate")[:20]
def item_title(self, item): def item_title(self, item):
"""article title""" """article title"""
@ -32,10 +41,18 @@ class ArticleFeed(Feed):
"""article author""" """article author"""
return item.author_name return item.author_name
def item_link(self, item):
"""item url"""
return item.url
def item_pubdate(self, item): def item_pubdate(self, item):
"""article publication date""" """article publication date"""
return item.pubdate return item.pubdate
def item_updateddate(self, item):
"""updated date"""
return item.updateddate
def item_categories(self, item): def item_categories(self, item):
"""article tags""" """article tags"""
categories = [] categories = []
@ -45,32 +62,203 @@ class ArticleFeed(Feed):
class EventFeed(Feed): class EventFeed(Feed):
"""Combined RSS feed for all the articles""" """Combined feed for all events and calls for papers"""
feed_type = Atom1Feed
link = "/events"
feed_url = "/events/feed"
feed_guid = f"https://{settings.DOMAIN}/events/feed"
title = "Aus GLAMR events" title = "Aus GLAMR events"
link = "/feeds/events"
description = "Australasian events for Galleries, Libraries, Archives, Museums, Records workers" description = "Australasian events for Galleries, Libraries, Archives, Museums, Records workers"
def items(self): def items(self):
"""event items for the feed""" """event and CFP items for the feed"""
return Event.objects.order_by("-start_date")[:20]
events = models.Event.objects.filter(approved=True)
cfps = models.CallForPapers.objects.all()
result_list = sorted(
chain(events, cfps),
key=attrgetter("pubdate"),
reverse=True,
)
return result_list[:20]
def item_title(self, item): def item_title(self, item):
"""event name""" """event or CFP name"""
date = item.start_date.strftime("%d %b %Y") return item.name
return f"{item.name} ({date})"
def item_description(self, item): def item_description(self, item):
"""event description""" """description or details"""
return item.description return (
item.description
if hasattr(item, "description")
else item.details
if hasattr(item, "details")
else None
)
def item_link(self, item):
"""item url"""
return item.url if hasattr(item, "url") else item.event.url
def item_pubdate(self, item): def item_pubdate(self, item):
"""date event was registered""" """date event/CFP was registered"""
return item.pub_date return item.pubdate
def item_categories(self, item): def item_categories(self, item):
"""event GLAMR category""" """event GLAMR category"""
return [_(item.category)] if hasattr(item, "category"):
return [models.Category(item.category).label]
# TODO: newsletter editions feed class EditionFeed(Feed):
"""Newsletter editions"""
feed_type = Atom1Feed
link = "/newsletter-editions"
feed_url = "/newsletter-editions/feed"
feed_guid = f"https://{settings.DOMAIN}/newsletter-editions/feed"
title = "Aus GLAMR Blog newsletter editions"
description = "Newsletters from Australasian Galleries, Libraries, Archives, Museums, Records and associated blogs"
def items(self):
"""each article in the feed"""
return models.Edition.objects.order_by("-pubdate")[:20]
def item_title(self, item):
"""article title"""
return item.title
def item_description(self, item):
"""article description"""
return item.description
def item_author_name(self, item):
"""article author"""
return item.author_name
def item_link(self, item):
"""item url"""
return item.url
def item_pubdate(self, item):
"""article publication date"""
return item.pubdate
def item_updateddate(self, item):
"""updated date"""
return item.updateddate
def item_categories(self, item):
"""newsletter category"""
return [models.Category(item.newsletter.category).label]
class CombinedFeed(Feed):
"""Combined Atom feed for everything"""
feed_type = Atom1Feed
link = "/"
feed_url = "/feed"
feed_guid = f"https://{settings.DOMAIN}/feed"
title = "Aus GLAMR"
description = "Latest news and opinion from Australasian Galleries, Libraries, Archives, Museums, Records professionals"
categories = [
"GLAM",
"GLAMR",
"Galleries",
"Libraries",
"Archives",
"Museums",
"Records",
]
feed_copyright = "Copyright is owned by individual authors"
ttl = 600
def items(self):
"""items for the feed"""
blog_objects = models.Blog.objects.filter(
approved=True, suspended=False, active=True
)
posts = models.Article.objects.all()
newsletters = models.Newsletter.objects.filter(approved=True, active=True)
editions = models.Edition.objects.all()
groups = models.Group.objects.filter(approved=True)
events = models.Event.objects.filter(approved=True)
cfps = models.CallForPapers.objects.all()
result_list = sorted(
chain(blog_objects, posts, newsletters, editions, groups, events, cfps),
key=attrgetter("pubdate"),
reverse=True,
)
return result_list[:30]
def item_title(self, item):
"""title or name"""
return item.name if hasattr(item, "name") else item.title
def item_description(self, item):
"""description"""
return (
item.description
if hasattr(item, "description")
else item.details
if hasattr(item, "details")
else None
)
def item_link(self, item):
"""item url"""
return item.url if hasattr(item, "url") else item.event.url
def item_guid(self, item):
"""guid"""
return item.url if hasattr(item, "url") else f"{item.event.url}-cfp-{item.id}"
def item_author_name(self, item):
"""author"""
return getattr(item, "author_name", None)
def item_pubdate(self, item):
"""date item was published"""
pubdate = getattr(item, "pubdate", None)
return pubdate
def item_updateddate(self, item):
"""updated date"""
if hasattr(item, "updateddate"):
return item.updateddate
if hasattr(item, "pubdate"):
return item.pubdate
return None
def item_categories(self, item):
"""GLAMR category or tags"""
if hasattr(item, "category"):
return [models.Category(item.category).label]
if hasattr(item, "tags"):
categories = []
for tag in item.tags.all():
categories.append(tag.name)
return categories
return None

28
blogs/views/public.py
View file

@ -51,9 +51,10 @@ class Blogs(View):
for blog in blogs: for blog in blogs:
blog.category_name = models.Category(blog.category).label blog.category_name = models.Category(blog.category).label
data = {"title": "Blogs and websites", "blogs": blogs} data = {"title": "Blogs and websites", "blogs": blogs, "category": category}
return render(request, "browse/blogs.html", data) return render(request, "browse/blogs.html", data)
class Articles(View): class Articles(View):
"""Blog articles""" """Blog articles"""
@ -65,8 +66,7 @@ class Articles(View):
return render(request, "browse/articles.html", data) return render(request, "browse/articles.html", data)
class Events(View):
class Conferences(View):
"""browse the list of conferences""" """browse the list of conferences"""
def get(self, request, category=None): def get(self, request, category=None):
@ -90,7 +90,7 @@ class Conferences(View):
.last() .last()
) )
data = {"title": "Upcoming events", "cons": cons} data = {"title": "Upcoming events", "cons": cons, "category": category}
return render(request, "browse/events.html", data) return render(request, "browse/events.html", data)
@ -124,19 +124,29 @@ class Groups(View):
for group in groups: for group in groups:
group.category_name = models.Category(group.category).label group.category_name = models.Category(group.category).label
group.reg_type = models.utils.GroupType(group.type).label group.reg_type = models.utils.GroupType(group.type).label
data = {"title": "Groups and discussion lists", "groups": groups} data = {
"title": "Groups and discussion lists",
"groups": groups,
"category": category,
}
return render(request, "browse/groups.html", data) return render(request, "browse/groups.html", data)
class Newsletters(View): class Newsletters(View):
"""browse the list of groups""" """browse the list of groups"""
def get(self, request): def get(self, request, category=None):
"""here they are""" """here they are"""
news = models.Newsletter.objects.filter(approved=True).order_by("name") if category:
news = models.Newsletter.objects.filter(
approved=True, category=category
).order_by("name")
else:
news = models.Newsletter.objects.filter(approved=True).order_by("name")
for letter in news: for letter in news:
letter.category_name = models.Category(letter.category).label letter.category_name = models.Category(letter.category).label
data = {"title": "Newsletters", "news": news} data = {"title": "Newsletters", "news": news, "category": category}
return render(request, "browse/newsletters.html", data) return render(request, "browse/newsletters.html", data)
@ -423,7 +433,7 @@ class Search(View):
class Browse(View): class Browse(View):
"""browse by clicking on a tag""" """browse blog articles by clicking on a tag"""
def get(self, request): def get(self, request):
"""display browse results""" """display browse results"""

2
docker-compose.yml
View file

@ -11,7 +11,7 @@ services:
web: web:
build: . build: .
env_file: .env env_file: .env
command: python manage.py runserver 0.0.0.0:8000 command: python manage.py runserver 0.0.0.0:8000 # only use in dev
# command: gunicorn --env DJANGO_SETTINGS_MODULE=ausglamr.settings ausglamr.wsgi --workers=10 --threads=4 -b 0.0.0.0:8000 # command: gunicorn --env DJANGO_SETTINGS_MODULE=ausglamr.settings ausglamr.wsgi --workers=10 --threads=4 -b 0.0.0.0:8000
volumes: volumes:
- .:/app - .:/app

8
glamr-dev
View file

@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Thanks Mouse Reeve for letting me steal your idea. # Thanks Mouse Reeve for letting me steal their idea.
# exit on errors # exit on errors
set -e set -e
@ -31,9 +31,9 @@ case "$CMD" in
runweb python manage.py announce runweb python manage.py announce
;; ;;
backup) backup)
/snap/bin/docker exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump ${DOCKER_PATH} exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump
/snap/bin/docker cp ausglamr_db_1:/tmp/ausglamr_backup.dump /home/hugh/backups/ ${DOCKER_PATH} cp ausglamr_db_1:/tmp/ausglamr_backup.dump ${BACKUPS_DIR}/
mv /home/hugh/backups/ausglamr_backup.dump /home/hugh/backups/ausglamr_backup_$(date +'%a').dump mv ${BACKUPS_DIR}/ausglamr_backup.dump ${BACKUPS_DIR}/ausglamr_backup_$(date +'%a').dump
;; ;;
black) black)
docker compose run --rm web black ausglamr blogs docker compose run --rm web black ausglamr blogs

14
robots.txt Normal file
View file

@ -0,0 +1,14 @@
User-agent: AdsBot-Google
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: CCBot
Disallow: /