From ff1d837eb64983e42d23be187c81fc13bfcbb1b1 Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Fri, 26 Jan 2024 11:00:26 +1100 Subject: [PATCH] another ridiculously large commit --- .env.example | 7 +- .gitignore | 7 +- README.md | 17 +- ausglamr/urls.py | 10 +- blogs/forms.py | 2 +- blogs/management/commands/check_feeds.py | 276 ++++++++++-------- .../management/commands/send_weekly_email.py | 34 ++- blogs/migrations/0001_initial.py | 32 +- blogs/models/blog.py | 2 +- blogs/models/event.py | 14 +- blogs/models/group.py | 8 +- blogs/models/newsletter.py | 32 +- blogs/static/css/custom.css | 13 +- blogs/static/favicon.svg | 15 + blogs/templates/browse/articles.html | 6 + blogs/templates/browse/blogs.html | 6 +- blogs/templates/browse/cfp.html | 8 +- blogs/templates/browse/editions.html | 8 +- blogs/templates/browse/events.html | 8 +- blogs/templates/browse/newsletters.html | 8 +- blogs/templates/contribute.html | 14 +- blogs/templates/help.html | 10 +- blogs/templates/index.html | 6 +- blogs/templates/layout.html | 1 + blogs/templates/subscribe.html | 15 +- blogs/tests/data/example.html | 11 + blogs/tests/data/good-example.html | 14 + blogs/tests/data/partial-example.html | 12 + blogs/tests/test_commands.py | 76 ++++- blogs/tests/test_models.py | 2 +- blogs/tests/test_utilities.py | 10 +- blogs/tests/test_views.py | 2 +- blogs/utilities.py | 4 +- blogs/views/__init__.py | 4 +- blogs/views/feeds.py | 228 +++++++++++++-- blogs/views/public.py | 28 +- docker-compose.yml | 2 +- glamr-dev | 8 +- robots.txt | 14 + 39 files changed, 708 insertions(+), 266 deletions(-) create mode 100644 blogs/static/favicon.svg create mode 100644 blogs/tests/data/example.html create mode 100644 blogs/tests/data/good-example.html create mode 100644 blogs/tests/data/partial-example.html create mode 100644 robots.txt diff --git a/.env.example b/.env.example index d5bec16..6d9b2f2 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,12 @@ POSTGRES_DB="ausglamr" PGPORT=5432 POSTGRES_HOST="db" -# mastodon +# mastodon bot MASTODON_ACCESS_TOKEN="" MASTODON_DOMAIN="https://example.com" + +# filepaths for backups cron - include a leading slash + +DOCKER_PATH="/usr/bin/docker" # example, check your path +BACKUPS_DIR="/home/my_user/backups" # where you want your backups to be stored \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9c2ea03..1b217cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ -data -fixtures \ No newline at end of file +/data +fixtures +/static +.env.dev +z_README_Hugh.md \ No newline at end of file diff --git a/README.md b/README.md index d508298..cc731b0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A django app running on Docker. Replaces _Aus GLAM Blogs_. * set up database backups (as cron jobs): -* `./glamr-dev backup` +* `./glamr-dev backup`: * `/snap/bin/docker exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump` * `/snap/bin/docker cp ausglamr_db_1:/tmp/ausglamr_backup.dump /home/hugh/backups/` @@ -20,10 +20,6 @@ A django app running on Docker. Replaces _Aus GLAM Blogs_. * set up cron jobs for management commands as below -## Migrating - -See `data/help.txt` - ## Admin Don't forget to add some Content Warnings for use by the Mastodon bot. @@ -33,6 +29,7 @@ Don't forget to add some Content Warnings for use by the Mastodon bot. Use `glamr-dev` to make your life easier (thanks to Mouse Reeve for the inspiration): * announce +* backup * check_feeds * manage [django management command] * makemigrations @@ -77,6 +74,8 @@ Run every 21 mins. This checks all blog feeds for any new posts, and adds them to the database as long as they don't have an exclusion tag and were not published during a time the blog was suspended. +Also checks newsletter articles if there is a feed. + Run every hour. ### queue_announcements @@ -89,4 +88,10 @@ Run daily. Does what you think. Creates a weekly email of the latest stuff, and send to everyone in Subscribers. -Run weekly. \ No newline at end of file +Run weekly. + +### Backups + +There is a `backup` command in `glamr-dev`. You can adjust the filepaths in your `.env` file. + +Run daily \ No newline at end of file diff --git a/ausglamr/urls.py b/ausglamr/urls.py index c41a90b..bec8f89 100644 --- a/ausglamr/urls.py +++ b/ausglamr/urls.py @@ -12,13 +12,17 @@ urlpatterns = [ re_path(r"^browse/?$", views.Browse.as_view(), name="browse"), re_path(r"^search/?$", views.Search.as_view(), name="search"), re_path(r"^help/?$", views.Help.as_view(), name="help"), + path("feed", views.CombinedFeed(), name="feed"), + path("blog-articles/feed", views.ArticleFeed(), name="article-feed"), + path("events/feed", views.EventFeed(), name="event-feed"), + path("newsletter-editions/feed", views.EditionFeed(), name="edition-feed"), path("contribute", views.Contribute.as_view(), name="contribute"), path("contact", views.Contact.as_view(), name="contact"), path("blogs", views.Blogs.as_view(), name="blogs"), path("blogs/", views.Blogs.as_view(), name="blog-category "), path("blog-articles", views.Articles.as_view(), name="blog-articles"), - path("events", views.Conferences.as_view(), name="events"), - path("events/", views.Conferences.as_view(), name="events-category"), + path("events", views.Events.as_view(), name="events"), + path("events/", views.Events.as_view(), name="events-category"), re_path(r"^cfps/?$", views.CallsForPapers.as_view(), name="cfps"), path("groups", views.Groups.as_view(), name="groups"), path("groups/", views.Groups.as_view(), name="group-category"), @@ -60,6 +64,4 @@ urlpatterns = [ views.UnsubscribeEmail.as_view(), name="unsubscribe-email", ), - path("feeds/blogs", views.ArticleFeed(), name="article-feed"), - path("feeds/events", views.EventFeed(), name="event-feed"), ] diff --git a/blogs/forms.py b/blogs/forms.py index 6f82496..3279350 100644 --- a/blogs/forms.py +++ b/blogs/forms.py @@ -115,7 +115,7 @@ class RegisterNewsletterForm(forms.ModelForm): model = Newsletter fields = [ "name", - "author", + "author_name", "category", "url", "feed", diff --git a/blogs/management/commands/check_feeds.py b/blogs/management/commands/check_feeds.py index 3686f19..2287563 100644 --- a/blogs/management/commands/check_feeds.py +++ b/blogs/management/commands/check_feeds.py @@ -47,6 +47,16 @@ class Command(BaseCommand): action="store_true", help="Suppress non-error messages", ) + parser.add_argument( + "-blogs", + action="store_true", + help="Only check blog posts", + ) + parser.add_argument( + "-newsletters", + action="store_true", + help="Only check editions", + ) def handle(self, *args, **options): """check feeds and update database""" @@ -56,89 +66,153 @@ class Command(BaseCommand): f"checking feeds at {django_timezone.localtime(django_timezone.now())}" ) - blogs = models.Blog.objects.filter( - approved=True, suspended=False, active=True - ).all() - for blog in blogs: - try: - data = feedparser.parse(blog.feed, agent=agent) + if not options["newsletters"]: + blogs = models.Blog.objects.filter( + approved=True, suspended=False, active=True + ).all() + for blog in blogs: + try: + data = feedparser.parse(blog.feed, agent=agent) - for article in data.entries: - if not models.Article.objects.filter( - Q(url=article.link) | Q(guid=getattr(article, "id", article.link)) - ).exists(): - if blog.suspension_lifted and ( - blog.suspension_lifted - > date_to_tz_aware(article.updated_parsed) - ): - continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension) - - taglist = getattr(article, "tags", None) or getattr( - article, "categories", [] - ) - - tags = [tag.term.lower() for tag in taglist] - - opt_out = False - # don't include posts with opt out tags - for tag in tags: - if ( - len( - {tag} - & { - "notglam", - "notglamr", - "notausglamblogs", - "notausglamr", - "notglamblogs", - "#notglam", - } - ) - > 0 + for article in data.entries: + if not models.Article.objects.filter( + Q(url=article.link) + | Q(guid=getattr(article, "id", article.link)) + ).exists(): + if blog.suspension_lifted and ( + blog.suspension_lifted + > date_to_tz_aware(article.updated_parsed) ): - opt_out = True - else: - continue + continue # don't ingest posts published prior to suspension being lifted (we should already have older ones from prior to suspension) - if not opt_out: - author_name = getattr(article, "author", None) or getattr( - blog, "author", None + taglist = getattr(article, "tags", None) or getattr( + article, "categories", [] + ) + + tags = [tag.term.lower() for tag in taglist] + + opt_out = False + # don't include posts with opt out tags + for tag in tags: + if ( + len( + {tag} + & { + "notglam", + "notglamr", + "notausglamblogs", + "notausglamr", + "notglamblogs", + "#notglam", + } + ) + > 0 + ): + opt_out = True + else: + continue + + if not opt_out: + author_name = getattr( + article, "author", None + ) or getattr(blog, "author", None) + + description = ( + html.strip_tags(article.summary) + if ( + hasattr(article, "summary") + and len(article.summary) > 0 + ) + else html.strip_tags(article.description) + if ( + hasattr(article, "description") + and len(article.summary) + ) + else html.strip_tags(article.content[0].value)[:200] + ) + description += "..." + + instance = models.Article.objects.create( + title=article.title, + author_name=author_name, + url=article.link, + description=description, + updateddate=date_to_tz_aware( + article.updated_parsed + ), + blog=blog, + pubdate=date_to_tz_aware(article.published_parsed), + guid=getattr(article, "id", article.link), + ) + + tags_to_add = get_tags( + getattr(article, "tags", None) + or getattr(article, "categories", []) + ) + + for tag in tags_to_add: + instance.tags.add(tag) + + instance.save() + + cutoff = django_timezone.now() - timedelta(days=3) + newish = instance.pubdate > cutoff + if newish: + instance.announce() + + blog.set_success( + updateddate=date_to_tz_aware(article.updated_parsed) + ) + + except Exception as e: + blog.set_failing() + logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}") + logging.info(article) + logging.error(e) + + if not options["blogs"]: + newsletters = models.Newsletter.objects.filter( + approved=True, active=True, feed__isnull=False + ).all() + for newsletter in newsletters: + try: + data = feedparser.parse(newsletter.feed, agent=agent) + + for edition in data.entries: + if not models.Edition.objects.filter( + Q(url=edition.link) + | Q(guid=getattr(edition, "id", edition.link)) + ).exists(): + author_name = getattr(edition, "author", None) or getattr( + edition, "author", None ) description = ( - html.strip_tags(article.summary) + html.strip_tags(edition.summary) if ( - hasattr(article, "summary") - and len(article.summary) > 0 + hasattr(edition, "summary") and len(edition.summary) ) - else html.strip_tags(article.description) + else html.strip_tags(edition.description) if ( - hasattr(article, "description") - and len(article.summary) + hasattr(edition, "description") + and len(edition.summary) ) - else html.strip_tags(article.content[0].value)[:200] + else html.strip_tags(edition.content[0].value)[:200] + + "..." ) description += "..." - instance = models.Article.objects.create( - title=article.title, + instance = models.Edition.objects.create( + title=edition.title, author_name=author_name, - url=article.link, + url=edition.link, description=description, - updateddate=date_to_tz_aware(article.updated_parsed), - blog=blog, - pubdate=date_to_tz_aware(article.published_parsed), - guid=getattr(article, "id", article.link), + updateddate=date_to_tz_aware(edition.updated_parsed), + newsletter=newsletter, + pubdate=date_to_tz_aware(edition.published_parsed), + guid=getattr(edition, "id", edition.link), ) - tags_to_add = get_tags( - getattr(article, "tags", None) - or getattr(article, "categories", []) - ) - - for tag in tags_to_add: - instance.tags.add(tag) - instance.save() cutoff = django_timezone.now() - timedelta(days=3) @@ -146,72 +220,16 @@ class Command(BaseCommand): if newish: instance.announce() - blog.set_success( - updateddate=date_to_tz_aware(article.updated_parsed) + newsletter.set_success( + updateddate=date_to_tz_aware(edition.updated_parsed) ) - except Exception as e: - blog.set_failing() - logging.error(f"ERROR WITH BLOG {blog.title} - {blog.url}") - logging.info(article) - logging.error(e) - - newsletters = models.Newsletter.objects.filter( - approved=True, active=True, feed__isnull=False - ).all() - for newsletter in newsletters: - try: - data = feedparser.parse(newsletter.feed, agent=agent) - - for edition in data.entries: - if not models.Edition.objects.filter( - Q(url=edition.link) | Q(guid=getattr(edition, "id", edition.link)) - ).exists(): - author_name = getattr(edition, "author", None) or getattr( - blog, "author", None - ) - - description = ( - html.strip_tags(edition.summary) - if ( - hasattr(edition, "summary") and len(edition.summary) - ) - else html.strip_tags(edition.description) - if ( - hasattr(edition, "description") and len(edition.summary) - ) - else html.strip_tags(edition.content[0].value)[:200] + "..." - ) - description += "..." - - instance = models.Edition.objects.create( - title=edition.title, - author_name=author_name, - url=edition.link, - description=description, - updateddate=date_to_tz_aware(edition.updated_parsed), - newsletter=newsletter, - pubdate=date_to_tz_aware(edition.published_parsed), - guid=getattr(edition, "id", edition.link), - ) - - instance.save() - - cutoff = django_timezone.now() - timedelta(days=3) - newish = instance.pubdate > cutoff - if newish: - instance.announce() - - newsletter.set_success( - updateddate=date_to_tz_aware(edition.updated_parsed) + except Exception as e: + newsletter.set_failing() + logging.error( + f"ERROR WITH NEWSLETTER {newsletter.name} - {newsletter.url}" ) - - except Exception as e: - newsletter.set_failing() - logging.error( - f"ERROR WITH NEWSLETTER {newsletter.name} - {newsletter.url}" - ) - logging.error(e) + logging.error(e) if not options["q"]: logging.info( diff --git a/blogs/management/commands/send_weekly_email.py b/blogs/management/commands/send_weekly_email.py index d96df78..d55c8a8 100644 --- a/blogs/management/commands/send_weekly_email.py +++ b/blogs/management/commands/send_weekly_email.py @@ -32,14 +32,15 @@ class Command(BaseCommand): approved=True, active=True, added__gte=cutoff ) articles = models.Article.objects.filter(pubdate__gte=cutoff) - events = models.Event.objects.filter(approved=True, pub_date__gte=cutoff) + events = models.Event.objects.filter(approved=True, pubdate__gte=cutoff) cfps = models.CallForPapers.objects.filter( event__approved=True, closing_date__gte=timezone.now().date() ) newsletters = models.Newsletter.objects.filter( - approved=True, pub_date__gte=cutoff + approved=True, pubdate__gte=cutoff ) - groups = models.Group.objects.filter(approved=True, pub_date__gte=cutoff) + editions = models.Edition.objects.filter(pubdate__gte=cutoff) + groups = models.Group.objects.filter(approved=True, pubdate__gte=cutoff) new_blogs = "" for blog in blogs: @@ -78,11 +79,33 @@ class Command(BaseCommand): if new_articles != "": new_articles = ( - "

New Articles

" + "

New Blog Posts

" + new_articles + "
" ) + new_editions = "" + for edition in editions: + title_string = f"

{edition.title}

" + author_string = ( + f"

{edition.author_name}

" if edition.author_name else "" + ) + description_string = ( + f"

{edition.description}

" + ) + + string_list = [title_string, author_string, description_string] + string = "".join(string_list) + + new_editions = new_editions + string + + if new_editions != "": + new_editions = ( + "

New Newsletter Editions

" + + new_editions + + "
" + ) + coming_events = "" for event in events: s_date = event.start_date @@ -195,6 +218,7 @@ class Command(BaseCommand): subject = f"{emoji} Fresh Aus GLAMR updates for the week of {dt.day} {dt:%B} {dt.year}" sections = [ new_articles, + new_editions, new_blogs, new_newsletters, new_groups, @@ -203,7 +227,7 @@ class Command(BaseCommand): ] body = "".join(sections) if body == "": - body = "

No new updates this week.

Why not spend the time you would have been reading, publishing your own blog post instead?

" + body = "

No new updates this week.

Why not spend some time publishing your own blog post instead?

" for subscriber in subscribers: opt_out = f"https://{settings.DOMAIN}/unsubscribe-email/{subscriber.token}/{subscriber.id}" diff --git a/blogs/migrations/0001_initial.py b/blogs/migrations/0001_initial.py index b08e1d3..5262c0b 100644 --- a/blogs/migrations/0001_initial.py +++ b/blogs/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-21 04:31 +# Generated by Django 4.2.7 on 2024-01-22 08:13 import blogs.models.blog from django.db import migrations, models @@ -50,6 +50,10 @@ class Migration(migrations.Migration): ("url", models.URLField(max_length=2000, unique=True)), ("description", models.TextField(blank=True, null=True)), ("updateddate", models.DateTimeField()), + ( + "pubdate", + models.DateTimeField(default=django.utils.timezone.now, null=True), + ), ("feed", models.URLField(max_length=2000)), ( "category", @@ -143,7 +147,7 @@ class Migration(migrations.Migration): "description", models.TextField(blank=True, max_length=250, null=True), ), - ("pub_date", models.DateTimeField()), + ("pubdate", models.DateTimeField()), ("start_date", models.DateField()), ( "announcements", @@ -219,7 +223,10 @@ class Migration(migrations.Migration): ), ("announced", models.BooleanField(default=False)), ("approved", models.BooleanField(default=False)), - ("pub_date", models.DateTimeField(default=None, null=True)), + ( + "pubdate", + models.DateTimeField(default=django.utils.timezone.now, null=True), + ), ], ), migrations.CreateModel( @@ -235,7 +242,7 @@ class Migration(migrations.Migration): ), ), ("name", models.CharField(max_length=100)), - ("author", models.CharField(max_length=100)), + ("author_name", models.CharField(max_length=100)), ( "category", models.CharField( @@ -275,7 +282,7 @@ class Migration(migrations.Migration): ("active", models.BooleanField(default=True, null=True)), ("failing", models.BooleanField(blank=True, default=False, null=True)), ("updateddate", models.DateTimeField()), - ("pub_date", models.DateTimeField(default=None, null=True)), + ("pubdate", models.DateTimeField(default=None, null=True)), ], ), migrations.CreateModel( @@ -350,7 +357,10 @@ class Migration(migrations.Migration): ("url", models.URLField(max_length=2000, unique=True)), ("description", models.TextField(blank=True, null=True)), ("updateddate", models.DateTimeField()), - ("pubdate", models.DateTimeField()), + ( + "pubdate", + models.DateTimeField(default=django.utils.timezone.now, null=True), + ), ("guid", models.CharField(max_length=2000)), ( "newsletter", @@ -376,7 +386,10 @@ class Migration(migrations.Migration): ), ("name", models.CharField(max_length=100)), ("details", models.TextField(blank=True, max_length=250, null=True)), - ("pub_date", models.DateTimeField(default=None, null=True)), + ( + "pubdate", + models.DateTimeField(default=django.utils.timezone.now, null=True), + ), ("opening_date", models.DateField()), ("closing_date", models.DateField()), ("announcements", models.IntegerField(default=0, null=True)), @@ -411,7 +424,10 @@ class Migration(migrations.Migration): ("url", models.URLField(max_length=2000, unique=True)), ("description", models.TextField(blank=True, null=True)), ("updateddate", models.DateTimeField()), - ("pubdate", models.DateTimeField()), + ( + "pubdate", + models.DateTimeField(default=django.utils.timezone.now, null=True), + ), ("guid", models.CharField(max_length=2000)), ( "blog", diff --git a/blogs/models/blog.py b/blogs/models/blog.py index 3100f21..e381cc1 100644 --- a/blogs/models/blog.py +++ b/blogs/models/blog.py @@ -29,6 +29,7 @@ class BlogData(models.Model): url = models.URLField(max_length=2000, unique=True) description = models.TextField(null=True, blank=True) updateddate = models.DateTimeField() + pubdate = models.DateTimeField(null=True, default=timezone.now) class Meta: """This is an abstract model for common data""" @@ -107,7 +108,6 @@ class Article(BlogData): """A blog post""" blog = models.ForeignKey(Blog, on_delete=models.CASCADE, related_name="articles") - pubdate = models.DateTimeField() guid = models.CharField(max_length=2000) tags = models.ManyToManyField("Tag", related_name="articles") diff --git a/blogs/models/event.py b/blogs/models/event.py index 411957a..e15024d 100644 --- a/blogs/models/event.py +++ b/blogs/models/event.py @@ -13,7 +13,7 @@ class Event(models.Model): category = models.CharField(choices=Category.choices, max_length=4) url = models.URLField(max_length=400, unique=True) description = models.TextField(null=True, blank=True, max_length=250) - pub_date = models.DateTimeField() # for RSS feed + pubdate = models.DateTimeField() # for RSS feed start_date = models.DateField() announcements = models.IntegerField(null=True, blank=True, default=0) activitypub_account_name = models.CharField(max_length=200, blank=True, null=True) @@ -21,8 +21,8 @@ class Event(models.Model): approved = models.BooleanField(default=False) def save(self, *args, **kwargs): - if not self.pub_date: - self.pub_date = timezone.now() + if not self.pubdate: + self.pubdate = timezone.now() super().save(*args, **kwargs) def __str__(self): @@ -58,7 +58,7 @@ class CallForPapers(models.Model): max_length=100 ) # "Call for papers", "call for participation" etc details = models.TextField(null=True, blank=True, max_length=250) - pub_date = models.DateTimeField(null=True, default=None) + pubdate = models.DateTimeField(null=True, default=timezone.now) opening_date = models.DateField() closing_date = models.DateField() announcements = models.IntegerField(null=True, default=0) @@ -77,9 +77,3 @@ class CallForPapers(models.Model): Announcement.objects.create(status=status) self.announcements = self.announcements + 1 super().save() - - def save(self, *args, **kwargs): - """save a CFP with pub_date""" - if not self.pub_date: - self.pub_date = timezone.now() - super().save(*args, **kwargs) diff --git a/blogs/models/group.py b/blogs/models/group.py index 1056de1..406fd27 100644 --- a/blogs/models/group.py +++ b/blogs/models/group.py @@ -16,9 +16,10 @@ class Group(models.Model): registration_url = models.URLField(max_length=400, unique=True) description = models.TextField(null=True, blank=True, max_length=250) contact_email = models.EmailField(blank=True, null=True) + announced = models.BooleanField(default=False) approved = models.BooleanField(default=False) - pub_date = models.DateTimeField(null=True, default=None) + pubdate = models.DateTimeField(null=True, default=timezone.now) def announce(self): """create a group announcement""" @@ -30,8 +31,3 @@ class Group(models.Model): Announcement.objects.create(status=status) self.announced = True super().save() - - def save(self, *args, **kwargs): - if not self.pub_date: - self.pub_date = timezone.now() - super().save(*args, **kwargs) diff --git a/blogs/models/newsletter.py b/blogs/models/newsletter.py index 47f68ec..3d42334 100644 --- a/blogs/models/newsletter.py +++ b/blogs/models/newsletter.py @@ -10,7 +10,7 @@ class Newsletter(models.Model): """a newsletter""" name = models.CharField(max_length=100) - author = models.CharField(max_length=100) + author_name = models.CharField(max_length=100) category = models.CharField(choices=Category.choices, max_length=4) url = models.URLField(max_length=400, unique=True) feed = models.URLField(max_length=1000, unique=True, blank=True, null=True) @@ -24,17 +24,21 @@ class Newsletter(models.Model): failing = models.BooleanField(default=False, blank=True, null=True) updateddate = models.DateTimeField() - pub_date = models.DateTimeField(null=True, default=None) + pubdate = models.DateTimeField(null=True, default=None) + + def __str__(self): + """display for admin dropdowns""" + return self.name def announce(self): """create a event announcement""" category = Category(self.category).label - name = self.name + name = self.author_name if self.activitypub_account_name: - name = f"{self.name} ({self.activitypub_account_name})" + name = f"{self.author_name} ({self.activitypub_account_name})" - status = f"{name} is a newsletter about {category} from {self.author}. Check it out:\n\n{self.url}" + status = f"{self.name} is a newsletter about {category} from {name}. Check it out:\n\n{self.url}" Announcement.objects.create(status=status) self.announced = True @@ -74,5 +78,21 @@ class Edition(models.Model): newsletter = models.ForeignKey( Newsletter, on_delete=models.CASCADE, related_name="editions" ) - pubdate = models.DateTimeField() + pubdate = models.DateTimeField(null=True, default=timezone.now) guid = models.CharField(max_length=2000) + + def announce(self): + """queue an edition announcement""" + + author = self.newsletter.activitypub_account_name or self.author_name + + if self.newsletter.activitypub_account_name: + author = f"{self.newsletter.activitypub_account_name} - " + elif self.author_name: + author = f"{self.author_name} - " + else: + author = "" + + status = f"📬 {self.title} ({author}{self.newsletter.name})\n\n{self.url}" + + Announcement.objects.create(status=status) diff --git a/blogs/static/css/custom.css b/blogs/static/css/custom.css index 0c0fbd3..fd6fc0f 100644 --- a/blogs/static/css/custom.css +++ b/blogs/static/css/custom.css @@ -199,7 +199,7 @@ footer .left p { .subscribe .pop, .contribute .pop { - min-height: 48rem; + min-height: 44rem; } .contribute { @@ -209,9 +209,6 @@ footer .left p { grid-template-columns: 33% 33% 33%; } -/* .contribute .pop { - min-height: 28rem; -} */ .pop { margin-top: 2em; @@ -228,12 +225,9 @@ footer .left p { grid-template-columns: 50% 50%; } - .subscribe .pop { - min-height: 34rem; - } - + .subscribe .pop, .contribute .pop { - min-height: 30rem; + min-height: 33rem; } } @@ -336,7 +330,6 @@ footer .left p { background-color: transparent; border-radius: 4px; border: 1px solid #aaa; - cursor: pointer; box-sizing: border-box; } diff --git a/blogs/static/favicon.svg b/blogs/static/favicon.svg new file mode 100644 index 0000000..73dead9 --- /dev/null +++ b/blogs/static/favicon.svg @@ -0,0 +1,15 @@ + + + + + + + + + A + + + A + + + diff --git a/blogs/templates/browse/articles.html b/blogs/templates/browse/articles.html index bc169db..948610a 100644 --- a/blogs/templates/browse/articles.html +++ b/blogs/templates/browse/articles.html @@ -1,5 +1,9 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block content %} {% for post in latest %}
@@ -25,5 +29,7 @@
{% endif %} + {% empty %} +

Oh no! There are no articles available. Try checking out some newsletter editions.

{% endfor %} {% endblock %} \ No newline at end of file diff --git a/blogs/templates/browse/blogs.html b/blogs/templates/browse/blogs.html index 29faa21..83fdc31 100644 --- a/blogs/templates/browse/blogs.html +++ b/blogs/templates/browse/blogs.html @@ -1,5 +1,9 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block content %}
@@ -23,7 +27,7 @@
{% empty %} -

Oh no! There are no blogs currently registered. Try checking out some newsletters.

+

Oh no! There are no blogs currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out some newsletters.

{% endfor %} {% endblock %} \ No newline at end of file diff --git a/blogs/templates/browse/cfp.html b/blogs/templates/browse/cfp.html index d73049b..85ace90 100644 --- a/blogs/templates/browse/cfp.html +++ b/blogs/templates/browse/cfp.html @@ -1,7 +1,13 @@ {% extends "layout.html" %} -{% block content %} +{% block feed %} + +{% endblock %} +{% block content %} +
+ +
Details Event diff --git a/blogs/templates/browse/editions.html b/blogs/templates/browse/editions.html index 220a6cd..54db3a6 100644 --- a/blogs/templates/browse/editions.html +++ b/blogs/templates/browse/editions.html @@ -1,5 +1,9 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block content %} {% for edition in latest %}
@@ -9,7 +13,7 @@ {% if edition.author_name %} {{ edition.author_name }} | {% endif %} - {{ edition.newsletter.title }} + {{ edition.newsletter.name }}

{% if edition.description %} @@ -20,5 +24,7 @@
{% endif %} + {% empty %} +

Oh no! There are no editions available. Try checking out some blog posts.

{% endfor %} {% endblock %} \ No newline at end of file diff --git a/blogs/templates/browse/events.html b/blogs/templates/browse/events.html index 382765b..4a1c38f 100644 --- a/blogs/templates/browse/events.html +++ b/blogs/templates/browse/events.html @@ -1,9 +1,13 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block content %}
- +
@@ -29,7 +33,7 @@
{% empty %} -

Oh no! There are no events currently registered. Try checking out some blogs.

+

Oh no! There are no events currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out some blogs.

{% endfor %} {% endblock %} \ No newline at end of file diff --git a/blogs/templates/browse/newsletters.html b/blogs/templates/browse/newsletters.html index ea9feef..6fe1f16 100644 --- a/blogs/templates/browse/newsletters.html +++ b/blogs/templates/browse/newsletters.html @@ -1,5 +1,9 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block content %}
@@ -19,11 +23,11 @@ {% include 'utils/rss-img.html' %} {% endif %} - {{pub.author}} + {{pub.author_name}} {{pub.category_name}}
{% empty %} -

Oh no! There are no newsletter currently registered. Try checking out some blogs.

+

Oh no! There are no newsletter currently registered{% if category %} under '{{category}}'{% endif %}. Try checking out some blogs.

{% endfor %} {% endblock %} \ No newline at end of file diff --git a/blogs/templates/contribute.html b/blogs/templates/contribute.html index 3080b36..56dd2d6 100644 --- a/blogs/templates/contribute.html +++ b/blogs/templates/contribute.html @@ -26,20 +26,18 @@ Register group -
-

Register an event

-

Conference, convention, seminar, workshop, talk, meet-up... -
Whatever you call it, you know what we mean.

- Register event -
-

Register a newsletter

Whether it uses Ghost, Write.as, Mailchimp or something else, if it's an email newsletter with some kind of GLAMR focus you can register it here.

Register newsletter
- +
+

Register an event

+

Conference, convention, seminar, workshop, talk, meet-up... +
Whatever you call it, you know what we mean.

+ Register event +

Register or update a Call for Papers

diff --git a/blogs/templates/help.html b/blogs/templates/help.html index 50dfc0d..8b3472b 100644 --- a/blogs/templates/help.html +++ b/blogs/templates/help.html @@ -30,7 +30,7 @@ {% with 'aus-glam-blogs' as anchor %} {% url 'help' as the_url %}

What happened to Aus GLAM Blogs?

🔗 -

Aus GLAM Blogs was focussed on blogs specifically. Aus GLAMR is the successor, now including more GLAMR content. All the blog articles from the original site have been migrated and will be inluded in your search results.

+

Aus GLAM Blogs was focussed on blogs specifically. Aus GLAMR is the successor, now including more GLAMR content. All the blog articles from the original site have been migrated and will be included in your search results.

{% endwith %} {% with 'pocket' as anchor %} @@ -44,6 +44,14 @@

Activitypub account name!!???

🔗

ActivityPub is the protocol used by Mastodon, BlueSky, BookWyrm, Threads, and other "fediverse" social media. If you include an account name, it will be mentioned when your registered thing is announced by the AusGLAMR Mastodon bot. If you'd like to join the "fediverse", try Aus GLAM Space.

{% endwith %} + + {% with 'aus' as anchor %} + {% url 'help' as the_url %} +

Why 'Aus' GLAMR?

🔗 +

The "Aus" is short for "Australasia". This is kind of a cheat - GLAMR content from Australia, Aotearoa/New Zealand, and our neighbours is welcome. I considered calling it "ANZ GLAMR" but I didn't want to do the Australian thing of pretending that Aotearoa/New Zealand is practically just another state of Australia with no cultural or institutional differences.

+

Opening it up to the whole world would likely result in the same thing that happens on every other English-language website: an assumed default of United States of America context, and a drowning out of anything else.

+

In short this aggregator is made by an Australian in Australia with that context in mind. Kiwis and other South-West Pacific neighbours are super welcome to contribute, but they also have their own context and may prefer their own thing. I can't speak for them.

+ {% endwith %}
{% endblock %} \ No newline at end of file diff --git a/blogs/templates/index.html b/blogs/templates/index.html index 716cb4a..e5b0b19 100644 --- a/blogs/templates/index.html +++ b/blogs/templates/index.html @@ -1,12 +1,16 @@ {% extends "layout.html" %} +{% block feed %} + +{% endblock %} + {% block intro %}{% endblock %} {% block content %}

Aus GLAMR is the place to find out what's happening in the Australasian cultural memory professions. Whether you work in galleries, libraries, archives, museums, or records, you'll find the latest blog posts, conferences and events, newsletters, and discussion groups here!

Browse one of the lists, search by keywords, or contribute by registering your blog, event, group or newsletter.

-

You can stay up to date by following the Mastodon bot or subscribing to email updates or to one of the RSS feeds - check out the subscribe page.

+

You can stay up to date by subscribing via the Mastodon bot, email updates or one of the RSS feeds - check out the subscribe page.

Please note that whilst off-topic or egregiously offensive content may be refused or removed, inclusion on this site does not imply endorsement by Hugh or newCardigan of the content of any particular listed publication or event.

{% endblock %} \ No newline at end of file diff --git a/blogs/templates/layout.html b/blogs/templates/layout.html index 6d6a0d4..0b9b8aa 100644 --- a/blogs/templates/layout.html +++ b/blogs/templates/layout.html @@ -11,6 +11,7 @@ + {% block feed %}{% endblock %}
diff --git a/blogs/templates/subscribe.html b/blogs/templates/subscribe.html index fdd4563..a58c47b 100644 --- a/blogs/templates/subscribe.html +++ b/blogs/templates/subscribe.html @@ -15,7 +15,7 @@

Get a weekly update listing the latest blog posts, open calls for papers, and upcoming events.

@@ -23,13 +23,13 @@

Subscribe via ActivityPub

-

Follow the @blogs@ausglam.space bot to see an announcement every time a new blog post , event, or call for papers is published.

+

Follow @ausglamr@ausglam.space to see an announcement every time something is added.

{% csrf_token %} {{ form.errors.username }} - +
@@ -37,11 +37,12 @@

Subscribe via RSS

-

Subscribe to the blogs feed in your favourite RSS reader to get a combined feed with every new article.

-

Or subscribe to the events feed to be the first to know about upcoming events.

+

Ok technically it's Atom. Subscribe to the feed to stay up to date using your favourite feed reader.

diff --git a/blogs/tests/data/example.html b/blogs/tests/data/example.html new file mode 100644 index 0000000..e40a32d --- /dev/null +++ b/blogs/tests/data/example.html @@ -0,0 +1,11 @@ + + + + + + My test website with no RSS feed + + +

Hello

+ + \ No newline at end of file diff --git a/blogs/tests/data/good-example.html b/blogs/tests/data/good-example.html new file mode 100644 index 0000000..ab4dae5 --- /dev/null +++ b/blogs/tests/data/good-example.html @@ -0,0 +1,14 @@ + + + + + + + My test website with an RSS feed + + + + +

Hello

+ + \ No newline at end of file diff --git a/blogs/tests/data/partial-example.html b/blogs/tests/data/partial-example.html new file mode 100644 index 0000000..6190266 --- /dev/null +++ b/blogs/tests/data/partial-example.html @@ -0,0 +1,12 @@ + + + + + + + My test website with an RSS feed + + +

Hello

+ + \ No newline at end of file diff --git a/blogs/tests/test_commands.py b/blogs/tests/test_commands.py index a2013df..017ff07 100644 --- a/blogs/tests/test_commands.py +++ b/blogs/tests/test_commands.py @@ -33,6 +33,27 @@ class FeedParserItemMock(object): self.id = id +class FeedParserEditionMock(object): + title = "" + author = "" + link = "" + summary = "" + updated_parsed = ((),) + published_parsed = ((),) + id = "" + + def __init__( + self, title, author, link, summary, updated_parsed, published_parsed, id + ): + self.title = title + self.author + self.link = link + self.summary = summary + self.updated_parsed = updated_parsed + self.published_parsed = published_parsed + self.id = id + + class FeedParserTagMock(object): term = "" @@ -62,6 +83,14 @@ class CommandsTestCase(TestCase): suspended=False, ) + models.Newsletter.objects.create( + name="My awesome newsletter", + url="https://test.news", + feed="https://test.news/feed", + category="ARC", + approved=True, + ) + tag_one = FeedParserTagMock(term="testing") tag_two = FeedParserTagMock(term="python") tag_three = FeedParserTagMock(term="notglam") @@ -113,15 +142,26 @@ class CommandsTestCase(TestCase): id="333", ) + edition = FeedParserEditionMock( + title="My amazing newsletter edition", + author="Hugh Rundle", + link="https://news.letter/1", + summary="A summary of my edition", + updated_parsed=updated_parsed, + published_parsed=published_parsed, + id="1", + ) + self.feedparser = FeedParserMock(entries=[article]) self.feedparser_old = FeedParserMock(entries=[article_three]) self.feedparser_exclude = FeedParserMock(entries=[article_two]) self.feedparser_new = FeedParserMock(entries=[article_four]) + self.feedparser_edition = FeedParserMock(entries=[edition]) def test_check_feeds(self): """test parse a feed for basic blog info""" - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} self.assertEqual(models.Article.objects.count(), 0) @@ -135,13 +175,31 @@ class CommandsTestCase(TestCase): article = models.Article.objects.all().first() self.assertEqual(article.title, "My amazing blog post") - # should be announced + # should be set to be announced + self.assertEqual(models.Announcement.objects.count(), 1) + + def test_check_edition_feeds(self): + """test parse a feed for newsletter edition info""" + + args = {"-q": True, "-newsletters": True} + opts = {} + + self.assertEqual(models.Edition.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser_edition): + value = call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Edition.objects.count(), 1) + edition = models.Edition.objects.all().first() + self.assertEqual(edition.title, "My amazing newsletter edition") + + # should be set to be announced self.assertEqual(models.Announcement.objects.count(), 1) def test_check_feeds_duplicate(self): """test we do not ingest the same post twice""" - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} self.assertEqual(models.Article.objects.count(), 0) @@ -164,7 +222,7 @@ class CommandsTestCase(TestCase): def test_check_feeds_new(self): """test we ingest new post if id is different""" - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} self.assertEqual(models.Article.objects.count(), 0) @@ -189,7 +247,7 @@ class CommandsTestCase(TestCase): def test_check_feeds_old_post(self): """test parse a feed with a post older than a week""" - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} self.assertEqual(models.Article.objects.count(), 0) @@ -212,7 +270,7 @@ class CommandsTestCase(TestCase): self.assertEqual(models.Tag.objects.count(), 0) with patch("feedparser.parse", return_value=self.feedparser_exclude): - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} value = call_command("check_feeds", *args, **opts) @@ -230,7 +288,7 @@ class CommandsTestCase(TestCase): self.blog.save() with patch("feedparser.parse", return_value=self.feedparser): - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} value = call_command("check_feeds", *args, **opts) @@ -248,7 +306,7 @@ class CommandsTestCase(TestCase): self.blog.save() with patch("feedparser.parse", return_value=self.feedparser): - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} value = call_command("check_feeds", *args, **opts) @@ -266,7 +324,7 @@ class CommandsTestCase(TestCase): self.blog.save() with patch("feedparser.parse", return_value=self.feedparser): - args = {"-q": True} + args = {"-q": True, "-blogs": True} opts = {} value = call_command("check_feeds", *args, **opts) diff --git a/blogs/tests/test_models.py b/blogs/tests/test_models.py index 5171685..5f15597 100644 --- a/blogs/tests/test_models.py +++ b/blogs/tests/test_models.py @@ -139,7 +139,7 @@ class NewsletterTestCase(TestCase): """set up test newsletter""" self.news = models.Newsletter.objects.create( name="Awesome news", - author="Hugh", + author_name="Hugh", url="https://test.com", category="ARC", ) diff --git a/blogs/tests/test_utilities.py b/blogs/tests/test_utilities.py index 9a36451..eb9b4ca 100644 --- a/blogs/tests/test_utilities.py +++ b/blogs/tests/test_utilities.py @@ -11,12 +11,12 @@ from blogs import models, utilities class FeedParserFeedMock(object): title = "" author = "" - summary = "" + subtitle = "" - def __init__(self, title, author, summary): + def __init__(self, title, author, subtitle): self.title = title self.author = author - self.summary = summary + self.subtitle = subtitle class FeedParserMock(object): @@ -48,11 +48,11 @@ class UtilityTests(TestCase): feed = FeedParserFeedMock( title="My amazing blog", author="Hugh Rundle", - summary="A short summary of my blog", + subtitle="A short summary of my blog", ) feed_partial = FeedParserFeedMock( - title="My amazing blog", author=None, summary=None + title="My amazing blog", author=None, subtitle=None ) self.feedparser = FeedParserMock(feed=feed) diff --git a/blogs/tests/test_views.py b/blogs/tests/test_views.py index 814d505..57c9059 100644 --- a/blogs/tests/test_views.py +++ b/blogs/tests/test_views.py @@ -202,7 +202,7 @@ class PublicTests(TestCase): view = views.RegisterNewsletter.as_view() form = forms.RegisterNewsletterForm() form.data["name"] = "My newsletter" - form.data["author"] = "Bob Bobson" + form.data["author_name"] = "Bob Bobson" form.data["url"] = "https://www.example.com" form.data["category"] = "LIB" diff --git a/blogs/utilities.py b/blogs/utilities.py index 5ccf81f..b5881b4 100644 --- a/blogs/utilities.py +++ b/blogs/utilities.py @@ -22,7 +22,9 @@ def get_feed_info(feed): blog["feed"] = feed blog["title"] = getattr(b.feed, "title", "") blog["author_name"] = getattr(b.feed, "author", None) - blog["description"] = getattr(b.feed, "subtitle", None) # summary for a FEED is "subtitle" + blog["description"] = getattr( + b.feed, "subtitle", None + ) # summary for a FEED is "subtitle" return blog diff --git a/blogs/views/__init__.py b/blogs/views/__init__.py index 7823dd4..e30db72 100644 --- a/blogs/views/__init__.py +++ b/blogs/views/__init__.py @@ -8,7 +8,7 @@ from .public import ( Browse, Contribute, Contact, - Conferences, + Events, ConfirmEmail, CallsForPapers, Editions, @@ -27,4 +27,4 @@ from .public import ( UnsubscribeEmail, ) -from .feeds import ArticleFeed, EventFeed +from .feeds import ArticleFeed, EditionFeed, EventFeed, CombinedFeed diff --git a/blogs/views/feeds.py b/blogs/views/feeds.py index 38ee387..3c50356 100644 --- a/blogs/views/feeds.py +++ b/blogs/views/feeds.py @@ -1,10 +1,15 @@ """rss feeds""" -from django.contrib.syndication.views import Feed -from django.utils.translation import gettext_lazy as _ +from itertools import chain +from operator import attrgetter -from blogs.models.blog import Article -from blogs.models.event import Event +from django.conf import settings + +from django.contrib.syndication.views import Feed +from django.utils.feedgenerator import Atom1Feed + + +from blogs import models # pylint: disable=R6301 @@ -12,13 +17,17 @@ from blogs.models.event import Event class ArticleFeed(Feed): """Combined RSS feed for all the articles""" - title = "Aus GLAMR Blogs" - link = "/feeds/blogs" + feed_type = Atom1Feed + link = "/blog-articles" + feed_url = "/blog-articles/feed" + feed_guid = f"https://{settings.DOMAIN}/blog-articles/feed" + + title = "Aus GLAMR Blog articles" description = "Posts from Australasian Galleries, Libraries, Archives, Museums, Records and associated blogs" def items(self): """each article in the feed""" - return Article.objects.order_by("-pubdate")[:20] + return models.Article.objects.order_by("-pubdate")[:20] def item_title(self, item): """article title""" @@ -32,10 +41,18 @@ class ArticleFeed(Feed): """article author""" return item.author_name + def item_link(self, item): + """item url""" + return item.url + def item_pubdate(self, item): """article publication date""" return item.pubdate + def item_updateddate(self, item): + """updated date""" + return item.updateddate + def item_categories(self, item): """article tags""" categories = [] @@ -45,32 +62,203 @@ class ArticleFeed(Feed): class EventFeed(Feed): - """Combined RSS feed for all the articles""" + """Combined feed for all events and calls for papers""" + + feed_type = Atom1Feed + link = "/events" + feed_url = "/events/feed" + feed_guid = f"https://{settings.DOMAIN}/events/feed" title = "Aus GLAMR events" - link = "/feeds/events" description = "Australasian events for Galleries, Libraries, Archives, Museums, Records workers" def items(self): - """event items for the feed""" - return Event.objects.order_by("-start_date")[:20] + """event and CFP items for the feed""" + + events = models.Event.objects.filter(approved=True) + cfps = models.CallForPapers.objects.all() + + result_list = sorted( + chain(events, cfps), + key=attrgetter("pubdate"), + reverse=True, + ) + return result_list[:20] def item_title(self, item): - """event name""" - date = item.start_date.strftime("%d %b %Y") - return f"{item.name} ({date})" + """event or CFP name""" + return item.name def item_description(self, item): - """event description""" - return item.description + """description or details""" + return ( + item.description + if hasattr(item, "description") + else item.details + if hasattr(item, "details") + else None + ) + + def item_link(self, item): + """item url""" + return item.url if hasattr(item, "url") else item.event.url def item_pubdate(self, item): - """date event was registered""" - return item.pub_date + """date event/CFP was registered""" + return item.pubdate def item_categories(self, item): """event GLAMR category""" - return [_(item.category)] + if hasattr(item, "category"): + return [models.Category(item.category).label] -# TODO: newsletter editions feed +class EditionFeed(Feed): + """Newsletter editions""" + + feed_type = Atom1Feed + link = "/newsletter-editions" + feed_url = "/newsletter-editions/feed" + feed_guid = f"https://{settings.DOMAIN}/newsletter-editions/feed" + + title = "Aus GLAMR Blog newsletter editions" + description = "Newsletters from Australasian Galleries, Libraries, Archives, Museums, Records and associated blogs" + + def items(self): + """each article in the feed""" + return models.Edition.objects.order_by("-pubdate")[:20] + + def item_title(self, item): + """article title""" + return item.title + + def item_description(self, item): + """article description""" + return item.description + + def item_author_name(self, item): + """article author""" + return item.author_name + + def item_link(self, item): + """item url""" + return item.url + + def item_pubdate(self, item): + """article publication date""" + return item.pubdate + + def item_updateddate(self, item): + """updated date""" + return item.updateddate + + def item_categories(self, item): + """newsletter category""" + return [models.Category(item.newsletter.category).label] + + +class CombinedFeed(Feed): + """Combined Atom feed for everything""" + + feed_type = Atom1Feed + link = "/" + feed_url = "/feed" + feed_guid = f"https://{settings.DOMAIN}/feed" + + title = "Aus GLAMR" + description = "Latest news and opinion from Australasian Galleries, Libraries, Archives, Museums, Records professionals" + categories = [ + "GLAM", + "GLAMR", + "Galleries", + "Libraries", + "Archives", + "Museums", + "Records", + ] + feed_copyright = "Copyright is owned by individual authors" + ttl = 600 + + def items(self): + """items for the feed""" + + blog_objects = models.Blog.objects.filter( + approved=True, suspended=False, active=True + ) + + posts = models.Article.objects.all() + + newsletters = models.Newsletter.objects.filter(approved=True, active=True) + + editions = models.Edition.objects.all() + + groups = models.Group.objects.filter(approved=True) + + events = models.Event.objects.filter(approved=True) + + cfps = models.CallForPapers.objects.all() + + result_list = sorted( + chain(blog_objects, posts, newsletters, editions, groups, events, cfps), + key=attrgetter("pubdate"), + reverse=True, + ) + + return result_list[:30] + + def item_title(self, item): + """title or name""" + return item.name if hasattr(item, "name") else item.title + + def item_description(self, item): + """description""" + return ( + item.description + if hasattr(item, "description") + else item.details + if hasattr(item, "details") + else None + ) + + def item_link(self, item): + """item url""" + return item.url if hasattr(item, "url") else item.event.url + + def item_guid(self, item): + """guid""" + + return item.url if hasattr(item, "url") else f"{item.event.url}-cfp-{item.id}" + + def item_author_name(self, item): + """author""" + return getattr(item, "author_name", None) + + def item_pubdate(self, item): + """date item was published""" + + pubdate = getattr(item, "pubdate", None) + return pubdate + + def item_updateddate(self, item): + """updated date""" + if hasattr(item, "updateddate"): + return item.updateddate + + if hasattr(item, "pubdate"): + return item.pubdate + + return None + + def item_categories(self, item): + """GLAMR category or tags""" + + if hasattr(item, "category"): + return [models.Category(item.category).label] + + if hasattr(item, "tags"): + categories = [] + for tag in item.tags.all(): + categories.append(tag.name) + return categories + + return None diff --git a/blogs/views/public.py b/blogs/views/public.py index 9c6905c..e4bccd8 100644 --- a/blogs/views/public.py +++ b/blogs/views/public.py @@ -51,9 +51,10 @@ class Blogs(View): for blog in blogs: blog.category_name = models.Category(blog.category).label - data = {"title": "Blogs and websites", "blogs": blogs} + data = {"title": "Blogs and websites", "blogs": blogs, "category": category} return render(request, "browse/blogs.html", data) + class Articles(View): """Blog articles""" @@ -65,8 +66,7 @@ class Articles(View): return render(request, "browse/articles.html", data) - -class Conferences(View): +class Events(View): """browse the list of conferences""" def get(self, request, category=None): @@ -90,7 +90,7 @@ class Conferences(View): .last() ) - data = {"title": "Upcoming events", "cons": cons} + data = {"title": "Upcoming events", "cons": cons, "category": category} return render(request, "browse/events.html", data) @@ -124,19 +124,29 @@ class Groups(View): for group in groups: group.category_name = models.Category(group.category).label group.reg_type = models.utils.GroupType(group.type).label - data = {"title": "Groups and discussion lists", "groups": groups} + data = { + "title": "Groups and discussion lists", + "groups": groups, + "category": category, + } return render(request, "browse/groups.html", data) class Newsletters(View): """browse the list of groups""" - def get(self, request): + def get(self, request, category=None): """here they are""" - news = models.Newsletter.objects.filter(approved=True).order_by("name") + if category: + news = models.Newsletter.objects.filter( + approved=True, category=category + ).order_by("name") + else: + news = models.Newsletter.objects.filter(approved=True).order_by("name") + for letter in news: letter.category_name = models.Category(letter.category).label - data = {"title": "Newsletters", "news": news} + data = {"title": "Newsletters", "news": news, "category": category} return render(request, "browse/newsletters.html", data) @@ -423,7 +433,7 @@ class Search(View): class Browse(View): - """browse by clicking on a tag""" + """browse blog articles by clicking on a tag""" def get(self, request): """display browse results""" diff --git a/docker-compose.yml b/docker-compose.yml index 2255066..40e84d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: web: build: . env_file: .env - command: python manage.py runserver 0.0.0.0:8000 + command: python manage.py runserver 0.0.0.0:8000 # only use in dev # command: gunicorn --env DJANGO_SETTINGS_MODULE=ausglamr.settings ausglamr.wsgi --workers=10 --threads=4 -b 0.0.0.0:8000 volumes: - .:/app diff --git a/glamr-dev b/glamr-dev index 7290dfe..fbdbd0e 100755 --- a/glamr-dev +++ b/glamr-dev @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Thanks Mouse Reeve for letting me steal your idea. +# Thanks Mouse Reeve for letting me steal their idea. # exit on errors set -e @@ -31,9 +31,9 @@ case "$CMD" in runweb python manage.py announce ;; backup) - /snap/bin/docker exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump - /snap/bin/docker cp ausglamr_db_1:/tmp/ausglamr_backup.dump /home/hugh/backups/ - mv /home/hugh/backups/ausglamr_backup.dump /home/hugh/backups/ausglamr_backup_$(date +'%a').dump + ${DOCKER_PATH} exec -u root ausglamr_db_1 pg_dump -v -Fc -U ausglamr -d "ausglamr" -f /tmp/ausglamr_backup.dump + ${DOCKER_PATH} cp ausglamr_db_1:/tmp/ausglamr_backup.dump ${BACKUPS_DIR}/ + mv ${BACKUPS_DIR}/ausglamr_backup.dump ${BACKUPS_DIR}/ausglamr_backup_$(date +'%a').dump ;; black) docker compose run --rm web black ausglamr blogs diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..34b89e6 --- /dev/null +++ b/robots.txt @@ -0,0 +1,14 @@ +User-agent: AdsBot-Google +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: CCBot +Disallow: / \ No newline at end of file