prepare for prod

ausglamr/settings.py

@@ -131,6 +131,8 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/4.2/howto/static-files/
 
+# ManifestStaticFilesStorage seems to be largely useless
+
 STORAGES = {
     "staticfiles": {
         "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
@@ -139,7 +141,7 @@ STORAGES = {
 
 MEDIA_URL = "media/"
 STATIC_URL = "static/"
-STATIC_ROOT = os.path.join(BASE_DIR, "static/")
+STATIC_ROOT = os.path.join(BASE_DIR, "static")
 STATICFILES_DIRS = []
 
 # Default primary key field type

blogs/templates/layout.html

@@ -60,6 +60,6 @@
             </section>
         </section>
     </footer>
-    <script src="/static/js/glamr.js"></script>
+    <script src="{% static 'js/glamr.js' %}" type="text/javascript"></script>
 </body>
 </html>

blogs/tests/test_commands.py

@@ -1,6 +1,6 @@
 """test utility functions"""
 
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 
 from unittest.mock import patch
 
@@ -66,14 +66,17 @@ class CommandsTestCase(TestCase):
         tag_two = FeedParserTagMock(term="python")
         tag_three = FeedParserTagMock(term="notglam")
 
+        updated_parsed = (datetime.now(timezone.utc) - timedelta(hours=3)).timetuple()
+        published_parsed = (datetime.now(timezone.utc) - timedelta(hours=3)).timetuple()
+
         article = FeedParserItemMock(
             title="My amazing blog post",
             tags=[tag_one, tag_two],
             author="Hugh Rundle",
             link="https://test.com/1",
             summary="A short summary of my post",
-            updated_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
+            updated_parsed=updated_parsed,
-            published_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0),
+            published_parsed=published_parsed,
             id="1",
         )
 
@@ -83,14 +86,37 @@ class CommandsTestCase(TestCase):
             author="Hugh Rundle",
             link="https://test.com/2",
             summary="A short summary of my next post",
-            updated_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
+            updated_parsed=updated_parsed,
-            published_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0),
+            published_parsed=published_parsed,
             id="999",
         )
 
-        self.feedparser = FeedParserMock(entries=[article])
+        article_three = FeedParserItemMock(
+            title="My really old blog post",
+            tags=[tag_one, tag_two],
+            author="Hugh Rundle",
+            link="https://test.com/2",
+            summary="A short summary of my next post",
+            updated_parsed=(2023, 1, 3, 19, 48, 21, 3, 1, 0),
+            published_parsed=(2023, 1, 3, 19, 48, 21, 3, 1, 0),
+            id="999",
+        )
 
+        article_four = FeedParserItemMock(
+            title="My new nice post",
+            tags=[tag_one, tag_two],
+            author="Hugh Rundle",
+            link="https://test.com/3",
+            summary="A short summary of my next post",
+            updated_parsed=updated_parsed,
+            published_parsed=published_parsed,
+            id="333",
+        )
+
+        self.feedparser = FeedParserMock(entries=[article])
+        self.feedparser_old = FeedParserMock(entries=[article_three])
         self.feedparser_exclude = FeedParserMock(entries=[article_two])
+        self.feedparser_new = FeedParserMock(entries=[article_four])
 
     def test_check_feeds(self):
         """test parse a feed for basic blog info"""
@@ -109,6 +135,77 @@ class CommandsTestCase(TestCase):
             article = models.Article.objects.all().first()
             self.assertEqual(article.title, "My amazing blog post")
 
+            # should be announced
+            self.assertEqual(models.Announcement.objects.count(), 1)
+
+    def test_check_feeds_duplicate(self):
+        """test we do not ingest the same post twice"""
+
+        args = {"-q": True}
+        opts = {}
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)
+            article = models.Article.objects.all().first()
+            self.assertEqual(article.title, "My amazing blog post")
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)
+
+    def test_check_feeds_new(self):
+        """test we ingest new post if id is different"""
+
+        args = {"-q": True}
+        opts = {}
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        with patch("feedparser.parse", return_value=self.feedparser):
+            call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)
+            article = models.Article.objects.all().first()
+            self.assertEqual(article.title, "My amazing blog post")
+
+        with patch("feedparser.parse", return_value=self.feedparser_new):
+            call_command("check_feeds", *args, **opts)
+
+            self.assertEqual(models.Article.objects.count(), 2)
+            self.assertEqual(models.Tag.objects.count(), 2)
+            article = models.Article.objects.all().last()
+            self.assertEqual(article.title, "My new nice post")
+
+    def test_check_feeds_old_post(self):
+        """test parse a feed with a post older than a week"""
+
+        args = {"-q": True}
+        opts = {}
+
+        self.assertEqual(models.Article.objects.count(), 0)
+        self.assertEqual(models.Tag.objects.count(), 0)
+
+        with patch("feedparser.parse", return_value=self.feedparser_old):
+            value = call_command("check_feeds", *args, **opts)
+
+            # should be ingested
+            self.assertEqual(models.Article.objects.count(), 1)
+            self.assertEqual(models.Tag.objects.count(), 2)
+
+            # should not be announced
+            self.assertEqual(models.Announcement.objects.count(), 0)
+
+
     def test_check_feeds_exclude_tag(self):
         """test parse a feed with exclude tag"""
 
@@ -185,9 +282,7 @@ class CommandsTestCase(TestCase):
         self.assertEqual(models.Tag.objects.count(), 0)
 
         self.blog.suspended = False
-        self.blog.suspension_lifted = datetime(
+        self.blog.suspension_lifted = datetime.now(timezone.utc) - timedelta(minutes=1)
-            2024, 1, 2, 21, 0, 0, 0, tzinfo=timezone.utc
-        )
         self.blog.save()
 
         with patch("feedparser.parse", return_value=self.feedparser):

blogs/views/public.py

@@ -308,7 +308,7 @@ class Search(View):
         )
 
         events = (
-            models.Event.objects.annotate(rank=SearchRank(conference_vector, query))
+            models.Event.objects.filter(approved=True).annotate(rank=SearchRank(conference_vector, query))
             .filter(rank__gte=0.1)
             .order_by("-rank")
         )
@@ -318,7 +318,7 @@ class Search(View):
         )
 
         cfps = (
-            models.CallForPapers.objects.annotate(rank=SearchRank(cfp_vector, query))
+            models.CallForPapers.objects.filter(approved=True).annotate(rank=SearchRank(cfp_vector, query))
             .filter(rank__gte=0.1)
             .order_by("-rank")
         )
@@ -328,7 +328,7 @@ class Search(View):
         )
 
         newsletters = (
-            models.Newsletter.objects.annotate(rank=SearchRank(news_vector, query))
+            models.Newsletter.objects.filter(approved=True).annotate(rank=SearchRank(news_vector, query))
             .filter(rank__gte=0.1)
             .order_by("-rank")
         )
@@ -338,7 +338,7 @@ class Search(View):
         )
 
         groups = (
-            models.Event.objects.annotate(rank=SearchRank(group_vector, query))
+            models.Event.objects.filter(approved=True).annotate(rank=SearchRank(group_vector, query))
             .filter(rank__gte=0.1)
             .order_by("-rank")
         )

docker-compose.yml

@@ -11,7 +11,8 @@ services:
   web:
     build: .
     env_file: .env
-    command: python manage.py runserver 0.0.0.0:8000
+    # command: python manage.py runserver 0.0.0.0:8000
+    command: gunicorn --env DJANGO_SETTINGS_MODULE=ausglamr.settings ausglamr.wsgi --workers=10 --threads=4 -b 0.0.0.0:8000
     volumes:
       - .:/app
     depends_on:
@@ -19,7 +20,7 @@ services:
     networks:
       - main
     ports:
-      - "8000:8000"
+      - "8282:8000"
 volumes:
   pgdata:
 networks:

requirements.txt

@@ -1,4 +1,5 @@
 beautifulsoup4==4.12.2
+gunicorn==21.2.0
 Django==4.2.7
 environs==9.5.0
 feedparser==6.0.10