diff --git a/ausglamr/settings.py b/ausglamr/settings.py index 20d6a44..9027ea8 100644 --- a/ausglamr/settings.py +++ b/ausglamr/settings.py @@ -131,6 +131,8 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ +# ManifestStaticFilesStorage seems to be largely useless + STORAGES = { "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage", @@ -139,7 +141,7 @@ STORAGES = { MEDIA_URL = "media/" STATIC_URL = "static/" -STATIC_ROOT = os.path.join(BASE_DIR, "static/") +STATIC_ROOT = os.path.join(BASE_DIR, "static") STATICFILES_DIRS = [] # Default primary key field type diff --git a/blogs/templates/layout.html b/blogs/templates/layout.html index 7388242..1a8df98 100644 --- a/blogs/templates/layout.html +++ b/blogs/templates/layout.html @@ -60,6 +60,6 @@ - + \ No newline at end of file diff --git a/blogs/tests/test_commands.py b/blogs/tests/test_commands.py index 775d2fd..eb55417 100644 --- a/blogs/tests/test_commands.py +++ b/blogs/tests/test_commands.py @@ -1,6 +1,6 @@ """test utility functions""" -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from unittest.mock import patch @@ -66,14 +66,17 @@ class CommandsTestCase(TestCase): tag_two = FeedParserTagMock(term="python") tag_three = FeedParserTagMock(term="notglam") + updated_parsed = (datetime.now(timezone.utc) - timedelta(hours=3)).timetuple() + published_parsed = (datetime.now(timezone.utc) - timedelta(hours=3)).timetuple() + article = FeedParserItemMock( title="My amazing blog post", tags=[tag_one, tag_two], author="Hugh Rundle", link="https://test.com/1", summary="A short summary of my post", - updated_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0), - published_parsed=(2024, 1, 1, 19, 48, 21, 3, 1, 0), + updated_parsed=updated_parsed, + published_parsed=published_parsed, id="1", ) @@ -83,14 +86,37 @@ class CommandsTestCase(TestCase): author="Hugh Rundle", link="https://test.com/2", summary="A short summary of my next post", - updated_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0), - published_parsed=(2024, 1, 2, 19, 48, 21, 3, 1, 0), + updated_parsed=updated_parsed, + published_parsed=published_parsed, id="999", ) - self.feedparser = FeedParserMock(entries=[article]) + article_three = FeedParserItemMock( + title="My really old blog post", + tags=[tag_one, tag_two], + author="Hugh Rundle", + link="https://test.com/2", + summary="A short summary of my next post", + updated_parsed=(2023, 1, 3, 19, 48, 21, 3, 1, 0), + published_parsed=(2023, 1, 3, 19, 48, 21, 3, 1, 0), + id="999", + ) + article_four = FeedParserItemMock( + title="My new nice post", + tags=[tag_one, tag_two], + author="Hugh Rundle", + link="https://test.com/3", + summary="A short summary of my next post", + updated_parsed=updated_parsed, + published_parsed=published_parsed, + id="333", + ) + + self.feedparser = FeedParserMock(entries=[article]) + self.feedparser_old = FeedParserMock(entries=[article_three]) self.feedparser_exclude = FeedParserMock(entries=[article_two]) + self.feedparser_new = FeedParserMock(entries=[article_four]) def test_check_feeds(self): """test parse a feed for basic blog info""" @@ -109,6 +135,77 @@ class CommandsTestCase(TestCase): article = models.Article.objects.all().first() self.assertEqual(article.title, "My amazing blog post") + # should be announced + self.assertEqual(models.Announcement.objects.count(), 1) + + def test_check_feeds_duplicate(self): + """test we do not ingest the same post twice""" + + args = {"-q": True} + opts = {} + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser): + call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) + article = models.Article.objects.all().first() + self.assertEqual(article.title, "My amazing blog post") + + with patch("feedparser.parse", return_value=self.feedparser): + call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) + + def test_check_feeds_new(self): + """test we ingest new post if id is different""" + + args = {"-q": True} + opts = {} + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser): + call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) + article = models.Article.objects.all().first() + self.assertEqual(article.title, "My amazing blog post") + + with patch("feedparser.parse", return_value=self.feedparser_new): + call_command("check_feeds", *args, **opts) + + self.assertEqual(models.Article.objects.count(), 2) + self.assertEqual(models.Tag.objects.count(), 2) + article = models.Article.objects.all().last() + self.assertEqual(article.title, "My new nice post") + + def test_check_feeds_old_post(self): + """test parse a feed with a post older than a week""" + + args = {"-q": True} + opts = {} + + self.assertEqual(models.Article.objects.count(), 0) + self.assertEqual(models.Tag.objects.count(), 0) + + with patch("feedparser.parse", return_value=self.feedparser_old): + value = call_command("check_feeds", *args, **opts) + + # should be ingested + self.assertEqual(models.Article.objects.count(), 1) + self.assertEqual(models.Tag.objects.count(), 2) + + # should not be announced + self.assertEqual(models.Announcement.objects.count(), 0) + + def test_check_feeds_exclude_tag(self): """test parse a feed with exclude tag""" @@ -185,9 +282,7 @@ class CommandsTestCase(TestCase): self.assertEqual(models.Tag.objects.count(), 0) self.blog.suspended = False - self.blog.suspension_lifted = datetime( - 2024, 1, 2, 21, 0, 0, 0, tzinfo=timezone.utc - ) + self.blog.suspension_lifted = datetime.now(timezone.utc) - timedelta(minutes=1) self.blog.save() with patch("feedparser.parse", return_value=self.feedparser): diff --git a/blogs/views/public.py b/blogs/views/public.py index 323fb67..c1b5e1b 100644 --- a/blogs/views/public.py +++ b/blogs/views/public.py @@ -308,7 +308,7 @@ class Search(View): ) events = ( - models.Event.objects.annotate(rank=SearchRank(conference_vector, query)) + models.Event.objects.filter(approved=True).annotate(rank=SearchRank(conference_vector, query)) .filter(rank__gte=0.1) .order_by("-rank") ) @@ -318,7 +318,7 @@ class Search(View): ) cfps = ( - models.CallForPapers.objects.annotate(rank=SearchRank(cfp_vector, query)) + models.CallForPapers.objects.filter(approved=True).annotate(rank=SearchRank(cfp_vector, query)) .filter(rank__gte=0.1) .order_by("-rank") ) @@ -328,7 +328,7 @@ class Search(View): ) newsletters = ( - models.Newsletter.objects.annotate(rank=SearchRank(news_vector, query)) + models.Newsletter.objects.filter(approved=True).annotate(rank=SearchRank(news_vector, query)) .filter(rank__gte=0.1) .order_by("-rank") ) @@ -338,7 +338,7 @@ class Search(View): ) groups = ( - models.Event.objects.annotate(rank=SearchRank(group_vector, query)) + models.Event.objects.filter(approved=True).annotate(rank=SearchRank(group_vector, query)) .filter(rank__gte=0.1) .order_by("-rank") ) diff --git a/docker-compose.yml b/docker-compose.yml index 989ee4c..425c772 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ services: web: build: . env_file: .env - command: python manage.py runserver 0.0.0.0:8000 + # command: python manage.py runserver 0.0.0.0:8000 + command: gunicorn --env DJANGO_SETTINGS_MODULE=ausglamr.settings ausglamr.wsgi --workers=10 --threads=4 -b 0.0.0.0:8000 volumes: - .:/app depends_on: @@ -19,7 +20,7 @@ services: networks: - main ports: - - "8000:8000" + - "8282:8000" volumes: pgdata: networks: diff --git a/requirements.txt b/requirements.txt index 601769d..6028417 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4==4.12.2 +gunicorn==21.2.0 Django==4.2.7 environs==9.5.0 feedparser==6.0.10