Add CommunityAggregatedFields

django/conftest.py

@@ -22,6 +22,7 @@
 from thunderstore.community.consts import PackageListingReviewStatus
 from thunderstore.community.models import (
     Community,
+    CommunityAggregatedFields,
     CommunitySite,
     PackageCategory,
     PackageListing,
@@ -101,6 +102,7 @@ def prime_testing_database(django_db_setup, django_db_blocker):
         [x.delete() for x in Webhook.objects.all()]
         [x.delete() for x in CommunitySite.objects.all()]
         [x.delete() for x in Community.objects.all()]
+        [x.delete() for x in CommunityAggregatedFields.objects.all()]
         [x.delete() for x in Site.objects.all()]
 
 

django/thunderstore/api/cyberstorm/serializers/community.py

@@ -9,5 +9,11 @@ class CyberstormCommunitySerializer(serializers.Serializer):
     datetime_created = serializers.DateTimeField()
     background_image_url = serializers.CharField(required=False)
     icon_url = serializers.CharField(required=False)
-    total_download_count = serializers.IntegerField()
-    total_package_count = serializers.IntegerField()
+    total_download_count = serializers.SerializerMethodField()
+    total_package_count = serializers.SerializerMethodField()
+
+    def get_total_download_count(self, obj) -> int:
+        return obj.aggregated.download_count
+
+    def get_total_package_count(self, obj) -> int:
+        return obj.aggregated.package_count

django/thunderstore/api/cyberstorm/tests/test_community_detail.py

@@ -2,22 +2,29 @@
 from rest_framework.test import APIClient
 
 from thunderstore.community.factories import PackageListingFactory
-from thunderstore.community.models import CommunitySite
+from thunderstore.community.models import CommunityAggregatedFields, CommunitySite
 
 
 @pytest.mark.django_db
 def test_api_cyberstorm_community_detail_success(
-    client: APIClient, community_site: CommunitySite
+    client: APIClient,
+    community_site: CommunitySite,
 ):
     PackageListingFactory(
-        community_=community_site.community, package_version_kwargs={"downloads": 0}
+        community_=community_site.community,
+        package_version_kwargs={"downloads": 0},
     )
     PackageListingFactory(
-        community_=community_site.community, package_version_kwargs={"downloads": 23}
+        community_=community_site.community,
+        package_version_kwargs={"downloads": 23},
     )
     PackageListingFactory(
-        community_=community_site.community, package_version_kwargs={"downloads": 42}
+        community_=community_site.community,
+        package_version_kwargs={"downloads": 42},
     )
+    CommunityAggregatedFields.create_missing()
+    community_site.community.refresh_from_db()
+    CommunityAggregatedFields.update_for_community(community_site.community)
 
     response = client.get(
         f"/api/cyberstorm/community/{community_site.community.identifier}/",
@@ -30,19 +37,20 @@ def test_api_cyberstorm_community_detail_success(
 
     assert c.name == response_data["name"]
     assert c.identifier == response_data["identifier"]
-    assert c.total_download_count == response_data["total_download_count"]
-    assert c.total_package_count == response_data["total_package_count"]
+    assert c.aggregated.download_count == response_data["total_download_count"]
+    assert c.aggregated.package_count == response_data["total_package_count"]
     assert c.background_image_url == response_data["background_image_url"]
     assert c.description == response_data["description"]
     assert c.discord_url == response_data["discord_url"]
 
 
 @pytest.mark.django_db
 def test_api_cyberstorm_community_detail_failure(
-    client: APIClient, community_site: CommunitySite
+    client: APIClient,
+    community_site: CommunitySite,
 ):
     response = client.get(
-        f"/api/cyberstorm/community/bad/",
+        "/api/cyberstorm/community/bad/",
         HTTP_HOST=community_site.site.domain,
     )
     assert response.status_code == 404

django/thunderstore/api/cyberstorm/tests/test_community_list.py

@@ -8,26 +8,35 @@
 from thunderstore.api.cyberstorm.views.community_list import CommunityPaginator
 from thunderstore.community.consts import PackageListingReviewStatus
 from thunderstore.community.factories import CommunityFactory, PackageListingFactory
-from thunderstore.community.models.community import Community
+from thunderstore.community.models.community import Community, CommunityAggregatedFields
 from thunderstore.community.models.community_site import CommunitySite
 
 
 @pytest.mark.django_db
 def test_api_cyberstorm_community_list_get_success(
-    client: APIClient, community_site: CommunitySite, dummy_image
+    client: APIClient,
+    community_site: CommunitySite,
+    dummy_image,
 ):
+    community1 = CommunityFactory(
+        aggregated_fields=CommunityAggregatedFields.objects.create(),
+    )
+    community2 = CommunityFactory(
+        aggregated_fields=CommunityAggregatedFields.objects.create(),
+        background_image=dummy_image,
+    )
 
-    community1 = CommunityFactory()
-    community2 = CommunityFactory(background_image=dummy_image)
-    for x in range(10):
+    for _ in range(10):
         PackageListingFactory(
-            community_=community1, package_version_kwargs={"downloads": 10}
+            community_=community1,
+            package_version_kwargs={"downloads": 10},
         )
         PackageListingFactory(
-            community_=community2, package_version_kwargs={"downloads": 5}
+            community_=community2,
+            package_version_kwargs={"downloads": 5},
         )
 
-    for x in range(10):
+    for _ in range(10):
         PackageListingFactory(
             community_=community1,
             package_version_kwargs={"downloads": 3},
@@ -39,6 +48,9 @@ def test_api_cyberstorm_community_list_get_success(
             package_version_kwargs={"downloads": 5},
         )
 
+    CommunityAggregatedFields.update_for_community(community1)
+    CommunityAggregatedFields.update_for_community(community2)
+
     response = client.get(
         "/api/cyberstorm/community/",
         HTTP_HOST=community_site.site.domain,
@@ -50,8 +62,8 @@ def test_api_cyberstorm_community_list_get_success(
     for index, c in enumerate((community_site.community, community1, community2)):
         assert results[index]["name"] == c.name
         assert results[index]["identifier"] == c.identifier
-        assert results[index]["total_download_count"] == c.total_download_count
-        assert results[index]["total_package_count"] == c.total_package_count
+        assert results[index]["total_download_count"] == c.aggregated.download_count
+        assert results[index]["total_package_count"] == c.aggregated.package_count
         assert results[index]["background_image_url"] == c.background_image_url
         assert results[index]["description"] == c.description
         assert results[index]["discord_url"] == c.discord_url
@@ -74,12 +86,13 @@ def test_api_cyberstorm_community_list_only_listed_communities_are_returned(
     assert (
         len(data["results"]) == 2
     )  # There is the test community, that is created in api_client
-    assert not (non_listed.identifier in [c["identifier"] for c in data["results"]])
+    assert not (non_listed.identifier in (c["identifier"] for c in data["results"]))
 
 
 @pytest.mark.django_db
 def test_api_cyberstorm_community_list_are_ordered_by_identifier_by_default(
-    api_client: APIClient, community
+    api_client: APIClient,
+    community,
 ) -> None:
     a = CommunityFactory(identifier="a")
     b = CommunityFactory(identifier="b")
@@ -96,7 +109,7 @@ def test_api_cyberstorm_community_list_pagination(
     api_client: APIClient,
 ) -> None:
 
-    for i in range(25):
+    for _ in range(25):
         CommunityFactory()
 
     total_count = Community.objects.count()
@@ -123,7 +136,8 @@ def test_api_cyberstorm_community_list_pagination(
 
 
 def __assert_communities(
-    data: Dict, communities: Union[Community, List[Community]]
+    data: Dict,
+    communities: Union[Community, List[Community]],
 ) -> None:
     """
     Check that expected communities are found in results

django/thunderstore/community/context_processors.py

@@ -1,7 +1,6 @@
 from typing import Optional
 
 from django.conf import settings
-from django.db.models import Count
 from django.templatetags.static import static
 
 from thunderstore.community.models.community import Community
@@ -28,15 +27,15 @@ def get_community_context(community: Optional[Community]):
                     "site_icon": url,
                     "site_icon_width": community.icon_width,
                     "site_icon_height": community.icon_height,
-                }
+                },
             )
         else:
             result.update(
                 {
                     "site_icon": f"{settings.PROTOCOL}{settings.PRIMARY_HOST}{static('icon.png')}",
                     "site_icon_width": "1000",
                     "site_icon_height": "1000",
-                }
+                },
             )
         return result
     else:
@@ -61,7 +60,8 @@ def community_site(request):
 
 def selectable_communities(request):
     return {
-        "selectable_communities": Community.objects.listed()
-        .annotate(package_count=Count("package_listings"))
-        .order_by("-package_count", "datetime_created")
+        "selectable_communities": Community.objects.listed().order_by(
+            "-aggregated_fields__package_count",
+            "datetime_created",
+        ),
     }

django/thunderstore/community/migrations/0024_add_community_aggregated_fields.py

@@ -0,0 +1,47 @@
+# Generated by Django 3.1.7 on 2023-10-04 06:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import django_extrafields.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("community", "0023_add_decompilation_visibility_option"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CommunityAggregatedFields",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("datetime_created", models.DateTimeField(auto_now_add=True)),
+                ("datetime_updated", models.DateTimeField(auto_now=True)),
+                ("download_count", models.PositiveIntegerField(default=0)),
+                ("package_count", models.PositiveIntegerField(default=0)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="community",
+            name="aggregated_fields",
+            field=django_extrafields.models.SafeOneToOneOrField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="community.communityaggregatedfields",
+            ),
+        ),
+    ]

django/thunderstore/community/migrations/0025_init_aggregated_fields.py

@@ -0,0 +1,60 @@
+# Generated by Django 3.1.7 on 2023-10-18 09:19
+
+from django.db import migrations
+from django.db.models import Q, Sum
+
+from thunderstore.community.consts import PackageListingReviewStatus
+
+
+def create_missing(Community, CommunityAggregatedFields):
+    communities = Community.objects.filter(aggregated_fields=None).order_by("pk")
+
+    created_afs = CommunityAggregatedFields.objects.bulk_create(
+        [CommunityAggregatedFields() for _ in range(communities.count())],
+    )
+
+    for community, aggregated_fields in zip(communities, created_afs):
+        community.aggregated_fields = aggregated_fields
+
+    Community.objects.bulk_update(communities, ["aggregated_fields"])
+
+
+def update_for_community(community):
+    listings = community.package_listings.exclude(package__is_active=False).exclude(
+        ~Q(package__versions__is_active=True)
+    )
+
+    if community.require_package_listing_approval:
+        listings = listings.exclude(
+            ~Q(review_status=PackageListingReviewStatus.approved)
+        )
+
+    community.aggregated_fields.package_count = listings.count()
+    community.aggregated_fields.download_count = sum(
+        listing.package.versions.aggregate(downloads=Sum("downloads"))["downloads"]
+        for listing in listings
+    )
+    community.aggregated_fields.save()
+
+
+def forwards(apps, schema_editor):
+    Community = apps.get_model("community", "Community")
+    CommunityAggregatedFields = apps.get_model("community", "CommunityAggregatedFields")
+
+    # Create CommunityAggregatedFields for Communities that have none.
+    create_missing(Community, CommunityAggregatedFields)
+
+    # Set initial values for aggregated fields
+    for community in Community.objects.all():
+        update_for_community(community)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("community", "0024_add_community_aggregated_fields"),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards, migrations.RunPython.noop),
+    ]

django/thunderstore/community/migrations/0026_schedule_aggregated_fields_refresh.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.1.7 on 2023-10-18 09:40
+
+import pytz
+from django.db import migrations
+
+
+def forwards(apps, schema_editor):
+    CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule")
+    PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
+
+    schedule, _ = CrontabSchedule.objects.get_or_create(
+        minute="1",
+        hour="*",
+        day_of_week="*",
+        day_of_month="*",
+        month_of_year="*",
+        timezone=pytz.timezone("UTC"),
+    )
+    PeriodicTask.objects.get_or_create(
+        crontab=schedule,
+        name="Update CommunityAggregatedFields",
+        task="thunderstore.community.tasks.update_community_aggregated_fields",
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("community", "0025_init_aggregated_fields"),
+        ("django_celery_beat", "0014_remove_clockedschedule_enabled"),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards, migrations.RunPython.noop),
+    ]