Fixed API handler for grants contribution statistics, to use GrantContributionIndex + updating the command compute_grant_contribution_index (#10760)

Author: nutrina

app/grants/management/commands/compute_grant_contribution_index.py

@@ -1,23 +1,32 @@
 import datetime as dt
-from datetime import timezone
+from datetime import datetime, timezone
 from time import sleep
 
 from django.core.management.base import BaseCommand
+from django.db.models import F
 
 import requests
-from grants.models import Contribution, GrantContributionIndex
+from grants.models import Contribution, Grant, GrantContributionIndex
 
 
 class Command(BaseCommand):
 
     help = "rebuilds the table GrantContributionIndex"
 
     def handle(self, *args, **kwargs):
+        self.stdout.write(f"{datetime.now()} Building query for contributions ...")
         contributions = (
             Contribution.objects.filter(
                 success=True,
                 subscription__network="mainnet",
                 subscription__grant__clr_calculations__latest=True,
+                subscription__contributor_profile__isnull=False,
+                created_on__gt=F(
+                    "subscription__grant__clr_calculations__grantclr__start_date"
+                ),
+                created_on__lt=F(
+                    "subscription__grant__clr_calculations__grantclr__end_date"
+                ),
             )
             .order_by(
                 "subscription__contributor_profile__id",
@@ -36,13 +45,49 @@ def handle(self, *args, **kwargs):
             )
         )
 
+        self.stdout.write(f"{datetime.now()} Building contribIndexList ...")
         contribIndexList = [
             GrantContributionIndex(
                 profile_id=contribInfo[0],
-                grant_id=contribInfo[1],
-                round_num=contribInfo[2],
+                round_num=contribInfo[1],
+                grant_id=contribInfo[2],
             )
             for contribInfo in contributions
         ]
 
-        GrantContributionIndex.objects.bulk_create(contribIndexList, batch_size=100)
+        count = len(contribIndexList)
+        self.stdout.write(f"{datetime.now()} Length of contribIndexList: {count}")
+        self.stdout.write(f"{datetime.now()} Clearing GrantContributionIndex ...")
+
+        first_id = 0
+        last_id = 0
+
+        try:
+            first_id = GrantContributionIndex.objects.all().order_by("id")[0].id
+            last_id = GrantContributionIndex.objects.all().order_by("-id")[0].id
+        except:
+            pass
+
+        self.stdout.write(
+            f"{datetime.now()} ... deleting {last_id - first_id + 1} records"
+        )
+
+        count_to_delete = last_id - first_id
+        count_deleted = 0
+        batch_size = 50000
+        for i in range(first_id, last_id, batch_size):
+            count_deleted += batch_size
+            self.stdout.write(
+                f"{datetime.now()} ... {(count_deleted / count_to_delete * 100):.2f}% deleting {count} records up to id {i + batch_size}"
+            )
+            GrantContributionIndex.objects.filter(id__lt=i + batch_size).delete()
+
+        self.stdout.write(f"{datetime.now()} Saving to GrantContributionIndex ...")
+        # GrantContributionIndex.objects.bulk_create(contribIndexList, batch_size=1000, ignore_conflicts=True)
+        for i in range(0, count, batch_size):
+            self.stdout.write(
+                f"{datetime.now()} {(i / count * 100):.2f}% Saving to GrantContributionIndex ..."
+            )
+            GrantContributionIndex.objects.bulk_create(
+                contribIndexList[i : i + batch_size], ignore_conflicts=True
+            )

app/grants/models/grant_contribution_index.py

@@ -7,8 +7,8 @@
 class GrantContributionIndex(SuperModel):
     """Stores the grants and round number to shich a user contributed to.
     The purpose of this table is to allow a a fast query. This will be used from 
-    the `contributor_statistics`API """
+    the `contributor_statistics` API """
 
     profile = models.ForeignKey('dashboard.Profile', help_text=_('Contributor'), on_delete=models.CASCADE, db_index=True)
-    grant = models.ForeignKey('Grant', help_text=_('The grant a user contributed to'), on_delete=models.CASCADE)
+    grant = models.ForeignKey('grants.Grant', help_text=_('The grant a user contributed to'), on_delete=models.CASCADE)
     round_num = models.IntegerField(help_text=_('The round number a user contributed to'))

app/grants/views_api_vc.py

@@ -18,13 +18,14 @@
 
 """
 import logging
+from datetime import datetime
 
 from django.conf import settings
 from django.db.models import Sum
 from django.http import JsonResponse
 from django.utils.translation import gettext_lazy as _
 
-from grants.models import Contribution, Grant
+from grants.models import Contribution, Grant, GrantContributionIndex
 from perftools.models import StaticJsonEnv
 from townsquare.models import SquelchProfile
 
@@ -75,46 +76,48 @@ def contributor_statistics(request):
 
     # Get number of grants the user contributed to
     num_grants_contribute_to = (
-        Contribution.objects.filter(profile_for_clr__handle=handle, success=True)
+        GrantContributionIndex.objects.filter(profile__handle=handle)
         .order_by("grant_id")
         .distinct("grant_id")
         .count()
     )
 
-    # Get the number of grants the user contributed to
+    # Get number of rounds the user contributed to
     num_rounds_contribute_to = (
+        GrantContributionIndex.objects.filter(profile__handle=handle)
+        .order_by("round_num")
+        .distinct("round_num")
+        .count()
+    )
+
+    # Get the total number of contributions
+    contribution_list = list(
         Contribution.objects.filter(
+            profile_for_clr__handle=handle,
             success=True,
-            subscription__contributor_profile__handle=handle,
             subscription__network="mainnet",
             subscription__grant__clr_calculations__latest=True,
         )
-        .order_by("subscription__grant__clr_calculations__grantclr__round_num")
-        .distinct("subscription__grant__clr_calculations__grantclr__round_num")
-        .count()
+        .order_by(
+            "id",
+        )
+        .distinct(
+            "id",
+        )
+        .values_list("id", "amount_per_period_usdt")
     )
 
-    total_contribution_amount = Contribution.objects.filter(
-        profile_for_clr__handle=handle, success=True
-    ).aggregate(Sum("amount_per_period_usdt"))["amount_per_period_usdt__sum"]
-    total_contribution_amount = (
-        total_contribution_amount if total_contribution_amount is not None else 0
-    )
+    total_contribution_amount = sum([i[1] for i in contribution_list])
 
     # GR14 contributor (and not squelched by FDD)
+    start = datetime.now()
     profile_squelch = SquelchProfile.objects.filter(
         profile__handle=handle, active=True
     ).values_list("profile_id", flat=True)
 
     num_gr14_contributions = (
-        Contribution.objects.filter(
-            success=True,
-            subscription__contributor_profile__handle=handle,
-            subscription__network="mainnet",
-            subscription__grant__clr_calculations__latest=True,
-            subscription__grant__clr_calculations__grantclr__round_num=14,
-        )
-        .exclude(subscription__contributor_profile_id__in=profile_squelch)
+        GrantContributionIndex.objects.filter(profile__handle=handle, round_num=14)
+        .exclude(profile_id__in=profile_squelch)
         .count()
     )