Skip to content

Commit ab7ffd5

Browse files
zlsghzlsgh
committed
Merge branch 'stable'
2 parents 05eb0cf + 4705b1d commit ab7ffd5

File tree

3 files changed

+153
-7
lines changed

3 files changed

+153
-7
lines changed

‎app/app/db.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ def db_for_read(self, model, **hints):
99
Reads go to a randomly-chosen replica if backend node
1010
Else go to default DB
1111
"""
12-
replicas = ['read_replica_1', 'read_replica_2', 'read_replica_3']
12+
replicas = ['read_replica_1', 'read_replica_2']
1313
return random.choice(replicas)
1414

1515
def db_for_write(self, model, **hints):
@@ -23,7 +23,7 @@ def allow_relation(self, obj1, obj2, **hints):
2323
Relations between objects are allowed if both objects are
2424
in the primary/replica pool.
2525
"""
26-
db_set = {'default', 'read_replica_1', 'read_replica_2', 'read_replica_3'}
26+
db_set = {'default', 'read_replica_1', 'read_replica_2'}
2727
if obj1._state.db in db_set and obj2._state.db in db_set:
2828
return True
2929
return True # TODO: be more stringent about this IFF we ever have a situation in which diff tables are on diff DBs

‎app/app/settings.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,6 @@
212212
'default': env.db(),
213213
'read_replica_1': env.db('READ_REPLICA_1_DATABASE_URL'),
214214
'read_replica_2': env.db('READ_REPLICA_2_DATABASE_URL'),
215-
'read_replica_3': env.db('READ_REPLICA_3_DATABASE_URL')
216215
}
217216
DATABASE_ROUTERS = ['app.db.PrimaryDBRouter']
218217

@@ -772,7 +771,7 @@ def callback(request):
772771
S3_REPORT_BUCKET = env('S3_REPORT_BUCKET', default='') # TODO
773772
S3_REPORT_PREFIX = env('S3_REPORT_PREFIX', default='') # TODO
774773

775-
S3_BSCI_SYBIL_BUCKET = env('S3_REPORT_BUCKET', default='') # TODO
774+
S3_BSCI_SYBIL_BUCKET = env('S3_BSCI_SYBIL_BUCKET', default='') # TODO
776775

777776
INSTALLED_APPS += env.list('DEBUG_APPS', default=[])
778777

‎app/grants/clr.py

Lines changed: 150 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,147 @@
2323
from django.utils import timezone
2424

2525
import numpy as np
26-
from grants.clr_data_src import fetch_grants, fetch_summed_contributions
26+
from grants.clr_data_src import fetch_contributions, fetch_grants
27+
28+
29+
def populate_data_for_clr(grants, contributions, clr_round):
30+
'''
31+
Populate Data needed to calculate CLR
32+
33+
Args:
34+
grants : grants list
35+
contributions : contributions list for those grants
36+
clr_round : GrantCLR
37+
38+
Returns:
39+
contrib_data_list: {
40+
'id': grant_id,
41+
'contributions': summed_contributions
42+
}
43+
44+
'''
45+
46+
contrib_data_list = []
47+
48+
if not clr_round:
49+
print('Error: populate_data_for_clr - missing clr_round')
50+
return contrib_data_list
51+
52+
clr_start_date = clr_round.start_date
53+
clr_end_date = clr_round.end_date
54+
55+
mechanism="profile"
56+
57+
# 3-4s to get all the contributions
58+
_contributions = list(contributions.filter(created_on__gte=clr_start_date, created_on__lte=clr_end_date).prefetch_related('profile_for_clr', 'subscription'))
59+
_contributions_by_id = {}
60+
for ele in _contributions:
61+
key = ele.normalized_data.get('id')
62+
if key not in _contributions_by_id.keys():
63+
_contributions_by_id[key] = []
64+
_contributions_by_id[key].append(ele)
65+
66+
# set up data to load contributions for each grant
67+
for grant in grants:
68+
grant_id = grant.defer_clr_to.pk if grant.defer_clr_to else grant.id
69+
70+
# contributions
71+
contribs = _contributions_by_id.get(grant.id, [])
72+
73+
# create arrays
74+
contributing_profile_ids = []
75+
contributions_by_id = {}
76+
for c in contribs:
77+
prof = c.profile_for_clr
78+
if prof:
79+
key = prof.id
80+
if key not in contributions_by_id.keys():
81+
contributions_by_id[key] = []
82+
contributions_by_id[key].append(c)
83+
contributing_profile_ids.append((prof.id, prof.trust_bonus))
84+
85+
contributing_profile_ids = list(set(contributing_profile_ids))
86+
87+
summed_contributions = []
88+
89+
# contributions
90+
if len(contributing_profile_ids) > 0:
91+
for profile_id, trust_bonus in contributing_profile_ids:
92+
sum_of_each_profiles_contributions = sum(ele.normalized_data.get('amount_per_period_usdt') for ele in contributions_by_id[profile_id]) * float(clr_round.contribution_multiplier)
93+
94+
summed_contributions.append({
95+
'id': str(profile_id),
96+
'sum_of_each_profiles_contributions': sum_of_each_profiles_contributions,
97+
'profile_trust_bonus': trust_bonus
98+
})
99+
100+
contrib_data_list.append({
101+
'id': grant_id,
102+
'contributions': summed_contributions
103+
})
104+
105+
return contrib_data_list
106+
107+
108+
def translate_data(grants_data):
109+
'''
110+
translates django grant data structure to a list of lists
111+
112+
args:
113+
django grant data structure
114+
{
115+
'id': (string) ,
116+
'contibutions' : [
117+
{
118+
contributor_profile (str) : summed_contributions
119+
}
120+
]
121+
}
122+
123+
returns:
124+
list of lists of grant data
125+
[[grant_id (str), user_id (str), contribution_amount (float)]]
126+
dictionary of profile_ids and trust scores
127+
{user_id (str): trust_score (float)}
128+
'''
129+
trust_dict = {}
130+
grants_list = []
131+
for g in grants_data:
132+
grant_id = g.get('id')
133+
for c in g.get('contributions'):
134+
profile_id = c.get('id')
135+
trust_bonus = c.get('profile_trust_bonus')
136+
if profile_id:
137+
val = [grant_id] + [c.get('id')] + [c.get('sum_of_each_profiles_contributions')]
138+
grants_list.append(val)
139+
trust_dict[profile_id] = trust_bonus
140+
141+
return grants_list, trust_dict
142+
143+
144+
def aggregate_contributions(grant_contributions):
145+
'''
146+
aggregates contributions by contributor, and calculates total contributions by unique pairs
147+
148+
args:
149+
list of lists of grant data
150+
[[grant_id (str), user_id (str), verification_status (str), trust_bonus (float), contribution_amount (float)]]
151+
152+
returns:
153+
aggregated contributions by pair nested dict
154+
{
155+
grant_id (str): {
156+
user_id (str): aggregated_amount (float)
157+
}
158+
}
159+
'''
160+
contrib_dict = {}
161+
for proj, user, amount in grant_contributions:
162+
if proj not in contrib_dict:
163+
contrib_dict[proj] = {}
164+
contrib_dict[proj][user] = contrib_dict[proj].get(user, 0) + amount
165+
166+
return contrib_dict
27167

28168

29169
def get_totals_by_pair(contrib_dict):
@@ -235,8 +375,15 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
235375
print(f"- starting fetch_grants at {round(time.time(),1)}")
236376
grants = fetch_grants(clr_round, network)
237377

238-
print(f"- starting get data and sum at {round(time.time(),1)}")
239-
curr_agg, trust_dict = fetch_summed_contributions(grants, clr_round, network)
378+
print(f"- starting fetch_contributions at {round(time.time(),1)}")
379+
contributions = fetch_contributions(clr_round, network)
380+
381+
print(f"- starting sum (of {contributions.count()} contributions) at {round(time.time(),1)}")
382+
grant_contributions_curr = populate_data_for_clr(grants, contributions, clr_round)
383+
curr_round, trust_dict = translate_data(grant_contributions_curr)
384+
385+
# this aggregates the data into the expected format
386+
curr_agg = aggregate_contributions(curr_round)
240387

241388
if len(curr_agg) == 0:
242389
print(f'- done - no Contributions for CLR {clr_round.round_num}. Exiting')

0 commit comments

Comments
 (0)