fix: adds bash script to install docker for celery servers (#10661)

* fix: adds bash script to install docker

* fix: adds docker-compose to docker install script

* Adding docker-compose to run celery workers

* fix: moves deps to test.txt and imports inside fns

* fix: only run calculate_trust_bonus on gitcoin_passport queue

Co-authored-by: Gerald Iakobinyi-Pich <gerald@gitcoin.co>

Author: gdixon

app/app/settings.py

@@ -537,7 +537,7 @@ def callback(request):
 CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND', default=CACHEOPS_REDIS)
 # https://docs.celeryproject.org/en/latest/userguide/configuration.html#std-setting-task_routes
 CELERY_ROUTES = [
-    ('dashboard.tasks.calculate_trust_bonus', {'queue': 'high_priority'}),
+    ('dashboard.tasks.calculate_trust_bonus', {'queue': 'gitcoin_passport'}),
     ('grants.tasks.process_grant_contribution', {'queue': 'high_priority'}),
     ('grants.tasks.batch_process_grant_contributions', {'queue': 'high_priority'}),
     ('kudos.tasks.mint_token_request', {'queue': 'high_priority'}),

app/dashboard/passport_reader.py

@@ -2,8 +2,6 @@
 import hashlib
 import json
 
-import dag_cbor
-
 # Making GET requests against the CERAMIC_URL to read streams
 import requests
 
@@ -47,6 +45,9 @@ def get_did(address, network="1"):
     return f"did:pkh:eip155:{network}:{address}"
 
 def get_stream_ids(did, ids=[CERAMIC_GITCOIN_PASSPORT_STREAM_ID]):
+    # delay import as this is only available in celery envs
+    import dag_cbor
+
     # encode the input genesis with cborg (Concise Binary Object Representation)
     input_bytes = dag_cbor.encode({"header":{"controllers":[did],"family":"IDX"}})
     # hash the input_bytes and pad with STREAMID_CODEC and type (as bytes)

app/dashboard/tasks.py

@@ -14,7 +14,6 @@
 from django.http import HttpRequest
 from django.utils import timezone
 
-import didkit
 from app.services import RedisService
 from app.utils import get_location_from_ip
 from celery import app, group
@@ -496,6 +495,9 @@ def calculate_trust_bonus(user_id, did, address):
     :return: None
     """
 
+    # delay import as this is only available in celery envs
+    import didkit
+
     try:
         # Verify that this DID is not associated to any other profile.
         # We want to avoid the same user creating multiple accounts and re-using the same did

bin/docker-install.bash

@@ -0,0 +1,23 @@
+#!/usr/local/bin/dumb-init /bin/bash
+
+# ==================================================
+# Install docker
+# ==================================================
+
+# add repo
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+
+# use docker-ce over default
+apt-cache policy docker-ce
+
+# install from docker-ce
+sudo apt-get install -y docker-ce
+
+# add current user to dockers usergroup
+sudo usermod -aG docker ${USER}
+
+# download and expose docker-compose
+sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose

docker-compose-celery.yml

@@ -0,0 +1,31 @@
+version: '3'
+
+services:
+  worker_1: &worker
+    restart: unless-stopped
+    logging:
+      options:
+        max-size: '100m'
+        max-file: '10'
+      driver: json-file
+
+    # image: gitcoin/web:034f6cde3c
+    image: gitcoin/web:817ff194e7
+
+    volumes:
+      - .:/code
+
+    working_dir: /code/app
+    command: ["celery", "-A", "taskapp", "worker", "-Q", "gitcoin_passport,high_priority,celery"]
+
+  worker_2:
+    <<: *worker
+    command: ["celery", "-A", "taskapp", "worker", "-Q", "gitcoin_passport,high_priority,celery"]
+
+  worker_3:
+    <<: *worker
+    command: ["celery", "-A", "taskapp", "worker", "-Q", "gitcoin_passport,celery"]
+
+  worker_4:
+    <<: *worker
+    command: ["celery", "-A", "taskapp", "worker", "-Q", "gitcoin_passport,celery"]

requirements/base.txt

@@ -121,5 +121,3 @@ unidecode==1.2.0
 drf-flex-fields==0.9.1
 pandas
 django-svg-image-form-field==1.0.1
-git+https://github.com/gdixon/dag-cbor.git#egg=dag-cbor
-didkit==0.2.1

requirements/test.txt

@@ -17,3 +17,5 @@ django-debug-toolbar==3.2.4
 ipfshttpclient==0.6.0
 dj_static==0.0.6
 livereload==2.6.3
+git+https://github.com/gdixon/dag-cbor.git#egg=dag-cbor
+didkit==0.2.1