Skip to content

Commit 8422b0f

Browse files
committed
Squashed commit
1 parent f1f1d09 commit 8422b0f

18 files changed

+2553
-1
lines changed

.gitattributes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Auto detect text files and perform LF normalization
2+
* text=auto

.gitignore

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Ignore sensitive configuration files and credentials
2+
.env
3+
config.toml
4+
5+
# Ignore virtual environments
6+
venv/
7+
8+
# Ignore Docker volumes and data
9+
data/
10+
postgres_data/
11+
logs/
12+
subgraph/
13+
contracts/
14+
15+
# Ignore Ruff
16+
.ruff_cache/

Dockerfile

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Dockerfile for Service Quality Oracle
2+
3+
# Use Python 3.9 slim as the base image for a lightweight container
4+
FROM python:3.9-slim
5+
6+
# Add metadata labels
7+
LABEL description="Service Quality Oracle" \
8+
version="0.1.0"
9+
10+
# Set working directory
11+
WORKDIR /app
12+
13+
# Set environment variables
14+
ENV PYTHONDONTWRITEBYTECODE=1 \
15+
PYTHONUNBUFFERED=1 \
16+
PYTHONPATH=/app \
17+
TZ=UTC
18+
19+
# Install system dependencies
20+
RUN apt-get update && apt-get install -y \
21+
gcc \
22+
tini \
23+
&& apt-get clean \
24+
&& rm -rf /var/lib/apt/lists/*
25+
26+
# Create a non-root user to run the application
27+
RUN groupadd -g 1000 oracle && \
28+
useradd -u 1000 -g oracle -s /bin/bash -m oracle
29+
30+
# Create necessary directories for persistent data with proper permissions
31+
RUN mkdir -p /app/data/output /app/logs && \
32+
chown -R oracle:oracle /app && \
33+
chmod -R 750 /app
34+
35+
# Copy requirements file separately to leverage Docker caching
36+
COPY requirements.txt .
37+
38+
# Install Python dependencies
39+
RUN pip install --no-cache-dir -r requirements.txt
40+
41+
# Copy the application code
42+
COPY --chown=oracle:oracle src/ ./src/
43+
COPY --chown=oracle:oracle scripts/ ./scripts/
44+
COPY --chown=oracle:oracle contracts/ ./contracts/
45+
46+
# Copy marker files for project root detection
47+
COPY --chown=oracle:oracle .gitignore ./
48+
COPY --chown=oracle:oracle pyproject.toml ./
49+
50+
# Copy the scheduler to the root directory
51+
COPY --chown=oracle:oracle src/models/scheduler.py ./
52+
53+
# Create healthcheck file
54+
RUN touch /app/healthcheck && chown oracle:oracle /app/healthcheck
55+
56+
# Switch to non-root user
57+
USER oracle
58+
59+
# Use Tini as entrypoint
60+
ENTRYPOINT ["/usr/bin/tini", "--"]
61+
62+
# Add healthcheck to verify the service is running
63+
HEALTHCHECK --interval=5m --timeout=30s --start-period=1m --retries=3 \
64+
CMD python -c "import os, time; assert os.path.exists('/app/healthcheck') and time.time() - os.path.getmtime('/app/healthcheck') < 3600, 'Healthcheck failed: file missing or too old'" || exit 1
65+
66+
# Run the scheduler
67+
CMD ["python", "scheduler.py"]

ELIGIBILITY_CRITERIA.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Upcoming Eligibility Criteria
2+
3+
We will announce changes to the eligibility criteria in the table below. Once the change goes live then it will be reflected in the eligibility criteria section of this document.
4+
5+
| Upcoming Requirement | Justification | Date Introduced (YYYY-MM-DD)|
6+
|----------------------|---------------|-----------------------------|
7+
| **Requirement 1:** | This is a placeholder for future criteria, watch this space to stay informed. We will also announce any upcoming requirements via our existing official channels. | YYYY-MM-DD |
8+
9+
> **Note**:
10+
>
11+
> When announcing new eligibility criteria we will allow a window for indexers to prepare their infrastructure before any new criteria goes live, refer to the date introduced column to see when new criteria will merge.
12+
13+
# Eligibility Criteria
14+
15+
The Service Quality Oracle determines which indexers are eligible to receive indexing rewards using a threshold rewards algorithm that operates by checking indexers meet the following criteria:
16+
17+
1. Indexers must be online for 5+ days in a given 28 day rolling period.
18+
1. To be online an indexer must serve at least 1 qualifying query on 10 different subgraphs
19+
1. A qualifying query is one where:
20+
1. The query response HTTP status was 200 OK, indicating query success.
21+
2. The query response latency was <5,000 ms.
22+
3. The query was served <50,000 blocks behind chainhead.
23+
4. The subgraph had at least 500 GRT in curation signal at the time that the query was served.
24+
25+
> **Note**:
26+
>
27+
> All four quality criteria must be satisfied simultaneously for a query to count towards the daily requirement.
28+
>
29+
> The above query criteria must be satisfied on 10+ subgraphs per day, for 5+ days in any given 28 day rolling window.
30+
>
31+
> Issuance eligibility is refreshed daily via the ServiceQualityOracle contract.
32+
>
33+
> Once an indexer has qualified for issuance via the ServiceQualityOracle contract, they can claim indexing rewards from the protocol for the duration of the qualification period (default is 14 days), even if the requirements change.
34+
35+
36+
37+
| Requirement | Justification |
38+
|-------------|---------------|
39+
| **Query Status:** The query must have a `200 OK` HTTP response status indicating query success | Indexer infrastructure needs to be capable of serving successful queries to benefit data consumers. |
40+
| **Query Latency:** The query response must be delivered to the gateway in `< 5,000 ms` | Fast query responses are important to data consumers. |
41+
| **Query Freshness:** The query must be served from a subgraph that is `< 50,000 blocks` behind chainhead | Data needs to be fresh to be useful to data consumers. |
42+
| **Subgraph Signal:** The subgraph needs to have `≥ 500 GRT` in curation signal at the time when the query was served. | Indexers are encouraged to serve data on subgraphs that have curation signal. This also creates an economic barrier against those that prefer to game the system. |

README.md

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,91 @@
1-
# service-quality-oracle
1+
# Service Quality Oracle
2+
3+
4+
## Overview
5+
6+
This repository implements a Docker container service for the Service Quality Oracle. The oracle consumes data from BigQuery, processes it to determine indexer issuance rewards eligibility, based on a defined threshold algorithm, and posts issuance eligibility data on-chain.
7+
8+
The oracle runs with the following functionality:
9+
- **BigQuery Integration**: Fetches indexer performance data from Google BigQuery
10+
- **Eligibility Processing**: Applies threshold algorithm to determine issuance rewards eligibility based on service quality
11+
- **Blockchain Integration**: Posts issuance eligibility updates to the ServiceQualityOracle contract
12+
- **Docker Deployment**: Containerized and running with health checks
13+
- **Scheduled Execution**: Runs daily at 10:00 UTC
14+
- **RPC Failover**: Automatic failover between multiple RPC providers for reliability
15+
16+
17+
## Eligibility Criteria
18+
19+
Please refer to the [ELIGIBILITY_CRITERIA.md](./ELIGIBILITY_CRITERIA.md) file to view the latest criteria for issuance. We are also posting upcoming criteria in that document.
20+
21+
22+
## Data Flow
23+
24+
The application follows this data flow:
25+
26+
1. **BigQuery Data Acquisition**: The `bigquery_fetch_and_save_indexer_issuance_eligibility_data_finally_return_eligible_indexers` function in `issuance_data_access_helper.py` fetches fresh data from BigQuery, processes it to determine eligibility, and returns the eligibility data list that would then be posted on chain.
27+
- This function also ensures that data is saved to local files in dated directories for auditing/historical reference over the data retention period.
28+
29+
2. **Blockchain Publication**: The eligible indexers list from step 1 is directly posted on-chain to a smart contract. Batching of transactions is performed if necessary.
30+
31+
## Getting Started
32+
33+
### Quick Start with Docker
34+
35+
1. **Clone the repository**:
36+
```bash
37+
git clone https://github.com/graphprotocol/service-quality-oracle.git
38+
cd service-quality-oracle
39+
```
40+
41+
2. **Set up environment variables/config.toml**:
42+
43+
3. **Build and run with Docker Compose**:
44+
```bash
45+
docker-compose up --build -d
46+
```
47+
48+
4. **Monitor logs**:
49+
```bash
50+
docker-compose logs -f
51+
```
52+
53+
5. **Check health status**:
54+
```bash
55+
docker-compose ps
56+
```
57+
58+
59+
## License
60+
61+
[License information to be determined.]
62+
63+
64+
## TODO List (only outstanding TODOs)
65+
66+
### Environment variables
67+
- [ ] Load and securely manage secrets
68+
69+
### Smart Contract Integration
70+
- [ ] Further verification steps to confirm successful on-chain updates
71+
72+
### Testing & Quality Assurance
73+
- [ ] Create unit tests for all components
74+
- [ ] Slack monitoring integration
75+
- [ ] Add notification logic for failed runs so we are aware in a slack channel
76+
- [ ] Initially we can notify for successful runs too
77+
- [ ] Create integration tests for the entire pipeline
78+
- [ ] Implement mocking for blockchain interactions in test environment
79+
- [ ] CI/CD pipeline?
80+
- [ ] Perform security review of code and dependencies
81+
- [ ] Ensure unused files, functions & dependencies are removed from codebase
82+
83+
### Documentation
84+
85+
- [ ] Documentation of all major components
86+
- [ ] Document operational procedures
87+
88+
### Production Readiness
89+
- [ ] Check error recovery mechanisms to see if they could be improved (RPC failover, retry logic)
90+
- [ ] Verify health check endpoints or processes (Docker healthcheck)
91+

config.toml.example

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Service Quality Oracle Configuration
2+
# This file separates sensitive secrets from non-sensitive configuration values
3+
4+
# =============================================================================
5+
# NON-SENSITIVE CONFIGURATION
6+
# =============================================================================
7+
8+
[bigquery]
9+
BIGQUERY_LOCATION_ID = ""
10+
BIGQUERY_PROJECT_ID = ""
11+
BIGQUERY_DATASET_ID = ""
12+
13+
[blockchain]
14+
BLOCKCHAIN_CONTRACT_ADDRESS = ""
15+
BLOCKCHAIN_FUNCTION_NAME = ""
16+
BLOCKCHAIN_CHAIN_ID =
17+
BLOCKCHAIN_RPC_URLS = [
18+
"",
19+
"",
20+
"",
21+
""
22+
]
23+
24+
[scheduling]
25+
SCHEDULED_RUN_TIME = "10:00"
26+
27+
[subgraph]
28+
SUBGRAPH_URL_PRE_PRODUCTION = ""
29+
SUBGRAPH_URL_PRODUCTION = ""
30+
31+
[processing]
32+
BATCH_SIZE = 125
33+
MAX_AGE_BEFORE_DELETION = 120
34+
35+
# =============================================================================
36+
# SENSITIVE CONFIGURATION
37+
# =============================================================================
38+
39+
[secrets]
40+
GOOGLE_APPLICATION_CREDENTIALS = "$GOOGLE_APPLICATION_CREDENTIALS"
41+
BLOCKCHAIN_PRIVATE_KEY = "$BLOCKCHAIN_PRIVATE_KEY"
42+
ETHERSCAN_API_KEY = "$ETHERSCAN_API_KEY"
43+
ARBITRUM_API_KEY = "$ARBITRUM_API_KEY"
44+
STUDIO_API_KEY = "$STUDIO_API_KEY"
45+
STUDIO_DEPLOY_KEY = "$STUDIO_DEPLOY_KEY"

docker-compose.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
services:
2+
issuance-oracle:
3+
build: .
4+
container_name: service-quality-oracle
5+
image: service-quality-oracle:latest
6+
7+
volumes:
8+
# Mount data directory to store data
9+
- ./data:/app/data
10+
- ./logs:/app/logs
11+
12+
# Mount config file as read-only
13+
- ./config.toml:/app/config.toml:ro
14+
15+
# Mount Google credentials file if using file-based auth (optional)
16+
- ./credentials.json:/app/credentials.json:ro
17+
18+
environment:
19+
- PYTHONPATH=/app
20+
- RUN_ON_STARTUP=true
21+
- TZ=UTC
22+
23+
# Setup enviroment variables
24+
# Environment variables go into process memory for this specific container only
25+
# Meaning they can't be accessed by other containers or processes outside of this container
26+
# More secure than storing secrets in a file that can be accessed by other containers or processes
27+
- GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}
28+
- BLOCKCHAIN_PRIVATE_KEY=${BLOCKCHAIN_PRIVATE_KEY}
29+
- ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}
30+
- ARBITRUM_API_KEY=${ARBITRUM_API_KEY}
31+
- STUDIO_API_KEY=${STUDIO_API_KEY}
32+
- STUDIO_DEPLOY_KEY=${STUDIO_DEPLOY_KEY}
33+
34+
# Block processes from gaining higher privileges/capabilities
35+
security_opt:
36+
- no-new-privileges:true
37+
38+
# Resource limits
39+
deploy:
40+
resources:
41+
limits:
42+
cpus: '1.0'
43+
memory: 1G
44+
reservations:
45+
memory: 512M
46+
47+
restart: unless-stopped
48+
49+
healthcheck:
50+
test: ["CMD", "python", "-c", "import os, time; assert os.path.exists('/app/healthcheck') and time.time() - os.path.getmtime('/app/healthcheck') < 3600, 'Healthcheck failed'"]
51+
interval: 5m
52+
timeout: 30s
53+
retries: 3
54+
start_period: 1m
55+
56+
logging:
57+
driver: "json-file"
58+
options:
59+
max-size: "10m"
60+
max-file: "3"

pyproject.toml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# This file is used to configure project settings and tools for the Service Quality Oracle project.
2+
# It defines configurations for linting, formatting, and other development tools to maintain code quality
3+
# and consistency across the project. Specifically, it sets up rules for Ruff, a fast Python linter and
4+
# formatter, to enforce coding standards, manage import sorting, and handle code complexity.
5+
6+
[tool.ruff]
7+
# Allow line lengths up to 115 characters.
8+
line-length = 115
9+
target-version = "py39"
10+
11+
# Enable auto-fixing for most issues, but not line length
12+
fix = true
13+
fix-only = false
14+
15+
[tool.ruff.lint]
16+
# Enable rules including isort (I) for import sorting and additional fixes
17+
select = ["E", "F", "B", "I", "UP", "N", "C", "W"]
18+
19+
# Exclude a variety of commonly ignored directories.
20+
exclude = [
21+
".git",
22+
".ruff_cache",
23+
"venv",
24+
"__pycache__",
25+
]
26+
27+
# Prevent auto-fixing line length issues
28+
unfixable = ["E501"]
29+
30+
# Allow unused variables when underscore-prefixed.
31+
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
32+
33+
# Specify rules to ignore in specific files
34+
[tool.ruff.lint.per-file-ignores]
35+
# Ignore E402 (import not at top) in scripts and specific modules
36+
"scripts/test_*.py" = ["E402"]
37+
"src/models/issuance_eligibility_oracle_core.py" = ["E402"]
38+
39+
# Use unsafe fixes to address typing and other modernization issues
40+
[tool.ruff.lint.isort]
41+
known-first-party = ["src"]
42+
43+
[tool.ruff.lint.mccabe]
44+
# Unlike Flake8, default to a complexity level of 10.
45+
max-complexity = 10
46+
47+
[tool.ruff.format]
48+
# Format SQL code in strings/docstrings
49+
docstring-code-format = true
50+
quote-style = "double"
51+
indent-style = "space"

0 commit comments

Comments
 (0)