From cb944831f767f8ae614dcc4ae9fa2e028d80ac64 Mon Sep 17 00:00:00 2001 From: Jonathan Chiang Date: Fri, 21 Feb 2025 15:04:46 -0500 Subject: [PATCH 1/5] added hotkey auth to purchase focus video endpoint --- purchase_focus_video.py | 35 ++++++++++++++++++++++++----------- validator-api/app.py | 31 +++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/purchase_focus_video.py b/purchase_focus_video.py index 4c3aadb..1132986 100644 --- a/purchase_focus_video.py +++ b/purchase_focus_video.py @@ -58,17 +58,19 @@ Remember to keep your wallet information secure and never share your private keys. """ -import os -import requests -import bittensor as bt -from bittensor import wallet as btcli_wallet import argparse -import time import json -from tabulate import tabulate -from datetime import datetime import multiprocessing +import os import sys +import time +from datetime import datetime + +import bittensor as bt +import requests +from aiohttp import BasicAuth +from bittensor import wallet as btcli_wallet +from tabulate import tabulate parser = argparse.ArgumentParser(description='Interact with the OMEGA Focus Videos API.') args = parser.parse_args() @@ -80,6 +82,7 @@ if SUBTENSOR_NETWORK == "test" else "https://sn24-api.omegatron.ai" ) +# API_BASE = "http://localhost:8000" CYAN = "\033[96m" GREEN = "\033[92m" @@ -185,7 +188,12 @@ def transfer_with_timeout(wallet, transfer_address_to, transfer_balance): else: return False, None, "Transfer process exited without result" -def purchase_video(video_id=None, wallet_name=None, wallet_hotkey=None, wallet_path=None): +def purchase_video( + video_id=None, + wallet_name=None, + wallet_hotkey=None, + wallet_path=None +): if not video_id: video_id = input(f"{CYAN}Enter focus video id: {RESET}") @@ -210,15 +218,20 @@ def purchase_video(video_id=None, wallet_name=None, wallet_hotkey=None, wallet_p return miner_hotkey = hotkey.ss58_address - + signature = f"0x{hotkey.sign(miner_hotkey).hex()}" print(f"Purchasing video {video_id}...") print(f"{RED}You will only have 2 minutes and 30 seconds to complete the transfer of TAO tokens, otherwise the purchase will be reverted.{RESET}") purchase_response = requests.post( - API_BASE + "/api/focus/purchase", - json={"video_id": video_id, "miner_hotkey": miner_hotkey}, + API_BASE + "/api/focus/purchase", + auth=BasicAuth(hotkey, signature), + json={ + "video_id": video_id, + # "miner_hotkey": miner_hotkey, + }, headers={"Content-Type": "application/json"}, timeout=60 ) + time.sleep(20) purchase_data = purchase_response.json() if purchase_response.status_code != 200: diff --git a/validator-api/app.py b/validator-api/app.py index e966db5..dae408c 100644 --- a/validator-api/app.py +++ b/validator-api/app.py @@ -202,11 +202,18 @@ class VideoPurchaseRevert(BaseModel): def get_hotkey(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str: - keypair = Keypair(ss58_address=credentials.username) - - if keypair.verify(credentials.username, credentials.password): - return credentials.username - + # print(f"Username: {credentials.username}, Password: {credentials.password}") + try: + keypair = Keypair(ss58_address=credentials.username) + # print(f"Keypair: {keypair}") + if keypair.verify(credentials.username, credentials.password): + return credentials.username + except Exception as e: + print(f"Error verifying keypair: {e}") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"Error verifying keypair: {e}" + ) raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Signature mismatch", @@ -670,17 +677,25 @@ async def _get_available_focus_video_list( async def purchase_video( request: Request, background_tasks: BackgroundTasks, - video_id: Annotated[str, Body()], - miner_hotkey: Annotated[str, Body()], + video_id: Annotated[str, Body(embed=True)], + # miner_hotkey: Annotated[str, Body()], + hotkey: Annotated[str, Depends(get_hotkey)], db: Session = Depends(get_db), ): + # print(f"purchase_video() with hotkey={hotkey}") + if not authenticate_with_bittensor(hotkey, metagraph) and not authenticate_with_commune(hotkey, commune_keys): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Valid hotkey required.", + ) + if await already_purchased_max_focus_tao(db): print("Purchases in the last 24 hours have reached the max focus tao limit.") raise HTTPException( 400, "Purchases in the last 24 hours have reached the max focus tao limit, please try again later.") # run with_lock True - availability = await check_availability(db, video_id, miner_hotkey, True) + availability = await check_availability(db, video_id, hotkey, True) print('availability', availability) if availability['status'] == 'success': amount = availability['price'] From c4f7a81a864b974418fdee70e1a3585a9aeeb9e8 Mon Sep 17 00:00:00 2001 From: Jonathan Chiang Date: Fri, 21 Feb 2025 17:03:36 -0500 Subject: [PATCH 2/5] added miner banning --- validator-api/app.py | 17 +-- .../validator_api/cron/confirm_purchase.py | 3 + .../database/models/miner_bans.py | 104 ++++++++++++++++++ 3 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 validator-api/validator_api/database/models/miner_bans.py diff --git a/validator-api/app.py b/validator-api/app.py index dae408c..ade7db2 100644 --- a/validator-api/app.py +++ b/validator-api/app.py @@ -56,6 +56,7 @@ from validator_api.utils.marketplace import (TASK_TYPE_MAP, get_max_focus_alpha_per_day, get_purchase_max_focus_alpha) +from validator_api.database.models.miner_bans import miner_banned_until from omega.protocol import AudioMetadata, VideoMetadata @@ -212,12 +213,8 @@ def get_hotkey(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) print(f"Error verifying keypair: {e}") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail=f"Error verifying keypair: {e}" + detail=f"Error verifying keypair: {e}, make sure Basic Auth username is your hotkey SS58 address and the password is your hotkey's signature hex string (not private key!)." ) - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Signature mismatch", - ) def check_commune_validator_hotkey(hotkey: str, modules_keys): @@ -670,8 +667,6 @@ async def _get_available_focus_video_list( """ return await get_all_available_focus(db) - # FV TODO: let's do proper miner auth here instead, and then from the retrieved hotkey, we can also - # retrieve the coldkey and use that to confirm the transfer @app.post("/api/focus/purchase") @limiter.limit("2/minute") async def purchase_video( @@ -683,11 +678,19 @@ async def purchase_video( db: Session = Depends(get_db), ): # print(f"purchase_video() with hotkey={hotkey}") + # this verifies that the miner owns the hotkey if not authenticate_with_bittensor(hotkey, metagraph) and not authenticate_with_commune(hotkey, commune_keys): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Valid hotkey required.", ) + + banned_until = miner_banned_until(db, hotkey) + if banned_until: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Miner is banned from purchasing focus videos until {banned_until} due to too many failed purchases in a row. Contact a team member if you believe this is an error.", + ) if await already_purchased_max_focus_tao(db): print("Purchases in the last 24 hours have reached the max focus tao limit.") diff --git a/validator-api/validator_api/cron/confirm_purchase.py b/validator-api/validator_api/cron/confirm_purchase.py index 0b8569c..e0cee09 100644 --- a/validator-api/validator_api/cron/confirm_purchase.py +++ b/validator-api/validator_api/cron/confirm_purchase.py @@ -11,6 +11,7 @@ import bittensor as bt from validator_api.utils.wallet import get_transaction_from_block_hash +from validator_api.database.models.miner_bans import increment_failed_purchases, reset_failed_purchases def extrinsic_already_confirmed(db: Session, extrinsic_id: str) -> bool: record = db.query(FocusVideoRecord).filter(FocusVideoRecord.extrinsic_id == extrinsic_id) @@ -146,6 +147,7 @@ async def confirm_video_purchased( if video is not None and video.processing_state == FocusVideoStateInternal.PURCHASED: print(f"Video <{video_id}> has been marked as PURCHASED. Stopping background task.") + reset_failed_purchases(db, video.miner_hotkey) return True elif video is not None and video.processing_state == FocusVideoStateInternal.SUBMITTED: print(f"Video <{video_id}> has been marked as SUBMITTED. Stopping background task.") @@ -161,6 +163,7 @@ async def confirm_video_purchased( # we got here because we could not confirm the payment in time, so we need to revert # the video back to the SUBMITTED state (i.e. mark available for purchase) print(f"Video <{video_id}> has NOT been marked as PURCHASED. Reverting to SUBMITTED state...") + increment_failed_purchases(db, video.miner_hotkey) video.processing_state = FocusVideoStateInternal.SUBMITTED video.updated_at = datetime.utcnow() db.add(video) diff --git a/validator-api/validator_api/database/models/miner_bans.py b/validator-api/validator_api/database/models/miner_bans.py new file mode 100644 index 0000000..6b0ffb2 --- /dev/null +++ b/validator-api/validator_api/database/models/miner_bans.py @@ -0,0 +1,104 @@ +from datetime import datetime +from typing import Optional + +from sqlalchemy import Column, String, DateTime, Integer +from pydantic import BaseModel, ConfigDict + +from validator_api.database import Base +from validator_api.config import DB_STRING_LENGTH +from sqlalchemy.orm import Session +from datetime import timedelta + +class MinerBan(Base): + __tablename__ = 'miner_bans' + + miner_hotkey = Column(String(DB_STRING_LENGTH), primary_key=True, nullable=False) + purchases_failed_in_a_row = Column(Integer, nullable=False) + banned_until = Column(DateTime(timezone=True), nullable=True) + + +class MinerBanModel(BaseModel): + model_config = ConfigDict(from_attributes=True) + + miner_hotkey: str + purchases_failed_since_last_ban: int + banned_until: Optional[datetime] + + +def miner_banned_until(db: Session, miner_hotkey: str) -> Optional[datetime]: + """ + Check if a miner is currently banned and return their ban expiration time if they are. + + Args: + db: Database session + miner_hotkey: The miner's hotkey to check + + Returns: + datetime: The banned_until time if the miner is currently banned + None: If the miner is not currently banned + """ + ban = db.query(MinerBan).filter( + MinerBan.miner_hotkey == miner_hotkey, + MinerBan.banned_until > datetime.utcnow() + ).first() + + return ban.banned_until if ban else None + +def get_or_create_miner(db: Session, miner_hotkey: str) -> MinerBan: + """ + Get a miner's ban record or create it if it doesn't exist. + + Args: + db: Database session + miner_hotkey: The miner's hotkey + + Returns: + MinerBan: The miner's ban record + """ + miner = db.query(MinerBan).filter( + MinerBan.miner_hotkey == miner_hotkey + ).first() + + if not miner: + miner = MinerBan( + miner_hotkey=miner_hotkey, + purchases_failed_in_a_row=0, + banned_until=None + ) + db.add(miner) + db.commit() + + return miner + +def increment_failed_purchases(db: Session, miner_hotkey: str): + """ + Increment the number of purchases failed in a row for a miner. + Creates the miner record if it doesn't exist. + + """ + miner = get_or_create_miner(db, miner_hotkey) + miner.purchases_failed_in_a_row += 1 + db.commit() + check_and_ban_miner(db, miner_hotkey) + +def reset_failed_purchases(db: Session, miner_hotkey: str): + """ + In the case of a successful purchase, reset the number of purchases failed in a row for a miner. + Creates the miner record if it doesn't exist. + """ + miner = get_or_create_miner(db, miner_hotkey) + miner.purchases_failed_in_a_row = 0 + miner.banned_until = None + db.commit() + +BAN_PURCHASES_FAILED_IN_A_ROW = 3 +def check_and_ban_miner(db: Session, miner_hotkey: str): + """ + If a miner fails more than BAN_PURCHASES_FAILED_IN_A_ROW purchases in a row, ban them for 24 hours. + Creates the miner record if it doesn't exist. + """ + miner = get_or_create_miner(db, miner_hotkey) + if miner.purchases_failed_in_a_row >= BAN_PURCHASES_FAILED_IN_A_ROW: + miner.purchases_failed_in_a_row = 0 + miner.banned_until = datetime.utcnow() + timedelta(hours=24) + db.commit() \ No newline at end of file From 460caf253e8d651927544578c68930158b8cf6e1 Mon Sep 17 00:00:00 2001 From: Salman Date: Fri, 21 Feb 2025 23:11:23 +0000 Subject: [PATCH 3/5] added miner hotkey auth to verify and revert purchase endpoints --- validator-api/app.py | 7 +++---- validator-api/validator_api/database/crud/focusvideo.py | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/validator-api/app.py b/validator-api/app.py index 82134fd..c6dbbc4 100644 --- a/validator-api/app.py +++ b/validator-api/app.py @@ -683,7 +683,6 @@ async def purchase_video( request: Request, background_tasks: BackgroundTasks, video_id: Annotated[str, Body(embed=True)], - # miner_hotkey: Annotated[str, Body()], hotkey: Annotated[str, Depends(get_hotkey)], db: Session = Depends(get_db), ): @@ -728,17 +727,17 @@ async def purchase_video( @limiter.limit("4/minute") async def revert_pending_purchase( request: Request, + miner_hotkey: Annotated[str, Depends(get_hotkey)], video: VideoPurchaseRevert, db: Session = Depends(get_db), ): - # run with_lock True - return mark_video_submitted(db, video.video_id, True) + return mark_video_submitted(db, video.video_id, miner_hotkey, with_lock=True) @app.post("/api/focus/verify-purchase") @limiter.limit("4/minute") async def verify_purchase( request: Request, - miner_hotkey: Annotated[str, Body()], + miner_hotkey: Annotated[str, Depends(get_hotkey)], video_id: Annotated[str, Body()], block_hash: Annotated[str, Body()], db: Session = Depends(get_db), diff --git a/validator-api/validator_api/database/crud/focusvideo.py b/validator-api/validator_api/database/crud/focusvideo.py index fd1c389..c16ecf5 100644 --- a/validator-api/validator_api/database/crud/focusvideo.py +++ b/validator-api/validator_api/database/crud/focusvideo.py @@ -453,12 +453,13 @@ def mark_video_rejected( db.add(video_record) db.commit() -def mark_video_submitted(db: Session, video_id: str, with_lock: bool = False): +def mark_video_submitted(db: Session, video_id: str, miner_hotkey: str, with_lock: bool = False): # Mark video as "SUBMITTED" if in the "PURCHASE_PENDING" state. video_record = db.query(FocusVideoRecord).filter( FocusVideoRecord.video_id == video_id, FocusVideoRecord.processing_state == FocusVideoStateInternal.PURCHASE_PENDING, - FocusVideoRecord.deleted_at.is_(None) + FocusVideoRecord.deleted_at.is_(None), + FocusVideoRecord.miner_hotkey == miner_hotkey # make sure the miner requesting the cancellation is the one who was trying to buy it! ) if with_lock: video_record = video_record.with_for_update() From 0cb27c9d0f5ff946a00b40716c055a38017c1197 Mon Sep 17 00:00:00 2001 From: Salman Date: Fri, 21 Feb 2025 23:26:36 +0000 Subject: [PATCH 4/5] skip vali check on purchasing endpoint --- validator-api/app.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/validator-api/app.py b/validator-api/app.py index c6dbbc4..fe8390f 100644 --- a/validator-api/app.py +++ b/validator-api/app.py @@ -686,14 +686,6 @@ async def purchase_video( hotkey: Annotated[str, Depends(get_hotkey)], db: Session = Depends(get_db), ): - # print(f"purchase_video() with hotkey={hotkey}") - # this verifies that the miner owns the hotkey - if not authenticate_with_bittensor(hotkey, metagraph) and not authenticate_with_commune(hotkey, commune_keys): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=f"Valid hotkey required.", - ) - banned_until = miner_banned_until(db, hotkey) if banned_until: raise HTTPException( From 2cb4afc70cea6e86de1fdec3db8dfc5757791e7c Mon Sep 17 00:00:00 2001 From: Salman Date: Fri, 21 Feb 2025 23:34:02 +0000 Subject: [PATCH 5/5] cleaned up miner bans file --- .../validator_api/database/models/miner_bans.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/validator-api/validator_api/database/models/miner_bans.py b/validator-api/validator_api/database/models/miner_bans.py index 6b0ffb2..d8efabe 100644 --- a/validator-api/validator_api/database/models/miner_bans.py +++ b/validator-api/validator_api/database/models/miner_bans.py @@ -16,15 +16,6 @@ class MinerBan(Base): purchases_failed_in_a_row = Column(Integer, nullable=False) banned_until = Column(DateTime(timezone=True), nullable=True) - -class MinerBanModel(BaseModel): - model_config = ConfigDict(from_attributes=True) - - miner_hotkey: str - purchases_failed_since_last_ban: int - banned_until: Optional[datetime] - - def miner_banned_until(db: Session, miner_hotkey: str) -> Optional[datetime]: """ Check if a miner is currently banned and return their ban expiration time if they are. @@ -78,8 +69,8 @@ def increment_failed_purchases(db: Session, miner_hotkey: str): """ miner = get_or_create_miner(db, miner_hotkey) miner.purchases_failed_in_a_row += 1 + check_and_ban_miner(miner) db.commit() - check_and_ban_miner(db, miner_hotkey) def reset_failed_purchases(db: Session, miner_hotkey: str): """ @@ -91,14 +82,12 @@ def reset_failed_purchases(db: Session, miner_hotkey: str): miner.banned_until = None db.commit() -BAN_PURCHASES_FAILED_IN_A_ROW = 3 -def check_and_ban_miner(db: Session, miner_hotkey: str): +BAN_PURCHASES_FAILED_IN_A_ROW = 5 +def check_and_ban_miner(miner: MinerBan): """ If a miner fails more than BAN_PURCHASES_FAILED_IN_A_ROW purchases in a row, ban them for 24 hours. Creates the miner record if it doesn't exist. """ - miner = get_or_create_miner(db, miner_hotkey) if miner.purchases_failed_in_a_row >= BAN_PURCHASES_FAILED_IN_A_ROW: miner.purchases_failed_in_a_row = 0 miner.banned_until = datetime.utcnow() + timedelta(hours=24) - db.commit() \ No newline at end of file