From 1b2c02c2c1d0d595467c01892c6f33625da2fbb6 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 2 Jan 2025 10:35:20 -0800 Subject: [PATCH 1/3] CV2-5892 gut video --- Makefile | 3 --- app/main/lib/shared_models/video_model.py | 3 +-- app/main/lib/similarity.py | 2 +- app/test/test_video_similarity.py | 2 +- docker-compose.yml | 12 ------------ manage.py | 6 ------ 6 files changed, 3 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 9a687196..55613ee9 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,6 @@ run_model: run_rq_worker: while true; do python manage.py run_rq_worker; done -run_video_matcher: - while true; do python manage.py run_video_matcher; done - test: wait python manage.py init_perl_functions coverage run --source=app/main/ manage.py test diff --git a/app/main/lib/shared_models/video_model.py b/app/main/lib/shared_models/video_model.py index ffeb6b8d..1517f955 100644 --- a/app/main/lib/shared_models/video_model.py +++ b/app/main/lib/shared_models/video_model.py @@ -14,7 +14,6 @@ import tmkpy from sqlalchemy.orm.exc import NoResultFound -from app.main.lib.shared_models.shared_model import SharedModel from app.main.lib.similarity_helpers import get_context_query, drop_context_from_record from app.main.lib.helpers import context_matches from app.main.lib import media_crud @@ -28,7 +27,7 @@ def _after_log(retry_state): app.logger.debug("Retrying video similarity...") -class VideoModel(SharedModel): +class VideoModel(): def overload_context_to_denote_content_type(self, task): return {**task, **{"context": {**task.get("context", {}), **{"content_type": "video"}}}} diff --git a/app/main/lib/similarity.py b/app/main/lib/similarity.py index 65e009ad..31de09db 100644 --- a/app/main/lib/similarity.py +++ b/app/main/lib/similarity.py @@ -78,7 +78,7 @@ def audio_model(): return AudioModel(app.config['AUDIO_MODEL']) def video_model(): - return VideoModel(app.config['VIDEO_MODEL']) + return VideoModel() def model_response_package(item, command): response_package = { diff --git a/app/test/test_video_similarity.py b/app/test/test_video_similarity.py index 5ef02f78..6cda833e 100644 --- a/app/test/test_video_similarity.py +++ b/app/test/test_video_similarity.py @@ -23,7 +23,7 @@ def respond(self, task): class TestVideoSimilarityBlueprint(BaseTestCase): def setUp(self): super().setUp() - self.model = VideoModel('video') + self.model = VideoModel() def test_get_tempfile(self): self.assertIsInstance(self.model.get_tempfile(), tempfile._TemporaryFileWrapper) diff --git a/docker-compose.yml b/docker-compose.yml index b59188e8..90489bc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,18 +97,6 @@ services: # - .env_file # environment: # MODEL_NAME: mdebertav3filipino - # video: - # build: . - # platform: linux/x86_64 - # command: ["make", "run_model"] - # volumes: - # - ".:/app" - # depends_on: - # - redis - # env_file: - # - .env_file - # environment: - # MODEL_NAME: video # audio: # build: . # platform: linux/x86_64 diff --git a/manage.py b/manage.py index 61ece43f..0fe97a34 100644 --- a/manage.py +++ b/manage.py @@ -260,12 +260,6 @@ def run_model(): model_config['options'] ) - -@manager.command -def run_video_matcher(): - """Runs the video matcher.""" - VideoMatcher.start_server() - @manager.command def init(): """Initializes the service.""" From 90917f4c578e67e631bd8259ac7853c6277e6e30 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 6 Jan 2025 07:04:40 -0800 Subject: [PATCH 2/3] remove more dependencies --- Dockerfile | 3 +-- app/main/config.py | 1 - production/Dockerfile | 4 +--- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8713fdf3..6ee05327 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ WORKDIR /app # Install dependencies ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y ffmpeg cmake swig libavcodec-dev libavformat-dev -RUN ln -s /usr/bin/ffmpeg /usr/local/bin/ffmpeg +RUN apt-get update && apt-get install -y cmake swig # Other configurations diff --git a/app/main/config.py b/app/main/config.py index 39c8ac92..818b0561 100644 --- a/app/main/config.py +++ b/app/main/config.py @@ -29,7 +29,6 @@ class Config: MODEL_NAME = os.getenv('MODEL_NAME') MAX_CLAUSE_COUNT = 1000 PERSISTENT_DISK_PATH = os.getenv('PERSISTENT_DISK_PATH', '/app/persistent_disk') - VIDEO_MODEL = os.getenv('VIDEO_MODEL', 'video-model') try: VIDEO_MODEL_L1_SCORE = float(os.getenv('video_model_l1_score', '0.7')) except: diff --git a/production/Dockerfile b/production/Dockerfile index 88563296..fc32b774 100755 --- a/production/Dockerfile +++ b/production/Dockerfile @@ -19,10 +19,8 @@ RUN chmod 755 /opt/bin/*.sh WORKDIR /app -RUN apt-get update && apt-get install -y ffmpeg cmake swig libavcodec-dev libavformat-dev -RUN apt-get update && apt-get install -y ffmpeg swig +RUN apt-get update && apt-get install -y cmake swig RUN apt-get clean -RUN ln -s /usr/bin/ffmpeg /usr/local/bin/ffmpeg COPY . . From 2cb7f5b1ccecb7cb7175c3b7ff62dbd26883ec85 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 6 Jan 2025 07:33:42 -0800 Subject: [PATCH 3/3] remove another mention of config var --- app/main/lib/graph_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/lib/graph_writer.py b/app/main/lib/graph_writer.py index 2787b52a..763c5cb9 100644 --- a/app/main/lib/graph_writer.py +++ b/app/main/lib/graph_writer.py @@ -30,7 +30,7 @@ def audio_model(): return SharedModel.get_client(app.config['AUDIO_MODEL']) def video_model(): - return SharedModel.get_client(app.config['VIDEO_MODEL']) + return SharedModel.get_client() def get_iterable_objects(graph, data_type): try: