Skip to content

Commit d019295

Browse files
InfantLabInfantLab
committed
Implement background job processing system for VideoAnnotator API
- Added BackgroundJobManager to manage background job processing within the API server. - Introduced JobProcessor for handling individual job processing through pipelines. - Created worker job processor to continuously poll for pending jobs and process them using BatchOrchestrator. - Developed integration tests for API server startup, basic endpoints, authentication flow, error handling, and background job processing. - Established context manager for managing the lifecycle of background job processing. - Enhanced logging and error handling throughout the job processing workflow.
1 parent 158d476 commit d019295

23 files changed

+1912
-508
lines changed

‎.claude/settings.local.json‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,20 @@
3636
"Bash(git fetch:*)",
3737
"Bash(git push:*)",
3838
"Bash(gh auth:*)",
39-
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_logging_db.db\" uv run python -c \"\nimport subprocess\nimport time\nimport requests\n\n# Start server in background\nserver_process = subprocess.Popen(\n [''uv'', ''run'', ''python'', ''api_server.py'', ''--port'', ''8003'', ''--log-level'', ''info''],\n env={''VIDEOANNOTATOR_DB_PATH'': ''test_logging_db.db''}\n)\n\ntry:\n # Wait for server to start\n time.sleep(4)\n \n print(''[TEST] Testing API requests with logging...'')\n \n # Test health endpoint (excluded from request logging)\n response = requests.get(''http://localhost:8003/health'', timeout=5)\n print(f''[TEST] Health check: {response.status_code}'')\n \n # Test debug endpoint (should be logged)\n response = requests.get(''http://localhost:8003/api/v1/debug/server-info'', timeout=5)\n print(f''[TEST] Debug endpoint: {response.status_code}'')\n \n # Test with authentication (should be logged with user info)\n headers = {''Authorization'': ''Bearer dev-token''}\n response = requests.get(''http://localhost:8003/api/v1/debug/token-info'', headers=headers, timeout=5)\n print(f''[TEST] Authenticated request: {response.status_code}'')\n \n # Test invalid endpoint (should generate 404 and be logged)\n response = requests.get(''http://localhost:8003/api/v1/nonexistent'', timeout=5)\n print(f''[TEST] Invalid endpoint: {response.status_code}'')\n \n print(''[TEST] API requests completed'')\n \nfinally:\n # Clean up\n server_process.terminate()\n server_process.wait()\n print(''[TEST] Server stopped'')\n\")"
39+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_logging_db.db\" uv run python -c \"\nimport subprocess\nimport time\nimport requests\n\n# Start server in background\nserver_process = subprocess.Popen(\n [''uv'', ''run'', ''python'', ''api_server.py'', ''--port'', ''8003'', ''--log-level'', ''info''],\n env={''VIDEOANNOTATOR_DB_PATH'': ''test_logging_db.db''}\n)\n\ntry:\n # Wait for server to start\n time.sleep(4)\n \n print(''[TEST] Testing API requests with logging...'')\n \n # Test health endpoint (excluded from request logging)\n response = requests.get(''http://localhost:8003/health'', timeout=5)\n print(f''[TEST] Health check: {response.status_code}'')\n \n # Test debug endpoint (should be logged)\n response = requests.get(''http://localhost:8003/api/v1/debug/server-info'', timeout=5)\n print(f''[TEST] Debug endpoint: {response.status_code}'')\n \n # Test with authentication (should be logged with user info)\n headers = {''Authorization'': ''Bearer dev-token''}\n response = requests.get(''http://localhost:8003/api/v1/debug/token-info'', headers=headers, timeout=5)\n print(f''[TEST] Authenticated request: {response.status_code}'')\n \n # Test invalid endpoint (should generate 404 and be logged)\n response = requests.get(''http://localhost:8003/api/v1/nonexistent'', timeout=5)\n print(f''[TEST] Invalid endpoint: {response.status_code}'')\n \n print(''[TEST] API requests completed'')\n \nfinally:\n # Clean up\n server_process.terminate()\n server_process.wait()\n print(''[TEST] Server stopped'')\n\")",
40+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_background_db.db\" uv run python -c \"\nimport subprocess\nimport time\nimport requests\n\nprint(''[TEST] Starting API server with integrated background processing...'')\n\n# Start server with test database\nserver_process = subprocess.Popen(\n [''uv'', ''run'', ''python'', ''api_server.py'', ''--port'', ''8001'', ''--log-level'', ''info''],\n env={''VIDEOANNOTATOR_DB_PATH'': ''test_background_db.db''}\n)\n\ntry:\n # Wait for server to start\n time.sleep(6)\n \n print(''[TEST] Testing server health...'')\n response = requests.get(''http://localhost:8001/health'', timeout=5)\n if response.status_code == 200:\n print(f''[OK] Server healthy: {response.json()}'')\n else:\n print(f''[ERROR] Health check failed: {response.status_code}'')\n \n print(''[TEST] Testing background jobs status...'')\n response = requests.get(''http://localhost:8001/api/v1/debug/background-jobs'', timeout=5)\n if response.status_code == 200:\n bg_status = response.json()\n print(f''[OK] Background processing status: {bg_status}'')\n else:\n print(f''[ERROR] Background jobs status failed: {response.status_code}'')\n \n print(''[TEST] Server started successfully - checking logs for errors'')\n\nfinally:\n # Clean up\n server_process.terminate()\n server_process.wait()\n print(''[CLEANUP] Server stopped'')\n\")",
41+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_background_db.db\" uv run python api_server.py --port 8001 --log-level info)",
42+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_refactored_db.db\" uv run python -c \"\nimport subprocess\nimport time\nimport requests\n\nprint(''[TEST] Testing refactored job processing system...'')\n\n# Start server with test database\nserver_process = subprocess.Popen(\n [''uv'', ''run'', ''python'', ''api_server.py'', ''--port'', ''8004'', ''--log-level'', ''info''],\n env={''VIDEOANNOTATOR_DB_PATH'': ''test_refactored_db.db''}\n)\n\ntry:\n # Wait for server to start\n time.sleep(4)\n \n # Test job submission\n test_content = b''fake video content for testing''\n files = {''video'': (''test_video.mp4'', test_content, ''video/mp4'')}\n data = {''selected_pipelines'': ''scene,person''}\n \n response = requests.post(''http://localhost:8004/api/v1/jobs/'', files=files, data=data, timeout=10)\n \n print(f''Job submission status: {response.status_code}'')\n if response.status_code == 201:\n job_data = response.json()\n job_id = job_data[''id'']\n print(f''Job created successfully! ID: {job_id}'')\n print(f''Status: {job_data[\"\"status\"\"]}'')\n print(f''Pipelines: {job_data[\"\"selected_pipelines\"\"]}'')\n \n # Test job retrieval\n response = requests.get(f''http://localhost:8004/api/v1/jobs/{job_id}'', timeout=5)\n if response.status_code == 200:\n print(''Job retrieval successful!'')\n \n # Test job listing\n response = requests.get(''http://localhost:8004/api/v1/jobs/'', timeout=5)\n if response.status_code == 200:\n jobs_data = response.json()\n print(f''Job listing successful! Total jobs: {jobs_data[\"\"total\"\"]}'')\n \n print(''All API tests passed!'')\n else:\n print(f''Job submission failed: {response.text}'')\n \nfinally:\n # Clean up\n server_process.terminate()\n server_process.wait()\n\")",
43+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_refactored_startup.db\" timeout 10 uv run python api_server.py --port 8005 --log-level info)",
44+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_logging_fix.db\" timeout 8 uv run python api_server.py --port 8006 --log-level info)",
45+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_clean_logging.db\" timeout 8 uv run python api_server.py --port 8007 --log-level info)",
46+
"Bash(taskkill:*)",
47+
"Bash(powershell:*)",
48+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_single_server.db\" timeout 6 uv run python api_server.py --port 8888 --log-level info)",
49+
"Bash(VIDEOANNOTATOR_DB_PATH=\"test_fix_output_dir.db\" uv run python -c \"\nimport subprocess\nimport time\nimport requests\nimport os\n\nprint(''[TEST] Testing JobProcessor with output_dir fix...'')\n\n# Start server in background\nserver_process = subprocess.Popen(\n [''uv'', ''run'', ''python'', ''api_server.py'', ''--port'', ''8900'', ''--log-level'', ''info''],\n env=dict(os.environ, VIDEOANNOTATOR_DB_PATH=''test_fix_output_dir.db''),\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE\n)\n\ntry:\n # Wait for server to start\n time.sleep(5)\n \n # Test job submission with a small test video\n test_content = b''fake video content for testing''\n files = {''video'': (''test_video.mp4'', test_content, ''video/mp4'')}\n data = {''selected_pipelines'': ''scene,person''}\n \n print(''[TEST] Submitting job...'')\n response = requests.post(''http://localhost:8900/api/v1/jobs/'', files=files, data=data, timeout=10)\n \n print(f''Job submission status: {response.status_code}'')\n if response.status_code == 201:\n job_data = response.json()\n job_id = job_data[''id'']\n print(f''Job created successfully! ID: {job_id}'')\n \n # Wait a few seconds and check job status\n time.sleep(8)\n \n response = requests.get(f''http://localhost:8900/api/v1/jobs/{job_id}'', timeout=5)\n if response.status_code == 200:\n updated_job = response.json()\n print(f''Job status: {updated_job[\"\"status\"\"]}'')\n if updated_job.get(''error_message''):\n print(f''Error message: {updated_job[\"\"error_message\"\"]}'')\n else:\n print(''[SUCCESS] Job processed without output_dir error!'')\n \n else:\n print(f''Job submission failed: {response.text}'')\n \nfinally:\n # Clean up\n server_process.terminate()\n server_process.wait()\n print(''[CLEANUP] Server stopped'')\n\")",
50+
"Bash(dir /o-d logserrors.log)",
51+
"Bash(dir test_*)",
52+
"Bash(del /F test_fix_output_dir.db)"
4053
],
4154
"deny": []
4255
}

‎.gitignore‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,4 +229,12 @@ performance_reports/
229229
benchmark_results/
230230

231231
# Coverage reports
232-
htmlcov/
232+
htmlcov/
233+
234+
# Database files
235+
videoannotator.db
236+
test_*.db
237+
*.sqlite
238+
*.sqlite3
239+
240+

‎.vscode/settings.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,23 @@
1111
"python.testing.pytestEnabled": true,
1212
"testExplorer.useNativeTesting": false,
1313
"workbench.colorCustomizations": {
14-
"activityBar.activeBackground": "#65c89b",
15-
"activityBar.background": "#65c89b",
14+
"activityBar.activeBackground": "#e2c386",
15+
"activityBar.background": "#e2c386",
1616
"activityBar.foreground": "#15202b",
1717
"activityBar.inactiveForeground": "#15202b99",
18-
"activityBarBadge.background": "#945bc4",
18+
"activityBarBadge.background": "#259a73",
1919
"activityBarBadge.foreground": "#e7e7e7",
2020
"commandCenter.border": "#15202b99",
21-
"sash.hoverBorder": "#65c89b",
22-
"statusBar.background": "#42b883",
21+
"sash.hoverBorder": "#e2c386",
22+
"statusBar.background": "#d8ae5d",
2323
"statusBar.foreground": "#15202b",
24-
"statusBarItem.hoverBackground": "#359268",
25-
"statusBarItem.remoteBackground": "#42b883",
24+
"statusBarItem.hoverBackground": "#ce9934",
25+
"statusBarItem.remoteBackground": "#d8ae5d",
2626
"statusBarItem.remoteForeground": "#15202b",
27-
"titleBar.activeBackground": "#42b883",
27+
"titleBar.activeBackground": "#d8ae5d",
2828
"titleBar.activeForeground": "#15202b",
29-
"titleBar.inactiveBackground": "#42b88399",
29+
"titleBar.inactiveBackground": "#d8ae5d99",
3030
"titleBar.inactiveForeground": "#15202b99"
3131
},
32-
"peacock.color": "#42b883",
32+
"peacock.color": "#d8ae5d",
3333
}

‎docs/CLIENT_TOKEN_SETUP_GUIDE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,6 @@ videoannotator-client # Automatically uses production config
574574
---
575575

576576
**Document Version**: v1.0
577-
**Last Updated**: January 2025
577+
**Last Updated**: 24 August 2025
578578
**Compatible With**: VideoAnnotator API v1.2.0+
579579
**Status**: Production ready with comprehensive security features

0 commit comments

Comments
 (0)