Add uploading files without reindex, minor fixes

Author: drunkbatya

Dockerfile

@@ -1,13 +1,16 @@
 FROM python:3.11-alpine3.17
 
+RUN apk update
+RUN apk add tzdata nginx-mod-http-fancyindex nginx-mod-http-headers-more bash
+
+ADD requirements.txt /app/
+RUN python3 -m pip install -r /app/requirements.txt
+
 COPY nginx/nginx.conf /etc/nginx/nginx.conf
 COPY nginx/nginx-theme /var/lib/nginx/html/nginx-theme
 ADD indexer /app
 COPY startup.sh /app/
-WORKDIR /app
 
-RUN apk update
-RUN apk add tzdata nginx-mod-http-fancyindex nginx-mod-http-headers-more bash
-RUN python3 -m pip install -r requirements.txt
+WORKDIR /app
+CMD ["/bin/bash", "/app/startup.sh"]
 
-CMD ["/bin/bash", "startup.sh"]

README.md

@@ -47,3 +47,10 @@ Upload files
         -F "files=@flipper-z-f7-full-0.73.1.json" \
         127.0.0.1:8000/firmware/uploadfiles
 ```
+
+Upload files without reindex
+```bash
+    curl -L -H "Token: YOUR_TOKEN" \
+        -F "files=@gcc-arm-none-eabi-12.3-arm64-darwin-flipper-24.tar.gz" \
+        127.0.0.1:8000/toolchain/uploadfilesraw
+```

indexer/main.py

@@ -5,7 +5,7 @@
 import uvicorn
 from fastapi import FastAPI, Request, Response
 from src import directories, file_upload, security
-from src.directories import indexes
+from src.repository import indexes, raw_file_upload_directories
 from src.settings import settings
 from pygelf import GelfTcpHandler
 
@@ -25,9 +25,17 @@ def startup_event() -> None:
         os.makedirs(settings.files_dir)
     for index in indexes:
         try:
+            index_path = os.path.join(settings.files_dir, index)
+            os.makedirs(index_path, exist_ok=True)
             indexes[index].reindex()
         except Exception:
             logging.exception(f"Init {index} reindex failed")
+    for raw_upload_dir in raw_file_upload_directories:
+        try:
+            dir_path = os.path.join(settings.files_dir, raw_upload_dir)
+            os.makedirs(dir_path, exist_ok=True)
+        except Exception:
+            logging.exception(f"Failed to create {dir_path}")
 
 
 app.include_router(file_upload.router)

indexer/src/file_upload.py

@@ -7,7 +7,7 @@
 from typing import List
 from fastapi import APIRouter, Form, UploadFile
 from fastapi.responses import JSONResponse
-from .directories import indexes
+from .repository import indexes, raw_file_upload_directories
 from .settings import settings
 
 
@@ -46,7 +46,7 @@ def save_files(path: str, files: List[UploadFile]) -> None:
             out_file.write(file.file.read())
 
 
-def move_files(dest_dir: str, source_dir: str, version_token: str) -> None:
+def move_files_for_indexed(dest_dir: str, source_dir: str, version_token: str) -> None:
     token_file_path = os.path.join(dest_dir, TOKEN_FILENAME)
     do_cleanup = False
     if version_token and os.path.isfile(token_file_path):
@@ -67,6 +67,13 @@ def move_files(dest_dir: str, source_dir: str, version_token: str) -> None:
         shutil.move(sourcefilepath, destfilepath)
 
 
+def move_files_raw(dest_dir: str, source_dir: str) -> None:
+    for file in os.listdir(source_dir):
+        sourcefilepath = os.path.join(source_dir, file)
+        destfilepath = os.path.join(dest_dir, file)
+        shutil.move(sourcefilepath, destfilepath)
+
+
 @router.post("/{directory}/uploadfiles")
 async def create_upload_files(
     directory: str,
@@ -101,7 +108,7 @@ async def create_upload_files(
         try:
             with tempfile.TemporaryDirectory() as temp_path:
                 save_files(temp_path, files)
-                move_files(final_path, temp_path, version_token)
+                move_files_for_indexed(final_path, temp_path, version_token)
             logging.info(f"Uploaded {len(files)} files")
         except Exception as e:
             logging.exception(e)
@@ -117,3 +124,34 @@ async def create_upload_files(
                 )
         else:
             return JSONResponse("File uploaded, reindexing isn't needed!")
+
+
+@router.post("/{directory}/uploadfilesraw")
+async def create_upload_files_raw(
+    directory: str,
+    files: List[UploadFile],
+):
+    """
+    A method to upload files in a certain directory without indexing
+    Args:
+        directory: Repository name
+        files: File list
+
+    Returns:
+        Upload status
+    """
+    if directory not in raw_file_upload_directories:
+        return JSONResponse(f"{directory} not found!", status_code=404)
+
+    project_root_path = os.path.join(settings.files_dir, directory)
+
+    async with lock:
+        try:
+            with tempfile.TemporaryDirectory() as temp_path:
+                save_files(temp_path, files)
+                move_files_raw(project_root_path, temp_path)
+            logging.info(f"Uploaded {len(files)} files")
+            return JSONResponse("File uploaded")
+        except Exception as e:
+            logging.exception(e)
+            return JSONResponse(str(e), status_code=500)

indexer/src/repository.py

@@ -155,3 +155,5 @@ def get_file_from_latest_version(
         file_parser=blackmagicFileParser,
     ),
 }
+
+raw_file_upload_directories = ["toolchain"]

indexer/src/settings.py

@@ -44,5 +44,5 @@ class Settings(BaseModel):
     qFlipper_github_repo=os.getenv("INDEXER_QFLIPPER_GITHUB_REPO"),
     blackmagic_github_token=os.getenv("INDEXER_BLACKMAGIC_GITHUB_TOKEN"),
     blackmagic_github_repo=os.getenv("INDEXER_BLACKMAGIC_GITHUB_REPO"),
-    private_paths=["reindex", "uploadfiles"],
+    private_paths=["reindex", "uploadfiles", "uploadfilesraw"],
 )

nginx/nginx.conf

@@ -37,7 +37,7 @@ http {
             fancyindex_localtime on;
             fancyindex_ignore "nginx-theme";
         }
-        location ~ ^/(qFlipper|firmware|blackmagic-firmware)/ {
+        location ~ ^/(qFlipper|firmware|blackmagic-firmware|toolchain)/ {
             more_set_headers 'Cache-Control: no-cache, max-age=0, s-max-age=0, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0';
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;