4.3 Release

Author: toddbruner

conversion/conversion.md

@@ -2,6 +2,11 @@
 
 This directory contains scripts for migrating data from version 3 of SCOT to version 4. They are grouped into three categories: database migrations, file migrations, and extra (optional) migrations. Three bash shell scripts have been provided for you to run the applicable migrations in each category.
 
+## Necessary tools to run conversion
+You will need the following installed to run conversions:
+- python 3.11 with the contents of requirements-test.txt from this repo installed via pip
+- mysql-shell
+
 ## Database Conversion
 This set of scripts migrates the core database data to the SCOT4 database by pulling data directly from the SCOT3 mongodb database. Almost all SCOT3 installations migrating to SCOT4 will want to do this. The `database_conversion.sh` script will run all of the necessary scripts for you.
 
@@ -45,4 +50,4 @@ The following environment variables should be set when running `file_conversion.
  - SCOT_ADMIN_APIKEY - a SCOT4 API key with admin priveleges (see above for one way to create one)
  - SCOT3_FILE_PREFIX (needed for file migration) - the directory under which the files were stored in the SCOT3 database, this defaults to the default in SCOT3, which was `/opt/scotfiles/`
  - SCOT_FILES_DIR (needed for file migration) - the directory on the current machine in which the old SCOT3 files are stored (with the same file structure that the SCOT3 installation had)
- - SCOT_CACHED_IMAGES_DIR (needed for cached images migration) - the directory on the current machine that contains the SCOT3 cached images in their original file structure (this is usually the /cached_images/ directory in the SCOT3 files)
+ - SCOT_CACHED_IMAGES_DIR (needed for cached images migration) - the directory on the current machine that contains the SCOT3 cached images in their original file structure (this is usually the /cached_images/ directory in the SCOT3 files)

conversion/database_migration/bulk_entries.py

@@ -73,6 +73,9 @@ def main(mongo_db=None, role_lookup=None):
         with tqdm.tqdm(total=scot3_entry_count) as pbar:
             bulk_array = []
             for entry in scot3_entries:
+                if entry.get('target') is None or entry['target'].get('type') is None:
+                    pbar.update(1) # Malformed data
+                    continue
                 if entry['target']['type'] == 'alert' or entry['target']['type'] == 'alertgroup' or entry['target']['type'] == 'feed':
                     pbar.update(1)
                     continue

conversion/database_migration/bulk_links.py

@@ -14,6 +14,9 @@ def main(mongo_db=None):
     scot3_links = mongo_db.link.find()
     with tqdm.tqdm(total=scot3_link_count) as pbar:
         for link in scot3_links:
+            if link.get('vertices') is None:
+                pbar.update(1)
+                continue
             if type(link.get('when')) is int:
                 new_link = [ datetime.fromtimestamp(link['when']).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), link['id'], link['vertices'][1]['type'], link['vertices'][1]['id'], link['vertices'][0]['type'], link['vertices'][0]['id']]
             else:

conversion/database_migration/bulk_roles.py

@@ -28,9 +28,9 @@ def main(mongo_db=None):
         for count, role in enumerate(tqdm.tqdm(unique_roles)):
             if role == '':
                 continue
-            role_row = [count+2, role, 'migrated from SCOT3', datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None)] #Using an offset of 3 since we have two default groups: admin and everyone, created by default and taking up role ids 1 & 2 respectively
+            role_row = [count+3, role, 'migrated from SCOT3', datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None)] #Using an offset of 3 since we have two default groups: admin and everyone, created by default and taking up role ids 1 & 2 respectively
             writer.writerow(role_row)
-            role_lookup[role] = count+2
+            role_lookup[role] = count+3
 
     return role_lookup
 

conversion/database_migration/conversion_utilities.py

@@ -22,6 +22,6 @@ def write_tag_source_links(thing=None, thing_type=None, tag_lookup=None, source_
         if thing_type == "signature":
             # Check if reference ID exists here
             target_data = thing.get('data').get('target')
-            if target_data is not None:
+            if target_data is not None and target_data.get('type') is not None and target_data.get('id') is not None:
                 new_link = [thing_type, thing.get('id'), target_data['type'], target_data['id']]
                 link_csv_writer.writerow(new_link)

conversion/database_migration/initial_scot4_database.sql

@@ -271,6 +271,10 @@ CREATE TABLE `auth_settings` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
+LOCK TABLES `auth_settings` WRITE;
+INSERT INTO `auth_settings` VALUES (1, 'local', '{}', 1, '2023-06-06 16:53:27','2023-06-06 16:53:27');
+UNLOCK TABLES;
+
 --
 -- Table structure for table `auth_storage`
 --
@@ -840,9 +844,9 @@ LOCK TABLES `links` WRITE;
 /*!40000 ALTER TABLE `links` ENABLE KEYS */;
 UNLOCK TABLES;
 
----
---- Table structure for table `metrics`
----
+--
+-- Table structure for table `metrics`
+--
 
 DROP TABLE IF EXISTS `metrics`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;

conversion/database_migration/scot3_scot4_tsv_import.py

@@ -74,4 +74,4 @@
     util.import_table(f"{staging_directory}/entity_class_associations.csv", {"table": "entity_class_entity_association", "columns": ['entity_id', 'entity_class_id'], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})    
 
     if os.path.isfile("./games.csv"):
-        util.import_table("./games.csv", { "table": "games", "columns": [], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})
+        util.import_table("./games.csv", { "table": "games", "columns": ["game_id", "game_name", "tooltip", "results", "created_date", "modified_date"], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})

conversion/extra_migration/update_admin_password_and_api_key.py

@@ -8,7 +8,7 @@ def main(db_session):
     new_admin_apikey = os.getenv('SCOT_ADMIN_APIKEY')
     db_obj = db_session.query(User).filter(User.username=='scot-admin').one_or_none()
     if new_admin_pw:
-        update_dict={'username':'scot-admin', 'password':os.environ['SCOT_ADMIN_PASSWORD'])
+        update_dict={'username':'scot-admin', 'password':os.environ['SCOT_ADMIN_PASSWORD']}
         crud.user.update(db_session=db_session, db_obj=db_obj, obj_in=update_dict)
     else:
         print('SCOT_ADMIN_PASSWORD not set, not resetting scot-admin password')

conversion/file_migration/migrate_cached_images.py

@@ -9,6 +9,9 @@
 import pathlib
 import re
 import urllib3
+
+from app.db.session import SessionLocal
+from app.models import Entry
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 def rewrite_cached_images(html, flaired_html, entry_id):

requirements-test.txt

@@ -7,4 +7,5 @@ tqdm==4.66.5
 pymongo==4.8.0
 splunk-sdk==2.0.2
 pytest-xdist==3.6.1
+Faker==27.0.0
 -r requirements.txt

requirements.txt

@@ -1,12 +1,13 @@
 fastapi==0.112.1
 SQLAlchemy==2.0.32
+pydantic_core==2.23.4
 pydantic_settings==2.4.0
+pydantic==2.9.2
 mysql-connector-python==9.1.0
 meilisearch==0.31.4
 msal==1.30.0
 python-dateutil==2.9.0.post0
 nh3==0.2.18
-Faker==27.0.0
 python-jose==3.3.0
 loguru==0.7.2
 email_validator==2.0.0

src/app/api/__init__.py

@@ -1,24 +1,13 @@
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.openapi.utils import get_openapi
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from app.api.endpoints import router
 from app.core.config import settings
 from app.db.session import SessionLocal
 
 
-def register_cors(app: FastAPI) -> None:
-    """ """
-    app.add_middleware(
-        CORSMiddleware,
-        allow_origins=["http://localhost:8080", "http://127.0.0.1:8080"],  # Remember we need to
-        allow_origin_regex=r"moz-extension:\/\/.*",  # Needed to make the Plugin function properly in Firefox, works fine in Chrome/Edge... no idea why
-        allow_credentials=True,
-        allow_methods=["*"],
-        allow_headers=["*"],
-    )
-
-
 async def db_middleware(request: Request, call_next) -> None:
     """
     Middleware that wraps each API call in a transaction
@@ -40,11 +29,6 @@ async def db_middleware(request: Request, call_next) -> None:
     return response
 
 
-def register_router(app: FastAPI) -> None:
-    """ """
-    app.include_router(router.api_router, prefix=settings.API_V1_STR)
-
-
 def create_app() -> FastAPI:
     """
     :return:
@@ -56,10 +40,46 @@ def create_app() -> FastAPI:
         docs_url=settings.DOCS_URL,
         openapi_url=settings.OPENAPI_URL,
         redoc_url=settings.REDOC_URL,
+        swagger_ui_parameters={
+            "filter": True
+        }
+    )
+
+    app.include_router(router.api_router, prefix=settings.API_V1_STR)
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:8080", "http://127.0.0.1:8080"],  # Remember we need to
+        allow_origin_regex=r"moz-extension:\/\/.*",  # Needed to make the Plugin function properly in Firefox, works fine in Chrome/Edge... no idea why
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
     )
 
-    register_cors(app)
-    register_router(app)
     app.add_middleware(BaseHTTPMiddleware, dispatch=db_middleware)
 
+    if not app.openapi_schema:
+        # will need to update all links for open source stuff
+        openapi_schema = get_openapi(
+            title="SCOT",
+            version="4.2.0",
+            summary="Sandia Cyber Omni Tracker Server",
+            description="The Sandia Cyber Omni Tracker (SCOT) is a cyber security incident response management system and knowledge base. Designed by cyber security incident responders, SCOT provides a new approach to manage security alerts, analyze data for deeper patterns, coordinate team efforts, and capture team knowledge. SCOT integrates with existing security applications to provide a consistent, easy to use interface that enhances analyst effectiveness.",
+            routes=app.routes,
+            # terms_of_service?
+            contact={
+                "name": "SCOT Development Team",
+                "url": "http://getscot.sandia.gov",
+                "email": "scot-dev@sandia.gov"
+            },
+            license_info={
+                "name": "LICENSE",
+                "identifier": "Apache-2.0",
+            },
+        )
+        openapi_schema["info"]["x-logo"] = {
+            "url": "https://sandialabs.github.io/scot4-docs/images/scot.png"
+        }
+        app.openapi_schema = openapi_schema
+
     return app

src/app/api/deps.py

@@ -96,6 +96,8 @@ def get_current_active_user(
     if not crud.user.is_active(current_user):
         raise HTTPException(status_code=400, detail="Inactive user")
     crud.user.update_last_activity(db, current_user)
+    # add user id to the session for some queries
+    db.info["user_id"] = current_user.id
     db.commit()  # Done here to prevent race conditions
     return current_user
 

src/app/api/endpoints/alert.py

@@ -16,23 +16,25 @@
     generic_post,
     generic_undelete,
     generic_history,
-    generic_search
+    generic_search,
+    generic_upvote_and_downvote,
 )
 
 router = APIRouter()
-description, examples = create_schema_details(schemas.AlertUpdate)
 
 # Create get, post, put, and delete endpoints
 generic_get(router, crud.alert, TargetTypeEnum.alert, schemas.Alert)
-generic_post(
-    router, crud.alert, TargetTypeEnum.alert, schemas.Alert, schemas.AlertCreate
-)
+generic_post(router, crud.alert, TargetTypeEnum.alert, schemas.Alert, schemas.AlertCreate)
 generic_delete(router, crud.alert, TargetTypeEnum.alert, schemas.Alert)
 generic_undelete(router, crud.alert, TargetTypeEnum.alert, schemas.Alert)
 generic_entries(router, TargetTypeEnum.alert)
 generic_entities(router, TargetTypeEnum.alert)
 generic_search(router, crud.alert, TargetTypeEnum.alert, schemas.AlertSearch, schemas.Alert)
 generic_history(router, crud.alert, TargetTypeEnum.alert)
+generic_upvote_and_downvote(router, crud.alert, TargetTypeEnum.alert, schemas.Alert)
+
+
+description, examples = create_schema_details(schemas.AlertUpdate)
 
 
 # Custom PUT so that you can modify alerts if you have access to the alertgroup
@@ -71,9 +73,7 @@ def update_alert(
         raise HTTPException(422, f"Validation error: {e}")
 
     try:
-        updated = crud.alert.update(
-            db_session=db, db_obj=_obj, obj_in=obj, audit_logger=audit_logger
-        )
+        updated = crud.alert.update(db, _obj, obj, audit_logger)
     except ValueError as e:
         raise HTTPException(422, f"Error when updating alert: {e}")
 

src/app/api/endpoints/alertgroup.py

@@ -17,52 +17,30 @@
     generic_history,
     generic_reflair,
     generic_search,
-    generic_export
+    generic_export,
+    generic_upvote_and_downvote,
+    generic_user_links
 )
 
 router = APIRouter()
-description, examples = create_schema_details(schemas.AlertAdd)
 
 # Create get, post, put, and delete endpoints
 generic_export(router, crud.alert_group, TargetTypeEnum.alertgroup)
-generic_get(
-    router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed
-)
-
-generic_post(
-    router,
-    crud.alert_group,
-    TargetTypeEnum.alertgroup,
-    schemas.AlertGroupDetailed,
-    schemas.AlertGroupDetailedCreate,
-)
-generic_put(
-    router,
-    crud.alert_group,
-    TargetTypeEnum.alertgroup,
-    schemas.AlertGroupDetailed,
-    schemas.AlertGroupUpdate,
-)
-generic_delete(
-    router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed
-)
-generic_undelete(
-    router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed
-)
+generic_get(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
+generic_post(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed, schemas.AlertGroupDetailedCreate)
+generic_put(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed, schemas.AlertGroupUpdate)
+generic_delete(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
+generic_undelete(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
 generic_entities(router, TargetTypeEnum.alertgroup)
 generic_history(router, crud.alert_group, TargetTypeEnum.alertgroup)
-generic_reflair(
-    router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed
-)
+generic_reflair(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
 generic_search(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupSearch, schemas.AlertGroup)
+generic_upvote_and_downvote(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroup)
+generic_user_links(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
 
 
-alertgroup_read_dep = Depends(
-    deps.PermissionCheckId(TargetTypeEnum.alertgroup, PermissionEnum.read)
-)
-alertgroup_modify_dep = Depends(
-    deps.PermissionCheckId(TargetTypeEnum.alertgroup, PermissionEnum.modify)
-)
+alertgroup_read_dep = Depends(deps.PermissionCheckId(TargetTypeEnum.alertgroup, PermissionEnum.read))
+alertgroup_modify_dep = Depends(deps.PermissionCheckId(TargetTypeEnum.alertgroup, PermissionEnum.modify))
 
 
 @router.get(
@@ -75,18 +53,20 @@ def read_alertgroup_alerts(
     *,
     db: Session = Depends(deps.get_db),
     id: Annotated[int, Path(...)],
-    current_user: models.User = Depends(deps.get_current_active_user),
     audit_logger: deps.AuditLogger = Depends(deps.get_audit_logger),
 ) -> Any:
     """
     Get all alerts for this alertgroup
     """
-    _alert_group = crud.alert_group.get(db_session=db, _id=id, audit_logger=audit_logger)
+    _alert_group = crud.alert_group.get(db, id, audit_logger)
     if not _alert_group:
         raise HTTPException(404, "Alert with id %s not found" % id)
     return _alert_group.alerts
 
 
+description, examples = create_schema_details(schemas.AlertAdd)
+
+
 @router.post(
     "/{id}/alerts",
     response_model=schemas.Alert,
@@ -99,16 +79,15 @@ def add_alertgroup_alert(
     db: Session = Depends(deps.get_db),
     id: Annotated[int, Path(...)],
     alert: Annotated[schemas.AlertAdd, Body(..., openapi_examples=examples)],
-    current_user: models.User = Depends(deps.get_current_active_user),
+    _: models.User = Depends(deps.get_current_active_user),
     audit_logger: deps.AuditLogger = Depends(deps.get_audit_logger),
 ) -> Any:
     """
     Add an alert to an alertgroup
     """
     try:
         alert.alertgroup_id = id
-        _alert = crud.alert.add_to_alert_group(db_session=db, obj_in=alert, audit_logger=audit_logger)
-        return _alert
+        return crud.alert.add_to_alert_group(db_session=db, obj_in=alert, audit_logger=audit_logger)
     except Exception as e:
         raise HTTPException(status_code=422, detail=str(e))
 
@@ -124,6 +103,7 @@ def add_alertgroup_column(
     db: Session = Depends(deps.get_db),
     id: int,
     column_name: str = Body(...),
-    values: Dict[str, str] = Body({})
+    values: Dict[str, str] = Body({}),
+    current_user: models.User = Depends(deps.get_current_active_user)
 ) -> Any:
-    return crud.alert_group.add_column(db, id, column_name, values)
+    return crud.alert_group.add_column(db, id, column_name, values, current_user=current_user)