INTEGRITY: Add possible merge button inside filesets dashboard

Author: ShivangNagta

db_functions.py

@@ -947,7 +947,7 @@ def set_process(
 
         # Mac files in set.dat are not represented properly and they won't find a candidate fileset for a match, so we can drop them.
         if len(candidate_filesets) == 0:
-            category_text = "Drop set fileset - A"
+            category_text = "Drop fileset - No Candidates"
             fileset_name = fileset["name"] if "name" in fileset else ""
             fileset_description = (
                 fileset["description"] if "description" in fileset else ""
@@ -988,7 +988,7 @@ def set_process(
 
             for set_fileset in set_filesets:
                 fileset = id_to_fileset_dict[set_fileset]
-                category_text = "Drop set fileset - B"
+                category_text = "Drop fileset - Duplicates"
                 fileset_name = fileset["name"] if "name" in fileset else ""
                 fileset_description = (
                     fileset["description"] if "description" in fileset else ""
@@ -1098,15 +1098,15 @@ def set_perform_match(
                 else:
                     category_text = "Mismatch"
                     log_text = f"Fileset:{fileset_id} mismatched with Fileset:{matched_fileset_id} with status:{status}. Try manual merge."
-                    print(
-                        f"Merge Fileset:{fileset_id} manually with Fileset:{matched_fileset_id}. Unmatched files: {len(unmatched_files)}."
-                    )
+                    print_text = f"Merge Fileset:{fileset_id} manually with Fileset:{matched_fileset_id}. Unmatched files: {len(unmatched_files)}."
                     mismatch_filesets += 1
-                    # print(f"Merge Fileset:{fileset_id} manually with Fileset:{matched_fileset_id}. Unmatched files: {', '.join(filename for filename in unmatched_files)}.")
-                    create_log(
-                        escape_string(category_text),
+                    add_manual_merge(
+                        [matched_fileset_id],
+                        fileset_id,
+                        category_text,
+                        log_text,
+                        print_text,
                         user,
-                        escape_string(log_text),
                         conn,
                     )
 
@@ -1134,11 +1134,16 @@ def set_perform_match(
             if not found_match:
                 category_text = "Manual Merge Required"
                 log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidate_filesets)}."
-                print(log_text)
-                create_log(
-                    escape_string(category_text), user, escape_string(log_text), conn
-                )
                 manual_merged_filesets += 1
+                add_manual_merge(
+                    candidate_filesets,
+                    fileset_id,
+                    category_text,
+                    log_text,
+                    log_text,
+                    user,
+                    conn,
+                )
 
     return (
         fully_matched_filesets,
@@ -1148,6 +1153,26 @@ def set_perform_match(
     )
 
 
+def add_manual_merge(
+    child_filesets, parent_fileset, category_text, log_text, print_text, user, conn
+):
+    """
+    Adds the manual merge entries to a table called possible_merges.
+    """
+    with conn.cursor() as cursor:
+        for child_fileset in child_filesets:
+            query = """
+                    INSERT INTO possible_merges
+                    (child_fileset, parent_fileset)
+                    VALUES
+                    (%s, %s)
+                """
+            cursor.execute(query, (child_fileset, parent_fileset))
+
+    create_log(escape_string(category_text), user, escape_string(log_text), conn)
+    print(print_text)
+
+
 def is_full_checksum_match(candidate_fileset, fileset, conn):
     """
     Return type - (Boolean, List of unmatched files)

fileset.py

@@ -164,6 +164,7 @@ def fileset():
             """
             html += f"<button type='button' onclick=\"location.href='/fileset/{id}/merge'\">Manual Merge</button>"
             html += f"<button type='button' onclick=\"location.href='/fileset/{id}/match'\">Match and Merge</button>"
+            html += f"<button type='button' onclick=\"location.href='/fileset/{id}/possible_merge'\">Possible Merges</button>"
             html += f"""
                     <form action="/fileset/{id}/mark_full" method="post" style="display:inline;">
                         <button type='submit'>Mark as full</button>
@@ -603,6 +604,75 @@ def merge_fileset(id):
     """
 
 
+@app.route("/fileset/<int:id>/possible_merge", methods=["GET", "POST"])
+def possible_merge_filesets(id):
+    base_dir = os.path.dirname(os.path.abspath(__file__))
+    config_path = os.path.join(base_dir, "mysql_config.json")
+    with open(config_path) as f:
+        mysql_cred = json.load(f)
+
+    connection = pymysql.connect(
+        host=mysql_cred["servername"],
+        user=mysql_cred["username"],
+        password=mysql_cred["password"],
+        db=mysql_cred["dbname"],
+        charset="utf8mb4",
+        cursorclass=pymysql.cursors.DictCursor,
+    )
+
+    try:
+        with connection.cursor() as cursor:
+            query = """
+                SELECT
+                    fs.*,
+                    g.name AS game_name,
+                    g.engine AS game_engine,
+                    g.platform AS game_platform,
+                    g.language AS game_language,
+                    g.extra AS extra
+                FROM
+                    fileset fs
+                LEFT JOIN
+                    game g ON fs.game = g.id
+                JOIN
+                    possible_merges pm ON pm.child_fileset = fs.id
+                WHERE pm.parent_fileset = %s
+            """
+            cursor.execute(query, (id,))
+            results = cursor.fetchall()
+
+            html = f"""
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <link rel="stylesheet" type="text/css" href="{{{{ url_for('static', filename='style.css') }}}}">
+            </head>
+            <body>
+            <h2>Possible Merges for fileset-'{id}'</h2>
+            <table>
+            <tr><th>ID</th><th>Game Name</th><th>Platform</th><th>Language</th><th>Extra</th><th>Details</th><th>Action</th></tr>
+            """
+            for result in results:
+                html += f"""
+                <tr>
+                    <td>{result["id"]}</td>
+                    <td>{result["game_name"]}</td>
+                    <td>{result["game_platform"]}</td>
+                    <td>{result["game_language"]}</td>
+                    <td>{result["extra"]}</td>
+                    <td><a href="/fileset?id={result["id"]}">View Details</a></td>
+                    <td><a href="/fileset/{id}/merge/confirm?target_id={result["id"]}">Select</a></td>
+                </tr>
+                """
+            html += "</table>\n"
+            html += "</body>\n</html>"
+
+            return render_template_string(html)
+
+    finally:
+        connection.close()
+
+
 @app.route("/fileset/<int:id>/merge/confirm", methods=["GET", "POST"])
 def confirm_merge(id):
     target_id = (

schema.py

@@ -132,6 +132,15 @@
             fileset INT NOT NULL
         )
     """,
+    "possible_merges": """
+        CREATE TABLE IF NOT EXISTS possible_merges (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            child_fileset INT,
+            parent_fileset INT,
+            FOREIGN KEY (child_fileset) REFERENCES fileset(id) ON DELETE CASCADE,
+            FOREIGN KEY (parent_fileset) REFERENCES fileset(id) ON DELETE CASCADE
+        )
+    """,
 }
 
 for table, definition in tables.items():