Update connector.py

Author: raphaeljafriLB

labelpandas/connector.py

@@ -25,7 +25,10 @@ def create_data_rows(
         Two items - the global_key, and a dictionary with "row_data", "global_key", "external_id" and "metadata_fields" keys
     """
     row_data = lb_client.upload_file(str(row[row_data_col])) if local_files else str(row[row_data_col])
-    data_row_dict = {"row_data" : row_data, "global_key" : str(row[global_key_col]), "external_id" : row[external_id_col], "metadata_fields" : [{"schema_id" : metadata_name_key_to_schema['lb_integration_source', "value" : "Pandas"]}]}
+    data_row_dict = {
+        "row_data" : row_data, "global_key" : str(row[global_key_col]), "external_id" : row[external_id_col], 
+        "metadata_fields" : [{"schema_id" : metadata_name_key_to_schema['lb_integration_source', "value" : "Pandas"]}]
+    }
     if metadata_index:
         for metadata_field_name in metadata_index.keys():
             name_key = f"{metadata_field_name}{divider}{row[metadata_field_name]}"
@@ -42,3 +45,24 @@ def get_unique_values_function(table, column_name:str):
 def add_column_function(table, column_name:str):
     table[column_name] = ""
     return table
+
+def create_upload_dict(table, local_files, lb_client, row, row_data_col, global_key_col, external_id_col, metadata_index, divider):
+    global_key_col = global_key_col if global_key_col else row_data_col
+    external_id_col = external_id_col if external_id_col else global_key_col       
+    metadata_schema_to_name_key = get_metadata_schema_to_name_key(lb_mdo=False, divider=divider, invert=False)
+    metadata_name_key_to_schema = get_metadata_schema_to_name_key(lb_mdo=False, divider=divider, invert=True) 
+    global_key_to_upload_dict = {}
+    futures = []
+    with ThreadPoolExecutor() as exc:
+        for index, row in table.iterrows():
+            futures.append(
+                exc.submit(
+                    connector.create_data_rows, local_files, self.lb_client, row, row_data_col, 
+                    global_key_col, external_id_col, metadata_index, metadata_name_key_to_schema,
+                    metadata_schema_to_name_key, divider
+                )
+            )
+        for f in as_completed(futures):
+            res = f.result()
+            global_key_to_upload_dict[str(res["global_key"])] = res    
+    return global_key_to_upload_dict