|
1 | 1 | from labelbase import Client as baseClient
|
2 | 2 | from labelbox import Client
|
3 | 3 | import pandas
|
4 |
| -from tqdm.autonotebook import tqdm |
5 | 4 | from concurrent.futures import ThreadPoolExecutor, as_completed
|
6 | 5 |
|
7 | 6 | def create_upload_dict(df:pandas.core.frame.DataFrame, lb_client:Client, base_client:baseClient, row_data_col:str,
|
@@ -30,20 +29,17 @@ def create_upload_dict(df:pandas.core.frame.DataFrame, lb_client:Client, base_cl
|
30 | 29 | global_key_to_upload_dict = {}
|
31 | 30 | futures = []
|
32 | 31 | with ThreadPoolExecutor() as exc:
|
33 |
| - for index, row in df.iterrows(): |
34 |
| - futures.append(exc.submit(create_data_rows, lb_client, base_client, row, metadata_name_key_to_schema, metadata_schema_to_name_key, row_data_col, global_key_col, external_id_col, metadata_index, local_files, divider)) |
35 | 32 | if verbose:
|
36 | 33 | print(f'Processing data rows...')
|
37 |
| - x = 0 |
38 |
| - for f in tqdm(as_completed(futures)): |
39 |
| - x+=1 |
| 34 | + for index, row in df.iterrows(): |
| 35 | + futures.append(exc.submit(create_data_rows, lb_client, base_client, row, metadata_name_key_to_schema, metadata_schema_to_name_key, row_data_col, global_key_col, external_id_col, metadata_index, local_files, divider)) |
| 36 | + x = 0 |
| 37 | + for f in as_completed(futures): |
| 38 | + if verbose: |
| 39 | + x += 1 |
40 | 40 | print(f"On Data Row number {x}")
|
41 |
| - res = f.result() |
42 |
| - global_key_to_upload_dict[str(res["global_key"])] = res |
43 |
| - else: |
44 |
| - for f in as_completed(futures): |
45 |
| - res = f.result() |
46 |
| - global_key_to_upload_dict[str(res["global_key"])] = res |
| 41 | + res = f.result() |
| 42 | + global_key_to_upload_dict[str(res["global_key"])] = res |
47 | 43 | if verbose:
|
48 | 44 | print(f'Generated upload list - {len(global_key_to_upload_dict)} data rows to upload')
|
49 | 45 | return global_key_to_upload_dict
|
|
0 commit comments