@@ -1485,6 +1485,7 @@ def items_and_annotation_chip_generator(
1485
1485
stride_size : int ,
1486
1486
cache_directory : str ,
1487
1487
query : Optional [str ] = None ,
1488
+ num_processes : int = 0 ,
1488
1489
) -> Iterable [Dict [str , str ]]:
1489
1490
"""Provides a generator of chips for all DatasetItems and BoxAnnotations in the dataset.
1490
1491
@@ -1498,6 +1499,7 @@ def items_and_annotation_chip_generator(
1498
1499
cache_directory: The s3 or local directory to store the image and annotations of a chip.
1499
1500
s3 directories must be in the format s3://s3-bucket/s3-key
1500
1501
query: Structured query compatible with the `Nucleus query language <https://nucleus.scale.com/docs/query-language-reference>`_.
1502
+ num_processes: The number of worker processes to use to chip and upload images. If unset, no parallel processing will occur.
1501
1503
1502
1504
Returns:
1503
1505
Generator where each element is a dict containing the location of the image chip (jpeg) and its annotations (json).
@@ -1522,22 +1524,26 @@ def items_and_annotation_chip_generator(
1522
1524
annotations = item [BOX_TYPE ]
1523
1525
item_ref_id = item [ITEM_KEY ][REFERENCE_ID_KEY ]
1524
1526
offsets = generate_offsets (w , h , chip_size , stride_size )
1525
- with Pool () as pool :
1526
- chip_args = [
1527
- (
1528
- offset ,
1529
- chip_size ,
1530
- w ,
1531
- h ,
1532
- item_ref_id ,
1533
- cache_directory ,
1534
- image ,
1535
- annotations ,
1536
- )
1537
- for offset in offsets
1538
- ]
1539
- for chip_result in pool .imap (process_chip , chip_args ):
1540
- yield chip_result
1527
+ chip_args = [
1528
+ (
1529
+ offset ,
1530
+ chip_size ,
1531
+ w ,
1532
+ h ,
1533
+ item_ref_id ,
1534
+ cache_directory ,
1535
+ image ,
1536
+ annotations ,
1537
+ )
1538
+ for offset in offsets
1539
+ ]
1540
+ if num_processes :
1541
+ with Pool (num_processes ) as pool :
1542
+ for chip_result in pool .imap (process_chip , chip_args ):
1543
+ yield chip_result
1544
+ else :
1545
+ for chip_arg in chip_args :
1546
+ yield process_chip (chip_arg )
1541
1547
1542
1548
def export_embeddings (
1543
1549
self ,
0 commit comments