diff --git a/cloudvolume/__init__.py b/cloudvolume/__init__.py index fdacfed2..6462eefb 100644 --- a/cloudvolume/__init__.py +++ b/cloudvolume/__init__.py @@ -65,7 +65,7 @@ from . import exceptions from . import secrets -__version__ = '12.1.1' +__version__ = '12.2.0' # Register plugins from .datasource.precomputed import register as register_precomputed diff --git a/cloudvolume/datasource/precomputed/image/tx.py b/cloudvolume/datasource/precomputed/image/tx.py index 03394b94..f67f01ae 100644 --- a/cloudvolume/datasource/precomputed/image/tx.py +++ b/cloudvolume/datasource/precomputed/image/tx.py @@ -205,17 +205,19 @@ def upload_aligned( secrets=secrets ) - parallel_execution( - cup, chunk_ranges, parallel, - progress, desc="Upload", - cleanup_shm=location - ) - - # If manual mode is enabled, it's the - # responsibilty of the user to clean up - if not use_shared_memory: - array_like.close() - shm.unlink(location) + try: + parallel_execution( + cup, chunk_ranges, parallel, + progress, desc="Upload", + cleanup_shm=location + ) + finally: + # If manual mode is enabled, it's the + # responsibilty of the user to clean up + if not use_shared_memory: + del renderbuffer + array_like.close() + shm.unlink(location) def child_upload_process( meta, cache, @@ -267,6 +269,7 @@ def updatefn(): secrets=secrets, ) finally: + del renderbuffer array_like.close() def threaded_upload_chunks( diff --git a/cloudvolume/frontends/precomputed.py b/cloudvolume/frontends/precomputed.py index d0d715df..b3637596 100644 --- a/cloudvolume/frontends/precomputed.py +++ b/cloudvolume/frontends/precomputed.py @@ -1129,7 +1129,8 @@ def upload_from_shared_memory(self, location, bbox, order='F', cutout_bbox=None) order=order, use_shared_memory=True, ) - mmap_handle.close() + del shared_image + # mmap_handle.close() def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): """ @@ -1190,6 +1191,7 @@ def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): order=order, use_file=True, ) + del shared_image mmap_handle.close() def viewer(self, port=1337): diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 79aaee56..75e6448e 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -127,8 +127,7 @@ def allocate_shm_file(filename, nbytes, dbytes, readonly): def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): """Create a shared memory numpy array. Requires /dev/shm to exist.""" - import posix_ipc - from posix_ipc import O_CREAT + from multiprocessing import shared_memory import psutil nbytes = Vec(*shape).rectVolume() * np.dtype(dtype).itemsize @@ -170,21 +169,19 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): # a threading condition where the condition of the shared memory # was adjusted between the check above and now. Better to make sure # that we don't accidently change anything if readonly is set. - flags = 0 if readonly else O_CREAT size = 0 if readonly else int(nbytes) try: - shared = posix_ipc.SharedMemory(location, flags=flags, size=size) - array_like = mmap.mmap(shared.fd, shared.size) - os.close(shared.fd) - renderbuffer = np.ndarray(buffer=array_like, dtype=dtype, shape=shape, order=order, **kwargs) + shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) + renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) + renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: if err.errno == errno.ENOMEM: # Out of Memory - posix_ipc.unlink_shared_memory(location) + unlink_shm(location) raise renderbuffer.setflags(write=(not readonly)) - return array_like, renderbuffer + return shm, renderbuffer def unlink(location): if EMULATE_SHM: @@ -192,10 +189,11 @@ def unlink(location): return unlink_shm(location) def unlink_shm(location): - import posix_ipc + from multiprocessing import shared_memory try: - posix_ipc.unlink_shared_memory(location) - except posix_ipc.ExistentialError: + shm = shared_memory.SharedMemory(name=location, create=False) + shm.unlink() + except FileNotFoundError: return False return True diff --git a/setup.py b/setup.py index 4f65e7fb..89aabe16 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def requirements(): setuptools.setup( name="cloud_volume", - version="12.1.1", + version="12.2.0", setup_requires=[ 'numpy<1.17; python_version<"3.5"', 'numpy; python_version>="3.5"', @@ -50,7 +50,6 @@ def requirements(): "blosc", ], ':sys_platform!="win32"': [ - "posix_ipc>=1.0.4", "psutil>=5.4.3", ], "mesh_viewer": [ 'vtk' ],