Merge pull request #237 from pynbody/remote-derived-array

Fix error when mixing derived and loaded arrays (pynbody server mode)

Author: apontzen

.github/workflows/integration-test.yaml

@@ -56,10 +56,9 @@ jobs:
       working-directory: test_tutorial_build
       run: |
         wget https://zenodo.org/record/10051592/files/reference_database.db?download=1 -O reference_database.db -nv
-        tangos diff data.db reference_database.db --property-tolerance dm_density_profile 1e-2 0
+        tangos diff data.db reference_database.db --property-tolerance dm_density_profile 1e-2 0 --property-tolerance gas_map 1e-2 0 --property-tolerance gas_map_sideon 1e-2 0 --property-tolerance gas_map_faceon 1e-2 0
       # --property-tolerance dm_density_profile here is because if a single particle crosses between bins
       # (which seems to happen due to differing library versions), the profile can change by this much
       #
-      # previously had:
+      # Images also occasionally differ by one or two particles. To deal with this previously had:
       # --ignore-value-of gas_map gas_map_faceon gas_map_sideon uvi_image uvi_image_sideon uvi_image_faceon
-      # ignore-value-of above is a clunky fix for the use of 'approximate fast' images -- is it needed any longer?

tangos/blocking.py

@@ -317,10 +317,11 @@ def _load_array(self, array_name, fam=None):
             log.logger.debug("Array received; waited %.2fs",time.time()-start_time)
         except KeyError:
             raise OSError("No such array %r available from the remote"%array_name)
-        if fam is None:
-            self[array_name] = data
-        else:
-            self[fam][array_name] = data
+        with self.auto_propagate_off:
+            if fam is None:
+                self[array_name] = data
+            else:
+                self[fam][array_name] = data
 
 
 _connection_active = False

tangos/parallel_tasks/pynbody_server.py

@@ -1,6 +1,7 @@
 import os
 import sys
 
+import numpy as np
 import numpy.testing as npt
 import pynbody
 
@@ -188,3 +189,24 @@ def _test_oserror_on_nonexistent_file():
 def test_oserror_on_nonexistent_file():
     pt.use("multiprocessing-2")
     pt.launch(_test_oserror_on_nonexistent_file)
+
+
+
+@pynbody.snapshot.tipsy.TipsySnap.derived_quantity
+def metals(sim):
+    """Derived array that will only be invoked for dm, since metals is present on disk for gas/stars"""
+    return pynbody.array.SimArray(np.ones(len(sim)))
+
+def _test_mixed_derived_loaded_arrays():
+    f_remote = handler.load_object('tiny.000640', 1, 1, mode='server')
+    f_local = handler.load_object('tiny.000640', 1, 1, mode=None)
+    assert (f_remote.dm['metals'] == f_local.dm['metals']).all()
+    assert (f_remote.st['metals'] == f_local.st['metals']).all()
+
+
+def test_mixed_derived_loaded_arrays():
+    """Sometimes an array is present on disk for some families but is derived for others. A notable real-world example
+    is the mass array for gas in ramses snapshots. Previously accessing this array in a remotesnap could cause errors,
+    specifically a "derived array is not writable" error on the server. This test ensures that the correct behaviour"""
+    pt.use("multiprocessing-2")
+    pt.launch(_test_mixed_derived_loaded_arrays)