Add ability to ignore the local cache (#119)

Fixes #44

Adds `ignore_cache` in various places to allow for the ignoring of cache. Useful for debugging and demos

Author: gordonwatts

.vscode/settings.json

@@ -79,5 +79,8 @@
         "xaod",
         "xrootd"
     ],
-    "python.analysis.typeCheckingMode": "basic"
+    "python.analysis.typeCheckingMode": "basic",
+    "python.testing.pytestArgs": [
+        "--no-cov"
+    ]
 }

README.md

@@ -90,6 +90,47 @@ If you'd like to be able to submit multiple queries and have them run on the `Se
 
 For documentation of `get_data` and `get_data_async` see the `servicex.py` source file.
 
+### The Local Cache
+
+To speed things up - especially when you run the same query multiple times, the `servicex` package will cache queries data that comes back from Servicex. You can control where this is stored with the `cache_path` in the `.servicex` file (see below).
+
+There are times when you want the system to ignore the cache when it is running. You can do this by using `ignore_cache()`:
+
+```python
+from servicex import ignore_cache
+
+with ignore_cache():
+  do_query():
+```
+
+If you are using a Jupyter notebook, the `with` statement can't really span cells. So use `ignore_cache().__enter__()` instead. Or you can do something like:
+
+```python
+from servicex import ignore_cache
+
+ic = ignore_cache()
+ic.__enter__()
+
+...
+
+ic.__exit__(None, None, None)
+```
+
+If you wish to disable the cache for a single dataset, use the `ignore_cache` parameter when you create it:
+
+```python
+ds = ServiceXDataset(dataset, ignore_cache=True)
+```
+
+Finally, you can ignore the cache for a dataset for a short period of time by using the same context manager pattern:
+
+```python
+ds = ServiceXData(dataset)
+with ds.ignore_cache():
+  do_query(ds)  # Cache is ignored
+do_query(ds)  # Cache is not ignored
+```
+
 ## Configuration
 
 As mentioned above, the `.servicex` file is read to pull a configuration. The search path for this file:

servicex/__init__.py

@@ -10,4 +10,4 @@
 )
 from .servicex_adaptor import ServiceXAdaptor  # NOQA
 from .minio_adaptor import MinioAdaptor  # NOQA
-from .cache import Cache  # NOQA
+from .cache import Cache, ignore_cache  # NOQA

servicex/cache.py

@@ -1,14 +1,53 @@
 import json
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Any
+from contextlib import contextmanager
 
 from .utils import ServiceXException, _query_cache_hash, sanitize_filename
 
+_ignore_cache = False
+
+
+@contextmanager
+def ignore_cache():
+    '''This will cause all caches to be ignored while it is invoked:
+
+    ```
+    with ignore_cache():
+        ServiceXDataset(...).get_data...()
+    ```
+
+    If you want to do this globally, you can just use the `__enter__()` method.
+    This is probably the only way to do this accross cells in a notebook.
+
+    ```
+    i = ignore_cache()
+    i.__enter__()
+    ... Query code, jupyter notebook cells, etc. go here
+    i.__exit(None, None, None)
+    ```
+
+    Note:
+
+    - The only time the cache is checked is when the query is actually made, not when
+      the servicex dataset object is created!
+    - Calls to this can be safely nested.
+    - Note that calling this doesn't clear the cache or delete anything. It
+      just prevents the cache lookup from working while it is in effect.
+    '''
+    global _ignore_cache
+    old_value = _ignore_cache
+    _ignore_cache = True
+    yield
+    _ignore_cache = old_value
+
 
 class Cache:
     '''
     Caching for all data returns from the system. It provides both in-memory
     and on-disk cache.
+
+    TODO: Rename this to be an adaptor, unifying how we name things
     '''
     _in_memory_cache = {}
 
@@ -17,22 +56,34 @@ def reset_cache(cls):
         'Reset the internal cache, usually used for testing'
         cls._in_memory_cache = {}
 
-    def __init__(self, cache_path: Path):
+    def __init__(self, cache_path: Path, ignore_cache: bool = False):
         '''
         Create the cache object
 
         Arguments:
 
             cache_path          The path to the cache directory. Only sub-directories
                                 will be created in this path.
+            ignore_cache        If true, then always ignore the cache for any queries
+                                against this dataset.
         '''
         self._path = cache_path
+        self._ignore_cache = ignore_cache
 
     @property
     def path(self) -> Path:
         'Return root path of cache directory'
         return self._path
 
+    @contextmanager
+    def ignore_cache(self):
+        '''Ignore the cache as long as we are held. Supports nesting.
+        '''
+        old_ignore = self._ignore_cache
+        self._ignore_cache = True
+        yield
+        self._ignore_cache = old_ignore
+
     def _query_cache_file(self, json: Dict[str, str]) -> Path:
         'Return the query cache file'
         h = _query_cache_hash(json)
@@ -47,6 +98,10 @@ def _files_cache_file(self, id: str) -> Path:
         return self._path / 'file_list_cache' / id
 
     def lookup_query(self, json: Dict[str, str]) -> Optional[str]:
+        global _ignore_cache
+        if _ignore_cache or self._ignore_cache:
+            return None
+
         f = self._query_cache_file(json)
         if not f.exists():
             return None
@@ -108,6 +163,10 @@ def set_inmem(self, id: str, v: Any):
         self._in_memory_cache[id] = v
 
     def lookup_inmem(self, id: str) -> Optional[Any]:
+        global _ignore_cache
+        if _ignore_cache or self._ignore_cache:
+            return None
+
         if id not in self._in_memory_cache:
             return None
         return self._in_memory_cache[id]

servicex/servicex.py

@@ -47,7 +47,8 @@ def __init__(self,
                  local_log: log_adaptor = None,
                  session_generator: Callable[[], Awaitable[aiohttp.ClientSession]] = None,
                  config_adaptor: Optional[ServiceXConfigAdaptor] = None,
-                 data_convert_adaptor: Optional[DataConverterAdaptor] = None):
+                 data_convert_adaptor: Optional[DataConverterAdaptor] = None,
+                 ignore_cache: bool = False):
         '''
         Create and configure a ServiceX object for a dataset.
 
@@ -81,6 +82,9 @@ def __init__(self,
             data_convert_adaptor        Manages conversions between root and parquet and `pandas`
                                         and `awkward`, including default settings for expected
                                         datatypes from the backend.
+            ignore_cache                Always ignore the cache on any query for this dataset. This
+                                        is only meaningful if no cache adaptor is provided. Defaults
+                                        to false - the cache is used if possible.
 
         Notes:
 
@@ -101,7 +105,7 @@ def __init__(self,
             else ServiceXConfigAdaptor()
 
         # Establish the cache that will store all our queries
-        self._cache = Cache(get_configured_cache_path(config.settings)) \
+        self._cache = Cache(get_configured_cache_path(config.settings), ignore_cache) \
             if cache_adaptor is None \
             else cache_adaptor
 
@@ -127,6 +131,15 @@ def __init__(self,
         self._converter = data_convert_adaptor if data_convert_adaptor is not None \
             else DataConverterAdaptor(config.get_default_returned_datatype(backend_type))
 
+    def ignore_cache(self):
+        '''Return a context manager that, as long as it is held, will cause any queries against just
+        this dataset to ignore any locally cached data.
+
+        Returns:
+            ContextManager: As long as this is held, the local query cache will be ignored.
+        '''
+        return self._cache.ignore_cache()
+
     @functools.wraps(ServiceXABC.get_data_rootfiles_async, updated=())
     @_wrap_in_memory_sx_cache
     async def get_data_rootfiles_async(self, selection_query: str) -> List[Path]:

tests/test_cache.py

@@ -1,7 +1,7 @@
 from servicex.utils import ServiceXException
 import pytest
 
-from servicex.cache import Cache
+from servicex.cache import Cache, ignore_cache
 
 
 @pytest.fixture()
@@ -27,6 +27,26 @@ def test_query_hit_1(tmp_path):
     assert c.lookup_query({'hi': 'there'}) == 'dude'
 
 
+def test_ic_query(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    with ignore_cache():
+        assert c.lookup_query({'hi': 'there'}) is None
+
+
+def test_ic_query_query_context(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    with c.ignore_cache():
+        assert c.lookup_query({'hi': 'there'}) is None
+
+
+def test_ic_query_ds_level(tmp_path):
+    c = Cache(tmp_path, ignore_cache=True)
+    c.set_query({'hi': 'there'}, 'dude')
+    assert c.lookup_query({'hi': 'there'}) is None
+
+
 def test_query_hit_2(tmp_path):
     c = Cache(tmp_path)
     c.set_query({'hi': 'there'}, 'dude1')
@@ -61,6 +81,14 @@ def test_files_hit(tmp_path):
     assert c.lookup_files('1234') == [['hi', '1'], ['there', '1']]
 
 
+def test_ic_files_hit(tmp_path):
+    'The file list should not be affected by cache ignores'
+    c = Cache(tmp_path)
+    c.set_files('1234', [('hi', '1'), ('there', '1')])
+    with ignore_cache():
+        assert c.lookup_files('1234') == [['hi', '1'], ['there', '1']]
+
+
 def test_files_hit_reloaded(tmp_path):
     c1 = Cache(tmp_path)
     c1.set_files('1234', [('hi', '1'), ('there', '1')])
@@ -80,6 +108,29 @@ def test_memory_hit(tmp_path):
     assert c.lookup_inmem('dude') is r
 
 
+def test_ic_memory_hit(tmp_path):
+    c = Cache(tmp_path)
+    r = 10
+    c.set_inmem('dude', r)
+    with ignore_cache():
+        assert c.lookup_inmem('dude') is None
+
+
+def test_ic_memory_hit_ds_context(tmp_path):
+    c = Cache(tmp_path)
+    r = 10
+    c.set_inmem('dude', r)
+    with c.ignore_cache():
+        assert c.lookup_inmem('dude') is None
+
+
+def test_ic_memory_hit_ds_level(tmp_path):
+    c = Cache(tmp_path, ignore_cache=True)
+    r = 10
+    c.set_inmem('dude', r)
+    assert c.lookup_inmem('dude') is None
+
+
 def test_memory_hit_accross(tmp_path):
     c1 = Cache(tmp_path)
     r = 10
@@ -131,3 +182,49 @@ def test_query_cache_status_bad(tmp_path):
 
     with pytest.raises(ServiceXException):
         c.lookup_query_status('111-222-333')
+
+
+def test_ic_query_cache_status(tmp_path):
+    'Query status should be cached and accessed *during* a query'
+    c = Cache(tmp_path)
+    info = {'request_id': '111-222-333', 'key': 'bogus'}
+    c.set_query_status(info)
+    with ignore_cache():
+        info1 = c.lookup_query_status('111-222-333')
+        assert info1['key'] == 'bogus'
+
+
+def test_ic_restore(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    with ignore_cache():
+        pass
+    assert c.lookup_query({'hi': 'there'}) == 'dude'
+
+
+def test_ic_nesting(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    with ignore_cache():
+        with ignore_cache():
+            pass
+        assert c.lookup_query({'hi': 'there'}) is None
+
+
+def test_ic_nesting_ds_context(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    with c.ignore_cache():
+        with c.ignore_cache():
+            pass
+        assert c.lookup_query({'hi': 'there'}) is None
+
+
+def test_ic_enter_exit(tmp_path):
+    c = Cache(tmp_path)
+    c.set_query({'hi': 'there'}, 'dude')
+    i = ignore_cache()
+    i.__enter__()
+    assert c.lookup_query({'hi': 'there'}) is None
+    i.__exit__(None, None, None)
+    assert c.lookup_query({'hi': 'there'}) == 'dude'