Add support for processing keywords (#43)

* add support for processing keywords

* additional supported processing keywords

* additional allowed processing keywords

* add support for processing keywords

* changelog

* test cutout export

* example showing how to request a cutout

* Update CHANGELOG.rst

Co-authored-by: Nabil Freij <nabil.freij@gmail.com>

* Update CHANGELOG.rst

Co-authored-by: Nabil Freij <nabil.freij@gmail.com>

* Update CHANGELOG.rst

Co-authored-by: Nabil Freij <nabil.freij@gmail.com>

* Update drms/json.py

Co-authored-by: Nabil Freij <nabil.freij@gmail.com>

* Update drms/json.py

* clarify no validation

* test rebin

* link to im_patch docs

* fix rebin test

* Apply suggestions from code review

* Update CHANGELOG.rst

Co-authored-by: Stuart Mumford <stuart@cadair.com>

Co-authored-by: Nabil Freij <nabil.freij@gmail.com>
Co-authored-by: Stuart Mumford <stuart@cadair.com>

Author: 3 people

CHANGELOG.rst

@@ -14,4 +14,11 @@ Backwards Incompatible Changes
 Deprecations and Removals
 -------------------------
 
-- ``Client.get()`` has been removed, use ``Client.query()`` instead.
+- ``Client.get()`` has been removed, use :meth:`drms.client.Client.query()` instead.
+
+Support for Processing Keywords
+--------------------------------
+
+- :meth:`drms.client.Client.export` now accepts a ``process`` keyword argument
+- This allows users to specify additional server-side processing options such as image cutouts
+- See the "Processing" section of the `JSOC Data Export page <http://jsoc.stanford.edu/ajax/exportdata.html>`__ for more information.

drms/client.py

@@ -1138,6 +1138,7 @@ def export(
         n=None,
         email=None,
         requestor=None,
+        process=None,
     ):
         """
         Submit a data export request.
@@ -1188,6 +1189,13 @@ def export(
             current default email address is used, which in this case
             has to be set before calling export() by using the
             :attr:`Client.email` attribute.
+        process : `dict`, None
+            Dictionary of processing commands. Each entry is also a `dict`
+            containing all of the applicable options for that processing
+            command. Note that only the name of the process, and not the
+            arguments, are validated by the `~drms.client.Client`. In the case of invalid
+            or malformed processing arguments, JSOC may still return
+            an unprocessed image without the export request failing.
         requestor : str, None or bool
             Export user ID. Default is None, in which case the user
             name is determined from the email address. If set to False,
@@ -1223,6 +1231,7 @@ def export(
             filenamefmt=filenamefmt,
             n=n,
             requestor=requestor,
+            process=process,
         )
         return ExportRequest(d, client=self)
 

drms/json.py

@@ -284,17 +284,7 @@ def check_address(self, email):
         req = self._json_request(self._server.url_check_address + query)
         return req.data
 
-    def exp_request(
-        self,
-        ds,
-        notify,
-        method='url_quick',
-        protocol='as-is',
-        protocol_args=None,
-        filenamefmt=None,
-        n=None,
-        requestor=None,
-    ):
+    def exp_request(self, *args, **kwargs):
         """
         Request data export.
 
@@ -317,6 +307,10 @@ def exp_request(
         filenamefmt : str, None
             Custom filename format string for exported files. This is
             ignored for 'url_quick'/'as-is' data exports.
+        process : `dict`, None
+            Dictionary of processing commands. Each entry is also a `dict`
+            containing all of the applicable options for that processing
+            command.
         n : int or None
             Limits the number of records requested. For positive
             values, the first n records of the record set are returned,
@@ -334,6 +328,21 @@ def exp_request(
             Dictionary containing the server response to the export
             request.
         """
+        req = self._json_request(self._exp_request_url(*args, **kwargs))
+        return req.data
+
+    def _exp_request_url(
+        self,
+        ds,
+        notify,
+        method='url_quick',
+        protocol='as-is',
+        protocol_args=None,
+        filenamefmt=None,
+        n=None,
+        process=None,
+        requestor=None,
+    ):
         method = method.lower()
         method_list = ['url_quick', 'url', 'url-tar', 'ftp', 'ftp-tar']
         if method not in method_list:
@@ -393,18 +402,34 @@ def exp_request(
         if filenamefmt is not None:
             d['filenamefmt'] = filenamefmt
 
-        if n is not None:
-            n = int(n)
-            d['process=n'] = f'{int(n)}'
+        n = int(n) if n is not None else 0
+        d['process=n'] = f'{n}'
+        if process is not None:
+            allowed_processes = [
+                'im_patch',
+                'resize',
+                'rebin',
+                'aia_scale_aialev1',
+                'aia_scale_orig',
+                'aia_scale_other',
+                'Maproj',
+                'HmiB2ptr',
+            ]
+            process_strings = {}
+            for p, opts in process.items():
+                if p not in allowed_processes:
+                    raise ValueError(f'{p} is not one of the allowed processing options: {allowed_processes}')
+                process_strings[p] = ','.join([f'{k}={v}' for k, v in opts.items()])
+            processes = '|'.join([f'{k},{v}' for k, v in process_strings.items()])
+            d['process=n'] = f'{d["process=n"]}|{processes}'
 
         if requestor is None:
             d['requestor'] = notify.split('@')[0]
         elif requestor is not False:
             d['requestor'] = requestor
 
-        query = f'?{urlencode(d)}'
-        req = self._json_request(self._server.url_jsoc_fetch + query)
-        return req.data
+        query = '?' + urlencode(d)
+        return self._server.url_jsoc_fetch + query
 
     def exp_status(self, requestid):
         """

drms/tests/test_jsoc_export.py

@@ -59,6 +59,93 @@ def test_export_fits_basic(jsoc_client_export):
         assert url.endswith('continuum.fits') or url.endswith('magnetogram.fits')
 
 
+@pytest.mark.jsoc
+@pytest.mark.export
+@pytest.mark.remote_data
+def test_export_im_patch(jsoc_client_export):
+    # TODO: check that this has actually done the export/processing properly?
+    # NOTE: processing exports seem to fail silently on the server side if
+    # the correct names/arguments are not passed. Not clear how to check
+    # that this has not happened.
+    process = {'im_patch': {
+        't_ref': '2015-10-17T04:33:30.000',
+        't': 0,
+        'r': 0,
+        'c': 0,
+        'locunits': 'arcsec',
+        'boxunits': 'arcsec',
+        'x': -517.2,
+        'y': -246,
+        'width': 345.6,
+        'height': 345.6,
+    }}
+    req = jsoc_client_export.export(
+        'aia.lev1_euv_12s[2015-10-17T04:33:30.000/1m@12s][171]{image}',
+        method='url',
+        protocol='fits',
+        process=process,
+        requestor=False,
+    )
+
+    assert isinstance(req, drms.ExportRequest)
+    assert req.wait(timeout=60)
+    assert req.has_succeeded()
+    assert req.protocol == 'fits'
+
+    for record in req.urls.record:
+        record = record.lower()
+        assert record.startswith('aia.lev1_euv_12s_mod')
+
+    for filename in req.urls.filename:
+        assert filename.endswith('image.fits')
+
+    for url in req.urls.url:
+        assert url.endswith('image.fits')
+
+
+@pytest.mark.jsoc
+@pytest.mark.export
+@pytest.mark.remote_data
+def test_export_rebin(jsoc_client_export):
+    # TODO: check that this has actually done the export/processing properly?
+    # NOTE: processing exports seem to fail silently on the server side if
+    # the correct names/arguments are not passed. Not clear how to check
+    # that this has not happened.
+    req = jsoc_client_export.export(
+        'hmi.M_720s[2020-10-17_22:12:00_TAI/24m]{magnetogram}',
+        method='url',
+        protocol='fits',
+        process={'rebin': {'method': 'boxcar', 'scale': 0.25}},
+        requestor=False,
+    )
+
+    assert isinstance(req, drms.ExportRequest)
+    assert req.wait(timeout=60)
+    assert req.has_succeeded()
+    assert req.protocol == 'fits'
+
+    for record in req.urls.record:
+        record = record.lower()
+        assert record.startswith('hmi.m_720s_mod')
+
+    for filename in req.urls.filename:
+        assert filename.endswith('magnetogram.fits')
+
+    for url in req.urls.url:
+        assert url.endswith('magnetogram.fits')
+
+
+@pytest.mark.jsoc
+@pytest.mark.export
+@pytest.mark.remote_data
+def test_export_invalid_process(jsoc_client_export):
+    with pytest.raises(ValueError, match='foobar is not one of the allowed processing options'):
+        jsoc_client_export.export(
+            'aia.lev1_euv_12s[2015-10-17T04:33:30.000/1m@12s][171]{image}',
+            process={'foobar': {}}
+        )
+
+
 @pytest.mark.jsoc
 @pytest.mark.export
 @pytest.mark.remote_data

examples/cutout_export_request.py

@@ -0,0 +1,74 @@
+"""
+===================================
+Exporting data with cutout requests
+===================================
+
+This example shows how to submit a data export request with a cutout request using the ``im_patch`` command.
+"""
+
+import os
+
+import drms
+
+###############################################################################
+# Create DRMS client, uses the JSOC baseurl by default, set debug=True to see the DRMS query URLs.
+
+client = drms.Client(verbose=True)
+
+# This example requires a registered export email address. You can register
+# JSOC exports at: http://jsoc.stanford.edu/ajax/register_email.html
+email = 'nabil.freij@gmail.com'
+
+# Download directory
+out_dir = os.path.join('downloads')
+
+# Create download directory if it does not exist yet.
+if not os.path.exists(out_dir):
+    os.makedirs(out_dir)
+
+###############################################################################
+# Construct the DRMS query string: "Series[harpnum][timespan]{data segments}"
+
+qstr = 'aia.lev1_euv_12s[2015-10-17T04:33:30.000/1m@12s][171]{image}'
+print(f'Data export query:\n  {qstr}\n')
+
+###############################################################################
+# Construct the dictionary specifying that we want to request a cutout.
+# This is done via the ``im_patch`` command.
+# We request a 345.6 arcsecond cutout (on both sides) centered on the coordinate (-517.2, -246) arcseconds as defined in the helioprojective frame of SDO at time ``t_ref``.
+# The ``t`` controls whether tracking is disabled (``1``) or enabled (``0``).
+# ``r`` controls the use of sub-pixel registration.
+# ``c`` controls whether off-limb pixels are filled with NaNs.
+# For additional details about ``im_patch``, see the `documentation <http://jsoc.stanford.edu/doxygen_html/group__im__patch.html>`_.
+process = {'im_patch': {
+    't_ref': '2015-10-17T04:33:30.000',
+    't': 0,
+    'r': 0,
+    'c': 0,
+    'locunits': 'arcsec',
+    'boxunits': 'arcsec',
+    'x': -517.2,
+    'y': -246,
+    'width': 345.6,
+    'height': 345.6,
+}}
+
+# Submit export request using the 'fits' protocol
+print('Submitting export request...')
+result = client.export(
+    qstr,
+    method='url',
+    protocol='fits',
+    email=email,
+    process=process,
+)
+
+# Print request URL.
+print(f'\nRequest URL: {result.request_url}')
+print(f'{int(len(result.urls))} file(s) available for download.\n')
+
+# Download selected files.
+result.wait()
+result.download(out_dir)
+print('Download finished.')
+print(f'\nDownload directory:\n  "{os.path.abspath(out_dir)}"\n')