Redo logger and fix examples (#113)

* Redo logger and fix examples

* Experimental change

* Incorrectly setup logger

* Changelog

* Tweak tutorial and went back to skip true

Author: nabobalis

.github/workflows/ci.yml

@@ -13,6 +13,13 @@ on:
       - '!*post*'
   pull_request:
   workflow_dispatch:
+  schedule:
+    #        ┌───────── minute (0 - 59)
+    #        │ ┌───────── hour (0 - 23)
+    #        │ │ ┌───────── day of the month (1 - 31)
+    #        │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
+    #        │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
+    - cron: '0 7 * * *'  # Every day at 07:00 UTC
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.gitignore

@@ -213,6 +213,7 @@ examples/**/*.asdf
 examples/**/*.csv
 examples/jsoc.stanford.edu/
 jsoc.stanford.edu/
+docs/sg_execution_times.rst
 
 ### Pycharm(?)
 .idea

.pre-commit-config.yaml

@@ -11,17 +11,17 @@ repos:
         args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable']
         exclude: ".*(.fits|.fts|.fit|.txt|tca.*|extern.*|.rst|.md|docs/conf.py)$"
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.1.6'
+    rev: 'v0.1.9'
     hooks:
       - id: ruff
         args: ['--fix', '--unsafe-fixes']
   -   repo: https://github.com/psf/black
-      rev: 23.11.0
+      rev: 23.12.1
       hooks:
       - id: black
         exclude: ".*(.fits|.fts|.fit|.txt|.csv)$"
   -   repo: https://github.com/PyCQA/isort
-      rev: 5.12.0
+      rev: 5.13.2
       hooks:
       - id: isort
         exclude: ".*(.fits|.fts|.fit|.txt|.csv)$"

CHANGELOG.rst

@@ -1,3 +1,12 @@
+0.7.1 (2023-12-28)
+==================
+
+Bug Fixes
+---------
+
+- Incorrect setup of the logger is now fixed. (`#113 <https://github.com/sunpy/drms/pull/113>`__)
+- Fixed missing environment variable in the docs. (`#113 <https://github.com/sunpy/drms/pull/113>`__)
+
 0.7.0 (2023-11-17)
 ==================
 

docs/conf.py

@@ -14,7 +14,6 @@
 project = "drms"
 author = "The SunPy Project"
 copyright = f"{datetime.datetime.now().year}, {author}"  # NOQA: A001
-
 # The full version, including alpha/beta/rc tags
 release = __version__
 is_development = ".dev" in __version__
@@ -37,25 +36,14 @@
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
 ]
-
-# Set automodapi to generate files inside the generated directory
 automodapi_toctreedirnm = "generated/api"
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
 source_suffix = ".rst"
-# The master toctree document.
 master_doc = "index"
-# The reST default role (used for this markup: `text`) to use for all
-# documents. Set to the "smart" one.
 default_role = "obj"
 
 # -- Options for hoverxref -----------------------------------------------------
 if os.environ.get("READTHEDOCS"):
-    # Building on Read the Docs
     hoverxref_api_host = "https://readthedocs.org"
     if os.environ.get("PROXIED_API_ENDPOINT"):
         # Use the proxied API endpoint
@@ -65,7 +53,6 @@
 hoverxref_tooltip_maxwidth = 600  # RTD main window is 696px
 hoverxref_auto_ref = True
 hoverxref_mathjax = True
-# hoverxref has to be applied to these
 hoverxref_domains = ["py"]
 hoverxref_role_types = {
     # roles with py domain
@@ -111,7 +98,9 @@
 }
 
 # -- Options for HTML output -------------------------------------------------
-# Render inheritance diagrams in SVG
+# JSOC email os env
+# see https://github.com/sunpy/sunpy/wiki/Home:-JSOC
+os.environ["JSOC_EMAIL"] = "jsoc@sunpy.org"
 graphviz_output_format = "svg"
 sphinx_gallery_conf = {
     "backreferences_dir": Path("generated") / "modules",

docs/tutorial.rst

@@ -281,11 +281,11 @@ Here, we are only interested in magnetograms and continuum intensity maps:
 
 which are stored as FITS files with varying dimensions.
 
-If we now want to submit an export request for a magnetogram and an intensity map of HARP number 4864, recorded at midnight on November 30, 2014, we can use the following export query string:
+If we now want to submit an export request for a magnetogram and an intensity map of HARP number 10490, recorded at  eight am on December 7th, 2023, we can use the following export query string:
 
 .. code-block:: python
 
-    >>> query_string = 'hmi.sharp_720s[4864][2014.11.30_00:00_TAI]{continuum, magnetogram}'  # doctest: +REMOTE_DATA
+    >>> query_string = 'hmi.sharp_720s[10490][2023.12.07_08:00:00_TAI]{continuum}'  # doctest: +REMOTE_DATA
 
 In order to obtain FITS files that include keyword data in their headers, we then need to use ``protocol='fits'`` when submitting the request using :meth:`drms.client.Client.export`:
 
@@ -300,9 +300,7 @@ We now need to wait for the server to prepare the requested files:
 .. code-block:: python
 
     >>> export_request.wait()  # doctest: +REMOTE_DATA
-    Export request pending. [id=..., status=2]
-    Waiting for 5 seconds...
-    ...
+    True
 
     >>> export_request.status  # doctest: +REMOTE_DATA
     0
@@ -325,10 +323,8 @@ Downloading the data works exactly like in the previous example, by using :meth:
 .. code-block:: python
 
     >>> export_request.download(out_dir)  # doctest: +REMOTE_DATA
-    Downloading file 1 of 2...
-    ...
-    Downloading file 2 of 2...
-    ...
+                                                  record                                                url                          download
+    0  warning=No FITS files were exported. The reque...  http://jsoc.stanford.edu/...  /...
 
 If you want to access an existing export request that you have submitted earlier, or if you submitted an export request using the `JSOC Export Data <http://jsoc.stanford.edu/ajax/exportdata.html>`__ webpage.
 You can use :meth:`drms.client.Client.export_from_id` with the corresponding ``ExportID`` to create an `drms.client.ExportRequest` instance for this particular request.

drms/__init__.py

@@ -9,14 +9,22 @@
 * Homepage: https://github.com/sunpy/drms
 * Documentation: https://docs.sunpy.org/projects/drms/en/stable/
 """
+import logging
 from pathlib import Path
 
-from .client import Client, ExportRequest, SeriesInfo
-from .config import ServerConfig, register_server
-from .exceptions import DrmsError, DrmsExportError, DrmsOperationNotSupported, DrmsQueryError
-from .json import HttpJsonClient, HttpJsonRequest, JsocInfoConstants
-from .utils import to_datetime
-from .version import version as __version__
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+from .client import Client, ExportRequest, SeriesInfo  # NOQA: E402
+from .config import ServerConfig, register_server  # NOQA: E402
+from .exceptions import DrmsError, DrmsExportError, DrmsOperationNotSupported, DrmsQueryError  # NOQA: E402
+from .json import HttpJsonClient, HttpJsonRequest, JsocInfoConstants  # NOQA: E402
+from .utils import to_datetime  # NOQA: E402
+from .version import version as __version__  # NOQA: E402
 
 
 def _get_bibtex():
@@ -53,4 +61,5 @@ def _get_bibtex():
     "SeriesInfo",
     "ServerConfig",
     "to_datetime",
+    "logger",
 ]

drms/client.py

@@ -1,7 +1,6 @@
 import os
 import re
 import time
-import logging
 from pathlib import Path
 from collections import OrderedDict
 from urllib.error import URLError, HTTPError
@@ -11,6 +10,8 @@
 import numpy as np
 import pandas as pd
 
+from drms import logger
+
 from .exceptions import DrmsExportError, DrmsOperationNotSupported, DrmsQueryError
 from .json import HttpJsonClient
 from .utils import _extract_series_name, _pd_to_numeric_coerce, _split_arg
@@ -455,7 +456,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
 
         while True:
             idstr = str(None) if self._requestid is None else (f"{self._requestid}")
-            logging.info(f"Export request pending. [id={idstr}, status={self._status}]")
+            logger.info(f"Export request pending. [id={idstr}, status={self._status}]")
 
             # Use the user-provided sleep value or the server's wait value.
             # In case neither is available, wait for 5 seconds.
@@ -471,7 +472,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
                 if t_start + timeout + wait_secs - time.time() < 0:
                     return False
 
-            logging.info(f"Waiting for {int(round(wait_secs))} seconds...")
+            logger.info(f"Waiting for {int(round(wait_secs))} seconds...")
             time.sleep(wait_secs)
 
             if self.has_finished():
@@ -481,7 +482,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
                 # Raise exception, if no retries are left.
                 if retries_notfound <= 0:
                     self._raise_on_error(notfound_ok=False)
-                logging.info(f"Request not found on server, {retries_notfound} retries left.")
+                logger.info(f"Request not found on server, {retries_notfound} retries left.")
                 retries_notfound -= 1
 
     def download(self, directory, *, index=None, fname_from_rec=None):
@@ -565,18 +566,18 @@ def download(self, directory, *, index=None, fname_from_rec=None):
             fpath = Path(out_dir) / filename
             fpath_new = self._next_available_filename(fpath)
             fpath_tmp = self._next_available_filename(f"{fpath_new}.part")
-            logging.info(f"Downloading file {int(i + 1)} of {int(ndata)}...")
-            logging.info(f"    record: {di.record}")
-            logging.info(f"  filename: {di.filename}")
+            logger.info(f"Downloading file {int(i + 1)} of {int(ndata)}...")
+            logger.info(f"    record: {di.record}")
+            logger.info(f"    filename: {di.filename}")
             try:
                 urlretrieve(di.url, fpath_tmp)
             except (HTTPError, URLError):
                 fpath_new = None
-                logging.info("  -> Error: Could not download file")
+                logger.info("    -> Error: Could not download file")
             else:
                 fpath_new = self._next_available_filename(fpath)
                 Path(fpath_tmp).rename(fpath_new)
-                logging.info(f"  -> {os.path.relpath(fpath_new)}")
+                logger.info(f"    -> {os.path.relpath(fpath_new)}")
             downloads.append(fpath_new)
 
         res = data[["record", "url"]].copy()
@@ -650,7 +651,7 @@ def _generate_filenamefmt(self, sname):
             si = self.info(sname)
         except Exception as e:  # NOQA: BLE001
             # Cannot generate filename format for unknown series.
-            logging.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
+            logger.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
             return None
 
         pkfmt_list = []
@@ -700,7 +701,7 @@ def _filename_from_export_record(self, rs, *, old_fname=None):
             si = self.info(sname)
         except Exception as e:  # NOQA: BLE001
             # Cannot generate filename for unknown series.
-            logging.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
+            logger.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
             return None
 
         if pkeys is not None:

drms/json.py

@@ -1,9 +1,10 @@
 import json as _json
-import logging
 from enum import Enum
 from urllib.parse import urlencode, quote_plus
 from urllib.request import HTTPError, urlopen
 
+from drms import logger
+
 from .config import ServerConfig, _server_configs
 from .utils import _split_arg
 
@@ -86,7 +87,7 @@ def __repr__(self):
         return f"<HttpJsonClient: {self._server.name}>"
 
     def _json_request(self, url):
-        logging.info(url)
+        logger.debug(f"URL for request: {url}")
         return HttpJsonRequest(url, self._server.encoding)
 
     @property

drms/main.py

@@ -1,21 +1,18 @@
 import sys
-import logging
 import argparse
 
 
 def main():
     import drms
 
     args = parse_args(sys.argv[1:])
-    # Create a Client instance
     client = drms.Client(server=args.server, email=args.email)
-    logging.info(f"client: {client}")
+    drms.logger.info(f"client: {client}")
 
 
 def parse_args(args):
     import drms
 
-    # Handle command line options
     parser = argparse.ArgumentParser(description="drms, access HMI, AIA and MDI data with python")
     parser.add_argument(
         "--version",

setup.cfg

@@ -67,6 +67,8 @@ console_scripts =
 [tool:pytest]
 testpaths = "drms" "docs"
 norecursedirs = ".tox" "build" "docs[\/]_build" "docs[\/]generated" "*.egg-info" "examples" ".history" "paper" "drms[\/]_dev"
+log_cli=true
+log_level=INFO
 doctest_plus = enabled
 doctest_optionflags = NORMALIZE_WHITESPACE FLOAT_CMP ELLIPSIS
 addopts = --doctest-rst -p no:unraisableexception -p no:threadexception

tox.ini

@@ -5,8 +5,6 @@ envlist =
     build_docs
     codestyle
 requires =
-    setuptools
-    pip
     tox-pypi-filter
 
 [testenv]
@@ -44,6 +42,7 @@ deps =
 extras =
     dev
 commands =
+    pip freeze --all --no-input
     sunpy: pytest -vvv -s -ra --pyargs sunpy.net.jsoc --timeout=120 --remote-data=any {posargs}
     !online: {env:PYTEST_COMMAND} {posargs}
     online: {env:PYTEST_COMMAND} --timeout=120 --remote-data=any {posargs}
@@ -53,7 +52,8 @@ changedir = docs
 description = Invoke sphinx-build to build the HTML docs
 extras = dev
 commands =
-    sphinx-build -j auto --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs}
+    pip freeze --all --no-input
+    sphinx-build -j 1 --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs}
     python -c 'import pathlib; print("Documentation available under file://\{0\}".format(pathlib.Path(r"{toxinidir}") / "docs" / "_build" / "html"/ "index.html"))'
 
 [testenv:codestyle]