fix skip pandas 0.24 (#6151)

* remove skip

* fix for pandas 0.24 + bump minimum version to 0.21

* Allow pandas>=0.18 again

* use LooseVersion instead of packaging

Author: massich

environment.yml

@@ -10,7 +10,7 @@ dependencies:
 - matplotlib
 - cython
 - pyqt>=5.9
-- pandas
+- pandas>=0.18
 - xlrd
 - scikit-learn
 - h5py

mne/datasets/sleep_physionet/_utils.py

@@ -7,6 +7,7 @@
 import os
 import os.path as op
 import numpy as np
+from distutils.version import LooseVersion
 
 from ...utils import _fetch_file, verbose, _TempDir, _check_pandas_installed
 from ..utils import _get_path
@@ -106,6 +107,8 @@ def _update_sleep_temazepam_records(fname=TEMAZEPAM_SLEEP_RECORDS):
 
     # Load and massage the data.
     data = pd.read_excel(subjects_fname, header=[0, 1])
+    if LooseVersion(pd.__version__) >= LooseVersion('0.24.0'):
+        data = data.set_index(('Subject - age - sex', 'Nr'))
     data.index.name = 'subject'
     data.columns.names = [None, None]
     data = (data.set_index([('Subject - age - sex', 'Age'),
@@ -126,7 +129,10 @@ def _update_sleep_temazepam_records(fname=TEMAZEPAM_SLEEP_RECORDS):
     data = data.set_index(['id', 'subject', 'age', 'sex', 'drug',
                            'lights off', 'night nr', 'record type']).unstack()
     data.columns = [l1 + '_' + l2 for l1, l2 in data.columns]
-    data = data.reset_index().drop(columns=['id'])
+    if LooseVersion(pd.__version__) < LooseVersion('0.21.0'):
+        data = data.reset_index().drop(labels=['id'], axis=1)
+    else:
+        data = data.reset_index().drop(columns=['id'])
 
     data['sex'] = (data.sex.astype('category')
                        .cat.rename_categories({1: 'male', 2: 'female'}))
@@ -175,8 +181,10 @@ def _update_sleep_age_records(fname=AGE_SLEEP_RECORDS):
                                      .str.split('.', expand=True)[0]
                                      .astype('category'))
 
-    # data = data.set_index(['subject', 'night', 'record type'])
-    data = data.reset_index().drop(columns=['id'])
+    if LooseVersion(pd.__version__) < LooseVersion('0.21.0'):
+        data = data.reset_index().drop(labels=['id'], axis=1)
+    else:
+        data = data.reset_index().drop(columns=['id'])
     data = data[['subject', 'night', 'record type', 'age', 'sex', 'lights off',
                  'sha', 'fname']]
 

mne/datasets/sleep_physionet/tests/test_physionet.py

@@ -4,20 +4,19 @@
 # License: BSD Style.
 
 import os.path as op
-
 import numpy as np
-from numpy.testing import assert_array_equal
-
 import pytest
 
+from distutils.version import LooseVersion
+from numpy.testing import assert_array_equal
+
 from mne.utils import run_tests_if_main, requires_good_network
 from mne.utils import requires_pandas, requires_version
 from mne.datasets.sleep_physionet import age, temazepam
 from mne.datasets.sleep_physionet._utils import _update_sleep_temazepam_records
 from mne.datasets.sleep_physionet._utils import _update_sleep_age_records
 from mne.datasets.sleep_physionet._utils import AGE_SLEEP_RECORDS
 from mne.datasets.sleep_physionet._utils import TEMAZEPAM_SLEEP_RECORDS
-# from mne.datasets.sleep_physionet._utils import BASE_URL  # XXX not used
 
 
 @pytest.fixture(scope='session')
@@ -71,7 +70,15 @@ def test_run_update_age_records(tmpdir):
     fname = op.join(str(tmpdir), "records.csv")
     _update_sleep_age_records(fname)
     data = pd.read_csv(fname)
-    pd.testing.assert_frame_equal(data, pd.read_csv(AGE_SLEEP_RECORDS))
+
+    if LooseVersion(pd.__version__) < LooseVersion('0.23.0'):
+        expected = pd.read_csv(AGE_SLEEP_RECORDS)
+        assert_array_equal(
+            data[['subject', 'sha', 'fname']].values,
+            expected[['subject', 'sha', 'fname']].values,
+        )
+    else:
+        pd.testing.assert_frame_equal(data, pd.read_csv(AGE_SLEEP_RECORDS))
 
 
 def test_sleep_physionet_age(physionet_tmpdir, mocker):
@@ -129,14 +136,22 @@ def test_sleep_physionet_age(physionet_tmpdir, mocker):
 @requires_good_network
 @requires_pandas
 @requires_version('xlrd', '0.9')
-@pytest.mark.skip(reason="Broken with new pandas 0.24 and xlrd")
 def test_run_update_temazepam_records(tmpdir):
     """Test Sleep Physionet URL handling."""
     import pandas as pd
     fname = op.join(str(tmpdir), "records.csv")
     _update_sleep_temazepam_records(fname)
     data = pd.read_csv(fname)
-    pd.testing.assert_frame_equal(data, pd.read_csv(TEMAZEPAM_SLEEP_RECORDS))
+
+    if LooseVersion(pd.__version__) < LooseVersion('0.23.0'):
+        expected = pd.read_csv(TEMAZEPAM_SLEEP_RECORDS)
+        assert_array_equal(
+            data[['subject', 'sha_Hypnogram', 'sha_PSG']].values,
+            expected[['subject', 'sha_Hypnogram', 'sha_PSG']].values,
+        )
+    else:
+        pd.testing.assert_frame_equal(
+            data, pd.read_csv(TEMAZEPAM_SLEEP_RECORDS))
 
 
 def test_sleep_physionet_temazepam(physionet_tmpdir, mocker):