delay import of sklearn ijson

Author: xadupre

azure-pipelines.yml

@@ -24,7 +24,7 @@ jobs:
   - script: pip install -r requirements-dev.txt
     displayName: 'Install Requirements dev'
   - script: |
-      ruff .
+      ruff check .
     displayName: 'Ruff'
   - script: |
       black --diff .
@@ -76,7 +76,7 @@ jobs:
   - script: pip install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn
     displayName: 'Install scikit-learn nightly'
   - script: |
-      ruff .
+      ruff check .
     displayName: 'Ruff'
   - script: |
       rstcheck -r ./_doc ./pandas_streaming
@@ -117,7 +117,7 @@ jobs:
   - script: pip install -r requirements-dev.txt
     displayName: 'Install Requirements dev'
   - script: |
-      ruff .
+      ruff check .
     displayName: 'Ruff'
   - script: |
       rstcheck -r ./_doc ./pandas_streaming

pandas_streaming/df/connex_split.py

@@ -2,7 +2,6 @@
 from logging import getLogger
 import pandas
 import numpy
-from sklearn.model_selection import train_test_split
 from .dataframe_helpers import dataframe_shuffle
 
 logger = getLogger("pandas-streaming")
@@ -61,6 +60,7 @@ def train_test_split_weights(
             raise ValueError(
                 f"test_size={test_size} or train_size={train_size} cannot be null (1)."
             )
+        from sklearn.model_selection import train_test_split
         return train_test_split(
             df, test_size=test_size, train_size=train_size, random_state=random_state
         )

pandas_streaming/df/dataframe.py

@@ -640,10 +640,10 @@ def _reservoir_sampling(
                 if len(indices) < n:
                     indices.append((i, ir))
                 else:
-                    x = nrandom.random()  # pylint: disable=E1101
+                    x = nrandom.random()
                     if x * n < (seen - n):
                         k = nrandom.randint(0, len(indices) - 1)
-                        indices[k] = (i, ir)  # pylint: disable=E1126
+                        indices[k] = (i, ir)
         indices = set(indices)
 
         def reservoir_iterate(sdf, indices, chunksize):

pandas_streaming/df/dataframe_helpers.py

@@ -155,13 +155,13 @@ def hash_floatl(c):
     }  # pylint: disable=R1721
     for c in cols:
         t = coltype[c]
-        if t == int:
+        if t == int: # noqa: E721
             df[c] = df[c].apply(hash_intl)
         elif t == numpy.int64:
             df[c] = df[c].apply(lambda x: numpy.int64(hash_intl(x)))
-        elif t == float:
+        elif t == float: # noqa: E721
             df[c] = df[c].apply(hash_floatl)
-        elif t == object:
+        elif t == object: # noqa: E721
             df[c] = df[c].apply(hash_strl)
         else:
             raise NotImplementedError(  # pragma: no cover

pandas_streaming/df/dataframe_io_helpers.py

@@ -5,7 +5,6 @@
     from ujson import dumps
 except ImportError:  # pragma: no cover
     from json import dumps
-import ijson
 
 
 class JsonPerRowsStream:
@@ -257,6 +256,8 @@ def enumerate_json_items(
     else:
         if hasattr(filename, "seek"):
             filename.seek(0)
+        import ijson
+
         parser = ijson.parse(filename)
         current = None
         curkey = None

pandas_streaming/df/dataframe_split.py

@@ -45,7 +45,7 @@ def sklearn_train_test_split(
         )
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=ImportWarning)
-        from sklearn.model_selection import train_test_split  # pylint: disable=C0415
+        from sklearn.model_selection import train_test_split
 
     opts = ["test_size", "train_size", "random_state", "shuffle", "stratify"]
     split_ops = {}

pyproject.toml

@@ -25,11 +25,11 @@ exclude = [
 # Same as Black.
 line-length = 88
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
 max-complexity = 10
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "_doc/examples/plot_first_example.py" = ["E402", "F811"]
 "_unittests/ut_df/test_dataframe_io_helpers.py" = ["E501"]
 "pandas_streaming/data/__init__.py" = ["F401"]