Add ruff, remove isort, and add more pre-commits (#237)

* add ruff

* release notes

Author: gsheni

.flake8

@@ -1,19 +1,34 @@
+exclude: |
+  (?x)
+  ^nlp_primitives/data/nltk-data/|
+  .html$|.csv$|.svg$|.md$|.txt$|.json$|.xml$|.pickle$|^.github/|
+  (LICENSE.*|README.*)
 default_stages: [commit]
-exclude: ^LICENSE/|\.(html|csv|svg|md|txt|json|tab|bib|adv)$
 repos:
-  - repo: https://github.com/MarcoGorelli/absolufy-imports
-    rev: v0.3.1
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: 'v4.3.0'
     hooks:
-      - id: absolufy-imports
-        files: ^nlp_primitives/
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.0.4
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: 'v0.10.1'
     hooks:
-      - id: isort
-        args: [--settings-path=pyproject.toml]
+      - id: validate-pyproject
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: 'v2.2.3'
+    hooks:
+      - id: add-trailing-comma
+        name: Add trailing comma
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.191'
+    hooks:
+      - id: ruff
+        args: ["--fix"]
   - repo: https://github.com/python/black
-    rev: 22.8.0
+    rev: 22.12.0
     hooks:
       - id: black
-        args: [--target-version=py310]
-        types_or: [python]
+        args: [--preview]
+        additional_dependencies: [".[jupyter]"]
+        types_or: [python, jupyter]

.pre-commit-config.yaml

@@ -26,4 +26,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE

@@ -18,14 +18,13 @@ clean:
 
 .PHONY: lint
 lint:
-	isort --check-only nlp_primitives
-	black nlp_primitives -t py310 --check
-	flake8 nlp_primitives
+	black . --check --preview
+	ruff .
 
 .PHONY: lint-fix
 lint-fix:
-	black -t py310 nlp_primitives
-	isort nlp_primitives
+	black . --preview
+	ruff . --fix
 
 .PHONY: test
 test:

Makefile

@@ -87,7 +87,7 @@ def __init__(self, random_seed=0, corpus=None, algorithm="randomized"):
         self.algorithm = algorithm
         if self.algorithm not in ["randomized", "arpack"]:
             raise ValueError(
-                "TruncatedSVD algorithm must be either 'randomized' or 'arpack'"
+                "TruncatedSVD algorithm must be either 'randomized' or 'arpack'",
             )
 
     def _create_trainer(self):

nlp_primitives/lsa.py

@@ -38,7 +38,6 @@ def __init__(self):
         self.n = 15
 
     def get_function(self):
-
         # For more info about the different parts of speech, see here: https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
         types = [
             "C",  # cardinal digits

nlp_primitives/part_of_speech_count.py

@@ -54,7 +54,7 @@ def elmo(col):
                     [
                         tf.compat.v1.global_variables_initializer(),
                         tf.compat.v1.tables_initializer(),
-                    ]
+                    ],
                 )
                 embeddings = session.run(self.embed(col.tolist()))
             return embeddings.transpose()

nlp_primitives/tensorflow/elmo.py

@@ -29,7 +29,10 @@ class UniversalSentenceEncoder(TransformPrimitive):
     return_type = ColumnSchema(logical_type=Double, semantic_tags={"numeric"})
 
     def __init__(self):
-        message = "In order to use the UniversalSentenceEncoder primitive install 'nlp_primitives[complete]'"
+        message = (
+            "In order to use the UniversalSentenceEncoder primitive install"
+            " 'nlp_primitives[complete]'"
+        )
         self.tf = import_or_raise("tensorflow", message)
         hub = import_or_raise("tensorflow_hub", message)
         self.tf.compat.v1.disable_eager_execution()
@@ -45,7 +48,7 @@ def universal_sentence_encoder(col):
                     [
                         self.tf.compat.v1.global_variables_initializer(),
                         self.tf.compat.v1.tables_initializer(),
-                    ]
+                    ],
                 )
                 embeddings = session.run(self.embed(col.tolist()))
             return embeddings.transpose()

nlp_primitives/tensorflow/universal_sentence_encoder.py

@@ -19,7 +19,7 @@ def test_primitive_func_1(self):
                 "Not diverse not diverse not",
                 "this is a semi diverse diverse example",
                 "a a",
-            ]
+            ],
         )
         primitive_instance = self.primitive()
         primitive_func = primitive_instance.get_function()

nlp_primitives/tests/test_diversity_score.py

@@ -21,7 +21,7 @@ def test_regular(self):
                 "The roller coaster was built in 1885.",
                 "When will humans go to mars?",
                 "Mitochondria is the powerhouse of the cell",
-            ]
+            ],
         )
         new_results = primitive(words)
         assert round(sum(new_results[:, 0]), 3) == 8.744

nlp_primitives/tests/test_elmo.py

@@ -21,15 +21,15 @@ def test_strings(self):
                 "She ate a pineapple",
                 "Consume Electrolytes, he told me.",
                 "Hello",
-            ]
+            ],
         )
         primitive_func = self.primitive().get_function()
 
         answers = pd.Series(
             [
                 [2.41e-03, 6.29e-04, 7.26e-03, -1.85e-19],
                 [1.28e-03, 5.51e-04, 5.37e-03, -1.20e-15],
-            ]
+            ],
         )
         results = primitive_func(x)
         np.testing.assert_array_almost_equal(
@@ -45,7 +45,7 @@ def test_strings_custom_corpus(self):
                 "She ate a pineapple",
                 "Consume Electrolytes, he told me.",
                 "Hello",
-            ]
+            ],
         )
         # Create a new corpus using only the first 10000 elements from Gutenberg
         gutenberg = nltk.corpus.gutenberg.sents()
@@ -62,7 +62,7 @@ def test_strings_custom_corpus(self):
                     1.56e-03,
                     0.0,
                 ],
-            ]
+            ],
         )
         results = primitive_func(x)
         np.testing.assert_array_almost_equal(
@@ -96,7 +96,11 @@ def test_with_featuretools(self, es):
         primitive_instance = self.primitive()
         transform.append(primitive_instance)
         valid_dfs(
-            es, aggregation, transform, self.primitive.name.upper(), multi_output=True
+            es,
+            aggregation,
+            transform,
+            self.primitive.name.upper(),
+            multi_output=True,
         )
 
     def test_bad_algorithm_input_value(self):
@@ -120,6 +124,8 @@ def test_args_strings(self):
 
         # Test all args
         args_string = self.primitive(
-            random_seed=100, corpus=custom_corpus, algorithm="arpack"
+            random_seed=100,
+            corpus=custom_corpus,
+            algorithm="arpack",
         ).get_args_string()
         assert args_string == ", random_seed=100, corpus=user_defined, algorithm=arpack"

nlp_primitives/tests/test_lsa.py

@@ -19,7 +19,7 @@ def test_sentences(self):
                 "Ab. Bb. Db.",
                 "And? Why! Box. Car? Rat.",
                 "Yep.",
-            ]
+            ],
         )
         primitive_func = self.primitive().get_function()
         answers = pd.Series([3.0, 4.0, 4.0])

nlp_primitives/tests/test_mean_characters_per_sentence.py

@@ -18,7 +18,7 @@ def test_regular_input(self):
                 "Hello. Hello! Hello? Hello.",
                 "and?",
                 "yes no",
-            ]
+            ],
         )
         expected = [4.0, 1.0, 1.0]
         actual = self.primitive().get_function()(x)
@@ -34,7 +34,7 @@ def test_multiline(self):
         x = pd.Series(
             [
                 "Yes\n, this is true!",
-            ]
+            ],
         )
 
         expected = [1.0]

nlp_primitives/tests/test_number_of_sentences.py

@@ -14,7 +14,7 @@ class TestPartOfSpeechCount(PrimitiveT):
 
     def test_strings(self):
         x = pd.Series(
-            ["This IS a STRING.", "Testing AAA", "Testing AAA-BBB", "Testing AA3"]
+            ["This IS a STRING.", "Testing AAA", "Testing AAA-BBB", "Testing AA3"],
         )
         primitive_func = self.primitive().get_function()
 
@@ -35,7 +35,7 @@ def test_strings(self):
                 [0.0, 0.0, 0.0, 0.0],
                 [0.0, 1.0, 1.0, 1.0],
                 [0.0, 0.0, 0.0, 0.0],
-            ]
+            ],
         )
 
         pd.testing.assert_series_equal(primitive_func(x), answers, check_names=False)
@@ -61,7 +61,7 @@ def test_nan(self):
                 [np.nan, 0.0, 0.0],
                 [np.nan, 0.0, 0.0],
                 [np.nan, 0.0, 0.0],
-            ]
+            ],
         )
         pd.testing.assert_series_equal(primitive_func(x), answers, check_names=False)
 
@@ -70,5 +70,9 @@ def test_with_featuretools(self, es):
         primitive_instance = self.primitive()
         transform.append(primitive_instance)
         valid_dfs(
-            es, aggregation, transform, self.primitive.name.upper(), multi_output=True
+            es,
+            aggregation,
+            transform,
+            self.primitive.name.upper(),
+            multi_output=True,
         )

nlp_primitives/tests/test_part_of_speech_count.py

@@ -14,7 +14,7 @@ class TestPolarityScore(PrimitiveT):
 
     def test_primitive_func_1(self):
         array = pd.Series(
-            ["He hates cars!", "She loves everything", "This is neutral", "!12323"]
+            ["He hates cars!", "She loves everything", "This is neutral", "!12323"],
         )
         primitive_instance = self.primitive()
         primitive_func = primitive_instance.get_function()

nlp_primitives/tests/test_polarity_score.py

@@ -19,7 +19,7 @@ def test_strings(self):
                 "This is second string",
                 "third string",
                 "This IS the fourth string.",
-            ]
+            ],
         )
         primitive_func = self.primitive().get_function()
         answers = pd.Series([3, 2, 0, 3])

nlp_primitives/tests/test_stopword_count.py

@@ -21,7 +21,7 @@ def test_regular(universal_sentence_encoder):
             "The roller coaster was built in 1885.",
             "When will humans go to mars?",
             "Mitochondria is the powerhouse of the cell",
-        ]
+        ],
     )
     a = pd.DataFrame(universal_sentence_encoder(sentences))
     a = a.mean().round(7).to_numpy()
@@ -43,7 +43,10 @@ def mock_remove_tensorflow():
 
 
 def test_without_tensorflow(universal_sentence_encoder, mock_remove_tensorflow):
-    err_message = "In order to use the UniversalSentenceEncoder primitive install 'nlp_primitives[complete]'"
+    err_message = (
+        "In order to use the UniversalSentenceEncoder primitive install"
+        " 'nlp_primitives[complete]'"
+    )
     with pytest.raises(ImportError) as error:
         UniversalSentenceEncoder()
     assert error.value.args[0] == err_message
@@ -57,7 +60,7 @@ def test_primitive_serialization(universal_sentence_encoder):
             "The roller coaster was built in 1885.",
             "When will humans go to mars?",
             "Mitochondria is the powerhouse of the cell",
-        ]
+        ],
     )
     serialized_primitive = serialize_primitive(universal_sentence_encoder)
     deserializer = PrimitivesDeserializer()
@@ -77,7 +80,7 @@ def test_feature_serialization(universal_sentence_encoder, tmpdir):
             "The roller coaster was built in 1885.",
             "When will humans go to mars?",
             "Mitochondria is the powerhouse of the cell",
-        ]
+        ],
     )
 
     es = ft.EntitySet("es")

nlp_primitives/tests/test_universal_sentence_encoder.py

@@ -107,10 +107,12 @@ def find_applicable_primitives(primitive):
     all_transform_primitives = list(get_transform_primitives().values())
     all_aggregation_primitives = list(get_aggregation_primitives().values())
     applicable_transforms = find_stackable_primitives(
-        all_transform_primitives, primitive
+        all_transform_primitives,
+        primitive,
     )
     applicable_aggregations = find_stackable_primitives(
-        all_aggregation_primitives, primitive
+        all_aggregation_primitives,
+        primitive,
     )
     return applicable_transforms, applicable_aggregations
 
@@ -153,13 +155,14 @@ def valid_dfs(
                 applicable_features.append(feat)
     if len(applicable_features) == 0:
         raise ValueError(
-            "No feature names with %s, verify the name attribute \
-                          is defined and/or generate_name() is defined to \
-                          return %s "
-            % (feature_substrings, feature_substrings)
+            "No feature names with %s, verify the name attribute                       "
+            "    is defined and/or generate_name() is defined to                       "
+            "    return %s " % (feature_substrings, feature_substrings),
         )
     df = ft.calculate_feature_matrix(
-        entityset=es, features=applicable_features, instance_ids=instance_ids
+        entityset=es,
+        features=applicable_features,
+        instance_ids=instance_ids,
     )
 
     ft.encode_features(df, applicable_features)

nlp_primitives/tests/test_utils.py

@@ -10,7 +10,7 @@ def clean_tokens(text: str) -> List[str]:
 
     # Remove stopwords and punctuation
     stopwords_and_punctuation = set(nltk.corpus.stopwords.words("english")).union(
-        set(string.punctuation)
+        set(string.punctuation),
     )
     text = [word for word in text if word not in stopwords_and_punctuation]