Skip to content

BUG: Pandas concat raises RuntimeWarning: '<' not supported between i… #61608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion pandas/core/indexes/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3105,7 +3105,19 @@ def union(self, other, sort: bool | None = None):
return result.sort_values()
return result

result = self._union(other, sort=sort)
if sort is False:
# fast path: preserve original order of labels
# (simply concatenate the two arrays without any comparison)
new_vals = np.concatenate([self._values, other._values])
result = Index(new_vals, name=self.name)
else:
# sort==True or sort==None: call into the subclass-specific union
# but guard against TypeError from mixed-type comparisons
try:
result = self._union(other, sort=sort)
except TypeError:
new_vals = np.concatenate([self._values, other._values])
result = Index(new_vals, name=self.name)

return self._wrap_setop_result(other, result)

Expand Down
13 changes: 4 additions & 9 deletions pandas/core/indexes/multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -3911,18 +3911,13 @@ def _union(self, other, sort) -> MultiIndex:
else:
result = self._get_reconciled_name_object(other)

if sort is not False:
# only sort if requested; if types are unorderable, skip silently
if sort:
try:
result = result.sort_values()
except TypeError:
if sort is True:
raise
warnings.warn(
"The values in the array are unorderable. "
"Pass `sort=False` to suppress this warning.",
RuntimeWarning,
stacklevel=find_stack_level(),
)
# mixed-type tuples: bail out on sorting
pass
return result

def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
Expand Down
7 changes: 5 additions & 2 deletions pandas/core/reshape/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -823,8 +823,11 @@ def _get_sample_object(
return objs[0], objs


def _concat_indexes(indexes) -> Index:
return indexes[0].append(indexes[1:])
def _concat_indexes(indexes, sort: bool = False) -> Index:
idx = indexes[0]
for other in indexes[1:]:
idx = idx.union(other, sort=sort)
return idx


def validate_unique_levels(levels: list[Index]) -> None:
Expand Down
24 changes: 7 additions & 17 deletions pandas/tests/frame/test_query_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,21 +160,13 @@ def test_query_empty_string(self):
df.query("")

def test_query_duplicate_column_name(self, engine, parser):
df = DataFrame(
{
"A": range(3),
"B": range(3),
"C": range(3)
}
).rename(columns={"B": "A"})
df = DataFrame({"A": range(3), "B": range(3), "C": range(3)}).rename(
columns={"B": "A"}
)

res = df.query('C == 1', engine=engine, parser=parser)
res = df.query("C == 1", engine=engine, parser=parser)

expect = DataFrame(
[[1, 1, 1]],
columns=["A", "A", "C"],
index=[1]
)
expect = DataFrame([[1, 1, 1]], columns=["A", "A", "C"], index=[1])

tm.assert_frame_equal(res, expect)

Expand Down Expand Up @@ -1140,9 +1132,7 @@ def test_query_with_nested_special_character(self, parser, engine):
[">=", operator.ge],
],
)
def test_query_lex_compare_strings(
self, parser, engine, op, func
):
def test_query_lex_compare_strings(self, parser, engine, op, func):
a = Series(np.random.default_rng(2).choice(list("abcde"), 20))
b = Series(np.arange(a.size))
df = DataFrame({"X": a, "Y": b})
Expand Down Expand Up @@ -1411,7 +1401,7 @@ def test_expr_with_column_name_with_backtick_and_hash(self):
def test_expr_with_column_name_with_backtick(self):
# GH 59285
df = DataFrame({"a`b": (1, 2, 3), "ab": (4, 5, 6)})
result = df.query("`a``b` < 2") # noqa
result = df.query("`a``b` < 2")
# Note: Formatting checks may wrongly consider the above ``inline code``.
expected = df[df["a`b"] < 2]
tm.assert_frame_equal(result, expected)
Expand Down
3 changes: 1 addition & 2 deletions scripts/check_for_inconsistent_pandas_namespace.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@
from typing import NamedTuple

ERROR_MESSAGE = (
"{path}:{lineno}:{col_offset}: "
"Found both '{prefix}.{name}' and '{name}' in {path}"
"{path}:{lineno}:{col_offset}: Found both '{prefix}.{name}' and '{name}' in {path}"
)


Expand Down
1 change: 1 addition & 0 deletions scripts/check_test_naming.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
NOTE: if this finds a false positive, you can add the comment `# not a test` to the
class or function definition. Though hopefully that shouldn't be necessary.
"""

from __future__ import annotations

import argparse
Expand Down
1 change: 1 addition & 0 deletions scripts/generate_pip_deps_from_conda.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
generated with this script:
$ python scripts/generate_pip_deps_from_conda.py --compare
"""

import argparse
import pathlib
import re
Expand Down
1 change: 1 addition & 0 deletions scripts/pandas_errors_documented.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

pre-commit run pandas-errors-documented --all-files
"""

from __future__ import annotations

import argparse
Expand Down
1 change: 1 addition & 0 deletions scripts/sort_whatsnew_note.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

pre-commit run sort-whatsnew-items --all-files
"""

from __future__ import annotations

import argparse
Expand Down
5 changes: 1 addition & 4 deletions scripts/tests/test_check_test_naming.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,7 @@
0,
),
(
"class Foo: # not a test\n"
" pass\n"
"def test_foo():\n"
" Class.foo()\n",
"class Foo: # not a test\n pass\ndef test_foo():\n Class.foo()\n",
"",
0,
),
Expand Down
8 changes: 2 additions & 6 deletions scripts/tests/test_inconsistent_namespace_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,10 @@
)

BAD_FILE_0 = (
"from pandas import Categorical\n"
"cat_0 = Categorical()\n"
"cat_1 = pd.Categorical()"
"from pandas import Categorical\ncat_0 = Categorical()\ncat_1 = pd.Categorical()"
)
BAD_FILE_1 = (
"from pandas import Categorical\n"
"cat_0 = pd.Categorical()\n"
"cat_1 = Categorical()"
"from pandas import Categorical\ncat_0 = pd.Categorical()\ncat_1 = Categorical()"
)
BAD_FILE_2 = (
"from pandas import Categorical\n"
Expand Down
20 changes: 9 additions & 11 deletions scripts/tests/test_validate_docstrings.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ def redundant_import(self, paramx=None, paramy=None) -> None:
--------
>>> import numpy as np
>>> import pandas as pd
>>> df = pd.DataFrame(np.ones((3, 3)),
... columns=('a', 'b', 'c'))
>>> df = pd.DataFrame(np.ones((3, 3)), columns=("a", "b", "c"))
>>> df.all(axis=1)
0 True
1 True
Expand All @@ -50,14 +49,14 @@ def unused_import(self) -> None:
Examples
--------
>>> import pandas as pdf
>>> df = pd.DataFrame(np.ones((3, 3)), columns=('a', 'b', 'c'))
>>> df = pd.DataFrame(np.ones((3, 3)), columns=("a", "b", "c"))
"""

def missing_whitespace_around_arithmetic_operator(self) -> None:
"""
Examples
--------
>>> 2+5
>>> 2 + 5
7
"""

Expand All @@ -66,14 +65,14 @@ def indentation_is_not_a_multiple_of_four(self) -> None:
Examples
--------
>>> if 2 + 5:
... pass
... pass
"""

def missing_whitespace_after_comma(self) -> None:
"""
Examples
--------
>>> df = pd.DataFrame(np.ones((3,3)),columns=('a','b', 'c'))
>>> df = pd.DataFrame(np.ones((3, 3)), columns=("a", "b", "c"))
"""

def write_array_like_with_hyphen_not_underscore(self) -> None:
Expand Down Expand Up @@ -227,13 +226,13 @@ def test_validate_all_ignore_errors(self, monkeypatch):
"errors": [
("ER01", "err desc"),
("ER02", "err desc"),
("ER03", "err desc")
("ER03", "err desc"),
],
"warnings": [],
"examples_errors": "",
"deprecated": True,
"file": "file1",
"file_line": "file_line1"
"file_line": "file_line1",
},
)
monkeypatch.setattr(
Expand Down Expand Up @@ -272,14 +271,13 @@ def test_validate_all_ignore_errors(self, monkeypatch):
None: {"ER03"},
"pandas.DataFrame.align": {"ER01"},
# ignoring an error that is not requested should be of no effect
"pandas.Index.all": {"ER03"}
}
"pandas.Index.all": {"ER03"},
},
)
# two functions * two not global ignored errors - one function ignored error
assert exit_status == 2 * 2 - 1



class TestApiItems:
@property
def api_doc(self):
Expand Down
Loading
Loading