Skip to content

[WIP][SPARK-52646][PS] Avoid CAST_INVALID_INPUT of __eq__ in ANSI mode #51370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+24 −1
Draft

[WIP][SPARK-52646][PS] Avoid CAST_INVALID_INPUT of __eq__ in ANSI mode #51370

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion python/pyspark/pandas/data_type_ops/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
DecimalType,
FractionalType,
IntegralType,
LongType,
MapType,
NullType,
NumericType,
Expand All @@ -52,6 +53,7 @@
extension_object_dtypes_available,
spark_type_to_pandas_dtype,
)
from pyspark.pandas.utils import is_ansi_mode_enabled

if extension_dtypes_available:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
Expand Down Expand Up @@ -392,10 +394,12 @@ def ge(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
raise TypeError(">= can not be applied to %s." % self.pretty_name)

def eq(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
from pyspark.pandas.internal import InternalField

if isinstance(right, (list, tuple)):
from pyspark.pandas.series import first_series, scol_for
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME, InternalField
from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME

if len(left) != len(right):
raise ValueError("Lengths must be equal")
Expand Down Expand Up @@ -482,6 +486,25 @@ def eq(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
else:
from pyspark.pandas.base import column_op

if is_ansi_mode_enabled(left._internal.spark_frame.sparkSession):
from pyspark.pandas.base import IndexOpsMixin

def are_both_numeric(left_dtype, right_dtype):
return pd.api.types.is_numeric_dtype(
left_dtype
) and pd.api.types.is_numeric_dtype(right_dtype)

left = transform_boolean_operand_to_numeric(left, spark_type=LongType())
right = transform_boolean_operand_to_numeric(right, spark_type=LongType())
left_dtype = left.dtype
if isinstance(right, (IndexOpsMixin, np.ndarray)):
right_dtype = right.dtype
else:
right_dtype = pd.Series([right]).dtype

if left_dtype != right_dtype and not are_both_numeric(left_dtype, right_dtype):
return left._with_new_scol(F.lit(False))

return column_op(PySparkColumn.__eq__)(left, right)

def ne(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
Expand Down