Skip to content

Commit e999f93

Browse files
anishshri-dbyhuang-db
anishshri-db
authored and
yhuang-db
committed
[SPARK-52329][SS] Remove private sql scoping tags for new transformWithState API
### What changes were proposed in this pull request? Remove private sql scoping tags for new transformWithState API ### Why are the changes needed? To allow users to use the new operator in their queries without scoping explicitly to sql ### Does this PR introduce _any_ user-facing change? Yes ### How was this patch tested? Existing unit tests ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#51033 from anishshri-db/task/SPARK-52329. Authored-by: Anish Shrigondekar <anish.shrigondekar@databricks.com> Signed-off-by: Jungtaek Lim <kabhwan.opensource@gmail.com>
1 parent e757b94 commit e999f93

File tree

3 files changed

+28
-28
lines changed

3 files changed

+28
-28
lines changed

‎sql/api/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
566566
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
567567
* SQL.
568568
*/
569-
private[sql] def transformWithState[U: Encoder](
569+
def transformWithState[U: Encoder](
570570
statefulProcessor: StatefulProcessor[K, V, U],
571571
timeMode: TimeMode,
572572
outputMode: OutputMode): Dataset[U]
@@ -595,7 +595,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
595595
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
596596
* SQL.
597597
*/
598-
private[sql] def transformWithState[U: Encoder](
598+
def transformWithState[U: Encoder](
599599
statefulProcessor: StatefulProcessor[K, V, U],
600600
eventTimeColumnName: String,
601601
outputMode: OutputMode): Dataset[U]
@@ -621,7 +621,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
621621
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
622622
* SQL.
623623
*/
624-
private[sql] def transformWithState[U: Encoder](
624+
def transformWithState[U: Encoder](
625625
statefulProcessor: StatefulProcessor[K, V, U],
626626
timeMode: TimeMode,
627627
outputMode: OutputMode,
@@ -657,7 +657,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
657657
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
658658
* SQL.
659659
*/
660-
private[sql] def transformWithState[U: Encoder](
660+
def transformWithState[U: Encoder](
661661
statefulProcessor: StatefulProcessor[K, V, U],
662662
eventTimeColumnName: String,
663663
outputMode: OutputMode,
@@ -686,7 +686,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
686686
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
687687
* SQL.
688688
*/
689-
private[sql] def transformWithState[U: Encoder, S: Encoder](
689+
def transformWithState[U: Encoder, S: Encoder](
690690
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
691691
timeMode: TimeMode,
692692
outputMode: OutputMode,
@@ -719,7 +719,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
719719
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
720720
* SQL.
721721
*/
722-
private[sql] def transformWithState[U: Encoder, S: Encoder](
722+
def transformWithState[U: Encoder, S: Encoder](
723723
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
724724
eventTimeColumnName: String,
725725
outputMode: OutputMode,
@@ -751,7 +751,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
751751
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
752752
* SQL.
753753
*/
754-
private[sql] def transformWithState[U: Encoder, S: Encoder](
754+
def transformWithState[U: Encoder, S: Encoder](
755755
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
756756
timeMode: TimeMode,
757757
outputMode: OutputMode,
@@ -793,7 +793,7 @@ abstract class KeyValueGroupedDataset[K, V] extends Serializable {
793793
* See [[org.apache.spark.sql.Encoder]] for more details on what types are encodable to Spark
794794
* SQL.
795795
*/
796-
private[sql] def transformWithState[U: Encoder, S: Encoder](
796+
def transformWithState[U: Encoder, S: Encoder](
797797
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
798798
outputMode: OutputMode,
799799
initialState: KeyValueGroupedDataset[K, S],

‎sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -137,22 +137,22 @@ class KeyValueGroupedDataset[K, V] private[sql] () extends sql.KeyValueGroupedDa
137137
}
138138

139139
/** @inheritdoc */
140-
private[sql] def transformWithState[U: Encoder](
140+
def transformWithState[U: Encoder](
141141
statefulProcessor: StatefulProcessor[K, V, U],
142142
timeMode: TimeMode,
143143
outputMode: OutputMode): Dataset[U] =
144144
transformWithStateHelper(statefulProcessor, timeMode, outputMode)
145145

146146
/** @inheritdoc */
147-
private[sql] def transformWithState[U: Encoder, S: Encoder](
147+
def transformWithState[U: Encoder, S: Encoder](
148148
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
149149
timeMode: TimeMode,
150150
outputMode: OutputMode,
151151
initialState: sql.KeyValueGroupedDataset[K, S]): Dataset[U] =
152152
transformWithStateHelper(statefulProcessor, timeMode, outputMode, Some(initialState))
153153

154154
/** @inheritdoc */
155-
override private[sql] def transformWithState[U: Encoder](
155+
override def transformWithState[U: Encoder](
156156
statefulProcessor: StatefulProcessor[K, V, U],
157157
eventTimeColumnName: String,
158158
outputMode: OutputMode): Dataset[U] =
@@ -163,7 +163,7 @@ class KeyValueGroupedDataset[K, V] private[sql] () extends sql.KeyValueGroupedDa
163163
eventTimeColumnName = eventTimeColumnName)
164164

165165
/** @inheritdoc */
166-
override private[sql] def transformWithState[U: Encoder, S: Encoder](
166+
override def transformWithState[U: Encoder, S: Encoder](
167167
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
168168
eventTimeColumnName: String,
169169
outputMode: OutputMode,
@@ -261,29 +261,29 @@ class KeyValueGroupedDataset[K, V] private[sql] () extends sql.KeyValueGroupedDa
261261
initialState)
262262

263263
/** @inheritdoc */
264-
override private[sql] def transformWithState[U: Encoder](
264+
override def transformWithState[U: Encoder](
265265
statefulProcessor: StatefulProcessor[K, V, U],
266266
timeMode: TimeMode,
267267
outputMode: OutputMode,
268-
outputEncoder: Encoder[U]) =
268+
outputEncoder: Encoder[U]): Dataset[U] =
269269
super.transformWithState(statefulProcessor, timeMode, outputMode, outputEncoder)
270270

271271
/** @inheritdoc */
272-
override private[sql] def transformWithState[U: Encoder](
272+
override def transformWithState[U: Encoder](
273273
statefulProcessor: StatefulProcessor[K, V, U],
274274
eventTimeColumnName: String,
275275
outputMode: OutputMode,
276-
outputEncoder: Encoder[U]) =
276+
outputEncoder: Encoder[U]): Dataset[U] =
277277
super.transformWithState(statefulProcessor, eventTimeColumnName, outputMode, outputEncoder)
278278

279279
/** @inheritdoc */
280-
override private[sql] def transformWithState[U: Encoder, S: Encoder](
280+
override def transformWithState[U: Encoder, S: Encoder](
281281
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
282282
timeMode: TimeMode,
283283
outputMode: OutputMode,
284284
initialState: sql.KeyValueGroupedDataset[K, S],
285285
outputEncoder: Encoder[U],
286-
initialStateEncoder: Encoder[S]) = super.transformWithState(
286+
initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
287287
statefulProcessor,
288288
timeMode,
289289
outputMode,
@@ -292,13 +292,13 @@ class KeyValueGroupedDataset[K, V] private[sql] () extends sql.KeyValueGroupedDa
292292
initialStateEncoder)
293293

294294
/** @inheritdoc */
295-
override private[sql] def transformWithState[U: Encoder, S: Encoder](
295+
override def transformWithState[U: Encoder, S: Encoder](
296296
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
297297
outputMode: OutputMode,
298298
initialState: sql.KeyValueGroupedDataset[K, S],
299299
eventTimeColumnName: String,
300300
outputEncoder: Encoder[U],
301-
initialStateEncoder: Encoder[S]) = super.transformWithState(
301+
initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
302302
statefulProcessor,
303303
outputMode,
304304
initialState,

‎sql/core/src/main/scala/org/apache/spark/sql/classic/KeyValueGroupedDataset.scala

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -444,29 +444,29 @@ class KeyValueGroupedDataset[K, V] private[sql](
444444
initialState)
445445

446446
/** @inheritdoc */
447-
override private[sql] def transformWithState[U: Encoder](
447+
override def transformWithState[U: Encoder](
448448
statefulProcessor: StatefulProcessor[K, V, U],
449449
timeMode: TimeMode,
450450
outputMode: OutputMode,
451-
outputEncoder: Encoder[U]) =
451+
outputEncoder: Encoder[U]): Dataset[U] =
452452
super.transformWithState(statefulProcessor, timeMode, outputMode, outputEncoder)
453453

454454
/** @inheritdoc */
455-
override private[sql] def transformWithState[U: Encoder](
455+
override def transformWithState[U: Encoder](
456456
statefulProcessor: StatefulProcessor[K, V, U],
457457
eventTimeColumnName: String,
458458
outputMode: OutputMode,
459-
outputEncoder: Encoder[U]) =
459+
outputEncoder: Encoder[U]): Dataset[U] =
460460
super.transformWithState(statefulProcessor, eventTimeColumnName, outputMode, outputEncoder)
461461

462462
/** @inheritdoc */
463-
override private[sql] def transformWithState[U: Encoder, S: Encoder](
463+
override def transformWithState[U: Encoder, S: Encoder](
464464
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
465465
timeMode: TimeMode,
466466
outputMode: OutputMode,
467467
initialState: sql.KeyValueGroupedDataset[K, S],
468468
outputEncoder: Encoder[U],
469-
initialStateEncoder: Encoder[S]) = super.transformWithState(
469+
initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
470470
statefulProcessor,
471471
timeMode,
472472
outputMode,
@@ -475,13 +475,13 @@ class KeyValueGroupedDataset[K, V] private[sql](
475475
initialStateEncoder)
476476

477477
/** @inheritdoc */
478-
override private[sql] def transformWithState[U: Encoder, S: Encoder](
478+
override def transformWithState[U: Encoder, S: Encoder](
479479
statefulProcessor: StatefulProcessorWithInitialState[K, V, U, S],
480480
outputMode: OutputMode,
481481
initialState: sql.KeyValueGroupedDataset[K, S],
482482
eventTimeColumnName: String,
483483
outputEncoder: Encoder[U],
484-
initialStateEncoder: Encoder[S]) = super.transformWithState(
484+
initialStateEncoder: Encoder[S]): Dataset[U] = super.transformWithState(
485485
statefulProcessor,
486486
outputMode,
487487
initialState,

0 commit comments

Comments
 (0)