Skip to content

Commit 1ea7f11

Browse files
LuciferYangyhuang-db
LuciferYang
authored and
yhuang-db
committed
Revert "[SPARK-52283][SQL] Declarative Pipelines DataflowGraph creation and resolution"
This reverts commit 26370d6.
1 parent 3e30ba9 commit 1ea7f11

31 files changed

+2
-5730
lines changed

‎common/utils/src/main/resources/error/error-conditions.json

Lines changed: 0 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,6 @@
8282
],
8383
"sqlState" : "XX000"
8484
},
85-
"APPEND_ONCE_FROM_BATCH_QUERY" : {
86-
"message" : [
87-
"Creating a streaming table from a batch query prevents incremental loading of new data from source. Offending table: '<table>'.",
88-
"Please use the stream() operator. Example usage:",
89-
"CREATE STREAMING TABLE <target table name> ... AS SELECT ... FROM stream(<source table name>) ..."
90-
],
91-
"sqlState" : "42000"
92-
},
9385
"ARITHMETIC_OVERFLOW" : {
9486
"message" : [
9587
"<message>.<alternative> If necessary set <config> to \"false\" to bypass this error."
@@ -1380,12 +1372,6 @@
13801372
},
13811373
"sqlState" : "42734"
13821374
},
1383-
"DUPLICATE_FLOW_SQL_CONF" : {
1384-
"message" : [
1385-
"Found duplicate sql conf for dataset '<datasetName>': '<key>' is defined by both '<flowName1>' and '<flowName2>'"
1386-
],
1387-
"sqlState" : "42710"
1388-
},
13891375
"DUPLICATE_KEY" : {
13901376
"message" : [
13911377
"Found duplicate keys <keyColumn>."
@@ -1957,12 +1943,6 @@
19571943
],
19581944
"sqlState" : "42818"
19591945
},
1960-
"INCOMPATIBLE_BATCH_VIEW_READ" : {
1961-
"message" : [
1962-
"View <datasetIdentifier> is a batch view and must be referenced using SparkSession#read. This check can be disabled by setting Spark conf pipelines.incompatibleViewCheck.enabled = false."
1963-
],
1964-
"sqlState" : "42000"
1965-
},
19661946
"INCOMPATIBLE_COLUMN_TYPE" : {
19671947
"message" : [
19681948
"<operator> can only be performed on tables with compatible column types. The <columnOrdinalNumber> column of the <tableOrdinalNumber> table is <dataType1> type which is not compatible with <dataType2> at the same column of the first table.<hint>."
@@ -2039,12 +2019,6 @@
20392019
],
20402020
"sqlState" : "42613"
20412021
},
2042-
"INCOMPATIBLE_STREAMING_VIEW_READ" : {
2043-
"message" : [
2044-
"View <datasetIdentifier> is a streaming view and must be referenced using SparkSession#readStream. This check can be disabled by setting Spark conf pipelines.incompatibleViewCheck.enabled = false."
2045-
],
2046-
"sqlState" : "42000"
2047-
},
20482022
"INCOMPATIBLE_VIEW_SCHEMA_CHANGE" : {
20492023
"message" : [
20502024
"The SQL query of view <viewName> has an incompatible schema change and column <colName> cannot be resolved. Expected <expectedNum> columns named <colName> but got <actualCols>.",
@@ -3145,12 +3119,6 @@
31453119
},
31463120
"sqlState" : "KD002"
31473121
},
3148-
"INVALID_NAME_IN_USE_COMMAND" : {
3149-
"message" : [
3150-
"Invalid name '<name>' in <command> command. Reason: <reason>"
3151-
],
3152-
"sqlState" : "42000"
3153-
},
31543122
"INVALID_NON_DETERMINISTIC_EXPRESSIONS" : {
31553123
"message" : [
31563124
"The operator expects a deterministic expression, but the actual expression is <sqlExprs>."
@@ -3416,12 +3384,6 @@
34163384
],
34173385
"sqlState" : "22023"
34183386
},
3419-
"INVALID_RESETTABLE_DEPENDENCY" : {
3420-
"message" : [
3421-
"Tables <upstreamResettableTables> are resettable but have a non-resettable downstream dependency '<downstreamTable>'. `reset` will fail as Spark Streaming does not support deleted source data. You can either remove the <resetAllowedKey>=false property from '<downstreamTable>' or add it to its upstream dependencies."
3422-
],
3423-
"sqlState" : "42000"
3424-
},
34253387
"INVALID_RESET_COMMAND_FORMAT" : {
34263388
"message" : [
34273389
"Expected format is 'RESET' or 'RESET key'. If you want to include special characters in key, please use quotes, e.g., RESET `key`."
@@ -5457,19 +5419,6 @@
54575419
],
54585420
"sqlState" : "58030"
54595421
},
5460-
"UNABLE_TO_INFER_PIPELINE_TABLE_SCHEMA" : {
5461-
"message" : [
5462-
"Failed to infer the schema for table <tableName> from its upstream flows.",
5463-
"Please modify the flows that write to this table to make their schemas compatible.",
5464-
"",
5465-
"Inferred schema so far:",
5466-
"<inferredDataSchema>",
5467-
"",
5468-
"Incompatible schema:",
5469-
"<incompatibleDataSchema>"
5470-
],
5471-
"sqlState" : "42KD9"
5472-
},
54735422
"UNABLE_TO_INFER_SCHEMA" : {
54745423
"message" : [
54755424
"Unable to infer schema for <format>. It must be specified manually."
@@ -5641,12 +5590,6 @@
56415590
],
56425591
"sqlState" : "42883"
56435592
},
5644-
"UNRESOLVED_TABLE_PATH" : {
5645-
"message" : [
5646-
"Storage path for table <identifier> cannot be resolved."
5647-
],
5648-
"sqlState" : "22KD1"
5649-
},
56505593
"UNRESOLVED_USING_COLUMN_FOR_JOIN" : {
56515594
"message" : [
56525595
"USING column <colName> cannot be resolved on the <side> side of the join. The <side>-side columns: [<suggestion>]."
@@ -6628,20 +6571,6 @@
66286571
],
66296572
"sqlState" : "P0001"
66306573
},
6631-
"USER_SPECIFIED_AND_INFERRED_SCHEMA_NOT_COMPATIBLE" : {
6632-
"message" : [
6633-
"Table '<tableName>' has a user-specified schema that is incompatible with the schema",
6634-
"inferred from its query.",
6635-
"<streamingTableHint>",
6636-
"",
6637-
"Declared schema:",
6638-
"<specifiedSchema>",
6639-
"",
6640-
"Inferred schema:",
6641-
"<inferredDataSchema>"
6642-
],
6643-
"sqlState" : "42000"
6644-
},
66456574
"VARIABLE_ALREADY_EXISTS" : {
66466575
"message" : [
66476576
"Cannot create the variable <variableName> because it already exists.",

‎project/SparkBuild.scala

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -418,8 +418,6 @@ object SparkBuild extends PomBuild {
418418

419419
enable(HiveThriftServer.settings)(hiveThriftServer)
420420

421-
enable(SparkDeclarativePipelines.settings)(pipelines)
422-
423421
enable(SparkConnectCommon.settings)(connectCommon)
424422
enable(SparkConnect.settings)(connect)
425423
enable(SparkConnectClient.settings)(connectClient)
@@ -886,57 +884,6 @@ object SparkConnectClient {
886884
)
887885
}
888886

889-
object SparkDeclarativePipelines {
890-
import BuildCommons.protoVersion
891-
892-
lazy val settings = Seq(
893-
// For some reason the resolution from the imported Maven build does not work for some
894-
// of these dependendencies that we need to shade later on.
895-
libraryDependencies ++= {
896-
val guavaVersion =
897-
SbtPomKeys.effectivePom.value.getProperties.get(
898-
"connect.guava.version").asInstanceOf[String]
899-
val guavaFailureAccessVersion =
900-
SbtPomKeys.effectivePom.value.getProperties.get(
901-
"guava.failureaccess.version").asInstanceOf[String]
902-
Seq(
903-
"com.google.guava" % "guava" % guavaVersion,
904-
"com.google.guava" % "failureaccess" % guavaFailureAccessVersion,
905-
"com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf"
906-
)
907-
},
908-
909-
(assembly / logLevel) := Level.Info,
910-
911-
// Exclude `scala-library` from assembly.
912-
(assembly / assemblyPackageScala / assembleArtifact) := false,
913-
914-
// SPARK-46733: Include `spark-connect-*.jar`, `unused-*.jar`,`guava-*.jar`,
915-
// `failureaccess-*.jar`, `annotations-*.jar`, `grpc-*.jar`, `protobuf-*.jar`,
916-
// `gson-*.jar`, `error_prone_annotations-*.jar`, `j2objc-annotations-*.jar`,
917-
// `animal-sniffer-annotations-*.jar`, `perfmark-api-*.jar`,
918-
// `proto-google-common-protos-*.jar` in assembly.
919-
// This needs to be consistent with the content of `maven-shade-plugin`.
920-
(assembly / assemblyExcludedJars) := {
921-
val cp = (assembly / fullClasspath).value
922-
val validPrefixes = Set("spark-connect", "unused-", "guava-", "failureaccess-",
923-
"annotations-", "grpc-", "protobuf-", "gson", "error_prone_annotations",
924-
"j2objc-annotations", "animal-sniffer-annotations", "perfmark-api",
925-
"proto-google-common-protos")
926-
cp filterNot { v =>
927-
validPrefixes.exists(v.data.getName.startsWith)
928-
}
929-
},
930-
931-
(assembly / assemblyMergeStrategy) := {
932-
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") => MergeStrategy.discard
933-
// Drop all proto files that are not needed as artifacts of the build.
934-
case m if m.toLowerCase(Locale.ROOT).endsWith(".proto") => MergeStrategy.discard
935-
case _ => MergeStrategy.first
936-
}
937-
)
938-
}
939-
940887
object SparkProtobuf {
941888
import BuildCommons.protoVersion
942889

‎sql/pipelines/pom.xml

Lines changed: 2 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616
~ limitations under the License.
1717
-->
1818

19-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2120
<modelVersion>4.0.0</modelVersion>
2221
<parent>
2322
<groupId>org.apache.spark</groupId>
@@ -45,125 +44,14 @@
4544
</dependency>
4645
<dependency>
4746
<groupId>org.apache.spark</groupId>
48-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
49-
<version>${project.version}</version>
50-
</dependency>
51-
<dependency>
52-
<groupId>org.apache.spark</groupId>
53-
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
54-
<version>${project.version}</version>
55-
<type>test-jar</type>
56-
<scope>test</scope>
57-
</dependency>
58-
<dependency>
59-
<groupId>org.apache.spark</groupId>
60-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
61-
<version>${project.version}</version>
62-
<type>test-jar</type>
63-
<scope>test</scope>
64-
</dependency>
65-
<dependency>
66-
<groupId>org.apache.spark</groupId>
67-
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
47+
<artifactId>spark-core_${scala.binary.version}</artifactId>
6848
<version>${project.version}</version>
6949
<type>test-jar</type>
7050
<scope>test</scope>
71-
<exclusions>
72-
<exclusion>
73-
<groupId>org.apache.spark</groupId>
74-
<artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
75-
</exclusion>
76-
</exclusions>
77-
</dependency>
78-
<dependency>
79-
<groupId>org.scala-lang.modules</groupId>
80-
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
8151
</dependency>
82-
<dependency>
83-
<groupId>org.scalacheck</groupId>
84-
<artifactId>scalacheck_${scala.binary.version}</artifactId>
85-
<scope>test</scope>
86-
</dependency>
87-
<dependency>
88-
<groupId>org.mockito</groupId>
89-
<artifactId>mockito-core</artifactId>
90-
<scope>test</scope>
91-
</dependency>
92-
<dependency>
93-
<groupId>net.bytebuddy</groupId>
94-
<artifactId>byte-buddy</artifactId>
95-
<scope>test</scope>
96-
</dependency>
97-
<dependency>
98-
<groupId>net.bytebuddy</groupId>
99-
<artifactId>byte-buddy-agent</artifactId>
100-
<scope>test</scope>
101-
</dependency>
102-
<dependency>
103-
<groupId>org.apache.spark</groupId>
104-
<artifactId>spark-tags_${scala.binary.version}</artifactId>
105-
</dependency>
106-
107-
<!--
108-
This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
109-
them will yield errors.
110-
-->
111-
<dependency>
112-
<groupId>org.apache.spark</groupId>
113-
<artifactId>spark-tags_${scala.binary.version}</artifactId>
114-
<type>test-jar</type>
115-
<scope>test</scope>
116-
</dependency>
117-
11852
</dependencies>
11953
<build>
12054
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
12155
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
122-
<plugins>
123-
<plugin>
124-
<groupId>kr.motd.maven</groupId>
125-
<artifactId>os-maven-plugin</artifactId>
126-
<version>1.7.0</version>
127-
<extensions>true</extensions>
128-
</plugin>
129-
<plugin>
130-
<groupId>org.xolstice.maven.plugins</groupId>
131-
<artifactId>protobuf-maven-plugin</artifactId>
132-
<version>0.6.1</version>
133-
<configuration>
134-
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
135-
</protocArtifact>
136-
<protoSourceRoot>src/main/protobuf</protoSourceRoot>
137-
<pluginId>grpc-java</pluginId>
138-
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}
139-
</pluginArtifact>
140-
</configuration>
141-
<executions>
142-
<execution>
143-
<phase>generate-sources</phase>
144-
<goals>
145-
<goal>compile</goal>
146-
<goal>compile-custom</goal>
147-
<goal>test-compile</goal>
148-
</goals>
149-
</execution>
150-
</executions>
151-
</plugin>
152-
<plugin>
153-
<!-- see http://davidb.github.com/scala-maven-plugin -->
154-
<groupId>net.alchim31.maven</groupId>
155-
<artifactId>scala-maven-plugin</artifactId>
156-
<version>4.9.2</version>
157-
<executions>
158-
<execution>
159-
<goals>
160-
<goal>compile</goal>
161-
<goal>testCompile</goal>
162-
</goals>
163-
</execution>
164-
</executions>
165-
</plugin>
166-
<!-- os-maven-plugin helps resolve ${os.detected.classifier} automatically -->
167-
</plugins>
16856
</build>
16957
</project>

‎sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/AnalysisWarning.scala

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/Language.scala

Lines changed: 0 additions & 26 deletions
This file was deleted.

0 commit comments

Comments
 (0)