Skip to content

Commit 95ab2dc

Browse files
YuukadesuYuukadesu
authored
Feat:Support ParquetSink (#3601)
* Feat:Support ParquetSink * Update code * Update code
1 parent 89bfeb2 commit 95ab2dc

File tree

7 files changed

+569
-1
lines changed

7 files changed

+569
-1
lines changed

‎pom.xml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@
7070
<grpc-context.version>1.59.1</grpc-context.version>
7171
<httpclient.version>4.5.13</httpclient.version>
7272
<httpcore.version>4.4.9</httpcore.version>
73+
<hadoop.version>3.4.1</hadoop.version>
74+
<hadoop-mapreduce-client-core.version>3.4.1</hadoop-mapreduce-client-core.version>
7375
<influxdb.version>2.24</influxdb.version>
7476
<inlong.version>1.13.0</inlong.version>
7577
<iotdb.version>1.3.0</iotdb.version>
@@ -88,6 +90,7 @@
8890
<javassist.version>3.30.1-GA</javassist.version>
8991
<jboss-logging.version>3.5.2.Final</jboss-logging.version>
9092
<jedis.version>5.1.4</jedis.version>
93+
<jetty-client.version>12.0.19</jetty-client.version>
9194
<jersey.version>3.1.3</jersey.version>
9295
<jgrapht.version>1.5.1</jgrapht.version>
9396
<json-smart.version>2.4.10</json-smart.version>
@@ -114,6 +117,7 @@
114117
<postgresql.version>42.7.2</postgresql.version>
115118
<protobuf.version>4.28.2</protobuf.version>
116119
<pulsar.version>3.2.2</pulsar.version>
120+
<parquet.version>1.12.0</parquet.version>
117121
<quartz.version>2.3.2</quartz.version>
118122
<!-- needs to be lower than 4.x.x, see https://github.com/apache/streampipes/issues/1495 -->
119123
<rdf4j.version>3.7.7</rdf4j.version>
@@ -286,6 +290,16 @@
286290
<groupId>org.apache.tsfile</groupId>
287291
<artifactId>tsfile</artifactId>
288292
<version>${tsfile.version}</version>
293+
<exclusions>
294+
<exclusion>
295+
<groupId>com.github.luben</groupId>
296+
<artifactId>zstd-jni</artifactId>
297+
</exclusion>
298+
<exclusion>
299+
<groupId>org.xerial.snappy</groupId>
300+
<artifactId>snappy-java</artifactId>
301+
</exclusion>
302+
</exclusions>
289303
</dependency>
290304
<dependency>
291305
<groupId>org.tukaani</groupId>
@@ -407,6 +421,104 @@
407421
<artifactId>flink-streaming-java_2.11</artifactId>
408422
<version>${flink.version}</version>
409423
</dependency>
424+
<dependency>
425+
<groupId>org.apache.hadoop</groupId>
426+
<artifactId>hadoop-common</artifactId>
427+
<version>${hadoop.version}</version>
428+
<exclusions>
429+
<exclusion>
430+
<groupId>org.eclipse.jetty</groupId>
431+
<artifactId>jetty-http</artifactId>
432+
</exclusion>
433+
<exclusion>
434+
<groupId>org.eclipse.jetty</groupId>
435+
<artifactId>jetty-io</artifactId>
436+
</exclusion>
437+
<exclusion>
438+
<groupId>org.eclipse.jetty</groupId>
439+
<artifactId>jetty-util</artifactId>
440+
</exclusion>
441+
<exclusion>
442+
<groupId>io.netty</groupId>
443+
<artifactId>netty-handler</artifactId>
444+
</exclusion>
445+
<exclusion>
446+
<groupId>io.netty</groupId>
447+
<artifactId>netty-transport</artifactId>
448+
</exclusion>
449+
<exclusion>
450+
<groupId>io.dropwizard.metrics</groupId>
451+
<artifactId>metrics-core</artifactId>
452+
</exclusion>
453+
<exclusion>
454+
<groupId>ch.qos.reload4j</groupId>
455+
<artifactId>reload4j</artifactId>
456+
</exclusion>
457+
<exclusion>
458+
<groupId>io.netty</groupId>
459+
<artifactId>netty-transport-native-epoll</artifactId>
460+
</exclusion>
461+
<exclusion>
462+
<groupId>org.codehaus.woodstox</groupId>
463+
<artifactId>stax2-api</artifactId>
464+
</exclusion>
465+
<exclusion>
466+
<groupId>org.xerial.snappy</groupId>
467+
<artifactId>snappy-java</artifactId>
468+
</exclusion>
469+
<exclusion>
470+
<groupId>org.apache.avro</groupId>
471+
<artifactId>avro</artifactId>
472+
</exclusion>
473+
<exclusion>
474+
<groupId>com.sun.xml.bind</groupId>
475+
<artifactId>jaxb-impl</artifactId>
476+
</exclusion>
477+
<exclusion>
478+
<groupId>commons-net</groupId>
479+
<artifactId>commons-net</artifactId>
480+
</exclusion>
481+
<exclusion>
482+
<groupId>com.github.luben</groupId>
483+
<artifactId>zstd-jni</artifactId>
484+
</exclusion>
485+
</exclusions>
486+
</dependency>
487+
<dependency>
488+
<groupId>org.apache.hadoop</groupId>
489+
<artifactId>hadoop-mapreduce-client-core</artifactId>
490+
<version>${hadoop-mapreduce-client-core.version}</version>
491+
<exclusions>
492+
<exclusion>
493+
<groupId>org.eclipse.jetty</groupId>
494+
<artifactId>jetty-http</artifactId>
495+
</exclusion>
496+
<exclusion>
497+
<groupId>org.eclipse.jetty</groupId>
498+
<artifactId>jetty-io</artifactId>
499+
</exclusion>
500+
<exclusion>
501+
<groupId>org.eclipse.jetty</groupId>
502+
<artifactId>jetty-util</artifactId>
503+
</exclusion>
504+
<exclusion>
505+
<groupId>com.sun.jersey.contribs</groupId>
506+
<artifactId>jersey-guice</artifactId>
507+
</exclusion>
508+
<exclusion>
509+
<groupId>io.netty</groupId>
510+
<artifactId>netty-all</artifactId>
511+
</exclusion>
512+
<exclusion>
513+
<groupId>org.apache.avro</groupId>
514+
<artifactId>avro</artifactId>
515+
</exclusion>
516+
<exclusion>
517+
<groupId>com.google.inject</groupId>
518+
<artifactId>guice</artifactId>
519+
</exclusion>
520+
</exclusions>
521+
</dependency>
410522
<dependency>
411523
<groupId>org.apache.httpcomponents</groupId>
412524
<artifactId>fluent-hc</artifactId>
@@ -437,6 +549,25 @@
437549
<artifactId>kafka-streams</artifactId>
438550
<version>${kafka.version}</version>
439551
</dependency>
552+
<dependency>
553+
<groupId>org.apache.parquet</groupId>
554+
<artifactId>parquet-avro</artifactId>
555+
<version>${parquet.version}</version>
556+
<exclusions>
557+
<exclusion>
558+
<groupId>org.xerial.snappy</groupId>
559+
<artifactId>snappy-java</artifactId>
560+
</exclusion>
561+
<exclusion>
562+
<groupId>org.apache.avro</groupId>
563+
<artifactId>avro</artifactId>
564+
</exclusion>
565+
<exclusion>
566+
<groupId>com.github.luben</groupId>
567+
<artifactId>zstd-jni</artifactId>
568+
</exclusion>
569+
</exclusions>
570+
</dependency>
440571
<dependency>
441572
<groupId>org.apache.pulsar</groupId>
442573
<artifactId>pulsar-client</artifactId>
@@ -458,6 +589,11 @@
458589
<artifactId>bcutil-jdk18on</artifactId>
459590
<version>${bcprov-jdk18on.version}</version>
460591
</dependency>
592+
<dependency>
593+
<groupId>org.eclipse.jetty</groupId>
594+
<artifactId>jetty-client</artifactId>
595+
<version>${jetty-client.version}</version>
596+
</dependency>
461597
<dependency>
462598
<groupId>org.eclipse.rdf4j</groupId>
463599
<artifactId>rdf4j-rio-turtle</artifactId>

‎streampipes-extensions/streampipes-sinks-databases-jvm/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,23 @@
122122
<groupId>org.antlr</groupId>
123123
<artifactId>antlr4-runtime</artifactId>
124124
</dependency>
125+
<dependency>
126+
<groupId>org.apache.parquet</groupId>
127+
<artifactId>parquet-avro</artifactId>
128+
</dependency>
129+
<dependency>
130+
<groupId>org.apache.hadoop</groupId>
131+
<artifactId>hadoop-common</artifactId>
132+
</dependency>
133+
<dependency>
134+
<groupId>org.apache.hadoop</groupId>
135+
<artifactId>hadoop-mapreduce-client-core</artifactId>
136+
</dependency>
137+
<dependency>
138+
<groupId>org.eclipse.jetty</groupId>
139+
<artifactId>jetty-client</artifactId>
140+
</dependency>
141+
125142

126143

127144

‎streampipes-extensions/streampipes-sinks-databases-jvm/src/main/java/org/apache/streampipes/sinks/databases/jvm/DatabaseSinksExtensionModuleExport.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.streampipes.sinks.databases.jvm.ditto.DittoSink;
2727
import org.apache.streampipes.sinks.databases.jvm.iotdb.IotDbSink;
2828
import org.apache.streampipes.sinks.databases.jvm.milvus.MilvusSink;
29+
import org.apache.streampipes.sinks.databases.jvm.parquet.ParquetSink;
2930
import org.apache.streampipes.sinks.databases.jvm.postgresql.PostgreSqlSink;
3031
import org.apache.streampipes.sinks.databases.jvm.redis.RedisSink;
3132
import org.apache.streampipes.sinks.databases.jvm.tsfile.TsFileSink;
@@ -48,7 +49,8 @@ public List<IStreamPipesPipelineElement<?>> pipelineElements() {
4849
new DittoSink(),
4950
new RedisSink(),
5051
new MilvusSink(),
51-
new TsFileSink()
52+
new TsFileSink(),
53+
new ParquetSink()
5254
);
5355
}
5456

0 commit comments

Comments
 (0)