Skip to content

Commit 6961ed9

Browse files
zqburdezqburde
committed
Add sparketl appconn
1 parent 84d710a commit 6961ed9

File tree

19 files changed

+1000
-10
lines changed

19 files changed

+1000
-10
lines changed

‎assembly/bin/appconn-refresh.sh

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ if [ -z $SOURCE_ROOT ]; then
1111
source ${SOURCE_ROOT}/conf/db.sh
1212
fi
1313

14+
function refresh() {
15+
while true; do
16+
response=$(curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load)
17+
if [[ $response == *"succeed"* ]]; then
18+
break
19+
else
20+
sleep 5
21+
fi
22+
done
23+
}
24+
1425
function isSuccess(){
1526
if [ $? -ne 0 ]; then
1627
echo "Failed to " + $1
@@ -39,18 +50,18 @@ if [ -z $1 ];then
3950
sh $SOURCE_ROOT/sbin/dss-start-all.sh
4051
else
4152
echo "You chose to wait for 5 minutes."
42-
echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
43-
curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
44-
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
53+
echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
54+
refresh
55+
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
4556
echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
4657
echo ""
4758
exit 0
4859
fi
4960
else
5061
APPCONN_NAME=$1
51-
echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
52-
curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
53-
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
62+
echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
63+
refresh
64+
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
5465
echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
5566
echo ""
5667
exit 0

‎assembly/dss-package/src/main/assembly/distribution.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,16 @@
210210
</includes>
211211
</fileSet>
212212

213+
<fileSet>
214+
<directory>
215+
${basedir}/../../dss-appconn/appconns/dss-sparketl-appconn/target/out/
216+
</directory>
217+
<outputDirectory>dss-appconns</outputDirectory>
218+
<includes>
219+
<include>**/*</include>
220+
</includes>
221+
</fileSet>
222+
213223
<fileSet>
214224
<directory>
215225
${basedir}/../../dss-appconn/linkis-appconn-engineplugin/target/

‎conf/dss-apps-server.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ wds.linkis.datasource.hikari.minimumIdle=10
1515
#### scriptis######
1616
wds.dss.scriptis.global.limits.exportResEnable=false
1717
wds.dss.scriptis.global.limits.exportTableEnable=false
18-
wds.dss.scriptis.global.limits.downloadResEnable=false
18+
wds.dss.scriptis.global.limits.downloadResEnable=true
1919
wds.dss.scriptis.global.limits.resCopyEnable=false
2020
wds.dss.scriptis.global.limits.proxyEnable=false
2121

‎dss-appconn/appconns/dss-sparketl-appconn/pom.xml

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>dss</artifactId>
7+
<groupId>com.webank.wedatasphere.dss</groupId>
8+
<version>1.1.2</version>
9+
<relativePath>../../../pom.xml</relativePath>
10+
</parent>
11+
<modelVersion>4.0.0</modelVersion>
12+
13+
<artifactId>dss-sparketl-appconn</artifactId>
14+
15+
<dependencies>
16+
<dependency>
17+
<groupId>com.webank.wedatasphere.dss</groupId>
18+
<artifactId>dss-appconn-core</artifactId>
19+
<version>${dss.version}</version>
20+
<exclusions>
21+
<exclusion>
22+
<artifactId>linkis-common</artifactId>
23+
<groupId>org.apache.linkis</groupId>
24+
</exclusion>
25+
<exclusion>
26+
<artifactId>json4s-jackson_2.11</artifactId>
27+
<groupId>org.json4s</groupId>
28+
</exclusion>
29+
<exclusion>
30+
<groupId>com.webank.wedatasphere.dss</groupId>
31+
<artifactId>dss-origin-sso-integration-standard</artifactId>
32+
</exclusion>
33+
</exclusions>
34+
</dependency>
35+
36+
<dependency>
37+
<groupId>com.webank.wedatasphere.dss</groupId>
38+
<artifactId>dss-development-process-standard</artifactId>
39+
<version>${dss.version}</version>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>com.webank.wedatasphere.dss</groupId>
44+
<artifactId>dss-development-process-standard-execution</artifactId>
45+
<version>${dss.version}</version>
46+
</dependency>
47+
48+
<dependency>
49+
<groupId>org.apache.linkis</groupId>
50+
<artifactId>linkis-module</artifactId>
51+
<version>${linkis.version}</version>
52+
<scope>provided</scope>
53+
<exclusions>
54+
<exclusion>
55+
<artifactId>httpclient</artifactId>
56+
<groupId>org.apache.httpcomponents</groupId>
57+
</exclusion>
58+
</exclusions>
59+
<optional>true</optional>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.apache.linkis</groupId>
63+
<artifactId>linkis-cs-common</artifactId>
64+
<version>${linkis.version}</version>
65+
<scope>compile</scope>
66+
</dependency>
67+
<dependency>
68+
<artifactId>linkis-bml-client</artifactId>
69+
<exclusions>
70+
<exclusion>
71+
<artifactId>gson</artifactId>
72+
<groupId>com.google.code.gson</groupId>
73+
</exclusion>
74+
</exclusions>
75+
<groupId>org.apache.linkis</groupId>
76+
<version>${linkis.version}</version>
77+
<scope>provided</scope>
78+
<optional>true</optional>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>org.apache.linkis</groupId>
83+
<artifactId>linkis-httpclient</artifactId>
84+
<version>${linkis.version}</version>
85+
<exclusions>
86+
<exclusion>
87+
<artifactId>linkis-common</artifactId>
88+
<groupId>org.apache.linkis</groupId>
89+
</exclusion>
90+
<exclusion>
91+
<artifactId>json4s-jackson_2.11</artifactId>
92+
<groupId>org.json4s</groupId>
93+
</exclusion>
94+
</exclusions>
95+
</dependency>
96+
97+
<dependency>
98+
<groupId>org.apache.linkis</groupId>
99+
<artifactId>linkis-storage</artifactId>
100+
<version>${linkis.version}</version>
101+
<scope>provided</scope>
102+
<exclusions>
103+
<exclusion>
104+
<artifactId>linkis-common</artifactId>
105+
<groupId>org.apache.linkis</groupId>
106+
</exclusion>
107+
</exclusions>
108+
</dependency>
109+
110+
<dependency>
111+
<groupId>com.webank.wedatasphere.dss</groupId>
112+
<artifactId>dss-common</artifactId>
113+
<version>${dss.version}</version>
114+
<scope>provided</scope>
115+
</dependency>
116+
117+
</dependencies>
118+
119+
<build>
120+
<plugins>
121+
<plugin>
122+
<groupId>org.apache.maven.plugins</groupId>
123+
<artifactId>maven-deploy-plugin</artifactId>
124+
</plugin>
125+
126+
<plugin>
127+
<groupId>net.alchim31.maven</groupId>
128+
<artifactId>scala-maven-plugin</artifactId>
129+
</plugin>
130+
<plugin>
131+
<groupId>org.apache.maven.plugins</groupId>
132+
<artifactId>maven-jar-plugin</artifactId>
133+
</plugin>
134+
<plugin>
135+
<groupId>org.apache.maven.plugins</groupId>
136+
<artifactId>maven-assembly-plugin</artifactId>
137+
<version>2.3</version>
138+
<inherited>false</inherited>
139+
<executions>
140+
<execution>
141+
<id>make-assembly</id>
142+
<phase>package</phase>
143+
<goals>
144+
<goal>single</goal>
145+
</goals>
146+
<configuration>
147+
<descriptors>
148+
<descriptor>src/main/assembly/distribution.xml</descriptor>
149+
</descriptors>
150+
</configuration>
151+
</execution>
152+
</executions>
153+
<configuration>
154+
<skipAssembly>false</skipAssembly>
155+
<finalName>out</finalName>
156+
<appendAssemblyId>false</appendAssemblyId>
157+
<attach>false</attach>
158+
<descriptors>
159+
<descriptor>src/main/assembly/distribution.xml</descriptor>
160+
</descriptors>
161+
</configuration>
162+
</plugin>
163+
</plugins>
164+
</build>
165+
166+
</project>

‎dss-appconn/appconns/dss-sparketl-appconn/src/main/assembly/distribution.xml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<!--
2+
~ Copyright 2019 WeBank
3+
~ Licensed under the Apache License, Version 2.0 (the "License");
4+
~ you may not use this file except in compliance with the License.
5+
~ You may obtain a copy of the License at
6+
~
7+
~ http://www.apache.org/licenses/LICENSE-2.0
8+
~
9+
~ Unless required by applicable law or agreed to in writing, software
10+
~ distributed under the License is distributed on an "AS IS" BASIS,
11+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
~ See the License for the specific language governing permissions and
13+
~ limitations under the License.
14+
~
15+
-->
16+
17+
<assembly
18+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
21+
<id>dss-sparketl-appconn</id>
22+
<formats>
23+
<format>dir</format>
24+
</formats>
25+
<includeBaseDirectory>true</includeBaseDirectory>
26+
<baseDirectory>sparketl</baseDirectory>
27+
28+
<dependencySets>
29+
<dependencySet>
30+
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
31+
<!-- Now, select which projects to include in this module-set. -->
32+
<outputDirectory>lib</outputDirectory>
33+
<useProjectArtifact>true</useProjectArtifact>
34+
<useTransitiveDependencies>true</useTransitiveDependencies>
35+
<unpack>false</unpack>
36+
<useStrictFiltering>true</useStrictFiltering>
37+
<useTransitiveFiltering>true</useTransitiveFiltering>
38+
</dependencySet>
39+
</dependencySets>
40+
41+
<fileSets>
42+
<fileSet>
43+
<directory>${basedir}/src/main/resources</directory>
44+
<includes>
45+
<include>appconn.properties</include>
46+
</includes>
47+
<fileMode>0777</fileMode>
48+
<outputDirectory>/</outputDirectory>
49+
<lineEnding>unix</lineEnding>
50+
</fileSet>
51+
52+
<fileSet>
53+
<directory>${basedir}/src/main/resources</directory>
54+
<includes>
55+
<include>log4j.properties</include>
56+
<include>log4j2.xml</include>
57+
</includes>
58+
<fileMode>0777</fileMode>
59+
<outputDirectory>conf</outputDirectory>
60+
<lineEnding>unix</lineEnding>
61+
</fileSet>
62+
63+
<fileSet>
64+
<directory>${basedir}/src/main/resources</directory>
65+
<includes>
66+
<include>init.sql</include>
67+
</includes>
68+
<fileMode>0777</fileMode>
69+
<outputDirectory>db</outputDirectory>
70+
<!-- <lineEnding>unix</lineEnding>-->
71+
</fileSet>
72+
73+
</fileSets>
74+
75+
</assembly>
76+

‎dss-appconn/appconns/dss-sparketl-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/sparketl/SparkEtlAppConn.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.webank.wedatasphere.dss.appconn.sparketl;
2+
3+
import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn;
4+
import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn;
5+
import com.webank.wedatasphere.dss.appconn.sparketl.standard.SparkEtlDevelopmentStandard;
6+
import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;
7+
import org.apache.linkis.common.conf.CommonVars;
8+
9+
10+
public class SparkEtlAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn {
11+
12+
public static final String SPARKETL_APPCONN_NAME = CommonVars.apply("wds.dss.appconn.sparketl.name", "sparketl").getValue();
13+
14+
private SparkEtlDevelopmentStandard developmentStandard;
15+
16+
@Override
17+
protected void initialize() {
18+
developmentStandard = new SparkEtlDevelopmentStandard();
19+
}
20+
21+
@Override
22+
public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() {
23+
return developmentStandard;
24+
}
25+
26+
27+
28+
29+
}

‎dss-appconn/appconns/dss-sparketl-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/sparketl/constant/SparkEtlTaskStatusEnum.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.webank.wedatasphere.dss.appconn.sparketl.constant;
2+
3+
public enum SparkEtlTaskStatusEnum {
4+
5+
/**
6+
* Task status enum
7+
*/
8+
SUBMITTED(1, "已提交", "SUBMITTED"),
9+
INITED(2, "初始化", "INITED"),
10+
RUNNING(3, "运行中", "RUNNING"),
11+
SUCCEED(4, "成功", "SUCCEED"),
12+
PASS_CHECKOUT(5, "通过校验", "PASS_CHECKOUT"),
13+
FAIL_CHECKOUT(6, "未通过校验", "FAIL_CHECKOUT"),
14+
FAILED(7, "失败", "FAILED"),
15+
TASK_NOT_EXIST(8, "Task不存在", "TASK_NOT_EXIST"),
16+
CANCELLED(9, "取消", "CANCELLED"),
17+
TIMEOUT(10, "超时", "TIMEOUT"),
18+
SCHEDULED(11, "调度中", "SCHEDULED"),
19+
SUBMIT_PENDING(12, "提交阻塞", "SUBMIT_PENDING");
20+
21+
private Integer code;
22+
private String message;
23+
private String state;
24+
25+
SparkEtlTaskStatusEnum(Integer code, String message, String state) {
26+
this.code = code;
27+
this.message = message;
28+
this.state = state;
29+
}
30+
31+
public Integer getCode() {
32+
return code;
33+
}
34+
35+
public String getMessage() {
36+
return message;
37+
}
38+
39+
public String getState() {
40+
return state;
41+
}
42+
43+
}

0 commit comments

Comments
 (0)