Skip to content

Commit 22b8269

Browse files
zqburdezqburde
authored
Merge pull request #1101 from WeDataSphere/master
Fix some bugs.
2 parents 82bc101 + 584b04f commit 22b8269

File tree

70 files changed

+20537
-16871
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+20537
-16871
lines changed

‎.github/workflows/dlc.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
11
{
2+
"ignorePatterns": [
3+
{
4+
"pattern": "https://github.com/WeBankFinTech/DataSphereStudio-Doc"
5+
},
6+
{
7+
"pattern": "http://localhost"
8+
},
9+
{
10+
"pattern": "http://127.0.0.1"
11+
}
12+
],
213
"timeout": "10s",
314
"retryOn429": true,
415
"retryCount": 10,

‎assembly/bin/appconn-install.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ function isSuccess(){
2626
fi
2727
}
2828

29-
PROC_NAME=DSSProjectServerApplication
29+
PROC_NAME=DSSServerApplication
3030
ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l`
3131
if [ $ProcNumber -le 0 ];then
3232
echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started."

‎assembly/bin/appconn-refresh.sh

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ if [ -z $SOURCE_ROOT ]; then
1111
source ${SOURCE_ROOT}/conf/db.sh
1212
fi
1313

14+
function refresh() {
15+
while true; do
16+
response=$(curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load)
17+
if [[ $response == *"succeed"* ]]; then
18+
break
19+
else
20+
sleep 5
21+
fi
22+
done
23+
}
24+
1425
function isSuccess(){
1526
if [ $? -ne 0 ]; then
1627
echo "Failed to " + $1
@@ -39,18 +50,18 @@ if [ -z $1 ];then
3950
sh $SOURCE_ROOT/sbin/dss-start-all.sh
4051
else
4152
echo "You chose to wait for 5 minutes."
42-
echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
43-
curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
44-
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
53+
echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
54+
refresh
55+
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
4556
echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
4657
echo ""
4758
exit 0
4859
fi
4960
else
5061
APPCONN_NAME=$1
51-
echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
52-
curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
53-
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
62+
echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
63+
refresh
64+
isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
5465
echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
5566
echo ""
5667
exit 0

‎assembly/bin/dssinstall.sh

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
#!/bin/sh
2+
#Actively load user env
3+
if [ -f "~/.bashrc" ];then
4+
echo "Warning! user bashrc file does not exist."
5+
else
6+
source ~/.bashrc
7+
fi
8+
9+
shellDir=`dirname $0`
10+
workDir=`cd ${shellDir}/..;pwd`
11+
12+
SERVER_IP=""
13+
SERVER_HOME=""
14+
15+
local_host="`hostname --fqdn`"
16+
LOCAL_IP=$(hostname -I | sed 's/^\([0-9.]\+\) .*/\1/')
17+
18+
#To be compatible with MacOS and Linux
19+
txt=""
20+
if [[ "$OSTYPE" == "darwin"* ]]; then
21+
txt="''"
22+
elif [[ "$OSTYPE" == "linux-gnu" ]]; then
23+
txt=""
24+
elif [[ "$OSTYPE" == "cygwin" ]]; then
25+
echo "dss not support Windows operating system"
26+
exit 1
27+
elif [[ "$OSTYPE" == "msys" ]]; then
28+
echo "dss not support Windows operating system"
29+
exit 1
30+
elif [[ "$OSTYPE" == "win32" ]]; then
31+
echo "dss not support Windows operating system"
32+
exit 1
33+
elif [[ "$OSTYPE" == "freebsd"* ]]; then
34+
txt=""
35+
else
36+
echo "Operating system unknown, please tell us(submit issue) for better service"
37+
exit 1
38+
fi
39+
40+
function isSuccess(){
41+
if [ $? -ne 0 ]; then
42+
echo "Failed to " + $1
43+
exit 1
44+
else
45+
echo "Succeed to " + $1
46+
fi
47+
}
48+
49+
function checkJava(){
50+
java -version
51+
isSuccess "execute java --version"
52+
}
53+
54+
checkJava
55+
56+
dos2unix -q ${workDir}/config/*
57+
isSuccess "execute dos2unix -q ${workDir}/config/*"
58+
dos2unix -q ${workDir}/bin/*
59+
60+
echo "step1:load config"
61+
source ${workDir}/config/config.sh
62+
source ${workDir}/config/db.sh
63+
64+
DSS_FILE_PATH="$workDir/$DSS_FILE_NAME"
65+
66+
if [ -z $DSS_FILE_NAME ]; then
67+
echo "DSS_FILE_NAME is null "
68+
exit 1
69+
fi
70+
71+
function replaceCommonIp() {
72+
if [ -z "$DSS_SERVER_INSTALL_IP" ]; then
73+
DSS_SERVER_INSTALL_IP=$LOCAL_IP
74+
fi
75+
76+
if [ -z "$DSS_SERVER_PORT" ]; then
77+
DSS_SERVER_PORT=9002
78+
fi
79+
80+
if [ -z "$DSS_APPS_SERVER_INSTALL_IP" ]; then
81+
DSS_APPS_SERVER_INSTALL_IP=$LOCAL_IP
82+
fi
83+
if [ -z "$DSS_APPS_SERVER_PORT" ]; then
84+
DSS_APPS_SERVER_PORT=9003
85+
fi
86+
87+
if [[ $GATEWAY_INSTALL_IP == "127.0.0.1" ]] || [ -z "$GATEWAY_INSTALL_IP" ]; then
88+
GATEWAY_INSTALL_IP=$LOCAL_IP
89+
fi
90+
91+
if [[ $EUREKA_INSTALL_IP == "127.0.0.1" ]] || [ -z "$EUREKA_INSTALL_IP" ]; then
92+
EUREKA_INSTALL_IP=$LOCAL_IP
93+
fi
94+
}
95+
##替换真实的IP
96+
replaceCommonIp
97+
98+
EUREKA_URL=http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/
99+
100+
## excecute sql
101+
source ${workDir}/bin/executeSQL.sh
102+
103+
function changeCommonConf(){
104+
sed -i "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $CONF_APPLICATION_YML
105+
sed -i "s#hostname:.*#hostname: $SERVER_IP#g" $CONF_APPLICATION_YML
106+
sed -i "s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g" $CONF_DSS_PROPERTIES
107+
sed -i "s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g" $CONF_DSS_PROPERTIES
108+
sed -i "s#wds.linkis.server.mybatis.datasource.password.*#***REMOVED***$MYSQL_PASSWORD#g" $CONF_DSS_PROPERTIES
109+
sed -i "s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g" $CONF_DSS_PROPERTIES
110+
sed -i "s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g" $CONF_DSS_PROPERTIES
111+
sed -i "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES
112+
sed -i "s#wds.linkis.gateway.wtss.url.*#wds.linkis.gateway.wtss.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES
113+
sed -i "s#wds.linkis.bml.auth.token.value.*#wds.linkis.bml.auth.token.value=$BML_AUTH#g" $CONF_DSS_PROPERTIES
114+
sed -i "s#linkis.configuration.linkisclient.auth.token.value.*#linkis.configuration.linkisclient.auth.token.value=$BML_AUTH#g" $CONF_DSS_PROPERTIES
115+
sed -i "s#wds.linkis.context.client.auth.value.*#wds.linkis.context.client.auth.value=$BML_AUTH#g" $CONF_DSS_PROPERTIES
116+
sed -i "s#wds.linkis.errorcode.auth.token.*#wds.linkis.errorcode.auth.token=$BML_AUTH#g" $CONF_DSS_PROPERTIES
117+
}
118+
119+
##function start
120+
function changeConf(){
121+
sed -i "s#spring.server.port=.*#spring.server.port=$SERVER_PORT#g" $CONF_SERVER_PROPERTIES
122+
if [[ $SERVER_NAME == "dss-framework-orchestrator-server" ]] || [[ $SERVER_NAME == "dss-workflow-server" ]]; then
123+
SERVER_FULL_NAME=$SERVER_NAME
124+
SERVER_FULL_NAME=$SERVER_NAME-$ENV_FLAG
125+
sed -i "s#spring.spring.application.name=.*#spring.spring.application.name=$SERVER_FULL_NAME#g" $CONF_SERVER_PROPERTIES
126+
fi
127+
sed -i "s#wds.dss.appconn.scheduler.project.store.dir.*#wds.dss.appconn.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g" $CONF_SERVER_PROPERTIES
128+
isSuccess "subsitution $CONF_SERVER_PROPERTIES"
129+
}
130+
##function end
131+
132+
133+
UPLOAD_PUBLIC_IPS=""
134+
##function start
135+
function uploadServiceFile(){
136+
if [[ $SERVER_IP == "127.0.0.1" ]]; then
137+
SERVER_IP=$local_host
138+
fi
139+
if [[ $UPLOAD_PUBLIC_IPS == *",${ENV_FLAG}-$SERVER_IP,"* ]]; then
140+
return 0
141+
fi
142+
echo "rfp:$SERVER_HOME"
143+
cp -rfp ${DSS_FILE_PATH}/* $SERVER_HOME
144+
cp -rfp ${workDir}/bin $SERVER_HOME
145+
cp -rfp ${workDir}/config/* $SERVER_HOME/conf
146+
sudo chown -R $deployUser:$deployUser $SERVER_HOME
147+
UPLOAD_PUBLIC_IPS="$UPLOAD_PUBLIC_IPS,${ENV_FLAG}-$SERVER_IP,"
148+
changeCommonConf
149+
}
150+
151+
##function start
152+
function installPackage(){
153+
if [[ $SERVER_IP == "127.0.0.1" ]]; then
154+
SERVER_IP=$local_host
155+
fi
156+
if [ -z $SERVER_NAME ]; then
157+
echo "ERROR:SERVER_NAME is null "
158+
exit 1
159+
fi
160+
uploadServiceFile
161+
# change configuration
162+
changeConf
163+
}
164+
165+
function dssWebInstall(){
166+
if ! test -e ${LINKIS_DSS_HOME}/wedatasphere-dss-web*.zip; then
167+
echo "**********Error: please put wedatasphere-dss-web-xxx.zip in ${LINKIS_DSS_HOME}! "
168+
exit 1
169+
else
170+
echo "Start to unzip dss web package."
171+
unzip -d ${LINKIS_DSS_HOME}/web/ -o ${LINKIS_DSS_HOME}/wedatasphere-dss-web-*.zip > /dev/null 2>&1
172+
sed -i "s#linkis_url.*#linkis_url=${LINKIS_GATEWAY_URL}#g" ${LINKIS_DSS_HOME}/web/config.sh
173+
isSuccess "Unzip dss web package to ${LINKIS_DSS_HOME}/web"
174+
fi
175+
}
176+
177+
##Install dss projects
178+
function installDssProject() {
179+
echo "step2:update config"
180+
SERVER_HOME=$DSS_INSTALL_HOME
181+
echo "create dir SERVER_HOME: $SERVER_HOME"
182+
sudo mkdir -p $SERVER_HOME
183+
isSuccess "Create the dir of $SERVER_HOME"
184+
sudo chown -R $deployUser:$deployUser $SERVER_HOME
185+
isSuccess "chown -R $deployUser:$deployUser $SERVER_HOME"
186+
187+
#echo ""
188+
SERVER_NAME=dss-server
189+
SERVER_IP=$DSS_SERVER_INSTALL_IP
190+
SERVER_PORT=$DSS_SERVER_PORT
191+
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
192+
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
193+
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
194+
###install dss-server
195+
installPackage
196+
#echo ""
197+
198+
SERVER_NAME=dss-apps-server
199+
SERVER_IP=$DSS_APPS_SERVER_INSTALL_IP
200+
SERVER_PORT=$DSS_APPS_SERVER_PORT
201+
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
202+
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
203+
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
204+
###install dss-apps-server
205+
installPackage
206+
#echo ""
207+
208+
#echo "-----------------DSS install end--------------------"
209+
#echo ""
210+
211+
}
212+
ENV_FLAG="dev"
213+
installDssProject
214+
215+
echo "Congratulations! You have installed DSS $DSS_VERSION successfully, please use sbin/dss-start-all.sh to start it!"

0 commit comments

Comments
 (0)