Merge pull request #826 from WeDataSphere/dev-1.1.4

Sync all the latest codes of WeDataSphere to dev-1.1.0

Author: wushengyeyouya

.github/workflows/build.yml

@@ -51,5 +51,6 @@ jobs:
       - name: Build frontend by node.js
         run: |
           cd web
-          npm install
+          npm install lerna -g
+          lerna bootstrap
           npm run build

.gitignore

@@ -17,8 +17,9 @@ dss-appconn/appconns/dss-orchestrator-framework-appconn/target
 dss-appconn/appconns/dss-schedulis-appconn/target
 dss-appconn/appconns/dss-sendemail-appconn/esb-email-support/target
 dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/target
-dss-appconn/appconns/dss-visualis-appconn/target
 dss-appconn/appconns/dss-workflow-appconn/target
+dss-appconn/appconns/dss-sso-appconn/target
+dss-appconn/appconns/dss-scriptis-appconn-appconn/target
 dss-appconn/dss-appconn-core/target
 dss-appconn/dss-appconn-loader/target
 dss-appconn/dss-schedule-appconn-core/target
@@ -30,7 +31,11 @@ dss-appconn/dss-scheduler-appconn/target
 
 # dss-apps
 dss-apps/dss-apiservice-server/target
-dss-apps/dss-datapipe-server/target
+dss-apps/dss-scriptis-server/target
+
+#dss-guide
+dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/
+dss-apps/dss-user-guide/dss-user-guide-server/target
 
 # dss-commons
 dss-commons/dss-common/target
@@ -44,10 +49,11 @@ dss-framework/dss-framework-project-server/target
 dss-framework/dss-framework-workspace-server/target
 dss-framework/framework-plugins/dss-framework-orchestrator-publish/target
 dss-framework/framework-plugins/dss-framework-release-server/target
+dss-framework/framework-plugins/dss-framework-migrate-server/target
 dss-framework/dss-framework-admin/target
 dss-framework/dss-framework-sql-template/target
 dss-framework/dss-framework-dbapi-server/target
-dss-framework/dss-framework-ctyun-service/target
+dss-framework/dss-framework-admin-service/target
 
 
 # dss-orchestrator
@@ -72,27 +78,23 @@ dss-standard/development-standard/development-process-standard/target
 dss-standard/development-standard/development-process-standard-execution/target
 dss-standard/structure-standard/dss-project-plugin/target
 dss-standard/structure-standard/dss-role-plugin/target
-dss-standard/structure-standard/dss-structure-integration-standard
+dss-standard/structure-standard/dss-structure-integration-standard/target
 dss-standard/structure-standard/spring-origin-dss-project-plugin/target
 
 #dss-data-api
-dss-data-api/dss-data-api-server/src/main/resources
-dss-data-api/dss-api-sql-template/target
-dss-data-api/dss-data-api-server/target
+dss-apps/dss-data-api/dss-data-api-server/src/main/resources
+dss-apps/dss-data-api/dss-api-sql-template/target
+dss-apps/dss-data-api/dss-data-api-server/target
 
 #dss-data-governance
-dss-data-governance/dss-data-governance-server/src/main/resources
-dss-data-governance/dss-data-governance-server/target
-dss-data-governance/dss-data-governance-common/target
-dss-data-governance/dss-data-asset-server/target
-dss-data-governance/dss-data-classification-server/target
-dss-data-governance/dss-data-warehouse-dao/target
-dss-data-governance/dss-data-warehouse-service/target
-dss-data-governance/dss-data-warehouse-server/target
-
-#dss-guide
-dss-guide/dss-guide-server/src/main/resources/
-dss-guide/dss-guide-server/target
+dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources
+dss-apps/dss-data-governance/dss-data-governance-server/target
+dss-apps/dss-data-governance/dss-data-governance-common/target
+dss-apps/dss-data-governance/dss-data-asset-server/target
+dss-apps/dss-data-governance/dss-data-classification-server/target
+dss-apps/dss-data-governance/dss-data-warehouse-dao/target
+dss-apps/dss-data-governance/dss-data-warehouse-service/target
+dss-apps/dss-data-governance/dss-data-warehouse-server/target
 
 
 # plugins
@@ -103,4 +105,6 @@ logs
 logs/*
 
 *.iml
-/plugins/dolphinscheduler/linkis-dolphinscheduler-client/target/
+/plugins/dolphinscheduler/dss-dolphinscheduler-client/target
+/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/target
+/plugins/dolphinscheduler/dss-dolphinscheduler-token/target

assembly/bin/appconn-install.sh

@@ -22,62 +22,80 @@ function isSuccess(){
       echo "Failed to " + $1
       exit 1
   else
-      echo "Succeed to" + $1
+      echo "Succeed to " + $1
   fi
 }
 
 PROC_NAME=DSSProjectServerApplication
-ProcNumber=`ps -ef |grep -w $PROC_NAME|grep -v grep|wc -l`
+ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l`
 if [ $ProcNumber -le 0 ];then
-   echo "${PROC_NAME} is not running,Please check whether DSS is installed"
-   exit 1000
+   echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started."
+   exit 10
 else
-   echo "Begine to install appconn"
+   echo "Begine to install new AppConn plugin..."
 fi
 
 ##choose install mysql mode
 function initInstallAppConn() {
-  echo "Please select the type of installation component?"
-  echo " 1: schedulis"
-  echo " 2: visualis"
-  echo " 3：Your AppConn Name"
-  echo " 4：exit"
-  read -p "Please input the choice:"  idx
-  if [[ '1' = "$idx" ]];then
-   APPCONN_NAME="schedulis"
-  elif [[ '2' = "$idx" ]];then
-    APPCONN_NAME="visualis"
-  elif [[ '4' = "$idx" ]];then
-    echo "no choice,exit!"
-    exit 1
+  if [ ! -z $1 ];then
+    APPCONN_NAME=$1
+    echo "Begin to install default appconn:$APPCONN_NAME"
   else
-     APPCONN_NAME=$idx
-  fi
-  echo "Current installation component is ${APPCONN_NAME}"
-
-  echo ""
-  echo "If this machine(127.0.0.1) is installed, enter 1"
-  echo "For others, you need to enter a complete IP address."
-  read -p "Please enter the ip of appconn: "  ip
-  APPCONN_INSTALL_IP=$ip
-  if [[ '1' = "$ip" ]];then
-   APPCONN_INSTALL_IP="127.0.0.1"
+    echo "Please input the name of installation AppConn, e.g: schedulis."
+    read -p "Please input the AppConn name:"  idx
+    if [[ 'exit' = "$idx" ]];then
+      echo "exit!"
+      exit 1
+    else
+      APPCONN_NAME=$idx
+    fi
+    echo "Current installation AppConn is ${APPCONN_NAME}"
+    echo ""
+    echo "Please input the installed IP address of $APPCONN_NAME."
+    echo "e.g: if you have installed $APPCONN_NAME with 192.168.1.1:8080, please input ip in 192.168.1.1"
+    read -p "Please input the IP: "  ip
+    APPCONN_INSTALL_IP=$ip
+    echo ""
+    echo "e.g: if you have installed $APPCONN_NAME with 192.168.1.1:8080, please input port in 8080"
+    read -p "Please input the port:"  port
+    APPCONN_INSTALL_PORT=$port
   fi
-  echo "You input ip is ${APPCONN_INSTALL_IP}"
 
   echo ""
-  read -p "Please enter the port of appconn:"  port
-  APPCONN_INSTALL_PORT=$port
-  echo "You input ip is ${APPCONN_INSTALL_PORT}"
+  replaceCommonIp
+  echo "The base url of $APPCONN_NAME is http://${APPCONN_INSTALL_IP}:${APPCONN_INSTALL_PORT}/."
 }
 
 function replaceCommonIp() {
  if [[ $APPCONN_INSTALL_IP == "127.0.0.1" ]] || [[ $APPCONN_INSTALL_IP == "0.0.0.0" ]];then
-    echo "APPCONN_INSTALL_IP is equals $APPCONN_INSTALL_IP, we will change it to ip address"
+    echo "since you input the ip to $APPCONN_INSTALL_IP, we will change it to real ip $LOCAL_IP."
     APPCONN_INSTALL_IP=$LOCAL_IP
  fi
 }
 
+function replaceDefaultAppconnSql() {
+    sed -i "s#EVENTCHECKER_JDBC_URL#$EVENTCHECKER_JDBC_URL#g" $DB_DML_PATH
+    sed -i "s#EVENTCHECKER_JDBC_USERNAME#$EVENTCHECKER_JDBC_USERNAME#g" $DB_DML_PATH
+    sed -i "s#EVENTCHECKER_JDBC_PASSWORD#$EVENTCHECKER_JDBC_PASSWORD#g" $DB_DML_PATH
+
+    sed -i "s#DATACHECKER_JOB_JDBC_URL#$DATACHECKER_JOB_JDBC_URL#g" $DB_DML_PATH
+    sed -i "s#DATACHECKER_JOB_JDBC_USERNAME#$DATACHECKER_JOB_JDBC_USERNAME#g" $DB_DML_PATH
+    sed -i "s#DATACHECKER_JOB_JDBC_PASSWORD#$DATACHECKER_JOB_JDBC_PASSWORD#g" $DB_DML_PATH
+
+    sed -i "s#DATACHECKER_BDP_JDBC_URL#$DATACHECKER_BDP_JDBC_URL#g" $DB_DML_PATH
+    sed -i "s#DATACHECKER_BDP_JDBC_USERNAME#$DATACHECKER_BDP_JDBC_USERNAME#g" $DB_DML_PATH
+    sed -i "s#DATACHECKER_BDP_JDBC_PASSWORD#$DATACHECKER_BDP_JDBC_PASSWORD#g" $DB_DML_PATH
+
+    sed -i "s#BDP_MASK_IP#127.0.0.1#g" $DB_DML_PATH
+    sed -i "s#BDP_MASK_PORT#8087#g" $DB_DML_PATH
+
+    sed -i "s#EMAIL_HOST#${EMAIL_HOST}#g" $DB_DML_PATH
+    sed -i "s#EMAIL_PORT#${EMAIL_PORT}#g" $DB_DML_PATH
+    sed -i "s#EMAIL_USERNAME#${EMAIL_USERNAME}#g" $DB_DML_PATH
+    sed -i "s#EMAIL_PASSWORD#${EMAIL_PASSWORD}#g" $DB_DML_PATH
+    sed -i "s#EMAIL_PROTOCOL#${EMAIL_PROTOCOL}#g" $DB_DML_PATH
+}
+
 ##choose execute mysql mode
 function executeSQL() {
   TEMP_DB_DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}/db
@@ -86,28 +104,22 @@ function executeSQL() {
   sed -i "s/APPCONN_INSTALL_IP/$APPCONN_INSTALL_IP/g"       $DB_DML_PATH
   sed -i "s/APPCONN_INSTALL_PORT/$APPCONN_INSTALL_PORT/g"   $DB_DML_PATH
   sed -i "s#DSS_INSTALL_HOME_VAL#$DSS_INSTALL_HOME#g" $DB_DML_PATH
+  if [ ! -z $1 ];then
+    replaceDefaultAppconnSql
+  fi
   mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
-  isSuccess "source $DB_DML_PATH"
-  echo "the table update finished"
+  isSuccess "initialize the database settings $DB_DML_PATH"
+  echo "initialize the $TEMP_DB_DML_PATH/init.sql for $APPCONN_NAME succeed."
 }
 
 echo ""
-echo "step1:Initialize installation settings"
-initInstallAppConn
-echo ""
-
-echo "step2:replaceIp"
-replaceCommonIp
-echo ""
-
-echo "step3:update database"
-executeSQL
+echo "Step1: get the AppConn basic settings."
+initInstallAppConn $1
 echo ""
 
-echo "step4:refresh appconn load"
-curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
+echo "Step2: initialize the database settings."
+executeSQL $1
 echo ""
 
-echo ""
-echo "step5:Restart the DSS service,please use sbin/dss-start-all.sh to restart it!"
-echo ""
+echo "Step3: load the plugin of $APPCONN_NAME AppConn in DSS."
+sh $SOURCE_ROOT/bin/appconn-refresh.sh $APPCONN_NAME

assembly/bin/appconn-refresh.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+if [ -z $SOURCE_ROOT ]; then
+  #Actively load user env
+  source ~/.bashrc
+  shellDir=`dirname $0`
+  workDir=`cd ${shellDir}/..;pwd`
+  SOURCE_ROOT=${workDir}
+  #load config
+  source ${SOURCE_ROOT}/conf/config.sh
+  source ${SOURCE_ROOT}/conf/db.sh
+fi
+
+function isSuccess(){
+  if [ $? -ne 0 ]; then
+      echo "Failed to " + $1
+      exit 1
+  else
+      echo "Succeed to " + $1
+  fi
+}
+
+if [ -z $1 ];then
+  if [ -z $APPCONN_NAME ]; then
+    APPCONN_NAME=''
+    echo "Please input the name of refreshing AppConn, e.g: schedulis."
+    read -p "Please input the AppConn name: "  APPCONN_NAME
+  fi
+
+  echo ""
+  echo "Try to refresh the plugin of $APPCONN_NAME AppConn in all DSS micro-services."
+  echo "The following 2 ways can take effect:"
+  echo "1. restart DSS, we will use ${SOURCE_ROOT}/sbin/dss-start-all.sh to restart it!"
+  echo "2. do nothing, just wait for 5 minutes. Since the DSS micro-services will refresh all the AppConn plugins every 10 minutes."
+  echo ""
+  read -p "Please input the choise: "  choise
+  if [[ '1' = "$choise" ]]; then
+    echo "You chose to restart DSS, now try to restart DSS..."
+    sh $SOURCE_ROOT/sbin/dss-start-all.sh
+  else
+    echo "You chose to wait for 5 minutes."
+    echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
+    curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
+    isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
+    echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
+    echo ""
+    exit 0
+  fi
+else
+  APPCONN_NAME=$1
+  echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
+  curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
+  isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
+  echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
+  echo ""
+  exit 0
+fi