Merge pull request #1100 from WeDataSphere/master

Merge code to dev-1.1.2

Author: zqburde

.github/workflows/dlc.json

@@ -1,9 +1,4 @@
 {
-  "ignorePatterns": [
-    {
-      "pattern": "https://github.com/WeBankFinTech/DataSphereStudio-Doc"
-    }
-  ],
   "timeout": "10s",
   "retryOn429": true,
   "retryCount": 10,
@@ -13,5 +8,4 @@
     401,
     403
   ]
-}
-
+}

.gitignore

@@ -1,33 +1,115 @@
+.idea
+.idea/*
+
 .DS_Store
-.cache
 
-# for ide
+assembly/target
+assembly/dss-package/target
+
+
+# dss-commons
+dss-commons/dss-sender-service/target
+
+# dss-appconn
+dss-appconn/appconns/dss-datachecker-appconn/target
+dss-appconn/appconns/dss-eventchecker-appconn/target
+dss-appconn/appconns/dss-orchestrator-framework-appconn/target
+dss-appconn/appconns/dss-schedulis-appconn/target
+dss-appconn/appconns/dss-sendemail-appconn/esb-email-support/target
+dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/target
+dss-appconn/appconns/dss-workflow-appconn/target
+dss-appconn/appconns/dss-sso-appconn/target
+dss-appconn/appconns/dss-scriptis-appconn-appconn/target
+dss-appconn/dss-appconn-core/target
+dss-appconn/dss-appconn-loader/target
+dss-appconn/dss-schedule-appconn-core/target
+dss-appconn/linkis-appconn-engineplugin/target
+dss-appconn/appconns/dss-dolphinscheduler-appconn/target
+dss-appconn/dss-appconn-manager/dss-appconn-manager-client/target
+dss-appconn/dss-appconn-manager/dss-appconn-manager-core/target
+dss-appconn/dss-scheduler-appconn/target
+dss-appconn/appconns/dss-scriptis-appconn/target/
+
+# dss-apps
+dss-apps/dss-apiservice-server/target
+dss-apps/dss-scriptis-server/target
+dss-apps/dss-apps-server/target
+dss-server/target
+
+#dss-guide
+dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/
+dss-apps/dss-user-guide/dss-user-guide-server/target
+
+# dss-commons
+dss-commons/dss-common/target
+dss-commons/dss-contextservice/target
+dss-commons/dss-bmlservice/target
+
+# dss-framework
+dss-framework/dss-appconn-framework/target
+dss-framework/dss-framework-common/target
+dss-framework/dss-framework-orchestrator-server/target
+dss-framework/dss-framework-project-server/target
+dss-framework/dss-framework-workspace-server/target
+dss-framework/framework-plugins/dss-framework-orchestrator-publish/target
+dss-framework/framework-plugins/dss-framework-release-server/target
+dss-framework/framework-plugins/dss-framework-migrate-server/target
+dss-framework/dss-framework-admin/target
+dss-framework/dss-framework-sql-template/target
+dss-framework/dss-framework-dbapi-server/target
+dss-framework/dss-framework-admin-service/target
+dss-framework/dss-framework-proxy-user-service/target
+
+
+# dss-orchestrator
+dss-orchestrator/dss-orchestrator-common/target
+dss-orchestrator/dss-orchestrator-core/target
+dss-orchestrator/dss-orchestrator-db/target
+dss-orchestrator/dss-orchestrator-loader/target
+dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/target
+dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/target
+dss-orchestrator/dss-orchestrator-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/target
+
+# dss-standars
+dss-standard/dss-standard-common/target
+dss-standard/sso-standard/origin-sso-integration-standard/target
+dss-standard/sso-standard/spring-origin-sso-integration-plugin/target
+dss-standard/sso-standard/sso-integration-standard/target
+dss-standard/development-standard/development-process-standard/target
+dss-standard/development-standard/development-process-standard-execution/target
+dss-standard/structure-standard/dss-project-plugin/target
+dss-standard/structure-standard/dss-role-plugin/target
+dss-standard/structure-standard/dss-structure-integration-standard/target
+dss-standard/structure-standard/spring-origin-dss-project-plugin/target
+
+#dss-data-api
+dss-apps/dss-data-api/dss-data-api-server/src/main/resources
+dss-apps/dss-data-api/dss-api-sql-template/target
+dss-apps/dss-data-api/dss-data-api-server/target
+
+#dss-data-governance
+dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources
+dss-apps/dss-data-governance/dss-data-governance-server/target
+dss-apps/dss-data-governance/dss-data-governance-common/target
+dss-apps/dss-data-governance/dss-data-asset-server/target
+dss-apps/dss-data-governance/dss-data-classification-server/target
+dss-apps/dss-data-governance/dss-data-warehouse-dao/target
+dss-apps/dss-data-governance/dss-data-warehouse-service/target
+dss-apps/dss-data-governance/dss-data-warehouse-server/target
+
+
+# plugins
+plugins/azkaban/linkis-jobtype/target
+plugins/linkis/dss-gateway-support/target
+
+logs
+logs/*
+
 *.iml
-*.ipr
-*.iws
-*.pyc
-*.pyo
-*.swp
-.idea/
-.idea_modules/
-.project
-.pydevproject
-.scala_dependencies
-.settings
-.classpath
-
-# For SBT
-.jvmopts
-
-# For Node.js
-node_modules/
-
-# generated file
-.mvn/wrapper/maven-wrapper.jar
-dist/
-out/
-target/
-
-# log folder
-logs/
-*.log
+/plugins/dolphinscheduler/dss-dolphinscheduler-client/target
+/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/target
+/plugins/dolphinscheduler/dss-dolphinscheduler-token/target

assembly/bin/appconn-uninstall.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+#Actively load user env
+source ~/.bashrc
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+SOURCE_ROOT=${workDir}
+
+#load config
+source ${SOURCE_ROOT}/conf/config.sh
+source ${SOURCE_ROOT}/conf/db.sh
+
+APPCONN_NAME=''
+
+LOCAL_IP="`ifconfig | grep 'inet' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $2}'`"
+
+function isSuccess(){
+  if [ $? -ne 0 ]; then
+      echo "Failed to " $1
+      exit 1
+  else
+      echo "Succeed to " $1
+  fi
+}
+
+
+PROC_NAME=DSSProjectServerApplication
+ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l`
+if [ $ProcNumber -le 0 ];then
+   echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started."
+   exit 10
+else
+   echo "Begin to uninstall AppConn plugin..."
+fi
+
+
+function getUninstallAppConn() {
+  echo "Please input the name of uninstallation AppConn, e.g: schedulis."
+  read -p "Please input the AppConn name:"  idx
+  if [[ 'exit' = "$idx" ]];then
+    echo "exit!"
+    exit 1
+  else
+    APPCONN_NAME=$idx
+  fi
+  echo "Current uninstallation AppConn is ${APPCONN_NAME}"
+  echo ""
+  echo -e "\e[1;31m Are you sure you want to uninstall AppConn ${APPCONN_NAME}, the workflow associated with the AppConn will not be available after uninstallation.\e[0m"
+  echo -e "\e[1;31mIf you want to uninstall AppConn ${APPCONN_NAME}, please enter 1, otherwise enter 0.\e[0m"
+  echo ""
+  read -p "Please input your choice:" idx
+  
+  if [[ '0' = "$idx" ]]; then
+    echo "exit!"
+    exit 1
+  fi
+}
+
+##choose execute mysql mode
+function executeSQL() {
+  TEMP_DB_DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}/db
+  DB_DML_PATH=$TEMP_DB_DML_PATH/uninstall.sql
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
+  isSuccess "complete the cleanup of the database $DB_DML_PATH"
+  echo "Implement the $TEMP_DB_DML_PATH/uninstall.sql for $APPCONN_NAME succeed."
+}
+
+
+function deleteDML() {
+  echo ""
+  echo -e "\e[1;31m If you want to delete AppConn ${APPCONN_NAME} file, please enter 1, otherwise enter 0.\e[0m"
+  echo ""
+  read -p "Please input your choice:" idx
+  if [[ '1' = "$idx" ]];then
+    DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}
+    suffix=.zip
+    rm -rf $DML_PATH$suffix
+    rm -rf $DML_PATH
+    isSuccess "complete the cleanup of the ${APPCONN_NAME} file."
+  fi
+}
+
+
+echo ""
+echo "Step1: Get the uninstall AppConn name."
+getUninstallAppConn
+echo ""
+
+echo "Step2: Delete AppConn $APPCONN_NAME database info."
+executeSQL
+echo ""
+
+echo "Step3: Clear the plugin of $APPCONN_NAME AppConn in DSS."
+deleteDML
+echo ""
+echo "Now try to delete the plugin of ${APPCONN_NAME} AppConn in all DSS micro-services."
+echo "The following 2 ways can take effect:"
+echo "1. Restart DSS, we will use sh $SOURCE_ROOT/sbin/dss-start-all.sh to restart, it will spend 1 minute."
+echo "2. Do nothing, just wait for 5 minutes. Since the DSS micro-services will refresh all the AppConn plugins every 10 minutes."
+echo ""
+read -p "Please input the choise: "  choise
+if [[ '1' = "$choise" ]]; then
+  echo "You chose to restart dss-framework-project micro-services, now try to restart ..."
+  sh $SOURCE_ROOT/sbin/dss-start-all.sh
+else
+  echo "You chose to wait for 5 minutes."
+fi

assembly/bin/install.sh

@@ -13,8 +13,7 @@ SERVER_IP=""
 SERVER_HOME=""
 
 local_host="`hostname --fqdn`"
-LOCAL_IP=$(hostname -I)
-LOCAL_IP=${LOCAL_IP// /}
+LOCAL_IP=$(hostname -I | awk '{print $1}')
 
 #To be compatible with MacOS and Linux
 txt=""
@@ -228,19 +227,12 @@ fi
 ##Install dss projects
 function installDssProject() {
   echo "step2:update config"
-#  if [ "$DSS_INSTALL_HOME" != "" ]
-#  then
-#	   rm -rf $DSS_INSTALL_HOME
-#  fi
-  #echo ""
-  #echo "-----------------DSS install start--------------------"
   SERVER_HOME=$DSS_INSTALL_HOME
   if [ "$SERVER_HOME" == "" ]
   then
     export SERVER_HOME=${workDir}/DSSInstall
   fi
   if [ -d $SERVER_HOME ] && [ "$SERVER_HOME" != "$workDir" ]; then
-    rm -r $SERVER_HOME-bak
     echo "mv  $SERVER_HOME  $SERVER_HOME-bak"
     mv  $SERVER_HOME  $SERVER_HOME-bak
   fi

assembly/config/config.sh

@@ -7,7 +7,7 @@ SERVER_HEAP_SIZE="512M"
 ### The install home path of DSS，Must provided
 DSS_INSTALL_HOME=/appcom/Install/DSSInstall
 
-DSS_VERSION=1.1.0
+DSS_VERSION=1.1.2
 
 DSS_FILE_NAME="dss-$DSS_VERSION"
 
@@ -33,35 +33,13 @@ GATEWAY_PORT=9001
 
 ### DSS_SERVER
 ### This service is used to provide dss-server capability.
+### dss-server
+DSS_SERVER_INSTALL_IP=127.0.0.1
+DSS_SERVER_PORT=9043
 
-### project-server
-DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP=127.0.0.1
-DSS_FRAMEWORK_PROJECT_SERVER_PORT=9002
-### orchestrator-server
-DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP=127.0.0.1
-DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT=9003
-### apiservice-server
-DSS_APISERVICE_SERVER_INSTALL_IP=127.0.0.1
-DSS_APISERVICE_SERVER_PORT=9004
-### dss-workflow-server
-DSS_WORKFLOW_SERVER_INSTALL_IP=127.0.0.1
-DSS_WORKFLOW_SERVER_PORT=9005
-### dss-flow-execution-server
-DSS_FLOW_EXECUTION_SERVER_INSTALL_IP=127.0.0.1
-DSS_FLOW_EXECUTION_SERVER_PORT=9006
-###dss-scriptis-server
-DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1
-DSS_SCRIPTIS_SERVER_PORT=9008
-
-###dss-data-api-server
-DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1
-DSS_DATA_API_SERVER_PORT=9208
-###dss-data-governance-server
-DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1
-DSS_DATA_GOVERNANCE_SERVER_PORT=9209
-###dss-guide-server
-DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1
-DSS_GUIDE_SERVER_PORT=9210
+### dss-apps-server
+DSS_APPS_SERVER_INSTALL_IP=127.0.0.1
+DSS_APPS_SERVER_PORT=9044
 
 ############## ############## dss_appconn_instance configuration   start   ############## ##############
 ####eventchecker表的地址，一般就是dss数据库