Merge remote-tracking branch 'origin/dev-1.1.4' into dev-1.1.4

Author: yuankang134

db/dss_dml.sql

@@ -203,7 +203,7 @@ select @node_ui_source_type:=id from dss_workflow_node_ui where `key`='source.ty
 select @node_ui_check_object:=id from dss_workflow_node_ui where `key`='check.object';
 select @node_ui_max_check_hours:=id from dss_workflow_node_ui where `key`='max.check.hours';
 select @node_ui_job_desc:=id from dss_workflow_node_ui where `key`='job.desc';
-select @node_ui_bindViewKey:=id from dss_workflow_node_ui where `key`='bindViewKey';
+select @node_ui_upStreams:=id from dss_workflow_node_ui where `key`='upStreams';
 select @node_ui_executeUser:=id from dss_workflow_node_ui where `key`='executeUser';
 select @node_ui_ReuseEngine:=id from dss_workflow_node_ui where `key`='ReuseEngine';
 
@@ -373,7 +373,7 @@ insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_to,55);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_itsm,55);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_subject,55);
-insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_bindViewKey,55);
+insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_upStreams,55);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,32);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,41);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (39,46);
@@ -388,7 +388,7 @@ insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (4
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_ReuseEngine,59);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_driver_memory,60);
 insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_spark_executor_memory,60);
-insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validata_id`) values (@node_ui_job_desc,61);
+insert  into `dss_workflow_node_ui_to_validate`(`ui_id`,`validate_id`) values (@node_ui_job_desc,61);
 
 
 DELETE FROM dss_workspace_appconn_role;

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/java/com/webank/wedatasphere/dss/appconn/sendemail/email/Email.java

@@ -41,4 +41,6 @@ public interface Email {
     String getBcc();
     void setBcc(String bcc);
 
+    String getEmailType();
+    void setEmailType(String emailType);
 }

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/conf/SendEmailAppConnConfiguration.scala

@@ -25,7 +25,8 @@ object SendEmailAppConnConfiguration {
 
   val EMAIL_HOOK_CLASSES = CommonVars("wds.dss.appconn.email.hook.classes",
     "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailItsmCheckHook," +
-      "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailContentLimitHook")
+      "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailVisualisContentLimitHook," +
+      "com.webank.wedatasphere.dss.appconn.sendemail.hook.SendEmailTableauCheckHook")
 
   val EMAIL_IMAGE_HEIGHT = CommonVars("wds.dss.appconn.email.image.height", 500)
   val EMAIL_IMAGE_WIDTH = CommonVars("wds.dss.appconn.email.image.width", 1920)
@@ -44,4 +45,6 @@ object SendEmailAppConnConfiguration {
   val EMAIL_SMTP_SSL_ENABLED = CommonVars("wds.dss.appconn.email.smtp.ssl.enable", "true")
   val EMAIL_SMTP_TIMEOUT: CommonVars[Integer] = CommonVars("wds.dss.appconn.email.smtp.timeout", 25000)
 
+  val EMAIL_ENTITY_CLASSES = CommonVars("wds.dss.appconn.email.hook.entity.classes","com.webank.wedatasphere.dss.appconn.sendemail.hook.entity.newvisualis.NewVisualisEmailInfo," +
+    "com.webank.wedatasphere.dss.appconn.sendemail.hook.entity.visualis.VisualisEmailInfo")
 }

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/cs/EmailCSHelper.scala

@@ -16,6 +16,10 @@
 
 package com.webank.wedatasphere.dss.appconn.sendemail.cs
 
+
+import java.util
+
+import com.google.gson.internal.LinkedTreeMap
 import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException
 import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext
 import org.apache.linkis.common.utils.Logging
@@ -25,6 +29,7 @@ import org.apache.linkis.cs.common.entity.enumeration.{ContextScope, ContextType
 import org.apache.linkis.cs.common.entity.source.CommonContextKey
 import org.apache.linkis.cs.common.utils.CSCommonUtils
 import org.apache.linkis.server.JSONUtils
+import org.springframework.util.CollectionUtils
 
 import scala.collection.JavaConversions._
 
@@ -57,4 +62,16 @@ object EmailCSHelper extends Logging{
     info(s"Job IDs is ${jobIds.toList}.")
     jobIds
   }
+
+  def getJobTypes(refContext: ExecutionRequestRefContext):util.ArrayList[String] = {
+    val jobTypes = new util.ArrayList[String]()
+    getJobIds(refContext).foreach { jobId =>
+      jobTypes.add(refContext.fetchLinkisJob(jobId).getParams.get("labels").asInstanceOf[LinkedTreeMap[_,_]].get("codeType").toString)
+    }
+    if (CollectionUtils.isEmpty(jobTypes)) {
+      throw new EmailSendFailedException(80003 ,"empty result set is not allowed")
+    }
+    info(s"Job Types is $jobTypes.")
+    jobTypes
+  }
 }

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/AbstractEmail.scala

@@ -29,6 +29,7 @@ class AbstractEmail extends Email {
   private var to: String = _
   private var cc: String = _
   private var bcc: String = _
+  private var emialType: String = _
 
   override def getContent: String = content
   override def setContent(content: String): Unit = this.content = content
@@ -52,4 +53,7 @@ class AbstractEmail extends Email {
 
   override def getBcc: String = bcc
   override def setBcc(bcc: String): Unit = this.bcc = bcc
+
+  override def getEmailType: String = emialType
+  override def setEmailType(emailType: String): Unit = this.emialType = emailType
 }

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/domain/MultiContentEmail.scala

@@ -30,4 +30,6 @@ class MultiContentEmail extends AbstractEmail {
 
   def getEmailContents: Array[EmailContent[_]] = emailContents.toIterator.toArray
 
+
+
 }

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/email/generate/MultiContentEmailGenerator.scala

@@ -16,11 +16,14 @@
 
 package com.webank.wedatasphere.dss.appconn.sendemail.email.generate
 
+import com.google.gson.internal.LinkedTreeMap
 import com.webank.wedatasphere.dss.appconn.sendemail.cs.EmailCSHelper
 import com.webank.wedatasphere.dss.appconn.sendemail.email.domain.{AbstractEmail, MultiContentEmail}
 import com.webank.wedatasphere.dss.appconn.sendemail.emailcontent.domain.PictureEmailContent
 import com.webank.wedatasphere.dss.appconn.sendemail.exception.EmailSendFailedException
+import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils
 import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef
+import org.apache.commons.lang3.StringUtils
 import org.apache.linkis.storage.resultset.ResultSetFactory
 
 class MultiContentEmailGenerator extends AbstractEmailGenerator {
@@ -45,6 +48,10 @@ class MultiContentEmailGenerator extends AbstractEmailGenerator {
               }
               multiContentEmail.addEmailContent(emailContent)
             }
+            if (StringUtils.isBlank(multiContentEmail.getEmailType)) {
+              val emailType = refContext.fetchLinkisJob(jobId).getParams.get("labels").asInstanceOf[LinkedTreeMap[_,_]].get("codeType").toString
+              multiContentEmail.setEmailType(emailType)
+            }
           }
         case "file" => throw new EmailSendFailedException(80003 ,"file content is not allowed") //addContentEmail(c => new FileEmailContent(new FsPath(c)))
         case "text" => throw new EmailSendFailedException(80003 ,"text content is not allowed")//addContentEmail(new TextEmailContent(_))

dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/PictureEmailContentParser.scala

@@ -45,7 +45,8 @@ object PictureEmailContentParser extends AbstractEmailContentParser[PictureEmail
     val imageUUID: String = UUID.randomUUID.toString
     val width: Int = bufferedImage.getWidth
     val height: Int = bufferedImage.getHeight
-    val imagesCuts = if (height > EMAIL_IMAGE_HEIGHT.getValue) {
+    // 只支持修改visualis图片大小，后续如果有新增其他类型的邮件需要修改图片大小，需要在if中加上该邮件类型
+    val imagesCuts = if (email.getEmailType.contains("visualis") && height > EMAIL_IMAGE_HEIGHT.getValue) {
       val numOfCut = Math.ceil(height.toDouble / EMAIL_IMAGE_HEIGHT.getValue).toInt
       val realHeight = height / numOfCut
       (0 until numOfCut).map(i => bufferedImage.getSubimage(0, i * realHeight, width, realHeight)).toArray
@@ -62,7 +63,7 @@ object PictureEmailContentParser extends AbstractEmailContentParser[PictureEmail
       var iHeight = image.getHeight
       var iWidth = image.getWidth
 
-      if (iWidth > EMAIL_IMAGE_WIDTH.getValue) {
+      if (email.getEmailType.contains("visualis") && iWidth > EMAIL_IMAGE_WIDTH.getValue) {
         iHeight = ((EMAIL_IMAGE_WIDTH.getValue.toDouble / iWidth.toDouble) * iHeight.toDouble).toInt
         iWidth = EMAIL_IMAGE_WIDTH.getValue
       }

dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java

@@ -18,6 +18,7 @@
 
 import com.webank.wedatasphere.dss.apiservice.core.exception.*;
 import com.webank.wedatasphere.dss.apiservice.core.vo.MessageVo;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.linkis.common.exception.WarnException;
 import org.apache.linkis.server.Message;
 import org.slf4j.Logger;
@@ -39,27 +40,15 @@ public static Message doAndResponse(TryOperation tryOperation, String method, St
         try {
             Message message = tryOperation.operateAndGetMessage();
             return setMethod(message, method);
-        } catch (ConstraintViolationException e) {
-            LOG.error("api error, method: " + method, e);
-            return Message.error(e.getMessage());
         } catch (WarnException e) {
             LOG.error("api error, method: " + method, e);
             return setMethod(Message.warn(e.getMessage()), method);
-        } catch (AssertException e) {
+        } catch (AssertException | ApiExecuteException | ApiServiceQueryException e) {
             LOG.error("api error, method: " + method, e);
             return setMethod(Message.error(e.getMessage()), method);
-
-        } catch (ApiExecuteException e) {
-        LOG.error("api error, method: " + method, e);
-        return setMethod(Message.error(e.getMessage()), method);
-        }
-        catch (ApiServiceQueryException e) {
-            LOG.error("api error, method: " + method, e);
-            return setMethod(Message.error(e.getMessage()), method);
-        }
-        catch (Exception e) {
+        } catch (Exception e) {
             LOG.error("api error, method: " + method, e);
-            return Message.error(e.getMessage());
+            return Message.error(ExceptionUtils.getRootCauseMessage(e));
         }
     }
 

dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java

@@ -20,9 +20,11 @@
 import com.webank.wedatasphere.dss.common.conf.DSSCommonConf;
 import com.webank.wedatasphere.dss.common.entity.Resource;
 import com.webank.wedatasphere.dss.common.exception.DSSErrorException;
+import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException;
 import com.webank.wedatasphere.dss.common.exception.ErrorCode;
 import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils;
 import com.webank.wedatasphere.dss.contextservice.service.ContextService;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.linkis.common.exception.ErrorException;
 import org.apache.linkis.cs.client.ContextClient;
 import org.apache.linkis.cs.client.builder.ContextClientFactory;
@@ -76,8 +78,9 @@ public String createContextID(String workspace, String project, String flow, Str
             return SerializeHelper.serializeContextID(contextID);
         } catch (Exception e) {
             logger.error("createContextID error. workspace : {}, project : {}, flow : {}, version : {}, user : {}", workspace, project, flow, version, user, e);
+            throw new DSSRuntimeException(50032, "Try to ask Linkis for creating a new contextId failed(向Linkis请求创建一个ContextID失败)! Linkis error msg: " +
+                    ExceptionUtils.getRootCauseMessage(e), e);
         }
-        return null;
     }
 
     @Override