Skip to content

Commit 14985b2

Browse files
committed
Revert "datachecker 支持 qualitis v5"
This reverts commit 2709ddc.
1 parent b653e35 commit 14985b2

File tree

1 file changed

+8
-20
lines changed
  • dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector

1 file changed

+8
-20
lines changed

dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,9 @@ public boolean validateTableStatusFunction(Properties props, Logger log, RefExec
149149
}).collect(Collectors.toList());
150150
boolean flag = allCheckRes.stream().allMatch(res -> res.equals(true));
151151
if (flag) {
152-
log.info("=============================Data Check End,check result:true==========================================");
152+
log.info("=============================Data Check End==========================================");
153153
if (null != action.getExecutionRequestRefContext()) {
154-
action.getExecutionRequestRefContext().appendLog("=============================Data Check End,check result:true==========================================");
154+
action.getExecutionRequestRefContext().appendLog("=============================Data Check End==========================================");
155155
}
156156
return true;
157157
}
@@ -160,9 +160,9 @@ public boolean validateTableStatusFunction(Properties props, Logger log, RefExec
160160
throw new RuntimeException("get DataChecker result failed", e);
161161
}
162162

163-
log.info("=============================Data Check End,check result:false==========================================");
163+
log.info("=============================Data Check End==========================================");
164164
if (null != action.getExecutionRequestRefContext()) {
165-
action.getExecutionRequestRefContext().appendLog("=============================Data Check End,,check result:false==========================================");
165+
action.getExecutionRequestRefContext().appendLog("=============================Data Check End==========================================");
166166
}
167167
return false;
168168
}
@@ -210,26 +210,22 @@ private boolean getDataCheckResult(Map<String, String> proObjectMap,
210210
if (null != action.getExecutionRequestRefContext()){
211211
action.getExecutionRequestRefContext().appendLog("start to check hive meta");
212212
}
213-
log.info("start to check hive meta");
214213
proObjectMap.put(DataChecker.SOURCE_TYPE, HIVE_SOURCE_TYPE);
215214
normalCheck= getJobTotalCount(dataObject, jobConn, log) > 0;
216215
if (null != action.getExecutionRequestRefContext()){
217216
action.getExecutionRequestRefContext().appendLog("check hive meta end,check result:"+normalCheck);
218217
}
219-
log.info("check hive meta end,check result:"+normalCheck);
220218

221219
} else {
222220
if (isCheckMask.test(proObjectMap)) {
223221
if (null != action.getExecutionRequestRefContext()){
224222
action.getExecutionRequestRefContext().appendLog("start to check maskis");
225223
}
226-
log.info("start to check maskis");
227224
proObjectMap.put(DataChecker.SOURCE_TYPE, MASK_SOURCE_TYPE);
228225
normalCheck= (getBdpTotalCount(dataObject, bdpConn, log, props) > 0 || "success".equals(fetchMaskCode(dataObject, log, props).get("maskStatus")));
229226
if (null != action.getExecutionRequestRefContext()){
230227
action.getExecutionRequestRefContext().appendLog("check maskis end,check result:"+normalCheck);
231228
}
232-
log.info("check maskis end,check result:"+normalCheck);
233229
}else {
234230
normalCheck = false;
235231
}
@@ -459,21 +455,15 @@ private int checkDops(CheckDataObject dataObject, Connection conn, Logger log){
459455
private boolean checkQualitisData(String objectNum,CheckDataObject dataObject, Logger log, RefExecutionAction action,Properties props,Connection conn, QualitisUtil qualitisUtil) {
460456
boolean systemCheck = Boolean.valueOf(props.getProperty(DataChecker.QUALITIS_SWITCH));
461457
String userCheckDefault=props.getProperty(DataChecker.QUALITIS_CHECK_DEFAULT);
462-
String userCheckStr = StringUtils.isBlank(props.getProperty(DataChecker.QUALITIS_CHECK)) ?
463-
userCheckDefault :
464-
props.getProperty(DataChecker.QUALITIS_CHECK);
465-
boolean userCheck = Boolean.valueOf(userCheckStr);
466-
log.info("systemCheck:{},userCheckDefault:{},userCheck:{}",systemCheck,userCheckDefault,userCheck);
458+
boolean userCheck = Boolean.valueOf(props.getProperty(DataChecker.QUALITIS_CHECK, userCheckDefault));
467459
if (systemCheck && userCheck ) {
468460
if (null != action.getExecutionRequestRefContext()){
469461
action.getExecutionRequestRefContext().appendLog("start to check dops");
470462
}
471-
log.info("start to check dops");
472463
int dopsState=checkDops(dataObject,conn,log);
473464
if (null != action.getExecutionRequestRefContext()){
474465
action.getExecutionRequestRefContext().appendLog("check dops end,check result:"+dopsState);
475466
}
476-
log.info("check dops end,check result:"+dopsState);
477467
if(dopsState==0){
478468
//没找到记录,直接通过校验
479469
return true;
@@ -483,7 +473,7 @@ private boolean checkQualitisData(String objectNum,CheckDataObject dataObject, L
483473
}
484474
// 其他情况,继续走qualitis校验
485475
if (null != action.getExecutionRequestRefContext()){
486-
action.getExecutionRequestRefContext().appendLog("Data Check Qualitis Start");
476+
action.getExecutionRequestRefContext().appendLog("start to check qualitis");
487477
}
488478
log.info(
489479
"=============================Data Check Qualitis Start==========================================");
@@ -519,22 +509,20 @@ private boolean checkQualitisData(String objectNum,CheckDataObject dataObject, L
519509
if (null != action.getExecutionRequestRefContext()){
520510
action.getExecutionRequestRefContext().appendLog("check qualitis end,check result:true");
521511
}
522-
log.info("check qualitis end,check result:true");
523512
return true;
524513
default:
525514
if (null != action.getExecutionRequestRefContext()){
526515
action.getExecutionRequestRefContext().appendLog("check qualitis end,check result:false");
527516
}
528-
log.info("check qualitis end,check result:false");
529517
return false;
530518
}
531519

532520
}
533521
if (null != action.getExecutionRequestRefContext()){
534-
action.getExecutionRequestRefContext().appendLog("Data Check Qualitis time out,check result set to false");
522+
action.getExecutionRequestRefContext().appendLog("check qualitis time out,check result set to false");
535523
}
536524
log.info(
537-
"=============================Data Check Qualitis time out,check result set to false==========================================");
525+
"=============================Data Check Qualitis time out==========================================");
538526
return false;
539527
} catch (Exception e) {
540528
if (null != action.getExecutionRequestRefContext()){

0 commit comments

Comments
 (0)