Skip to content

Commit b344fe9

Browse files
author
仲珊丽
committed
Merge branch 'feature-sprint2' of https://10.30.90.89:39999/luban/dataspherestudio into feature-sprint2
2 parents 2e96773 + 1ea1915 commit b344fe9

File tree

40 files changed

+1465
-422
lines changed

40 files changed

+1465
-422
lines changed

dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblDetailInfo.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ public class HiveTblDetailInfo implements Serializable {
2121

2222
@Data
2323
public static class HiveTblBasicInfo extends HiveTblSimpleInfo {
24-
private String store;
25-
private String comment;
26-
private Set<String> labels;
27-
private Boolean isParTbl;
24+
private String store; //存储量
25+
private Boolean isParTbl; //是否分区表
26+
private String tableType; //Hive表类型 tableType: EXTERNAL_TABLE, MANAGED_TABLE
27+
private String location; //Hive表存储路径
2828
}
2929

3030
@Data

dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/entity/HiveTblSimpleInfo.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import lombok.Data;
44

55
import java.util.List;
6+
import java.util.Set;
67

78
/**
9+
* @author suyc
810
* @Classname HiveTblSimpleInfo
911
* @Description TODO
1012
* @Date 2021/8/24 10:17
@@ -14,10 +16,12 @@
1416
public class HiveTblSimpleInfo {
1517
private String guid;
1618
private String name;
19+
private String dbName;
1720
private String qualifiedName;
21+
private List<String> columns;
1822
private String createTime;
1923
private String owner;
20-
private String dbName;
24+
private String comment;
25+
private Set<String> labels;
2126
private List<HiveTblDetailInfo.HiveClassificationInfo> classifications;
22-
2327
}

dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/restful/AssetTblRestful.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ public Message searchHiveTbl(@RequestParam(required = false) String classificati
6969
@RequestParam(defaultValue = "") String keyword,
7070
@RequestParam(defaultValue = DEFAULT_LIMIT) int limit,
7171
@RequestParam(defaultValue = DEFAULT_OFFSET) int offset) throws Exception {
72-
List<HiveTblSimpleInfo> hiveTblBasicList = assetService.searchHiveTable(classification,'*'+query+'*',limit,offset);
73-
if(hiveTblBasicList ==null || keyword ==null || keyword.trim().equals("")) {
74-
return Message.ok().data("result",hiveTblBasicList);
72+
List<HiveTblSimpleInfo> hiveTblSimpleInfoList = assetService.searchHiveTable(classification,query.trim(),limit,offset);
73+
if(hiveTblSimpleInfoList ==null || keyword ==null || keyword.trim().equals("")) {
74+
return Message.ok().data("result",hiveTblSimpleInfoList);
7575
}
7676
else {
7777
Pattern regex = Pattern.compile(keyword);
78-
return Message.ok().data("result",hiveTblBasicList.stream().filter(ele -> regex.matcher(ele.getOwner()).find()).collect(Collectors.toList()));
78+
return Message.ok().data("result",hiveTblSimpleInfoList.stream().filter(ele -> regex.matcher(ele.getOwner()).find()).collect(Collectors.toList()));
7979
}
8080
}
8181

dss-data-governance/dss-data-asset-server/src/main/java/com/webank/wedatasphere/dss/data/asset/service/impl/AssetServiceImpl.java

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.apache.atlas.model.instance.AtlasClassification;
1919
import org.apache.atlas.model.instance.AtlasEntity;
2020
import org.apache.atlas.model.instance.AtlasEntityHeader;
21+
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
2122
import org.apache.atlas.model.lineage.AtlasLineageInfo;
2223
import org.apache.atlas.model.typedef.AtlasClassificationDef;
2324
import org.slf4j.Logger;
@@ -31,6 +32,7 @@
3132
import java.util.Map;
3233
import java.util.Objects;
3334
import java.util.Set;
35+
import java.util.regex.Pattern;
3436
import java.util.stream.Collectors;
3537

3638
/**
@@ -72,37 +74,49 @@ public List<HiveTblSimpleInfo> searchHiveTable(String classification, String que
7274
int limit, int offset) throws DataGovernanceException {
7375
List<AtlasEntityHeader> atlasEntityHeaders = null;
7476
try {
75-
atlasEntityHeaders = atlasService.searchHiveTable(classification, query, true, limit, offset);
77+
atlasEntityHeaders = atlasService.searchHiveTable(classification, "*" + query + "*", true, limit, offset);
7678
} catch (AtlasServiceException ex) {
7779
throw new DataGovernanceException(ex.getMessage());
7880
}
7981

8082
if (atlasEntityHeaders != null) {
83+
//columns 根据keyword来正则匹配过滤
84+
Pattern regex = Pattern.compile(query);
8185
return atlasEntityHeaders.parallelStream().filter(Objects::nonNull).map(atlasEntityHeader -> {
82-
HiveTblSimpleInfo hiveTblBasic = new HiveTblSimpleInfo();
83-
hiveTblBasic.setGuid(atlasEntityHeader.getGuid());
84-
hiveTblBasic.setName(stringValueOfObject(atlasEntityHeader.getAttribute("name")));
86+
HiveTblSimpleInfo hiveTblSimpleInfo = new HiveTblSimpleInfo();
87+
hiveTblSimpleInfo.setGuid(atlasEntityHeader.getGuid());
88+
hiveTblSimpleInfo.setName(stringValueOfObject(atlasEntityHeader.getAttribute("name")));
8589
String qualifiedName =stringValueOfObject(atlasEntityHeader.getAttribute("qualifiedName"));
86-
hiveTblBasic.setQualifiedName(qualifiedName);
87-
hiveTblBasic.setOwner(stringValueOfObject(atlasEntityHeader.getAttribute("owner")));
90+
hiveTblSimpleInfo.setQualifiedName(qualifiedName);
91+
hiveTblSimpleInfo.setOwner(stringValueOfObject(atlasEntityHeader.getAttribute("owner")));
8892
Object createTime = atlasEntityHeader.getAttribute("createTime");
8993
if (createTime != null) {
90-
hiveTblBasic.setCreateTime(DateUtil.unixToTimeStr((Double) createTime));
94+
hiveTblSimpleInfo.setCreateTime(DateUtil.unixToTimeStr((Double) createTime));
9195
}
9296
if(null != qualifiedName && qualifiedName.split("\\.").length >0){
9397
String dbName = qualifiedName.split("\\.")[0];
94-
hiveTblBasic.setDbName(dbName);
98+
hiveTblSimpleInfo.setDbName(dbName);
9599
}
100+
hiveTblSimpleInfo.setLabels(atlasEntityHeader.getLabels());
96101

97102
try {
98103
AtlasEntity atlasEntity = atlasService.getHiveTblByGuid(atlasEntityHeader.getGuid());
104+
105+
//comment
106+
hiveTblSimpleInfo.setComment(stringValueOfObject(atlasEntity.getAttribute("comment")));
107+
List<Map<String,Object>> atlasRelatedObjectIdListForColumns = (List<Map<String,Object>>)atlasEntity.getRelationshipAttribute("columns");
108+
if(null != query && !query.trim().equalsIgnoreCase("")) {
109+
hiveTblSimpleInfo.setColumns(atlasRelatedObjectIdListForColumns.stream().map(columnMap -> columnMap.getOrDefault("displayText","").toString())
110+
.filter(columnName -> regex.matcher(columnName).find()).collect(Collectors.toList()));
111+
}
112+
//classifications
99113
List<HiveTblDetailInfo.HiveClassificationInfo> classificationInfoList = getClassificationInfoList(atlasEntity);
100-
hiveTblBasic.setClassifications(classificationInfoList);
114+
hiveTblSimpleInfo.setClassifications(classificationInfoList);
101115
} catch (AtlasServiceException ex) {
102116
logger.error(ex.getMessage());
103117
}
104118

105-
return hiveTblBasic;
119+
return hiveTblSimpleInfo;
106120
}).collect(Collectors.toList());
107121
}
108122
return null;
@@ -155,8 +169,8 @@ public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceExce
155169
}
156170

157171
private HiveTblDetailInfo.HiveTblBasicInfo getBasicInfo(String guid, AtlasEntity atlasEntity) throws AtlasServiceException {
158-
Map<String, Object> hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid);
159-
Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition");
172+
Map<String, Object> hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid);
173+
Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition");
160174
int storage = 0;
161175
String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0];
162176
String tableName = db_name.split("\\.")[1];
@@ -169,14 +183,16 @@ private HiveTblDetailInfo.HiveTblBasicInfo getBasicInfo(String guid, AtlasEntity
169183

170184
HiveTblDetailInfo.HiveTblBasicInfo basic = new HiveTblDetailInfo.HiveTblBasicInfo();
171185
basic.setName(tableName);
172-
basic.setOwner(String.valueOf(atlasEntity.getAttributes().get("owner")));
186+
basic.setOwner(String.valueOf(atlasEntity.getAttributes().getOrDefault("owner","NULL")));
173187
basic.setCreateTime(new java.text.SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(atlasEntity.getCreateTime()));
174188
basic.setStore(String.valueOf(storage));
175-
basic.setComment(String.valueOf(atlasEntity.getAttributes().get("comment")));
189+
basic.setComment(String.valueOf(atlasEntity.getAttributes().getOrDefault("comment","NULL")));
176190
Set<String> labels = atlasEntity.getLabels();
177191
basic.setLabels(labels);
178192
basic.setIsParTbl(isPartTable);
179193
basic.setGuid(guid);
194+
basic.setTableType(hiveTblAttributesMap.getOrDefault("tableType","NULL").toString());
195+
basic.setLocation(hiveTblAttributesMap.getOrDefault("location","NULL").toString());
180196

181197
return basic;
182198
}
@@ -288,8 +304,8 @@ public String getTbSelect(String guid) throws DataGovernanceException {
288304
for (AtlasEntity hiveColumnsByGuid : hiveColumnsByGuids) {
289305
fields.add((String) hiveColumnsByGuid.getAttributes().get("name"));
290306
}
291-
Map<String, Object> hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid);
292-
Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition");
307+
Map<String, Object> hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid);
308+
Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition");
293309
if (isPartTable == true) {
294310
List<String> partguids = new ArrayList<>();
295311
List<LinkedTreeMap<String, String>> partitionKeys = (List<LinkedTreeMap<String, String>>) atlasEntity.getAttributes().get("partitionKeys");
@@ -355,8 +371,8 @@ public String getTbCreate(String guid) throws DataGovernanceException {
355371
sql.append(field);
356372
}
357373
sql.append(") @$ ");
358-
Map<String, Object> hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid);
359-
Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition");
374+
Map<String, Object> hiveTblAttributesMap = atlasService.getHiveTblAttributesByGuid(guid);
375+
Boolean isPartTable = (Boolean) hiveTblAttributesMap.get("isPartition");
360376
if (isPartTable == true) {
361377
sql.append("PARTITIONED BY @$ ( @$ ");
362378
List<String> partguids = new ArrayList<>();

dss-data-governance/dss-data-governance-common/src/main/java/com/webank/wedatasphere/dss/data/common/atlas/AtlasService.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77
import com.google.gson.JsonElement;
88
import com.google.gson.JsonObject;
99
import com.google.gson.JsonParseException;
10+
import com.google.gson.internal.LinkedTreeMap;
1011
import com.webank.wedatasphere.dss.data.common.conf.AtlasConf;
1112
import org.apache.atlas.AtlasServiceException;
1213
import org.apache.atlas.model.discovery.AtlasSearchResult;
1314
import org.apache.atlas.model.instance.AtlasClassification;
1415
import org.apache.atlas.model.instance.AtlasEntity;
1516
import org.apache.atlas.model.instance.AtlasEntityHeader;
17+
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
1618
import org.apache.atlas.model.lineage.AtlasLineageInfo;
1719
import org.apache.atlas.model.typedef.AtlasClassificationDef;
1820
import org.apache.atlas.model.typedef.AtlasTypesDef;
@@ -189,17 +191,21 @@ public String getHiveTblNameById(String guid) throws AtlasServiceException {
189191
}
190192

191193
/**
192-
* 根据guid来获取hive tbl名称 和 是否分区表
194+
* 根据guid来获取hive tbl名称 和 是否分区表、是否外部表、外部表路径
193195
*/
194-
public Map<String, Object> getHiveTblNameAndIsPartById(String guid) throws AtlasServiceException {
195-
Map<String, Object> result = new HashMap<>(2);
196+
public Map<String, Object> getHiveTblAttributesByGuid(String guid) throws AtlasServiceException {
197+
Map<String, Object> result = new HashMap<>(4);
196198

197-
String jsonStr = atlasClient.getEntityByGuidForString(guid, true, false);
199+
String jsonStr = atlasClient.getEntityByGuidForString(guid, false, false);
198200
AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class);
199201

200202
result.put("tblName", atlasEntityWithExtInfo.getEntity().getAttribute("qualifiedName").toString().split("@")[0]);
201203
result.put("isPartition", ((List) atlasEntityWithExtInfo.getEntity().getAttribute("partitionKeys")).size() > 0);
202-
204+
result.put("tableType",atlasEntityWithExtInfo.getEntity().getAttribute("tableType"));
205+
Map<String,Object> sdMap = (LinkedTreeMap)atlasEntityWithExtInfo.getEntity().getRelationshipAttribute("sd");
206+
if(null != sdMap) {
207+
result.put("location",atlasEntityWithExtInfo.getReferredEntities().get(sdMap.get("guid")).getAttribute("location"));
208+
}
203209
return result;
204210
}
205211

0 commit comments

Comments
 (0)