Skip to content

Commit 27b40f3

Browse files
det101det101
committed
review fix
Stream close
1 parent 3bbc39f commit 27b40f3

File tree

10 files changed

+353
-340
lines changed

10 files changed

+353
-340
lines changed

‎dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,25 +49,25 @@ public final class AzkabanSecurityService extends AppJointUrlImpl implements Sch
4949

5050
static {
5151
Utils.defaultScheduler().scheduleAtFixedRate(()->{
52-
LOGGER.info("开始读取用户token文件");
52+
LOGGER.info("load azkaban-user.xml");
5353
Properties properties = new Properties();
5454
try {
5555
properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME));
5656
userToken = properties;
5757
} catch (IOException e) {
58-
LOGGER.error("读取文件失败:",e);
58+
LOGGER.error("load error:",e);
5959
}
6060
},0,10, TimeUnit.MINUTES);
6161
}
6262

6363
public void reloadToken(){
64-
LOGGER.info("开始读取用户token文件");
64+
LOGGER.info("reload azkaban-user.xml");
6565
Properties properties = new Properties();
6666
try {
6767
properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME));
6868
userToken = properties;
6969
} catch (IOException e) {
70-
LOGGER.error("读取文件失败:",e);
70+
LOGGER.error("reload error:",e);
7171
}
7272
}
7373

‎dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
@Consumes(MediaType.APPLICATION_JSON)
4141
public class UserManagerApi {
4242

43-
private ExecutorService executor = Executors.newCachedThreadPool();
4443
private LubanAuthorizationClient client = new LubanAuthorizationClient();
4544
private Logger logger = LoggerFactory.getLogger(this.getClass());
4645

‎dss-server/src/main/test/HttpTest.java

Lines changed: 185 additions & 185 deletions
Large diffs are not rendered by default.

‎dss-server/src/main/test/com/webank/HttpTest.java

Lines changed: 141 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -1,141 +1,141 @@
1-
//package com.webank;
2-
//
3-
//import com.webank.wedatasphere.dss.schedulis.exception.SchedulisSchedulerException;
4-
//import com.webank.wedatasphere.dss.schedulis.linkisjob.AbstractLinkisAzkabanJob;
5-
//import com.webank.wedatasphere.dss.schedulis.linkisjob.LinkisAzkabanJobFactory;
6-
//import org.apache.commons.io.IOUtils;
7-
//import org.apache.http.NameValuePair;
8-
//import org.apache.http.client.CookieStore;
9-
//import org.apache.http.client.entity.UrlEncodedFormEntity;
10-
//import org.apache.http.client.methods.CloseableHttpResponse;
11-
//import org.apache.http.client.methods.HttpGet;
12-
//import org.apache.http.client.methods.HttpPost;
13-
//import org.apache.http.client.protocol.HttpClientContext;
14-
//import org.apache.http.cookie.Cookie;
15-
//import org.apache.http.impl.client.BasicCookieStore;
16-
//import org.apache.http.impl.client.CloseableHttpClient;
17-
//import org.apache.http.impl.client.HttpClients;
18-
//import org.apache.http.message.BasicNameValuePair;
19-
//import org.apache.http.protocol.HTTP;
20-
//import org.apache.http.util.EntityUtils;
21-
//import org.junit.Test;
22-
//
23-
//import java.io.File;
24-
//import java.io.IOException;
25-
//import java.util.ArrayList;
26-
//import java.util.List;
27-
//
28-
//
29-
//public class HttpTest {
30-
//
31-
// //@Test
32-
// public Cookie test01() throws IOException {
33-
// HttpPost httpPost = new HttpPost("http://127.0.0.1:8088/checkin");
34-
// List<NameValuePair> params = new ArrayList<>();
35-
// params.add(new BasicNameValuePair("username", "neiljianliu"));
36-
// params.add(new BasicNameValuePair("userpwd", "*****"));
37-
// params.add(new BasicNameValuePair("action", "login"));
38-
// httpPost.setEntity(new UrlEncodedFormEntity(params));
39-
// CookieStore cookieStore = new BasicCookieStore();
40-
// CloseableHttpClient httpClient = null;
41-
// CloseableHttpResponse response = null;
42-
// HttpClientContext context = null;
43-
// try {
44-
// httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
45-
// context = HttpClientContext.create();
46-
// response = httpClient.execute(httpPost, context);
47-
// } finally {
48-
// IOUtils.closeQuietly(response);
49-
// IOUtils.closeQuietly(httpClient);
50-
// }
51-
// List<Cookie> cookies = context.getCookieStore().getCookies();
52-
// return cookies.get(0);
53-
// }
54-
//
55-
// @Test
56-
// public void test02() throws IOException {
57-
// List<String> strings = new ArrayList<>();
58-
// strings.add("1");
59-
// strings.add("bbb");
60-
// strings.add("ccc");
61-
// }
62-
//
63-
// public void print(String string) throws IOException, IllegalAccessException{
64-
// switch (string){
65-
// case "1":throw new IOException("a");
66-
// case "2":throw new IllegalAccessException("b");
67-
// default:
68-
// System.out.println(string);
69-
// }
70-
// }
71-
// @Test
72-
// public void test03() throws IOException, SchedulisSchedulerException {
73-
// Cookie cookie = test01();
74-
// List<NameValuePair> params = new ArrayList<>();
75-
// params.add(new BasicNameValuePair("ajax","fetchProjectPage"));
76-
// params.add(new BasicNameValuePair("start","0"));
77-
// params.add(new BasicNameValuePair("length","10"));
78-
// params.add(new BasicNameValuePair("projectsType","personal"));
79-
// params.add(new BasicNameValuePair("pageNum","1"));
80-
// params.add(new BasicNameValuePair("order","orderProjectName"));
81-
// CookieStore cookieStore = new BasicCookieStore();
82-
// cookieStore.addCookie(cookie);
83-
// HttpClientContext context = HttpClientContext.create();
84-
// CloseableHttpResponse response = null;
85-
// CloseableHttpClient httpClient = null;
86-
// try {
87-
// String finalUrl = "http://127.0.0.1:8088/index" + "?" + EntityUtils.toString(new UrlEncodedFormEntity(params));
88-
// HttpGet httpGet = new HttpGet(finalUrl);
89-
// httpGet.addHeader(HTTP.CONTENT_ENCODING, "UTF-8");
90-
// httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
91-
// response = httpClient.execute(httpGet, context);
92-
// /*Header[] allHeaders = context.getRequest().getAllHeaders();
93-
// Optional<Header> header = Arrays.stream(allHeaders).filter(f -> "Cookie".equals(f.getAppJointName())).findFirst();
94-
// header.ifPresent(AzkabanUtils.handlingConsumerWrapper(this::parseCookie));*/
95-
// } catch (Exception e) {
96-
// throw new SchedulisSchedulerException(90002, e.getMessage());
97-
// } finally {
98-
// IOUtils.closeQuietly(response);
99-
// IOUtils.closeQuietly(httpClient);
100-
// }
101-
// }
102-
//
103-
// @Test
104-
// public void test04(){
105-
///* A a = new A();
106-
// Tuple2<String, String> hello = a.hello();
107-
// Tuple2<String, String> stringStringTuple2 = new Tuple2<>("","");*/
108-
// }
109-
//
110-
// @Test
111-
// public void test05(){
112-
// //String subFlowPath = "C:\\Users\\v_wbjftang\\linuxDownloads\\project_0926_tjf\\flow2\\subFlows\\subFlow21\\subFlows\\subFlow211";
113-
// String subFlowPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211";
114-
// int indexOf = subFlowPath.indexOf("subFlows");
115-
// if(indexOf != -1){
116-
// subFlowPath = subFlowPath.substring(0, indexOf-1);
117-
// }
118-
// String substring1 = subFlowPath.substring(0, subFlowPath.lastIndexOf("/"));
119-
// System.out.println(substring1);
120-
// String allPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211";
121-
// String substring = allPath.substring(substring1.length() +1 );
122-
// System.out.println("\\" + File.separator);
123-
// //String s = substring.replaceAll("\\\\subFlows\\\\", "....");
124-
// String s = substring.replaceAll("\\" + File.separator +"subFlows" + "\\" + File.separator, "....");
125-
// System.out.println(s);
126-
// }
127-
// @Test
128-
// public void test06(){
129-
///* String nodeType = "spark.sql";
130-
// try {
131-
// AbstractLinkisAzkabanJob job = LinkisAzkabanJobFactory.createJob(nodeType);
132-
// } catch (ClassNotFoundException e) {
133-
// e.printStackTrace();
134-
// } catch (IllegalAccessException e) {
135-
// e.printStackTrace();
136-
// } catch (InstantiationException e) {
137-
// e.printStackTrace();
138-
// }
139-
// System.out.println("sdf");*/
140-
// }
141-
//}
1+
package com.webank;
2+
3+
import com.webank.wedatasphere.dss.schedulis.exception.SchedulisSchedulerException;
4+
import com.webank.wedatasphere.dss.schedulis.linkisjob.AbstractLinkisAzkabanJob;
5+
import com.webank.wedatasphere.dss.schedulis.linkisjob.LinkisAzkabanJobFactory;
6+
import org.apache.commons.io.IOUtils;
7+
import org.apache.http.NameValuePair;
8+
import org.apache.http.client.CookieStore;
9+
import org.apache.http.client.entity.UrlEncodedFormEntity;
10+
import org.apache.http.client.methods.CloseableHttpResponse;
11+
import org.apache.http.client.methods.HttpGet;
12+
import org.apache.http.client.methods.HttpPost;
13+
import org.apache.http.client.protocol.HttpClientContext;
14+
import org.apache.http.cookie.Cookie;
15+
import org.apache.http.impl.client.BasicCookieStore;
16+
import org.apache.http.impl.client.CloseableHttpClient;
17+
import org.apache.http.impl.client.HttpClients;
18+
import org.apache.http.message.BasicNameValuePair;
19+
import org.apache.http.protocol.HTTP;
20+
import org.apache.http.util.EntityUtils;
21+
import org.junit.Test;
22+
23+
import java.io.File;
24+
import java.io.IOException;
25+
import java.util.ArrayList;
26+
import java.util.List;
27+
28+
29+
public class HttpTest {
30+
31+
//@Test
32+
public Cookie test01() throws IOException {
33+
HttpPost httpPost = new HttpPost("http://127.0.0.1:8088/checkin");
34+
List<NameValuePair> params = new ArrayList<>();
35+
params.add(new BasicNameValuePair("username", "neiljianliu"));
36+
params.add(new BasicNameValuePair("userpwd", "*****"));
37+
params.add(new BasicNameValuePair("action", "login"));
38+
httpPost.setEntity(new UrlEncodedFormEntity(params));
39+
CookieStore cookieStore = new BasicCookieStore();
40+
CloseableHttpClient httpClient = null;
41+
CloseableHttpResponse response = null;
42+
HttpClientContext context = null;
43+
try {
44+
httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
45+
context = HttpClientContext.create();
46+
response = httpClient.execute(httpPost, context);
47+
} finally {
48+
IOUtils.closeQuietly(response);
49+
IOUtils.closeQuietly(httpClient);
50+
}
51+
List<Cookie> cookies = context.getCookieStore().getCookies();
52+
return cookies.get(0);
53+
}
54+
55+
@Test
56+
public void test02() throws IOException {
57+
List<String> strings = new ArrayList<>();
58+
strings.add("1");
59+
strings.add("bbb");
60+
strings.add("ccc");
61+
}
62+
63+
public void print(String string) throws IOException, IllegalAccessException{
64+
switch (string){
65+
case "1":throw new IOException("a");
66+
case "2":throw new IllegalAccessException("b");
67+
default:
68+
System.out.println(string);
69+
}
70+
}
71+
@Test
72+
public void test03() throws IOException, SchedulisSchedulerException {
73+
Cookie cookie = test01();
74+
List<NameValuePair> params = new ArrayList<>();
75+
params.add(new BasicNameValuePair("ajax","fetchProjectPage"));
76+
params.add(new BasicNameValuePair("start","0"));
77+
params.add(new BasicNameValuePair("length","10"));
78+
params.add(new BasicNameValuePair("projectsType","personal"));
79+
params.add(new BasicNameValuePair("pageNum","1"));
80+
params.add(new BasicNameValuePair("order","orderProjectName"));
81+
CookieStore cookieStore = new BasicCookieStore();
82+
cookieStore.addCookie(cookie);
83+
HttpClientContext context = HttpClientContext.create();
84+
CloseableHttpResponse response = null;
85+
CloseableHttpClient httpClient = null;
86+
try {
87+
String finalUrl = "http://127.0.0.1:8088/index" + "?" + EntityUtils.toString(new UrlEncodedFormEntity(params));
88+
HttpGet httpGet = new HttpGet(finalUrl);
89+
httpGet.addHeader(HTTP.CONTENT_ENCODING, "UTF-8");
90+
httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
91+
response = httpClient.execute(httpGet, context);
92+
/*Header[] allHeaders = context.getRequest().getAllHeaders();
93+
Optional<Header> header = Arrays.stream(allHeaders).filter(f -> "Cookie".equals(f.getAppJointName())).findFirst();
94+
header.ifPresent(AzkabanUtils.handlingConsumerWrapper(this::parseCookie));*/
95+
} catch (Exception e) {
96+
throw new SchedulisSchedulerException(90002, e.getMessage());
97+
} finally {
98+
IOUtils.closeQuietly(response);
99+
IOUtils.closeQuietly(httpClient);
100+
}
101+
}
102+
103+
@Test
104+
public void test04(){
105+
/* A a = new A();
106+
Tuple2<String, String> hello = a.hello();
107+
Tuple2<String, String> stringStringTuple2 = new Tuple2<>("","");*/
108+
}
109+
110+
@Test
111+
public void test05(){
112+
//String subFlowPath = "C:\\Users\\v_wbjftang\\linuxDownloads\\project_0926_tjf\\flow2\\subFlows\\subFlow21\\subFlows\\subFlow211";
113+
String subFlowPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211";
114+
int indexOf = subFlowPath.indexOf("subFlows");
115+
if(indexOf != -1){
116+
subFlowPath = subFlowPath.substring(0, indexOf-1);
117+
}
118+
String substring1 = subFlowPath.substring(0, subFlowPath.lastIndexOf("/"));
119+
System.out.println(substring1);
120+
String allPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211";
121+
String substring = allPath.substring(substring1.length() +1 );
122+
System.out.println("\\" + File.separator);
123+
//String s = substring.replaceAll("\\\\subFlows\\\\", "....");
124+
String s = substring.replaceAll("\\" + File.separator +"subFlows" + "\\" + File.separator, "....");
125+
System.out.println(s);
126+
}
127+
@Test
128+
public void test06(){
129+
/* String nodeType = "spark.sql";
130+
try {
131+
AbstractLinkisAzkabanJob job = LinkisAzkabanJobFactory.createJob(nodeType);
132+
} catch (ClassNotFoundException e) {
133+
e.printStackTrace();
134+
} catch (IllegalAccessException e) {
135+
e.printStackTrace();
136+
} catch (InstantiationException e) {
137+
e.printStackTrace();
138+
}
139+
System.out.println("sdf");*/
140+
}
141+
}

‎dss-server/src/main/test/com/webank/ReqTest.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package com.webank
22

3-
//import dispatch.Req
3+
import dispatch.Req
44

55

66
object ReqTest extends App{
7-
// private val url: Req = dispatch.url("")
8-
// print(url == url.setMethod("POST"))
7+
private val url: Req = dispatch.url("")
8+
print(url == url.setMethod("POST"))
99
}

‎dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
public class LinuxServer {
44

5-
private String linuxHost; //服务器ip,用逗号分隔
6-
private String linuxLoginUser; //服务器用户名
7-
private String linuxLoginPassword; //服务器密码
5+
private String linuxHost; //master ip，Comma separated
6+
private String linuxLoginUser;
7+
private String linuxLoginPassword;
88

99

1010
public String getLinuxHost() {

‎dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ protected String getString(Process process) throws IOException, InterruptedExcep
8080
if (status != 0){
8181
logger.error("shell error: "+status);
8282
}
83+
br.close();
8384
return Command.SUCCESS;
8485
}
8586

‎dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.dom4j.io.XMLWriter;
1111

1212
import java.io.File;
13+
import java.io.FileInputStream;
1314
import java.io.FileOutputStream;
1415
import java.io.IOException;
1516
import java.util.Iterator;
@@ -41,8 +42,19 @@ private void xmlHandler(String azkPath, AuthorizationBody body) throws DocumentE
4142
SAXReader reader = new SAXReader();
4243

4344
File file = new File(azkPath);
45+
Document document;
46+
FileInputStream fis = null;
47+
try {
48+
fis = new FileInputStream(file);
49+
document = reader.read(fis);
50+
}catch (DocumentException e){
51+
throw e;
52+
}finally {
53+
if (fis != null) {
54+
fis.close();
55+
}
56+
}
4457

45-
Document document = reader.read(file);
4658
Element root = document.getRootElement();
4759

4860
Iterator it = root.elementIterator("user");

‎dss-user-manager/target/classes/default/HdfsPath.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@
22

33
user=$1
44
dir=$2
5+
echo $1 $2
56
hdfs dfs -mkdir $dir
6-
hdfs dfs -chown $user:$user
7+
hdfs dfs -chown $user:$user $dir

0 commit comments

Comments
 (0)