Skip to content

Commit 90fb072

Browse files
wgzhaowgzhao
committed
Add support for proxy and proxy authenication
1 parent 6b382ee commit 90fb072

File tree

4 files changed

+71
-23
lines changed

4 files changed

+71
-23
lines changed

‎docs/src/main/sphinx/reader/httpreader.md

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,11 @@ HttpReader 插件实现了读取Restful API 数据的能力
173173
"parameter": {
174174
"connection": [
175175
{
176-
"url": "http://127.0.0.1:9090/mock/17/LDJSC/ASSET"
176+
"url": "http://127.0.0.1:9090/mock/17/LDJSC/ASSET",
177+
"proxy": {
178+
"host": "http://127.0.0.1:3128",
179+
"auth": "user:pass"
180+
}
177181
}
178182
],
179183
"reqParams": {
@@ -182,7 +186,12 @@ HttpReader 插件实现了读取Restful API 数据的能力
182186
},
183187
"resultKey":"result",
184188
"method": "GET",
185-
"column": ["CURR_DATE","DEPT","TOTAL_MANAGED_MARKET_VALUE","TOTAL_MANAGED_MARKET_VALUE_GROWTH"]
189+
"column": ["CURR_DATE","DEPT","TOTAL_MANAGED_MARKET_VALUE","TOTAL_MANAGED_MARKET_VALUE_GROWTH"],
190+
"username": "user",
191+
"password": "passw0rd",
192+
"headers": {
193+
"X-Powered-by": "DataX"
194+
}
186195
}
187196
},
188197
"writer": {
@@ -307,10 +316,30 @@ bin/datax.py job/httpreader2stream.json
307316
| resultKey | 否 | string | 无 | 要获取结果的那个key值，如果是获取整个返回值，则可以不用填写 |
308317
| method | 否 | string | get | 请求模式，仅支持GET，POST两种，不区分大小写 |
309318
| column | 是 | list | 无 | 要获取的key，如果配置为 `"*"` ，则表示获取所有key的值 |
319+
| username | 否 | string | 无 | 接口请求需要的认证帐号（如有) |
320+
| password | 否 | string | 无 | 接口请求需要的密码（如有) |
321+
| proxy | 否 | map | 无 | 代理地址,详见下面描述 |
322+
| headers | 否 | map | 无 | 定制的请求头信息 |
323+
324+
#### proxy
325+
326+
如果访问的接口需要通过代理，则可以配置 `proxy` 配置项，该配置项是一个 json 字典，包含一个必选的 `host` 字段和一个可选的 `auth` 字段。
327+
328+
```json
329+
{
330+
"proxy": {
331+
"host": "http://127.0.0.1:8080",
332+
"auth": "user:pass"
333+
}
334+
}
335+
```
336+
337+
`host` 是代理地址，包含代理类型，目前仅支持 `http` 代理，其他类型比如 `socks5` 不支持。 如果代理需要认证，则可以配置 `auth` , 它由 用户名和密码组成，两者之间用冒号(:) 隔开。
310338

311339
### 限制说明
312340

313341
1. 返回的结果必须是JSON类型
314342
2. 当前所有key的值均当作字符串类型
315-
3. 暂不支持接口认证和Token鉴权模式
343+
3. 暂不支持接口Token鉴权模式
316344
4. 暂不支持分页获取
345+
5. 代理仅支持 `http` 模式

‎plugin/reader/httpreader/src/main/java/com/alibaba/datax/plugin/reader/httpreader/HttpReader.java

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@
3131
import org.apache.http.impl.client.HttpClients;
3232
import org.apache.http.ssl.SSLContextBuilder;
3333
import org.apache.http.util.EntityUtils;
34-
import org.slf4j.Logger;
35-
import org.slf4j.LoggerFactory;
3634

3735
import javax.net.ssl.HostnameVerifier;
3836
import javax.net.ssl.SSLContext;
@@ -56,7 +54,6 @@ public class HttpReader
5654
public static class Job
5755
extends Reader.Job
5856
{
59-
private static final Logger log = LoggerFactory.getLogger(Job.class);
6057
private Configuration originConfig = null;
6158

6259
@Override
@@ -68,7 +65,7 @@ public void init()
6865
@Override
6966
public void destroy()
7067
{
71-
68+
//
7269
}
7370

7471
@Override
@@ -83,23 +80,19 @@ public List<Configuration> split(int adviceNumber)
8380
public static class Task
8481
extends Reader.Task
8582
{
86-
private static final Logger log = LoggerFactory.getLogger(Task.class);
8783
private Configuration readerSliceConfig = null;
88-
private String url;
89-
private URI fullUri;
9084
private URIBuilder uriBuilder;
9185
private HttpHost proxy;
9286
private String username;
9387
private String password;
94-
private String token;
88+
private String proxyAuth;
9589

9690
@Override
9791
public void init()
9892
{
9993
this.readerSliceConfig = this.getPluginJobConf();
10094
this.username = readerSliceConfig.getString(Key.USERNAME, null);
10195
this.password = readerSliceConfig.getString(Key.PASSWORD, null);
102-
this.token = readerSliceConfig.getString(Key.TOKEN, null);
10396
Configuration conn =
10497
readerSliceConfig.getListConfiguration(Key.CONNECTION).get(0);
10598
uriBuilder = new URIBuilder(URI.create(conn.getString(Key.URL)));
@@ -214,10 +207,10 @@ record = recordSender.createRecord();
214207
}
215208
}
216209

217-
218210
private void createProxy(Configuration proxyConf)
219211
{
220-
String host = proxyConf.getString(Key.ADDRESS);
212+
String host = proxyConf.getString(Key.HOST);
213+
this.proxyAuth = proxyConf.getString(Key.AUTH);
221214
if (host.startsWith("socks")) {
222215
throw DataXException.asDataXException(
223216
HttpReaderErrorCode.NOT_SUPPORT, "sockes 代理暂时不支持"
@@ -265,19 +258,32 @@ else if ("post".equalsIgnoreCase(method)) {
265258
private CloseableHttpClient createCloseableHttpClient()
266259
{
267260
HttpClientBuilder httpClientBuilder = HttpClients.custom();
268-
261+
CredentialsProvider provider = null;
269262
if (this.password != null) {
270-
log.info("set authentication with user:{}", this.username);
271263
httpClientBuilder = HttpClientBuilder.create();
272264
// setup BasicAuth
273-
CredentialsProvider provider = new BasicCredentialsProvider();
265+
provider = new BasicCredentialsProvider();
274266
// Create the authentication scope
275-
AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM);
267+
HttpHost target = new HttpHost(uriBuilder.getHost(), uriBuilder.getPort());
268+
AuthScope scope = new AuthScope(target);
276269
// Create credential pair
277270
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(this.username, this.password);
278271
// Inject the credentials
279272
provider.setCredentials(scope, credentials);
280273
// Set the default credentials provider
274+
}
275+
276+
if (this.proxyAuth != null) {
277+
String[] up = this.proxyAuth.split(":");
278+
provider = new BasicCredentialsProvider();
279+
if (up.length == 1) {
280+
provider.setCredentials(new AuthScope(this.proxy), new UsernamePasswordCredentials(up[0], null));
281+
}
282+
if (up.length == 2) {
283+
provider.setCredentials(new AuthScope(this.proxy), new UsernamePasswordCredentials(up[0], up[1]));
284+
}
285+
}
286+
if (provider != null) {
281287
httpClientBuilder.setDefaultCredentialsProvider(provider);
282288
}
283289
httpClientBuilder.setSSLSocketFactory(ignoreSSLErrors());

‎plugin/reader/httpreader/src/main/java/com/alibaba/datax/plugin/reader/httpreader/Key.java

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,34 @@
22

33
public class Key
44
{
5+
// 获取返回json的那个key值
56
public static final String RESULT_KEY = "resultKey";
7+
// 连接信息
68
public static final String CONNECTION = "connection";
9+
// 配置连接代理
710
public static final String PROXY = "proxy";
8-
public static final String ADDRESS = "host";
11+
// 代理地址
12+
public static final String HOST = "host";
13+
// 代理认证信息，格式为 username:password
914
public static final String AUTH = "auth";
15+
// 请求的地址
1016
public static final String URL = "url";
17+
// 接口认证帐号
1118
public static final String USERNAME = "username";
19+
// 接口认证密码
1220
public static final String PASSWORD = "password";
21+
// 接口认证token
1322
public static final String TOKEN = "token";
23+
// 接口请求参数
1424
public static final String REQUEST_PARAMETERS = "reqParams";
25+
// 请求的定制头信息
1526
public static final String HEADERS = "headers";
16-
public static final String TIMEOUT = "timeout";
17-
public static final String CONNECTPATTERN = "connectPattern";
18-
public static final String MAXTRAVERSALLEVEL = "maxTraversalLevel";
27+
// 请求超时参数，单位为秒
28+
public static final String TIMEOUT_SEC = "timeout";
29+
// 请求方法，仅支持get，post两种模式
1930
public static final String METHOD = "method";
31+
// 结果集编码，默认为UTF8
2032
public static final String ENCODING = "encoding";
33+
// 要获取的字段，即json中的key，key值允许包含子路径，比如 info.user.age
2134
public static final String COLUMN = "column";
2235
}

‎plugin/reader/httpreader/src/main/resources/plugin_job_template.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
{
66
"url": "",
77
"proxy": {
8-
"address": "socks5://127.0.0.1:8080",
8+
"host": "socks5://127.0.0.1:8080",
99
"auth": "username:password"
1010
}
1111
}

0 commit comments

Comments
 (0)