Skip to content

Commit 3d79c14

Browse files
BonySmokeBonySmoke
authored
feat: support query parameters in httpjson (#692)
This patches makes these changes: 1. Support query parameters in user-provided URLs as this can be useful in filtering data before futher processing, significantly decreasing the request duration. 2. Cover new functionality with unit tests. 3. Modify existing tests to work with the new changes. Signed-off-by: Oleh Neichev <oleg.neichev@gmail.com>
1 parent 8c9bd0f commit 3d79c14

File tree

2 files changed

+90
-55
lines changed

2 files changed

+90
-55
lines changed

‎pkg/providers/httpjson/httpjson.go

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -64,26 +64,30 @@ func GetXpathFromUri(uri string) (xpathExpression string, err error) {
6464

6565
func GetUrlFromUri(uri string, protocol string) (string, error) {
6666
// Remove httpjson:// prefix
67-
trimmedStr := strings.TrimPrefix(uri, "httpjson://")
68-
// Attempt to split uri on argument
69-
uriParts := strings.Split(trimmedStr, "?")
70-
urlDomain := ""
71-
if len(uriParts) == 1 {
72-
// Attempt to split uri on parameter
73-
urlDomain = strings.Split(trimmedStr, "#")[0]
74-
} else {
75-
urlDomain = uriParts[0]
67+
trimmedUrl := strings.TrimPrefix(uri, "httpjson://")
68+
// Remove xpath suffix
69+
trimmedUrl = strings.Split(trimmedUrl, "#/")[0]
70+
71+
fullURL := fmt.Sprintf("%s://%s", protocol, trimmedUrl)
72+
73+
parsedUrl, parseErr := url.Parse(fullURL)
74+
if parseErr != nil {
75+
return "", fmt.Errorf("invalid URL: %s, error: %s", fullURL, parseErr.Error())
7676
}
77-
if urlDomain == "" {
77+
78+
if parsedUrl.Host == "" {
7879
return "", fmt.Errorf("no domain found in uri: %s", uri)
7980
}
80-
fullURL := fmt.Sprintf("%s://%s", protocol, urlDomain)
81-
_, err := url.Parse(fullURL)
82-
if err != nil {
83-
return "", fmt.Errorf("invalid domain: %s", err.Error())
81+
82+
query := parsedUrl.Query()
83+
84+
for _, key := range []string{"insecure", "floatAsInt"} {
85+
query.Del(key)
8486
}
8587

86-
return fullURL, nil
88+
parsedUrl.RawQuery = query.Encode()
89+
90+
return parsedUrl.String(), nil
8791
}
8892

8993
func (p *provider) GetJsonDoc(url string) error {
@@ -116,7 +120,11 @@ func (p *provider) GetString(uri string) (string, error) {
116120
returnValue := ""
117121
var values []string
118122
node, err := jsonquery.Query(p.docs[url], xpathQuery)
119-
if err != nil || node == nil {
123+
if err != nil {
124+
return "", fmt.Errorf("error querying the xpath expression %s, error: %s", xpathQuery, err)
125+
}
126+
127+
if node == nil {
120128
return "", fmt.Errorf("unable to query doc for value with xpath query using %v", uri)
121129
}
122130

‎vals_httpjson_test.go

Lines changed: 66 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func Test_HttpJson(t *testing.T) {
124124
expected := "chartify"
125125
actual := vals["value"]
126126
if actual != expected {
127-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
127+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
128128
}
129129
})
130130

@@ -137,18 +137,18 @@ func Test_HttpJson(t *testing.T) {
137137
expected := "go-yaml"
138138
actual := vals["value"]
139139
if actual != expected {
140-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
140+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
141141
}
142142
})
143143

144144
t.Run("Error getting document from location jsonquery.LoadURL", func(t *testing.T) {
145145
config := createProvider("httpjson://boom.github.com/users/helmfile/repos?insecure=true#", "//owner", "false")
146146
_, err := Load(config)
147147
if err != nil {
148-
expected := "error fetching json document at http://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value"
148+
expected := "error fetching json document at http://boom.github.com/users/helmfile/repos: Get \"http://boom.github.com/users/helmfile/repos\": dial tcp: lookup boom.github.com: no such host"
149149
actual := err.Error()
150150
if actual != expected {
151-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
151+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
152152
}
153153
}
154154
})
@@ -161,7 +161,7 @@ func Test_HttpJson(t *testing.T) {
161161
expected := "unable to query doc for value with xpath query using " + uri + "//boom"
162162
actual := err.Error()
163163
if actual != expected {
164-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
164+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
165165
}
166166
}
167167
})
@@ -174,19 +174,19 @@ func Test_HttpJson(t *testing.T) {
174174
expected := "unable to query doc for value with xpath query using " + uri + "//boom"
175175
actual := err.Error()
176176
if actual != expected {
177-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
177+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
178178
}
179179
}
180180
})
181181

182182
t.Run("Get Avatar URL with child nodes causing error", func(t *testing.T) {
183-
config := createProvider(prefixAndPath+"?insecure=true&mode=singleparam#", "//owner", "false")
183+
config := createProvider(prefixAndPath+"?insecure=true#", "//owner", "false")
184184
_, err := Load(config)
185185
if err != nil {
186186
expected := "location //owner has child nodes at " + server.URL + ", please use a more granular query"
187187
actual := err.Error()
188188
if actual != expected {
189-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
189+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
190190
}
191191
}
192192
})
@@ -200,7 +200,7 @@ func Test_HttpJson(t *testing.T) {
200200
expected := "251296379"
201201
actual := vals["value"]
202202
if actual != expected {
203-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
203+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
204204
}
205205
})
206206

@@ -211,7 +211,7 @@ func Test_HttpJson(t *testing.T) {
211211
expected := "unable to convert possible float to int for value: chartify"
212212
actual := err.Error()
213213
if actual != expected {
214-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
214+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
215215
}
216216
}
217217
})
@@ -225,7 +225,7 @@ func Test_HttpJson(t *testing.T) {
225225
expected := "chartify.database1.io,chartify.database2.io,chartify.database3.io,chartify.database4.io,chartify.database5.io"
226226
actual := vals["value"]
227227
if actual != expected {
228-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
228+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
229229
}
230230
})
231231
}
@@ -243,7 +243,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
243243
}
244244
expected := "http://boom.com/path"
245245
if returnValue != expected {
246-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
246+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
247247
}
248248
})
249249
t.Run("GetUrlFromUri: valid (https)", func(t *testing.T) {
@@ -253,37 +253,64 @@ func Test_HttpJson_UnitTests(t *testing.T) {
253253
}
254254
expected := "https://boom.com/path"
255255
if returnValue != expected {
256-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
256+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
257257
}
258258
})
259259
t.Run("GetUrlFromUri: invalid character in host name (https)", func(t *testing.T) {
260260
_, err := httpjson.GetUrlFromUri("httpjson://supsupsup^boom#///*[1]/name", "https")
261-
if err != nil {
262-
expected := "invalid domain: parse \"https://supsupsup^boom\": invalid character \"^\" in host name"
263-
actual := err.Error()
264-
if actual != expected {
265-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
266-
}
261+
expected := "invalid URL: https://supsupsup^boom, error: parse \"https://supsupsup^boom\": invalid character \"^\" in host name"
262+
263+
if err == nil {
264+
t.Fatalf("expected an error %q, got nil", expected)
265+
}
266+
267+
actual := err.Error()
268+
if actual != expected {
269+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
267270
}
268271
})
269272
t.Run("GetUrlFromUri: no domain provided (http)", func(t *testing.T) {
270273
_, err := httpjson.GetUrlFromUri("httpjson://?insecure=true#///*[1]/name", "http")
271-
if err != nil {
272-
expected := "no domain found in uri: httpjson://?insecure=true#///*[1]/name"
273-
actual := err.Error()
274-
if actual != expected {
275-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
276-
}
274+
275+
expected := "no domain found in uri: httpjson://?insecure=true#///*[1]/name"
276+
277+
if err == nil {
278+
t.Fatalf("expected an error %q, got nil", expected)
279+
}
280+
281+
actual := err.Error()
282+
if actual != expected {
283+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
277284
}
278285
})
279286
t.Run("GetUrlFromUri: no domain provided (https)", func(t *testing.T) {
280287
_, err := httpjson.GetUrlFromUri("httpjson://#///*[1]/name", "https")
281-
if err != nil {
282-
expected := "no domain found in uri: httpjson://#///*[1]/name"
283-
actual := err.Error()
284-
if actual != expected {
285-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
286-
}
288+
289+
expected := "no domain found in uri: httpjson://#///*[1]/name"
290+
291+
if err == nil {
292+
t.Fatalf("expected an error %q, got nil", expected)
293+
}
294+
295+
actual := err.Error()
296+
if actual != expected {
297+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
298+
}
299+
})
300+
t.Run("GetUrlFromUri: query params are preserved", func(t *testing.T) {
301+
url, _ := httpjson.GetUrlFromUri("httpjson://domain.com/path/?param=value#///*[1]/name", "https")
302+
303+
expected := "https://domain.com/path/?param=value"
304+
if expected != url {
305+
t.Errorf("unexpected url: expected=%q, got=%q", expected, url)
306+
}
307+
})
308+
t.Run("GetUrlFromUri: special query params are removed", func(t *testing.T) {
309+
url, _ := httpjson.GetUrlFromUri("httpjson://domain.com/path/?insecure=true&floatAsInt=false&param=value#///*[1]/name", "http")
310+
311+
expected := "http://domain.com/path/?param=value"
312+
if expected != url {
313+
t.Errorf("unexpected url: expected=%q, got=%q", expected, url)
287314
}
288315
})
289316

@@ -295,7 +322,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
295322
}
296323
expected := "//*[1]/name"
297324
if returnValue != expected {
298-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
325+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
299326
}
300327
})
301328
t.Run("GetXpathFromUri: valid (https)", func(t *testing.T) {
@@ -305,7 +332,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
305332
}
306333
expected := "//*[1]/name"
307334
if returnValue != expected {
308-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
335+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, returnValue)
309336
}
310337
})
311338
t.Run("GetXpathFromUri: no xpath provided (http)", func(t *testing.T) {
@@ -314,7 +341,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
314341
expected := "no xpath expression found in uri: httpjson://blah.blah/blah?insecure=true"
315342
actual := err.Error()
316343
if actual != expected {
317-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
344+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
318345
}
319346
}
320347
})
@@ -324,7 +351,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
324351
expected := "no xpath expression found in uri: httpjson://blah.blah/blah"
325352
actual := err.Error()
326353
if actual != expected {
327-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
354+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
328355
}
329356
}
330357
})
@@ -334,7 +361,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
334361
expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah?insecure=true#/"
335362
actual := err.Error()
336363
if actual != expected {
337-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
364+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
338365
}
339366
}
340367
})
@@ -344,7 +371,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
344371
expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah#/"
345372
actual := err.Error()
346373
if actual != expected {
347-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
374+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
348375
}
349376
}
350377
})
@@ -354,7 +381,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
354381
expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah?insecure=true#/hello^sup"
355382
actual := err.Error()
356383
if actual != expected {
357-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
384+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
358385
}
359386
}
360387
})
@@ -364,7 +391,7 @@ func Test_HttpJson_UnitTests(t *testing.T) {
364391
expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah#/hello^sup"
365392
actual := err.Error()
366393
if actual != expected {
367-
t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual)
394+
t.Errorf("unexpected value for key %q: expected=%q, got=%q", "value", expected, actual)
368395
}
369396
}
370397
})

0 commit comments

Comments
 (0)