add cli httpclient with support for proxy configuration (#672)

hdiniz · web-flow · commit 2ad7fb3e1462 · 2020-05-19T16:23:15.000+02:00
* add configuration for cli httpclient

* add httpclient roundtripper with proxy and user-agent

* add cli httpclient

* set downloader user-agent tp equal cli httpclient

* use cli httpclient on board list command

* add unit tests for httpclient

* add copyright notice

* upgrade downloader to v2 with httpclient injection

* change go.mod to point to upstream downloader project v2
diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go
@@ -19,7 +19,7 @@ import (
 	"fmt"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 	semver "go.bug.st/relaxed-semver"
 )
 
diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go
@@ -18,7 +18,7 @@ package librariesmanager
 import (
 	"net/url"
 
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // LibraryIndexURL is the URL where to get library index.
diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go
@@ -20,7 +20,7 @@ import (
 	"os"
 
 	"github.com/arduino/go-paths-helper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // ArchivePath returns the path of the Archive of the specified DownloadResource relative
diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go
@@ -23,9 +23,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/arduino/arduino-cli/httpclient"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 type EchoHandler struct{}
@@ -53,7 +54,9 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 		URL:             srv.URL,
 	}
 
-	d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
+	httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})
+
+	d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
 	require.NoError(t, err)
 	err = d.Run()
 	require.NoError(t, err)
diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go
@@ -22,7 +22,7 @@ import (
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 func TestDownloadAndChecksums(t *testing.T) {
diff --git a/cli/globals/globals.go b/cli/globals/globals.go
@@ -16,11 +16,8 @@
 package globals
 
 import (
-	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/arduino/arduino-cli/version"
 )
@@ -31,18 +28,3 @@ var (
 	// DefaultIndexURL is the default index url
 	DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
 )
-
-// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
-// and adds the subComponent if specified
-func NewHTTPClientHeader(subComponent string) http.Header {
-	if subComponent != "" {
-		subComponent = " " + subComponent
-	}
-	userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
-		VersionInfo.Application,
-		VersionInfo.VersionString,
-		subComponent,
-		runtime.GOARCH, runtime.GOOS, runtime.Version(),
-		VersionInfo.Commit)
-	return http.Header{"User-Agent": []string{userAgentValue}}
-}
diff --git a/commands/board/list.go b/commands/board/list.go
@@ -23,8 +23,8 @@ import (
 	"regexp"
 	"sync"
 
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/commands"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/pkg/errors"
 	"github.com/segmentio/stats/v4"
@@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
 	url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
 	retVal := []*rpc.BoardListItem{}
 	req, _ := http.NewRequest("GET", url, nil)
-	req.Header = globals.NewHTTPClientHeader("")
 	req.Header.Set("Content-Type", "application/json")
 
 	// TODO: use proxy if set
 
-	if res, err := http.DefaultClient.Do(req); err == nil {
+	httpClient, err := httpclient.New()
+
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize http client")
+	}
+
+	if res, err := httpClient.Do(req); err == nil {
 		if res.StatusCode >= 400 {
 			if res.StatusCode == 404 {
 				return nil, ErrNotFound
diff --git a/commands/download.go b/commands/download.go
@@ -16,30 +16,24 @@
 package commands
 
 import (
-	"net/url"
 	"time"
 
-	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // GetDownloaderConfig returns the downloader configuration based on
 // current settings.
 func GetDownloaderConfig() (*downloader.Config, error) {
-	res := &downloader.Config{
-		RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),
+
+	httpClient, err := httpclient.New()
+	if err != nil {
+		return nil, err
 	}
-	if viper.IsSet("network.proxy") {
-		proxy := viper.GetString("network.proxy")
-		if _, err := url.Parse(proxy); err != nil {
-			return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error())
-		}
-		res.ProxyURL = proxy
-		logrus.Infof("Using proxy %s", proxy)
+
+	res := &downloader.Config{
+		HttpClient: *httpClient,
 	}
 	return res, nil
 }
diff --git a/commands/instances.go b/commands/instances.go
@@ -33,7 +33,7 @@ import (
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // this map contains all the running Arduino Core Services instances
diff --git a/go.mod b/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.4.0
 	go.bug.st/cleanup v1.0.0
-	go.bug.st/downloader v1.2.0
+	go.bug.st/downloader/v2 v2.0.1
 	go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
 	go.bug.st/serial v1.0.0
 	go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect
diff --git a/go.sum b/go.sum
@@ -205,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA=
 go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk=
-go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho=
-go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
+go.bug.st/downloader/v2 v2.0.1 h1:F/ZgVDrUHo2tHIeOVbcl4rgGaMmQgAEjn5c8yi5t+Iw=
+go.bug.st/downloader/v2 v2.0.1/go.mod h1:VZW2V1iGKV8rJL2ZEGIDzzBeKowYv34AedJz13RzVII=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE=
 go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=
diff --git a/httpclient/httpclient.go b/httpclient/httpclient.go
@@ -0,0 +1,40 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package httpclient
+
+import (
+	"net/http"
+)
+
+// New returns a default http client for use in the cli API calls
+func New() (*http.Client, error) {
+	config, err := DefaultConfig()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return NewWithConfig(config), nil
+}
+
+// NewWithConfig creates a http client for use in the cli API calls with a given configuration
+func NewWithConfig(config *Config) *http.Client {
+	transport := newHTTPClientTransport(config)
+
+	return &http.Client{
+		Transport: transport,
+	}
+}
diff --git a/httpclient/httpclient_config.go b/httpclient/httpclient_config.go
@@ -0,0 +1,64 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package httpclient
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"runtime"
+
+	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/spf13/viper"
+)
+
+// Config is the configuration of the http client
+type Config struct {
+	UserAgent string
+	Proxy     *url.URL
+}
+
+// DefaultConfig returns the default http client config
+func DefaultConfig() (*Config, error) {
+	var proxy *url.URL
+	var err error
+	if viper.IsSet("network.proxy") {
+		proxyConfig := viper.GetString("network.proxy")
+		if proxy, err = url.Parse(proxyConfig); err != nil {
+			return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error())
+		}
+	}
+
+	return &Config{
+		UserAgent: UserAgent(),
+		Proxy:     proxy,
+	}, nil
+}
+
+// UserAgent returns the user agent for the cli http client
+func UserAgent() string {
+	subComponent := viper.GetString("network.user_agent_ext")
+	if subComponent != "" {
+		subComponent = " " + subComponent
+	}
+
+	return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
+		globals.VersionInfo.Application,
+		globals.VersionInfo.VersionString,
+		subComponent,
+		runtime.GOARCH, runtime.GOOS, runtime.Version(),
+		globals.VersionInfo.Commit)
+}
diff --git a/httpclient/httpclient_test.go b/httpclient/httpclient_test.go
@@ -0,0 +1,70 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package httpclient
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUserAgentHeader(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, r.Header.Get("User-Agent"))
+	}))
+	defer ts.Close()
+
+	client := NewWithConfig(&Config{
+		UserAgent: "test-user-agent",
+	})
+
+	request, err := http.NewRequest("GET", ts.URL, nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+
+	b, err := ioutil.ReadAll(response.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, "test-user-agent", string(b))
+}
+
+func TestProxy(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer ts.Close()
+
+	proxyURL, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+
+	client := NewWithConfig(&Config{
+		Proxy: proxyURL,
+	})
+
+	request, err := http.NewRequest("GET", "http://arduino.cc", nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusNoContent, response.StatusCode)
+}
diff --git a/httpclient/httpclient_transport.go b/httpclient/httpclient_transport.go

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ import (`
`19`	`19`	`"fmt"`
`20`	`20`
`21`	`21`	`"github.com/arduino/arduino-cli/arduino/cores"`
`22`		`- "go.bug.st/downloader"`
	`22`	`+ "go.bug.st/downloader/v2"`
`23`	`23`	`semver "go.bug.st/relaxed-semver"`
`24`	`24`	`)`
`25`	`25`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ package librariesmanager`
`18`	`18`	`import (`
`19`	`19`	`"net/url"`
`20`	`20`
`21`		`- "go.bug.st/downloader"`
	`21`	`+ "go.bug.st/downloader/v2"`
`22`	`22`	`)`
`23`	`23`
`24`	`24`	`// LibraryIndexURL is the URL where to get library index.`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ import (`
`20`	`20`	`"os"`
`21`	`21`
`22`	`22`	`"github.com/arduino/go-paths-helper"`
`23`		`- "go.bug.st/downloader"`
	`23`	`+ "go.bug.st/downloader/v2"`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`// ArchivePath returns the path of the Archive of the specified DownloadResource relative`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ import (`
`22`	`22`
`23`	`23`	`"github.com/arduino/go-paths-helper"`
`24`	`24`	`"github.com/stretchr/testify/require"`
`25`		`- "go.bug.st/downloader"`
	`25`	`+ "go.bug.st/downloader/v2"`
`26`	`26`	`)`
`27`	`27`
`28`	`28`	`func TestDownloadAndChecksums(t *testing.T) {`
Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ import (`
`33`	`33`	`paths "github.com/arduino/go-paths-helper"`
`34`	`34`	`"github.com/sirupsen/logrus"`
`35`	`35`	`"github.com/spf13/viper"`
`36`		`- "go.bug.st/downloader"`
	`36`	`+ "go.bug.st/downloader/v2"`
`37`	`37`	`)`
`38`	`38`
`39`	`39`	`// this map contains all the running Arduino Core Services instances`