- User-Agent header added

Author: giventocode

blobporter.go

@@ -71,6 +71,9 @@ func init() {
 	var defaultNumberOfWorkers = runtime.NumCPU() * numOfWorkersFactor
 	var defaultNumberOfReaders = runtime.NumCPU() * numOfReadersFactor
 
+	// set user agent info
+	util.SetUserAgentInfo(programVersion)
+
 	blockSizeStr = "8MB" // default size for blob blocks
 	const (
 		dblockSize = 8 * util.MB

sources/http.go

@@ -187,6 +187,8 @@ func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, p
 
 			header := fmt.Sprintf("bytes=%v-%v", p.Offset, p.Offset-1+uint64(p.BytesToRead))
 			req.Header.Set("Range", header)
+			userAgent, _ := util.GetUserAgentInfo()
+			req.Header.Set("User-Agent", userAgent)
 
 			//set the close header only when the block is larger than the blob
 			//to minimize the number of open when transfering small files.

targets/azureblock.go

@@ -121,13 +121,17 @@ func (t *AzureBlock) ProcessWrittenPart(result *pipeline.WorkerResult, listInfo
 func (t *AzureBlock) WritePart(part *pipeline.Part) (duration time.Duration, startTime time.Time, numOfRetries int, err error) {
 
 	headers := make(map[string]string)
+	userAgent, _ := util.GetUserAgentInfo()
+	headers["User-Agent"] = userAgent
+	
 	//if the max retries is exceeded, panic will happen, hence no error is returned.
 	duration, startTime, numOfRetries = util.RetriableOperation(func(r int) error {
 		//computation of the MD5 happens is done by the readers.
 		if part.IsMD5Computed() {
 			headers["Content-MD5"] = part.MD5()
 		}
 
+
 		if part.NumberOfBlocks == 1 {
 			if err := t.StorageClient.CreateBlockBlobFromReader(t.Container,
 				part.TargetAlias,

targets/azurepage.go

@@ -53,8 +53,11 @@ func (t *AzurePage) PreProcessSourceInfo(source *pipeline.SourceInfo, blockSize
 	}
 
 	//if the max retries is exceeded, panic will happen, hence no error is returned.
+	headers := make(map[string]string)
+	userAgent, _ := util.GetUserAgentInfo()
+	headers["User-Agent"] = userAgent
 	util.RetriableOperation(func(r int) error {
-		if err := (*t.StorageClient).PutPageBlob(t.Container, (*source).TargetAlias, size, nil); err != nil {
+		if err := (*t.StorageClient).PutPageBlob(t.Container, (*source).TargetAlias, size, headers); err != nil {
 			t.resetClient()
 			return err
 		}
@@ -95,6 +98,9 @@ func (t *AzurePage) WritePart(part *pipeline.Part) (duration time.Duration, star
 		if part.IsMD5Computed() {
 			headers["Content-MD5"] = part.MD5()
 		}
+		userAgent, _ := util.GetUserAgentInfo()
+		headers["User-Agent"] = userAgent
+
 		if err := (*t.StorageClient).PutPage(t.Container, part.TargetAlias, offset, endByte, "update", part.Data, headers); err != nil {
 			util.PrintfIfDebug("WritePart -> |%v|%v|%v|%v", part.Offset, len(part.Data), part.TargetAlias, err)
 			t.resetClient()

util/util.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -380,7 +381,7 @@ func getStorageHTTPClient() *http.Client {
 var c *http.Client
 var mtx sync.Mutex
 
-//NewHTTPClient  creates a new HTTP client with the configured timeout and MaxIdleConnsPerHost = 50, keep alive dialer.
+//NewHTTPClient  creates a shared HTTP client with the configured timeout and MaxIdleConnsPerHost = 50, keep alive dialer.
 func NewHTTPClient() *http.Client {
 	mtx.Lock()
 	defer mtx.Unlock()
@@ -410,7 +411,7 @@ func getSuggestion(err error) string {
 	case strings.Contains(err.Error(), "Client.Timeout"):
 		return "Try increasing the timeout using the -s option or reducing the number of workers and readers, options: -r and -g"
 	case strings.Contains(err.Error(), "too many open files"):
-		return "Try increasing the number of open files allowed. For debian systems you can try: ulimit -Sn 2048 "
+		return "Try reducing the number of sources or batch size"
 	default:
 		return ""
 	}
@@ -441,3 +442,21 @@ func GetFileNameFromURL(sourceURI string) (string, error) {
 
 	return parts[len(parts)-1], nil
 }
+
+//UserAgent TODO
+var userAgent string
+
+//GetUserAgentInfo TODO
+func GetUserAgentInfo() (string, error) {
+	if userAgent == "" {
+		return "", fmt.Errorf("User agent is not set")
+	}
+
+	return userAgent, nil
+
+}
+
+//SetUserAgentInfo TODO
+func SetUserAgentInfo(programVersion string) {
+	userAgent = fmt.Sprintf("%s/%s/(%s %s)/", "BlobPorter", programVersion, runtime.GOOS, runtime.GOARCH)
+}