- based url/endpoint suffix is exposed as an option.

- download transfer improvements
- doc and help update
- fix error message when account or key env variables are missing

Author: giventocode

args.go

@@ -61,7 +61,6 @@ type arguments struct {
 	storageAccountKey        string
 	storageClientHTTPTimeout int
 	baseBlobURL              string
-	targetBaseBlobURL        string
 	sourceParameters         map[string]string
 	sourceAuthorization      string
 	quietMode                bool
@@ -206,11 +205,13 @@ func (p *paramParserValidator) getTargetRules() ([]parseAndValidationRule, error
 			p.pvKeepDirStructueIsFalseWarning}, nil
 	case transfer.PageBlob:
 		return []parseAndValidationRule{
+			p.pvTargetBaseURL,
 			p.pvTargetContainerIsReq,
 			p.pvBlockSizeCheckForPageBlobs,
 			p.pvTargetBlobAuthInfoIsReq}, nil
 	case transfer.BlockBlob:
 		return []parseAndValidationRule{
+			p.pvTargetBaseURL,
 			p.pvTargetContainerIsReq,
 			p.pvBlockSizeCheckForBlockBlobs,
 			p.pvTargetBlobAuthInfoIsReq}, nil
@@ -239,6 +240,7 @@ func (p *paramParserValidator) getSourceRules() ([]parseAndValidationRule, error
 			p.pvSourceInfoForS3IsReq}, nil
 	case transfer.Blob:
 		return []parseAndValidationRule{
+			p.pvSourceBaseURL,
 			p.pvSourceInfoForBlobIsReq,
 			p.pvSetEmptyPrefixIfNone}, nil
 	case transfer.Perf:
@@ -266,7 +268,7 @@ func (p *paramParserValidator) pvgUseExactMatch() error {
 func (p *paramParserValidator) pvgTransferStatusPathIsPresent() error {
 
 	if p.args.transferStatusPath != "" {
-		if !p.args.quietMode{
+		if !p.args.quietMode {
 			fmt.Printf("Transfer is resumable. Transfer status file:%v \n", p.args.transferStatusPath)
 		}
 		tracker, err := internal.NewTransferTracker(p.args.transferStatusPath)
@@ -345,6 +347,16 @@ func (p *paramParserValidator) pvgTransferType() error {
 }
 
 //Transfer segment specific rules...
+func (p *paramParserValidator) pvSourceBaseURL() error {
+	p.params.blobSource.baseBlobURL = p.args.baseBlobURL
+
+	return nil
+}
+func (p *paramParserValidator) pvTargetBaseURL() error {
+	p.params.blobTarget.baseBlobURL = p.args.baseBlobURL
+
+	return nil
+}
 func (p *paramParserValidator) pvSetTargetAliases() error {
 	p.params.targetAliases = p.args.blobNames
 
@@ -447,7 +459,7 @@ func (p *paramParserValidator) pvSetEmptyPrefixIfNone() error {
 // storage account that is the target in all other cases. And the second with the URI provided, as when blob to blob transfers occur.
 func (p *paramParserValidator) pvSourceInfoForBlobIsReq() error {
 
-	//if the scenarios is download then check if download is via short-mode
+	//if the scenario is download check for short-mode download
 	if p.params.transferType == transfer.BlobToFile {
 		p.params.blobSource.accountName = p.args.storageAccountName
 		if p.params.blobSource.accountName == "" {
@@ -477,7 +489,7 @@ func (p *paramParserValidator) pvSourceInfoForBlobIsReq() error {
 	err := p.pvSourceURIISReq()
 
 	if err != nil {
-		return err
+		return fmt.Errorf("%s \n Or the account name or key were not set", err)
 	}
 
 	var burl *url.URL
@@ -492,7 +504,7 @@ func (p *paramParserValidator) pvSourceInfoForBlobIsReq() error {
 	p.params.blobSource.accountName = host[0]
 
 	if p.params.blobSource.accountName == "" {
-		return fmt.Errorf("Invalid source Azure Blob URL. Account name could be parsed from the domain")
+		return fmt.Errorf("Invalid source Azure Blob URL. Account name could not be parsed from the domain")
 	}
 
 	segments := strings.Split(burl.Path, "/")
@@ -540,7 +552,7 @@ func (p *paramParserValidator) pvSourceInfoForS3IsReq() error {
 	segments := strings.Split(burl.Path, "/")
 
 	if len(segments) < 2 {
-		return fmt.Errorf("Invalid S3 endpoint URL. Bucket not specified. The format is s3://[END_POINT]/[BUCKET]/[PREFIX]")		
+		return fmt.Errorf("Invalid S3 endpoint URL. Bucket not specified. The format is s3://[END_POINT]/[BUCKET]/[PREFIX]")
 	}
 
 	p.params.s3Source.bucket = segments[1]

blobporter.go

@@ -45,7 +45,8 @@ func init() {
 		numberOfHandlersPerFileMsg = "Number of open handles for concurrent reads and writes per file."
 		numberOfFilesInBatchMsg    = "Maximum number of files in a transfer.\n\tIf the number is exceeded new transfers are created"
 		readTokenExpMsg            = "Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources."
-		transferStatusFileMsg      = "Transfer status file location. If set, blobporter will use this file to track the status of the transfer.\n\tIn case of failure and if the option is set the same status file, source files that were transferred will be skipped.\n\tIf the transfer is successful a summary will be appended."
+		transferStatusFileMsg      = "Transfer status file location. If set, blobporter will use this file to track the status of the transfer.\n\tIn case of failure the transfer will skip files already transferred.\n\tIf the transfer is successful a summary will be appended."
+		baseURLMsg                 = "Endpoint suffix to be used for blob sources or targets. Default is blob.core.windows.net"
 	)
 
 	flag.Usage = func() {
@@ -69,6 +70,7 @@ func init() {
 		util.PrintUsageDefaults("x", "files_per_transfer", strconv.Itoa(argsUtil.args.numberOfFilesInBatch), numberOfFilesInBatchMsg)
 		util.PrintUsageDefaults("o", "read_token_exp", strconv.Itoa(defaultReadTokenExp), readTokenExpMsg)
 		util.PrintUsageDefaults("l", "transfer_status", "", transferStatusFileMsg)
+		util.PrintUsageDefaults("u", "endpoint_suffix", "", baseURLMsg) //when empty the azutil will use the default base url, hence not setting it here.
 	}
 
 	util.StringListVarAlias(&argsUtil.args.sourceURIs, "f", "source_file", "", fileMsg)
@@ -91,6 +93,7 @@ func init() {
 	util.IntVarAlias(&argsUtil.args.numberOfFilesInBatch, "x", "files_per_transfer", defaultNumberOfFilesInBatch, numberOfFilesInBatchMsg)
 	util.IntVarAlias(&argsUtil.args.readTokenExp, "o", "read_token_exp", defaultReadTokenExp, readTokenExpMsg)
 	util.StringVarAlias(&argsUtil.args.transferStatusPath, "l", "transfer_status", "", transferStatusFileMsg)
+	util.StringVarAlias(&argsUtil.args.baseBlobURL, "u", "endpoint_suffix", "", baseURLMsg)
 }
 
 var dataTransferred uint64

docs/gettingstarted.rst

@@ -89,6 +89,10 @@ Command Options
  -l, --transfer_status      (string) Transfer status file location.
                             If set, blobporter will use this file to track the status of the transfer. 
 
-                            In case of failure and the same file is referrenced, the source files that were transferred will be skipped.
+                            By referencing the same status file after a failure, the transfer will skip files already transferred.
 
                             If the transfer is successful a summary will be appended.
+
+ -u, --endpoint_suffix      Endpoint suffix to be used for blob sources or targets.
+
+                            Default is blob.core.windows.net

internal/const.go

@@ -6,7 +6,7 @@ import (
 )
 
 //ProgramVersion blobporter version
-const ProgramVersion = "0.6.14"
+const ProgramVersion = "0.6.15"
 
 //HTTPClientTimeout  HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations.
 var HTTPClientTimeout = 90

sources/http.go

@@ -170,12 +170,6 @@ func (f *HTTPSource) ExecuteReader(partitionsQ chan pipeline.PartsPartition, par
 			req.Header.Set("Range", header)
 			req.Header.Set("User-Agent", internal.UserAgentStr)
 
-			//set the close header only when the block is larger than the blob
-			//to minimize the number of open when transferring small files.
-			if p.BytesToRead < p.BlockSize {
-				req.Close = true
-			}
-
 			if res, err = f.HTTPClient.Do(req); err != nil || res.StatusCode != 206 {
 				var status int
 				if res != nil {