Merge pull request #69 from Azure/dev

- v0.5.13

Author: giventocode

README.md

@@ -29,7 +29,7 @@ Sources and targets are decoupled, this design enables the composition of variou
 Download, extract and set permissions:
 
 ```bash
-wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.5.12/bp_linux.tar.gz
+wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.5.13/bp_linux.tar.gz
 tar -xvf bp_linux.tar.gz linux_amd64/blobporter
 chmod +x ~/linux_amd64/blobporter
 cd ~/linux_amd64
@@ -46,7 +46,7 @@ export ACCOUNT_KEY=<STORAGE_ACCOUNT_KEY>
 
 ### Windows
 
-Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.5.12/bp_windows.zip)
+Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.5.13/bp_windows.zip)
 
 Set environment variables (if using the command prompt):
 

blobporter.go

@@ -49,7 +49,7 @@ const (
 	// User can use environment variables to specify storage account information
 	storageAccountNameEnvVar = "ACCOUNT_NAME"
 	storageAccountKeyEnvVar  = "ACCOUNT_KEY"
-	programVersion           = "0.5.12" // version number to show in help
+	programVersion           = "0.5.13" // version number to show in help
 )
 
 const numOfWorkersFactor = 8

inttest._m.sh

@@ -25,8 +25,8 @@ fi
 echo "Creating working files..."
 mkdir -p $WORKING_DIR
 
-dd if=/dev/urandom of=$F1 bs=64M count=1 iflag=fullblock
-dd if=/dev/urandom of=$P1 bs=64M count=1 iflag=fullblock
+dd if=/dev/urandom of=$F1 bs=64M count=8 iflag=fullblock
+dd if=/dev/urandom of=$P1 bs=64M count=2 iflag=fullblock
 
 
 #Scenario 1 - Simple upload/download with default values
@@ -47,14 +47,14 @@ calculateMD5 $P1 $DOWN_P1
 
 #Scenario 3 - Silent page upload/download with default values
 #Upload vhd
-./blobporter -f $P1 -c $CONT -n $DOWN_p1  -t file-pageblob -q
+./blobporter -f $P1 -c $CONT -n $DOWN_P1 -t file-pageblob -q
 #Download vhd
 ./blobporter -n $DOWN_P1 -c $CONT -t pageblob-file -q
 calculateMD5 $P1 $DOWN_P1
 
 #Scenario 4 - Silent page upload/download with default values and md5
 #Upload vhd
-./blobporter -f $P1 -c $CONT -t -n $DOWN_p1  file-pageblob -q -m
+./blobporter -f $P1 -c $CONT -n $DOWN_P1 -t file-pageblob -q -m
 #Download vhd
 ./blobporter -n $DOWN_P1 -c $CONT -t pageblob-file -q -m
 calculateMD5 $P1 $DOWN_P1
@@ -80,3 +80,24 @@ calculateMD5 vd1.mp4 vd1f.mp4
 #Download file
 ./blobporter -n $DOWN_F1 -c $CONT -t blob-file -b 16MB
 calculateMD5 $F1 $DOWN_F1
+
+
+#Scenario 7 - Simple upload/download with file smaller than the block size
+dd if=/dev/urandom of=$F1 bs=16M count=1 iflag=fullblock
+
+#Upload file
+./blobporter -f $F1 -c $CONT -n $DOWN_F1 -b 32MB
+
+#Download file
+./blobporter -n $DOWN_F1 -c $CONT -t blob-file -b 32MB
+calculateMD5 $F1 $DOWN_F1
+
+#Scenario 8 - Simple upload/download with file equal than the block size
+dd if=/dev/urandom of=$F1 bs=36M count=1 iflag=fullblock
+
+#Upload file
+./blobporter -f $F1 -c $CONT -n $DOWN_F1 -b 32MB
+
+#Download file
+./blobporter -n $DOWN_F1 -c $CONT -t blob-file -b 32MB
+calculateMD5 $F1 $DOWN_F1

sources/http.go

@@ -149,7 +149,12 @@ func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, p
 
 			header := fmt.Sprintf("bytes=%v-%v", p.Offset, p.Offset-1+uint64(p.BytesToRead))
 			req.Header.Set("Range", header)
-			req.Close = true
+
+			//set the close header only when the block is larger than the blob
+			//to minimize the number of open when transfering small files.
+			if p.BytesToRead < p.BlockSize {
+				req.Close = true
+			}
 
 			if res, err = f.HTTPClient.Do(req); err != nil || res.StatusCode != 206 {
 				var status int
@@ -158,9 +163,8 @@ func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, p
 					err = fmt.Errorf("Invalid status code in the response. Status: %v Bytes: %v", status, header)
 				}
 
-				if util.Verbose {
-					fmt.Printf("EH|R|%v|%v|%v|%v|%v\n", p.BlockID, p.BytesToRead, status, err, header)
-				}
+				util.PrintfIfDebug("ExecuteReader -> |%v|%v|%v|%v|%v", p.BlockID, p.BytesToRead, status, err, header)
+
 				return err
 			}
 			p.Data, err = ioutil.ReadAll(res.Body)
@@ -172,9 +176,9 @@ func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, p
 			if f.includeMD5 {
 				p.MD5()
 			}
-			if util.Verbose {
-				fmt.Printf("OK|R|%v|%v|%v|%v|%v\n", p.BlockID, p.BytesToRead, res.StatusCode, res.ContentLength, header)
-			}
+
+			util.PrintfIfDebug("ExecuteReader -> |%v|%v|%v|%v|%v", p.BlockID, p.BytesToRead, res.StatusCode, res.ContentLength, header)
+
 			return nil
 		})
 

sources/multifile.go

@@ -195,9 +195,7 @@ func (f *MultiFilePipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartiti
 				log.Fatal(err)
 			}
 
-			if util.Verbose {
-				fmt.Printf("OKR|R|%v|%v|%v|%v/n", part.BlockID, bytesRead, part.TargetAlias, part.BytesToRead)
-			}
+			util.PrintfIfDebug("ExecuteReader -> %v|%v|%v|%v", part.BlockID, bytesRead, part.TargetAlias, part.BytesToRead)
 
 			if f.includeMD5 {
 				part.MD5()

targets/azureblock.go

@@ -51,7 +51,7 @@ func (t *AzureBlock) CommitList(listInfo *pipeline.TargetCommittedListInfo, numb
 	if util.Verbose {
 		fmt.Printf("Final BlockList:\n")
 		for j := 0; j < numberOfBlocks; j++ {
-			fmt.Printf("   [%2d]: ID=%s, Status=%s", j, blockList[j].ID, blockList[j].Status)
+			fmt.Printf("   [%2d]: ID=%s, Status=%s\n", j, blockList[j].ID, blockList[j].Status)
 		}
 	}
 

targets/azurepage.go

@@ -88,16 +88,13 @@ func (t *AzurePage) WritePart(part *pipeline.Part) (duration time.Duration, star
 			headers["Content-MD5"] = part.MD5()
 		}
 		if err := (*t.StorageClient).PutPage(t.Container, part.TargetAlias, offset, endByte, "update", part.Data, headers); err != nil {
-			if util.Verbose {
-				fmt.Printf("EH|S|%v|%v|%v|%v\n", part.Offset, len(part.Data), part.TargetAlias, err)
-			}
+			util.PrintfIfDebug("WritePart -> |%v|%v|%v|%v", part.Offset, len(part.Data), part.TargetAlias, err)
 			t.resetClient()
 			return err
 		}
 
-		if util.Verbose {
-			fmt.Printf("OKA|S|%v|%v|%v|%v\n", part.Offset, len(part.Data), part.TargetAlias, err)
-		}
+		util.PrintfIfDebug("WritePart -> |%v|%v|%v", part.Offset, len(part.Data), part.TargetAlias)
+
 		return nil
 	})
 

util/util.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net"
 	"os"
 	"regexp"
 	"strconv"
@@ -199,10 +200,8 @@ func RetriableOperation(operation func(r int) error) (duration time.Duration, st
 		}
 		retries++
 
-		if Verbose {
-			fmt.Printf(" R %v ", retries)
-			fmt.Println(err.Error())
-		}
+		PrintfIfDebug("RetriableOperation -> |%v|%v", retries, err)
+
 		time.Sleep(retrySleepDuration)
 	}
 }
@@ -391,12 +390,17 @@ func getStorageHTTPClient() *http.Client {
 
 }
 
-//NewHTTPClient  creates a new HTTP client with the configured timeout and MaxIdleConnsPerHost = 100
+//NewHTTPClient  creates a new HTTP client with the configured timeout and MaxIdleConnsPerHost = 50, keep alive dialer.
+
 func NewHTTPClient() *http.Client {
 
 	c := http.Client{
 		Timeout: time.Duration(HTTPClientTimeout) * time.Second,
 		Transport: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   30 * time.Second, // dial timeout
+				KeepAlive: 30 * time.Second,
+			}).Dial,
 			MaxIdleConns:        maxIdleConns,
 			MaxIdleConnsPerHost: maxIdleConnsPerHost}}