-fir perf source pipeline call

Author: giventocode

args.go

@@ -88,7 +88,7 @@ type validatedParameters struct {
 	filesPerPipeline       int
 	useExactMatch          bool
 	numberOfHandlesPerFile int
-	numberOfIlesInBatch    int
+	numberOfFilesInBatch   int
 	blobSource             blobParams
 	blobTarget             blobParams
 	perfSourceDefinitions  []sources.SourceDefinition
@@ -277,7 +277,7 @@ func (p *paramParserValidator) pvgBatchLimits() error {
 	if p.args.numberOfFilesInBatch < 1 {
 		return fmt.Errorf("Invalid value for option -x, the value must be greater than 1")
 	}
-	p.params.numberOfIlesInBatch = p.args.numberOfFilesInBatch
+	p.params.numberOfFilesInBatch = p.args.numberOfFilesInBatch
 	return nil
 }
 func (p *paramParserValidator) pvgHTTPTimeOut() error {

blobporter.go

@@ -5,18 +5,18 @@ import (
 	"fmt"
 	"log"
 	"math"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"strconv"
 	"sync/atomic"
 	"time"
-
 	"github.com/Azure/blobporter/pipeline"
 	"github.com/Azure/blobporter/transfer"
 	"github.com/Azure/blobporter/util"
 )
 
-const programVersion = "0.5.32"
+const programVersion = "0.5.33"
 
 var argsUtil paramParserValidator
 
@@ -138,6 +138,10 @@ func displayFinalWrapUpSummary(duration time.Duration, targetRetries int32, thre
 }
 
 func main() {
+	go func() {
+		//runtime.SetBlockProfileRate(5)
+		//log.Println(http.ListenAndServe("localhost:6060", nil))
+	}()
 
 	flag.Parse()
 

pipelinefactory.go

@@ -33,29 +33,6 @@ func newTransferPipelines(params *validatedParameters) ([]pipeline.SourcePipelin
 
 }
 
-/*
-const openFileLimitForLinux = 1024
-
-//validates if the number of readers needs to be adjusted to accommodate max filehandle limits in Debian systems
-func adjustReaders(numOfSources int) error {
-	if runtime.GOOS != "linux" {
-		return nil
-	}
-
-	if (numOfSources * numberOfHandlesPerFile) > openFileLimitForLinux {
-		numberOfHandlesPerFile = openFileLimitForLinux / (numOfSources * numberOfHandlesPerFile)
-
-		if numberOfHandlesPerFile == 0 {
-			return fmt.Errorf("The number of files will cause the process to exceed the limit of open files allowed by the OS. Reduce the number of files to be transferred")
-		}
-
-		fmt.Printf("Warning! Adjusting the number of handles per file (-h) to %v\n", numberOfHandlesPerFile)
-
-	}
-
-	return nil
-}
-*/
 type pipelinesFactory struct {
 	source    transfer.TransferSegment
 	target    transfer.TransferSegment
@@ -133,7 +110,7 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) {
 			NumOfPartitions:  p.valParams.numberOfReaders, //TODO make this more explicit by numofpartitions as param..
 			MD5:              p.valParams.calculateMD5,
 			KeepDirStructure: p.valParams.keepDirStructure,
-			FilesPerPipeline: p.valParams.numberOfIlesInBatch}, nil
+			FilesPerPipeline: p.valParams.numberOfFilesInBatch}, nil
 	case transfer.HTTP:
 		return sources.HTTPSourceParams{
 			SourceURIs:    p.valParams.sourceURIs,
@@ -151,7 +128,7 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) {
 			SourceParams: sources.SourceParams{
 				CalculateMD5:      p.valParams.calculateMD5,
 				UseExactNameMatch: p.valParams.useExactMatch,
-				FilesPerPipeline:  p.valParams.numberOfIlesInBatch,
+				FilesPerPipeline:  p.valParams.numberOfFilesInBatch,
 				//default to always true so blob names are kept
 				KeepDirStructure: p.valParams.keepDirStructure}}, nil
 	case transfer.Blob:
@@ -164,10 +141,11 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) {
 			SourceParams: sources.SourceParams{
 				CalculateMD5:      p.valParams.calculateMD5,
 				UseExactNameMatch: p.valParams.useExactMatch,
-				FilesPerPipeline:  p.valParams.numberOfIlesInBatch,
+				FilesPerPipeline:  p.valParams.numberOfFilesInBatch,
 				KeepDirStructure:  p.valParams.keepDirStructure}}, nil
 	case transfer.Perf:
 		return sources.PerfSourceParams{
+			BlockSize:   p.valParams.blockSize,
 			Definitions: p.valParams.perfSourceDefinitions,
 			SourceParams: sources.SourceParams{
 				CalculateMD5: p.valParams.calculateMD5}}, nil

transfer/transfer.go

@@ -84,8 +84,8 @@ const (
 	BlobToPage             = "blob-pageblob"
 	S3ToBlock              = "s3-blockblob"
 	S3ToPage               = "s3-pageblob"
-	PerfToBlock            = "perf-block"
-	PerfToPage             = "perf-page"
+	PerfToBlock            = "perf-blockblob"
+	PerfToPage             = "perf-pageblob"
 	BlobToPerf             = "blob-perf"
 	none                   = "none"
 )
@@ -158,9 +158,9 @@ func ParseTransferDefinition(str string) (Definition, error) {
 		return S3ToBlock, nil
 	case "s3-pageblob":
 		return S3ToPage, nil
-	case "perf-block":
+	case "perf-blockblob":
 		return PerfToBlock, nil
-	case "perf-page":
+	case "perf-pageblob":
 		return PerfToPage, nil
 	case "blob-perf":
 		return BlobToPerf, nil