Skip to content

Commit c790228

Browse files
giventocodegiventocode
committed
- default http client uses Dialer without context
- contexts are now request scoped. - unit test for arg parsing
1 parent c9791e5 commit c790228

File tree

6 files changed

+217
-49
lines changed

6 files changed

+217
-49
lines changed

‎args.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ func newParamParserValidator() paramParserValidator {
133133
numberOfWorkers: defaultNumberOfWorkers,
134134
blockSizeStr: defaultBlockSizeStr,
135135
dedupeLevelOptStr: defaultDedupeLevelStr,
136-
transferDef: transfer.FileToBlock,
136+
transferDefStr: string(transfer.FileToBlock),
137137
numberOfHandlesPerFile: defaultNumberOfHandlesPerFile,
138138
hTTPClientTimeout: defaultHTTPClientTimeout,
139139
numberOfFilesInBatch: defaultNumberOfFilesInBatch}
@@ -268,9 +268,10 @@ func (p *paramParserValidator) pvgCalculateReadersAndWorkers() error {
268268
func (p *paramParserValidator) pvgParseBlockSize() error {
269269
var err error
270270
p.params.blockSize, err = util.ByteCountFromSizeString(p.args.blockSizeStr)
271-
if err != nil {
271+
if err != nil || p.params.blockSize <= 0 {
272272
return fmt.Errorf("Invalid block size specified: %v. Parse Error: %v ", p.args.blockSizeStr, err)
273273
}
274+
274275
return nil
275276
}
276277
func (p *paramParserValidator) pvgBatchLimits() error {

‎args_test.go

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"testing"
7+
8+
"github.com/Azure/blobporter/transfer"
9+
"github.com/Azure/blobporter/util"
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
func TestBasicUpload(t *testing.T) {
14+
val := newParamParserValidator()
15+
val.args.sourceURIs = []string{"data"}
16+
val.args.storageAccountName = "myaccount"
17+
val.args.storageAccountKey = "mykey"
18+
val.args.containerName = "mycont"
19+
20+
err := val.parseAndValidate()
21+
assert.NoError(t, err, "un expected error, all params should be set")
22+
assert.Equal(t, val.args.storageAccountName, val.params.blobTarget.accountName, "account name is not set")
23+
assert.Equal(t, val.args.storageAccountKey, val.params.blobTarget.accountKey, "account key is not set")
24+
assert.Equal(t, val.args.containerName, val.params.blobTarget.container, "container is not set")
25+
assert.Equal(t, val.args.sourceURIs[0], val.params.sourceURIs[0], "source is missing")
26+
}
27+
func TestBasicUploadWithAlias(t *testing.T) {
28+
val := newParamParserValidator()
29+
val.args.sourceURIs = []string{"data"}
30+
val.args.blobNames = []string{"data2"}
31+
val.args.storageAccountName = "myaccount"
32+
val.args.storageAccountKey = "mykey"
33+
val.args.containerName = "mycont"
34+
35+
err := val.parseAndValidate()
36+
assert.NoError(t, err, "un expected error, all params should be set")
37+
assert.Equal(t, val.args.storageAccountName, val.params.blobTarget.accountName, "account name is not set")
38+
assert.Equal(t, val.args.storageAccountKey, val.params.blobTarget.accountKey, "account key is not set")
39+
assert.Equal(t, val.args.containerName, val.params.blobTarget.container, "container is not set")
40+
assert.Equal(t, val.args.sourceURIs[0], val.params.sourceURIs[0], "source is missing")
41+
assert.Equal(t, val.args.blobNames[0], val.params.targetAliases[0], "target alias is missing")
42+
}
43+
func TestBasicUploaBlockSizeLimits(t *testing.T) {
44+
val := newParamParserValidator()
45+
val.args.sourceURIs = []string{"data"}
46+
val.args.blobNames = []string{"data2"}
47+
val.args.storageAccountName = "myaccount"
48+
val.args.storageAccountKey = "mykey"
49+
val.args.containerName = "mycont"
50+
val.args.blockSizeStr = "16MB"
51+
bs, err := util.ByteCountFromSizeString("16MB")
52+
assert.NoError(t, err, "byte size could not be parsed")
53+
54+
err = val.parseAndValidate()
55+
assert.NoError(t, err, "unexpected error, all params should be set")
56+
assert.Equal(t, val.args.storageAccountName, val.params.blobTarget.accountName, "account name is not set")
57+
assert.Equal(t, val.args.storageAccountKey, val.params.blobTarget.accountKey, "account key is not set")
58+
assert.Equal(t, val.args.containerName, val.params.blobTarget.container, "container is not set")
59+
assert.Equal(t, val.args.sourceURIs[0], val.params.sourceURIs[0], "source is missing")
60+
assert.Equal(t, val.args.blobNames[0], val.params.targetAliases[0], "target alias is missing")
61+
assert.Equal(t, bs, val.params.blockSize, "blocksize don't match")
62+
63+
val.args.blockSizeStr = "0"
64+
err = val.parseAndValidate()
65+
assert.Error(t, err, "expected to fail as it is an invalid block size")
66+
67+
val.args.blockSizeStr = "101MB"
68+
err = val.parseAndValidate()
69+
assert.Error(t, err, "expected to fail as it is an invalid block size")
70+
}
71+
func TestBasicUploaBlockSizeLimitsForPageBlobs(t *testing.T) {
72+
val := newParamParserValidator()
73+
val.args.sourceURIs = []string{"data"}
74+
val.args.blobNames = []string{"data2"}
75+
val.args.storageAccountName = "myaccount"
76+
val.args.storageAccountKey = "mykey"
77+
val.args.containerName = "mycont"
78+
val.args.blockSizeStr = "2MB"
79+
val.args.transferDefStr = "file-pageblob"
80+
bs, err := util.ByteCountFromSizeString("2MB")
81+
assert.NoError(t, err, "byte size could not be parsed")
82+
83+
err = val.parseAndValidate()
84+
assert.NoError(t, err, "unexpected error, all params should be set")
85+
assert.Equal(t, val.args.storageAccountName, val.params.blobTarget.accountName, "account name is not set")
86+
assert.Equal(t, val.args.storageAccountKey, val.params.blobTarget.accountKey, "account key is not set")
87+
assert.Equal(t, val.args.containerName, val.params.blobTarget.container, "container is not set")
88+
assert.Equal(t, val.args.sourceURIs[0], val.params.sourceURIs[0], "source is missing")
89+
assert.Equal(t, val.args.blobNames[0], val.params.targetAliases[0], "target alias is missing")
90+
assert.Equal(t, bs, val.params.blockSize, "blocksize don't match")
91+
assert.Equal(t, transfer.FileToPage, string(val.params.transferType), "transfer definition does not match")
92+
93+
val.args.blockSizeStr = "0"
94+
err = val.parseAndValidate()
95+
assert.Error(t, err, "expected to fail as it is an invalid block size")
96+
97+
val.args.blockSizeStr = "513"
98+
err = val.parseAndValidate()
99+
assert.Error(t, err, "expected to fail as it is an invalid block size")
100+
101+
val.args.blockSizeStr = "512"
102+
err = val.parseAndValidate()
103+
assert.NoError(t, err, "unexpected error, 512 is a valid page size")
104+
105+
//test auto adjusment
106+
val.args.blockSizeStr = "5MB"
107+
err = val.parseAndValidate()
108+
assert.NoError(t, err, "unexpected error, the block size should adjusted to 4MB")
109+
110+
bs, err = util.ByteCountFromSizeString("4MB")
111+
assert.NoError(t, err, "byte size could not be parsed")
112+
113+
assert.Equal(t, bs, val.params.blockSize, "block size does not match")
114+
115+
}
116+
func TestShortOptionDownload(t *testing.T) {
117+
val := newParamParserValidator()
118+
val.args.blobNames = []string{"data"}
119+
val.args.storageAccountName = "myaccount"
120+
val.args.storageAccountKey = "mykey"
121+
val.args.containerName = "mycont"
122+
val.args.transferDefStr = "blob-file"
123+
124+
err := val.parseAndValidate()
125+
assert.NoError(t, err, "un expected error, all params should be set")
126+
assert.Equal(t, val.args.storageAccountName, val.params.blobSource.accountName, "account name is not set")
127+
assert.Equal(t, val.args.storageAccountKey, val.params.blobSource.accountKey, "account key is not set")
128+
assert.Equal(t, val.args.containerName, val.params.blobSource.container, "container is not set")
129+
assert.Equal(t, val.args.blobNames[0], val.params.blobSource.prefixes[0], "blobname is missing")
130+
}
131+
132+
func TestLongOptionDownload(t *testing.T) {
133+
val := newParamParserValidator()
134+
back := os.Getenv(sourceAuthorizationEnvVar)
135+
136+
tempval := back
137+
if tempval == "" {
138+
tempval = "TEST_KEY"
139+
}
140+
os.Setenv(sourceAuthorizationEnvVar, tempval)
141+
142+
defer os.Setenv(sourceAuthorizationEnvVar, back)
143+
144+
a := "myaccount"
145+
c := "mycontainer"
146+
b := "myblob"
147+
val.args.sourceURIs = []string{fmt.Sprintf("http://%s.blob.core.windows.net/%s/%s", a, c, b)}
148+
val.args.sourceAuthorization = "mykey"
149+
val.args.transferDefStr = "blob-file"
150+
151+
err := val.parseAndValidate()
152+
assert.NoError(t, err, "un expected error, all params should be set")
153+
assert.Equal(t, a, val.params.blobSource.accountName, "account name is not set")
154+
assert.Equal(t, tempval, val.params.blobSource.accountKey, "account key is not set")
155+
assert.Equal(t, c, val.params.blobSource.container, "container is not set")
156+
assert.Equal(t, b, val.params.blobSource.prefixes[0], "blobname is missing")
157+
}

‎blobporter.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,18 @@ import (
55
"fmt"
66
"log"
77
"math"
8-
_ "net/http/pprof"
98
"net/url"
109
"os"
1110
"strconv"
1211
"sync/atomic"
1312
"time"
13+
1414
"github.com/Azure/blobporter/pipeline"
1515
"github.com/Azure/blobporter/transfer"
1616
"github.com/Azure/blobporter/util"
1717
)
1818

19-
const programVersion = "0.5.33"
19+
const programVersion = "0.5.34"
2020

2121
var argsUtil paramParserValidator
2222

@@ -138,10 +138,6 @@ func displayFinalWrapUpSummary(duration time.Duration, targetRetries int32, thre
138138
}
139139

140140
func main() {
141-
go func() {
142-
//runtime.SetBlockProfileRate(5)
143-
//log.Println(http.ListenAndServe("localhost:6060", nil))
144-
}()
145141

146142
flag.Parse()
147143

‎targets/azureblock.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ type AzureBlock struct {
2020
azutil *util.AzUtil
2121
}
2222

23-
2423
//NewAzureBlockPipeline TODO
2524
func NewAzureBlockPipeline(params AzureTargetParams) pipeline.TargetPipeline {
2625

@@ -90,9 +89,11 @@ func (t *AzureBlock) PreProcessSourceInfo(source *pipeline.SourceInfo, blockSize
9089
}
9190

9291
//clean uncommitted blocks for large files only (>1GB)
93-
if source.Size > util.GB {
94-
return t.azutil.CleanUncommittedBlocks(source.TargetAlias)
95-
}
92+
/*
93+
if source.Size > util.GB {
94+
return t.azutil.CleanUncommittedBlocks(source.TargetAlias)
95+
}
96+
*/
9697

9798
return nil
9899
}
@@ -138,6 +139,7 @@ func (t *AzureBlock) WritePart(part *pipeline.Part) (duration time.Duration, sta
138139

139140
return
140141
}
141-
err = t.azutil.PutBlock(part.TargetAlias, part.BlockID, bytes.NewReader(part.Data))
142+
reader := bytes.NewReader(part.Data)
143+
err = t.azutil.PutBlock(t.container, part.TargetAlias, part.BlockID, reader)
142144
return
143145
}

0 commit comments

Comments
 (0)