Fix: Last batch was not sending the last items from the source.

Author: giventocode

blobporter.go

@@ -54,7 +54,7 @@ const (
 	s3AccessKeyEnvVar = "S3_ACCESS_KEY"
 	s3SecretKeyEnvVar = "S3_SECRET_KEY"
 
-	programVersion = "0.5.23" // version number to show in help
+	programVersion = "0.5.25" // version number to show in help
 )
 
 const numOfWorkersFactor = 8

sources/http.go

@@ -49,10 +49,11 @@ func newHTTPSource(sourceListManager objectListManager, pipelineFactory sourceHT
 
 	for b := 0; b < numOfBatches; b++ {
 
+		start = b * numOfFilesInBatch
+
 		if filesSent < numOfFilesInBatch {
 			numOfFilesInBatch = filesSent
 		}
-		start = b * numOfFilesInBatch
 
 		httpSource := HTTPPipeline{Sources: sourceInfos[start : start+numOfFilesInBatch], HTTPClient: util.NewHTTPClient(), includeMD5: includeMD5}
 

sources/s3info.go

@@ -77,7 +77,7 @@ func (s *s3InfoProvider) getSourceInfo() ([]pipeline.SourceInfo, error) {
 				if !s.params.KeepDirStructure {
 					targetAlias = path.Base(obj.Key)
 				}
-
+				
 				sourceURIs = append(sourceURIs, pipeline.SourceInfo{
 					SourceName:  u.String(),
 					Size:        uint64(obj.Size),
@@ -118,7 +118,7 @@ func (s *s3InfoProvider) getObjectLists() (map[string][]minio.ObjectInfo, error)
 			if object.Err != nil {
 				return nil, object.Err
 			}
-
+			fmt.Printf("Object from s3 %+v\n", object)
 			list = append(list, object)
 		}