- readme update

- sync and lock fixes for the file handel pool

Author: giventocode

README.md

@@ -29,7 +29,7 @@ Sources and targets are decoupled, this design enables the composition of variou
 Download, extract and set permissions:
 
 ```bash
-wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.5.22/bp_linux.tar.gz
+wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.5.23/bp_linux.tar.gz
 tar -xvf bp_linux.tar.gz linux_amd64/blobporter
 chmod +x ~/linux_amd64/blobporter
 cd ~/linux_amd64
@@ -46,7 +46,7 @@ export ACCOUNT_KEY=<STORAGE_ACCOUNT_KEY>
 
 ### Windows
 
-Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.5.22/bp_windows.zip)
+Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.5.23/bp_windows.zip)
 
 Set environment variables (if using the command prompt):
 

targets/multifile.go

@@ -8,8 +8,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Azure/blobporter/util"
-
 	"github.com/Azure/blobporter/pipeline"
 )
 
@@ -116,13 +114,16 @@ func (h *fileHandleManager) getHandle(path string) (*os.File, error) {
 
 	if !isInit {
 		fh, err = h.initFile(path)
-		if err != nil {
-			return nil, err
-		}
 	}
 	h.initTracker.Store(path, true)
 	h.Unlock()
 
+	if err != nil {
+		return nil, err
+	}
+
+
+
 	if !h.cacheEnabled {
 		//we don't have handle from the initialization, so open a new one.
 		if fh != nil {
@@ -135,42 +136,48 @@ func (h *fileHandleManager) getHandle(path string) (*os.File, error) {
 
 	//try to get it from the cache
 	var fhQ chan *os.File
-	var ok bool
+	var incache bool
 	var val interface{}
-	val, ok = h.fileHandlesQMap.Load(path)
-	if ok {
+
+	val, incache = h.fileHandlesQMap.Load(path)
+	if incache {
 		fhQ = val.(chan *os.File)
 		fh := <-fhQ
+		
 		return fh, nil
 	}
 
-	//at this point we need to create a new handle and create the pool handles if not full if it hasn't be created.
-	if fh, err = os.OpenFile(path, os.O_WRONLY, os.ModeAppend); err != nil {
-		return nil, err
+	//if init, the handle from the creation is not available (e.i. fh == nil)
+	if isInit {
+		if fh, err = os.OpenFile(path, os.O_WRONLY, os.ModeAppend); err != nil {
+			return nil, err
+		}
+
 	}
 
-	//the pool is full return the handle
+	h.Lock()
+	defer h.Unlock()
+	//ok not in the cache so we need to check if the cache is full
 	if h.cachedFileHandles+h.numOfHandlesPerFile > maxFileHandlesInCache {
-		util.PrintfIfDebug("getHandle -> Cache is full: %v\n", h.cachedFileHandles+h.numOfHandlesPerFile)
 		return fh, nil
 	}
 
-	//prime the pool with additional handles
-	if !isInit {
-		fhQ = make(chan *os.File, h.numOfHandlesPerFile)
-		h.cachedFileHandles = h.cachedFileHandles + 1
-		for i := 1; i < h.numOfHandlesPerFile; i++ {
-			var fhi *os.File
-			if fhi, err = os.OpenFile(path, os.O_WRONLY, os.ModeAppend); err != nil {
-				return nil, err
-			}
-			fhQ <- fhi
-			h.cachedFileHandles = h.cachedFileHandles + 1
-		}
+	//add items to the cache
+	fhQ = make(chan *os.File, h.numOfHandlesPerFile)
+	h.cachedFileHandles = h.cachedFileHandles + 1
+	for i := 1; i < h.numOfHandlesPerFile; i++ {
+		var fhi *os.File
 
-		h.fileHandlesQMap.Store(path, fhQ)
+		if fhi, err = os.OpenFile(path, os.O_WRONLY, os.ModeAppend); err != nil {
+			return nil, err
+		}
+		
+		fhQ <- fhi
+		h.cachedFileHandles = h.cachedFileHandles + 1
 	}
 
+	h.fileHandlesQMap.Store(path, fhQ)
+
 	return fh, nil
 
 }
@@ -187,14 +194,18 @@ func (h *fileHandleManager) returnHandle(path string, fileHandle *os.File) error
 	var ok bool
 
 	val, ok = h.fileHandlesQMap.Load(path)
+
 	if ok {
 		fhq = val.(chan *os.File)
+		//fmt.Printf("return to dequeue %v\n", path)
 		fhq <- fileHandle
+		//fmt.Printf("after to dequeue %v\n", path)
 		h.fileHandlesQMap.Store(path, fhq)
 		return nil
 	}
 
 	//not found in the map, which is the case when the cache is at capacity
+	//fmt.Printf("close (not in the map) %v", path)
 	return fileHandle.Close()
 }