Use cache multipart upload (#93)

* add cache multipart upload for huge files

Signed-off-by: Ayman <enkhalifapro@gmail.com>

* test changes

Signed-off-by: Ayman <enkhalifapro@gmail.com>

* clean up

Signed-off-by: Ayman <enkhalifapro@gmail.com>

* clean up

Signed-off-by: Ayman <enkhalifapro@gmail.com>

---------

Signed-off-by: Ayman <enkhalifapro@gmail.com>
Co-authored-by: Ayman <enkhalifapro@gmail.com>

Author: khalifapro

cmd/git/git.go

@@ -3453,18 +3453,17 @@ func (j *DSGit) createYearCacheFile(cache []CommitCache, path string) error {
 	if err != nil {
 		return err
 	}
-	file, err := os.ReadFile(cacheFile)
+
+	err = j.cacheProvider.UpdateMultiPartFileByKey(j.endpoint, cacheFile)
 	if err != nil {
 		return err
 	}
+
 	err = os.Remove(cacheFile)
 	if err != nil {
 		return err
 	}
-	err = j.cacheProvider.UpdateFileByKey(j.endpoint, cacheFile, file)
-	if err != nil {
-		return err
-	}
+
 	if len(nextYearCache) > 0 {
 		CurrentCacheYear = nextYearCache[0].CommitDate.Year()
 		if err = j.createYearCacheFile(nextYearCache, path); err != nil {

go.mod

@@ -3,7 +3,7 @@ module github.com/LF-Engineering/insights-datasource-git
 go 1.17
 
 require (
-	github.com/LF-Engineering/insights-datasource-shared v1.5.26
+	github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211
 	github.com/LF-Engineering/lfx-event-schema v0.1.37
 	github.com/aws/aws-lambda-go v1.27.1
 	github.com/aws/aws-sdk-go v1.42.25

go.sum

@@ -1,6 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/LF-Engineering/insights-datasource-shared v1.5.26 h1:zA4Vc/gTjcVJPr4uARisSl4MQiF1Br9XT9i71G5Gza0=
-github.com/LF-Engineering/insights-datasource-shared v1.5.26/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
+github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211 h1:Km27TFC9d8JJSE7IWkcOOVv6v3HqonsyOe0JK1F4p1Q=
+github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
 github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
 github.com/LF-Engineering/lfx-event-schema v0.1.37 h1:ny46D2NdCXokvJZ01GJcw2RfQM64ousJjaYsrRj5zzg=
 github.com/LF-Engineering/lfx-event-schema v0.1.37/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=