reduce memory usage in large repos (#94)

Signed-off-by: Ayman <enkhalifapro@gmail.com>
Co-authored-by: Ayman <enkhalifapro@gmail.com>

Author: khalifapro

cmd/git/git.go

@@ -3453,7 +3453,7 @@ func (j *DSGit) createYearCacheFile(cache []CommitCache, path string) error {
 	if err != nil {
 		return err
 	}
-
+	cachedCommits = make(map[string]CommitCache)
 	err = j.cacheProvider.UpdateMultiPartFileByKey(j.endpoint, cacheFile)
 	if err != nil {
 		return err
@@ -3463,7 +3463,7 @@ func (j *DSGit) createYearCacheFile(cache []CommitCache, path string) error {
 	if err != nil {
 		return err
 	}
-
+	loadCacheToMemory(records)
 	if len(nextYearCache) > 0 {
 		CurrentCacheYear = nextYearCache[0].CommitDate.Year()
 		if err = j.createYearCacheFile(nextYearCache, path); err != nil {
@@ -3684,6 +3684,48 @@ func (j *DSGit) getYearCache(lastSync string) {
 	}
 }
 
+func loadCacheToMemory(records [][]string) {
+	lastSync := os.Getenv("LAST_SYNC")
+	for i, record := range records {
+		if i == 0 {
+			continue
+		}
+		orphaned, err := strconv.ParseBool(record[5])
+		if err != nil {
+			orphaned = false
+		}
+		if lastSync != "" {
+			orphaned = true
+		}
+
+		var fromDL bool
+		if len(record) > 6 {
+			fromDL, err = strconv.ParseBool(record[6])
+			if err != nil {
+				fromDL = false
+			}
+		}
+
+		var content string
+		if len(record) > 7 {
+			if record[7] != "" {
+				content = record[7]
+			}
+		}
+
+		cachedCommits[record[4]] = CommitCache{
+			Timestamp:      record[0],
+			EntityID:       record[1],
+			SourceEntityID: record[2],
+			FileLocation:   record[3],
+			Hash:           record[4],
+			Orphaned:       orphaned,
+			FromDL:         fromDL,
+			Content:        content,
+		}
+	}
+}
+
 func isCommitCreated(id string) bool {
 	_, ok := cachedCommits[id]
 	return ok

go.mod

@@ -3,7 +3,7 @@ module github.com/LF-Engineering/insights-datasource-git
 go 1.17
 
 require (
-	github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211
+	github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230410030513-945f1d5a92a4
 	github.com/LF-Engineering/lfx-event-schema v0.1.37
 	github.com/aws/aws-lambda-go v1.27.1
 	github.com/aws/aws-sdk-go v1.42.25

go.sum

@@ -1,6 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211 h1:Km27TFC9d8JJSE7IWkcOOVv6v3HqonsyOe0JK1F4p1Q=
-github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230408041536-83d398cfb211/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
+github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230410030513-945f1d5a92a4 h1:q9DZVrh19QzM0s6EEbAHFkf51Ubw4jw5g1qlKASlHvc=
+github.com/LF-Engineering/insights-datasource-shared v1.5.30-0.20230410030513-945f1d5a92a4/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
 github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
 github.com/LF-Engineering/lfx-event-schema v0.1.37 h1:ny46D2NdCXokvJZ01GJcw2RfQM64ousJjaYsrRj5zzg=
 github.com/LF-Engineering/lfx-event-schema v0.1.37/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=