Improve efficiency when processing response (#258)

* Improve efficiency when processing response

* Gofmt

* Bring back warnings logs, but as debug

* Revert "Bring back warnings logs, but as debug"

This reverts commit 58103d3.

* Bring back warnings logs, but as debug

* Fix lint

* Fix lint

Author: ivanahuckova

pkg/googlesheets/googlesheets.go

@@ -81,7 +81,7 @@ func (gs *GoogleSheets) GetSpreadsheets(ctx context.Context, config models.Datas
 		return nil, err
 	}
 
-	fileNames := map[string]string{}
+	fileNames := make(map[string]string, len(files))
 	for _, i := range files {
 		fileNames[i.Id] = i.Name
 	}
@@ -180,16 +180,18 @@ func (gs *GoogleSheets) transformSheetToDataFrame(sheet *sheets.GridData, meta m
 		if column.HasMixedTypes() {
 			warning := fmt.Sprintf("Multiple data types found in column %q. Using string data type", column.Header)
 			warnings = append(warnings, warning)
-			log.DefaultLogger.Warn(warning)
+			log.DefaultLogger.Debug(warning)
 		}
 
 		if column.HasMixedUnits() {
 			warning := fmt.Sprintf("Multiple units found in column %q. Formatted value will be used", column.Header)
 			warnings = append(warnings, warning)
-			log.DefaultLogger.Warn(warning)
+			log.DefaultLogger.Debug(warning)
 		}
 	}
 
+	// We want to show the warnings only once per column
+	warningsIncludeConverterErrorForColumns := make(map[int]bool, len(columns))
 	for rowIndex := start; rowIndex < len(sheet.RowData); rowIndex++ {
 		for columnIndex, cellData := range sheet.RowData[rowIndex].Values {
 			if columnIndex >= len(columns) {
@@ -202,8 +204,10 @@ func (gs *GoogleSheets) transformSheetToDataFrame(sheet *sheets.GridData, meta m
 			}
 
 			err := inputConverter.Set(columnIndex, rowIndex-start, cellData)
-			if err != nil {
+			if err != nil && !warningsIncludeConverterErrorForColumns[columnIndex] {
+				log.DefaultLogger.Debug("unsuccessful converting of cell data", "err", err)
 				warnings = append(warnings, err.Error())
+				warningsIncludeConverterErrorForColumns[columnIndex] = true
 			}
 		}
 	}
@@ -212,7 +216,6 @@ func (gs *GoogleSheets) transformSheetToDataFrame(sheet *sheets.GridData, meta m
 	meta["spreadsheetId"] = qm.Spreadsheet
 	meta["range"] = qm.Range
 	frame.Meta = &data.FrameMeta{Custom: meta}
-	log.DefaultLogger.Debug("frame.Meta: %s", spew.Sdump(frame.Meta))
 	return frame, nil
 }
 
@@ -287,13 +290,13 @@ func getUniqueColumnName(formattedName string, columnIndex int, columns map[stri
 }
 
 func getColumnDefinitions(rows []*sheets.RowData) ([]*ColumnDefinition, int) {
-	columns := []*ColumnDefinition{}
-	columnMap := map[string]bool{}
 	if len(rows) < 1 {
-		return columns, 0
+		return []*ColumnDefinition{}, 0
 	}
 	headerRow := rows[0].Values
 
+	columns := make([]*ColumnDefinition, 0, len(headerRow))
+	columnMap := make(map[string]bool, len(headerRow))
 	start := 0
 	if len(rows) > 1 {
 		start = 1

pkg/googlesheets/googlesheets_bench_test.go

@@ -0,0 +1,42 @@
+package googlesheets
+
+import (
+	"testing"
+
+	"github.com/grafana/google-sheets-datasource/pkg/models"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/stretchr/testify/require"
+)
+
+// To avoid compiler optimizations eliminating the function under test
+// we are storing the result to a package level variable
+var Frame *data.Frame
+
+func BenchmarkTransformMixedSheetToDataFrame(b *testing.B) {
+	sheet, err := loadTestSheet("./testdata/mixed-data.json")
+	require.NoError(b, err)
+	gsd := &GoogleSheets{}
+	qm := models.QueryModel{Spreadsheet: "someId"}
+	meta := make(map[string]any)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		frame, err := gsd.transformSheetToDataFrame(sheet.Sheets[0].Data[0], meta, "ref1", &qm)
+		require.NoError(b, err)
+		Frame = frame
+	}
+}
+
+func BenchmarkTransformMixedSheetWithInvalidDateToDataFrame(b *testing.B) {
+	sheet, err := loadTestSheet("./testdata/invalid-date-time.json")
+	require.NoError(b, err)
+	gsd := &GoogleSheets{}
+	qm := models.QueryModel{Spreadsheet: "someId"}
+	meta := make(map[string]any)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		frame, err := gsd.transformSheetToDataFrame(sheet.Sheets[0].Data[0], meta, "ref1", &qm)
+		require.NoError(b, err)
+		Frame = frame
+	}
+}