From 5b5e3a8b49d643a54b0a1188107f33e6d0346371 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 21 Mar 2025 17:19:35 -0400 Subject: [PATCH 1/2] feat: Adding basic benchmarks --- test/benchmarks/helpers/helpers.go | 195 ++++++++++++++++++++++ test/benchmarks/integration_bench_test.go | 126 ++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 test/benchmarks/helpers/helpers.go create mode 100644 test/benchmarks/integration_bench_test.go diff --git a/test/benchmarks/helpers/helpers.go b/test/benchmarks/helpers/helpers.go new file mode 100644 index 0000000000..d7486961f5 --- /dev/null +++ b/test/benchmarks/helpers/helpers.go @@ -0,0 +1,195 @@ +// Package helpers provides helper functions for the integration benchmarks. +package helpers + +import ( + "context" + "io" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gruntwork-io/terragrunt/cli" + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/pkg/log" + "github.com/stretchr/testify/require" +) + +// RunTerragruntCommand runs a Terragrunt command and logs the output to io.Discard. +func RunTerragruntCommand(b *testing.B, args ...string) { + b.Helper() + + writer := io.Discard + errwriter := io.Discard + + opts := options.NewTerragruntOptionsWithWriters(writer, errwriter) + app := cli.NewApp(opts) //nolint:contextcheck + + ctx := log.ContextWithLogger(context.Background(), opts.Logger) + + err := app.RunContext(ctx, args) + require.NoError(b, err) +} + +// GenerateNUnits generates n units in the given temporary directory. +func GenerateNUnits(b *testing.B, dir string, n int, tgConfig string, tfConfig string) { + b.Helper() + + for i := range n { + unitDir := filepath.Join(dir, "unit-"+strconv.Itoa(i)) + require.NoError(b, os.MkdirAll(unitDir, 0755)) + + // Create an empty `terragrunt.hcl` file + unitTerragruntConfigPath := filepath.Join(unitDir, "terragrunt.hcl") + require.NoError(b, os.WriteFile(unitTerragruntConfigPath, []byte(tgConfig), 0644)) + + // Create an empty `main.tf` file + unitMainTfPath := filepath.Join(unitDir, "main.tf") + require.NoError(b, os.WriteFile(unitMainTfPath, []byte(tfConfig), 0644)) + } +} + +// GenerateEmptyUnits generates n empty units in the given temporary directory. +func GenerateEmptyUnits(b *testing.B, dir string, n int) { + b.Helper() + + emptyRootConfig := `` + includeRootConfig := `include "root" { + path = find_in_parent_folders("root.hcl") +} +` + emptyMainTf := `` + + rootTerragruntConfigPath := filepath.Join(dir, "root.hcl") + + // Create an empty `root.hcl` file + require.NoError(b, os.WriteFile(rootTerragruntConfigPath, []byte(emptyRootConfig), 0644)) + + // Generate n units + GenerateNUnits(b, dir, n, includeRootConfig, emptyMainTf) +} + +// GenerateDependencyTrain generates a dependency train in the given temporary directory. +// A dependency train is a set of units where each unit depends on the previous unit (except for the first unit). +func GenerateDependencyTrain(b *testing.B, dir string, n int) { + b.Helper() + + if n <= 1 { + b.Fatalf("n must be greater than 1") + } + + // Create the root config + rootConfig := `` + + rootConfigPath := filepath.Join(dir, "root.hcl") + require.NoError(b, os.WriteFile(rootConfigPath, []byte(rootConfig), 0644)) + + // Create the first unit + unitDir := filepath.Join(dir, "unit-0") + require.NoError(b, os.MkdirAll(unitDir, 0755)) + + // Create an empty `terragrunt.hcl` file + unitTerragruntConfigPath := filepath.Join(unitDir, "terragrunt.hcl") + require.NoError(b, os.WriteFile(unitTerragruntConfigPath, []byte(`include "root" { + path = find_in_parent_folders("root.hcl") +} + +terraform { + source = "." +} + +inputs = { + unit_count = `+strconv.Itoa(n)+` +} +`), 0644)) + + // Create an empty `main.tf` file + unitMainTfPath := filepath.Join(unitDir, "main.tf") + require.NoError(b, os.WriteFile(unitMainTfPath, []byte(`variable "unit_count" { + type = number +} + +resource "null_resource" "this" { + triggers = { + always_run = timestamp() + } +} + +output "unit_count" { + value = var.unit_count +} +`), 0644)) + + // Create N-1 units + for i := 1; i < n; i++ { + unitDir := filepath.Join(dir, "unit-"+strconv.Itoa(i)) + require.NoError(b, os.MkdirAll(unitDir, 0755)) + + // Create an empty `terragrunt.hcl` file + unitTerragruntConfigPath := filepath.Join(unitDir, "terragrunt.hcl") + require.NoError(b, os.WriteFile(unitTerragruntConfigPath, []byte(`include "root" { + path = find_in_parent_folders("root.hcl") +} + +terraform { + source = "." +} + +dependency "unit_`+strconv.Itoa(i-1)+`" { + config_path = "../unit-`+strconv.Itoa(i-1)+`" + + mock_outputs = { + unit_count = 0 + } + + mock_outputs_allowed_terraform_commands = ["plan"] +} + +inputs = { + unit_count = dependency.unit_`+strconv.Itoa(i-1)+`.outputs.unit_count + 1 +} + +`), 0644)) + + // Create an empty `main.tf` file + unitMainTfPath := filepath.Join(unitDir, "main.tf") + require.NoError(b, os.WriteFile(unitMainTfPath, []byte(`variable "unit_count" { + type = number +} + +output "unit_count" { + value = var.unit_count +} + `), 0644)) + } +} + +func PlanApplyDestroy(b *testing.B, dir string) { + b.Helper() + + // Measure plan time + planStart := time.Now() + + RunTerragruntCommand(b, "terragrunt", "run-all", "plan", "--non-interactive", "--working-dir", dir) + + planDuration := time.Since(planStart) + + // Measure apply time + applyStart := time.Now() + + RunTerragruntCommand(b, "terragrunt", "run-all", "apply", "-auto-approve", "--non-interactive", "--working-dir", dir) + + applyDuration := time.Since(applyStart) + + // Measure destroy time + destroyStart := time.Now() + + RunTerragruntCommand(b, "terragrunt", "run-all", "destroy", "-auto-approve", "--non-interactive", "--working-dir", dir) + + destroyDuration := time.Since(destroyStart) + + b.ReportMetric(float64(planDuration.Seconds()), "plan_s/op") + b.ReportMetric(float64(applyDuration.Seconds()), "apply_s/op") + b.ReportMetric(float64(destroyDuration.Seconds()), "destroy_s/op") +} diff --git a/test/benchmarks/integration_bench_test.go b/test/benchmarks/integration_bench_test.go new file mode 100644 index 0000000000..5b3673c344 --- /dev/null +++ b/test/benchmarks/integration_bench_test.go @@ -0,0 +1,126 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gruntwork-io/terragrunt/test/benchmarks/helpers" + "github.com/stretchr/testify/require" +) + +func BenchmarkEmptyTerragruntPlanApply(b *testing.B) { + emptyMainTf := `` + + emptyRootConfig := `` + includeRootConfig := `include "root" { + path = find_in_parent_folders("root.hcl") +} + +terraform { + source = "." +} +` + + b.Run("10 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + // Create a temporary directory for the test + tmpDir := b.TempDir() + rootTerragruntConfigPath := filepath.Join(tmpDir, "root.hcl") + // Create an empty `root.hcl` file + require.NoError(b, os.WriteFile(rootTerragruntConfigPath, []byte(emptyRootConfig), 0644)) + + // Create 10 units + helpers.GenerateNUnits(b, tmpDir, 10, includeRootConfig, emptyMainTf) + + b.StartTimer() + + helpers.PlanApplyDestroy(b, tmpDir) + } + }) + + b.Run("100 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + // Create a temporary directory for the test + tmpDir := b.TempDir() + rootTerragruntConfigPath := filepath.Join(tmpDir, "root.hcl") + + // Create an empty `root.hcl` file + require.NoError(b, os.WriteFile(rootTerragruntConfigPath, []byte(emptyRootConfig), 0644)) + + // Create 100 units + helpers.GenerateNUnits(b, tmpDir, 100, includeRootConfig, emptyMainTf) + + b.StartTimer() + + helpers.PlanApplyDestroy(b, tmpDir) + } + }) + + b.Run("1000 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + // Create a temporary directory for the test + tmpDir := b.TempDir() + rootTerragruntConfigPath := filepath.Join(tmpDir, "root.hcl") + + // Create an empty `root.hcl` file + require.NoError(b, os.WriteFile(rootTerragruntConfigPath, []byte(emptyRootConfig), 0644)) + + // Create 1000 units + helpers.GenerateNUnits(b, tmpDir, 1000, includeRootConfig, emptyMainTf) + + b.StartTimer() + + helpers.PlanApplyDestroy(b, tmpDir) + } + }) +} + +// TODO: Enable this once it's fixed. +// +// func BenchmarkDependencyTrainPlanApply(b *testing.B) { +// b.Run("10 units", func(b *testing.B) { +// for i := 0; i < b.N; i++ { +// b.StopTimer() +// +// tmpDir := b.TempDir() +// helpers.GenerateDependencyTrain(b, tmpDir, 10) +// +// b.StartTimer() +// +// helpers.PlanApplyDestroy(b, tmpDir) +// } +// }) +// +// b.Run("100 units", func(b *testing.B) { +// for i := 0; i < b.N; i++ { +// b.StopTimer() +// +// tmpDir := b.TempDir() +// helpers.GenerateDependencyTrain(b, tmpDir, 100) +// +// b.StartTimer() +// +// helpers.PlanApplyDestroy(b, tmpDir) +// } +// }) +// +// b.Run("1000 units", func(b *testing.B) { +// for i := 0; i < b.N; i++ { +// b.StopTimer() +// +// tmpDir := b.TempDir() +// helpers.GenerateDependencyTrain(b, tmpDir, 1000) +// +// b.StartTimer() +// +// helpers.PlanApplyDestroy(b, tmpDir) +// } +// }) +// } From 5ae0d39c589e3b20b0b125ff55ae73b6152138f2 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:02:39 -0400 Subject: [PATCH 2/2] feat: Slightly better --- test/benchmarks/helpers/helpers.go | 13 ++++ test/benchmarks/integration_bench_test.go | 84 +++++++++++------------ 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/test/benchmarks/helpers/helpers.go b/test/benchmarks/helpers/helpers.go index d7486961f5..48f1b748bc 100644 --- a/test/benchmarks/helpers/helpers.go +++ b/test/benchmarks/helpers/helpers.go @@ -193,3 +193,16 @@ func PlanApplyDestroy(b *testing.B, dir string) { b.ReportMetric(float64(applyDuration.Seconds()), "apply_s/op") b.ReportMetric(float64(destroyDuration.Seconds()), "destroy_s/op") } + +func Plan(b *testing.B, dir string) { + b.Helper() + + // Measure plan time + planStart := time.Now() + + RunTerragruntCommand(b, "terragrunt", "run-all", "plan", "--non-interactive", "--working-dir", dir) + + planDuration := time.Since(planStart) + + b.ReportMetric(float64(planDuration.Seconds()), "plan_s/op") +} diff --git a/test/benchmarks/integration_bench_test.go b/test/benchmarks/integration_bench_test.go index 5b3673c344..4fb68add83 100644 --- a/test/benchmarks/integration_bench_test.go +++ b/test/benchmarks/integration_bench_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -func BenchmarkEmptyTerragruntPlanApply(b *testing.B) { +func BenchmarkEmptyTerragruntPlanApplyDestroy(b *testing.B) { emptyMainTf := `` emptyRootConfig := `` @@ -82,45 +82,43 @@ terraform { }) } -// TODO: Enable this once it's fixed. -// -// func BenchmarkDependencyTrainPlanApply(b *testing.B) { -// b.Run("10 units", func(b *testing.B) { -// for i := 0; i < b.N; i++ { -// b.StopTimer() -// -// tmpDir := b.TempDir() -// helpers.GenerateDependencyTrain(b, tmpDir, 10) -// -// b.StartTimer() -// -// helpers.PlanApplyDestroy(b, tmpDir) -// } -// }) -// -// b.Run("100 units", func(b *testing.B) { -// for i := 0; i < b.N; i++ { -// b.StopTimer() -// -// tmpDir := b.TempDir() -// helpers.GenerateDependencyTrain(b, tmpDir, 100) -// -// b.StartTimer() -// -// helpers.PlanApplyDestroy(b, tmpDir) -// } -// }) -// -// b.Run("1000 units", func(b *testing.B) { -// for i := 0; i < b.N; i++ { -// b.StopTimer() -// -// tmpDir := b.TempDir() -// helpers.GenerateDependencyTrain(b, tmpDir, 1000) -// -// b.StartTimer() -// -// helpers.PlanApplyDestroy(b, tmpDir) -// } -// }) -// } +func BenchmarkDependencyTrainPlan(b *testing.B) { + b.Run("10 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + tmpDir := b.TempDir() + helpers.GenerateDependencyTrain(b, tmpDir, 10) + + b.StartTimer() + + helpers.Plan(b, tmpDir) + } + }) + + b.Run("100 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + tmpDir := b.TempDir() + helpers.GenerateDependencyTrain(b, tmpDir, 100) + + b.StartTimer() + + helpers.Plan(b, tmpDir) + } + }) + + b.Run("1000 units", func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + + tmpDir := b.TempDir() + helpers.GenerateDependencyTrain(b, tmpDir, 1000) + + b.StartTimer() + + helpers.Plan(b, tmpDir) + } + }) +}