chore: using configured stan modelId (#1237)

Author: Ubisoft-potato

manifests/bucketeer/charts/batch/templates/deployment.yaml

@@ -144,6 +144,8 @@ spec:
               value: "{{ .Values.env.nonPersistentRedis.poolMaxActive }}"
             - name: BUCKETEER_BATCH_EXPERIMENT_LOCK_TTL
               value: "{{ .Values.env.experimentLockTTL }}"
+            - name: BUCKETEER_BATCH_STAN_MODEL_ID
+              value: "{{ .Values.httpstan.modelId }}"
 
           volumeMounts:
             - name: service-cert-secret

manifests/bucketeer/charts/batch/values.yaml

@@ -97,6 +97,10 @@ httpstan:
     pullPolicy: IfNotPresent
   config:
   port: 8080
+  # The model ID is generated when the HTTP Stan model is compiled.
+  # We must update the ID every time we compile the model.
+  # Since we don't change the model so often, this ID won't change.
+  modelId: y3qsnd7m
   healthPath: v1/health
   resources: {}
 

manifests/bucketeer/values.dev.yaml

@@ -161,7 +161,7 @@ batch-server:
       addr: localenv-redis-headless.default.svc.cluster.local:6379
       poolMaxIdle: 25
       poolMaxActive: 25
-    experimentLockTTL: 10m
+    experimentLockTTL: 1m
 
   tls:
     service:

pkg/batch/cmd/server/server.go

@@ -80,6 +80,7 @@ type server struct {
 	oauthIssuer        *string
 	stanHost           *string
 	stanPort           *string
+	stanModelID        *string
 	// MySQL
 	mysqlUser        *string
 	mysqlPass        *string
@@ -136,6 +137,7 @@ func RegisterCommand(r cli.CommandRegistry, p cli.ParentCommand) cli.Command {
 		oauthIssuer:      cmd.Flag("oauth-issuer", "The issuer url").Required().String(),
 		stanHost:         cmd.Flag("stan-host", "httpstan host.").Default("localhost").String(),
 		stanPort:         cmd.Flag("stan-port", "httpstan port.").Default("8080").String(),
+		stanModelID:      cmd.Flag("stan-model-id", "httpstan modelId.").Required().String(),
 		mysqlUser:        cmd.Flag("mysql-user", "MySQL user.").Required().String(),
 		mysqlPass:        cmd.Flag("mysql-pass", "MySQL password.").Required().String(),
 		mysqlHost:        cmd.Flag("mysql-host", "MySQL host.").Required().String(),
@@ -456,6 +458,7 @@ func (s *server) Run(ctx context.Context, metrics metrics.Metrics, logger *zap.L
 		),
 		calculator.NewExperimentCalculate(
 			stan.NewStan(*s.stanHost, *s.stanPort),
+			*s.stanModelID,
 			environmentClient,
 			experimentClient,
 			eventCounterClient,

pkg/batch/jobs/calculator/experiment_calculate.go

@@ -52,6 +52,7 @@ type experimentCalculate struct {
 
 func NewExperimentCalculate(
 	httpStan *stan.Stan,
+	stanModelID string,
 	environmentClient environmentclient.Client,
 	experimentClient experimentclient.Client,
 	ecClient ecclient.Client,
@@ -69,6 +70,7 @@ func NewExperimentCalculate(
 	}
 	calculator := experimentcalc.NewExperimentCalculator(
 		httpStan,
+		stanModelID,
 		environmentClient,
 		ecClient,
 		experimentClient,
@@ -154,8 +156,9 @@ func (e *experimentCalculate) calculateExperimentWithLock(ctx context.Context,
 		)
 		return nil
 	}
-
-	defer func() {
+	if calcErr := e.calculateExperiment(ctx, env, experiment); calcErr != nil {
+		// To prevent calculating the same experiment multiple times in a short time,
+		// we set the TTL for the lock key and only unlock it when an error occurs so that it can retry.
 		unlocked, unlockErr := e.experimentLock.Unlock(ctx, env.Id, experiment.Id, lockValue)
 		if unlockErr != nil {
 			e.logger.Error("Failed to release lock",
@@ -170,9 +173,9 @@ func (e *experimentCalculate) calculateExperimentWithLock(ctx context.Context,
 				zap.String("experimentId", experiment.Id),
 			)
 		}
-	}()
-
-	return e.calculateExperiment(ctx, env, experiment)
+		return calcErr
+	}
+	return nil
 }
 
 func (e *experimentCalculate) calculateExperiment(ctx context.Context,

pkg/experimentcalculator/experimentcalc/experiment_calculator.go

@@ -50,8 +50,8 @@ const (
 )
 
 type ExperimentCalculator struct {
-	httpStan *stan.Stan
-	modelID  string
+	httpStan    *stan.Stan
+	stanModelID string
 
 	environmentClient  envclient.Client
 	eventCounterClient ecclient.Client
@@ -65,6 +65,7 @@ type ExperimentCalculator struct {
 
 func NewExperimentCalculator(
 	httpStan *stan.Stan,
+	stanModelID string,
 	environmentClient envclient.Client,
 	eventCounterClient ecclient.Client,
 	experimentClient experimentclient.Client,
@@ -74,24 +75,9 @@ func NewExperimentCalculator(
 	logger *zap.Logger,
 ) *ExperimentCalculator {
 	registerMetrics(metrics)
-	var compiledModel stan.ModelCompileResp
-	for i := 0; i < 3; i++ {
-		resp, err := httpStan.CompileModel(context.TODO(), stan.ModelCode())
-		if err != nil {
-			logger.Warn("ExperimentCalculator failed to compile model, retrying...",
-				zap.Error(err),
-			)
-			time.Sleep(1 * time.Second)
-		} else {
-			compiledModel = resp
-			break
-		}
-	}
-	modelID := compiledModel.Name[len("models/"):]
 	return &ExperimentCalculator{
-		httpStan: httpStan,
-		modelID:  modelID,
-
+		httpStan:           httpStan,
+		stanModelID:        stanModelID,
 		environmentClient:  environmentClient,
 		eventCounterClient: eventCounterClient,
 		experimentClient:   experimentClient,
@@ -462,11 +448,11 @@ func (e ExperimentCalculator) binomialModelSample(
 				NumWarmup:  1000,
 				RandomSeed: 1234,
 			}
-			fitResp, err := e.httpStan.CreateFit(ctx, e.modelID, req)
+			fitResp, err := e.httpStan.CreateFit(ctx, e.stanModelID, req)
 			if err != nil {
 				e.logger.Error("Failed to create fit",
 					log.FieldsFromImcomingContext(ctx).AddFields(
-						zap.String("modelId", e.modelID),
+						zap.String("modelId", e.stanModelID),
 						zap.Error(err),
 					)...,
 				)
@@ -489,7 +475,7 @@ func (e ExperimentCalculator) binomialModelSample(
 				}
 				time.Sleep(50 * time.Millisecond)
 			}
-			result, err := e.httpStan.GetFitResult(ctx, e.modelID, fitId)
+			result, err := e.httpStan.GetFitResult(ctx, e.stanModelID, fitId)
 			if err != nil {
 				e.logger.Error("Failed to get fit result",
 					log.FieldsFromImcomingContext(ctx).AddFields(
@@ -500,7 +486,7 @@ func (e ExperimentCalculator) binomialModelSample(
 				return
 			}
 			fit := e.httpStan.ExtractFromFitResult(ctx, result)
-			constrainedNames, _ := e.httpStan.StanParams(ctx, e.modelID, req.Data)
+			constrainedNames, _ := e.httpStan.StanParams(ctx, e.stanModelID, req.Data)
 			samplesChan <- fit.Select(constrainedNames)
 		}(i)
 	}

pkg/experimentcalculator/experimentcalc/experiment_calculator_test.go

@@ -36,11 +36,16 @@ var (
 	jpLocation = time.FixedZone("Asia/Tokyo", 9*60*60)
 )
 
+const (
+	stanModelID = "y3qsnd7m"
+)
+
 func creatExperimentCalculator(mockController *gomock.Controller) *ExperimentCalculator {
 	registerer := metricsmock.NewMockRegisterer(mockController)
 	registerer.EXPECT().MustRegister(gomock.Any()).Return()
 	return NewExperimentCalculator(
 		stan.NewStan("localhost", "8080"),
+		stanModelID,
 		envclient.NewMockClient(mockController),
 		ecclient.NewMockClient(mockController),
 		experimentclient.NewMockClient(mockController),