config(cody): Various fixes and updating the naming rules for ModelConfig resource IDs (#63436)

The `internal/modelconfig` package provides the schema by which
Sourcegraph instances will keep track of LLM providers, models, and
various configuration settings. The `cmd/cody-gateway-config` tool
writes a JSON file contains the model configuration data for Cody
Gateway.

This PR fixes an assortment if problems with these components.

d8963da - Due to a logic error, we were
saving the model ID of "unknown" into `models.json`. We now put the
actual `ModelID` from the `ModelRef`.

d9baa65 - Updates the model information
that gets rendered, so that it matches what is hard-coded into the
`sourcegraph/cody` repo. (We were missing any reference to the newly
added Google Gemini models.)

c28780c - Relaxes the resource ID
regular expression so that it is now legal to add periods. So
"gemini-1.5-latest" is now considered a valid `ModelID`.

... however, the validation checks were incorrectly passing because
there was a bug in the regular expression. And after writing some unit
tests for the `validateModelRef` function, I found several other
problems with that regular expression 😅 . But we should be much closer
to things working as intended now.


## Test plan

Added tests

Author: chrsmith

cmd/cody-gateway-config/dotcom_models.go

@@ -64,7 +64,7 @@ func getAnthropicModels() []types.Model {
 		// Pro Anthropic models.
 		newModel(
 			modelIdentity{
-				MRef:        mRef(anthropic_06_2023, "claude-3-5-sonnet"),
+				MRef:        mRef(anthropic_06_2023, "claude-3.5-sonnet"),
 				Name:        "claude-3-5-sonnet-20240620",
 				DisplayName: "Claude 3.5 Sonnet",
 			},
@@ -145,6 +145,43 @@ func getAnthropicModels() []types.Model {
 	}
 }
 
+func getGoogleModels() []types.Model {
+	const (
+		// Gemini API versions.
+		// https://ai.google.dev/gemini-api/docs/api-versions
+		geminiV1 = "google::v1"
+	)
+
+	return []types.Model{
+		newModel(
+			modelIdentity{
+				MRef:        mRef(geminiV1, "gemini-1.5-pro-latest"),
+				Name:        "gemini-1.5-pro-latest",
+				DisplayName: "Gemini 1.5 Pro",
+			},
+			modelMetadata{
+				Capabilities: chatAndEdit,
+				Category:     types.ModelCategoryAccuracy,
+				Status:       types.ModelStatusStable,
+				Tier:         types.ModelTierPro,
+			},
+			expandedCtxWindow),
+		newModel(
+			modelIdentity{
+				MRef:        mRef(geminiV1, "gemini-1.5-flash-latest"),
+				Name:        "google/gemini-1.5-flash-latest",
+				DisplayName: "Gemini 1.5 Flash",
+			},
+			modelMetadata{
+				Capabilities: chatAndEdit,
+				Category:     types.ModelCategorySpeed,
+				Status:       types.ModelStatusStable,
+				Tier:         types.ModelTierPro,
+			},
+			expandedCtxWindow),
+	}
+}
+
 func getMistralModels() []types.Model {
 	// Not sure if there is a canonical API reference, since Mixtral offers 3rd
 	// party LLMs as a service.
@@ -209,7 +246,7 @@ func getOpenAIModels() []types.Model {
 			},
 			modelMetadata{
 				Capabilities: chatAndEdit,
-				Category:     types.ModelCategoryAccuracy,
+				Category:     types.ModelCategoryBalanced,
 				Status:       types.ModelStatusStable,
 				Tier:         types.ModelTierPro,
 			},
@@ -238,6 +275,7 @@ func GetCodyFreeProModels() ([]types.Model, error) {
 	// ================================================
 	var allModels []types.Model
 	allModels = append(allModels, getAnthropicModels()...)
+	allModels = append(allModels, getGoogleModels()...)
 	allModels = append(allModels, getMistralModels()...)
 	allModels = append(allModels, getOpenAIModels()...)
 

cmd/cody-gateway-config/utils.go

@@ -38,7 +38,7 @@ func newModel(identity modelIdentity, metadata modelMetadata, ctxWindow types.Co
 	// before writing it out.
 	modelID := types.ModelID("unknown")
 	parts := strings.Split(string(identity.MRef), "::")
-	if len(parts) != 3 {
+	if len(parts) == 3 {
 		modelID = types.ModelID(parts[2])
 	}
 

internal/modelconfig/embedded/models.json

@@ -21,11 +21,11 @@
   ],
   "models": [
     {
-      "modelId": "unknown",
+      "modelId": "claude-3-sonnet",
       "modelRef": "anthropic::2023-06-01::claude-3-sonnet",
       "displayName": "Claude 3 Sonnet",
       "string": "claude-3-sonnet-20240229",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "balanced",
       "status": "stable",
       "tier": "free",
@@ -35,11 +35,11 @@
       }
     },
     {
-      "modelId": "unknown",
-      "modelRef": "anthropic::2023-06-01::claude-3-5-sonnet",
+      "modelId": "claude-3.5-sonnet",
+      "modelRef": "anthropic::2023-06-01::claude-3.5-sonnet",
       "displayName": "Claude 3.5 Sonnet",
       "string": "claude-3-5-sonnet-20240620",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "accuracy",
       "status": "stable",
       "tier": "pro",
@@ -49,11 +49,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "claude-3-opus",
       "modelRef": "anthropic::2023-06-01::claude-3-opus",
       "displayName": "Claude 3 Opus",
       "string": "claude-3-opus-20240229",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "accuracy",
       "status": "stable",
       "tier": "pro",
@@ -63,11 +63,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "claude-3-haiku",
       "modelRef": "anthropic::2023-06-01::claude-3-haiku",
       "displayName": "Claude 3 Haiku",
       "string": "claude-3-haiku-20240307",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "speed",
       "status": "stable",
       "tier": "pro",
@@ -77,11 +77,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "claude-2.1",
       "modelRef": "anthropic::2023-01-01::claude-2.1",
       "displayName": "Claude 2.1",
       "string": "claude-2.1",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "balanced",
       "status": "deprecated",
       "tier": "free",
@@ -91,11 +91,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "claude-2.0",
       "modelRef": "anthropic::2023-01-01::claude-2.0",
       "displayName": "Claude 2.0",
       "string": "claude-2.0",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "balanced",
       "status": "deprecated",
       "tier": "free",
@@ -105,11 +105,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "claude-instant-1.2",
       "modelRef": "anthropic::2023-01-01::claude-instant-1.2",
       "displayName": "Claude Instant",
       "string": "claude-instant-1.2",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "balanced",
       "status": "deprecated",
       "tier": "free",
@@ -119,11 +119,39 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "gemini-1.5-pro-latest",
+      "modelRef": "google::v1::gemini-1.5-pro-latest",
+      "displayName": "Gemini 1.5 Pro",
+      "string": "gemini-1.5-pro-latest",
+      "capabilities": ["autocomplete", "chat"],
+      "category": "accuracy",
+      "status": "stable",
+      "tier": "pro",
+      "contextWindow": {
+        "maxInputTokens": 30000,
+        "maxOutputTokens": 4000
+      }
+    },
+    {
+      "modelId": "gemini-1.5-flash-latest",
+      "modelRef": "google::v1::gemini-1.5-flash-latest",
+      "displayName": "Gemini 1.5 Flash",
+      "string": "google/gemini-1.5-flash-latest",
+      "capabilities": ["autocomplete", "chat"],
+      "category": "speed",
+      "status": "stable",
+      "tier": "pro",
+      "contextWindow": {
+        "maxInputTokens": 30000,
+        "maxOutputTokens": 4000
+      }
+    },
+    {
+      "modelId": "mixtral-8x7b-instruct",
       "modelRef": "mistral::v1::mixtral-8x7b-instruct",
       "displayName": "Mixtral 8x7B",
       "string": "mixtral-8x7b-instruct",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "speed",
       "status": "stable",
       "tier": "pro",
@@ -133,11 +161,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "mixtral-8x22b-instruct",
       "modelRef": "mistral::v1::mixtral-8x22b-instruct",
       "displayName": "Mixtral 8x22B",
       "string": "mixtral-8x22b-instruct",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "accuracy",
       "status": "stable",
       "tier": "pro",
@@ -147,11 +175,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "gpt-4o",
       "modelRef": "openai::2024-02-01::gpt-4o",
       "displayName": "GPT-4o",
       "string": "gpt-4o",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "accuracy",
       "status": "stable",
       "tier": "pro",
@@ -161,12 +189,12 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "gpt-4-turbo",
       "modelRef": "openai::2024-02-01::gpt-4-turbo",
       "displayName": "GPT-4 Turbo",
       "string": "gpt-4-turbo",
-      "capabilities": ["chat", "edit"],
-      "category": "accuracy",
+      "capabilities": ["autocomplete", "chat"],
+      "category": "balanced",
       "status": "stable",
       "tier": "pro",
       "contextWindow": {
@@ -175,11 +203,11 @@
       }
     },
     {
-      "modelId": "unknown",
+      "modelId": "gpt-3.5-turbo",
       "modelRef": "openai::2024-02-01::gpt-3.5-turbo",
       "displayName": "GPT-3.5 Turbo",
       "string": "gpt-3.5-turbo",
-      "capabilities": ["chat", "edit"],
+      "capabilities": ["autocomplete", "chat"],
       "category": "speed",
       "status": "stable",
       "tier": "pro",

internal/modelconfig/validation.go

@@ -16,7 +16,7 @@ import (
 
 // resourceIDRE is a regular expression for verifying resource IDs are
 // of a simple format.
-var resourceIDRE = regexp.MustCompile(`[a-z][a-z-]*[a-z]`)
+var resourceIDRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_\-\.]*[a-z0-9]$`)
 
 func validateProvider(p types.Provider) error {
 	if l := len(p.DisplayName); l < 5 || l > 40 {
@@ -36,17 +36,23 @@ func validateProvider(p types.Provider) error {
 func validateModelRef(ref types.ModelRef) error {
 	parts := strings.Split(string(ref), "::")
 	if len(parts) != 3 {
-		return errors.New("invalid number of parts")
+		return errors.New("modelRef syntax error")
 	}
 	if !resourceIDRE.MatchString(parts[0]) {
 		return errors.New("invalid ProviderID")
 	}
+	if !resourceIDRE.MatchString(parts[2]) {
+		return errors.New("invalid ModelID")
+	}
 	// We don't impose any constraints on the API Version ID, because
 	// while it's something Sourcegraph manages, there are lots of exotic
 	// but reasonable forms it could take. e.g. "2024-06-01" or
 	// "v1+beta2/with-git-lfs-context-support".
-	if !resourceIDRE.MatchString(parts[2]) {
-		return errors.New("invalid ModelID")
+	//
+	// But we still want to impose some basic standards, defined here.
+	apiVersion := parts[1]
+	if strings.ContainsAny(apiVersion, `:;*% $#\"',!@`) {
+		return errors.New("invalid APIVersionID")
 	}
 
 	return nil

internal/modelconfig/validation_test.go

@@ -11,6 +11,61 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/modelconfig/types"
 )
 
+// Tests for various corner cases and regressions.
+func TestValidationMethods(t *testing.T) {
+	t.Run("ValidateModelRef", func(t *testing.T) {
+		tests := []struct {
+			MRef string
+			// WantErr is the text of the expected error message.
+			// If empty, we expect no error.
+			WantErr string
+		}{
+			// Valid MRefs.
+			{"foo::bar::baz", ""},
+			{"foo-dashed::bar-dashed::baz-dashed-twice", ""},
+			{"foo.dotted::bar.dotted::baz.dotted.twice", ""},
+			{"provider_id::api_id::model_id", ""},
+
+			{"provider::api-version-can-totally/have-slashes::model", ""},
+
+			{"anthropic::2023-06-01::claude-3.5-sonnet", ""},
+
+			// Expected failure with older-style model references.
+			{"claude-2", "modelRef syntax error"},
+			{"anthropic/claude-2", "modelRef syntax error"},
+
+			// Generic validation errors.
+			{"a::b::c::d", "modelRef syntax error"},
+
+			{"provider/v1::api-version::model", "invalid ProviderID"},
+			{"CAPS_PROVIDER::v1::model", "invalid ProviderID"},
+			{"g o o g l e::v1::gemini-1.5", "invalid ProviderID"},
+
+			{"foo::name-with!exclamatnions-should::be-ok", "invalid APIVersionID"},
+			{"google::version one::gemini-1.5", "invalid APIVersionID"},
+
+			{"provider::api-version::model/v1", "invalid ModelID"},
+			{"provider::apiver::CAPS_MODEL", "invalid ModelID"},
+			{"anthropic::2023-01-01::claude instant", "invalid ModelID"},
+			{"google::v1::Gemini 1.5", "invalid ModelID"},
+		}
+
+		for _, test := range tests {
+			ref := types.ModelRef(test.MRef)
+			gotErr := validateModelRef(ref)
+
+			var gotErrText string
+			if gotErr != nil {
+				gotErrText = gotErr.Error()
+			}
+
+			assert.Equal(
+				t, test.WantErr, gotErrText,
+				"didn't get expected validation error for mref %q", test.MRef)
+		}
+	})
+}
+
 // Confirm that the model data currently in the repo is well-formed and valid.
 func TestEmbeddedModelConfig(t *testing.T) {
 	loadModelConfig := func(t *testing.T) types.ModelConfiguration {