ref(grouping): Add separate test-only grouping config (#95741)

When we're testing anything having to do with grouping config upgrade or transition, it's helpful to have a config which we know will produce a different hash than the default config produces. We've been using the legacy config for that, because it doesn't parameterize messages, but it's about to disappear.

This therefore replaces its use in tests with a new config, explicitly for the purpose of generating a different hash, which is just the default config with message parameterization turned off. So that this new config doesn't count as valid in prod, it's only registered when `tests/grouping/__init__.py` is loaded (i.e., when grouping tests are run).

Author: lobsterkatie

tests/sentry/event_manager/grouping/test_assign_to_group.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from sentry.conf.server import DEFAULT_GROUPING_CONFIG, LEGACY_GROUPING_CONFIG
+from sentry.conf.server import DEFAULT_GROUPING_CONFIG
 from sentry.event_manager import _create_group
 from sentry.eventstore.models import Event
 from sentry.grouping.ingest.hashing import (
@@ -22,6 +22,7 @@
 from sentry.testutils.pytest.fixtures import django_db_all
 from sentry.testutils.pytest.mocking import capture_results
 from sentry.testutils.skips import requires_snuba
+from tests.sentry.grouping import NO_MSG_PARAM_CONFIG
 
 pytestmark = [requires_snuba]
 
@@ -265,7 +266,7 @@ def test_new_group(
         event_data=event_data,
         project=project,
         primary_config=DEFAULT_GROUPING_CONFIG,
-        secondary_config=LEGACY_GROUPING_CONFIG,
+        secondary_config=NO_MSG_PARAM_CONFIG,
         in_transition=in_transition,
     )
 
@@ -316,14 +317,14 @@ def test_existing_group_no_new_hash(
     event_data = {"message": "testing, testing, 123"}
 
     # Set the stage by creating a group with the soon-to-be-secondary hash
-    existing_event = save_event_with_grouping_config(event_data, project, LEGACY_GROUPING_CONFIG)
+    existing_event = save_event_with_grouping_config(event_data, project, NO_MSG_PARAM_CONFIG)
 
     # Now save a new, identical, event with an updated grouping config
     results = get_results_from_saving_event(
         event_data=event_data,
         project=project,
         primary_config=DEFAULT_GROUPING_CONFIG,
-        secondary_config=LEGACY_GROUPING_CONFIG,
+        secondary_config=NO_MSG_PARAM_CONFIG,
         in_transition=in_transition,
         existing_group_id=existing_event.group_id,
     )
@@ -378,13 +379,14 @@ def test_existing_group_new_hash_exists(
     project = default_project
     event_data = {"message": "testing, testing, 123"}
 
-    # Set the stage by creating a group tied to the new hash (and possibly the legacy hash as well)
+    # Set the stage by creating a group tied to the new hash (and, depending on the value of
+    # `secondary_hash_exists`, a secondary hash tied to the same group as well)
     if secondary_hash_exists:
         existing_event_with_secondary_hash = save_event_with_grouping_config(
-            event_data, project, LEGACY_GROUPING_CONFIG
+            event_data, project, NO_MSG_PARAM_CONFIG
         )
         existing_event_with_primary_hash = save_event_with_grouping_config(
-            event_data, project, DEFAULT_GROUPING_CONFIG, LEGACY_GROUPING_CONFIG, True
+            event_data, project, DEFAULT_GROUPING_CONFIG, NO_MSG_PARAM_CONFIG, True
         )
         group_id = existing_event_with_primary_hash.group_id
 
@@ -407,7 +409,7 @@ def test_existing_group_new_hash_exists(
         event_data=event_data,
         project=project,
         primary_config=DEFAULT_GROUPING_CONFIG,
-        secondary_config=LEGACY_GROUPING_CONFIG,
+        secondary_config=NO_MSG_PARAM_CONFIG,
         in_transition=in_transition,
         existing_group_id=group_id,
     )

tests/sentry/event_manager/grouping/test_grouphash_metadata.py

@@ -4,7 +4,7 @@
 from typing import Any
 from unittest.mock import ANY, MagicMock, patch
 
-from sentry.conf.server import DEFAULT_GROUPING_CONFIG, LEGACY_GROUPING_CONFIG
+from sentry.conf.server import DEFAULT_GROUPING_CONFIG
 from sentry.eventstore.models import Event
 from sentry.grouping.ingest.grouphash_metadata import create_or_update_grouphash_metadata_if_needed
 from sentry.models.grouphash import GroupHash
@@ -13,6 +13,7 @@
 from sentry.testutils.helpers.eventprocessing import save_new_event
 from sentry.testutils.helpers.options import override_options
 from sentry.testutils.skips import requires_snuba
+from tests.sentry.grouping import NO_MSG_PARAM_CONFIG
 
 pytestmark = [requires_snuba]
 
@@ -100,22 +101,23 @@ def test_stores_expected_properties(self):
     def test_stores_expected_properties_for_secondary_hashes(self):
         project = self.project
 
-        project.update_option("sentry:grouping_config", LEGACY_GROUPING_CONFIG)
+        project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
 
-        # Ensure the legacy grouphash doesn't have metadata added when it's created as the primary
-        # grouphash, so we can test how the metadata code handles it when it's a seconary grouphash
+        # Ensure the older grouphash doesn't have metadata added when it's created as the primary
+        # grouphash, so we can test how the metadata code handles it when that same grouphash comes
+        # up as a secondary grouphash
         with override_options({"grouping.grouphash_metadata.ingestion_writes_enabled": False}):
             event1 = save_new_event(
                 {"message": "Dogs are great! 1231", "platform": "python"}, self.project
             )
-            legacy_config_grouphash = GroupHash.objects.filter(
+            older_config_grouphash = GroupHash.objects.filter(
                 project=self.project, hash=event1.get_primary_hash()
             ).first()
-            assert legacy_config_grouphash and not legacy_config_grouphash.metadata
+            assert older_config_grouphash and not older_config_grouphash.metadata
 
         # Update the project's grouping config, and set it in transition mode
         project.update_option("sentry:grouping_config", DEFAULT_GROUPING_CONFIG)
-        project.update_option("sentry:secondary_grouping_config", LEGACY_GROUPING_CONFIG)
+        project.update_option("sentry:secondary_grouping_config", NO_MSG_PARAM_CONFIG)
         project.update_option("sentry:secondary_grouping_expiry", time() + 3600)
 
         event2 = save_new_event(
@@ -124,40 +126,41 @@ def test_stores_expected_properties_for_secondary_hashes(self):
         default_config_grouphash = GroupHash.objects.filter(
             project=self.project, hash=event2.get_primary_hash()
         ).first()
-        # The events should end up in the same group, but their hashes should be different, because
-        # the legacy config won't parameterize the number in the message, while the new one will
+        # The events should end up in the same group, but their hashes should be different (because
+        # NO_MSG_PARAM_CONFIG won't parameterize the number in the message, while the default config
+        # will)
         assert event1.group_id == event2.group_id
         assert (
             default_config_grouphash
-            and default_config_grouphash.hash != legacy_config_grouphash.hash
+            and default_config_grouphash.hash != older_config_grouphash.hash
         )
 
         # This time metadata was added
-        legacy_config_grouphash.refresh_from_db()
-        assert legacy_config_grouphash.metadata
+        older_config_grouphash.refresh_from_db()
+        assert older_config_grouphash.metadata
         self.assert_metadata_values(
-            legacy_config_grouphash,
+            older_config_grouphash,
             {
                 "schema_version": GROUPHASH_METADATA_SCHEMA_VERSION,
-                "latest_grouping_config": LEGACY_GROUPING_CONFIG,
+                "latest_grouping_config": NO_MSG_PARAM_CONFIG,
                 "platform": "python",
             },
         )
         # No hash basis or hashing metadata because secondary grouphashes don't come with variants
-        assert legacy_config_grouphash.metadata.hash_basis is None
-        assert legacy_config_grouphash.metadata.hashing_metadata is None
+        assert older_config_grouphash.metadata.hash_basis is None
+        assert older_config_grouphash.metadata.hashing_metadata is None
 
     @override_options({"grouping.grouphash_metadata.backfill_sample_rate": 1.0})
     @patch("sentry.grouping.ingest.grouphash_metadata.metrics.incr")
     def test_does_grouping_config_update(self, mock_metrics_incr: MagicMock):
-        self.project.update_option("sentry:grouping_config", LEGACY_GROUPING_CONFIG)
+        self.project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
 
         event1 = save_new_event({"message": "Dogs are great!"}, self.project)
         grouphash1 = GroupHash.objects.filter(
             project=self.project, hash=event1.get_primary_hash()
         ).first()
 
-        self.assert_metadata_values(grouphash1, {"latest_grouping_config": LEGACY_GROUPING_CONFIG})
+        self.assert_metadata_values(grouphash1, {"latest_grouping_config": NO_MSG_PARAM_CONFIG})
 
         # Update the grouping config. Since there's nothing to parameterize in the message, the
         # hash should be the same under both configs, meaning we'll hit the same grouphash.
@@ -177,21 +180,21 @@ def test_does_grouping_config_update(self, mock_metrics_incr: MagicMock):
             "grouping.grouphash_metadata.db_hit",
             tags={
                 "reason": "old_grouping_config",
-                "current_config": LEGACY_GROUPING_CONFIG,
+                "current_config": NO_MSG_PARAM_CONFIG,
                 "new_config": DEFAULT_GROUPING_CONFIG,
             },
         )
 
     @override_options({"grouping.grouphash_metadata.backfill_sample_rate": 0.415})
     def test_updates_obey_sample_rate(self):
-        self.project.update_option("sentry:grouping_config", LEGACY_GROUPING_CONFIG)
+        self.project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
 
         event1 = save_new_event({"message": "Dogs are great!"}, self.project)
         grouphash1 = GroupHash.objects.filter(
             project=self.project, hash=event1.get_primary_hash()
         ).first()
 
-        self.assert_metadata_values(grouphash1, {"latest_grouping_config": LEGACY_GROUPING_CONFIG})
+        self.assert_metadata_values(grouphash1, {"latest_grouping_config": NO_MSG_PARAM_CONFIG})
 
         # Update the grouping config. Since there's nothing to parameterize in the message, the
         # hash should be the same under both configs, meaning we'll hit the same grouphash.
@@ -208,9 +211,7 @@ def test_updates_obey_sample_rate(self):
             assert grouphash1 == grouphash2
 
             # Grouping config wasn't updated
-            self.assert_metadata_values(
-                grouphash2, {"latest_grouping_config": LEGACY_GROUPING_CONFIG}
-            )
+            self.assert_metadata_values(grouphash2, {"latest_grouping_config": NO_MSG_PARAM_CONFIG})
 
         # Under the sample rate cutoff, so record should be updated
         with patch("sentry.grouping.ingest.grouphash_metadata.random.random", return_value=0.1231):
@@ -288,7 +289,7 @@ def test_does_schema_update(self, mock_metrics_incr: MagicMock):
     @override_options({"grouping.grouphash_metadata.backfill_sample_rate": 1.0})
     @patch("sentry.grouping.ingest.grouphash_metadata.metrics.incr")
     def test_does_both_updates(self, mock_metrics_incr: MagicMock):
-        self.project.update_option("sentry:grouping_config", LEGACY_GROUPING_CONFIG)
+        self.project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
 
         with patch(
             "sentry.grouping.ingest.grouphash_metadata.GROUPHASH_METADATA_SCHEMA_VERSION", "11"
@@ -301,7 +302,7 @@ def test_does_both_updates(self, mock_metrics_incr: MagicMock):
             self.assert_metadata_values(
                 grouphash1,
                 {
-                    "latest_grouping_config": LEGACY_GROUPING_CONFIG,
+                    "latest_grouping_config": NO_MSG_PARAM_CONFIG,
                     "schema_version": "11",
                     "hashing_metadata": {
                         "message_source": "message",
@@ -346,7 +347,7 @@ def test_does_both_updates(self, mock_metrics_incr: MagicMock):
                 "grouping.grouphash_metadata.db_hit",
                 tags={
                     "reason": "config_and_schema",
-                    "current_config": LEGACY_GROUPING_CONFIG,
+                    "current_config": NO_MSG_PARAM_CONFIG,
                     "new_config": DEFAULT_GROUPING_CONFIG,
                     "current_version": "11",
                     "new_version": "12",

tests/sentry/event_manager/test_event_manager_grouping.py

@@ -8,11 +8,7 @@
 import pytest
 
 from sentry import audit_log
-from sentry.conf.server import (
-    DEFAULT_GROUPING_CONFIG,
-    LEGACY_GROUPING_CONFIG,
-    SENTRY_GROUPING_CONFIG_TRANSITION_DURATION,
-)
+from sentry.conf.server import DEFAULT_GROUPING_CONFIG, SENTRY_GROUPING_CONFIG_TRANSITION_DURATION
 from sentry.event_manager import _get_updated_group_title
 from sentry.eventtypes.base import DefaultEvent
 from sentry.grouping.api import get_grouping_config_dict_for_project
@@ -26,6 +22,7 @@
 from sentry.testutils.pytest.fixtures import django_db_all
 from sentry.testutils.silo import assume_test_silo_mode_of
 from sentry.testutils.skips import requires_snuba
+from tests.sentry.grouping import NO_MSG_PARAM_CONFIG
 
 pytestmark = [requires_snuba]
 
@@ -151,7 +148,7 @@ def test_auto_updates_grouping_config_even_if_config_is_gone(self):
         assert self.project.get_option("sentry:secondary_grouping_config") is None
 
     def test_auto_updates_grouping_config(self):
-        self.project.update_option("sentry:grouping_config", LEGACY_GROUPING_CONFIG)
+        self.project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
 
         save_new_event({"message": "Adopt don't shop"}, self.project)
         assert self.project.get_option("sentry:grouping_config") == DEFAULT_GROUPING_CONFIG
@@ -164,7 +161,7 @@ def test_auto_updates_grouping_config(self):
 
         assert audit_log_entry.data == {
             "sentry:grouping_config": DEFAULT_GROUPING_CONFIG,
-            "sentry:secondary_grouping_config": LEGACY_GROUPING_CONFIG,
+            "sentry:secondary_grouping_config": NO_MSG_PARAM_CONFIG,
             "sentry:secondary_grouping_expiry": ANY,  # tested separately below
             "id": self.project.id,
             "slug": self.project.slug,
@@ -440,8 +437,8 @@ def test_records_avg_calculations_per_event_metrics(self, mock_metrics_incr: Mag
         project = self.project
 
         cases: list[Any] = [
-            ["Dogs are great!", LEGACY_GROUPING_CONFIG, None, None, 1],
-            ["Adopt don't shop", DEFAULT_GROUPING_CONFIG, LEGACY_GROUPING_CONFIG, time() + 3600, 2],
+            ["Dogs are great!", NO_MSG_PARAM_CONFIG, None, None, 1],
+            ["Adopt don't shop", DEFAULT_GROUPING_CONFIG, NO_MSG_PARAM_CONFIG, time() + 3600, 2],
         ]
 
         for (
@@ -485,10 +482,10 @@ def test_adds_correct_tags_to_avg_calculations_per_event_metrics(
         project = self.project
 
         in_transition_cases: list[Any] = [
-            [LEGACY_GROUPING_CONFIG, None, None, "False"],  # Not in transition
+            [NO_MSG_PARAM_CONFIG, None, None, "False"],  # Not in transition
             [
                 DEFAULT_GROUPING_CONFIG,
-                LEGACY_GROUPING_CONFIG,
+                NO_MSG_PARAM_CONFIG,
                 time() + 3600,
                 "True",
             ],  # In transition
@@ -541,7 +538,7 @@ def test_records_hash_comparison_metric(
 ):
     project = default_project
     project.update_option("sentry:grouping_config", DEFAULT_GROUPING_CONFIG)
-    project.update_option("sentry:secondary_grouping_config", LEGACY_GROUPING_CONFIG)
+    project.update_option("sentry:secondary_grouping_config", NO_MSG_PARAM_CONFIG)
     project.update_option("sentry:secondary_grouping_expiry", time() + 3600)
 
     with mock.patch(

tests/sentry/grouping/__init__.py

@@ -22,7 +22,7 @@
 from sentry.grouping.component import BaseGroupingComponent
 from sentry.grouping.enhancer import Enhancements
 from sentry.grouping.fingerprinting import FingerprintingRules
-from sentry.grouping.strategies.configurations import CONFIGURATIONS
+from sentry.grouping.strategies.configurations import CONFIGURATIONS, register_strategy_config
 from sentry.grouping.variants import BaseVariant
 from sentry.models.project import Project
 from sentry.stacktraces.processing import normalize_stacktraces_for_grouping
@@ -32,6 +32,16 @@
 GROUPING_INPUTS_DIR = path.join(path.dirname(__file__), "grouping_inputs")
 FINGERPRINT_INPUTS_DIR = path.join(path.dirname(__file__), "fingerprint_inputs")
 
+# Create a grouping config to be used only in tests, in which message parameterization is turned
+# off. This lets us easily force an event to have different hashes for different configs. (We use a
+# purposefully old date so that it can be used as a secondary config.)
+NO_MSG_PARAM_CONFIG = "no-msg-param-tests-only:2012-12-31"
+register_strategy_config(
+    id=NO_MSG_PARAM_CONFIG,
+    base=DEFAULT_GROUPING_CONFIG,
+    initial_context={"normalize_message": False},
+)
+
 
 class GroupingInput:
     def __init__(self, inputs_dir: str, filename: str):

tests/sentry/grouping/test_benchmark.py

@@ -1,7 +1,12 @@
 import pytest
 
 from sentry.grouping.strategies.configurations import CONFIGURATIONS
-from tests.sentry.grouping import GROUPING_INPUTS_DIR, GroupingInput, get_grouping_inputs
+from tests.sentry.grouping import (
+    GROUPING_INPUTS_DIR,
+    NO_MSG_PARAM_CONFIG,
+    GroupingInput,
+    get_grouping_inputs,
+)
 
 GROUPING_INPUTS = get_grouping_inputs(GROUPING_INPUTS_DIR)
 
@@ -18,7 +23,8 @@ def benchmark_available() -> bool:
 @pytest.mark.skipif(not benchmark_available(), reason="requires pytest-benchmark")
 @pytest.mark.parametrize(
     "config_name",
-    sorted(CONFIGURATIONS.keys()),
+    # NO_MSG_PARAM_CONFIG is only used in tests, so no need to benchmark it
+    sorted(set(CONFIGURATIONS.keys()) - {NO_MSG_PARAM_CONFIG}),
     ids=lambda config_name: config_name.replace("-", "_"),
 )
 def test_benchmark_grouping(config_name, benchmark):

tests/sentry/grouping/test_grouphash_metadata.py

@@ -24,6 +24,7 @@
 from sentry.utils import json
 from tests.sentry.grouping import (
     GROUPING_INPUTS_DIR,
+    NO_MSG_PARAM_CONFIG,
     GroupingInput,
     dump_variant,
     get_snapshot_path,
@@ -38,7 +39,8 @@
 @with_grouping_inputs("grouping_input", GROUPING_INPUTS_DIR)
 @pytest.mark.parametrize(
     "config_name",
-    set(CONFIGURATIONS.keys()) - {DEFAULT_GROUPING_CONFIG},
+    # The default config is tested below, and NO_MSG_PARAM_CONFIG is only meant for use in unit tests
+    set(CONFIGURATIONS.keys()) - {DEFAULT_GROUPING_CONFIG, NO_MSG_PARAM_CONFIG},
     ids=lambda config_name: config_name.replace("-", "_"),
 )
 def test_hash_basis_with_legacy_configs(
@@ -97,7 +99,8 @@ def test_hash_basis_with_current_default_config(
 @django_db_all
 @pytest.mark.parametrize(
     "config_name",
-    CONFIGURATIONS.keys(),
+    # NO_MSG_PARAM_CONFIG is only meant for use in unit tests
+    set(CONFIGURATIONS.keys()) - {NO_MSG_PARAM_CONFIG},
     ids=lambda config_name: config_name.replace("-", "_"),
 )
 def test_unknown_hash_basis(