Fixes for MIPRO: Don't fail silently on bootstrapping! (#8548)

* Fixes for MIPRO: If `datasets` wasn't installed, bootstrapping would fail silently in 3.0.0b{1/2}

* test relax

Author: okhat

dspy/primitives/module.py

@@ -169,8 +169,8 @@ def __getattribute__(self, name):
 
             if forward_called_directly:
                 logger.warning(
-                    f"Calling {self.__class__.__name__}.forward() directly is discouraged. "
-                    f"Please use {self.__class__.__name__}() instead."
+                    f"Calling module.forward(...) on {self.__class__.__name__} directly is discouraged. "
+                    f"Please use module(...) instead."
                 )
 
         return attr

dspy/propose/grounded_proposer.py

@@ -422,7 +422,7 @@ def propose_instruction_for_predictor(
 
         with dspy.settings.context(lm=self.prompt_model):
             self.prompt_model.kwargs["temperature"] = modified_temp
-            proposed_instruction = instruction_generator.forward(
+            proposed_instruction = instruction_generator(
                 demo_candidates=demo_candidates,
                 pred_i=pred_i,
                 demo_set_i=demo_set_i,

dspy/teleprompt/mipro_optimizer_v2.py

@@ -423,31 +423,33 @@ def _bootstrap_fewshot_examples(self, program: Any, trainset: list, seed: int, t
 
         zeroshot = self.max_bootstrapped_demos == 0 and self.max_labeled_demos == 0
 
-        try:
-            effective_max_errors = (
-                self.max_errors if self.max_errors is not None else dspy.settings.max_errors
-            )
+        # try:
+        effective_max_errors = (
+            self.max_errors if self.max_errors is not None else dspy.settings.max_errors
+        )
 
-            demo_candidates = create_n_fewshot_demo_sets(
-                student=program,
-                num_candidate_sets=self.num_fewshot_candidates,
-                trainset=trainset,
-                max_labeled_demos=(LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT if zeroshot else self.max_labeled_demos),
-                max_bootstrapped_demos=(
-                    BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT if zeroshot else self.max_bootstrapped_demos
-                ),
-                metric=self.metric,
-                max_errors=effective_max_errors,
-                teacher=teacher,
-                teacher_settings=self.teacher_settings,
-                seed=seed,
-                metric_threshold=self.metric_threshold,
-                rng=self.rng,
-            )
-        except Exception as e:
-            logger.info(f"Error generating few-shot examples: {e}")
-            logger.info("Running without few-shot examples.")
-            demo_candidates = None
+        demo_candidates = create_n_fewshot_demo_sets(
+            student=program,
+            num_candidate_sets=self.num_fewshot_candidates,
+            trainset=trainset,
+            max_labeled_demos=(LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT if zeroshot else self.max_labeled_demos),
+            max_bootstrapped_demos=(
+                BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT if zeroshot else self.max_bootstrapped_demos
+            ),
+            metric=self.metric,
+            max_errors=effective_max_errors,
+            teacher=teacher,
+            teacher_settings=self.teacher_settings,
+            seed=seed,
+            metric_threshold=self.metric_threshold,
+            rng=self.rng,
+        )
+        # NOTE: Bootstrapping is essential to MIPRO!
+        # Failing silently here makes the rest of the optimization far weaker as a result!
+        # except Exception as e:
+        #     logger.info(f"!!!!\n\n\n\n\nError generating few-shot examples: {e}")
+        #     logger.info("Running without few-shot examples.!!!!\n\n\n\n\n")
+        #     demo_candidates = None
 
         return demo_candidates
 

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "openai>=0.28.1",
     "datasets>=2.14.6", # needed for Bootstrap's Hasher
     "regex>=2023.10.3",
+    "datasets>=2.14.6", # needed for Bootstrap's Hasher
     "ujson>=5.8.0",
     "tqdm>=4.66.1",
     "requests>=2.31.0",

tests/primitives/test_base_module.py

@@ -501,7 +501,7 @@ def forward(self, x):
     module = TestModule()
     module.forward("test")
     captured = capsys.readouterr()
-    assert "Calling TestModule.forward() directly is discouraged" in captured.err
+    assert "directly is discouraged" in captured.err
 
 
 def test_forward_through_call_no_warning(capsys):
@@ -512,4 +512,4 @@ def forward(self, x):
     module = TestModule()
     module(x="test")
     captured = capsys.readouterr()
-    assert "Calling TestModule.forward() directly is discouraged" not in captured.err
+    assert "directly is discouraged" not in captured.err