Merge pull request #98 from LogicNet-Subnet/pre-release

Pre release to main

Author: trungthanhnguyen0502

logicnet/__init__.py

@@ -5,7 +5,7 @@
 from . import miner
 from . import utils
 
-__version__ = "1.5.4"
+__version__ = "1.5.5"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

logicnet/utils/config.py

@@ -151,14 +151,14 @@ def add_args(cls, parser):
             "--proxy.proxy_client_url",
             type=str,
             help="The url initialize credentials for proxy.",
-            default="https://logicnet.aitprotocol.ai/proxy_client",
+            default="https://proxy-client.logicnet.ai/",
         )
 
         parser.add_argument(
             "--storage.storage_url",
             type=str,
             help="The url initialize to store miner's information.",
-            default="https://logicnet.aitprotocol.ai/proxy_client/store_miner_information",
+            default="https://proxy-client.logicnet.ai/store_miner_information",
         )
 
         parser.add_argument(

logicnet/validator/challenger/challenger.py

@@ -55,63 +55,23 @@ def get_atom_logic_problem(self) -> Tuple[str, str]:
         Returns:
             (atom_logic_question, atom_logic_answer) as a tuple of strings.
         """
-        resources = ['mathgenerator', 'gsm8k', 'mmlustem']
-
-        if len(self.dataset_weight) == 3:
-            selected_resource = random.choices(resources, weights=self.dataset_weight, k=1)[0]
-        else:
-            bt.logging.warning("Invalid dataset weight configuration provided. Using default weights.")
-            selected_resource = random.choices(resources, weights=DATASET_WEIGHT, k=1)[0]
-
-        bt.logging.debug(f"Selected resource: {selected_resource}")
         try:
             # Select an atom question and answer from the Mathgenerator
-            if selected_resource == 'mathgenerator':
-                selected_topic = random.choice(topics)
-                subtopic = selected_topic["subtopic"]
-                topic = selected_topic["topic"]
-                atom_question, atom_answer = eval(f"mathgenerator.{topic}.{subtopic}()")
-                if atom_question is None or atom_answer is None:
-                    raise ValueError("Failed to get atom logic problem")
-                bt.logging.debug("Generating math problem using Mathgenerator.")
-                subtopic = subtopic.replace("_", " ").capitalize()
-                topic = topic.replace("_", " ").capitalize()
-                atom_question = atom_question.replace("$", "").strip()
-                atom_question = (
-                    f"Find the solution of this math problem:\n---\n"
-                    f"Topic: {topic}, Subtopic: {subtopic}.\n{atom_question}\n---\n"
-                )
-
-            # Select an atom question and answer from the GSM8K
-            elif selected_resource == 'gsm8k':
-                ds = load_dataset("openai/gsm8k", "main")
-                bt.logging.debug("Generating problem using GSM8K dataset.")
-                data_set = ds['train']
-                bt.logging.info(f"Loaded GSM8K dataset with {len(data_set['question'])} entries")
-                random_index = random.randint(0, len(data_set['question']) - 1)
-                question = data_set['question'][random_index]
-                answer = data_set['answer'][random_index]
-                if "####" in answer:
-                    answer = answer.split("####")[1]
-                atom_question = f"Find the solution of this question:\n---\n{question}\n---\n"
-                atom_answer = answer
-
-            # Select an atom question and answer from the MMLU-STEM
-            else:
-                ds = load_dataset("TIGER-Lab/MMLU-STEM")
-                bt.logging.debug("Generating problem using MMLU-STEM dataset.")
-                data_set = ds['test']
-                data_set = data_set.filter(lambda x: "Statement" not in x['question'])
-                bt.logging.info(f"Loaded MMLU-STEM dataset with {len(data_set['question'])} entries")
-                random_index = random.randint(0, len(data_set['question']) - 1)
-                question = data_set['question'][random_index]
-                answer_id = data_set['answer'][random_index]
-                answer_choice = data_set['choices'][random_index]
-                atom_question = f"Find the solution of this question:\n---\n{question}\n---\n"
-                atom_answer = answer_choice[answer_id]
-
+            selected_topic = random.choice(topics)
+            subtopic = selected_topic["subtopic"]
+            topic = selected_topic["topic"]
+            atom_question, atom_answer = eval(f"mathgenerator.{topic}.{subtopic}()")
+            if atom_question is None or atom_answer is None:
+                raise ValueError("Failed to get atom logic problem")
+            bt.logging.debug("Generating math problem using Mathgenerator.")
+            subtopic = subtopic.replace("_", " ").capitalize()
+            topic = topic.replace("_", " ").capitalize()
+            atom_question = atom_question.replace("$", "").strip()
+            atom_question = (
+                f"Find the solution of this math problem:\n---\n"
+                f"Topic: {topic}, Subtopic: {subtopic}.\n{atom_question}\n---\n"
+            )
         except Exception as e:
-            bt.logging.error(f"Error accessing dataset {selected_resource}: {e}. Attempting to load an alternative dataset.")
             self.retry_count += 1
             if self.retry_count > 3:
                 bt.logging.error("Max retries reached. Returning a default question and answer.")

logicnet/validator/challenger/math_generator/topics.py

@@ -72,7 +72,6 @@
     ("trig_differentiation", "calculus"),
     ("definite_integral", "calculus"),
     ("is_prime", "basic_math"),
-    ("bcd_to_decimal", "computer_science"),
     ("complex_to_polar", "misc"),
     ("set_operation", "misc"),
     ("base_conversion", "misc"),
@@ -84,13 +83,11 @@
     ("complex_quadratic", "algebra"),
     ("is_leap_year", "misc"),
     ("minutes_to_hours", "misc"),
-    ("decimal_to_bcd", "computer_science"),
     ("circumference", "geometry"),
     ("combine_like_terms", "algebra"),
     ("signum_function", "misc"),
     ("conditional_probability", "statistics"),
     ("arc_length", "geometry"),
-    # ("binomial_distribution", "misc"),
     ("stationary_points", "calculus"),
     ("expanding", "algebra"),
     ("area_of_circle", "geometry"),

logicnet/validator/rewarder.py

@@ -38,7 +38,6 @@ def __call__(self, uids, responses: list[LogicSynapse], base_synapse: LogicSynap
         """
         # Get the unique task UID from the base_synapse
         task_uid = base_synapse.task_uid
-
         valid_uids = [
             uid for uid, response in zip(uids, responses) if response.is_success
         ]
@@ -311,7 +310,7 @@ def _get_correctness_by_llm(self, question: str, ground_truth: str, response: st
     def _compare_numerical_answers(self, ground_truth: str, miner_answer: str):
         try:
             # Remove formatting characters from the answers
-            formatting_chars = ['$', '$$', '\\[', '\\]', '%']
+            formatting_chars = ['$', '$$', '\\[', '\\]', '%', 'm^2', 'm^3']
             for char in formatting_chars:
                 ground_truth = ground_truth.replace(char, '')
                 miner_answer = miner_answer.replace(char, '')
@@ -323,19 +322,26 @@ def _compare_numerical_answers(self, ground_truth: str, miner_answer: str):
             if len(gt_values) == 0:
                 return None
 
-            if len(gt_values) > 0 and len(miner_values) == 0:
-                return 0.0
+            gt_value = None
+            miner_value = None
 
             if len(gt_values) == 1 and len(miner_values) == 1:
                 # Single numerical value found in both answers
                 gt_value = gt_values[0]
                 miner_value = miner_values[0]
+            else:
+                try:
+                    if "**" not in ground_truth and "**" not in miner_answer and "^" not in ground_truth and "^" not in miner_answer:
+                        gt_value = sympy.sympify(ground_truth.strip())
+                        miner_value = sympy.sympify(miner_answer.strip())
+                except Exception as e:
+                    return None
+
+            if gt_value is not None and miner_value is not None:
                 abs_difference = abs(gt_value - miner_value)
                 epsilon = 1e-8
                 gt_abs = abs(gt_value) + epsilon
                 relative_error = abs_difference / gt_abs
-                # Logs for debugging
-                bt.logging.info(f"[CORRECTNESS DEBUG FOR NUMERICAL COMPARISON]: Absolute difference: {abs_difference}, Relative error: {relative_error}")
                 correctness_score = max(0.0, 1.0 - relative_error)
                 correctness_score = min(correctness_score, 1.0)
                 return correctness_score

neurons/validator/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.5.4"
+__version__ = "1.5.5"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

neurons/validator/validator.py

@@ -543,8 +543,7 @@ def _log_wandb(self, log):
                 "question",
                 "logic_question",
                 "ref_ground_truth",
-                "ground_truth_answer",
-                
+                "ground_truth_answer"                
             ]
         Each row in the table is for one UID, containing lists of their tasks, responses, scores, etc.
         """
@@ -570,7 +569,7 @@ def _log_wandb(self, log):
                 "question": log["question"],
                 "logic_question": log["logic_question"],
                 "ref_ground_truth": log["ref_ground_truth"],
-                "ground_truth": log["ground_truth"],
+                "ground_truth": log["ground_truth"]
             }
 
             # 3) Log to wandb