diff --git a/dataloop/error_data_expansion.py b/dataloop/error_data_expansion.py index 933ee92..3b43bbd 100644 --- a/dataloop/error_data_expansion.py +++ b/dataloop/error_data_expansion.py @@ -210,7 +210,7 @@ def call_generation_api(messages: List[Dict], clients: List[OpenAI], models: lis def extract_tools_from_instruction(instruction: str) -> str: """从instruction中提取工具定义""" - tools_pattern = r'\n(.*?)\n' + tools_pattern = r'\s*(.*?)\s*' tools_match = re.search(tools_pattern, instruction, re.DOTALL) if tools_match: @@ -356,8 +356,8 @@ def parse_generated_sample(api_response: str) -> Tuple[Optional[str], Optional[s """解析API生成的样本,提取INPUT和OUTPUT部分""" try: # 查找INPUT和OUTPUT部分 - input_match = re.search(r'INPUT:\s*(.*?)(?=OUTPUT:|$)', api_response, re.DOTALL) - output_match = re.search(r'OUTPUT:\s*(.*?)$', api_response, re.DOTALL) + input_match = re.search(r'INPUT:\s*(.*?)(?=\nOUTPUT:|$)', api_response, re.DOTALL) + output_match = re.search(r'\nOUTPUT:\s*(.*?)$', api_response, re.DOTALL) input_content = input_match.group(1).strip() if input_match else None output_content = output_match.group(1).strip() if output_match else None @@ -372,15 +372,52 @@ def validate_generated_sample(input_content: str, output_content: str, original_ if not input_content or not output_content: return False - # 检查INPUT是否包含对话格式 - # if "<|im_start|>" not in input_content or "<|im_end|>" not in input_content: - # return False - # 检查OUTPUT是否包含tool_call格式 if "" not in output_content: return False + + try: + # 获取允许的工具名称列表 + valid_tool_names = set() + if original_tools and original_tools.strip(): + try: + tools_data = json.loads(original_tools) + if isinstance(tools_data, list): + for tool in tools_data: + if isinstance(tool, dict) and 'name' in tool: + valid_tool_names.add(tool['name']) + except json.JSONDecodeError: + # 如果original_tools不是JSON格式,可能无法验证,这里选择放行或仅做基本检查 + # 为了安全起见,如果解析失败但有工具调用,我们无法验证名称,这可能是有风险的 + pass + + # 提取生成内容中的所有工具调用 + # + # {"name": "func", ...} + # + tool_calls = re.findall(r'\s*(.*?)\s*', output_content, re.DOTALL) - # 可以添加更多验证逻辑 + if not tool_calls: + return False + + for tool_call_str in tool_calls: + try: + tool_call = json.loads(tool_call_str) + name = tool_call.get("name") + if not name: + return False + + # 如果我们成功解析了原始工具列表,就进行名称验证 + if valid_tool_names and name not in valid_tool_names: + #print(f"Validation failed: Hallucinated tool '{name}'") + return False + except json.JSONDecodeError: + return False + + except Exception as e: + print(f"Validation error: {e}") + return False + return True def generate_single_sample(index: int, error_sample: dict, clients: List[OpenAI], models: list,