import abc from typing import Any from result import ai_score from spider.evaluation import print_scores as print_spider_scores from executable_evaluation.evaluation import get_result_score_name from executable_evaluation.evaluation import MODEL_RESULT_SQL_EXEC_SUCCESS class ResultProcessor(metaclass=abc.ABCMeta): @abc.abstractmethod def print_score_summary(self): pass @abc.abstractmethod def print_score_statistic(self): pass @abc.abstractmethod def print_one_score(self, index: int): pass @abc.abstractmethod def get_result_count(self): pass @abc.abstractstaticmethod def get_processor_name(): pass class ResultProcessors(): def __init__(self, processors) -> None: self.processors = processors def print_score_summary(self): scores = {} for processor in self.processors: score = processor.print_score_summary() scores[processor.get_processor_name()] = score return scores def print_score_statistic(self): for processor in self.processors: processor.print_score_statistic() def print_one_score(self, index: int): scores = {} for processor in self.processors: score = processor.print_one_score(index) scores[processor.get_processor_name()] = score return scores def get_processors(self): return self.processors class AIResult(): def __init__(self, result: Any) -> None: self.result = result def print_score_summary(self): score = ai_score(self.result) print("ai_score: {}".format(str(score))) return score def print_score_statistic(self): return def is_result_valid(self): return (self.result is not None) and ('result' in self.result) def print_one_score(self, index: int): if self.is_result_valid() and (index < len(self.result['result'])): print("openai: {}".format(self.result['result'][index])) else: print("openai: unknown") return self.result['result'][index] def get_result_count(self): if self.is_result_valid(): return len(self.result['result']) else: return 0 @staticmethod def get_processor_name(): return "ai_result" class SpiderResult(): def __init__(self, result: Any, evaluation_type: str) -> None: self.result = result self.evaluation_type = evaluation_type def print_score_summary(self): print("execution_match_score: {}".format(str(self.result['scores']['all']['exec']))) return self.result['scores']['all']['exec'] def print_score_statistic(self): print_spider_scores(self.result['scores'], self.evaluation_type, False) def print_one_score(self, index: int): print("spider_exec: {}".format(str(self.result["exec_scores_list"][index]))) return self.result["exec_scores_list"][index] def get_result_count(self): return len(self.result["exec_scores_list"]) @staticmethod def get_processor_name(): return "spider_result" class ExecutableResult(): def __init__(self, result: Any) -> None: self.result = result def print_score_summary(self): result_score = self.result[get_result_score_name(MODEL_RESULT_SQL_EXEC_SUCCESS)] print("executable_score: {}".format(str(result_score))) return result_score def print_score_statistic(self): return def print_one_score(self, index: int): result = self.result['result'][index][0] print("executable result: {}".format(result)) data = self.result['result'][index][1] if result != MODEL_RESULT_SQL_EXEC_SUCCESS: print("executable error: {}".format(data)) return print("executable data:") for row in data: print(row) return {"executable": result, "data":data} def get_result_count(self): return len(self.result['result']) @staticmethod def get_processor_name(): return "executable_result"