import abc import sys import threading from typing import Any from aievaluation.evaluation import evaluate as ai_evaluate from spider.evaluation import evaluate as spider_evaluate, build_foreign_key_map_from_json from executable_evaluation.evaluation import evaluate as executable_evaluate from process_result import AIResult, SpiderResult, ExecutableResult from utils import is_file, try_get_default_ssl_ca_path from spider.tidb import TiDB class Evaluator(metaclass=abc.ABCMeta): @abc.abstractmethod def run(self): pass @abc.abstractstaticmethod def get_evaluator_name(): pass # threadLock = threading.Lock() # # def loop(evaluator, results): # result = evaluator.run() # threadLock.acquire() # results[result.get_processor_name()] = result # threadLock.release() class Evaluators(): def __init__(self, evaluators) -> None: self.evaluators = evaluators def run(self): results = {} # threads=[] for i in range(len(self.evaluators)): result = self.evaluators[i].run() results[result.get_processor_name()] = result # t=threading.Thread(target=loop,args=(self.evaluators[i], results)) # threads.append(t) # # for t in threads: # t.start() # for t in threads: # t.join() print("all evaluators finished",flush=True) return results def get_evaluators(self): return self.evaluators class AIEvaluator(): def __init__(self, gold, ai_pred, benchmark_path) -> None: self.gold = gold self.ai_pred = ai_pred self.benchmark_path = benchmark_path def run(self): print("start ai evaluation") ai_result = ai_evaluate(self.gold, self.ai_pred, self.benchmark_path) if (ai_result is not None) and ('result' in ai_result): ai_result_cnt = len(ai_result['result']) print('======== ai evaluation result list({}) begin ========'.format(ai_result_cnt)) for idx in range(ai_result_cnt): print('ai result[{}] {}'.format(idx, ai_result['result'][idx])) print('======== ai evaluation result list({}) end ========'.format(ai_result_cnt)) ai_result_processor = AIResult(ai_result) print("end ai evaluation",flush=True) return ai_result_processor @staticmethod def get_evaluator_name(): return "ai_evaluator" def get_tidb_accessor(db_config): print('configured ssl ca path is {}'.format(db_config['ssl_ca_path'])) ssl_ca_path = db_config['ssl_ca_path'] if ssl_ca_path == '': ssl_ca_path = try_get_default_ssl_ca_path() assert ssl_ca_path != '' and is_file(ssl_ca_path) is True, 'ssl ca path {} for db connection is invalid'.format(ssl_ca_path) print('applied ssl ca path is {}'.format(ssl_ca_path)) db_accessor = TiDB({'host': db_config['host'], 'port': db_config['port'], 'user': db_config['user'], 'passwd': db_config['password'], 'ssl_ca': ssl_ca_path}) return db_accessor class SpiderEvaluator(): def __init__(self, args: Any, db_config, processed_path) -> None: self.args = args self.db_config = db_config self.processed_path = processed_path def run(self): # only evaluting exact match needs this argument kmaps = None if self.args.etype in ['all', 'match']: assert self.args.table is not None, 'table argument must be non-None if exact set match is evaluated' kmaps = build_foreign_key_map_from_json(self.processed_path["table"]) db_accessor = get_tidb_accessor(self.db_config) print("start spider evaluation") spider_result = spider_evaluate(db_accessor, self.processed_path["gold"], self.processed_path["pred"], self.args.etype, kmaps, self.args.plug_value, self.args.keep_distinct, self.args.progress_bar_for_each_datapoint) spider_result_processor = SpiderResult(spider_result, self.args.etype) print("end spider evaluation",flush=True) db_accessor.cursor.close() db_accessor.conn.close() return spider_result_processor @staticmethod def get_evaluator_name(): return "spider_evaluator" class ExecutableEvaluator(): def __init__(self, db_config, db_id_list, pred_list) -> None: self.db_config = db_config self.db_id_list = db_id_list self.pred_list = pred_list self.db_accessor = get_tidb_accessor(self.db_config) def run(self): print("start executable evaluation") result = executable_evaluate(self.db_accessor, self.db_id_list, self.pred_list) result_processor = ExecutableResult(result) print("end executable evaluation",flush=True) return result_processor @staticmethod def get_evaluator_name(): return "executable_evaluator"