import yaml import datetime from result import save_result, save_result_for_question_evaluation, write_gold, write_pred from process_result import ResultProcessors, SpiderResult, AIResult, ExecutableResult from process_evaluation import SpiderEvaluator, AIEvaluator, ExecutableEvaluator, Evaluators from preprocess_file import process_file, process_file_for_question_evaluation from aievaluation.process_file import process_benchmark from aievaluation.execution import execute as ai_execute class Commands(): @classmethod def evaluate_on_dataset(cls, args): start = datetime.datetime.now() processed_path = process_file(args.db, args.table, args.benchmark, args.model_meta, args.dataset_meta, args.template, args.enable_sqlite) benchmark = process_benchmark(processed_path["benchmark"]) # get gold list gold = benchmark["gold_list"] write_gold(processed_path["gold"], gold) gold_list_cnt = len(gold) print('======== gold list({}) begin ========'.format(gold_list_cnt)) for idx in range(gold_list_cnt): print('gold[{}] {}'.format(idx, gold[idx])) print('======== gold list({}) end ========'.format(gold_list_cnt)) with open(processed_path["dataset_meta"], 'r') as f: dataset_meta = yaml.safe_load(f) ai_pred = ai_execute(dataset_meta['need_generate_prompt'], processed_path["model_meta"], processed_path["template"], benchmark) write_pred(processed_path["pred"], ai_pred) ai_pred_list_cnt = len(ai_pred) print('======== ai pred list({}) begin ========'.format(ai_pred_list_cnt)) for idx in range(ai_pred_list_cnt): print('ai pred[{}] {}'.format(idx, ai_pred[idx])) print('======== ai pred list({}) end ========'.format(ai_pred_list_cnt)) evaluator_list = [] if dataset_meta['spider'] is True: db_config = dataset_meta['db'] spider_evaluator = SpiderEvaluator(args, db_config, processed_path) evaluator_list.append(spider_evaluator) if dataset_meta['openai'] is True: ai_evaluator = AIEvaluator(gold, ai_pred, processed_path["benchmark"]) evaluator_list.append(ai_evaluator) if dataset_meta['executable'] is True: db_config = dataset_meta['db'] executable_evaluator = ExecutableEvaluator(db_config, benchmark['db_id_list'], ai_pred) evaluator_list.append(executable_evaluator) evaluators = Evaluators(evaluator_list) try: results = evaluators.run() except Exception as e: print('evaluators.run() failed, {}'.format(e)) # result print("save dataset evaluation results") result_list = [] valid_result_types = [SpiderResult.get_processor_name(), AIResult.get_processor_name(), ExecutableResult.get_processor_name()] for result_type in valid_result_types: if result_type in results: result_list.append(results[result_type]) result_processors = ResultProcessors(result_list) assert save_result(args, result_processors, gold, ai_pred, start) is True, "save dataset evaluation file failed" @classmethod def evaluate_on_log(cls, args): pass @classmethod def evaluate_on_question(cls, args): start = datetime.datetime.now() processed_path = process_file_for_question_evaluation(args.model_meta, args.dataset_meta, args.template) with open(processed_path["dataset_meta"], 'r') as f: dataset_meta = yaml.safe_load(f) db_config = dataset_meta['db'] db_id_list = [db_config['executable_evaluation_db']] question_list = [args.question] benchmark = { "db_id_list": db_id_list, "question_list": question_list, "prompt_list": [""], } ai_pred = ai_execute(dataset_meta['need_generate_prompt'], processed_path["model_meta"], processed_path["template"], benchmark) ai_pred_list_cnt = len(ai_pred) print('======== ai pred list({}) begin ========'.format(ai_pred_list_cnt)) for idx in range(ai_pred_list_cnt): print('ai pred[{}] {}'.format(idx, ai_pred[idx])) print('======== ai pred list({}) end ========'.format(ai_pred_list_cnt)) executable_evaluator = ExecutableEvaluator(db_config, benchmark['db_id_list'], ai_pred) result = executable_evaluator.run() # result print("save question evaluation results") result_processors = ResultProcessors([result]) assert save_result_for_question_evaluation(args, result_processors, question_list, ai_pred, start) is True, "save question evaluation file failed" def register_cmds(parser): # Commands management subparsers = parser.add_subparsers(help="All evaluation execution subcommands") parser.add_argument("--evaluation_id",dest='id', type=str, default='undefined', help="evaluation id") parser.add_argument("--evaluation_name",dest='name', type=str, default='undefined', help="evaluation name") parser.add_argument("--model_meta",dest='model_meta', type=str, default='undefined', help="meta of model") parser.add_argument("--template",dest='template', type=str, default='undefined', help="template") parser.add_argument("--dataset_meta",dest='dataset_meta', type=str, default='undefined', help="meta of dataset") parser.add_argument("--output",dest='output', type=str, help="the result write to output path. support s3/native path") def help(args): parser.print_help() parser.set_defaults(func=help) sub = subparsers.add_parser("dataset") sub.add_argument('--benchmark', dest='benchmark', type=str, help="the benchmark file") sub.add_argument('--spider.table', dest='table', type=str, help="the tables.json schema file") sub.add_argument('--spider.etype', dest='etype', type=str, default='exec', help="evaluation type, exec for test suite accuracy, match for the original exact set match accuracy", choices=('all', 'exec', 'match')) sub.add_argument('--spider.plug_value', dest='plug_value', default=False, action='store_true', help='whether to plug in the gold value into the predicted query; suitable if your model does not predict values.') sub.add_argument('--spider.keep_distinct', dest='keep_distinct', default=False, action='store_true', help='whether to keep distinct keyword during evaluation. default is false.') sub.add_argument('--spider.progress_bar_for_each_datapoint', dest='progress_bar_for_each_datapoint', default=False, action='store_true', help='whether to print progress bar of running test inputs for each datapoint') sub.add_argument('--spider.enable_sqlite', dest='enable_sqlite', default=False, action='store_true', help="whether to calculate execution accuracy with sqlite or not",) sub.add_argument("--spider.db", dest='db', type=str, default='database', help="") sub.set_defaults(func=Commands.evaluate_on_dataset) sub = subparsers.add_parser("baseline") sub.add_argument("--start_date", required=True) sub.add_argument("--end_date", required=True) sub.set_defaults(func=Commands.evaluate_on_log) sub = subparsers.add_parser("question") sub.add_argument("--question", required=True) sub.set_defaults(func=Commands.evaluate_on_question)