import os import sys import datetime import json from preprocess_file import upload_file from spider.evaluation import print_scores as print_spider_scores def save_result(args, processors, gold, pred, start): # init glod list glist = [] for l in gold: if len(l.strip()) != 0: glist.append(l.strip().split('\t')[0]) # init pred list plist = [] for l in pred: if len(l.strip()) != 0: plist.append(l.strip().split('\t')[0]) result = open("result.txt", "w") temp = sys.stdout sys.stdout = result print_result(args, processors, glist, plist, start) sys.stdout = temp result.close() upload_file("result.txt", args.output) return True def print_result(args, processors, gold_list, pred_list, start): assert len(gold_list) == len(pred_list), "number of sessions must equal, len(gold_list)={}, len(pred_list)={}".format(len(gold_list), len(pred_list)) for processor in processors.get_processors(): assert len(gold_list) == processor.get_result_count(), "number of sessions must equal, len(gold_list)={}, result_count={}, result={}".format(len(gold_list), processor.get_result_count(), processor.get_processor_name()) processors.print_score_summary() print() print() print("id: {}".format(args.id)) print("name: {}".format(args.name)) print('start time: {}'.format(start.strftime('%Y-%m-%d %H:%M:%S'))) end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print('end time: {}'.format(end)) print() print() print("-------------------- scores --------------------") print() print() processors.print_score_statistic() print() print() print("-------------------- detail --------------------") print() print() for index in range(len(gold_list)): print() print() print("gold: {}".format(gold_list[index])) print("pred: {}".format(pred_list[index])) processors.print_one_score(index) def save_result_for_question_evaluation(args, processors, question_list, pred, start): result = open("result.txt", "w") temp = sys.stdout sys.stdout = result resultJson = print_result_for_question_evaluation(args, processors, question_list, pred, start) sys.stdout = temp result.close() upload_file("result.txt", args.output) with open("result.json", "w") as f: json.dump(resultJson, f) upload_file("result.json", '{}.json'.format(args.output)) return True def print_result_for_question_evaluation(args, processors, question_list, pred_list, start): assert len(question_list) == len(pred_list), "number of sessions must equal, len(question_list)={}, len(pred_list)={}".format(len(question_list), len(pred_list)) for processor in processors.get_processors(): assert len(question_list) == processor.get_result_count(), "number of sessions must equal, len(question_list)={}, result_count={}, result={}".format(len(question_list), processor.get_result_count(), processor.get_processor_name()) scores = processors.print_score_summary() print() print() print("id: {}".format(args.id)) print("name: {}".format(args.name)) print('start time: {}'.format(start.strftime('%Y-%m-%d %H:%M:%S'))) end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print('end time: {}'.format(end)) print() print() print("-------------------- detail --------------------") print() print() detail_scores = [] for index in range(len(question_list)): print() print() print("question: {}".format(question_list[index])) print("pred: {}".format(pred_list[index])) detail_score = {} detail_score["question"]=question_list[index] detail_score["sql"]=pred_list[index] detail_score["result"]=processors.print_one_score(index) detail_scores.append(detail_score) resultJson = {} resultJson["id"] = args.id resultJson["name"] = args.name resultJson["startTime"] = start.strftime('%Y-%m-%d %H:%M:%S') resultJson["endTime"] = end resultJson["scores"] = scores resultJson["detail"] = detail_scores return resultJson def ai_score(ai_result): if (ai_result is None) or (('result' in ai_result) is False): return 0 match_count = 0 total = 0 for r in ai_result['result']: total = total + 1 if "true" in r.split(',')[0].lower(): match_count = match_count + 1 return match_count / total def write_gold(gold_file, gold_list): file = open(gold_file, "w") for gold in gold_list: file.write(gold) file.write("\n") file.close def write_pred(pred_file, pred_list): file = open(pred_file, "w") for pred in pred_list: file.write(pred.replace('\n', '')) file.write('\n') file.close