from genericpath import exists import os import torch.nn as nn import torch import logging from tqdm import tqdm, trange import timeit import collections import json import math from bs4 import BeautifulSoup from copy import deepcopy import string import re from torch.utils.tensorboard import SummaryWriter from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler) from transformers import ( BasicTokenizer, ) from transformers import ( AdamW, get_linear_schedule_with_warmup, ) def reorganize_batch_web(args, batch_web): dic = {} dic['input_ids'] = batch_web[0].cuda() dic['attention_mask'] = batch_web[1].cuda() dic['token_type_ids'] = batch_web[2].cuda() dic['xpath_tags_seq'] = batch_web[3].cuda() dic['xpath_subs_seq'] = batch_web[4].cuda() dic['start_positions'] = batch_web[5].cuda() dic['end_positions'] = batch_web[6].cuda() if 'box' in args.embedding_mode: dic['bbox'] = batch_web[7].cuda() # new added dic['embedding_mode'] = args.embedding_mode return dic def train(args, dataset_web, model, tokenizer): # torch.cuda.set_device(args.local_rank) # Log when executing on clusters try: from azureml.core.run import Run aml_run = Run.get_context() except: aml_run = None # Open tensorboard writer = SummaryWriter(f'{args.output_dir}/output/{args.exp_name}') # Count batch gpu_nums = torch.cuda.device_count() batch = args.batch_per_gpu * gpu_nums dataloader_web = DataLoader( dataset_web, batch_size=batch, num_workers=args.num_workers, pin_memory=False, shuffle=True, ) # Get warmup steps total_step = args.epoch * len(dataloader_web) warmup_steps = int(args.warmup_ratio * total_step) # Prepare optimizers no_decay = ["bias", "LayerNorm.weight"] optimizer_grouped_parameters = [ { "params": [ p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) ], "weight_decay": args.weight_decay, }, { "params": [ p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) ], "weight_decay": 0.0, }, ] optimizer = AdamW( optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon ) scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=warmup_steps, num_training_steps=total_step ) # Transfer the parameters to cuda model = model.cuda() # Prepare fp16 if args.fp16: try: from apex import amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) model, optimizer = amp.initialize( model, optimizer, opt_level=args.fp16_opt_level ) logging.info('Successfully load fp16 mode') # Parallel or Distribute if gpu_nums > 1: model = torch.nn.DataParallel(model) # Record some training info logging.info("***** Running training *****") # logging.info(" Num examples in dataset_doc = %d", len(dataset_doc)) logging.info(" Num examples in dataset_web = %d", len(dataset_web)) # logging.info(" Num steps for each epoch for doc = %d", len(dataloader_doc)) logging.info(" Num steps for each epoch for web = %d", len(dataloader_web)) logging.info(" Num Epochs = %d", args.epoch) logging.info( " Instantaneous batch size per GPU = %d", args.batch_per_gpu ) logging.info(" Total optimization steps = %d", total_step) # Start training model.zero_grad() train_iterator = trange( 0, int(args.epoch), desc="Epoch", ) global_step = 0 for now_epoch, _ in enumerate(tqdm(train_iterator, desc="Iteration")): # tqdm for epoch # epoch_iterator_doc = iter(dataloader_doc) epoch_iterator_web = iter(dataloader_web) min_step = len(epoch_iterator_web) for now_step in tqdm(range(min_step), desc="Iteration"): # tqdm for step # batch_doc = epoch_iterator_doc.next() batch_web = epoch_iterator_web.next() batch_web = reorganize_batch_web(args, batch_web) model.train() # loss_doc = model(**batch_doc)[0] loss_web = model(**batch_web)[0] loss = loss_web if gpu_nums > 1: loss = loss.mean() # loss_doc = loss_doc.mean() loss_web = loss_web.mean() if args.fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() if args.fp16: torch.nn.utils.clip_grad_norm_( amp.master_params(optimizer), args.max_grad_norm ) else: torch.nn.utils.clip_grad_norm_( model.parameters(), args.max_grad_norm ) if global_step % args.accumulation == 0: optimizer.step() model.zero_grad() scheduler.step() global_step += 1 if global_step % args.log_step == 0: logging.info(f'epoch: {now_epoch} | step: {now_step+1} | total_step: {global_step} | loss: {loss} | lr: {scheduler.get_lr()[0]}') writer.add_scalar('loss', loss, global_step//args.log_step) # writer.add_scalar('loss_doc', loss_doc, global_step//args.log_step) writer.add_scalar('loss_web', loss_web, global_step//args.log_step) writer.add_scalar('lr', scheduler.get_lr()[0], global_step//args.log_step) if aml_run is not None: aml_run.log('loss', loss.item()) # aml_run.log('loss_doc', loss_doc.item()) aml_run.log('loss_web', loss_web.item()) aml_run.log('lr', scheduler.get_lr()[0]) if global_step % args.save_step == 0: # Save model checkpoint output_dir = os.path.join(args.output_dir, 'output', args.exp_name, f'step-{global_step}') os.makedirs(output_dir, exist_ok=True) model_to_save = ( model.module if hasattr(model, "module") else model ) # Take care of distributed/parallel training model_to_save.save_pretrained(output_dir) tokenizer.save_pretrained(output_dir) torch.save(args, os.path.join(output_dir, "training_args.bin")) logging.info("Saving model checkpoint to %s", output_dir) torch.save( optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt") ) torch.save( scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt") ) logging.info( "Saving optimizer and scheduler states to %s", output_dir ) if global_step % 1000 == 0: # eval print('Start eval!') from data.datasets.websrc import get_websrc_dataset dataset_web, examples, features = get_websrc_dataset(args, tokenizer, evaluate=True, output_examples=True) evaluate(args, dataset_web, examples, features, model, tokenizer, global_step) RawResult = collections.namedtuple("RawResult", ["unique_id", "start_logits", "end_logits"]) def to_list(tensor): return tensor.detach().cpu().tolist() def _get_best_indexes(logits, n_best_size): """Get the n-best logits from a list.""" index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) best_indexes = [] for i in range(len(index_and_score)): if i >= n_best_size: break best_indexes.append(index_and_score[i][0]) return best_indexes def _get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): def _strip_spaces(text): ns_chars = [] ns_to_s_map = collections.OrderedDict() for (i, c) in enumerate(text): if c == " ": continue ns_to_s_map[len(ns_chars)] = i ns_chars.append(c) ns_text = "".join(ns_chars) return ns_text, ns_to_s_map # We first tokenize `orig_text`, strip whitespace from the result # and `pred_text`, and check if they are the same length. If they are # NOT the same length, the heuristic has failed. If they are the same # length, we assume the characters are one-to-one aligned. tokenizer = BasicTokenizer(do_lower_case=do_lower_case) tok_text = " ".join(tokenizer.tokenize(orig_text)) start_position = tok_text.find(pred_text) if start_position == -1: # if verbose_logging: # logging.info( # "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) return orig_text end_position = start_position + len(pred_text) - 1 (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) if len(orig_ns_text) != len(tok_ns_text): if verbose_logging: logging.info("Length not equal after stripping spaces: '%s' vs '%s'", orig_ns_text, tok_ns_text) return orig_text # We then project the characters in `pred_text` back to `orig_text` using # the character-to-character alignment. tok_s_to_ns_map = {} for (i, tok_index) in tok_ns_to_s_map.items(): tok_s_to_ns_map[tok_index] = i orig_start_position = None if start_position in tok_s_to_ns_map: ns_start_position = tok_s_to_ns_map[start_position] if ns_start_position in orig_ns_to_s_map: orig_start_position = orig_ns_to_s_map[ns_start_position] if orig_start_position is None: if verbose_logging: logging.info("Couldn't map start position") return orig_text orig_end_position = None if end_position in tok_s_to_ns_map: ns_end_position = tok_s_to_ns_map[end_position] if ns_end_position in orig_ns_to_s_map: orig_end_position = orig_ns_to_s_map[ns_end_position] if orig_end_position is None: if verbose_logging: logging.info("Couldn't map end position") return orig_text output_text = orig_text[orig_start_position:(orig_end_position + 1)] return output_text def _compute_softmax(scores): """Compute softmax probability over raw logits.""" if not scores: return [] max_score = None for score in scores: if max_score is None or score > max_score: max_score = score exp_scores = [] total_sum = 0.0 for score in scores: x = math.exp(score - max_score) exp_scores.append(x) total_sum += x probs = [] for score in exp_scores: probs.append(score / total_sum) return probs class EvalOpts: r""" The options which the matrix evaluation process needs. Arguments: data_file (str): the SQuAD-style json file of the dataset in evaluation. root_dir (str): the root directory of the raw WebSRC dataset, which contains the HTML files. pred_file (str): the prediction file which contain the best predicted answer text of each question from the model. tag_pred_file (str): the prediction file which contain the best predicted answer tag id of each question from the model. result_file (str): the file to write down the matrix evaluation results of each question. out_file (str): the file to write down the final matrix evaluation results of the whole dataset. """ def __init__(self, data_file, root_dir, pred_file, tag_pred_file, result_file='', out_file=""): self.data_file = data_file self.root_dir = root_dir self.pred_file = pred_file self.tag_pred_file = tag_pred_file self.result_file = result_file self.out_file = out_file def write_predictions(all_examples, all_features, all_results, n_best_size, max_answer_length, do_lower_case, output_prediction_file, output_tag_prediction_file, output_nbest_file, verbose_logging, tokenizer): r""" Compute and write down the final results, including the n best results. Arguments: all_examples (list[SRCExample]): all the SRC Example of the dataset; note that we only need it to provide the mapping from example index to the question-answers id. all_features (list[InputFeatures]): all the features for the input doc spans. all_results (list[RawResult]): all the results from the models. n_best_size (int): the number of the n best buffer and the final n best result saved. max_answer_length (int): constrain the model to predict the answer no longer than it. do_lower_case (bool): whether the model distinguish upper and lower case of the letters. output_prediction_file (str): the file which the best answer text predictions will be written to. output_tag_prediction_file (str): the file which the best answer tag predictions will be written to. output_nbest_file (str): the file which the n best answer predictions including text, tag, and probabilities will be written to. verbose_logging (bool): if true, all of the warnings related to data processing will be printed. """ logging.info("Writing predictions to: %s" % output_prediction_file) logging.info("Writing nbest to: %s" % output_nbest_file) example_index_to_features = collections.defaultdict(list) for feature in all_features: example_index_to_features[feature.example_index].append(feature) unique_id_to_result = {} for result in all_results: unique_id_to_result[result.unique_id] = result _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name "PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit", "tag_ids"]) all_predictions = collections.OrderedDict() all_tag_predictions = collections.OrderedDict() all_nbest_json = collections.OrderedDict() for (example_index, example) in enumerate(all_examples): features = example_index_to_features[example_index] prelim_predictions = [] for (feature_index, feature) in enumerate(features): result = unique_id_to_result[feature.unique_id] start_indexes = _get_best_indexes(result.start_logits, n_best_size) end_indexes = _get_best_indexes(result.end_logits, n_best_size) # if we could have irrelevant answers, get the min score of irrelevant for start_index in start_indexes: for end_index in end_indexes: # We could hypothetically create invalid predictions, e.g., predict # that the start of the span is in the question. We throw out all # invalid predictions. if start_index >= len(feature.tokens): continue if end_index >= len(feature.tokens): continue if start_index not in feature.token_to_orig_map: continue if end_index not in feature.token_to_orig_map: continue if not feature.token_is_max_context.get(start_index, False): continue if end_index < start_index: continue length = end_index - start_index + 1 if length > max_answer_length: continue tag_ids = set(feature.token_to_tag_index[start_index: end_index + 1]) prelim_predictions.append( _PrelimPrediction( feature_index=feature_index, start_index=start_index, end_index=end_index, start_logit=result.start_logits[start_index], end_logit=result.end_logits[end_index], tag_ids=list(tag_ids))) prelim_predictions = sorted( prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True) _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name "NbestPrediction", ["text", "start_logit", "end_logit", "tag_ids"]) seen_predictions = {} nbest = [] for pred in prelim_predictions: if len(nbest) >= n_best_size: break feature = features[pred.feature_index] if pred.start_index > 0: # this is a non-null prediction tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] orig_doc_start = feature.token_to_orig_map[pred.start_index] orig_doc_end = feature.token_to_orig_map[pred.end_index] orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] tok_text = " ".join(tok_tokens) # De-tokenize WordPieces that have been split off. tok_text = tok_text.replace(" ##", "") tok_text = tok_text.replace("##", "") # Clean whitespace tok_text = tok_text.strip() tok_text = " ".join(tok_text.split()) orig_text = " ".join(orig_tokens) final_text = _get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) if final_text in seen_predictions: continue seen_predictions[final_text] = True else: final_text = "" seen_predictions[final_text] = True nbest.append( _NbestPrediction( text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit, tag_ids=pred.tag_ids)) # In very rare edge cases we could have no valid predictions. So we # just create a nonce prediction in this case to avoid failure. if not nbest: nbest.append( _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0, tag_ids=[-1])) assert len(nbest) >= 1 total_scores = [] best_non_null_entry = None for entry in nbest: total_scores.append(entry.start_logit + entry.end_logit) if not best_non_null_entry: if entry.text: best_non_null_entry = entry probs = _compute_softmax(total_scores) nbest_json = [] for (i, entry) in enumerate(nbest): output = collections.OrderedDict() output["text"] = entry.text output["probability"] = probs[i] output["start_logit"] = entry.start_logit output["end_logit"] = entry.end_logit output["tag_ids"] = entry.tag_ids nbest_json.append(output) assert len(nbest_json) >= 1 best = nbest_json[0]["text"].split() best = ' '.join([w for w in best if (w[0] != '<' or w[-1] != '>') and w != "" and w != tokenizer.sep_token and w != tokenizer.cls_token]) all_predictions[example.qas_id] = best all_tag_predictions[example.qas_id] = nbest_json[0]["tag_ids"] all_nbest_json[example.qas_id] = nbest_json with open(output_prediction_file, "w+") as writer: writer.write(json.dumps(all_predictions, indent=4) + "\n") with open(output_nbest_file, "w+") as writer: writer.write(json.dumps(all_nbest_json, indent=4) + "\n") with open(output_tag_prediction_file, 'w+') as writer: writer.write(json.dumps(all_tag_predictions, indent=4) + '\n') return def make_qid_to_has_ans(dataset): r""" Pick all the questions which has answer in the dataset and return the list. """ qid_to_has_ans = {} for domain in dataset: for w in domain['websites']: for qa in w['qas']: qid_to_has_ans[qa['id']] = bool(qa['answers']) return qid_to_has_ans def normalize_answer(s): """Lower text and remove punctuation, articles and extra whitespace.""" def remove_articles(text): regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) return re.sub(regex, ' ', text) def white_space_fix(text): return ' '.join(text.split()) def remove_punc(text): exclude = set(string.punctuation) return ''.join(ch for ch in text if ch not in exclude) def lower(text): return text.lower() return white_space_fix(remove_articles(remove_punc(lower(s)))) def compute_exact(a_gold, a_pred): r""" Calculate the exact match. """ if normalize_answer(a_gold) == normalize_answer(a_pred): return 1 return 0 def get_raw_scores(dataset, preds, tag_preds, root_dir): r""" Calculate all the three matrix (exact match, f1, POS) for each question. Arguments: dataset (dict): the dataset in use. preds (dict): the answer text prediction for each question in the dataset. tag_preds (dict): the answer tags prediction for each question in the dataset. root_dir (str): the base directory for the html files. Returns: tuple(dict, dict, dict): exact match, f1, pos scores for each question. """ exact_scores = {} f1_scores = {} pos_scores = {} for websites in dataset: for w in websites['websites']: f = os.path.join(root_dir, websites['domain'], w['page_id'][0:2], 'processed_data', w['page_id'] + '.html') for qa in w['qas']: qid = qa['id'] gold_answers = [a['text'] for a in qa['answers'] if normalize_answer(a['text'])] gold_tag_answers = [a['element_id'] for a in qa['answers']] additional_tag_information = [a['answer_start'] for a in qa['answers']] if not gold_answers: # For unanswerable questions, only correct answer is empty string gold_answers = [''] if qid not in preds: print('Missing prediction for %s' % qid) continue a_pred, t_pred = preds[qid], tag_preds[qid] # Take max over all gold answers exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) pos_scores[qid] = max(compute_pos(f, t, a, t_pred) for t, a in zip(gold_tag_answers, additional_tag_information)) return exact_scores, f1_scores, pos_scores def get_tokens(s): r""" Get the word list in the input. """ if not s: return [] return normalize_answer(s).split() def compute_f1(a_gold, a_pred): r""" Calculate the f1 score. """ gold_toks = get_tokens(a_gold) pred_toks = get_tokens(a_pred) common = collections.Counter(gold_toks) & collections.Counter(pred_toks) num_same = sum(common.values()) if len(gold_toks) == 0 or len(pred_toks) == 0: # If either is no-answer, then F1 is 1 if they agree, 0 otherwise return int(gold_toks == pred_toks) if num_same == 0: return 0 precision = 1.0 * num_same / len(pred_toks) recall = 1.0 * num_same / len(gold_toks) f1 = (2 * precision * recall) / (precision + recall) return f1 def compute_pos(f, t_gold, addition, t_pred): r""" Calculate the POS score. Arguments: f (str): the html file on which the question is based. t_gold (int): the gold answer tag id provided by the dataset (the value correspond to the key element_id). addition (int): the addition information used for yes/no question provided by the dataset (the value corresponding to the key answer_start). t_pred (list[int]): the tag ids of the tags corresponding the each word in the predicted answer. Returns: float: the POS score. """ h = BeautifulSoup(open(f), "lxml") p_gold, e_gold = set(), h.find(tid=t_gold) if e_gold is None: if len(t_pred) != 1: return 0 else: t = t_pred[0] e_pred, e_prev = h.find(tid=t), h.find(tid=t-1) if (e_pred is not None) or (addition == 1 and e_prev is not None) or\ (addition == 0 and e_prev is None): return 0 else: return 1 else: p_gold.add(e_gold['tid']) for e in e_gold.parents: if int(e['tid']) < 2: break p_gold.add(e['tid']) p = None for t in t_pred: p_pred, e_pred = set(), h.find(tid=t) if e_pred is not None: p_pred.add(e_pred['tid']) if e_pred.name != 'html': for e in e_pred.parents: if int(e['tid']) < 2: break p_pred.add(e['tid']) else: p_pred.add(str(t)) if p is None: p = p_pred else: p = p & p_pred return len(p_gold & p) / len(p_gold | p) def make_pages_list(dataset): r""" Record all the pages which appears in the dataset and return the list. """ pages_list = [] last_page = None for domain in dataset: for w in domain['websites']: for qa in w['qas']: if last_page != qa['id'][:4]: last_page = qa['id'][:4] pages_list.append(last_page) return pages_list def make_eval_dict(exact_scores, f1_scores, pos_scores, qid_list=None): r""" Make the dictionary to show the evaluation results. """ if qid_list is None: total = len(exact_scores) return collections.OrderedDict([ ('exact', 100.0 * sum(exact_scores.values()) / total), ('f1', 100.0 * sum(f1_scores.values()) / total), ('pos', 100.0 * sum(pos_scores.values()) / total), ('total', total), ]) else: total = len(qid_list) if total == 0: return collections.OrderedDict([ ('exact', 0), ('f1', 0), ('pos', 0), ('total', 0), ]) return collections.OrderedDict([ ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), ('pos', 100.0 * sum(pos_scores[k] for k in qid_list) / total), ('total', total), ]) def merge_eval(main_eval, new_eval, prefix): for k in new_eval: main_eval['%s_%s' % (prefix, k)] = new_eval[k] def evaluate_on_squad(opts): with open(opts.data_file) as f: dataset_json = json.load(f) dataset = dataset_json['data'] if isinstance(opts.pred_file, str): with open(opts.pred_file) as f: preds = json.load(f) else: preds = opts.pred_file if isinstance(opts.tag_pred_file, str): with open(opts.tag_pred_file) as f: tag_preds = json.load(f) else: tag_preds = opts.tag_pred_file qid_to_has_ans = make_qid_to_has_ans(dataset) has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] exact, f1, pos = get_raw_scores(dataset, preds, tag_preds, opts.root_dir) out_eval = make_eval_dict(exact, f1, pos) if has_ans_qids: has_ans_eval = make_eval_dict(exact, f1, pos, qid_list=has_ans_qids) merge_eval(out_eval, has_ans_eval, 'HasAns') if no_ans_qids: no_ans_eval = make_eval_dict(exact, f1, pos, qid_list=no_ans_qids) merge_eval(out_eval, no_ans_eval, 'NoAns') print(json.dumps(out_eval, indent=2)) pages_list, write_eval = make_pages_list(dataset), deepcopy(out_eval) for p in pages_list: pages_ans_qids = [k for k, _ in qid_to_has_ans.items() if p in k] page_eval = make_eval_dict(exact, f1, pos, qid_list=pages_ans_qids) merge_eval(write_eval, page_eval, p) if opts.result_file: with open(opts.result_file, 'w') as f: w = {} for k, v in qid_to_has_ans.items(): w[k] = {'exact': exact[k], 'f1': f1[k], 'pos': pos[k]} json.dump(w, f) if opts.out_file: with open(opts.out_file, 'w') as f: json.dump(write_eval, f) print('****** result ******') print(out_eval) return out_eval def evaluate(args, dataset_web, examples, features, model, tokenizer, step=0): gpu_nums = torch.cuda.device_count() batch = args.batch_per_gpu * gpu_nums eval_sampler = SequentialSampler(dataset_web) eval_dataloader = DataLoader(dataset_web, sampler=eval_sampler, batch_size=batch, num_workers=8) # Eval! logging.info("***** Running evaluation *****") logging.info(" Num examples = %d", len(dataset_web)) logging.info(" Batch size = %d", batch) model = model.cuda() all_results = [] start_time = timeit.default_timer() for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.cuda() for t in batch) with torch.no_grad(): inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'token_type_ids': batch[2], 'xpath_tags_seq': batch[4], 'xpath_subs_seq': batch[5], } feature_indices = batch[3] outputs = model(**inputs) for i, feature_index in enumerate(feature_indices): eval_feature = features[feature_index.item()] unique_id = int(eval_feature.unique_id) result = RawResult(unique_id=unique_id, start_logits=to_list(outputs[0][i]), end_logits=to_list(outputs[1][i])) all_results.append(result) eval_time = timeit.default_timer() - start_time logging.info(" Evaluation done in total %f secs (%f sec per example)", eval_time, eval_time / len(dataset_web)) # Compute predictions # output_dir = os.path.join(args.output_dir, 'output', args.exp_name, f'step-{global_step}') output_prediction_file = os.path.join(args.output_dir,"output", args.exp_name, f"predictions_{step}.json") output_tag_prediction_file = os.path.join(args.output_dir,"output", args.exp_name, f"tag_predictions_{step}.json") output_nbest_file = os.path.join(args.output_dir,"output", args.exp_name, f"nbest_predictions_{step}.json") output_result_file = os.path.join(args.output_dir,"output", args.exp_name, f"qas_eval_results_{step}.json") output_file = os.path.join(args.output_dir,"output", args.exp_name, f"eval_matrix_results_{step}") write_predictions(examples, features, all_results, args.n_best_size, args.max_answer_length, args.do_lower_case, output_prediction_file, output_tag_prediction_file, output_nbest_file, args.verbose_logging, tokenizer) # Evaluate evaluate_options = EvalOpts(data_file=args.web_eval_file, root_dir=args.root_dir, pred_file=output_prediction_file, tag_pred_file=output_tag_prediction_file, result_file=output_result_file, out_file=output_file) results = evaluate_on_squad(evaluate_options) return results