fix NER output; fix BERT Tokenizer

pull/2/head
Kai Labusch 5 years ago
parent 9bf2e6f51b
commit 3eabe5054a

@ -271,7 +271,7 @@ def ner(model_id):
if token == '[UNK]': if token == '[UNK]':
orig_pos = len("".join([pred['word'] for pred in output_sentence])) orig_pos = len("".join([pred['word'] for pred in output_sentence]))
output_sentence.append({'word': original_text[orig_pos], 'prediction': 'O'}) output_sentence.append({'word': original_text[orig_pos], 'prediction': last_prediction})
continue continue
token = token[2:] if token.startswith('##') else token token = token[2:] if token.startswith('##') else token

Loading…
Cancel
Save