more correct interpretation of BERT tokens

pull/2/head
Kai Labusch 5 years ago
parent ba188d1daa
commit 29eb154fb8

@ -238,6 +238,9 @@ def ner(model_id=None):
continue continue
if not token.startswith('##') and word_pred == 'X':
word_pred = 'O'
token = token[2:] if token.startswith('##') else token token = token[2:] if token.startswith('##') else token
word += token word += token

Loading…
Cancel
Save