qurator-spk/sbb_ner
1
0
Fork 0
mirror of https://github.com/qurator-spk/sbb_ner.git synced 2025-10-31 09:34:12 +01:00
Code Issues Releases Wiki Activity

implement demo interface;fix eval bug in ner webapp; beautify textline_rec code

Browse source
This commit is contained in:
Kai Labusch 2019-08-22 13:57:32 +02:00
parent 3c12e5538a
commit 9560f70580
7 changed files with 168 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

2
qurator/sbb_ner/models/bert.py
View file

@ -385,6 +385,8 @@ def model_predict(dataloader, device, label_map, model):
temp_2.pop() # skip last token since its [SEP] temp_2.pop() # skip last token since its [SEP]
y_pred.append(temp_2) y_pred.append(temp_2)
break break
else:
y_pred.append(temp_2)
return y_pred return y_pred

120
qurator/sbb_ner/webapp/app.py
View file

@ -93,12 +93,16 @@ class NERPredictor:
features = [convert_examples_to_features(ex, self._label_to_id, self._max_seq_length, self._bert_tokenizer) features = [convert_examples_to_features(ex, self._label_to_id, self._max_seq_length, self._bert_tokenizer)
for ex in examples] for ex in examples]
assert len(sentences) == len(features)
data_loader = NerProcessor.make_data_loader(None, self._batch_size, self._local_rank, self._label_to_id, data_loader = NerProcessor.make_data_loader(None, self._batch_size, self._local_rank, self._label_to_id,
self._max_seq_length, self._bert_tokenizer, features=features, self._max_seq_length, self._bert_tokenizer, features=features,
sequential=True) sequential=True)
prediction_tmp = model_predict(data_loader, self._device, self._label_map, self._model) prediction_tmp = model_predict(data_loader, self._device, self._label_map, self._model)
assert len(sentences) == len(prediction_tmp)
prediction = [] prediction = []
for fe, pr in zip(features, prediction_tmp): for fe, pr in zip(features, prediction_tmp):
prediction.append((fe.tokens[1:-1], pr)) prediction.append((fe.tokens[1:-1], pr))
@ -185,125 +189,47 @@ def fulltext(ppn):
if row_data.text is None: if row_data.text is None:
continue continue
text += html.escape(str(row_data.text)) + '<br><br><br>' text += row_data.text + " "
ret = {'text': text, 'ppn': ppn} ret = {'text': text, 'ppn': ppn}
return jsonify(ret) return jsonify(ret)
@app.route('/digisam-tokenized/<ppn>') @app.route('/tokenized', methods=['GET', 'POST'])
def tokenized(ppn): def tokenized():
df = digisam.get(ppn) raw_text = request.json['text']
if len(df) == 0: sentences = tokenizer.parse_text(raw_text)
return 'bad request!', 400
text = '' result = [(sen, i) for i, (sen, _) in enumerate(sentences)]
for row_index, row_data in df.iterrows():
if row_data.text is None: return jsonify(result)
continue
sentences = tokenizer.parse_text(row_data.text)
for sen, _ in sentences:
text += html.escape(str(sen)) + '<br>'
text += '<br><br><br>'
ret = {'text': text, 'ppn': ppn}
return jsonify(ret)
@app.route('/ner-bert-tokens/<model_id>/<ppn>') @app.route('/ner-bert-tokens/<model_id>', methods=['GET', 'POST'])
def ner_bert_tokens(model_id, ppn): def ner_bert_tokens(model_id):
df = digisam.get(ppn) raw_text = request.json['text']
if len(df) == 0: sentences = tokenizer.parse_text(raw_text)
return 'bad request!', 400
text = '' prediction = predictor_store.get(model_id).classify_text(sentences)
for row_index, row_data in df.iterrows():
if row_data.text is None: output = []
continue
sentences = tokenizer.parse_text(row_data.text) for tokens, word_predictions in prediction:
prediction = predictor_store.get(model_id).classify_text(sentences) output_sentence = []
for tokens, word_predictions in prediction: for token, word_pred in zip(tokens, word_predictions):
for token, word_pred in zip(tokens, word_predictions): output_sentence.append({'token': html.escape(token), 'prediction': word_pred})
text += html.escape("{}({})".format(token, word_pred)) output.append(output_sentence)
text += '<br>' return jsonify(output)
text += '<br><br><br>'
ret = {'text': text, 'ppn': ppn}
return jsonify(ret)
@app.route('/digisam-ner/<model_id>/<ppn>')
def digisam_ner(model_id, ppn):
df = digisam.get(ppn)
if len(df) == 0:
return 'bad request!', 400
text = ''
for row_index, row_data in df.iterrows():
if row_data.text is None:
continue
sentences = tokenizer.parse_text(row_data.text)
prediction = predictor_store.get(model_id).classify_text(sentences)
for tokens, word_predictions in prediction:
last_prediction = 'O'
for token, word_pred in zip(tokens, word_predictions):
if token == '[UNK]':
continue
if not token.startswith('##'):
text += ' '
token = token[2:] if token.startswith('##') else token
if word_pred != 'X':
last_prediction = word_pred
if last_prediction == 'O':
text += html.escape(token)
elif last_prediction.endswith('PER'):
text += '<font color="red">' + html.escape(token) + '</font>'
elif last_prediction.endswith('LOC'):
text += '<font color="green">' + html.escape(token) + '</font>'
elif last_prediction.endswith('ORG'):
text += '<font color="blue">' + html.escape(token) + '</font>'
text += '<br>'
text += '<br><br><br>'
ret = {'text': text, 'ppn': ppn}
return jsonify(ret)
@app.route('/ner/<model_id>', methods=['GET', 'POST']) @app.route('/ner/<model_id>', methods=['GET', 'POST'])

10
qurator/sbb_ner/webapp/static/index.html
View file

@ -35,9 +35,9 @@
<div class="form-group row ml-2"> <div class="form-group row ml-2">
<label for="task" class="col-sm-2 col-form-label">Task:</label> <label for="task" class="col-sm-2 col-form-label">Task:</label>
<select id="task" class="selectpicker col-md-auto" onchange="task_select()"> <select id="task" class="selectpicker col-md-auto" onchange="task_select()">
<option value="2">Wort- und Satztokenisierung</option> <option value="tokenize">Wort- und Satztokenisierung</option>
<option value="3" selected>Named Entity Recognition</option> <option value="ner" selected>Named Entity Recognition</option>
<option value="4">BERT Tokens</option> <option value="bert-tokens">BERT Tokens</option>
</select> </select>
</div> </div>
<div class="form-group row ml-2" id="model_select"> <div class="form-group row ml-2" id="model_select">
@ -48,8 +48,7 @@
<div class="form-group row ml-2"> <div class="form-group row ml-2">
<label for="inputtext" class="col-sm-2 col-form-label">Input text:</label> <label for="inputtext" class="col-sm-2 col-form-label">Input text:</label>
<!-- <input id="inputtext" class="col-sm-8" type="text" rows=10/> --> <textarea id="inputtext" class=" col-sm-8 form-control" rows="10" required></textarea>
<textarea id="inputtext" class=" col-sm-8 form-control" rows="3" required></textarea>
</div> </div>
<div class="form-group row ml-2"> <div class="form-group row ml-2">
@ -75,5 +74,6 @@
</div> </div>
<script src="js/ner.js"></script> <script src="js/ner.js"></script>
<script src="js/ner-demo.js"></script>
</body> </body>
</html> </html>

34
qurator/sbb_ner/webapp/static/js/ner-demo.js Normal file
View file

@ -0,0 +1,34 @@
$(document).ready(function(){
$('#nerform').submit(
function(e){
e.preventDefault();
var task = $('#task').val();
var model_id = $('#model').val();
var input_text = $('#inputtext').val()
do_task(task, model_id, input_text);
}
);
$.get( "/models")
.done(
function( data ) {
var tmp="";
$.each(data,
function(index, item){
selected=""
if (item.default) {
selected = "selected"
}
tmp += '<option value="' + item.id + '" ' + selected + ' >' + item.name + '</option>'
});
$('#model').html(tmp);
}
);
task_select()
});

138
qurator/sbb_ner/webapp/static/js/ner-ds-sbb.js
View file

@ -4,7 +4,31 @@ $(document).ready(function(){
$('#nerform').submit( $('#nerform').submit(
function(e){ function(e){
e.preventDefault(); e.preventDefault();
load_ppn();
var task = $('#task').val();
var model_id = $('#model').val();
var spinner_html =
`<div class="d-flex justify-content-center">
<div class="spinner-border align-center" role="status">
<span class="sr-only">Loading...</span>
</div>
</div>`;
var ppn = $('#ppn').val()
$("#resultregion").html(spinner_html);
$.get( "/digisam-fulltext/" + ppn)
.done(function( data ) {
do_task(task, model_id, data.text)
})
.fail(
function() {
console.log('Failed.');
$("#resultregion").html('Failed.');
});
} }
); );
@ -41,115 +65,3 @@ $(document).ready(function(){
task_select() task_select()
}); });
function task_select() {
var task = $('#task').val();
if (task < 3) {
$('#model_select').hide()
}
else {
$('#model_select').show()
}
$("#resultregion").html("");
$("#legende").html("");
}
function load_ppn() {
var ppn = $('#ppn').val()
var text_region_html =
`<div class="card">
<div class="card-header">
Ergebnis:
</div>
<div class="card-block">
<div id="textregion" style="overflow-y:scroll;height: 65vh;"></div>
</div>
</div>`;
var legende_html =
`<div class="card">
<div class="card-header">
Legende:
<div class="ml-2" >[<font color="red">Person</font>]</div>
<div class="ml-2" >[<font color="green">Ort</font>]</div>
<div class="ml-2" >[<font color="blue">Organisation</font>]</div>
<div class="ml-2" >[keine Named Entity]</div>
</div>
</div>`;
var spinner_html =
`<div class="d-flex justify-content-center">
<div class="spinner-border align-center" role="status">
<span class="sr-only">Loading...</span>
</div>
</div>`;
$("#legende").html("");
var task = $('#task').val();
var model_id = $('#model').val();
console.log("Task: " + task);
if (task == 1) {
$("#resultregion").html(spinner_html);
$.get( "/digisam-fulltext/" + ppn)
.done(function( data ) {
$("#resultregion").html(text_region_html)
$("#textregion").html(data.text)
})
.fail(
function() {
console.log('Failed.');
$("#resultregion").html('Failed.');
});
}
else if (task == 2) {
$("#resultregion").html(spinner_html);
$.get( "/digisam-tokenized/" + ppn,
function( data ) {
$("#resultregion").html(text_region_html)
$("#textregion").html(data.text)
}).fail(
function() {
console.log('Failed.')
$("#resultregion").html('Failed.')
});
}
else if (task == 3) {
$("#resultregion").html(spinner_html);
$.get( "/digisam-ner/" + model_id + "/" + ppn,
function( data ) {
$("#resultregion").html(text_region_html)
$("#textregion").html(data.text)
$("#legende").html(legende_html)
}).fail(
function(a,b,c) {
console.log('Failed.')
$("#resultregion").html('Failed.')
});
}
else if (task == 4) {
$("#resultregion").html(spinner_html);
$.get( "/digisam-ner-bert-tokens/" + model_id + "/" + ppn,
function( data ) {
$("#resultregion").html(text_region_html)
$("#textregion").html(data.text)
}).fail(
function(a,b,c) {
console.log('Failed.')
$("#resultregion").html('Failed.')
});
}
}

168
qurator/sbb_ner/webapp/static/js/ner.js
View file

@ -1,39 +1,9 @@
$(document).ready(function(){
$('#nerform').submit(
function(e){
e.preventDefault();
do_task();
}
);
$.get( "/models")
.done(
function( data ) {
var tmp="";
$.each(data,
function(index, item){
selected=""
if (item.default) {
selected = "selected"
}
tmp += '<option value="' + item.id + '" ' + selected + ' >' + item.name + '</option>'
});
$('#model').html(tmp);
}
);
task_select()
});
function task_select() { function task_select() {
var task = $('#task').val(); var task = $('#task').val();
if (task < 3) { if ((task != "ner") && (task != "bert-tokens")){
$('#model_select').hide() $('#model_select').hide()
} }
else { else {
@ -44,10 +14,9 @@ function task_select() {
$("#legende").html(""); $("#legende").html("");
} }
function do_task(task, model_id, input_text) {
function do_task() { var post_data = { "text" : input_text }
var input_text = $('#inputtext').val()
var text_region_html = var text_region_html =
`<div class="card"> `<div class="card">
@ -55,7 +24,7 @@ function do_task() {
Ergebnis: Ergebnis:
</div> </div>
<div class="card-block"> <div class="card-block">
<div id="textregion" style="overflow-y:scroll;height: 65vh;"></div> <div id="textregion" style="overflow-y:scroll;height: 55vh;"></div>
</div> </div>
</div>`; </div>`;
@ -79,31 +48,45 @@ function do_task() {
$("#legende").html(""); $("#legende").html("");
var task = $('#task').val(); if (task == "fulltext") {
var model_id = $('#model').val(); $("#resultregion").html(text_region_html)
$("#textregion").html(input_text)
}
else if (task == "tokenize") {
// if (task == 2) { $("#resultregion").html(spinner_html)
// $("#resultregion").html(spinner_html);
//
// $.get( "/digisam-tokenized/" + ppn,
// function( data ) {
// $("#resultregion").html(text_region_html)
// $("#textregion").html(data.text)
// }).fail(
// function() {
// console.log('Failed.')
// $("#resultregion").html('Failed.')
// });
// }
// else
//
if (task == 3) {
$("#resultregion").html(spinner_html); $.ajax(
{
url: "/tokenized",
data: JSON.stringify(post_data),
type: 'POST',
contentType: "application/json",
success:
function( data ) {
text_html = ""
data.forEach(
function(sentence) {
post_data = { "text" : input_text } text_html += JSON.stringify(sentence)
console.log(post_data) text_html += '<br/>'
}
)
$("#resultregion").html(text_region_html)
$("#textregion").html(text_html)
$("#legende").html(legende_html)
}
,
error:
function(error) {
console.log(error);
}
})
}
else if (task == "ner") {
$("#resultregion").html(spinner_html)
$.ajax({ $.ajax({
url: "/ner/" + model_id, url: "/ner/" + model_id,
@ -141,43 +124,40 @@ function do_task() {
console.log(error); console.log(error);
} }
}); });
// $.post( "/ner/" + model_id, post_data).done(
// function( data ) {
//
// text_region_html = ""
// data.forEach(
// function(sentence) {
// sentence.forEach(
// function(token) {
// text_region_html += token.word + "(" + token.prediction + ") "
// })
// }
// )
//
// $("#resultregion").html(text_region_html)
// $("#textregion").html(data.text)
// $("#legende").html(legende_html)
// }).fail(
// function(a,b,c) {
// console.log('Failed.')
// $("#resultregion").html('Failed.')
// });
} }
// else else if (task == "bert-tokens") {
// $("#resultregion").html(spinner_html);
// if (task == 4) {
// $("#resultregion").html(spinner_html); $.ajax(
// {
// $.get( "/digisam-ner-bert-tokens/" + model_id + "/" + ppn, url: "/ner-bert-tokens/" + model_id,
// function( data ) { data: JSON.stringify(post_data),
// $("#resultregion").html(text_region_html) type: 'POST',
// $("#textregion").html(data.text) contentType: "application/json",
// }).fail( success:
// function(a,b,c) { function( data ) {
// console.log('Failed.') text_html = ""
// $("#resultregion").html('Failed.') data.forEach(
// }); function(sentence) {
// } sentence.forEach(
function(part) {
if (text_html != "") text_html += ' '
text_html += part.token + "(" + part.prediction + ")"
})
text_html += '<br/>'
}
)
$("#resultregion").html(text_region_html)
$("#textregion").html(text_html)
$("#legende").html(legende_html)
}
,
error:
function(error) {
console.log(error);
}
})
}
} }

9
qurator/sbb_ner/webapp/static/ner-ds-sbb.html
View file

@ -35,10 +35,10 @@
<div class="form-group row ml-2"> <div class="form-group row ml-2">
<label for="task" class="col-sm-2 col-form-label">Task:</label> <label for="task" class="col-sm-2 col-form-label">Task:</label>
<select id="task" class="selectpicker col-md-auto" onchange="task_select()"> <select id="task" class="selectpicker col-md-auto" onchange="task_select()">
<option value="1">OCR-Text aus ALTO Datei</option> <option value="fulltext">OCR-Text aus ALTO Datei</option>
<option value="2">Wort- und Satztokenisierung</option> <option value="tokenize">Wort- und Satztokenisierung</option>
<option value="3" selected>Named Entity Recognition</option> <option value="ner" selected>Named Entity Recognition</option>
<option value="4">BERT Tokens</option> <option value="bert-tokens">BERT Tokens</option>
</select> </select>
</div> </div>
<div class="form-group row ml-2" id="model_select"> <div class="form-group row ml-2" id="model_select">
@ -72,6 +72,7 @@
</div> </div>
</div> </div>
<script src="js/ner.js"></script>
<script src="js/ner-ds-sbb.js"></script> <script src="js/ner-ds-sbb.js"></script>
</body> </body>
</html> </html>