search

coli-saar / am-parser

Modular implementation of an AM dependency parser in AllenNLP.
Apache License 2.0
30 stars 10 forks source link

Lowering worse than not lowering #75

Closed namednil closed 5 years ago

namednil commented 5 years ago

Row 18 in our spreadsheet contains a new UCCA model from last night. I accidentally didn't use edge lowering in post-processing. When I did, I got this instead:

{"time": 2.5911805629730225,
 "n": 1049,
 "labels": {"f": 0.0, "r": 0.0, "c": 0, "s": 0, "g": 0, "p": 0.0},
 "cpu": 0.5157659000000003,
 "exact": 2,
 "all": {"f": 0.6241449305971113, "r": 0.5956241322165987, "c": 26597, "s": 40573, "g": 44654, "p": 0.6555344687353659},
 "properties": {"f": 0.0, "r": 0.0, "c": 0, "s": 0, "g": 0, "p": 0.0},
 "anchors": {"f": 0.9191877816139037, "r": 0.9138934140097743, "c": 15708, "s": 16990, "g": 17188, "p": 0.9245438493231313},
 "tops": {"f": 0.0, "r": 0.0, "c": 0, "s": 0, "g": 1049, "p": 0.0},
 "attributes": {"f": 0.0, "r": 0.0, "c": 0, "s": 0, "g": 1155, "p": 0.0},
 "edges": {"f": 0.4458593510082915, "r": 0.4310426727891695, "c": 10889, "s": 23583, "g": 25262, "p": 0.4617309078573549}}
namednil commented 5 years ago

Unclear why this occured. Below are the results on the dev set.

From Mario's e-mail: With lowering:

{"n": 1049,
 "exact": 2,
 "tops": {"g": 1049, "s": 1325, "c": 1045, "p": 0.7886792452830189, "r": 0.996186844613918, "f": 0.8803706823925863},
 "labels": {"g": 0, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "properties": {"g": 0, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "anchors": {"g": 17188, "s": 16990, "c": 15617, "p": 0.9191877575044144, "r": 0.9085990225738888, "f": 0.9138627187079408},
 "edges": {"g": 25262, "s": 23724, "c": 14474, "p": 0.6100994773225425, "r": 0.5729554271237431, "f": 0.5909443514473522},
 "attributes": {"g": 1155, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "all": {"g": 44654, "s": 42039, "c": 31136, "p": 0.7406455909988344, "r": 0.6972723608187397, "f": 0.7183048227653905},

Without lowering:

{"n": 1049,
 "exact": 2,
 "tops": {"g": 1049, "s": 1327, "c": 1045, "p": 0.787490580256217, "r": 0.996186844613918, "f": 0.8796296296296297},
 "labels": {"g": 0, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "properties": {"g": 0, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "anchors": {"g": 17188, "s": 16990, "c": 15725, "p": 0.9255444379046498, "r": 0.9148824761461485, "f": 0.9201825735853474},
 "edges": {"g": 25262, "s": 23082, "c": 12904, "p": 0.5590503422580365, "r": 0.51080674530916, "f": 0.5338408075459209},
 "attributes": {"g": 1155, "s": 0, "c": 0, "p": 0.0, "r": 0.0, "f": 0.0},
 "all": {"g": 44654, "s": 41399, "c": 29674, "p": 0.7167805985651827, "r": 0.6645317328794733, "f": 0.6896679953052189},
 "time": 24.02891492843628,
 "cpu": 23.902485000000002}