Closed ntlind closed 1 month ago
wrote the following tests to confirm that we get the same output as sklearn:
def test_sklearn_output():
from sklearn.metrics import roc_auc_score
n_datums = 100
y_true = [0] * (n_datums - 1) + [1]
y_score = [0.99] * (n_datums - 1) + [0.01]
output = roc_auc_score(y_true, y_score, multi_class="ovr")
assert output == 0
def test_perfect_matches():
n_datums = 100
classifications = []
for i in range(n_datums):
classifications.append(
Classification(
uid=f"uid{i}",
groundtruths=["low_risk"],
predictions=["low_risk", "high_risk"],
scores=[0.99, 0.01],
)
)
loader = DataLoader()
loader.add_data(classifications)
evaluator = loader.finalize()
metrics = evaluator.evaluate(as_dict=True)
assert metrics[MetricType.ROCAUC][0]["value"] == 0
valor version checks
Reproducible Example
Issue Description
Accuracy, precision, etc. are equal to 1 using the API service, but ROCAUC seems off.
making the code adjustment below seems to fix the issue:
Expected Behavior
check to make sure we're handling zeroes correctly in
lite