Striveworks / valor

Valor is a centralized evaluation store which makes it easy to measure, explore, and rank model performance.
https://striveworks.github.io/valor/
Other
38 stars 4 forks source link

BUG: Valor Core Classification Fix Future Warning #738

Closed jqu-striveworks closed 1 month ago

jqu-striveworks commented 2 months ago

valor version checks

Reproducible Example

from valor_core import classification, schemas, enums

def evaluate_image_clf_groundtruths():
    return [
        schemas.GroundTruth(
            datum=schemas.Datum(
                uid="uid0",
                metadata={
                    "height": 900,
                    "width": 300,
                },
            ),
            annotations=[
                schemas.Annotation(
                    labels=[
                        schemas.Label(key="dataset1", value="ant"),
                    ],
                ),
            ],
        ),
        schemas.GroundTruth(
            datum=schemas.Datum(
                uid="uid1",
                metadata={
                    "height": 900,
                    "width": 300,
                },
            ),
            annotations=[
                schemas.Annotation(
                    labels=[
                        schemas.Label(key="dataset2", value="egg"),
                    ],
                ),
            ],
        ),
    ]

def evaluate_image_clf_predictions():
    return [
        schemas.Prediction(
            datum=schemas.Datum(
                uid="uid0",
                metadata={
                    "height": 900,
                    "width": 300,
                },
            ),
            annotations=[
                schemas.Annotation(
                    labels=[
                        schemas.Label(key="dataset1", value="ant", score=0.15),
                        schemas.Label(key="dataset1", value="bee", score=0.48),
                        schemas.Label(key="dataset1", value="cat", score=0.37),
                    ],
                )
            ],
        ),
        schemas.Prediction(
            datum=schemas.Datum(
                uid="uid1",
                metadata={
                    "height": 900,
                    "width": 300,
                },
            ),
            annotations=[
                schemas.Annotation(
                    labels=[
                        schemas.Label(key="dataset2", value="egg", score=0.15),
                        schemas.Label(key="dataset2", value="milk", score=0.48),
                        schemas.Label(key="dataset2", value="flour", score=0.37),
                    ],
                )
            ],
        ),
    ]

groundtruths = evaluate_image_clf_groundtruths()
predictions = evaluate_image_clf_predictions()

metrics_to_return = [
    enums.MetricType.DetailedPrecisionRecallCurve
]

metrics_out = classification.evaluate_classification(
    groundtruths=groundtruths,
    predictions=predictions,
    metrics_to_return=metrics_to_return,
)

def foo(k, _class, _metric):
    if k == 'dataset1':
        m = metrics_out.metrics[0]['value'][_class]
    elif k == 'dataset2':
        m = metrics_out.metrics[1]['value'][_class]

    for threshold in [x / 100 for x in range(5, 100, 5)]:
        print(f"{threshold:.2f}: {(m[threshold][_metric])}")

Issue Description

valor_core/utilities.py:26: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.

Expected Behavior

No more warnings