Skip to content

Commit

Permalink
[#179] Factor away _get_aggregate_metrics()
Browse files Browse the repository at this point in the history
This function is now simple enough that we can just inline it in the one
place where it's called.
  • Loading branch information
riley-harper committed Dec 11, 2024
1 parent df9b463 commit 7817ed5
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 42 deletions.
2 changes: 2 additions & 0 deletions hlink/linking/core/model_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# https://github.com/ipums/hlink
import math

import numpy as np


def mcc(tp: int, tn: int, fp: int, fn: int) -> float:
"""
Expand Down
30 changes: 6 additions & 24 deletions hlink/linking/model_exploration/link_step_train_test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,14 +658,14 @@ def _capture_prediction_results(
fn_count,
tn_count,
) = _get_confusion_matrix(predictions, dep_var)
test_precision, test_recall, test_mcc = _get_aggregate_metrics(
tp_count, fp_count, fn_count, tn_count
)
precision = metrics_core.precision(tp_count, fp_count)
recall = metrics_core.recall(tp_count, fn_count)
mcc = metrics_core.mcc(tp_count, tn_count, fp_count, fn_count)

result = ThresholdTestResult(
precision=test_precision,
recall=test_recall,
mcc=test_mcc,
precision=precision,
recall=recall,
mcc=mcc,
pr_auc=pr_auc,
model_id=model,
alpha_threshold=alpha_threshold,
Expand Down Expand Up @@ -764,24 +764,6 @@ def _get_confusion_matrix(
)


def _get_aggregate_metrics(
true_positives: int, false_positives: int, false_negatives: int, true_negatives: int
) -> tuple[float, float, float]:
"""
Given the counts of true positives, false positives, false negatives, and
true negatives for a model run, compute several metrics to evaluate the
model's quality.
Return a tuple of (precision, recall, Matthews Correlation Coefficient).
"""
precision = metrics_core.precision(true_positives, false_positives)
recall = metrics_core.recall(true_positives, false_negatives)
mcc = metrics_core.mcc(
true_positives, true_negatives, false_positives, false_negatives
)
return precision, recall, mcc


# The outer list entries hold results from each outer fold, the inner list has a ThresholdTestResult per threshold
# matrix entry. We need to get data for each threshold entry together. Basically we need to invert the data.
def _combine_by_threshold_matrix_entry(
Expand Down
36 changes: 36 additions & 0 deletions hlink/tests/core/model_metrics_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# This file is part of the ISRDI's hlink.
# For copyright and licensing information, see the NOTICE and LICENSE files
# in this project's top-level directory, and also on-line at:
# https://github.com/ipums/hlink

from hlink.linking.core.model_metrics import mcc, precision, recall


def test_mcc_example() -> None:
tp = 3112
fp = 205
fn = 1134
tn = 33259

mcc_score = mcc(tp, tn, fp, fn)
assert abs(mcc_score - 0.8111208) < 0.0001, "expected MCC to be near 0.8111208"


def test_precision_example() -> None:
tp = 3112
fp = 205

precision_score = precision(tp, fp)
assert (
abs(precision_score - 0.9381972) < 0.0001
), "expected precision to be near 0.9381972"


def test_recall_example() -> None:
tp = 3112
fn = 1134

recall_score = recall(tp, fn)
assert (
abs(recall_score - 0.7329251) < 0.0001
), "expected recall to be near 0.7329251"
18 changes: 0 additions & 18 deletions hlink/tests/model_exploration_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
_custom_param_grid_builder,
_get_model_parameters,
_get_confusion_matrix,
_get_aggregate_metrics,
)


Expand Down Expand Up @@ -1016,20 +1015,3 @@ def test_get_confusion_matrix(spark: SparkSession) -> None:
assert false_positives == 3
assert false_negatives == 2
assert true_negatives == 1


def test_get_aggregate_metrics() -> None:
true_positives = 3112
false_positives = 205
false_negatives = 1134
true_negatives = 33259

precision, recall, mcc = _get_aggregate_metrics(
true_positives, false_positives, false_negatives, true_negatives
)

assert (
abs(precision - 0.9381972) < 0.0001
), "expected precision to be near 0.9381972"
assert abs(recall - 0.7329251) < 0.0001, "expected recall to be near 0.7329251"
assert abs(mcc - 0.8111208) < 0.0001, "expected MCC to be near 0.8111208"

0 comments on commit 7817ed5

Please sign in to comment.