Skip to content

Commit

Permalink
[#179] Include F-measure in ThresholdTestResults
Browse files Browse the repository at this point in the history
This required fixing a bug in core.model_metrics.f_measure where it errored out
instead of returning NaN when its denominator was 0.
  • Loading branch information
riley-harper committed Dec 12, 2024
1 parent 1ecef81 commit 7f0c48c
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 2 deletions.
5 changes: 4 additions & 1 deletion hlink/linking/core/model_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@


def f_measure(true_pos: int, false_pos: int, false_neg: int) -> float:
return 2 * true_pos / (2 * true_pos + false_pos + false_neg)
denominator = 2 * true_pos + false_pos + false_neg
if denominator == 0:
return math.nan
return 2 * true_pos / denominator


def mcc(true_pos: int, true_neg: int, false_pos: int, false_neg: int) -> float:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,9 @@ def make_threshold_matrix(self) -> list[list[float]]:
class ThresholdTestResult:
precision: float
recall: float
pr_auc: float
mcc: float
f_measure: float
pr_auc: float
model_id: str
alpha_threshold: float
threshold_ratio: float
Expand Down Expand Up @@ -661,11 +662,13 @@ def _capture_prediction_results(
precision = metrics_core.precision(tp_count, fp_count)
recall = metrics_core.recall(tp_count, fn_count)
mcc = metrics_core.mcc(tp_count, tn_count, fp_count, fn_count)
f_measure = metrics_core.f_measure(tp_count, fp_count, fn_count)

result = ThresholdTestResult(
precision=precision,
recall=recall,
mcc=mcc,
f_measure=f_measure,
pr_auc=pr_auc,
model_id=model,
alpha_threshold=alpha_threshold,
Expand Down
9 changes: 9 additions & 0 deletions hlink/tests/core/model_metrics_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ def test_f_measure_example() -> None:
), "expected F-measure to be near 0.8229539"


def test_f_measure_all_zeroes() -> None:
"""
When true_pos, false_pos, and false_neg are all 0, f_measure is undefined and
returns NaN to indicate this.
"""
f_measure_score = f_measure(0, 0, 0)
assert math.isnan(f_measure_score)


@given(true_pos=NonNegativeInt, false_pos=NonNegativeInt, false_neg=NonNegativeInt)
def test_f_measure_between_0_and_1(
true_pos: int, false_pos: int, false_neg: int
Expand Down

0 comments on commit 7f0c48c

Please sign in to comment.