diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bba4717..c8c58e3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.1 +current_version = 0.1.2 commit = False tag = False allow_dirty = False diff --git a/README.md b/README.md index 209182e..febc47c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # Kyle - a Calibration Toolkit +## Note: +This library is currently in the alpha stage and breaking changes can happen at any time. Some +central features are currently missing and will be added soon. + +## Overview This library contains utils for measuring and visualizing calibration of probabilistic classifiers as well as for recalibrating them. Currently, only methods for recalibration through post-processing are supported, although we plan to include calibration specific training algorithms as well in the future. @@ -10,9 +15,19 @@ notebook (the notebook with executed cells can be found in the docu). Apart from tools for analysing models, kyle also offers support for developing and testing custom calibration metrics and algorithms. In order not to have to rely on evaluation data sets and trained models for delivering labels and confidence -vectors, with kyle custom samplers based on [fake classifiers](our paper/review) can be constructed. These samplers can +vectors, with kyle custom samplers based on fake classifiers can be constructed. A note explaining the +theory behind fake classifiers will be published soon. +These samplers can also be fit on some data set in case you want to mimic it. Using the fake classifiers, an arbitrary number of ground truth labels and miscalibrated confidence vectors can be generated to help you analyse your algorithms (common use cases -will be analysis of variance and bias of calibration metrics and benchmarking of recalibration algorithms). Several -pre-configured fake classifiers mimicking common models, e.g. vision models trained on MNIST and CIFAR10, are implemented -in kyle and can be used out of the box. +will be analysis of variance and bias of calibration metrics and benchmarking of recalibration algorithms). + + +Currently, several algorithms in kyle use the [calibration framework library](https://github.com/fabiankueppers/calibration-framework) under the hood although this is subject +to change. + +## Installation +Kyle can be installed from pypi, e.g. with +``` +pip install kyle-calibration +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 1ccbd01..139812f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -132,14 +132,14 @@ def lineno_from_object_name(source_file, object_name): master_doc = "index" # General information about the project. -project = "kyle" +package_name = "kyle-calibration" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The full version, including alpha/beta/rc tags. -version = pkg_resources.get_distribution(project).version +version = pkg_resources.get_distribution(package_name).version release = version # The short X.Y version. major_v, minor_v = version.split(".")[:2] @@ -300,7 +300,7 @@ def lineno_from_object_name(source_file, object_name): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "kyle", "", ["Miguel and Mischa"], 1)] +man_pages = [("index", "kyle", "", ["appliedAI"], 1)] # If true, show URL addresses after external links. # man_show_urls = False diff --git a/setup.py b/setup.py index f35f14d..45a7287 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ license="MIT", url="https://github.com/appliedAI-Initiative/kyle", include_package_data=True, - version="0.1.1", + version="0.1.2", description="appliedAI classifier calibration library", install_requires=open("requirements.txt").readlines(), setup_requires=["wheel"], diff --git a/src/kyle/__init__.py b/src/kyle/__init__.py index 485f44a..b3f4756 100644 --- a/src/kyle/__init__.py +++ b/src/kyle/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/src/kyle/calibration/calibration_methods/calibration_methods.py b/src/kyle/calibration/calibration_methods/calibration_methods.py index 8bc4870..25989c8 100644 --- a/src/kyle/calibration/calibration_methods/calibration_methods.py +++ b/src/kyle/calibration/calibration_methods/calibration_methods.py @@ -85,7 +85,7 @@ def __init__(self): class HistogramBinning(NetcalBasedCalibration[bn.HistogramBinning]): def __init__(self, bins=20): - super().__init__(bn.HistogramBinning(bins=20)) + super().__init__(bn.HistogramBinning(bins=bins)) class ClassWiseCalibration(BaseCalibrationMethod): diff --git a/src/kyle/evaluation/discrete.py b/src/kyle/evaluation/discrete.py index 2f60d21..e4e407f 100644 --- a/src/kyle/evaluation/discrete.py +++ b/src/kyle/evaluation/discrete.py @@ -1,4 +1,4 @@ -from typing import Sequence, Union +from typing import Sequence, Union, Dict import matplotlib.pyplot as plt import numpy as np @@ -321,3 +321,15 @@ def plot_confidence_distributions( plt.legend(loc="best") if new_fig: plt.show() + + def plot_gt_distribution(self, label_names: Dict[int, str] = None): + class_labels, counts = np.unique(self.y_true, return_counts=True) + if label_names is not None: + class_labels = [label_names.get(l, l) for l in class_labels] + + fig, ax = plt.subplots() + ax.pie(counts, labels=class_labels, autopct="%1.1f%%", startangle=90) + ax.axis("equal") # Equal aspect ratio ensures that pie is drawn as a circle. + ax.set_title("Ground Truth Distribution") + fig.show() +