Merge branch 'release/v0.1.2'

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.1
+current_version = 0.1.2
 commit = False
 tag = False
 allow_dirty = False

README.md

@@ -1,5 +1,10 @@
 # Kyle - a Calibration Toolkit
 
+## Note:
+This library is currently in the alpha stage and breaking changes can happen at any time. Some
+central features are currently missing and will be added soon.
+
+## Overview
 This library contains utils for measuring and visualizing calibration of probabilistic classifiers as well as for 
 recalibrating them. Currently, only methods for recalibration through post-processing are supported, although we plan
 to include calibration specific training algorithms as well in the future.
@@ -10,9 +15,19 @@ notebook (the notebook with executed cells can be found in the docu).
 
 Apart from tools for analysing models, kyle also offers support for developing and testing custom calibration metrics
 and algorithms. In order not to have to rely on evaluation data sets and trained models for delivering labels and confidence 
-vectors, with kyle custom samplers based on [fake classifiers](our paper/review) can be constructed. These samplers can
+vectors, with kyle custom samplers based on fake classifiers can be constructed. A note explaining the
+theory behind fake classifiers will be published soon.
+These samplers can
 also be fit on some data set in case you want to mimic it. Using the fake classifiers, an arbitrary number of ground 
 truth labels and miscalibrated confidence vectors can be generated to help you analyse your algorithms (common use cases
-will be analysis of variance and bias of calibration metrics and benchmarking of recalibration algorithms). Several
-pre-configured fake classifiers mimicking common models, e.g. vision models trained on MNIST and CIFAR10, are implemented
-in kyle and can be used out of the box. 
+will be analysis of variance and bias of calibration metrics and benchmarking of recalibration algorithms).
+
+
+Currently, several algorithms in kyle use the [calibration framework library](https://github.com/fabiankueppers/calibration-framework) under the hood although this is subject 
+to change.
+
+## Installation
+Kyle can be installed from pypi, e.g. with
+```
+pip install kyle-calibration
+```

docs/conf.py

@@ -132,14 +132,14 @@ def lineno_from_object_name(source_file, object_name):
 master_doc = "index"
 
 # General information about the project.
-project = "kyle"
+package_name = "kyle-calibration"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-version = pkg_resources.get_distribution(project).version
+version = pkg_resources.get_distribution(package_name).version
 release = version
 # The short X.Y version.
 major_v, minor_v = version.split(".")[:2]
@@ -300,7 +300,7 @@ def lineno_from_object_name(source_file, object_name):
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [("index", "kyle", "", ["Miguel and Mischa"], 1)]
+man_pages = [("index", "kyle", "", ["appliedAI"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False

setup.py

@@ -15,7 +15,7 @@
  license="MIT",
  url="https://github.com/appliedAI-Initiative/kyle",
  include_package_data=True,
- version="0.1.1",
+ version="0.1.2",
  description="appliedAI classifier calibration library",
  install_requires=open("requirements.txt").readlines(),
  setup_requires=["wheel"],

src/kyle/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.1.2"

src/kyle/calibration/calibration_methods/calibration_methods.py

@@ -85,7 +85,7 @@ def __init__(self):
 
 class HistogramBinning(NetcalBasedCalibration[bn.HistogramBinning]):
  def __init__(self, bins=20):
- super().__init__(bn.HistogramBinning(bins=20))
+ super().__init__(bn.HistogramBinning(bins=bins))
 
 
 class ClassWiseCalibration(BaseCalibrationMethod):

src/kyle/evaluation/discrete.py

@@ -1,4 +1,4 @@
-from typing import Sequence, Union
+from typing import Sequence, Union, Dict
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -321,3 +321,15 @@ def plot_confidence_distributions(
  plt.legend(loc="best")
  if new_fig:
  plt.show()
+
+ def plot_gt_distribution(self, label_names: Dict[int, str] = None):
+ class_labels, counts = np.unique(self.y_true, return_counts=True)
+ if label_names is not None:
+ class_labels = [label_names.get(l, l) for l in class_labels]
+
+ fig, ax = plt.subplots()
+ ax.pie(counts, labels=class_labels, autopct="%1.1f%%", startangle=90)
+ ax.axis("equal") # Equal aspect ratio ensures that pie is drawn as a circle.
+ ax.set_title("Ground Truth Distribution")
+ fig.show()
+