refactor: rename tool and update README (#63)

1. Rename tool to `tti-eval`
2. Add some juicy stuff to README

pyproject.toml

@@ -4,11 +4,11 @@ requires = ["poetry-core"]
 
 [tool.poetry]
 authors = ["Cord Technologies Limited <[email protected]>"]
-description = "Working with multiple CLIP models"
-name = "clip-eval"
+description = "Evaluating text-to-image and image-to-text retrieval models."
+name = "tti-eval"
 readme = "README.md"
 version = "0.1.0"
-packages = [{include = "clip_eval", from = "."}]
+packages = [{include = "tti_eval", from = "."}]
 
 [tool.poetry.dependencies]
 autofaiss = "^2.17.0"
@@ -41,7 +41,7 @@ ipdb = "^0.13.13"
 pytest = "^8.1.1"
 
 [tool.poetry.scripts]
-clip-eval = "clip_eval.cli.main:cli"
+tti-eval = "tti_eval.cli.main:cli"
 
 [tool.ruff]
 line-length = 120

sources/dataset-definition-schema.json

@@ -22,7 +22,7 @@
  "type": "string"
  },
  "module_path": {
- "default": "../../clip_eval/dataset/types/__init__.py",
+ "default": "../../tti_eval/dataset/types/__init__.py",
  "format": "path",
  "title": "Module Path",
  "type": "string"

sources/model-definition-schema.json

@@ -10,7 +10,7 @@
  "type": "string"
  },
  "module_path": {
- "default": "../../clip_eval/model/types/__init__.py",
+ "default": "../../tti_eval/model/types/__init__.py",
  "format": "path",
  "title": "Module Path",
  "type": "string"

tests/common/test_embedding_definition.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from clip_eval.common import EmbeddingDefinition, Embeddings, Split
-from clip_eval.compute import compute_embeddings_from_definition
+from tti_eval.common import EmbeddingDefinition, Embeddings, Split
+from tti_eval.compute import compute_embeddings_from_definition
 
 if __name__ == "__main__":
  def_ = EmbeddingDefinition(

tests/evaluation/test_knn.py

@@ -1,8 +1,8 @@
 import numpy as np
 
-from clip_eval.common import ClassArray, Embeddings, ProbabilityArray
-from clip_eval.evaluation.knn import WeightedKNNClassifier
-from clip_eval.evaluation.utils import normalize, softmax
+from tti_eval.common import ClassArray, Embeddings, ProbabilityArray
+from tti_eval.evaluation.knn import WeightedKNNClassifier
+from tti_eval.evaluation.utils import normalize, softmax
 
 
 def slow_knn_predict(

clip_eval/cli/main.py → tti_eval/cli/main.py

@@ -3,17 +3,17 @@
 import matplotlib.pyplot as plt
 from typer import Option, Typer
 
-from clip_eval.common import Split
-from clip_eval.compute import compute_embeddings_from_definition
-from clip_eval.utils import read_all_cached_embeddings
+from tti_eval.common import Split
+from tti_eval.compute import compute_embeddings_from_definition
+from tti_eval.utils import read_all_cached_embeddings
 
 from .utils import (
  parse_raw_embedding_definitions,
  select_existing_embedding_definitions,
  select_from_all_embedding_definitions,
 )
 
-cli = Typer(name="clip-eval", no_args_is_help=True, rich_markup_mode="markdown")
+cli = Typer(name="tti-eval", no_args_is_help=True, rich_markup_mode="markdown")
 
 
 @cli.command(
@@ -84,13 +84,13 @@ def evaluate_embeddings(
  all_: Annotated[bool, Option("--all", "-a", help="Evaluate all models.")] = False,
  save: Annotated[bool, Option("--save", "-s", help="Save evaluation results to a CSV file.")] = False,
 ):
- from clip_eval.evaluation import (
+ from tti_eval.evaluation import (
  I2IRetrievalEvaluator,
  LinearProbeClassifier,
  WeightedKNNClassifier,
  ZeroShotClassifier,
  )
- from clip_eval.evaluation.evaluator import export_evaluation_to_csv, run_evaluation
+ from tti_eval.evaluation.evaluator import export_evaluation_to_csv, run_evaluation
 
  model_datasets = model_datasets or []
 
@@ -117,7 +117,7 @@ def animate_embeddings(
  interactive: Annotated[bool, Option(help="Interactive plot instead of animation.")] = False,
  reduction: Annotated[str, Option(help="Reduction type [pca, tsne, umap (default)].")] = "umap",
 ):
- from clip_eval.plotting.animation import build_animation, save_animation_to_file
+ from tti_eval.plotting.animation import build_animation, save_animation_to_file
 
  defs = select_existing_embedding_definitions(by_dataset=True, count=2)
  res = build_animation(defs[0], defs[1], interactive=interactive, reduction=reduction)
@@ -135,8 +135,8 @@ def list_models_datasets(
  Option("--all", "-a", help="List all models and datasets that are available via the tool."),
  ] = False,
 ):
- from clip_eval.dataset import DatasetProvider
- from clip_eval.model import ModelProvider
+ from tti_eval.dataset import DatasetProvider
+ from tti_eval.model import ModelProvider
 
  if all_:
  datasets = DatasetProvider.list_dataset_titles()

clip_eval/cli/utils.py → tti_eval/cli/utils.py

@@ -5,10 +5,10 @@
 from InquirerPy.base.control import Choice
 from natsort import natsorted, ns
 
-from clip_eval.common import EmbeddingDefinition
-from clip_eval.dataset import DatasetProvider
-from clip_eval.model import ModelProvider
-from clip_eval.utils import read_all_cached_embeddings
+from tti_eval.common import EmbeddingDefinition
+from tti_eval.dataset import DatasetProvider
+from tti_eval.model import ModelProvider
+from tti_eval.utils import read_all_cached_embeddings
 
 
 @overload

clip_eval/common/base.py → tti_eval/common/base.py

@@ -7,7 +7,7 @@
 from pydantic import BaseModel, model_validator
 from pydantic.functional_validators import AfterValidator
 
-from clip_eval.constants import NPZ_KEYS, PROJECT_PATHS
+from tti_eval.constants import NPZ_KEYS, PROJECT_PATHS
 
 from .numpy_types import ClassArray, EmbeddingArray
 from .string_utils import safe_str

clip_eval/compute.py → tti_eval/compute.py

@@ -1,8 +1,8 @@
 from torch.utils.data import DataLoader
 
-from clip_eval.common import EmbeddingDefinition, Embeddings, Split
-from clip_eval.dataset import Dataset, DatasetProvider
-from clip_eval.model import Model, ModelProvider
+from tti_eval.common import EmbeddingDefinition, Embeddings, Split
+from tti_eval.dataset import Dataset, DatasetProvider
+from tti_eval.model import Model, ModelProvider
 
 
 def compute_embeddings(model: Model, dataset: Dataset, batch_size: int = 50) -> Embeddings:

clip_eval/constants.py → tti_eval/constants.py

@@ -6,10 +6,12 @@
 load_dotenv()
 
 # If the cache directory is not explicitly specified, use the `.cache` directory located in the project's root.
-_CLIP_EVAL_ROOT_DIR = Path(__file__).parent.parent
-CACHE_PATH = Path(os.environ.get("CLIP_EVAL_CACHE_PATH", _CLIP_EVAL_ROOT_DIR / ".cache"))
-_OUTPUT_PATH = Path(os.environ.get("CLIP_EVAL_OUTPUT_PATH", _CLIP_EVAL_ROOT_DIR / "output"))
-_SOURCES_PATH = _CLIP_EVAL_ROOT_DIR / "sources"
+_TTI_EVAL_ROOT_DIR = Path(__file__).parent.parent
+CACHE_PATH = Path(os.environ.get("TTI_EVAL_CACHE_PATH", _TTI_EVAL_ROOT_DIR / ".cache"))
+_OUTPUT_PATH = Path(
+ os.environ.get("TTI_EVAL_OUTPUT_PATH", _TTI_EVAL_ROOT_DIR / "output")
+)
+_SOURCES_PATH = _TTI_EVAL_ROOT_DIR / "sources"
 
 
 class PROJECT_PATHS:

clip_eval/dataset/base.py → tti_eval/dataset/base.py

@@ -5,8 +5,8 @@
 from pydantic import BaseModel, ConfigDict
 from torch.utils.data import Dataset as TorchDataset
 
-from clip_eval.common import Split
-from clip_eval.constants import CACHE_PATH, SOURCES_PATH
+from tti_eval.common import Split
+from tti_eval.constants import CACHE_PATH, SOURCES_PATH
 
 DEFAULT_DATASET_TYPES_LOCATION = (
  Path(relpath(str(__file__), SOURCES_PATH.DATASET_INSTANCE_DEFINITIONS)).parent / "types" / "__init__.py"

clip_eval/dataset/provider.py → tti_eval/dataset/provider.py

@@ -4,7 +4,7 @@
 
 from natsort import natsorted, ns
 
-from clip_eval.constants import CACHE_PATH, SOURCES_PATH
+from tti_eval.constants import CACHE_PATH, SOURCES_PATH
 
 from .base import Dataset, DatasetDefinitionSpec, Split
 from .utils import load_class_from_path

tti_eval/dataset/types/__init__.py

@@ -0,0 +1,2 @@
+from tti_eval.dataset.types.encord_ds import EncordDataset
+from tti_eval.dataset.types.hugging_face import HFDataset

clip_eval/dataset/types/encord_ds.py → tti_eval/dataset/types/encord_ds.py

@@ -12,8 +12,8 @@
 from PIL import Image as PILImage
 from tqdm.auto import tqdm
 
-from clip_eval.dataset import Dataset, Split
-from clip_eval.dataset.utils import collect_async, download_file, simple_random_split
+from tti_eval.dataset import Dataset, Split
+from tti_eval.dataset.utils import collect_async, download_file, simple_random_split
 
 
 class EncordDataset(Dataset):

clip_eval/dataset/types/hugging_face.py → tti_eval/dataset/types/hugging_face.py

@@ -1,6 +1,6 @@
 from datasets import ClassLabel, DatasetDict, Sequence, Value, load_dataset
 
-from clip_eval.dataset import Dataset, Split
+from tti_eval.dataset import Dataset, Split
 
 
 class HFDataset(Dataset):

clip_eval/dataset/utils.py → tti_eval/dataset/utils.py

@@ -10,7 +10,7 @@
 import requests
 from tqdm.auto import tqdm
 
-from clip_eval.common import Split
+from tti_eval.common import Split
 
 T = TypeVar("T")
 G = TypeVar("G")

clip_eval/evaluation/base.py → tti_eval/evaluation/base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any
 
-from clip_eval.common import ClassArray, Embeddings, ProbabilityArray
+from tti_eval.common import ClassArray, Embeddings, ProbabilityArray
 
 from .utils import normalize
 

clip_eval/evaluation/evaluator.py → tti_eval/evaluation/evaluator.py

@@ -4,16 +4,16 @@
 from natsort import natsorted, ns
 from tabulate import tabulate
 
-from clip_eval.common import EmbeddingDefinition, Split
-from clip_eval.constants import OUTPUT_PATH
-from clip_eval.evaluation import (
+from tti_eval.common import EmbeddingDefinition, Split
+from tti_eval.constants import OUTPUT_PATH
+from tti_eval.evaluation import (
  EvaluationModel,
  I2IRetrievalEvaluator,
  LinearProbeClassifier,
  WeightedKNNClassifier,
  ZeroShotClassifier,
 )
-from clip_eval.utils import read_all_cached_embeddings
+from tti_eval.utils import read_all_cached_embeddings
 
 
 def print_evaluation_results(

clip_eval/evaluation/image_retrieval.py → tti_eval/evaluation/image_retrieval.py

@@ -4,7 +4,7 @@
 import numpy as np
 from autofaiss import build_index
 
-from clip_eval.common import Embeddings
+from tti_eval.common import Embeddings
 
 from .base import EvaluationModel
 

clip_eval/evaluation/knn.py → tti_eval/evaluation/knn.py

@@ -4,7 +4,7 @@
 import numpy as np
 from autofaiss import build_index
 
-from clip_eval.common import ClassArray, Embeddings, ProbabilityArray
+from tti_eval.common import ClassArray, Embeddings, ProbabilityArray
 
 from .base import ClassificationModel
 from .utils import softmax

clip_eval/evaluation/linear_probe.py → tti_eval/evaluation/linear_probe.py

@@ -4,8 +4,8 @@
 import numpy as np
 from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
 
-from clip_eval.common import ClassArray, Embeddings, ProbabilityArray
-from clip_eval.evaluation.base import ClassificationModel
+from tti_eval.common import ClassArray, Embeddings, ProbabilityArray
+from tti_eval.evaluation.base import ClassificationModel
 
 logger = logging.getLogger("multiclips")
 

clip_eval/evaluation/utils.py → tti_eval/evaluation/utils.py

@@ -1,7 +1,7 @@
 import numpy as np
 import numpy.typing as npt
 
-from clip_eval.common.numpy_types import DType
+from tti_eval.common.numpy_types import DType
 
 
 def normalize(x: npt.NDArray[DType]) -> npt.NDArray[DType]:

clip_eval/evaluation/zero_shot.py → tti_eval/evaluation/zero_shot.py

@@ -1,8 +1,8 @@
 import numpy as np
 
-from clip_eval.common import ClassArray, Embeddings, ProbabilityArray
-from clip_eval.evaluation.base import ClassificationModel
-from clip_eval.evaluation.utils import softmax
+from tti_eval.common import ClassArray, Embeddings, ProbabilityArray
+from tti_eval.evaluation.base import ClassificationModel
+from tti_eval.evaluation.utils import softmax
 
 
 class ZeroShotClassifier(ClassificationModel):

clip_eval/model/base.py → tti_eval/model/base.py

@@ -8,8 +8,8 @@
 from pydantic import BaseModel, ConfigDict
 from torch.utils.data import DataLoader
 
-from clip_eval.common.numpy_types import ClassArray, EmbeddingArray
-from clip_eval.constants import CACHE_PATH, SOURCES_PATH
+from tti_eval.common.numpy_types import ClassArray, EmbeddingArray
+from tti_eval.constants import CACHE_PATH, SOURCES_PATH
 
 DEFAULT_MODEL_TYPES_LOCATION = (
  Path(relpath(str(__file__), SOURCES_PATH.MODEL_INSTANCE_DEFINITIONS)).parent / "types" / "__init__.py"

clip_eval/model/provider.py → tti_eval/model/provider.py

@@ -3,8 +3,8 @@
 
 from natsort import natsorted, ns
 
-from clip_eval.constants import SOURCES_PATH
-from clip_eval.dataset.utils import load_class_from_path
+from tti_eval.constants import SOURCES_PATH
+from tti_eval.dataset.utils import load_class_from_path
 
 from .base import Model, ModelDefinitionSpec
 

tti_eval/model/types/__init__.py

@@ -0,0 +1,3 @@
+from tti_eval.model.types.hugging_face_clip import ClosedCLIPModel
+from tti_eval.model.types.local_clip_model import LocalCLIPModel
+from tti_eval.model.types.open_clip_model import OpenCLIPModel

clip_eval/model/types/hugging_face_clip.py → tti_eval/model/types/hugging_face_clip.py

@@ -9,9 +9,9 @@
 from transformers import AutoProcessor as HF_AutoProcessor
 from transformers import AutoTokenizer as HF_AutoTokenizer
 
-from clip_eval.common import ClassArray, EmbeddingArray
-from clip_eval.dataset import Dataset
-from clip_eval.model import Model
+from tti_eval.common import ClassArray, EmbeddingArray
+from tti_eval.dataset import Dataset
+from tti_eval.model import Model
 
 
 class ClosedCLIPModel(Model):

clip_eval/model/types/local_clip_model.py → tti_eval/model/types/local_clip_model.py

@@ -1,4 +1,4 @@
-from clip_eval.model.types.open_clip_model import OpenCLIPModel
+from tti_eval.model.types.open_clip_model import OpenCLIPModel
 
 
 class LocalCLIPModel(OpenCLIPModel):

clip_eval/model/types/open_clip_model.py → tti_eval/model/types/open_clip_model.py

@@ -7,9 +7,9 @@
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 
-from clip_eval.common import ClassArray, EmbeddingArray
-from clip_eval.dataset import Dataset
-from clip_eval.model import Model
+from tti_eval.common import ClassArray, EmbeddingArray
+from tti_eval.dataset import Dataset
+from tti_eval.model import Model
 
 
 class OpenCLIPModel(Model):

clip_eval/plotting/animation.py → tti_eval/plotting/animation.py

@@ -8,11 +8,11 @@
 from matplotlib.widgets import Slider
 from PIL import Image
 
-from clip_eval.common import EmbeddingDefinition
-from clip_eval.common.base import SafeName
-from clip_eval.common.numpy_types import ClassArray, N2Array
-from clip_eval.constants import OUTPUT_PATH
-from clip_eval.dataset import DatasetProvider, Split
+from tti_eval.common import EmbeddingDefinition
+from tti_eval.common.base import SafeName
+from tti_eval.common.numpy_types import ClassArray, N2Array
+from tti_eval.constants import OUTPUT_PATH
+from tti_eval.dataset import DatasetProvider, Split
 
 from .reduction import REDUCTIONS, reduction_from_string
 

clip_eval/plotting/reduction.py → tti_eval/plotting/reduction.py

@@ -6,7 +6,7 @@
 from sklearn.decomposition import PCA
 from sklearn.manifold import TSNE
 
-from clip_eval.common import EmbeddingArray, EmbeddingDefinition, ReductionArray, Split
+from tti_eval.common import EmbeddingArray, EmbeddingDefinition, ReductionArray, Split
 
 
 class Reducer: