Fixes (#25)

- version
- pyupgrade
- deep=True by default
- fix test
- Add size test

.pre-commit-config.yaml

@@ -11,4 +11,9 @@ repos:
 - repo: https://github.com/timothycrosley/isort
  rev: 5.4.2
  hooks:
- - id: isort
+ - id: isort
+- repo: https://github.com/asottile/pyupgrade
+ rev: v2.7.2
+ hooks:
+ - id: pyupgrade
+ args: [--py36-plus]

setup.py

@@ -14,10 +14,14 @@
 with (source_root / "requirements_test.txt").open(encoding="utf8") as f:
  test_requirements = f.readlines()
 
+__version__ = None
+with (source_root / "src/compressio/version.py").open(encoding="utf8") as f:
+ exec(f.read())
+
 
 setup(
  name="compressio",
- version="0.1.3",
+ version=__version__,
  url="https://github.com/dylan-profiler/compressio",
  description="compressio",
  author="Ian Eaves, Simon Brugman",

src/compressio/__init__.py

@@ -12,6 +12,8 @@
  SparseCompressor,
 )
 
+from .version import __version__
+
 __all__ = [
  "Compress",
  "storage_size",
@@ -22,4 +24,5 @@
  "DefaultCompressor",
  "SparseCompressor",
  "type_compressions",
+ "__version__",
 ]

src/compressio/compression_algorithms/type_compressions.py

@@ -54,7 +54,7 @@ def compress_sparse_missing(series: pd.Series) -> pd.Series:
  series[series.notnull()], dtype=test_dtype, fill_value=fill_value
  )
  )
- if new_series.memory_usage() < series.memory_usage():
+ if new_series.memory_usage(deep=True) < series.memory_usage(deep=True):
  return new_series
 
  return series
@@ -127,7 +127,7 @@ def compress_complex(series: pd.Series) -> pd.Series:
 def compress_object(series: pd.Series) -> pd.Series:
  try:
  new_series = series.astype("category")
- if new_series.memory_usage() < series.memory_usage():
+ if new_series.memory_usage(deep=True) < series.memory_usage(deep=True):
  return new_series
  except: # noqa
  pass
@@ -137,7 +137,7 @@ def compress_object(series: pd.Series) -> pd.Series:
 def compress_datetime(series: pd.Series) -> pd.Series:
  try:
  new_series = series.astype("category")
- if new_series.memory_usage() < series.memory_usage():
+ if new_series.memory_usage(deep=True) < series.memory_usage(deep=True):
  return new_series
  except: # noqa
  pass

src/compressio/diagnostics.py

@@ -1,4 +1,3 @@
-"""Work in progress"""
 from functools import singledispatch
 
 import pandas as pd
@@ -11,18 +10,18 @@
 
 
 @singledispatch
-def storage_size(data: pdT, deep=True) -> Quantity:
+def storage_size(data: pdT) -> Quantity:
  raise TypeError(f"Can't compute memory size of objects with type {type(data)}")
 
 
 @storage_size.register(pd.Series) # type: ignore
-def _(data: pd.Series, deep) -> Quantity:
- return Quantity(value=data.memory_usage(deep=deep), units="byte")
+def _(data: pd.Series) -> Quantity:
+ return Quantity(value=data.memory_usage(deep=True), units="byte")
 
 
 @storage_size.register(pd.DataFrame) # type: ignore
-def _(data: pd.DataFrame, deep=False) -> Quantity:
- return Quantity(value=data.memory_usage(deep=deep).sum(), units="byte")
+def _(data: pd.DataFrame) -> Quantity:
+ return Quantity(value=data.memory_usage(deep=True).sum(), units="byte")
 
 
 @singledispatch
@@ -65,19 +64,15 @@ def _(
  compress_report(data[column], typeset, compressor, with_inference, units)
 
 
-def savings(
- original_data: pdT, new_data: pdT, units="megabyte", deep=False
-) -> Quantity:
- original_size = storage_size(original_data, deep)
- new_size = storage_size(new_data, deep)
+def savings(original_data: pdT, new_data: pdT, units: str = "megabyte",) -> Quantity:
+ original_size = storage_size(original_data)
+ new_size = storage_size(new_data)
  return (original_size - new_size).to(units)
 
 
-def savings_report(
- original_data: pdT, new_data: pdT, units="megabyte", deep=False
-) -> None:
- original_size = storage_size(original_data, deep).to(units)
- new_size = storage_size(new_data, deep).to(units)
+def savings_report(original_data: pdT, new_data: pdT, units: str = "megabyte",) -> None:
+ original_size = storage_size(original_data).to(units)
+ new_size = storage_size(new_data).to(units)
  reduction = original_size - new_size
  print(f"Original size: {original_size}")
  print(f"Compressed size: {new_size}")

src/compressio/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.4"

tests/test_compression_algorithms.py

@@ -2,7 +2,6 @@
 import pandas as pd
 import pytest
 from pandas.testing import assert_series_equal
-from visions import StandardSet
 
 from compressio.compression_algorithms import (
  compress_complex,

tests/test_compression_func_default.py

@@ -1,7 +1,6 @@
 import numpy as np
 import pandas as pd
 import pytest
-import visions
 from pandas.testing import assert_series_equal
 from visions import StandardSet
 
@@ -17,9 +16,9 @@
  (
  pd.Series([10.0, 100.0, np.iinfo(np.int16).max * 1.0], dtype=np.float64),
  np.float64,
- np.int16,
+ "int16",
  ),
- (pd.Series([np.nan, 1], dtype=np.float64), np.float64, pd.Int8Dtype),
+ (pd.Series([np.nan, 1], dtype=np.float64), np.float64, "Int8"),
  (
  pd.Series([True, False, None, None, None, None, True, False] * 1000),
  np.object,
@@ -35,8 +34,7 @@ def test_compress_series(series, before, expected):
  compressor=DefaultCompressor(),
  with_inference=True,
  )
- assert compressed_series.dtype == expected or isinstance(
- compressed_series.dtype, expected
- )
+
+ assert str(compressed_series.dtype) == expected
 
  assert_series_equal(series, compressed_series, check_dtype=False)

tests/test_size.py

@@ -0,0 +1,10 @@
+import pandas as pd
+from compressio import storage_size, savings
+
+
+def test_size_formatting():
+ series1 = pd.Series([True, False] * 10000, dtype=bool)
+ series2 = pd.Series([True, False] * 5000, dtype=bool)
+ assert series1.memory_usage(deep=True) == 20128
+ assert str(storage_size(series1)) == "20128 byte"
+ assert str(savings(series1, series2)) == "0.01 megabyte"