prepare for differentiable topk use for another lib

lucidrains · Aug 18, 2023 · 860081f · 860081f · johndpope · Aug 18, 2023
1 parent 8648bed
commit 860081f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/colt5_attention/topk.py b/colt5_attention/topk.py
@@ -13,11 +13,17 @@ def topk(
     eps_init = None,
     eps_decay = 1.,
     mask = None,
-    fused = False
+    fused = False,
+    non_differentiable = False
 ):
     """
     differentiable top-k on last dimension
     """
+
+    if non_differentiable:
+        values, indices = torch.topk(x, k = k, dim = -1)
+        return TopkReturn(values, indices, None, None)
+
     assert coor_descent_k_ratio >= 1.
     assert k > 0
 

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'CoLT5-attention',
   packages = find_packages(),
-  version = '0.10.14',
+  version = '0.10.15',
   license='MIT',
   description = 'Conditionally Routed Attention',
   long_description_content_type = 'text/markdown',