fix: replace unidecode with anyascii

Author: ghisvail

nlstruct/data_utils.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 import regex
-from unidecode import unidecode
+from anyascii import anyascii
 
 import functools
 import textwrap
@@ -369,10 +369,10 @@ def regex_multisub_with_spans(patterns, replacements, text, deltas=None, return_
 
 def run_unidecode(text, return_deltas=False):
     if not return_deltas:
-        return unidecode(text), None
+        return anyascii(text), None
     begins, ends, deltas = [], [], []
     new_text = ""
-    for i, (old_char, new_char) in enumerate((char, unidecode(char)) for char in text):
+    for i, (old_char, new_char) in enumerate((char, anyascii(char)) for char in text):
         if len(old_char) != len(new_char):
             begins.append(i)
             ends.append(i + 1)

nlstruct/datasets/base.py

@@ -12,7 +12,7 @@
 import torch
 from sklearn.datasets._base import _sha256
 from tqdm import tqdm
-from unidecode import unidecode
+from anyascii import anyascii
 import warnings
 
 from ..data_utils import mix, loop
@@ -101,7 +101,7 @@ def __init__(self,
                     if synonym_preprocess_fn is not None:
                         synonym = synonym_preprocess_fn(synonym)
                     if do_unidecode:
-                        synonym = unidecode(synonym)
+                        synonym = anyascii(synonym)
                     if subs:
                         for pattern, replacement in subs:
                             synonym = re.sub(pattern, replacement, synonym)

setup.py

@@ -27,8 +27,8 @@
             "fire",
             "torch>=1.0.0",
             "numpy",
-            "unidecode>=1.1.2",
             "einops>=0.3.0",
+            "anyascii>=0.3.2",
             "transformers>=4.3.0",
             "tqdm>=4.56.0",
             "scikit-learn",