fixed bug (hyphen at end of char class) in regexes.py

totalgood · Sep 3, 2018 · fd112fe · fd112fe
1 parent a138a73
commit fd112fe
Show file tree

Hide file tree

Showing 58 changed files with 19 additions and 9 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.gitignore b/.gitignore
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
diff --git a/.travis.yml b/.travis.yml
diff --git a/AUTHORS.rst b/AUTHORS.rst
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
diff --git a/CHANGES.rst b/CHANGES.rst
diff --git a/LICENSE.txt b/LICENSE.txt
diff --git a/README.rst b/README.rst
diff --git a/docs/Makefile b/docs/Makefile
diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore
diff --git a/docs/authors.rst b/docs/authors.rst
diff --git a/docs/changelog.rst b/docs/changelog.rst
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/index.rst b/docs/index.rst
diff --git a/docs/license.rst b/docs/license.rst
diff --git a/pytest.ini b/pytest.ini
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.cfg b/setup.cfg
diff --git a/setup.py b/setup.py
diff --git a/src/pugnlp/__init__.py b/src/pugnlp/__init__.py
diff --git a/src/pugnlp/charlist.py b/src/pugnlp/charlist.py
diff --git a/src/pugnlp/constants.py b/src/pugnlp/constants.py
diff --git a/src/pugnlp/data.py b/src/pugnlp/data.py
diff --git a/src/pugnlp/data/ascii-equivalents.csv b/src/pugnlp/data/ascii-equivalents.csv
diff --git a/src/pugnlp/data/ascii-equivalents.ods b/src/pugnlp/data/ascii-equivalents.ods
diff --git a/src/pugnlp/data/emojione-activities.txt b/src/pugnlp/data/emojione-activities.txt
diff --git a/src/pugnlp/data/emojione-animals-and-nature.txt b/src/pugnlp/data/emojione-animals-and-nature.txt
diff --git a/src/pugnlp/data/emojione-flags.txt b/src/pugnlp/data/emojione-flags.txt
diff --git a/src/pugnlp/data/emojione-food-and-drink.txt b/src/pugnlp/data/emojione-food-and-drink.txt
diff --git a/src/pugnlp/data/emojione-objects.txt b/src/pugnlp/data/emojione-objects.txt
diff --git a/src/pugnlp/data/emojione-smilies.txt b/src/pugnlp/data/emojione-smilies.txt
diff --git a/src/pugnlp/data/emojione-symbols.txt b/src/pugnlp/data/emojione-symbols.txt
diff --git a/src/pugnlp/data/emojione-travel-and-places.txt b/src/pugnlp/data/emojione-travel-and-places.txt
diff --git a/src/pugnlp/data/emoticons-from-wikipedia.csv b/src/pugnlp/data/emoticons-from-wikipedia.csv
diff --git a/src/pugnlp/data/emoticons-from-wikipedia.ods b/src/pugnlp/data/emoticons-from-wikipedia.ods
diff --git a/src/pugnlp/data/goodreads-omniscient-books.txt b/src/pugnlp/data/goodreads-omniscient-books.txt
diff --git a/src/pugnlp/data/tlds-from-iana.csv b/src/pugnlp/data/tlds-from-iana.csv
diff --git a/src/pugnlp/data/uri-schemes.csv b/src/pugnlp/data/uri-schemes.csv
diff --git a/src/pugnlp/data/uri-schemes.xhtml.csv b/src/pugnlp/data/uri-schemes.xhtml.csv
diff --git a/src/pugnlp/data/wsj_pugnlp.detector_morse.Detector.json.gz b/src/pugnlp/data/wsj_pugnlp.detector_morse.Detector.json.gz
diff --git a/src/pugnlp/detector_morse.py b/src/pugnlp/detector_morse.py
diff --git a/src/pugnlp/futil.py b/src/pugnlp/futil.py
@@ -369,7 +369,7 @@ def touch_p(filepath, times=None, mkdir=True):
 def sudo_yield_file_lines(file_path='/etc/NetworkManager/system-connections/*'):
     r"""Cat a file iterating/yielding one line at a time,
 
-    shell will exeucte: `sudo cat $file_path` so if your shell doesn't have sudo or cat, no joy
+    shell will execute: `sudo cat $file_path` so if your shell doesn't have sudo or cat, no joy
     Input:
                     file_path(str): glob stars are fine
 

diff --git a/src/pugnlp/penn_treebank_tokenizer.py b/src/pugnlp/penn_treebank_tokenizer.py
diff --git a/src/pugnlp/plots.py b/src/pugnlp/plots.py
diff --git a/src/pugnlp/regexes.py b/src/pugnlp/regexes.py
@@ -156,18 +156,18 @@
 nonphrase = re.compile(r"[^-\w\s/&']")
 parenthetical_time = re.compile(r'([^(]*)\(\s*(\d+)\s*(?:min)?\s*\)([^(]*)', re.IGNORECASE)
 
-fqdn = r'(\b[a-zA-Z0-9-.]+\b([.]' + r'|'.join(constants.tld_iana) + r'\b)\b)'  # noqa
-fqdn_popular = r'(\b[a-zA-Z0-9-.]+\b([.]' + r'|'.join(constants.tld_popular) + r'\b)\b)'
-username = r'(\b[a-zA-Z0-9-.!#$%&*+-/=?^_`{|}~]+\b)'
+fqdn = r'(\b[-.a-zA-Z0-9]+\b([.]' + r'|'.join(constants.tld_iana) + r'\b)\b)'  # noqa
+fqdn_popular = r'(\b[-.a-zA-Z0-9]+\b([.]' + r'|'.join(constants.tld_popular) + r'\b)\b)'
+username = r'(\b[-.a-zA-Z0-9!#$%&*+/=?^_`{|}~]+\b)'
 
 email = re.compile(r'(\b' + username + r'\b@\b' + fqdn + r'\b)')
 email_popular = re.compile(r'(\b' + username + r'\b@\b' + fqdn_popular + r'\b)')
 
 # TODO: unmatched surrounding symbols are accepted/consumed, likewise for multiple dots/ats
 at = r'(([-@="_(\[{\|\s]+(at|At|AT)[-@="_)\]\}\|\s]+)|[@])'
 dot = r'(([-.="_(\[{\|\s]+(dot|dt|Dot|DOT)[-.="_)\]\}\|\s]+)|[.])'
-fqdn_obfuscated = r'(\b(([a-zA-Z0-9-]+' + dot + r'){1,7})(' + r'|'.join(constants.tld_iana) + r')\b)'
-fqdn_popular_obfuscated = r'(\b(([a-zA-Z0-9-]+' + dot + r'){1,7})(' + r'|'.join(constants.tld_popular) + r')\b)'
+fqdn_obfuscated = r'(\b(([-a-zA-Z0-9]+' + dot + r'){1,7})(' + r'|'.join(constants.tld_iana) + r')\b)'
+fqdn_popular_obfuscated = r'(\b(([-a-zA-Z0-9]+' + dot + r'){1,7})(' + r'|'.join(constants.tld_popular) + r')\b)'
 username_obfuscated = r'(([a-zA-Z0-9!#$%&*+/?^`~]+' + dot + r'?){1,7})'
 email_obfuscated = re.compile(r'(\b' + username_obfuscated + at + fqdn_obfuscated + r'\b)')
 email_popular_obfuscated = re.compile(r'(\b' + username_obfuscated + at + fqdn_popular_obfuscated + r'\b)')

diff --git a/src/pugnlp/scripts/__init__.py b/src/pugnlp/scripts/__init__.py
diff --git a/src/pugnlp/scripts/bon_lsi.py b/src/pugnlp/scripts/bon_lsi.py
diff --git a/src/pugnlp/segmentation.py b/src/pugnlp/segmentation.py
diff --git a/src/pugnlp/skeleton.py b/src/pugnlp/skeleton.py
diff --git a/src/pugnlp/stats.py b/src/pugnlp/stats.py
diff --git a/src/pugnlp/tutil.py b/src/pugnlp/tutil.py
diff --git a/src/pugnlp/util.py b/src/pugnlp/util.py
@@ -100,18 +100,28 @@ def remove_invalid_chars(str_or_seq, valid_regex=r'\w'):
     return seq[0] if isinstance(str_or_seq, str) else seq
 
 
-def clean_columns(columns, valid_regex=r'\w', lower=True):
+def clean_columns(columns, valid_regex=r'\w', lower=True, max_len=32):
+    """ Ensure all column name strings are valid python variable/attribute names 
+
+    >>> df = pd.DataFrame(np.zeros((2, 3)), columns=['WAT??', "Don't do th!s, way too long. ya-think????", 'ok-this123.456'])
+    >>> df.columns = clean_columns(df.columns, max_len=12)
+    >>> df.head()
+       wat  dont_do_ths_  okthis123456
+    0  0.0           0.0           0.0
+    1  0.0           0.0           0.0
+    """
     rettype = None
     if isinstance(columns, str):
         rettype = type(columns)
         columns = [columns]
 
     columns = [c.strip() for c in columns]
-    # # unneccessary because these are invalid characters removed below
+    # # unnecessary because these are invalid characters removed below
     # columns = [(c[1:-1] if c[0] in '\'"' and c[-1] == c[0] else c) for c in columns]
     # columns = [(c[1:-1] if c[0] in '{([<' and c[-1] in '})]>' else c) for c in columns]
-    columns = [re.sub('\s', '_', c).lower() for c in columns]
+    columns = [re.sub('\s+', '_', c).lower() for c in columns]
     columns = remove_invalid_chars(columns, valid_regex=r'\w')
+    columns = [c[:max_len] for c in columns]
     columns = np.array(columns) if rettype is None else rettype(columns[0])
     return columns
 

diff --git a/test-requirements.txt b/test-requirements.txt
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py
diff --git a/tests/travis_install.sh b/tests/travis_install.sh
diff --git a/tox.ini b/tox.ini