[BUGFIX] Fix vocab determinism in py35 (#1166) (#1167)

Author: eric-haibin-lin

src/gluonnlp/vocab/vocab.py

@@ -23,6 +23,7 @@
 import json
 import uuid
 import warnings
+import sys
 from typing import Dict, Hashable, List, Optional
 
 from mxnet import nd
@@ -37,6 +38,9 @@
 _DEPR_EOS = object()
 
 
+def _is_py35():
+    return sys.version_info[0] == 3 and sys.version_info[1] == 5
+
 class Vocab:
     """Indexing and embedding attachment for text tokens.
 
@@ -222,7 +226,10 @@ def __init__(self, counter: Optional[Counter] = None, max_size: Optional[int] =
 
         # Handle special tokens
         special_tokens = []
-        for special_token_name, special_token in kwargs.items():
+        special_iter = kwargs.items()
+        if _is_py35():
+            special_iter = sorted(special_iter)
+        for special_token_name, special_token in special_iter:
             # Test if kwarg specifies a special token
             if not special_token_name.endswith('_token'):
                 raise ValueError('{} is invalid. Only keyword arguments '

tests/unittest/test_vocab_embed.py

@@ -1457,3 +1457,12 @@ def test_vocab_remapped_unknown_token_idx(unknown_token, padding_token, eos_toke
 
     v = Vocab(token_to_idx={unknown_token: 1})
     assert v['UNKNOWNWORD'] == 1
+
+def test_vocab_consistency():
+    v0 = nlp.Vocab({'a': 1}, mask_token='[MASK]', sep_token='[SEP]',
+                   cls_token='[CLS]')
+    v1 = nlp.Vocab({'a': 1}, mask_token='[MASK]', sep_token='[SEP]',
+                   cls_token='[CLS]')
+    assert v0[v0.mask_token] == v1[v1.mask_token]
+    assert v0[v0.sep_token] == v1[v1.sep_token]
+    assert v0[v0.cls_token] == v1[v1.cls_token]