1

Author: [email protected]

app/exp_config.py

@@ -3,8 +3,8 @@
 TEST_PERCENT = 0.1
 MAX_SEQ_LENGTH = 512
 
-SYSTEM_CONFIG = {'per_device_train_batch_size': 16,
-                 'gradient_accumulation_steps': 8}
+SYSTEM_CONFIG = {'per_device_train_batch_size': 8,
+                 'gradient_accumulation_steps': 16}
 
 # logging_steps: logging when training roberta
 # the frequency of evaluating on the val set for roberta
@@ -14,4 +14,4 @@
 
 TRAIN_DEBUG_CONFIG = {'epoch': 1,
                       'logging_steps': 1,
-                      'eval_steps': 2}
+                      'eval_steps': 2}

app/run_pos_dep_constit_ner.sh

@@ -0,0 +1,14 @@
+# bash run_pos_dep_constit_ner.sh
+set -e
+
+# pre-train model
+bash train_on_cn_novel_dep.sh     '0' 1 0 0 15
+bash train_on_cn_novel_pos.sh     '0' 1 0 0 15
+bash train_on_cn_novel_constit.sh '0' 1 0 0 15
+bash train_on_cn_novel_ner.sh     '0' 1 0 0 15
+
+# non-pre-train model
+bash train_on_cn_novel_dep.sh     '0' 1 0 1 15
+bash train_on_cn_novel_pos.sh     '0' 1 0 1 15
+bash train_on_cn_novel_constit.sh '0' 1 0 1 15
+bash train_on_cn_novel_ner.sh     '0' 1 0 1 15

app/train_cn_roberta.py

@@ -61,7 +61,9 @@ def args_parse():
                                  'None',
                                  'likelihood_rank',
                                  'pos',
-                                 'dep'
+                                 'dep',
+                                 'constit',  # phrase structure tree, constituency tree,
+                                 'ner'
                                  ],
                         required=True)
     parser.add_argument('--is_change_apply_to_test', type=int, default=0)
@@ -95,6 +97,9 @@ def main():
         if x in semantic_change:
             is_change_apply_to_test = True
 
+    if 'pos' in semantic_change or 'dep' in semantic_change:
+        assert char_freq_ranges == [0]
+
     # read char frequencies
     char_freq_rank = {}
     with open(char_freq_txt_path, 'r') as f:
@@ -116,6 +121,23 @@ def main():
         classifier_name += '_no_pretrain'
 
     semantic_change_str = '_'.join(semantic_change)
+
+    # set save path
+    # save path
+    save_name = f'{dataset_name}_{classifier_name}_{semantic_change_str}' \
+                f'_{is_change_apply_to_train}_{is_change_apply_to_test}'
+    if is_debug:
+        save_name = save_name + '_debug.csv'
+    else:
+        save_name = save_name + '.csv'
+    save_path = os.path.join(save_dir, save_name)
+
+    if os.path.isfile(save_path):
+        print(f"=" * 78)
+        print(f"{save_path} exist. Skip Training!!!")
+        print(f"=" * 78)
+        return
+
     semantic_modifier = SemanticModifier(semantic_change, char_freq_rank=char_freq_rank)
 
     tokenizer = AutoTokenizer.from_pretrained(hugginface_model_id)
@@ -240,10 +262,6 @@ def main():
             shutil.rmtree(tmp_ckpts_dir)
             print(f"Remove temp dir {tmp_ckpts_dir} SUCCESS!!!!")
 
-    # save path
-    save_path = os.path.join(save_dir,
-                             f'{dataset_name}_{classifier_name}_{semantic_change_str}_{is_change_apply_to_train}'
-                             f'_{is_change_apply_to_test}.csv')
     exp_recorder.save_to_disk(save_path)
 
     # # Save model

app/train_on_cn_novel_constit.sh

@@ -0,0 +1,28 @@
+# bash train_on_cn_novel_constit.sh '0' 1 1 0 1
+
+char_freq_ranges=${1:-0}
+is_change_apply_to_test=${2:-1}
+is_debug=${3:-0}
+re_init_weights=${4:-0}
+repeat=${5:-1}
+is_change_apply_to_train=${6:-1}
+
+data_dir=../data/5billion
+save_dir=../result/
+dataset_name=cn_novel_5billion
+classifier_name=cn_roberta
+char_freq_txt_path=../data/sort_char.txt
+semantic_change='constit'
+
+python3.6 train_cn_roberta.py --classifier_name $classifier_name \
+                              --dataset_name $dataset_name \
+                              --data_dir $data_dir \
+                              --save_dir $save_dir \
+                              --char_freq_txt_path $char_freq_txt_path \
+                              --is_debug $is_debug \
+                              --repeat $repeat \
+                              --char_freq_ranges $char_freq_ranges \
+                              --semantic_change $semantic_change \
+                              --is_change_apply_to_test $is_change_apply_to_test \
+                              --re_init_weights $re_init_weights \
+                              --is_change_apply_to_train $is_change_apply_to_train

app/train_on_cn_novel_ner.sh

@@ -0,0 +1,28 @@
+# bash train_on_cn_novel_ner.sh '0' 1 1 0 1
+
+char_freq_ranges=${1:-0}
+is_change_apply_to_test=${2:-1}
+is_debug=${3:-0}
+re_init_weights=${4:-0}
+repeat=${5:-1}
+is_change_apply_to_train=${6:-1}
+
+data_dir=../data/5billion
+save_dir=../result/
+dataset_name=cn_novel_5billion
+classifier_name=cn_roberta
+char_freq_txt_path=../data/sort_char.txt
+semantic_change='ner'
+
+python3.6 train_cn_roberta.py --classifier_name $classifier_name \
+                              --dataset_name $dataset_name \
+                              --data_dir $data_dir \
+                              --save_dir $save_dir \
+                              --char_freq_txt_path $char_freq_txt_path \
+                              --is_debug $is_debug \
+                              --repeat $repeat \
+                              --char_freq_ranges $char_freq_ranges \
+                              --semantic_change $semantic_change \
+                              --is_change_apply_to_test $is_change_apply_to_test \
+                              --re_init_weights $re_init_weights \
+                              --is_change_apply_to_train $is_change_apply_to_train

core/semantic_modifier.py

@@ -1,19 +1,32 @@
+import os
 import ipdb
 import copy
 import random
 import numpy as np
+import glob
 import spacy
 from tqdm import tqdm
+import pickle
+import hashlib
+import benepar
 
 
 class SemanticModifier:
     def __init__(self, semantic_change, char_freq_rank=None):
         self.semantic_change = semantic_change
         self.char_freq_rank = char_freq_rank
         self.max_freq = max(self.char_freq_rank.values())
-        if 'pos' or 'dep' in semantic_change:
+        if 'pos' in semantic_change or \
+                'dep' in semantic_change or \
+                'constit' in semantic_change or \
+                'ner' in semantic_change:
             # self.spacy_parser = spacy.load("zh_core_web_sm")
             self.spacy_parser = spacy.load("zh_core_web_trf")
+            # if 'constit' in semantic_change:
+            benepar_model = 'benepar_zh2'
+            self.spacy_parser.add_pipe("benepar", config={"model": benepar_model})
+            print(f'Spacy add benepar pipe done! Model: {benepar_model}')
+            self.spacy_results = {}
         else:
             self.spacy_parser = None
 
@@ -89,21 +102,66 @@ def change_texts(self, texts, char_freq_range):
                         appear_set.add(x)
                 split_text = new_split_text
 
-            if 'pos' in self.semantic_change or 'dep' in self.semantic_change:
+            if 'pos' in self.semantic_change or \
+                    'dep' in self.semantic_change or \
+                    'constit' in self.semantic_change or \
+                    'ner' in self.semantic_change:
                 # from spacy.lang.zh.examples import sentences
                 # example_sentence = sentences[0]
-                parse_res = self.spacy_parser(text.replace(' ', ''))
-                new_text = []
-                for token in parse_res:
-                    if 'pos' in self.semantic_change:
-                        new_text.append(token.pos_)
-                    elif 'dep' in self.semantic_change:
-                        new_text.append(token.dep_)
+                to_parse_text = text.replace(' ', '')
+                model_name = self.spacy_parser.meta['name'] + '_' + self.spacy_parser.meta['lang']
+                text_md5 = hashlib.md5(f'{model_name}_{to_parse_text}'.encode()).hexdigest()
+
+                if text_md5 in self.spacy_results:
+                    split_text = self.spacy_results[text_md5]
+                else:
+                    text_pickle_path = f'../spacy_temp/{text_md5}.pkl'
+                    if os.path.isfile(text_pickle_path):
+                        parse_res = pickle.load(open(text_pickle_path, 'rb'))
+                    else:
+                        parse_res = self.spacy_parser(to_parse_text)
+                        pickle.dump(parse_res, open(text_pickle_path, 'wb'))
+                    new_text = []
+                    # Reference: https://spacy.io/usage/linguistic-features#dependency-parse
+
+                    if 'pos' in self.semantic_change or 'dep' in self.semantic_change:
+                        for token in parse_res:
+                            if 'pos' in self.semantic_change:
+                                new_text.append(token.pos_)
+                            elif 'dep' in self.semantic_change:
+                                new_text.append(token.dep_)
+                                new_text.append(str(token.idx))
+                                new_text.append(str(token.head.idx))
+                                # new_text.append(token.head.text)
+                            else:
+                                raise Exception
+                        new_text = ' '.join(new_text)
+                    elif 'constit' in self.semantic_change:
+                        new_text = []
+                        tokens = [str(x) for x in parse_res]
+                        tokens = ''.join(tokens)
+                        tokens_set = set(list(tokens))
+                        for sen in parse_res.sents:
+                            parse_string = sen._.parse_string
+                            new_text.append(parse_string)
+                        new_text = '<s>'.join(new_text)
+                        for token in tokens_set:
+                            new_text = new_text.replace(str(token), '')
+                        new_text = new_text.replace(' ', '')
+                    elif 'ner' in self.semantic_change:
+                        new_text = []
+                        for ent in parse_res.ents:
+                            new_text.append(ent.label_)
+                            new_text.append(str(ent.start_char))
+                            new_text.append(str(ent.end_char))
+                        new_text = ' '.join(new_text)
                     else:
                         raise Exception
-                split_text = new_text
+                    split_text = new_text
+                    self.spacy_results[text_md5] = split_text
                 # 这个pos/dep tag的数量和原本中文的数量是对不上的，因为会对中文做分词，所以会短一点
-            processed_texts.append(' '.join(split_text))
+
+            processed_texts.append(split_text)
 
         assert len(processed_texts) == len(texts)