kobert_sentiment_classification.py

# -*- coding: utf-8 -*-
"""koberNaver2myVersion.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1g0Uq1BjWHBh6-WDSaGLL4RwVijbPWE_0
"""

!pip install mxnet
!pip install gluonnlp pandas tqdm
!pip install sentencepiece
!pip install transformers==3
!pip install torch

!pip install git+https://git@github.com/SKTBrain/KoBERT.git@master

import torch
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import gluonnlp as nlp
import numpy as np
from tqdm import tqdm, tqdm_notebook

from kobert.utils import get_tokenizer
from kobert.pytorch_kobert import get_pytorch_kobert_model

from transformers import AdamW
from transformers.optimization import get_cosine_schedule_with_warmup

##GPU 사용 시
device = torch.device("cuda:0")
#device = torch.device("cpu")

bertmodel, vocab = get_pytorch_kobert_model()

!wget "https://drive.google.com/uc?export=download&id=1fhEBkjvl0g7EHYPjN-r8wefEUi_pZ7RZ" -O datatest.tsv
!wget "https://drive.google.com/uc?export=download&id=1rmjklQfdEPaDgpQu1UKi64iagytxrC92" -O datatrain.tsv

dataset_train = nlp.data.TSVDataset("datatrain.tsv")

dataset_test = nlp.data.TSVDataset("datatest.tsv")

tokenizer = get_tokenizer()
tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)

class BERTDataset(Dataset):
    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len,
                 pad, pair):
        transform = nlp.data.BERTSentenceTransform(
            bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)

        self.sentences = [transform([i[sent_idx]]) for i in dataset]
        self.labels = [np.int32(int(float(i[label_idx]))) for i in dataset]

    def __getitem__(self, i):
        return (self.sentences[i] + (self.labels[i], ))

    def __len__(self):
        return (len(self.labels))

## Setting parameters
max_len = 64
batch_size = 64
warmup_ratio = 0.1
num_epochs = 8
max_grad_norm = 1
log_interval = 200
learning_rate =  5e-5

print(dataset_train[:5])

data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)
data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)

#print(data_train[0])

train_dataloader = torch.utils.data.DataLoader(data_train, batch_size=batch_size, num_workers=5)
test_dataloader = torch.utils.data.DataLoader(data_test, batch_size=batch_size, num_workers=5)

class BERTClassifier(nn.Module):
    def __init__(self,
                 bert,
                 hidden_size = 768,
                 num_classes=2,
                 dr_rate=None,
                 params=None):
        super(BERTClassifier, self).__init__()
        self.bert = bert
        self.dr_rate = dr_rate
                 
        self.classifier = nn.Linear(hidden_size , num_classes)
        if dr_rate:
            self.dropout = nn.Dropout(p=dr_rate)
    
    def gen_attention_mask(self, token_ids, valid_length):
        attention_mask = torch.zeros_like(token_ids)
        for i, v in enumerate(valid_length):
            attention_mask[i][:v] = 1
        return attention_mask.float()

    def forward(self, token_ids, valid_length, segment_ids):
        attention_mask = self.gen_attention_mask(token_ids, valid_length)
        
        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device))
        if self.dr_rate:
            out = self.dropout(pooler)
        return self.classifier(out)

model = BERTClassifier(bertmodel,  dr_rate=0.5).to(device)

# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]

optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
loss_fn = nn.CrossEntropyLoss()

t_total = len(train_dataloader) * num_epochs
warmup_step = int(t_total * warmup_ratio)

scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=t_total)

def calc_accuracy(X,Y):
    max_vals, max_indices = torch.max(X, 1)
    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]
    return train_acc

for e in range(num_epochs):
    train_acc = 0.0
    test_acc = 0.0
    model.train()
    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(train_dataloader)):
        optimizer.zero_grad()
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length= valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        loss = loss_fn(out, label)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
        optimizer.step()
        scheduler.step()  # Update learning rate schedule
        train_acc += calc_accuracy(out, label)
        if batch_id % log_interval == 0:
            print("epoch {} batch id {} loss {} train acc {}".format(e+1, batch_id+1, loss.data.cpu().numpy(), train_acc / (batch_id+1)))
    print("epoch {} train acc {}".format(e+1, train_acc / (batch_id+1)))
    model.eval()
    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(test_dataloader)):
        token_ids = token_ids.long().to(device)
        segment_ids = segment_ids.long().to(device)
        valid_length= valid_length
        label = label.long().to(device)
        out = model(token_ids, valid_length, segment_ids)
        loss = loss_fn(out, label)
        loss.backward()
        test_acc += calc_accuracy(out, label)
    print("epoch {} test acc {}".format(e+1, test_acc / (batch_id+1)))
    print("epoch {} test loss {}".format(e+1,loss.data.cpu().numpy() ))

import pandas as pd

def getSentimentValue(comment, tok, max_len, batch_size, device):
  commnetslist = []
  emo_list = []
  for c in comment:
    commnetslist.append( [c, 5] )
  pdData = pd.DataFrame( commnetslist, columns = [['댓글', '감성']] )
  pdData = pdData.values
  test_set = BERTDataset(pdData, 0, 1, tok, max_len, True, False) 
  test_input = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=5)
  for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(test_input):#enumerate(tqdm_notebook(test_input)):
    token_ids = token_ids.long().to(device)
    segment_ids = segment_ids.long().to(device)
    valid_length= valid_length 
    out = model(token_ids, valid_length, segment_ids)
    #print("out one:",out)
  
    for e in out:
      #print(e)
      if e[0]>e[1]: # 부정
        value = 0
      else: #긍정
        value = 1
      emo_list.append(value)

  return emo_list

from google.colab import drive
import json

tlist = [] # database table name list
for i in range(1,6):
    t = "daum"
    tlist.append(t+"after"+str(i))
    tlist.append(t+"before"+str(i))
for i in range(1,6):
    t = "naver"
    tlist.append(t+"after"+str(i))
    tlist.append(t+"before"+str(i))

drive.mount("/content/drive")
path = "/content/drive/My Drive/ColabNotebooks/jsonComment/" # comment  json file directory path
for tablename in tlist:
    
    with open(path+tablename+'-dict.json', encoding="utf-8") as json_file:
        data = json.load(json_file) # load json file
    
    for date in data.keys(): 
        #print(">>",date,"/",tablename)
        
        day_article = data[date]
        for article in day_article.keys():
            #if len(day_article[article][0])>100: print("카운팅:",len(day_article[article][0]))
            comments = day_article[article][0]
            if len(comments)==1: continue
            
            comments.remove("")
            
            emotion = getSentimentValue(comments, tok, max_len, batch_size, device)
            
            data[date][article][-1].extend(emotion)
            #data[date][article][-1].append(emotion) # dictionary value append sentiment
    print(tablename, "--comment save")
    result_path = "/content/drive/My Drive/ColabNotebooks/predictComment/" #  result save json path
    #drive.mount("/Colab Notebooks/predictComment")
    with open(result_path+"finish-"+tablename+"-dict.json", "w", encoding="utf-8") as json_file:
        json.dump(data, json_file, indent="\t",ensure_ascii = False)