(연습)Multilingual-BERT를 이용해 크롤링한 영화 댓글 감성 분류 하기 #COLAB

#Crawling test(html 구조 파악하기)
import requests
from bs4 import BeautifulSoup 
 
test_url = "https://movie.naver.com/movie/bi/mi/pointWriteFormList.nhn?code=136990&type=after&page=1"
resp = requests.get(test_url)
html = BeautifulSoup(resp.content, 'html.parser')
html
 

 

 

 

 

 

 

 

 

html 결과

 

 

 

 

 

 

 

 

 

 

#영화 군도
gundo_url = 'https://movie.naver.com/movie/bi/mi/pointWriteFormList.nhn?code=99752&type=after&page='
gundo_page = 2000
 
gundo_texts = []
gundo_scores = []
 
def organize_data(x):
  a = x.find('div', {'class':'score_result'})
  lists = a.findAll('li')
  for li in lists:
    
    
 
    review_text = li.find('p').getText().strip()
    score = li.find('em').getText()
    
    
    
    gundo_texts.append(review_text)
    gundo_scores.append(score)
 
 
for i in range(gundo_page):
  requested = requests.get(gundo_url+str(i+1))
 
 
gundo_reviews = []
for text in gundo_texts:
  if '관람객' == text[0:3]:
    a = text[3:]
    a = a.replace('\n','')
    a = a.replace('\t','')
    a = a.replace('\r','')
        
 
    gundo_reviews.append(a)
 
  else:
    gundo_reviews.append(text)
 
 
 
num_scores = []
for i in gundo_scores:
  num_scores.append(int(i))
 
 
#부정 = 0, 긍정 = 1
gundo_labels = [] 
for i in num_scores:
  if i <= 7:
    gundo_labels.append(0)
  else:
    gundo_labels.append(1)
 
 
 
#영화 인크레더블2
incre_url = 'https://movie.naver.com/movie/bi/mi/pointWriteFormList.nhn?code=136990&type=after&page='
incre_page = 715
 
incre_texts = []
incre_scores = []
 
def organize_data(x):
  a = x.find('div', {'class':'score_result'})
  lists = a.findAll('li')
  for li in lists:
    
    
 
    review_text = li.find('p').getText().strip()
    score = li.find('em').getText()
    
    
    
    incre_texts.append(review_text)
    incre_scores.append(score)
 
 
for i in range(incre_page):
  requested = requests.get(incre_url+str(i+1))
  html = BeautifulSoup(requested.content, 'html.parser')
  organize_data(html)
 
incre_reviews = []
for text in incre_texts:
  if '관람객' == text[0:3]:
    b = text[3:]
    b = b.replace('\n','')
    b = b.replace('\t','')
    b = b.replace('\r','')
        
 
    incre_reviews.append(b)
 
  else:
    incre_reviews.append(text)
 
 
num_scores = []
for i in incre_scores:
  num_scores.append(int(i))
 
incre_labels = []
for i in num_scores:
  if i <= 7:
    incre_labels.append(0)
  else:
    incre_labels.append(1)
 
 
#어차피 뒤에서 shuffle 해줄거니까 일단 그냥 붙이자
reviews = gundo_reviews + incre_reviews
labels = gundo_labels + incre_labels
 
 
import numpy as np
 
labels = np.array(labels)

 

 

 

 

총 데이터 크기 27150

 

 

 

 

 

reviews들을 모아온 결과

 

 

 

 

 

 

 

 

 

 

 

!pip3 install http://download.pytorch.org/whl/cu80/torch-0.3.0.post4-cp36-cp36m-linux_x86_64.whl
!pip install pytorch-transformers
!pip install  torch==1.1.0
!pip install transformers==2.2.2
!pip install sentencepiece==0.1.82
 
import torch
import tensorflow as tf
import numpy as np
import pandas as pd
from transformers import BertTokenizer
from transformers import BertForSequenceClassification, AdamW, BertConfig
from torch.utils.data import TensorDataset, DataLoader, RandomSampler
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
 
 
 
review_train, review_test, label_train, label_test = train_test_split(reviews, labels, random_state=100, test_size=0.3, shuffle = True )
review_train_sent = ['[CLS] ' + str(review) + ' [SEP]' for review in review_train]
review_test_sent = ['[CLS] ' + str(review) + ' [SEP]' for review in review_test]
 
 
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)
tokenized_review_train = [tokenizer.tokenize(sent) for sent in review_train_sent]
tokenized_review_test = [tokenizer.tokenize(sent) for sent in review_test_sent]
 
train_review_token_id = [tokenizer.convert_tokens_to_ids(sent) for sent in tokenized_review_train]
test_review_token_id = [tokenizer.convert_tokens_to_ids(sent) for sent in tokenized_review_test]
 
 
max_len = 128
train_review_token_id = pad_sequences(train_review_token_id, maxlen=max_len, dtype='long', padding='post',truncating='post')
test_review_token_id = pad_sequences(test_review_token_id, maxlen=max_len, dtype='long', padding='post',truncating='post')
 
 
mask_train = []
attention_mask_train = []
for sent in train_review_token_id:
  mask_train = [float(i>0) for i in sent]
  attention_mask_train.append(mask_train)
 
 
mask_test = []
attention_mask_test = []
for sent in test_review_token_id:
  mask_test = [float(i>0) for i in sent]
  attention_mask_test.append(mask_test)
 
train_review_token_id = torch.tensor(train_review_token_id)
attention_mask_train = torch.tensor(attention_mask_train)
label_train = torch.tensor(label_train)
 
 
test_review_token_id = torch.tensor(test_review_token_id)
attention_mask_test = torch.tensor(attention_mask_test)
label_test = torch.tensor(label_test)
 
 
 
 

 

 

 

 

 

 

텐서의 생김새

 

 

 

 

 

 

#batch size
batch_size = 32
 
 
#데이터들 묶어주고 데이터로더 형식으로 변환해주기
train_data = TensorDataset(train_review_token_id, attention_mask_train, label_train)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, batch_size=batch_size, sampler=train_sampler)
 
test_data = TensorDataset(test_review_token_id, attention_mask_test,   label_test)
test_dataloader = DataLoader(test_data, batch_size=batch_size)
 
 
#GPU를 사용하기 위한 device 설정
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
#모델설정
model = BertForSequenceClassification.from_pretrained("bert-base-multilingual-cased", num_labels=2)
model.to(device)
 
# 옵티마이저 설정
optimizer = AdamW(model.parameters(),
                  lr = 2e-5, # 학습률
                  eps = 1e-8 # 0으로 나누는 것을 방지하기 위한 epsilon 값
                )
 
 
# 에폭수
epochs = 3
 
# 총 훈련 스텝 : 배치반복 횟수 * 에폭
total_steps = len(train_dataloader) * epochs
 
 
 
 
# 정확도 계산 함수
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
 
    return np.sum(pred_flat == labels_flat) / len(labels_flat)
 
 
 
 
 
 

 

 

 

 

 

# training 에폭만큼 반복
for epoch_i in range(0, epochs):
    
    # ========================================m
    #               Training
    # ========================================
    
    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
    print('Training...')
 
 
    # 로스 초기화
    total_loss = 0
 
    # 훈련모드로 변경
    model.train()
        
    # 데이터로더에서 배치만큼 반복하여 가져옴
    for step, batch in enumerate(train_dataloader):
        
        
        # 배치를 GPU에 넣음
        batch = tuple(t.to(device) for t in batch)
        
        # 배치에서 데이터 추출
        b_input_ids, b_input_mask, b_labels = batch
 
        # Forward 수행                
        outputs = model(b_input_ids, 
                        token_type_ids=None, 
                        attention_mask=b_input_mask, 
                        labels = b_labels
                        )
        
        
        # 로스 구함
        loss = outputs[0]
          # 총 로스 계산
        total_loss += loss.item()
 
        # Backward 수행으로 그래디언트 계산
        loss.backward()
 
 
        # 그래디언트를 통해 가중치 파라미터 업데이트
        optimizer.step()
 
        # 그래디언트 초기화
        model.zero_grad() #we need to set the gradients to zero before starting to do backpropragation because PyTorch accumulates the gradients on subsequent backward passes
 
    # 평균 로스 계산
    avg_train_loss = total_loss / len(train_dataloader)
    print(avg_train_loss)            
 
        
 
 
       
 
    # ========================================
    #               test
    # ========================================
 
print("")
print("Running test...")
 
accuracies = []
predictions = []
 
    # 평가모드로 변경
model.eval()
 
    # 변수 초기화
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
 
    # 데이터로더에서 배치만큼 반복하여 가져옴
for batch in test_dataloader:
 
        # 배치를 GPU에 넣음
  batch = tuple(t.to(device) for t in batch)
        
        # 배치에서 데이터 추출
  b_input_ids, b_input_mask, b_labels = batch
        
        # 그래디언트 계산 안함
  with torch.no_grad():     
            # Forward 수행
        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
             # 로짓 구함
        logits = outputs[0]
 
        # CPU로 데이터 이동
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()
 
        true_labels.append(label_ids)
        predictions.append(logits)
 
 
        
        # 출력 로짓과 라벨을 비교하여 정확도 계산
        tmp_eval_accuracy = flat_accuracy(logits, label_ids)
        eval_accuracy += tmp_eval_accuracy
        nb_eval_steps += 1
        print('Accuracy: {0:.5f}'.format(eval_accuracy/nb_eval_steps))
        accuracies.append(eval_accuracy/nb_eval_steps)
 
 
print("complete!")
        
       

 

 

eval_accuracy = 84%

 

 

 

 

#새로운 영화 테스트
 
def convert_input_data(sentences):
  tokenized_text = [tokenizer.tokenize(sent) for sent in sentences]
  Max_len = 128
  input_ids = [tokenizer.convert_tokens_to_ids(sent) for sent in tokenized_text]
  input_ids = pad_sequences(input_ids, maxlen = Max_len, padding = 'post', truncating='post',dtype = 'long')
  attention_mask = []
  for seq in input_ids:
    seq_mask = [float(i>0) for i in seq]
    attention_mask.append(seq_mask)
 
  inputs = torch.tensor(input_ids)
  masks = torch.tensor(attention_mask)
 
  return inputs, masks
 
 
 
def test_sentences(sentences):
  model.eval()
  inputs, masks = convert_input_data(sentences)
  b_input_ids = inputs.to(device)
  b_input_masks = masks.to(device)
  with torch.no_grad():
    outputs = model(b_input_ids,
                    token_type_ids = None,
                    attention_mask = b_input_masks)
    logits = outputs[0]
    logits = logits.detach().cpu().numpy()
    return logits
 
 
 
 
 
 
 
 
 
 

 

 

 

 

일단은 분류를 어느정도 해내는 것 같다.