How do I test BERT with my own input instead of using a database? - bert-language-model

Question

So I am new to BERT and language classification and I wrote this code following an online course, but they use a database in order to test and train the model:
# imports and setup for BERT
!pip install transformers
import os
import gdown
import torch
import numpy as np
import transformers
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import pad_sequences
from transformers import BertTokenizer
from transformers import get_linear_schedule_with_warmup
from transformers import BertForSequenceClassification, AdamW, BertConfig
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
%matplotlib inline
# gdown.download('https://drive.google.com/uc?id=1q4U2gVY9tWEPdT6W-pdQpKmo152QqWLE', 'finance_train.csv', True)
# gdown.download('https://drive.google.com/uc?id=1nIBqAsItwVEGVayYTgvybz7HeK0asom0', 'finance_test.csv', True)
!wget 'https://storage.googleapis.com/inspirit-ai-data-bucket-1/Data/AI%20Scholars/Sessions%206%20-%2010%20(Projects)/Project%20-%20NLP%2BFinance/finance_test.csv'
!wget 'https://storage.googleapis.com/inspirit-ai-data-bucket-1/Data/AI%20Scholars/Sessions%206%20-%2010%20(Projects)/Project%20-%20NLP%2BFinance/finance_train.csv'
def get_finance_train():
df_train = pd.read_csv("finance_train.csv")
return df_train
def get_finance_test():
df_test = pd.read_csv("finance_test.csv")
return df_test
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat) / len(labels_flat)
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case = True)
print ("Train and Test Files Loaded as train.csv and test.csv")
LABEL_MAP = {0 : "negative", 1 : "neutral", 2 : "positive"}
NONE = 4 * [None]
RND_SEED=2020
df_train = get_finance_train()
sentences = df_train["Sentence"].values
labels = df_train["Label"].values
input_ids = []
sentences_with_special_tokens = []
tokenized_texts = []
attention_masks = []
for sentence in sentences:
new_sentence = "[CLS] " + sentence + " [SEP]"
sentences_with_special_tokens.append(new_sentence)
for sentence in sentences_with_special_tokens:
tokenized_sentence = tokenizer.tokenize(sentence)
tokenized_texts.append(tokenized_sentence)
for token in tokenized_texts:
ids = tokenizer.convert_tokens_to_ids(token)
input_ids.append(ids)
input_ids = pad_sequences(input_ids,
maxlen=128,
dtype="long",
truncating="post",
padding="post")
for id in input_ids:
mask=[float(i>0) for i in id]
attention_masks.append(mask)
X_train, X_val, y_train, y_val =train_test_split(input_ids, labels, test_size=0.15, random_state=RND_SEED)
train_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids, test_size=0.15,random_state=RND_SEED)
train_inputs = torch.tensor(np.array(X_train));
validation_inputs = torch.tensor(np.array(X_val));
train_masks = torch.tensor(np.array(train_masks));
validation_masks = torch.tensor(np.array(validation_masks));
train_labels = torch.tensor(np.array(y_train));
validation_labels = torch.tensor(np.array(y_val));
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels);
train_sampler = RandomSampler(train_data); # Samples data randonly for training
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size);
validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels);
validation_sampler = SequentialSampler(validation_data); # Samples data sequentially
validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size);
#Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top.
model = BertForSequenceClassification.from_pretrained(
"bert-base-uncased", # Use the 12-layer BERT small model, with an uncased vocab.
num_labels = 3,
output_attentions = False, # Whether the model returns attentions weights.
output_hidden_states = False, # Whether the model returns all hidden-states.
);
# Given that this a huge neural network, we need to explicity specify
# in pytorch to run this model on the GPU.
model.cuda();
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
torch.cuda.get_device_name(0)
optimizer = AdamW(model.parameters(),
lr = 2e-5,
eps = 1e-8
)
epochs = 4
# Total number of training steps is [number of batches] x [number of epochs].
# (Note that this is not the same as the number of training samples).
total_steps = len(train_dataloader) * epochs
# Create the learning rate scheduler.
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0, # Default value in run_glue.py
num_training_steps = total_steps)
# We'll store training and validation loss,
# validation accuracy, and timings.
training_loss = []
validation_loss = []
training_stats = []
for epoch_i in range(0, epochs):
# Training
print('Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training the model')
# Reset the total loss for epoch.
total_train_loss = 0
# Put the model into training mode.
model.train()
# For each batch of training data
for step, batch in enumerate(train_dataloader):
# Progress update every 40 batches.
if step % 20 == 0 and not step == 0:
# Report progress.
print(' Batch {:>5,} of {:>5,}. '.format(step, len(train_dataloader)))
# STEP 1 & 2: Unpack this training batch from our dataloader.
# As we unpack the batch, we'll also copy each tensor to the GPU using the
# `to` method.
# `batch` contains three pytorch tensors:
# [0]: input ids
# [1]: attention masks
# [2]: labels
b_input_ids = batch[0].to(device)
b_input_mask = batch[1].to(device)
b_labels = batch[2].to(device)
# STEP 3
# Always clear any previously calculated gradients before performing a
# backward pass.
model.zero_grad()
# STEP 4
# Perform a forward pass (evaluate the model on this training batch).
# It returns the loss (because we provided labels) and
# the "logits"--the model outputs prior to activation.
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
loss = outputs[0]
logits = outputs[1]
# Accumulate the training loss over all of the batches so that we can
# calculate the average loss at the end. `loss` is a Tensor containing a
# single value; the `.item()` function just returns the Python value
# from the tensor.
total_train_loss += loss.item()
# STEP 5
# Perform a backward pass to calculate the gradients.
loss.backward()
# Clip the norm of the gradients to 1.0.
# This is to help prevent the "exploding gradients" problem.
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
# STEP 6
# Update parameters and take a step using the computed gradient
optimizer.step()
# Update the learning rate.
scheduler.step()
# Calculate the average loss over all of the batches.
avg_train_loss = total_train_loss / len(train_dataloader)
print(" Average training loss: {0:.2f}".format(avg_train_loss))
# Validation
# After the completion of each training epoch, measure our performance on
# our validation set.
print("Evaluating on Validation Set")
# Put the model in evaluation mode
model.eval()
# Tracking variables
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
# Evaluate data for one epoch
for batch in validation_dataloader:
#Step 1 and Step 2
# Unpack this validation batch from our dataloader.
b_input_ids = batch[0].to(device)
b_input_mask = batch[1].to(device)
b_labels = batch[2].to(device)
# Tell pytorch not to bother with constructing the compute graph during
# the forward pass, since this is only needed for backprop (training).
with torch.no_grad():
# Forward pass, calculate logit predictions.
# The "logits" are the output
# values prior to applying an activation function like the softmax.
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
loss = outputs[0]
logits = outputs[1]
# Accumulate the validation loss.
total_eval_loss += loss.item()
# Move logits and labels to CPU
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu').numpy()
# Calculate the accuracy for this batch of test sentences, and
# accumulate it over all batches.
total_eval_accuracy += flat_accuracy(logits, label_ids)
# Report the final accuracy for this validation run.
avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
print("Validation Accuracy: {0:.2f}".format(avg_val_accuracy))
# Calculate the average loss over all of the batches.
avg_val_loss = total_eval_loss / len(validation_dataloader)
print("Validation Loss: {0:.2f}".format(avg_val_loss))
training_loss.append(avg_train_loss)
validation_loss.append(avg_val_loss)
# Record all statistics from this epoch.
training_stats.append(
{
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': avg_val_loss,
'Valid. Accur.': avg_val_accuracy
}
)
print("Training complete!")
Here is what I have:
import tensorflow as tf
df_test = "The stock is growing, the company is doing well. The company is close to bankruptcy and the stock price is falling"
sentences_with_special_tokens = []
tokenized_texts = []
input_ids = []
attention_masks = []
for sentence in df_test:
new_sentence = "[CLS] " + sentence + " [SEP]"
sentences_with_special_tokens.append(new_sentence)
print(sentences_with_special_tokens)
tokenized_texts = []
for sentence in sentences_with_special_tokens:
tokenized_sentence = tokenizer.tokenize(sentence)
tokenized_texts.append(tokenized_sentence)
print(tokenized_texts)
for token in tokenized_texts:
ids = tokenizer.convert_tokens_to_ids(token)
input_ids.append(ids)
print(input_ids)
input_ids = pad_sequences(input_ids,
maxlen=128,
dtype="long",
truncating="post",
padding="post")
for id in input_ids:
mask=[float(i>0) for i in id]
attention_masks.append(mask)
print(attention_masks)
I want to input df_test into the model and for it to return the percentages for each of the labels
LABEL_MAP = {0 : "negative", 1 : "neutral", 2 : "positive"}
I looked online and I found this on hugging face for single label classification:
import torch
from transformers import BertTokenizer, BertForSequenceClassification
model = BertForSequenceClassification.from_pretrained("textattack/bert-base-uncased-yelp-polarity")
inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
with torch.no_grad():
logits = model(**inputs).logits
predicted_class_id = logits.argmax().item()
model.config.id2label[predicted_class_id]
But when I tried implementing this into my code, it tells me that it needs all of the tensors on the same device and I have at least two devices.
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument index in method wrapper__index_select)