Train

!pip install datasets
!pip install transformers==4.28.0
!pip install evaluate

!nvidia-smi

import torch

torch.cuda.is_available(), torch.cuda.device_count(), torch.cuda.current_device()

from datasets import load_dataset

dataset = load_dataset("financial_phrasebank", 'sentences_allagree')

dataset["train"][100]

set([item["label"] for item in dataset["train"]])

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")

def tokenize_function(examples):
    return tokenizer(examples["sentence"], padding="max_length", truncation=True)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000,1200)) 

small_train_dataset, small_eval_dataset 

print(small_train_dataset[0]["sentence"])
print(small_train_dataset[0]["label"])
print(small_train_dataset[0]["input_ids"])
print(small_train_dataset[0]["attention_mask"])

print(small_eval_dataset[0]["sentence"])
print(small_eval_dataset[0]["label"])
print(small_eval_dataset[0]["input_ids"])
print(small_eval_dataset[0]["attention_mask"]) 

from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english",
    ignore_mismatched_sizes=True,
    num_labels=3)

from sklearn.metrics import f1_score
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score

y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 0]
y_pred = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]

from sklearn.metrics import confusion_matrix

TP, FP, FN, TN = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()

# True Positive (TP) : The prediction was positive and the real sample was also positive.
print("True Positive (TP):", TP)
# True Negative (TN) : The prediction was negative and the real sample was also negative.
print("True Negative (TN):", TN)
# False Positive (FP) : The prediction was positive and the real sample was also negative.
print("False Positive (FP):", FP)
# False Negative (FN) : The prediction was negative and the real sample was also positive.
print("False Negative (FN):", FN)

# Recall (R) = TP / (TP + FN)
print(recall_score(y_true, y_pred))

# Precizitás (P) = TP / (TP + FP)
print(precision_score(y_true, y_pred))

# Accuracy (A) = (TP + TN) / (TP + TN + FP + FN) - Correct classificaton / all classifications
print(accuracy_score(y_true, y_pred))

# F1 = 2 * P * R / (P + R)
print(f1_score(y_true, y_pred))

import numpy as np
import evaluate

metric = evaluate.load("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir="test_trainer",
    evaluation_strategy="epoch",
    num_train_epochs=2
    )

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=small_train_dataset,
    eval_dataset=small_eval_dataset,
    compute_metrics=compute_metrics,
)

trainer.train()

from transformers import pipeline

model_old = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
pipeline_old = pipeline("text-classification", model=model_old, tokenizer=tokenizer)

text = small_eval_dataset[0:10]["sentence"]
labels = small_eval_dataset[0:10]["label"]
text, labels

pipeline_old(text)

device = "cuda:0" if torch.cuda.is_available() else "cpu"
device

model_cpu = model.to("cpu")

pipeline_new = pipeline("text-classification", model=model_cpu, tokenizer=tokenizer)

print(labels)
pipeline_new(text)

import torch
from transformers import DistilBertTokenizer, DistilBertForSequenceClassification

tokenizer_hc = DistilBertTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
model_hc = model

inputs = tokenizer(text, return_tensors="pt",padding=True,truncation=True)
with torch.no_grad():
    logits = model_hc(**inputs).logits

predicted_class_ids = [l.argmax().item() for l in logits]
predicted_class_ids, labels