Update app.py

app.py

@@ -33,21 +33,69 @@ def preprocess_data(examples):
 
 encoded_dataset = dataset.map(preprocess_data, batched=True, remove_columns=dataset['train'].column_names)
 
-example = encoded_dataset['train'][9500]
-console.log(example['labels'])
-
 encoded_dataset.set_format("torch")
 
+id2label = {idx:label for idx, label in enumerate(labels)}
+label2id = {label:idx for idx, label in enumerate(labels)}
+
+model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased", 
+                                                           num_labels=len(labels),
+                                                           id2label=id2label,
+                                                           label2id=label2id)
 
+batch_size = 8
+metric_name = "f1"
 
+from transformers import TrainingArguments, Trainer
 
+args = TrainingArguments(
+    f"bert-finetuned-sem_eval-english",
+    evaluation_strategy = "epoch",
+    save_strategy = "epoch",
+    learning_rate=2e-5,
+    per_device_train_batch_size=batch_size,
+    per_device_eval_batch_size=batch_size,
+    num_train_epochs=5,
+    weight_decay=0.01,
+    load_best_model_at_end=True,
+    metric_for_best_model=metric_name,
+    #push_to_hub=True,
+)
 
-def sentiment_score(review):
-  tokens = tokenizer.encode(review, return_tensors='pt')
-  result = model(tokens)
-  return int(torch.argmax(result.logits))
 
-dataset['sentiment'] = dataset['train']['text'].apply(lambda x: sentiment_score(x[:512]))
+from sklearn.metrics import f1_score, roc_auc_score, accuracy_score
+from transformers import EvalPrediction
+import torch
+    
+# source: https://jesusleal.io/2021/04/21/Longformer-multilabel-classification/
+def multi_label_metrics(predictions, labels, threshold=0.5):
+    # first, apply sigmoid on predictions which are of shape (batch_size, num_labels)
+    sigmoid = torch.nn.Sigmoid()
+    probs = sigmoid(torch.Tensor(predictions))
+    # next, use threshold to turn them into integer predictions
+    y_pred = np.zeros(probs.shape)
+    y_pred[np.where(probs >= threshold)] = 1
+    # finally, compute metrics
+    y_true = labels
+    f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='micro')
+    roc_auc = roc_auc_score(y_true, y_pred, average = 'micro')
+    accuracy = accuracy_score(y_true, y_pred)
+    # return as dictionary
+    metrics = {'f1': f1_micro_average,
+               'roc_auc': roc_auc,
+               'accuracy': accuracy}
+    return metrics
+
+def compute_metrics(p: EvalPrediction):
+    preds = p.predictions[0] if isinstance(p.predictions, 
+            tuple) else p.predictions
+    result = multi_label_metrics(
+        predictions=preds, 
+        labels=p.label_ids)
+    return result
+
+
+console.log(encoded_dataset['train']['input_ids'][0])
 
 """