Supervised versus Unsupervised MultiTask Learning for C

    import torch 
    from torch import nn 
    from torchtext import data 
    from sklearn.metrics import classification_report 
    
    class LSTMClassifier(nn.Module): 
    def __init__(self, input_dim, hidden_dim, output_dim, n_layers=2, dropout=0.5): 
        super().__init__() 
        self.hidden_dim = hidden_dim 
        self.n_layers = n_layers 
        self.dropout = nn.Dropout(p=dropout) 
    
        # LSTM layer 
        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=n_layers, batch_first=True)  
    
        # Fully connected layer 
        self.fc = nn.Linear(hidden_dim, output_dim) 
    
    def forward(self, x): 
        h0 = nn.Parameter(torch.zeros(self.n_layers, x.size(0), self.hidden_dim).to(x.device)) 
        c0 = nn.Parameter(torch.zeros(self.n_layers, x.size(0), self.hidden_dim).to(x.device)) 
        lstm_out, (hn, cn) = self.lstm(x, (h0,c0)) 
    
        out = lstm_out[:, -1, :]  
        out = self.dropout(out) 
        out = self.fc(out) 
    
        return out 
    
    def main(): 
    TEXT = data.Field() 
    LABEL = data.LabelField() 
    train_data, test_data = datasets.IMDB.splits(TEXT, LABEL) 
    
    MAX_VOCAB_SIZE = 25_000 
    TEXT.build_vocab(train_data, max_size=MAX_VOCAB_SIZE) 
    LABEL.build_vocab(train_data) 
    
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 
    INPUT_DIM = len(TEXT.vocab) 
    EMBEDDING_DIM = 300 
    HIDDEN_DIM = 256 
    OUTPUT_DIM = 2 
    N_LAYERS = 2 
    BATCH_SIZE = 32 
    DROPOUT = 0.5 
    
    model = LSTMClassifier(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, DROPOUT)  
    
    loss_fn = nn.CrossEntropyLoss()  
    optimizer = optim.Adam(model.parameters())  
    
    train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data, valid_data, test_data), 
                                                                                 batch_size=BATCH_SIZE, 
                                                                                 device=device) 
    
    def accuracy(preds, y): 
        """Compute the accuracy of prediction.""" 
        correct = (torch.max(pred, dim=1)[1].squeeze() == y).float().sum() 
        acc = correct / float(y.shape[0]) 
        return acc 
    
    best_valid_loss = float('inf') 
    
    for epoch in range(N_EPOCHS): 
        start_time = time.time() 
        train_loss = train(model, train_iterator, optimizer, criterion=loss_fn) 
        end_time = time.time() 
    
        valid_loss = evaluate(model, valid_iterator, criterion=loss_fn) 
    
        if valid_loss < best_valid_loss: 
            best_valid_loss = valid_loss 
            torch.save(model.state_dict(), f'model_{epoch}.pt') 
    
        print(f'\tEpoch {epoch+1}, Train Loss: {train_loss:.3f}, Valid Loss: {valid_loss:.3f}') 
    
    model.load_state_dict(torch.load(f'model_{best_valid_loss}.pt'))  
    
    test_loss, test_acc = evaluate(model, test_iterator, criterion=loss_fn)  
    
    y_true = [] 
    y_pred = [] 
    with torch.no_grad(): 
        for batch in test_iterator: 
            text, label = batch.text, batch.label 
            predictions = model(text).argmax(1).tolist() 
            labels = label.tolist() 
    
            y_true += labels 
            y_pred += predictions 
    
    report = classification_report(y_true, y_pred, target_names=['Negative', 'Positive'], digits=4) 
    print(report)
    
    if __name__ == '__main__': 
    main()
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    
    代码解读

    import pandas as pd 
    import numpy as np  
    import transformers   
    from sklearn.model_selection import train_test_split 
    from sklearn.metrics import precision_recall_fscore_support, accuracy_score  
    
    class BERTClassifier(transformers.PreTrainedModel):
    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels
    
        self.bert = transformers.BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
    
        self.init_weights()
    
    def forward(self, inputs):
        outputs = self.bert(**inputs)
        pooled_output = outputs[1]
    
        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
    
        return logits
    
    
    def read_dataset(file_path):
    df = pd.read_csv(file_path)
    X = df['content'].values.tolist()
    y = df['sentiment'].values
    encoder = {'neg': 0, 'pos': 1, 'neu': 2}
    le = LabelEncoder()
    le.fit(encoder.keys())
    y = le.transform([x[0] for x in y])
    return list(zip(X, y)), len(le.classes_)
    
    def load_dataset(tokenizer, file_path):
    dataset, num_labels = read_dataset(file_path)
    examples = tokenizer.batch_encode_plus([x[0] for x in dataset], padding='max_length', truncation=True, max_length=512)
    features = []
    for i, example in enumerate(examples['input_ids']):
        feature = {}
        feature["input_ids"] = example
        feature["attention_mask"] = examples["attention_mask"][i]
        feature["token_type_ids"] = [0]*len(example)
        feature["label"] = dataset[i][1]
        features.append(feature)
    return features, num_labels
    
    
    def tokenize_and_align_labels(batch, tokenizer, labels):
    tokenized_inputs = tokenizer(batch["sentence"], padding="max_length", truncation=True, max_length=512)
    labels = [{"label": str(l)} for l in labels]
    aligned_labels = []
    for label in labels:
        label["start_position"] = None
        label["end_position"] = None
        try:
            char_start_positions = [m.start() for m in re.finditer(label["label"], tokenized_inputs["input_ids"][0])]
            char_end_positions = [m.end() for m in re.finditer(label["label"], tokenized_inputs["input_ids"][0])]
            word_start_positions = [int(np.floor(idx/float(len(tokenized_inputs["input_ids"][0]))*len(char_start_positions))) for idx, pos in enumerate(char_start_positions)]
            word_end_positions = [int(min(np.ceil(idx/float(len(tokenized_inputs["input_ids"][0]))*len(char_end_positions))+1, len(char_end_positions))) for idx, pos in enumerate(char_end_positions)]
            matched_indices = [(wst, wen) for wst, wen in zip(word_start_positions, word_end_positions) if wen > wst][:len(char_start_positions)]
            if not matched_indices or len(matched_indices)!= len(char_start_positions):
                raise ValueError("Something is wrong")
    
            subwords_in_chars = [[m.start(), m.end()] for m in re.finditer('[A-Za-z]+', tokenized_inputs["tokens"][0])]
            matched_subwords = set()
            for st, en in matched_indices:
                found = False
                for sst, sen in subwords_in_chars:
                    if st >= sst and en <= sen:
                        matched_subwords.add((sst, sen))
                        found = True
                        break
                if not found:
                    continue
    
            labeled_indices = sorted([(i,j) for i, j in matched_subwords], key=lambda x: x[0]-x[1])
            assert all([isinstance(li[0], int) and isinstance(li[1], int) for li in labeled_indices]), "Wrong indices"
            words_with_label = [" ".join(tokenized_inputs["tokens"][0][li[0]:li[1]]) for li in labeled_indices]
            labeled_texts = [' '.join(words_with_label[:k])+f"[LABEL]{words_with_label[-1]}" for k in range(1, len(words_with_label)+1)]
            alignment_map = [-1]*len(tokenized_inputs["input_ids"][0])
            for wi, li in enumerate(labeled_indices[:-1]):
                align_start = sum(len(tk) for tk in tokenized_inputs["tokens"][0][:li[1]] + tokenized_inputs["tokens"][0][:li[0]][::-1]) - len(tokenized_inputs["tokens"][0][:li[0]])
                align_end = align_start + len("[LABEL]") 
                alignment_map[align_start:align_end] = [wi]*len(alignment_map[align_start:align_end])
            last_wi = len(labeled_indices)-1
            align_start = sum(len(tk) for tk in tokenized_inputs["tokens"][0][:labeled_indices[last_wi][1]])
            align_end = align_start + len(words_with_label[-1])
            alignment_map[align_start:] = [last_wi]*len(alignment_map[align_start:])
            label["start_position"] = alignment_map.index(-1)
            label["end_position"] = alignment_map.index(-1)
        except Exception as e:
            pass
    
        aligned_labels.append(label)
    
    tokenized_inputs["labels"] = aligned_labels
    return tokenized_inputs
    
    
    def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    
    precision, recall, f1, _ = precision_recall_fscore_support(labels, predictions, average='binary')
    acc = accuracy_score(labels, predictions)
    return {"accuracy": acc, "precision": precision, "recall": recall, "f1": f1}
    
    
    if __name__ == "__main__":
    SAVE_DIR = "./models/"
    MODEL_NAME = "bert-base-uncased"
    NUM_LABELS = 2
    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    
    
    _, num_labels = load_dataset(None, './datasets/twitter.csv')
    print("# Labels:", num_labels)
    
    tokenizer = transformers.BertTokenizer.from_pretrained(MODEL_NAME)
    data_collator = DataCollatorForTokenClassification(tokenizer)
    train_data, val_data = load_dataset(tokenizer, './datasets/twitter.csv')
    
    train_dataset = Dataset.from_dict(train_data)
    val_dataset = Dataset.from_dict(val_data)
    
    train_dataset = train_dataset.map(tokenize_and_align_labels, batched=True, fn_kwargs={"tokenizer": tokenizer})
    val_dataset = val_dataset.map(tokenize_and_align_labels, batched=True, fn_kwargs={"tokenizer": tokenizer})
    
    model = BERTClassifier.from_pretrained(MODEL_NAME, num_labels=NUM_LABELS)
    model.to(DEVICE)
    
    training_args = TrainingArguments(
        output_dir=SAVE_DIR,          # output directory
        num_train_epochs=10,              # total number of training epochs
        per_device_train_batch_size=16,  # batch size per device during training
        per_device_eval_batch_size=16,   # batch size for evaluation
        warmup_steps=500,                # number of warmup steps for learning rate scheduler
        weight_decay=0.01,               # strength of weight decay
        logging_dir="./logs/",            # directory for storing logs
        save_total_limit=1,
        fp16=True,                        # use FP16 instead of FP32
        seed=42,                          
    )
    
    trainer = Trainer(
        model=model,                         # the instantiated 🤗 Transformers model to be trained
        args=training_args,                  # training arguments, defined above
        train_dataset=train_dataset,         # training dataset
        eval_dataset=val_dataset,           # evaluation dataset
        data_collator=data_collator,             # collator that will dynamically pad the batches
        callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],       # callback that performs early stopping
        compute_metrics=compute_metrics      # function that computes metrics at evaluation
    )
    
    trainer.train()
    trainer.evaluate()
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    
    代码解读