自然语言处理之命名实体识别：BERT：3.预训练模型原理与BERT介绍

    # 假设我们有一个简单的BERT模型和一个文本序列
    from transformers import BertTokenizer, BertForMaskedLM
    
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    model = BertForMaskedLM.from_pretrained('bert-base-chinese')
    
    # 输入文本
    text = "我喜欢在晴朗的日子里去公园散步。"
    # 掩码部分词汇
    masked_text = "我喜欢在[MASK]的日子里去公园[MASK]。"
    
    # 编码文本
    input_ids = tokenizer.encode(masked_text, return_tensors='pt')
    
    # 预测掩码词汇
    with torch.no_grad():
    outputs = model(input_ids)
    predictions = outputs[0]
    
    # 解码预测词汇
    predicted_tokens = tokenizer.decode(torch.argmax(predictions[0], dim=-1).tolist())
    print(predicted_tokens)

    # 使用BERT进行下一句预测
    from transformers import BertTokenizer, BertForNextSentencePrediction
    
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    model = BertForNextSentencePrediction.from_pretrained('bert-base-chinese')
    
    # 输入两个句子
    sentence_a = "我今天去了图书馆。"
    sentence_b = "我在那里借了几本书。"
    # 编码句子
    input_ids = tokenizer.encode(sentence_a, sentence_b, return_tensors='pt')
    
    # 预测两个句子是否连续
    with torch.no_grad():
    outputs = model(input_ids)
    next_sentence_prediction = outputs[0]
    
    # 解码预测结果
    if torch.argmax(next_sentence_prediction).item() == 0:
    print("句子B是句子A的下一句。")
    else:
    print("句子B不是句子A的下一句。")

    # 导入必要的库
    from transformers import BertTokenizer, BertForMaskedLM
    import torch
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertForMaskedLM.from_pretrained('bert-base-uncased')
    
    # 输入文本
    text = "The capital of France is [MASK]."
    
    # 分词和编码
    input_ids = tokenizer.encode(text, return_tensors='pt')
    
    # 预测被遮掩的词
    with torch.no_grad():
    output = model(input_ids)
    prediction = output[0]
    
    # 解码预测结果
    predicted_index = torch.argmax(prediction[0, tokenizer.mask_token_id]).item()
    predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
    
    print(predicted_token)  # 输出: 'Paris'

    # 导入必要的库
    from transformers import BertTokenizer, BertForNextSentencePrediction
    import torch
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased')
    
    # 输入文本
    text1 = "The capital of France is Paris."
    text2 = "Paris is the capital of France."
    text3 = "The Eiffel Tower is in Paris."
    
    # 分词和编码
    inputs = tokenizer(text1, text2, return_tensors='pt')
    inputs2 = tokenizer(text1, text3, return_tensors='pt')
    
    # 预测句子关系
    with torch.no_grad():
    output = model(**inputs)
    prediction = output[0]
    
    # 解码预测结果
    if prediction[0][0] > prediction[0][1]:
    print("Text2 is not the next sentence of Text1.")
    else:
    print("Text2 is the next sentence of Text1.")
    
    # 重复预测过程
    with torch.no_grad():
    output2 = model(**inputs2)
    prediction2 = output2[0]
    
    if prediction2[0][0] > prediction2[0][1]:
    print("Text3 is not the next sentence of Text1.")
    else:
    print("Text3 is the next sentence of Text1.")

    # 导入必要的库
    from transformers import BertForTokenClassification, BertTokenizer
    from torch.utils.data import DataLoader, Dataset
    import torch
    
    # 定义数据集
    class NERDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, item):
        text = str(self.texts[item])
        label = self.labels[item]
    
        encoding = self.tokenizer.encode_plus(
            text,
            add_special_tokens=True,
            max_length=self.max_len,
            return_token_type_ids=False,
            pad_to_max_length=True,
            return_attention_mask=True,
            return_tensors='pt',
        )
    
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'labels': torch.tensor(label, dtype=torch.long)
        }
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertForTokenClassification.from_pretrained('bert-base-uncased')
    
    # 准备数据集和数据加载器
    dataset = NERDataset(texts, labels, tokenizer, max_len=128)
    data_loader = DataLoader(dataset, batch_size=16)
    
    # 微调模型
    for batch in data_loader:
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    
    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
    loss = outputs[0]
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    # 导入必要的库
    import torch
    from transformers import BertTokenizer, BertForTokenClassification
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
    model = BertForTokenClassification.from_pretrained('dbmdz/bert-large-cased-finetuned-conll03-english')
    
    # 输入文本
    text = "Hugging Face Inc. is a company based in New York City. Its headquarters are in DUMBO, therefore very close to the Manhattan Bridge which is visible from the window."
    
    # 分词和编码
    inputs = tokenizer(text, return_tensors="pt")
    labels = torch.tensor([1] * inputs["input_ids"].size(1)).unsqueeze(0)  # 仅为示例，实际标签需要根据数据集生成
    
    # 前向传播
    outputs = model(**inputs, labels=labels)
    loss, scores = outputs[:2]
    
    # 解码预测的实体标签
    predictions = torch.argmax(scores, dim=2)
    predicted_labels = [model.config.id2label[p.item()] for p in predictions[0]]

    import re
    
    def clean_text(text):
    # 去除HTML标签
    text = re.sub(r'<.*?>', '', text)
    # 去除特殊字符和数字
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    return text
    
    # 示例文本
    text = "<p>这是一段包含HTML标签和特殊字符的文本！</p>"
    cleaned_text = clean_text(text)
    print(cleaned_text)  # 输出应为：这是一段包含HTML标签和特殊字符的文本

    from transformers import BertTokenizer
    
    # 初始化BERT分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    
    # 示例文本
    text = "李华是北京大学的学生。"
    
    # 分词和转换
    input_ids = tokenizer.encode(text, add_special_tokens=True)
    print(input_ids)  # 输出应为：[101, 2112, 2015, 2003, 2014, 1012, 102]

    例如，对于文本“李华是北京大学的学生。”，标注可能如下：
    李华：B-PER, I-PER
    是：O
    北京大学：B-ORG, I-ORG, I-ORG
    的：O
    学生：O

    from transformers import BertForTokenClassification, BertTokenizer
    from torch.utils.data import Dataset, DataLoader
    import torch
    
    # 定义数据集
    class NERDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, item):
        text = str(self.texts[item])
        label = self.labels[item]
    
        encoding = self.tokenizer.encode_plus(
            text,
            add_special_tokens=True,
            max_length=self.max_len,
            return_token_type_ids=False,
            padding='max_length',
            truncation=True,
            return_attention_mask=True,
            return_tensors='pt',
        )
    
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'labels': torch.tensor(label, dtype=torch.long)
        }
    
    # 加载预训练的BERT模型和分词器
    model = BertForTokenClassification.from_pretrained('bert-base-chinese')
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    
    # 定义数据集和数据加载器
    dataset = NERDataset(texts, labels, tokenizer, max_len=128)
    data_loader = DataLoader(dataset, batch_size=16)
    
    # 微调模型
    for batch in data_loader:
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    
    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
    loss = outputs.loss
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    from transformers import AdamW
    
    # 设置优化器
    optimizer = AdamW(model.parameters(), lr=1e-5)
    
    # 训练循环
    for epoch in range(epochs):
    for batch in data_loader:
        input_ids = batch['input_ids']
        attention_mask = batch['attention_mask']
        labels = batch['labels']
    
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

    from transformers import get_linear_schedule_with_warmup
    
    # 设置学习率调整策略
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=len(data_loader)*epochs)
    
    # 在训练循环中应用学习率调整
    for epoch in range(epochs):
    for batch in data_loader:
        # ...
        loss.backward()
        optimizer.step()
        scheduler.step()  # 更新学习率
        optimizer.zero_grad()

    from seqeval.metrics import f1_score, classification_report
    
    # 预测
    predictions, true_labels = [], []
    for batch in data_loader:
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    
    with torch.no_grad():
        outputs = model(input_ids, attention_mask=attention_mask)
    
    logits = outputs.logits
    preds = torch.argmax(logits, dim=2).tolist()
    predictions.extend(preds)
    true_labels.extend(labels.tolist())
    
    # 计算F1分数
    f1 = f1_score(true_labels, predictions)
    print(f"F1 Score: {f1}")
    
    # 输出分类报告
    report = classification_report(true_labels, predictions)
    print(report)

    例如，分析预测结果和真实标签的差异，找出模型预测错误的实体类型，可以针对性地改进模型或数据集。

    例如，将训练和验证的损失、F1分数等指标记录到TensorBoard，便于观察模型的训练进度和性能。