自然语言处理之命名实体识别：BERT：15.命名实体识别在机器翻译中的应用

    # 导入必要的库
    from transformers import BertTokenizer, BertForTokenClassification
    import torch
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
    model = BertForTokenClassification.from_pretrained('dbmdz/bert-large-cased-finetuned-conll03-english')
    
    # 输入文本
    text = "Hugging Face Inc. is a company based in New York City. Its headquarters are in DUMBO, therefore very close to the Manhattan Bridge which is visible from the window."
    
    # 分词和编码
    inputs = tokenizer(text, return_tensors="pt")
    labels = torch.tensor([1] * inputs["input_ids"].size(1)).unsqueeze(0)  # 仅为示例，实际中需要使用真实的标签
    
    # 前向传播
    outputs = model(**inputs, labels=labels)
    loss, scores = outputs[:2]
    
    # 解码预测的实体标签
    predictions = torch.argmax(scores, dim=2)
    predicted_labels = [model.config.id2label[p.item()] for p in predictions[0]]
    
    # 打印预测结果
    print(predicted_labels)

    {
    "text": "Hugging Face Inc. is a company based in New York City.",
    "entities": [
        {"start": 0, "end": 14, "label": "ORG"},
        {"start": 32, "end": 41, "label": "LOC"}
    ]
    }

    import pycrfsuite
    
    # 训练数据
    train_data = [
    # 句子1
    [
        ('李华', 'Person'),
        ('昨天', 'O'),
        ('去了', 'O'),
        ('北京', 'Location'),
        ('的', 'O'),
        ('清华大学', 'Organization'),
    ],
    # 句子2
    [
        ('张三', 'Person'),
        ('今天', 'O'),
        ('去了', 'O'),
        ('上海', 'Location'),
        ('的', 'O'),
        ('复旦大学', 'Organization'),
    ],
    ]
    
    # 特征函数
    def word2features(sent, i):
    word = sent[i][0]
    features = [
        'bias',
        'word.lower=' + word.lower(),
        'word[-3:]=' + word[-3:],
        'word[-2:]=' + word[-2:],
        'word.isupper=%s' % word.isupper(),
        'word.istitle=%s' % word.istitle(),
        'word.isdigit=%s' % word.isdigit(),
    ]
    if i > 0:
        word1 = sent[i-1][0]
        features.extend([
            '-1:word.lower=' + word1.lower(),
            '-1:word.istitle=%s' % word1.istitle(),
            '-1:word.isupper=%s' % word1.isupper(),
        ])
    else:
        features.append('BOS')
    if i < len(sent)-1:
        word1 = sent[i+1][0]
        features.extend([
            '+1:word.lower=' + word1.lower(),
            '+1:word.istitle=%s' % word1.istitle(),
            '+1:word.isupper=%s' % word1.isupper(),
        ])
    else:
        features.append('EOS')
    return features
    
    # 句子转换为特征
    def sent2features(sent):
    return [word2features(sent, i) for i in range(len(sent))]
    
    # 句子转换为标签
    def sent2labels(sent):
    return [label for token, label in sent]
    
    # 准备训练数据
    X_train = [sent2features(s) for s in train_data]
    y_train = [sent2labels(s) for s in train_data]
    
    # 训练CRF模型
    trainer = pycrfsuite.Trainer(verbose=False)
    for xseq, yseq in zip(X_train, y_train):
    trainer.append(xseq, yseq)
    trainer.set_params({
    'c1': 1.0,   # coefficient for L1 penalty
    'c2': 1e-3,  # coefficient for L2 penalty
    'max_iterations': 50,  # stop earlier
    'feature.possible_transitions': True
    })
    trainer.train('crf_model')
    
    # 使用模型进行预测
    tagger = pycrfsuite.Tagger()
    tagger.open('crf_model')
    print(tagger.tag(sent2features([('李华', 'Person'), ('去了', 'O'), ('北京', 'Location'), ('的', 'O'), ('清华大学', 'Organization')])))
    
    # 输出预测结果
    # ['Person', 'O', 'Location', 'O', 'Organization']

    from transformers import BertTokenizer, BertForTokenClassification
    import torch
    
    # 加载预训练的BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    model = BertForTokenClassification.from_pretrained('bert-base-chinese', num_labels=3)
    
    # 输入文本
    text = "李华昨天去了北京的清华大学。"
    
    # 分词和编码
    inputs = tokenizer.encode(text, return_tensors="pt")
    labels = torch.tensor([1, 2, 0, 0, 1, 2, 0]).unsqueeze(0)  # 1: Person, 2: Location, 0: O
    
    # 前向传播
    outputs = model(inputs, labels=labels)
    loss, scores = outputs[:2]
    
    # 预测
    predictions = torch.argmax(scores, dim=2)
    
    # 解码预测结果
    predicted_labels = [tokenizer.convert_ids_to_tokens([i])[0] for i in predictions[0]]
    print(predicted_labels)
    
    # 输出预测结果
    # ['[CLS]', 'Person', 'O', 'O', 'Location', 'Organization', 'O', '[SEP]']

    from transformers import BertForTokenClassification, BertTokenizer, Trainer, TrainingArguments
    from datasets import load_dataset
    
    # 加载数据集
    dataset = load_dataset("conll2003")
    
    # 加载预训练的BERT模型和分词器
    model = BertForTokenClassification.from_pretrained("bert-base-cased")
    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
    
    # 准备训练参数
    training_args = TrainingArguments(
    output_dir='./results',          # 输出目录
    num_train_epochs=3,              # 训练轮数
    per_device_train_batch_size=16,  # 每个GPU的训练批次大小
    per_device_eval_batch_size=64,   # 每个GPU的评估批次大小
    warmup_steps=500,                # 预热步数
    weight_decay=0.01,               # 权重衰减
    logging_dir='./logs',            # 日志目录
    )
    
    # 创建Trainer实例
    trainer = Trainer(
    model=model,                         # 要训练的模型
    args=training_args,                  # 训练参数
    train_dataset=dataset['train'],       # 训练数据集
    eval_dataset=dataset['validation'],   # 验证数据集
    tokenizer=tokenizer,                  # 分词器
    )
    
    # 开始训练
    trainer.train()

    # 导入必要的库
    import torch
    from transformers import BertTokenizer, BertForTokenClassification
    
    # 初始化BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
    model = BertForTokenClassification.from_pretrained('dbmdz/bert-large-cased-finetuned-conll03-english')
    
    # 输入文本
    text = "New York is a city in the United States."
    
    # 分词和编码
    inputs = tokenizer(text, return_tensors="pt")
    with torch.no_grad():
    outputs = model(**inputs)
    
    # 获取预测标签
    predicted_labels = torch.argmax(outputs.logits, dim=2)
    
    # 解码预测标签
    label_list = ['O', 'B-MISC', 'I-MISC', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']
    predicted_labels = [label_list[label.item()] for label in predicted_labels[0]]
    
    # 打印预测结果
    print(predicted_labels)

    # 假设的实体对齐字典
    entity_alignment = {
    "New York": "纽约",
    "United States": "美国"
    }
    
    # 输入文本和NER结果
    text = "New York is a city in the United States."
    ner_results = ["B-LOC", "O", "O", "O", "O", "O", "O", "O", "B-LOC"]
    
    # 分词
    tokens = tokenizer.tokenize(text)
    
    # 实体对齐和翻译
    translated_text = []
    for token, ner_label in zip(tokens, ner_results):
    if ner_label.startswith("B-"):
        # 如果是实体的开始，查找对齐字典
        entity = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(tokenizer.encode(token)))
        if entity in entity_alignment:
            translated_text.append(entity_alignment[entity])
        else:
            translated_text.append(token)
    else:
        translated_text.append(token)
    
    # 合并翻译结果
    translated_text = " ".join(translated_text)
    print(translated_text)

    import re
    
    def clean_text(text):
    """
    清洗文本，去除HTML标签、特殊字符和数字。
    
    参数:
    text (str): 需要清洗的原始文本。
    
    返回:
    str: 清洗后的文本。
    """
    # 去除HTML标签
    text = re.sub(r'<[^>]+>', '', text)
    # 去除非字母字符
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # 去除数字
    text = re.sub(r'\d+', '', text)
    return text
    
    # 示例文本
    text = "<p>这是一段包含HTML标签<p>和一些特殊字符!@#以及数字123的文本。</p>"
    # 清洗文本
    cleaned_text = clean_text(text)
    print(cleaned_text)

    import jieba
    
    def tokenize(text):
    """
    使用jieba库对中文文本进行分词。
    
    参数:
    text (str): 需要分词的中文文本。
    
    返回:
    list: 分词后的词语列表。
    """
    return list(jieba.cut(text))
    
    # 示例文本
    text = "这是一段需要分词的中文文本。"
    # 分词
    tokens = tokenize(text)
    print(tokens)

    from transformers import BertTokenizer, BertForTokenClassification
    import torch
    
    # 加载预训练的BERT模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    model = BertForTokenClassification.from_pretrained('bert-base-chinese', num_labels=3)
    
    # 示例文本
    text = "李华是北京大学的学生。"
    
    # 分词并转换为BERT输入格式
    inputs = tokenizer(text, return_tensors="pt")
    # 预测实体标签
    outputs = model(**inputs)
    # 获取预测结果
    predictions = torch.argmax(outputs.logits, dim=2)
    
    # 解码预测结果
    def decode_predictions(tokens, predictions):
    """
    解码BERT的实体识别预测结果。
    
    参数:
    tokens (list): 分词后的文本。
    predictions (list): 预测的实体标签。
    
    返回:
    list: 解码后的实体列表。
    """
    # 将分词和预测标签组合
    results = [(token, prediction) for token, prediction in zip(tokens, predictions[0].tolist())]
    # 过滤掉特殊标记
    results = [(token, label) for token, label in results if token not in ['[CLS]', '[SEP]']]
    # 解码实体标签
    decoded_results = [(token, tokenizer.convert_ids_to_tokens([label])[0]) for token, label in results]
    return decoded_results
    
    # 解码预测结果
    decoded_results = decode_predictions(tokenizer.tokenize(text), predictions.tolist())
    print(decoded_results)

    entity_translation_dict = {
    "北京大学": "Peking University",
    "李华": "Li Hua"
    }
    
    def translate_entities(text, entity_translation_dict):
    """
    翻译文本中的命名实体。
    
    参数:
    text (str): 原始文本。
    entity_translation_dict (dict): 实体翻译字典。
    
    返回:
    str: 实体翻译后的文本。
    """
    for entity, translation in entity_translation_dict.items():
        text = text.replace(entity, translation)
    return text
    
    # 示例文本
    text = "李华是北京大学的学生。"
    # 翻译实体
    translated_text = translate_entities(text, entity_translation_dict)
    print(translated_text)

    def replace_entities(translated_text, entity_translation_dict):
    """
    替换翻译结果中的命名实体。
    
    参数:
    translated_text (str): 翻译后的文本。
    entity_translation_dict (dict): 实体翻译字典。
    
    返回:
    str: 替换实体后的文本。
    """
    for entity, correct_translation in entity_translation_dict.items():
        if correct_translation in translated_text:
            translated_text = translated_text.replace(correct_translation, entity_translation_dict[entity])
    return translated_text
    
    # 示例文本
    translated_text = "Li Hua is North University's student."
    # 替换实体
    corrected_text = replace_entities(translated_text, entity_translation_dict)
    print(corrected_text)

    # 导入必要的库
    import torch
    from transformers import BertForTokenClassification, BertTokenizer, AdamW
    from torch.utils.data import DataLoader, TensorDataset
    
    # 加载预训练的BERT模型和分词器
    model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased')
    tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
    
    # 准备数据集
    # 假设我们有以下数据样例
    # text: ['I', 'love', 'living', 'in', 'Berlin']
    # labels: ['O', 'O', 'O', 'O', 'B-LOC']
    # 其中，'O'表示非实体，'B-LOC'表示地点实体的开始
    
    # 将文本和标签转换为模型可以接受的格式
    input_ids = tokenizer(text, is_split_into_words=True, return_tensors='pt')['input_ids']
    labels = torch.tensor([labels])
    
    # 创建数据加载器
    dataset = TensorDataset(input_ids, labels)
    dataloader = DataLoader(dataset, batch_size=16)
    
    # 设置优化器
    optimizer = AdamW(model.parameters(), lr=5e-5)
    
    # 训练模型
    model.train()
    for epoch in range(3):  # 训练3轮
    for batch in dataloader:
        input_ids, labels = batch
        outputs = model(input_ids, labels=labels)
        loss = outputs.loss
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

    # 导入必要的库
    import torch
    from transformers import BertTokenizer, BertForTokenClassification, pipeline
    
    # 初始化NER模型
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertForTokenClassification.from_pretrained('dbmdz/bert-large-cased-finetuned-conll03-english')
    
    # 创建NER管道
    ner = pipeline('ner', model=model, tokenizer=tokenizer)
    
    # 输入文本
    text = "Microsoft is based in Redmond, Washington."
    
    # 使用NER管道处理文本
    ner_results = ner(text)
    
    # 输出NER结果
    print(ner_results)
    
    # 假设的翻译结果
    translated_text = "微软总部位于华盛顿州雷德蒙德。"
    
    # 将NER结果应用到翻译结果中，确保实体翻译的准确性
    for entity in ner_results:
    if entity['entity'] == 'I-ORG' or entity['entity'] == 'B-ORG':
        translated_text = translated_text.replace(entity['word'], '微软')
    elif entity['entity'] == 'I-LOC' or entity['entity'] == 'B-LOC':
        translated_text = translated_text.replace(entity['word'], '雷德蒙德')
    elif entity['entity'] == 'I-GPE' or entity['entity'] == 'B-GPE':
        translated_text = translated_text.replace(entity['word'], '华盛顿州')
    
    # 输出最终翻译结果
    print(translated_text)