第十二章：huggingface的resume训练的Demo

    WEIGHTS_NAME = "pytorch_model.bin"
    WEIGHTS_INDEX_NAME = "pytorch_model.bin.index.json"
    ADAPTER_CONFIG_NAME = "adapter_config.json"
    ADAPTER_WEIGHTS_NAME = "adapter_model.bin"
    ADAPTER_SAFE_WEIGHTS_NAME = "adapter_model.safetensors"
    TF2_WEIGHTS_NAME = "tf_model.h5"
    TF2_WEIGHTS_INDEX_NAME = "tf_model.h5.index.json"
    TF_WEIGHTS_NAME = "model.ckpt"
    FLAX_WEIGHTS_NAME = "flax_model.msgpack"
    FLAX_WEIGHTS_INDEX_NAME = "flax_model.msgpack.index.json"
    SAFE_WEIGHTS_NAME = "model.safetensors"
    SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json"
    CONFIG_NAME = "config.json"
    FEATURE_EXTRACTOR_NAME = "preprocessor_config.json"
    IMAGE_PROCESSOR_NAME = FEATURE_EXTRACTOR_NAME
    GENERATION_CONFIG_NAME = "generation_config.json"
    MODEL_CARD_NAME = "modelcard.json"

    这些参数是用于指定在使用 Hugging Face 库时保存和加载模型和配置文件时使用的文件名。以下是每个参数的详细解释：
    1. `WEIGHTS_NAME`: 保存模型权重的文件名。通常是一个二进制文件，包含了模型的所有参数。 "pytorch_model.bin"
    2. `WEIGHTS_INDEX_NAME`: 保存模型权重索引的文件名。也提供了有关权重文件的额外信息。"pytorch_model.bin.index.json"
    3. `ADAPTER_CONFIG_NAME`: 适配器模型的配置文件名。适配器是指在预训练模型上添加的额外层，用于进行特定任务的微调或迁移学习。"adapter_config.json"
    4. `ADAPTER_WEIGHTS_NAME`: 保存适配器模型权重的文件名。"adapter_model.bin"
    5. `ADAPTER_SAFE_WEIGHTS_NAME`: 保存适配器模型安全权重的文件名。 "adapter_model.safetensors"
    6. `TF2_WEIGHTS_NAME`: TensorFlow 2.x 格式的模型权重文件名。"tf_model.h5"
    7. `TF2_WEIGHTS_INDEX_NAME`: TensorFlow 2.x 格式的模型权重索引文件名。"tf_model.h5.index.json"
    8. `TF_WEIGHTS_NAME`: TensorFlow 1.x 或 2.x 兼容的模型权重文件名。"model.ckpt"
    9. `FLAX_WEIGHTS_NAME`: 使用 Flax 框架保存的模型权重文件名。Flax 是一个基于 JAX 的深度学习库。"flax_model.msgpack"
    10. `FLAX_WEIGHTS_INDEX_NAME`: Flax 模型权重索引文件名。"flax_model.msgpack.index.json"
    11. `SAFE_WEIGHTS_NAME`: 安全模型权重文件名。类似于 `ADAPTER_SAFE_WEIGHTS_NAME`，用于在某些情况下以安全格式保存模型权重。
    12. `SAFE_WEIGHTS_INDEX_NAME`: 安全模型权重索引文件名。"model.safetensors"
    13. `CONFIG_NAME`: 模型配置文件名。这个文件通常包含了模型的架构、超参数和其他配置信息。"config.json"
    14. `FEATURE_EXTRACTOR_NAME`: 特征提取器的配置文件名。特征提取器通常用于从原始数据中提取特征，例如文本的词嵌入。"preprocessor_config.json"
    15. `IMAGE_PROCESSOR_NAME`: 图像处理器的配置文件名。类似于 `FEATURE_EXTRACTOR_NAME`，用于处理图像数据的特征提取器。
    16. `GENERATION_CONFIG_NAME`: 生成模型的配置文件名。用于生成文本或其他类型的数据的模型的配置信息。"generation_config.json"
    17. `MODEL_CARD_NAME`: 模型卡片的文件名。模型卡片通常包含了模型的描述、使用方式、限制和其他相关信息，用于帮助用户了解和使用模型。"modelcard.json"
    这些参数提供了在 Hugging Face 库中加载和保存模型及其相关配置时所需的文件命名约定。

    # 定义训练参数
    training_args = TrainingArguments(
    output_dir='./out_dirs',
    	...
    save_safetensors=True,  # 默认为False
    
    )
    
    # state_dict可为None，safe_serialization=self.args.save_safetensors参数为True，output_dir为文件路径
    self.model.save_pretrained( output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors)

    # 开始训练
    trainer.train()
    trainer.save_pretrained( output_dir, state_dict=None, safe_serialization=True)

    self.do_grad_scaling = False
    if (args.fp16 or args.bf16) and not (self.is_deepspeed_enabled or is_sagemaker_mp_enabled()):
    # deepspeed and SageMaker Model Parallel manage their own half precision
    if self.sharded_ddp is not None:
        if args.half_precision_backend == "cuda_amp":
            self.use_cuda_amp = True
            self.amp_dtype = torch.float16 if args.fp16 else torch.bfloat16
            #  bf16 does not need grad scaling
            self.do_grad_scaling = self.amp_dtype == torch.float16

    # 定义训练参数
    training_args = TrainingArguments(
    output_dir='./out_dirs',
    per_device_train_batch_size=3,
    ...
    learning_rate=2e-4,
    fp16=True
    
    
    )

    if self.do_grad_scaling:
    torch.save(self.scaler.state_dict(), os.path.join(output_dir, SCALER_NAME))

    torch.save(self.optimizer.state_dict(), os.path.join(output_dir, OPTIMIZER_NAME))
    
    torch.save(self.lr_scheduler.state_dict(), os.path.join(output_dir, SCHEDULER_NAME))

    # Save the Trainer state
    if self.args.should_save:
    self.state.save_to_json(os.path.join(output_dir, TRAINER_STATE_NAME)) # TRAINER_STATE_NAME=trainer_state.json

    # Save RNG state in non-distributed training
     rng_states = {
     "python": random.getstate(),
     "numpy": np.random.get_state(),
     "cpu": torch.random.get_rng_state(),
     }
     if torch.cuda.is_available():
     if self.args.parallel_mode == ParallelMode.DISTRIBUTED:
         # In non distributed, we save the global CUDA RNG state (will take care of DataParallel)
         rng_states["cuda"] = torch.cuda.random.get_rng_state_all()
     else:
         rng_states["cuda"] = torch.cuda.random.get_rng_state()
    
     if is_torch_tpu_available():
     rng_states["xla"] = xm.get_rng_state()
    
     # A process can arrive here before the process 0 has a chance to save the model, in which case output_dir may
     # not yet exist.
     os.makedirs(output_dir, exist_ok=True)
    
     if self.args.world_size <= 1:
     torch.save(rng_states, os.path.join(output_dir, "rng_state.pth"))
     else:
     torch.save(rng_states, os.path.join(output_dir, f"rng_state_{self.args.process_index}.pth"))

    def _inner_training_loop(
        self, batch_size=None, args=None, resume_from_checkpoint=None, trial=None, ignore_keys_for_eval=None
    ):
    		for epoch in range(epochs_trained, num_train_epochs): 
    			...
    			for step, inputs in enumerate(epoch_iterator):
    				 ...
    				 self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) # 有条件的进入
    		 ...
    		 self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)

    import os
    os.environ["CUDA_VISIBLE_DEVICES"] = "0"
    
    from torch.utils.data import Dataset
    
    import torch
    from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments
    import random
    
    # 随机种子
    seed = 42
    random.seed(seed)
    torch.manual_seed(seed)
    
    # 示例数据,给更多数据便于常看数据是否继承
    
    train_data = [
    ["Work is my source of happiness", 1], ["I am full of joy in my work", 1], ["Work makes me feel satisfied and happy", 1],
    ["I feel pain in my work", 0], ["I detest work pressure", 0],["I love every workday", 1],  
    ["Work makes me feel utterly exhausted", 0], ["I am not interested in work", 0], ["Work makes me feel tired", 0],
    ["I am full of passion and enthusiasm for my work", 1], ["I like my job", 1], ["I really like my job", 1],
    ["I feel disappointed and discouraged about work", 0], ["I hate the repetitiveness of work", 0],
    ["Work leaves me physically and mentally exhausted", 0], ["Work makes me feel fulfilled and happy", 1], 
    ["I have no passion for work", 0], ["Work is my motivation", 1],["Work makes me feel stressed", 0], 
    ["I feel tired and negative about work", 0], ["I despise the daily repetitive tasks at work", 0], 
    ["I am full of love for my work", 1], ["Work is my life", 1], ["I am passionate about my work", 1],
    ["Work makes me feel depressed and hopeless", 0], ["I have lost interest and motivation in work", 0],
    ["I feel tired and resistant towards work", 0], ["Work makes me feel unbearable and intolerable exhaustion", 0], 
    ["I find work monotonous and dull", 0], ["Work leaves me physically and mentally exhausted and helpless", 0], 
    ["Work makes me feel satisfied", 1], ["I enjoy the work process", 1], ["Work makes me feel excited", 1], 
    ["I feel I have lost enthusiasm for work", 0], ["Work makes me feel utterly tired and bored", 0], 
    ["Work makes me feel valuable", 1], ["I love my job", 1], ["Work makes me feel happy", 1], ["I am immersed in my work", 1],  
    ["I am full of passion for my work", 1], ["Work makes me feel happy", 1], ["I do not like work", 0], 
    ["Work makes me feel proud", 1], ["Work is my passion", 1], ["I am full of enthusiasm for my work", 1],
    ["I am tired of work life", 0], ["I do not like going to work", 0], ["I am full of zeal for my work", 1]]
    
    train_texts = [d[0] for d in train_data]
    train_labels = [d[1] for d in train_data]  # 1代表正面，0代表负面
    
    # 加载预训练的Bert tokenizer
    tokenizer = BertTokenizer.from_pretrained("/project/huggingface_demo/bert-base-uncased")
    # 数据编码
    train_encodings = tokenizer(train_texts, truncation=True, padding=True)
    
    class SentimentDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels
    
    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item
    
    def __len__(self):
        return len(self.labels)
    
    # 创建数据集
    train_dataset = SentimentDataset(train_encodings, train_labels)
    
    # 加载预训练的Bert模型
    model = BertForSequenceClassification.from_pretrained("/project/huggingface_demo/bert-base-uncased", num_labels=2)  # 最好给绝对路径
    
    
    # 定义训练参数
    training_args = TrainingArguments(
    output_dir='./out_dirs',
    per_device_train_batch_size=3,
    per_device_eval_batch_size=3,
    num_train_epochs=2,
    logging_dir="./logs",
    report_to="none",
    gradient_accumulation_steps=2,  # 梯度迭代累计
    save_strategy ="steps", # 权重保存策略，默认no，不保存
    save_steps=2,
    save_total_limit = 10,
    learning_rate=2e-4,
    
    )
    
    # 定义Trainer对象
    trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
       
    )
    
    
    
    # 开始训练
    #trainer.train()
    
    trainer.train(resume_from_checkpoint=True)  # resume训练方法
    
    print('ok')
    
    # 以下这种方式也可以重载，但trainer.train(resume_from_checkpoint=True)已有方法
    # trainer.model = trainer.model.from_pretrained("/project/llava_module/huggingface_demo/out_dirs/checkpoint-80")
    # trainer.optimizer.load_state_dict(torch.load("/project/llava_module/huggingface_demo/out_dirs/checkpoint-80/optimizer.pt"))
    # trainer.lr_scheduler.load_state_dict(torch.load("/project/llava_module/huggingface_demo/out_dirs/checkpoint-80/scheduler.pt"))