第三章：huggingface的Trainner参数解读

     if args is None:
     output_dir = "tmp_trainer"
     logger.info(f"No `TrainingArguments` passed, using `output_dir={output_dir}`.")
     args = TrainingArguments(output_dir=output_dir)
     self.args = args

    from transformers import BertForSequenceClassification
    from transformers import Trainer, TrainingArguments
    
    # 定义一个函数，用于实例化模型
    def model_init():
    return BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=2)
    
    # 定义训练参数
    training_args = TrainingArguments(
    "model-init-demo",
    evaluation_strategy="epoch",
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=3,
    logging_dir="./logs",
    logging_steps=100,
    save_steps=1000,
    save_total_limit=2,
    )
    
    # 初始化Trainer，传入model_init参数
    trainer = Trainer(
    model_init=model_init,
    args=training_args,
    )
    
    # 训练模型
    trainer.train()

    if model is None:
    if model_init is not None:
        self.model_init = model_init
        model = self.call_model_init()
    else:
        raise RuntimeError("`Trainer` requires either a `model` or `model_init` argument")
    else: 
    if model_init is not None:
        warnings.warn(
            "`Trainer` requires either a `model` or `model_init` argument, but not both. `model_init` will"
            " overwrite your model when calling the `train` method. This will become a fatal error in the next"
            " release.",
            FutureWarning,
        )
    self.model_init = model_init

    # Seed must be set before instantiating the model when using model
    enable_full_determinism(self.args.seed) if self.args.full_determinism else set_seed(self.args.seed)

    
    def enable_full_determinism(seed: int, warn_only: bool = False):
    """
    Helper function for reproducible behavior during distributed training. See
    - https://pytorch.org/docs/stable/notes/randomness.html for pytorch
    - https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism for tensorflow
    """
    # set seed first
    set_seed(seed)
    
    if is_torch_available():
        # Enable PyTorch deterministic mode. This potentially requires either the environment
        # variable 'CUDA_LAUNCH_BLOCKING' or 'CUBLAS_WORKSPACE_CONFIG' to be set,
        # depending on the CUDA version, so we set them both here
        os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
        os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
        torch.use_deterministic_algorithms(True, warn_only=warn_only)
        # Enable CUDNN deterministic mode
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    if is_tf_available():
        tf.config.experimental.enable_op_determinism()

    def set_seed(seed: int):
    """
    Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch` and/or `tf` (if installed).
    
    Args:
        seed (`int`): The seed to set.
    """
    random.seed(seed)
    np.random.seed(seed)
    if is_torch_available():
        torch.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
        # ^^ safe to call this function even if cuda is not available
    if is_tf_available():
        tf.random.set_seed(seed)

    per_gpu_train_batch_size: Optional[int] = field(
    default=None,
    metadata={
        "help": (
            "Deprecated, the use of `--per_device_train_batch_size` is preferred. "
            "Batch size per GPU/TPU core/CPU for training."
        )
    },
    )

    @property
    def train_batch_size(self) -> int:
    """
    The actual batch size for training (may differ from `per_gpu_train_batch_size` in distributed training).
    """
    if self.per_gpu_train_batch_size:
        logger.warning(
            "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future "
            "version. Using `--per_device_train_batch_size` is preferred."
        )
    per_device_batch_size = self.per_gpu_train_batch_size or self.per_device_train_batch_size
    train_batch_size = per_device_batch_size * max(1, self.n_gpu)
    return train_batch_size

    # Internal variables to help with automatic batch size reduction
    self._train_batch_size = args.train_batch_size

    default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)
    self.data_collator = data_collator if data_collator is not None else default_collator

    def torch_default_data_collator(features: List[InputDataClass]) -> Dict[str, Any]:
    import torch
    
    if not isinstance(features[0], Mapping):
        features = [vars(f) for f in features]
    first = features[0]
    batch = {}
    
    # Special handling for labels.
    # Ensure that tensor is created with the correct type
    # (it should be automatically the case, but let's make sure of it.)
    if "label" in first and first["label"] is not None:
        label = first["label"].item() if isinstance(first["label"], torch.Tensor) else first["label"]
        dtype = torch.long if isinstance(label, int) else torch.float
        batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype)
    elif "label_ids" in first and first["label_ids"] is not None:
        if isinstance(first["label_ids"], torch.Tensor):
            batch["labels"] = torch.stack([f["label_ids"] for f in features])
        else:
            dtype = torch.long if type(first["label_ids"][0]) is int else torch.float
            batch["labels"] = torch.tensor([f["label_ids"] for f in features], dtype=dtype)
    
    # Handling of all other possible keys.
    # Again, we will use the first element to figure out which key/values are not None for this model.
    for k, v in first.items():
        if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
            if isinstance(v, torch.Tensor):
                batch[k] = torch.stack([f[k] for f in features])
            elif isinstance(v, np.ndarray):
                batch[k] = torch.tensor(np.stack([f[k] for f in features]))
            else:
                batch[k] = torch.tensor([f[k] for f in features])
    
    return batch

    @dataclass
    class DataCollatorWithPadding:
     
    tokenizer: PreTrainedTokenizerBase
    padding: Union[bool, str, PaddingStrategy] = True
    max_length: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None
    return_tensors: str = "pt"
    
    def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
        batch = self.tokenizer.pad(
            features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors=self.return_tensors,
        )
        if "label" in batch:
            batch["labels"] = batch["label"]
            del batch["label"]
        if "label_ids" in batch:
            batch["labels"] = batch["label_ids"]
            del batch["label_ids"]
        return batch