AI Mass人工智能大模型即服务时代：大模型如何改变物联网

    import tensorflow as tf
    
    # hyperparameters
    hidden_size = 256
    num_layers = 3
    embedding_dim = 128
    max_sequence_length = 10
    vocab_size = 10000
    
    class Seq2seqModel(tf.keras.Model):
    def __init__(self):
        super().__init__()
    
        # encoder
        self.encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size+1, output_dim=embedding_dim)
        self.encoder_lstm = tf.keras.layers.LSTM(units=hidden_size, return_state=True, return_sequences=False, dropout=0.5)
    
        # decoder
        self.decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size+1, output_dim=embedding_dim)
        self.decoder_lstm = tf.keras.layers.LSTM(units=hidden_size, return_state=True, return_sequences=True, dropout=0.5)
        self.output_dense = tf.keras.layers.Dense(units=vocab_size+1)
    
    def call(self, input_data, target_data):
        """Forward pass"""
    
        batch_size = tf.shape(input_data)[0]
    
        # encode the input data using LSTM layers
        encoder_inputs = self.encoder_embedding(input_data)    # (batch_size, max_sequence_length, embedding_dim)
        _, state_h, state_c = self.encoder_lstm(encoder_inputs)   # both state vectors have shape (batch_size, hidden_size)
    
        # initialize the decoder with zeros and set the initial states to that of the encoder
        decoder_inputs = tf.expand_dims([vocab_size]*batch_size, 1)     # start tokens for each sample in batch
        decoder_states = [state_h, state_c]                              # first input is the previous state from the encoder
    
        # decode the encoded data using LSTM layers
        for t in range(max_sequence_length):
            decoder_outputs, state_h, state_c = self.decoder_lstm(
                inputs=tf.expand_dims(decoder_inputs[:,t], axis=-1), 
                initial_state=[state_h, state_c])
    
            outputs = self.output_dense(decoder_outputs)
            predicted_id = tf.argmax(outputs, axis=1)
            decoder_inputs = tf.concat([decoder_inputs, 
                                        tf.one_hot(predicted_id, depth=vocab_size+1)],
                                        axis=-1)
    
        # loss function
        masks = tf.math.logical_not(tf.equal(target_data, vocab_size))    # mask out padding values
        predictions_padded = tf.boolean_mask(decoder_inputs, masks)        # remove padding and create tensors for computing accuracy and perplexity
        targets_padded = tf.boolean_mask(target_data, masks)              # remove padding 
        crossentropy = tf.keras.losses.sparse_categorical_crossentropy(y_true=targets_padded, y_pred=predictions_padded, from_logits=True)
        loss = tf.reduce_mean(crossentropy)
    
        return {"loss": loss}
    
    model = Seq2seqModel()
    optimizer = tf.keras.optimizers.Adam(lr=1e-3)
    checkpoint_dir = './training_checkpoints'
    ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model)
    manager = tf.train.CheckpointManager(ckpt, checkpoint_dir, max_to_keep=3)
    ckpt.restore(manager.latest_checkpoint).expect_partial()
    if manager.latest_checkpoint:
      print("Restored from {}".format(manager.latest_checkpoint))
    else:
      print("Initializing from scratch.")
    
    
    @tf.function
    def train_step(input_data, target_data):
    with tf.GradientTape() as tape:
        results = model(input_data, target_data)
        loss = results["loss"]
    
    variables = model.trainable_variables
    gradients = tape.gradient(loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))
    
    return loss
    
    for epoch in range(EPOCHS):
    total_loss = 0
    for input_batch, target_batch in dataset:
        loss = train_step(input_batch, target_batch)
        total_loss += loss
    
    avg_loss = total_loss / len(dataset)
    template = 'Epoch {} Loss {:.4f}'
    print(template.format(epoch+1, avg_loss))
    save_path = ckpt.save(file_prefix=checkpoint_dir+'/ckpt')
    print('Saved checkpoint for step {}: {}'.format(int(ckpt.step), save_path))
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    
    代码解读