Ngram模型在机器翻译中的应用

    import math
    import collections
    
    def ngrams(sentence, n):
    words = sentence.split()
    for i in range(len(words)-n+1):
        yield tuple(words[i:i+n])
    
    class NGramLM():
    
    def __init__(self, sentences, n, alpha=0.1):
        self.sentences = list(map(str.lower, sentences)) # convert to lowercase and split into sentences
        self.n = n
        self.alpha = alpha
    
        # build dictionary and count frequency of each word sequence
        self._count = collections.Counter()
        self._vocab = set()
        for sentence in self.sentences:
            for ng in ngrams(sentence, self.n):
                self._count[ng] += 1
                self._vocab |= set(ng)
        
        self._total_word_count = sum([self._count[ng] for ng in self._count if all(''not in token for token in ng)]) + len(self._vocab)*self.alpha
    
    def train(self, smoothing='laplace', weighting='none'):
        # initialize parameters with random values
        params = {}
        for ng in self._vocab:
            params[(tuple(ng[:-1]), ng[-1])] = [math.random(), math.random()]
    
        # training loop
        num_sentences = float(len(self.sentences))
        total_loss = 0.0
        for sentence in self.sentences:
            loss = 0.0
            
            # compute negative log likelihood loss and gradient update for every n-gram in sentence
            features = list(ngrams(sentence, self.n))
            targets = [f[-1] for f in features]
    
            for i in range(len(features)):
                f = features[i]
                target = targets[i]
                
                context = tuple(f[:-1])
    
                # calculate probability using linear interpolation between current parameter estimate and fixed uniform distribution
                p = ((params[context][target == t])[0] + self.alpha)/(float(self._total_word_count)+self.alpha*(self.n**len(self._vocab)))
                
                loss -= math.log(max(p, 1e-20)) # add small value to prevent underflow errors
        
                grad = [- (1 if target == t else 0) / max(p, 1e-20)
                          for t in sorted(set(targets))]
                
                # smooth by adding Laplace or weighted version
                if smoothing=='laplace':
                    grad = [(g+self.alpha)/float(self._total_word_count+self.alpha*self.n**(len(self._vocab)))
                            for g in grad]
                elif smoothing=='weighted':
                    pass
                    
                # scale gradients for learning rate and update parameters
                delta = [d*min(1/num_sentences, 1./(self._count[f]+1e-10))
                         for d in grad]
                params[context] = [a-d for a,d in zip(params[context],delta)]
    
            total_loss += loss
        
        return params
    
    def evaluate(self, model):
        score = 0.0
        total_word_count = 0.0
        for sentence in self.sentences:
            ngram_counts = collections.defaultdict(int)
            ngram_scores = []
            
            # generate predictions from language model
            for i in range(len(sentence)-self.n+1):
                feature = tuple(sentence[i:i+self.n])
                
                context = tuple(feature[:-1])
                next_word = feature[-1]
    
                if context in model:
                    prob = model[context][next_word][0]/model[context][None][0]
                    ngram_counts[next_word] += int(prob > 0.5)
                    ngram_scores.append(prob)
    
            # use n-best candidates for prediction
            scores = sorted([(s,l) for l,s in enumerate(ngram_scores)], reverse=True)[:len(feature)]
            pred = sorted([s for s,_ in scores], reverse=True)[0]
            
            total_word_count += len(feature)
            score += sum([score for _,score in scores])
            
        print("perplexity:", math.exp(-score/total_word_count))
        
        return {'perplexity': math.exp(-score/total_word_count)}

    from nltk.tokenize import sent_tokenize
    import re
    
    train_data = "The quick brown fox jumps over the lazy dog." 
    test_data = "The slow blue elephant leaps over the sleepy rat."
    
    sents = [' '.join(re.findall('\w+', s)) for s in sent_tokenize(train_data)]
    print(sents) #[u'the quick brown fox jumps over the lazy dog']
    
    lm = NGramLM(sents, n=2) # initialize object with n=2, default alpha=0.1
    model = lm.train(smoothing='laplace') # train on training data with laplace smoothing
    
    results = lm.evaluate(model) # evaluate test data using trained model
    print(results['perplexity']) # perplexity should be around 179.8736

    sents = [' '.join(re.findall('\w+', s)) for s in sent_tokenize(train_data)]
    print(sents) #[u'the quick brown fox jumps over the lazy dog']

    lm = NGramLM(sents, n=2) # initialize object with n=2, default alpha=0.1
    model = lm.train(smoothing='laplace') # train on training data with laplace smoothing

    for context,dist in model.items():
    print('{} -> {}'.format(context, dist))
    
    #{()} -> [(0.067400187489713293, 0.27331216303774024), ('jumps', 0.26687836962259763), ('over', 0.26687836962259763), ('lazy', 0.13343918481129881), ('dog.', 0.13343918481129881)}
    #(('the', 'quick'), 0.067400187489713293) -> [('jumps', 0.047138452236325245), ('over', 0.047138452236325245), ('lazy', 0.027564892222103653), ('dog.', 0.027564892222103653)]
    #(('quick', 'brown'), 0.067400187489713293) -> [('fox', 0.03030870242781265), ('jumps', 0.016269337639611778), ('over', 0.016269337639611778), ('lazy', 0.013782446111051827), ('dog.', 0.013782446111051827)]
    #(('brown', 'fox'), 0.067400187489713293) -> [('jumps', 0.01910926629664261), ('over', 0.01910926629664261), ('lazy', 0.013782446111051827), ('dog.', 0.013782446111051827)]
    #('fox', ()) -> [('jumps', 0.15366741288707247), ('over', 0.15366741288707247), ('lazy', 0.07683370644353624), ('dog.', 0.07683370644353624)]
    #('jumps', ()) -> [('over', 0.23094269363686373), ('lazy', 0.07683370644353624), ('dog.', 0.07683370644353624)]
    #('over', ()) -> [('lazy', 0.10620186648927612), ('dog.', 0.10620186648927612)]
    #('lazy', ()) -> [('dog.', 0.10620186648927612)]
    #('dog.', ()) -> []

    results = lm.evaluate(model) # evaluate test data using trained model
    print(results['perplexity']) # perplexity should be around 179.8736

    {'perplexity': 179.87361710221525}
    179.87361710221525