数据挖掘#特征工程（一）总结

 def cv_loop(X, y, model, N):

    
     mean_auc = 0.
    
     for i in range(N):
    
     X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(
    
                                    X, y, test_size=.20,
    
                                    random_state = i*SEED)
    
     model.fit(X_train, y_train)
    
     preds = model.predict_proba(X_cv)[:,1]
    
     auc = metrics.auc_score(y_cv, preds)
    
     print "AUC (fold %d/%d): %f" % (i + 1, N, auc)
    
     mean_auc += auc
    
     return mean_auc/N
    
   
    
 score_hist = []
    
 N = 10
    
 good_features = set([])
    
 # Greedy feature selection loop
    
 while len(score_hist) < 2 or score_hist[-1][0] > score_hist[-2][0]:
    
     scores = []
    
     for f in range(len(Xts)):
    
     if f not in good_features:
    
         feats = list(good_features) + [f]
    
         Xt = sparse.hstack([Xts[j] for j in feats]).tocsr()
    
         score = cv_loop(Xt, y, model, N)
    
         scores.append((score, f))
    
         print "Feature: %i Mean AUC: %f" % (f, score)
    
     good_features.add(sorted(scores)[-1][1])
    
     score_hist.append(sorted(scores)[-1])
    
     print "Current features: %s" % sorted(list(good_features))
    
      
    
 # Remove last added feature from good_features
    
 good_features.remove(score_hist[-1][1])

 # Encode the data and keep it in a list for easy access during feature selection

    
 OHs = [OneHotEncoder(X[:,[i]]) for i in range(X.shape[1])]
    
 Xts = [o[0] for o in OHs]
    
 getX = lambda gne: sparse.hstack([Xts[i] for i in find(gne)]).tocsr()
    
   
    
 def mutate(gene, mutation_rate=1e-2):
    
     """
    
     Mutation method. Randomly flips the bit of a gene segment at a rate of
    
     mutation rate
    
     """
    
     sel = random.rand(len(gene)) <= mutation_rate
    
     gene[sel] = -gene[sel]
    
     return gene
    
      
    
 class Gene(Individual):
    
     """
    
     Gene class used for feature selection
    
     Implements a fitness function and reproduce function as required by the
    
     Individual base class
    
     """
    
     def fitness(self):
    
     """
    
     The fitness of an group of features is equal to 1 - mean(cross val scores)
    
     """
    
     cv_args = {'X': getX(self.gene),
    
                'y': y,
    
                'score_func': metrics.auc_score,
    
                'cv': cv,
    
                'n_jobs': N_JOBS}
    
     cv_scores = cross_validation.cross_val_score(model, **cv_args)
    
     return 1 - mean(cv_scores)
    
      
    
     def reproduce(self, other, n_times=1, mutation_rate=mutation_rate):
    
     """
    
     Takes another Gene and randomly swaps the genetic material between
    
     this gene and other gene at some cut point n_times. Afterwords, it
    
     mutates the resulting genetic information and creates two new Gene
    
     objects as children
    
      
    
     The high level description:
    
         copy the genes from self and other as g1, g2
    
         do n_times:
    
             randomly generate integer j along the length of the gene
    
             set g1 to g1[:j] + g2[j:]
    
             set g2 to g2[:j] + g1[j:]
    
         mutate g1 and g2
    
         return [Gene(g1), Gene(g2)]
    
     """
    
     lg = len(self.gene)
    
     g1 = copy(self.gene)
    
     g2 = copy(other.gene)
    
     for i in xrange(n_times):
    
         j = random.randint(lg)
    
         g1 = hstack((g1[:j], g2[j:]))
    
         g2 = hstack((g2[:j], g1[j:]))
    
     g1 = mutate(g1, mutation_rate)
    
     g2 = mutate(g2, mutation_rate)
    
     return [Gene(g1), Gene(g2)]
    
  
    
 # Do the feature selection
    
 print "Creating the initial gene pool... be patient"
    
 n_genes = ga_args['population_size']
    
 start_genes = (random.rand(n_genes, len(Xts)) > 0.5).astype(bool)
    
 start_genes = sorted([Gene(g) for g in start_genes])
    
 print 'Running the genetic algorithm...'
    
 gene_pool = ga.evolve(start_genes, n_generations)

 ind = [ i for i in self._indices(x)] # x: feature, a list of indices

    
 if dropout == 1:
    
     dropped = None
    
 else:
    
     dropped = [random.random() > dropout for i in xrange(0,len(ind))]
    
 #######predict stage######## 
    
 for j, i in enumerate(self._indices(x)):           
    
     if dropped != None and dropped[j]:
    
     continue          
    
     wTx += w[i]       
    
  
    
 if dropped != None: 
    
     wTx /= dropout #keep rate
    
   
    
 #######update parameters######
    
 if dropped != None and dropped[j]: 
    
     continue #不更新参数

 # for simplicity, we treat both integer and categorical features as categorical

    
 # INPUT:
    
 #     csv_row: a csv dictionary, ex: {'Lable': '1', 'I1': '357', 'I2': '', ...}
    
 #     D: the max index that we can hash to
    
 # OUTPUT:
    
 #     x: a list of indices that its value is 1
    
 def get_x(csv_row, D):
    
     fullind = []
    
     for key, value in csv_row.items():
    
         s = key + '=' + value
    
         fullind.append(hash(s) % D)
    
     if interaction == True:
    
         indlist2 = []
    
         for i in range(len(fullind)):
    
             for j in range(i+1,len(fullind)):
    
                 indlist2.append(fullind[i] ^ fullind[j]) # Creating interactions using XOR
    
         fullind = fullind + indlist2
    
     x = {}
    
     x[0] = 1  # 0 is the index of the bias term
    
     for index in fullind:
    
         if(not x.has_key(index)):
    
             x[index] = 0
    
         x[index] += 1
    
      
    
     return x  # x contains indices of features that have a value as number of occurences

 with_hash = lambda x, D: (x, hash(x, D))

    
 l = [with_hash("%s=%s" % (i,x[i]), D) for i in x]
    
   
    
 if self.interactions:
    
     k = x.keys()
    
     v = x.values()
    
     L = len(k)
    
     for i in xrange(0, L):
    
         l.extend([with_hash("%s=%s+%s=%s" % (k[i], v[i], k[j], v[j]), D) for j in xrange(i+1, L)])

 def group_data(data, degree=3, hash=hash):

    
     """
    
     numpy.array -> numpy.array
    
      
    
     Groups all columns of data into all combinations of triples
    
     """
    
     new_data = []
    
     m,n = data.shape
    
     for indicies in combinations(range(n), degree):
    
     new_data.append([hash(tuple(v)) for v in data[:,indicies]])
    
     return array(new_data).T

 def group_data(data, degree=3):

    
   new_data = []
    
   m,n = data.shape
    
   for indices in combinations(range(n), degree):
    
     group_ids = all_data.groupby( \
    
       list(all_data.columns[list(indices)])) \
    
       .grouper.group_info[0]
    
     new_data.append(group_ids)
    
   return array(new_data).T