零基础数据挖掘——金融风控（三）特征工程

 import datatime

    
 #转化成时间格式
    
 for data in [train, testA]:
    
     data['issueDate'] = pd.to_datetime(data['issueDate'],format='%Y-%m-%d')
    
     startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
    
     #构造时间特征
    
     data['issueDateDT'] = data['issueDate'].apply(lambda x: x-startdate).dt.days
    
 sns.histplot(data_train['issueDateDT'],kde=True)

 for data in [data_train, data_testA]:

    
     data['issueDate'] = pd.to_datetime(data['issueDate'],format='%Y-%m-%d')
    
     data['issueDateM'] = data['issueDate'].dt.month
    
 data_train['issueDateM'].value_counts(dropna=False).sort_index()

 #定义一个函数：非零值取空格前的数字，且设置为int8格式

    
 def employmentLength_to_int(s):
    
     if pd.isnull(s):
    
     return s
    
     else:
    
     return np.int8(s.split()[0])
    
 for data in [data_train, data_testA]:
    
     data['employmentLength'].replace(to_replace='10+ years', value='10 years', inplace=True)
    
     data['employmentLength'].replace('< 1 year', '0 years', inplace=True)
    
     data['employmentLength'] = data['employmentLength'].apply(employmentLength_to_int)
    
 data_train['employmentLength'].value_counts(dropna=False).sort_index()

 #取最后4个数字作为年份

    
 for data in [data_train, data_testA]:
    
     data['earliesCreditLine'] = data['earliesCreditLine'].apply(lambda s: int(s[-4:]))
    
 data['earliesCreditLine'].value_counts(dropna=False).sort_index()

 data_train.fillna(0, inplace=True) # 填充 0

    
 data_train.fillna(train_data.mean(),inplace=True) # 填充均值
    
 data_train.fillna(train_data.median(),inplace=True) # 填充中位数
    
 data_train.fillna(train_data.mode(),inplace=True) # 填充众数
    
 data_train.fillna(method='pad', inplace=True) # 填充前一条数据的值
    
 data_train.fillna(method='bfill', inplace=True) # 填充后一条数据的值
    
 data_train.fillna(method='bfill', limit = 2, inplace=True) ## 填充后一条数据的值,最多连着填充两个

 #填充KNN数据：先利用knn计算临近的k个数据，然后填充他们的均值

    
 from fancyimpute import KNN
    
 data_train_x = pd.DataFrame(KNN(k=6).fit_transform(data_train_x), columns=features)

 #用插值法拟合出缺失的数据，然后进行填充。

    
 for f in features: 
    
     data_train[f] = data_train[f].interpolate()

    data_train.dropna(inplace=True)

 #1判断异常值；

    
 def find_outliers_by_3segama(data,fea):  ##增加一列名为fea+'_outliers'来判断异常值/正常值
    
     data_std = np.std(data[fea])
    
     data_mean = np.mean(data[fea])
    
     outliers_cut_off = data_std 
    
     lower_rule = data_mean - outliers_cut_off
    
     upper_rule = data_mean + outliers_cut_off
    
     data[fea+'_outliers'] = data[fea].apply(lambda x:str('异常值') if x > upper_rule or x < lower_rule else '正常值')
    
     return data
    
 #2分析变量异常值和目标变量的关系
    
 numerical_fea = list(data_train.select_dtypes(exclude=['object']).columns)
    
 for fea in numerical_fea:
    
     data_train = find_outliers_by_3segama(data_train,fea)
    
     print(data_train[fea+'_outliers'].value_counts())
    
     print(data_train.groupby(fea+'_outliers')['isDefault'].sum())
    
 print('*'*10)
    
 #3删除异常值
    
 for fea in numerical_fea:
    
     data_train = data_train[data_train[fea+'_outliers']=='正常值']
    
 data_train = data_train.reset_index(drop=True)  #重置索引
    
 #4进一步删除用于判断异常值的列
    
 out_features = [f for f in data_train.columns if '_outliers' in f]
    
 data_train = data_train.drop(out_features, axis = 1)

 #1判断异常值；

    
 def find_outliers_by_boxplot(data,fea):   
    
     data_q1 = data[fea].quantile(0.25)
    
     data_q3 = data[fea].quantile(0.75)
    
     iqr = data_q3 - data_q1
    
     lower_rule = data_q1 - 1.5*iqr
    
     upper_rule = data_q3 + 1.5*iqr
    
     data[fea+'_outliers'] = data[fea].apply(lambda x:str('异常值') if x > upper_rule or x < lower_rule else '正常值')
    
     return data

 data_train.duplicated()  #判断是否有重复值

    
 data_train.drop_duplicates()  #删除重复值

* 处理缺失值：当数据源可能存在缺失值，此时可以把null单独作为一个分箱。
* 处理异常值：当数据中存在离群点时，可以把其通过分箱离散化处理，从而提高变量的鲁棒性（抗干扰能力）。例如，age若出现200这种异常值，可分入“age > 60”这个分箱里，排除影响。
* 业务解释性：我们习惯于线性判断变量的作用，当x越来越大，y就越来越大。但实际x与y之间经常存在着非线性关系，此时可经过WOE变换。

* （1）最小分箱占比不低于5%
* （2）箱内不能全部是好客户
* （3）连续箱单调

 data_train['loanAmnt_bin1'] = np.floor_divide(data_train['loanAmnt'], 10000)

    
 data_train['loanAmnt_bin1'].value_counts()

 data_train['loanAmnt_bin2'] = np.floor(np.log10(data_train['loanAmnt']))

    
 data_train['loanAmnt_bin2'].value_counts()

 data_train['loanAmnt_bin3'] = pd.qcut(data_train['loanAmnt'], 10, labels=False)

    
 data_train['loanAmnt_bin3'].value_counts()

 ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]

    
 # 如果按年龄分成18-25, 26-35, 36-60, 61以上的若干组，可以使用pandas中的cut
    
 bins = [18, 25, 35, 60, 100]         # 定义箱子的边
    
 cats = pd.cut(ages, bins)
    
 print(cats)   # 这是个categories对象    通过bin分成了四个区间， 然后返回每个年龄属于哪个区间
    
 # codes属性
    
 print(cats.codes)    #  这里返回一个数组，指明每一个年龄属于哪个区间
    
 print(cats.categories)
    
 print(pd.value_counts(cats))   # 返回结果是每个区间年龄的个数
    
  
    
 # 与区间的数学符号一致， 小括号表示开放，中括号表示封闭， 可以通过right参数改变
    
 print(pd.cut(ages, bins, right=False))

 hot_features = ['grade','subGrade']

    
 data_train_hot = pd.get_dummies(data_train, columns=hot_features)

 from sklearn.preprocessing import LabelEncoder

    
 le = LabelEncoder()
    
 data_train['subGrade'] = le.fit_transform(data_train['subGrade'])
    
 data_testA['subGrade'] = le.transform(data_testA['subGrade'])

 for data in [data_train, data_testA]:

    
     data['grade'] = data['grade'].map({'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,'G':7})
    
 data_train['grade']

 from sklearn.feature_selection import VarianceThreshold

    
 #其中参数threshold为方差的阈值
    
 VarianceThreshold(threshold=3).fit_transform(train,target_train)

 from sklearn.feature_selection import SelectKBest

    
 from scipy.stats import pearsonr
    
 #选择K个最好的特征，返回选择特征后的数据
    
 #第一个参数为计算评估特征是否好的函数，该函数输入特征矩阵和目标向量，
    
 #输出二元组（评分，P值）的数组，数组第i项为第i个特征的评分和P值。在此定义为计算相关系数
    
 #参数k为选择的特征个数
    
  
    
 SelectKBest(k=5).fit_transform(train,target_train)

 from sklearn.feature_selection import SelectKBest

    
 from sklearn.feature_selection import chi2
    
 #参数k为选择的特征个数
    
  
    
 SelectKBest(chi2, k=5).fit_transform(train,target_train)

 from sklearn.feature_selection import SelectKBest

    
 from minepy import MINE
    
 #由于MINE的设计不是函数式的，定义mic方法将其为函数式的，
    
 #返回一个二元组，二元组的第2项设置成固定的P值0.5
    
 def mic(x, y):
    
     m = MINE()
    
     m.compute_score(x, y)
    
     return (m.mic(), 0.5)
    
 #参数k为选择的特征个数
    
 SelectKBest(lambda X, Y: array(map(lambda x:mic(x, Y), X.T)).T, k=2).fit_transform(train,target_train)

 from sklearn.feature_selection import RFE

    
 from sklearn.linear_model import LogisticRegression
    
 #递归特征消除法，返回特征选择后的数据
    
 #参数estimator为基模型
    
 #参数n_features_to_select为选择的特征个数
    
  
    
 RFE(estimator=LogisticRegression(), n_features_to_select=2).fit_transform(train,target_train)

 from sklearn.feature_selection import SelectFromModel

    
 from sklearn.linear_model import LogisticRegression
    
 #带L1惩罚项的逻辑回归作为基模型的特征选择
    
  
    
 SelectFromModel(LogisticRegression(penalty="l1", C=0.1)).fit_transform(train,target_train)

 from sklearn.feature_selection import SelectFromModel

    
 from sklearn.ensemble import GradientBoostingClassifier
    
 #GBDT作为基模型的特征选择
    
 SelectFromModel(GradientBoostingClassifier()).fit_transform(train,target_train)

 for col in ['grade', 'subGrade']:

    
     #将训练数据按col分组，取目标值'isDefault'数据，计算不同'grade'的均值，加上索引列，将col的列名改成col_taregt_mean
    
     temp_dict = train.groupby([col])['isDefault'].agg(['mean']).reset_index().rename(columns={'mean': col + '_target_mean'})
    
     #将temp_dict转化为字典{grade: isDefault_mean}
    
     temp_dict.index = temp_dict[col].values
    
     temp_dict = temp_dict[col + '_target_mean'].to_dict()
    
     #通过字典新建一个col_target_mean列
    
     train[col + '_target_mean'] = train[col].map(temp_dict)
    
     testA[col + '_target_mean'] = testA[col].map(temp_dict)

 from sklearn.decomposition import PCA

    
 pca = PCA(n_components=4)  #4个主成分OR保留0.9的信息
    
 pca_result = pca.fit_transform(df[feat_cols].values)

 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

    
 LDA = LinearDiscriminantAnalysis(n_components=2)
    
 lda_result = LDA.fit_transform(data_train)

 from sklearn.manifold import TSNE

    
 tsne = TSNE(n_components=3, n_iter=300).fit_transform(df[feat_cols][:6000].values)