阿里云天池金融风控训练营【Task3 特征工程】学习笔记

    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import seaborn as sns  #基于matplotlib的图形可视化
    import datetime
    from tqdm import tqdm  #进度提示信息
    from sklearn.preprocessing import LabelEncoder  
    from sklearn.feature_selection import SelectKBest
    from sklearn.feature_selection import chi2
    from sklearn.preprocessing import MinMaxScaler
    import xgboost as xgb
    import lightgbm as lgb  #基于直方图的切分点算法
    from catboost import CatBoostRegressor #基于对称决策树为基学习器实现
    import warnings
    from sklearn.model_selection import StratifiedKFold, KFold
    from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, log_loss
    warnings.filterwarnings('ignore')

1. 

    df.fillna(0,inplace=True) #填充0

    df.fillna(df.mean(),inplace=True) #填充均值
    df.fillna(df.median(),inplace=True) #填充中位数
    df.fillna(df.mode(),inplace=True) #填充众数

    df.fillna(method='pad',inplace=True) #填充前一条数据的值，但前一条不一定有值

    df.fillna(method='bfill',inplace=True) #填充后一条数据的值，但后一条不一定有值

    for f in features: #插值法填充

        df[f] = df[f].interpolate()
    df.dropna(inplace=True)

    from fancyimpute import KNN

    
    train_data_x = pd.DataFrame(KNN(k=6).fit_transform(train_data_x), columns=features)

    #转化成时间格式

    for data in [data_train, data_test_a]:
    data['issueDate'] = pd.to_datetime(data['issueDate'],format='%Y-%m-%d')
    startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
    #构造时间特征
    data['issueDateDT'] = data['issueDate'].apply(lambda x: x-startdate).dt.days

    # 部分类别特征

    cate_features = ['grade', 'subGrade', 'employmentTitle', 'homeOwnership', 'verificationStatus', 'purpose', 'postCode', 'regionCode', \
                 'applicationType', 'initialListStatus', 'title', 'policyCode']
    for f in cate_features:
    print(f, '类型数：', data[f].nunique())

    ###正态分布标准差

    import numpy as np
    import matplotlib.pyplot as plt
    
    def find_outliter_std(random_data,c):
    anomalies = []
    random_data_std = np.std(random_data) 
    random_data_mean = np.mean(random_data) 
    anomaly_cut_off = random_data_std * c 
    lower_limit = random_data_mean - anomaly_cut_off 
    upper_limit = random_data_mean + anomaly_cut_off 
    #print(lower_limit)
    for outlier in random_data: 
        if outlier > upper_limit or outlier < lower_limit: 
            anomalies.append(1) 
        else:
            anomalies.append(0)
    res4=pd.merge(pd.DataFrame(random_data),pd.DataFrame(anomalies),left_index=True,right_index=True)
    res4.columns=['values','is_outliter']
    return res4

    import numpy as np

    import pandas as pd
    import scipy
    def find_outliter_quantile(random_data,c):
    anomalies = []
    random_data_q1 = np.quantile(random_data,0.25)
    random_data_q3 = np.quantile(random_data,0.75) 
    anomaly_cut_off = (random_data_q3-random_data_q1) * c #c一般为1.5或者3，但是个人认为可以根据实际业务情况进行更改，比如涨幅超过1倍的情况就是极端异常的情况，那我们可以c取值为1
    lower_limit = random_data_q1 - anomaly_cut_off 
    upper_limit = random_data_q3 + anomaly_cut_off 
    for outlier in random_data: 
        if outlier > upper_limit or outlier < lower_limit: 
            anomalies.append(1) 
        else:
            anomalies.append(0)
    res5=pd.merge(pd.DataFrame(random_data),pd.DataFrame(anomalies),left_index=True,right_index=True)
    res5.columns=['values','is_outliter']
    return res5
    
    if __name__=='__main__':
    	data=np.array([1338,1338,2,1047,688,520,1408,225,206,280])
    	outliers=find_outliter_quantile(random_data,c)

    from sklearn.cluster import DBSCAN

    import numpy as np
    def dbscan(random_data):
    	outlier_detection = DBSCAN(min_samples = 2, eps = 3)
    	clusters = outlier_detection.fit_predict(random_data)
    	print (list(clusters).count(-1))
    	res2=pd.DataFrame(random_data,list(clusters)).reset_index()
    	res2.columns=['is_outliter','values']
    	return res2

    ####k-means方法异常点检测#####

    from sklearn import preprocessing
    from sklearn.cluster import KMeans
    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt
    
    def kmeans(data):
    	data_scale = preprocessing.scale(data)#处理后需要转为dataframe，或者直接手动求均值、方差转换
    	model = KMeans(n_clusters = 2,max_iter = 1000) # k=2，一类认为是异常的，一类是不异常的，最大迭代次数可以根据实际情况进行修改
    	model.fit(data_scale)
    	data_scale = pd.DataFrame(data_scale)
    	#data_scale = pd.DataFrame(data_scale,index = data.index)
    	data_scale['labels'] = model.labels_
    	res3=pd.merge(data_scale,pd.DataFrame(data),left_index=True,right_index=True) 
    	res3.columns=['distance','labels','values']
    	#print (res3)
    	return res3

    #孤立森林

    #最好对数据集去重，在确定孤立点，因为如果多个异常点聚集，那么就不孤立了
    from sklearn.ensemble import IsolationForest
    import numpy as np
    def  isolationforest(random_data):
    	clf = IsolationForest(behaviour = 'new',max_samples='auto', random_state = 1, contamination= 'auto')
    	preds = clf.fit_predict(random_data)
    	print (list(preds).count(-1))
    	res1=pd.DataFrame(random_data,list(preds)).reset_index()
    	res1.columns=['is_outliter','values']
    	return res1

    # 通过除法映射到间隔均匀的分箱中，每个分箱的取值范围都是loanAmnt/1000

    data['loanAmnt_bin1'] = np.floor_divide(data['loanAmnt'], 1000)
    ## 通过对数函数映射到指数宽度分箱
    data['loanAmnt_bin2'] = np.floor(np.log10(data['loanAmnt']))

    data['loanAmnt_bin3'] = pd.qcut(data['loanAmnt'], 10, labels=False)

    #进行转换
    for col in tqdm(['employmentTitle', 'postCode', 'title','subGrade']):
    le = LabelEncoder()
    le.fit(list(data_train[col].astype(str).values) + list(data_test_a[col].astype(str).values))
    data_train[col] = le.transform(list(data_train[col].astype(str).values))
    data_test_a[col] = le.transform(list(data_test_a[col].astype(str).values))

        from sklearn.feature_selection import VarianceThreshold

        #其中参数threshold为方差的阈值
        VarianceThreshold(threshold=3).fit_transform(train,target_train)

        from sklearn.feature_selection import SelectKBest

        from scipy.stats import pearsonr
        #选择K个最好的特征，返回选择特征后的数据
        #第一个参数为计算评估特征是否好的函数，该函数输入特征矩阵和目标向量，
        #输出二元组（评分，P值）的数组，数组第i项为第i个特征的评分和P值。在此定义为计算相关系数
        #参数k为选择的特征个数
        SelectKBest(k=5).fit_transform(train,target_train)

        from sklearn.feature_selection import SelectKBest

        from sklearn.feature_selection import chi2
        #参数k为选择的特征个数
        SelectKBest(chi2, k=5).fit_transform(train,target_train)

        from sklearn.feature_selection import SelectKBest

        from minepy import MINE
        #由于MINE的设计不是函数式的，定义mic方法将其为函数式的，
        #返回一个二元组，二元组的第2项设置成固定的P值0.5
        def mic(x, y):
            m = MINE()
            m.compute_score(x, y)
            return (m.mic(), 0.5)
        #参数k为选择的特征个数
        SelectKBest(lambda X, Y: array(map(lambda x:mic(x, Y), X.T)).T, k=2).fit_transform(train,target_train)

        from sklearn.feature_selection import RFE

        from sklearn.linear_model import LogisticRegression
        #递归特征消除法，返回特征选择后的数据
        #参数estimator为基模型
        #参数n_features_to_select为选择的特征个数
        RFE(estimator=LogisticRegression(), n_features_to_select=2).fit_transform(train,target_train)

        from sklearn.feature_selection import SelectFromModel

        from sklearn.linear_model import LogisticRegression
        #带L1惩罚项的逻辑回归作为基模型的特征选择
        
        SelectFromModel(LogisticRegression(penalty="l1", C=0.1)).fit_transform(train,target_train)

        from sklearn.feature_selection import SelectFromModel

        from sklearn.ensemble import GradientBoostingClassifier
        #GBDT作为基模型的特征选择
        SelectFromModel(GradientBoostingClassifier()).fit_transform(train,target_train)