数据挖掘（五）

    import os
    import pandas as pd
    data_folder = os.path.join(os.getcwd(), 'adult')
    adult_filename = os.path.join(data_folder, 'adult.data')
    # 加载数据集文件，文件末尾有两处空行，pandas默认会把倒数第二行空行作为一条有效数据读取
    adult = pd.read_csv(adult_filename, header=None,
    names=["Age", "Work-Class", "fnlwgt", "Education", "Education-Num", 
            "Marital-Status", "Occupation", "Relationship", "Race", "Sex", 
            "Capital-gain", "Capital-loss", "Hours-per-week", "Native-Country", "Earnings-Raw"])
    # 删除包含无效数据的行，inplace=True表示改动当前数据框，而不是新建一个
    adult.dropna(how='all', inplace=True)
    # 查看所有特征的名称
    adult.columns

    # pandas提供了常见统计量的计算方法
    adult['Hours-per-week'].describe()

    adult['Education-Num'].median()

    #使用unique函数获得所有工作的情况，发现部分数据确实，但是不会影响这里的计算
    adult['Work-Class'].unique()

    # 创建LongHours时长特征用来表示一个人每周工作时长是否多于40个小时，把连续值转换为类别型特征
    adult['LongHours'] = adult['Hours-per-week'] > 40

    import numpy as np
    # np.arange(30)返回0~29的连续整数数组
    # reshape方法将数组变成10行3列的矩阵，可以看成10个个体、3个特征的数据集
    X = np.arange(30).reshape((10, 3))
    # 把第二列的数值都为1，那么第一、三列特征值的方差很大，第二列的方差为0
    X[:,1] = 1
    # 创建VarianceThreshold转换器处理数据集后第二列消失了
    from sklearn.feature_selection import VarianceThreshold
    vt = VarianceThreshold()
    Xt = vt.fit_transform(X)
    # 输出每一列的方差
    print(vt.variances_)

    # 从pandas数据框中抽取一部分数据，判断税前收入Earnings-Raw是否达到5万美元，创建目标类别列表，如果达到类别为True
    X = adult[['Age', 'Education-Num', 'Capital-gain', 'Capital-loss', 'Hours-per-week']].values
    y = (adult['Earnings-Raw'] == ' >50K').values
    # 使用SelectKBest转化器类用卡方函数打分
    from sklearn.feature_selection import SelectKBest, chi2
    transformer = SelectKBest(score_func=chi2, k=3)
    # 对相同的数据集进行预处理和转结果为分类效果较好的三个特征
    Xt_chi2 = transformer.fit_transform(X, y)
    # 输出每一列的相关性，相关性最好的是第一、三、五列分别对应着Age年龄、Capital-Gain资本收益和Hours-per-week每周工作时长三个特征
    print(transformer.scores_)

    # 从SciPy导入personr库
    from scipy.stats import pearsonr
    # 包装器函数处理多维数组
    def multivariate_presonr(X, y):
    scores, pvalues = [], []
    # X.shape得到矩阵的行列元组，X.shape[1]获得列数
    for column in range(X.shape[1]):
        cur_score, cur_p = pearsonr(X[:, column], y)
        scores.append(abs(cur_score))
        pvalues.append(cur_p)
    return (np.array(scores), np.array(pvalues))
    # 根据皮尔逊相关系数对特征进行排序
    transformer = SelectKBest(score_func=multivariate_presonr, k=3)
    Xt_pearson = transformer.fit_transform(X, y)
    print(transformer.scores_)

    from sklearn.tree import DecisionTreeClassifier
    from sklearn.model_selection import cross_val_score
    clf = DecisionTreeClassifier(random_state=14)
    scores_chi2 = cross_val_score(clf, Xt_chi2, y, scoring='accuracy')
    scores_pearson = cross_val_score(clf, Xt_pearson, y, scoring='accuracy')
    print('Chi2性能:{0:.3f}, Pearson性能:{1:.3f}'.format(scores_chi2.mean(), scores_pearson.mean()))

    import os
    import pandas as pd
    import numpy as np
    from collections import defaultdict
    
    # 把字符串转换为数字，如果失败捕获异常返回NaN
    def convert_number(x):
    try:
        return float(x)
    except ValueError:
        return np.nan
    # 数据集有几个问题，加载过程需要做些处理，前三个特征是高度、宽度、宽高比，最后一列是数据的类别，其余特征取值为1表示图像的URL、alt属性值
    # 1.前几个特征是数值，但是pandas会把它们当成字符串，我们需要编写将字符串转换为数字的函数
    # 2.数据集中有些值缺失，缺失的值用问号表示，问号不会被转换为浮点型数据，也可以转换为NaN
    data_folder = os.path.join(os.getcwd(), 'advertisements')
    data_filename = os.path.join(data_folder, 'ad.data')
    # 用字典存储所有特征及其转换结果
    converters = defaultdict(convert_number)
    # 把最后一列的值转化为0或1表示每条数据的类别
    converters[1558] = lambda x: 1 if x.strip() == "ad." else 0
    ads = pd.read_csv(data_filename, header=None, converters=converters)
    # 处理特殊符号和NaN
    ads = ads.replace('?', np.nan)
    ads = ads.replace(' ?', np.nan)
    ads = ads.replace('   ?', np.nan)
    ads = ads.replace('     ?', np.nan)
    ads = ads.replace(np.nan, 0)
    # 查看前5行数据
    print(ads[:5])

    X = ads.drop(1558, axis=1).values
    y = ads[1558]

    from sklearn.decomposition import PCA
    pca = PCA(n_components=5)
    Xd = pca.fit_transform(X)
    # Xd矩阵只要5个特征(3279, 5)
    Xd.shape
    # 查看每个特征的方差
    np.set_printoptions(precision=3, suppress=True)
    # 第一个特征的方差对数据集总体方差的贡献率为87.8%
    pca.explained_variance_ratio_

    from sklearn.model_selection import cross_val_score
    from sklearn.tree import DecisionTreeClassifier
    clf = DecisionTreeClassifier(random_state=14)
    scores_reduced = cross_val_score(clf, Xd, y, scoring='accuracy')
    print('正确率:{:.4f}'.format(np.mean(scores_reduced)))

    from matplotlib import pyplot as plt
    # 获取数据集中类别的所有取值，是广告和不是广告
    classes = set(y)
    # 在图形中用什么颜色表示这两个类别
    colors = {'red', 'green'}
    # zip函数把这两个列表组合起来同时遍历
    for cur_class, color in zip(classes, colors):
    # 给当前类别的个体创建遮罩层
    mask = (y == cur_class).values
    # scatter函数显示位置
    plt.scatter(Xd[mask, 0], Xd[mask, 1], marker='o', color=color, label=int(cur_class))
    # 创建图示显示图像    
    plt.legend()
    plt.show()

    # 导入用来设置API的TransformerMixin类
    from sklearn.base import TransformerMixin
    from sklearn.utils import as_float_array
    
    # 自定义转换器类继承TransformerMixin类，定义fit和transform方法
    class MeanDiscrete(TransformerMixin):
    def fit(self, X, y=None):
        X = as_float_array(X)
        self.mean = np.mean(X, axis=0)
        # fit函数返回本身确保在转换器中能够进行链式调用
        return self
    
    def transform(self, X):
        X = as_float_array(X)
        # 输入必须是numpy数组，检查输入的数据列数是否一致
        assert X.shape[1] == self.mean.shape[0]
        return X > self.mean
    
    mean_discrete = MeanDiscrete()
    X_mean = mean_discrete.fit_transform(X)

    import numpy as np
    from numpy.testing import assert_array_equal
    
    # 测试函数以test_开头便于工具自动查找并运行测试
    def test_meandiscrete():
    X_test = np.array([[ 0,  2],
                        [ 3,  5],
                        [ 6,  8],
                        [ 9, 11],
                        [12, 14],
                        [15, 17],
                        [18, 20],
                        [21, 23],
                        [24, 26],
                        [27, 29]])
    # 创建转换函数用测试数据进行训练
    mean_discrete = MeanDiscrete()
    mean_discrete.fit(X_test)
    assert_array_equal(mean_discrete.mean, np.array([13.5, 15.5]))
    X_transformed = mean_discrete.transform(X_test)
    X_expected = np.array([[ 0,  0],
                            [ 0, 0],
                            [ 0, 0],
                            [ 0, 0],
                            [ 0, 0],
                            [ 1, 1],
                            [ 1, 1],
                            [ 1, 1],
                            [ 1, 1],
                            [ 1, 1]])
    assert_array_equal(X_transformed, X_expected)
    test_meandiscrete()

    from sklearn.pipeline import Pipeline
    pipeline = Pipeline([('mean_discrete', MeanDiscrete()),
                     ('classifier', DecisionTreeClassifier(random_state=14))])
    scores_mean_discrete = cross_val_score(pipeline, X, y, scoring='accuracy')
    print("Mean Discrete performance: {0:.3f}".format(scores_mean_discrete.mean()))

    import os
    import pandas as pd
    import numpy as np
    from sklearn.feature_selection import VarianceThreshold, SelectKBest, chi2
    from scipy.stats import pearsonr
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.model_selection import cross_val_score
    
    def multivariate_presonr(X, y):
    scores, pvalues = [], []
    for column in range(X.shape[1]):
        # 接收两个数组作为参数, 第一个参数是一维数组，返回两个特征的皮尔逊相关系数和p值
        cur_score, cur_p = pearsonr(X[:, column], y)
        scores.append(abs(cur_score))
        pvalues.append(cur_p)
    return (np.array(scores), np.array(pvalues)) 
    # X = np.arange(30).reshape((10, 3))
    # X[:, 1] = 1
    # vt = VarianceThreshold()
    # Xt = vt.fit_transform(X)
    # print(vt.variances_)
    data_folder = os.path.join(os.getcwd(), 'adult')
    adult_filename = os.path.join(data_folder, 'adult.data')
    adult = pd.read_csv(adult_filename, header=None,
    names=["Age", "Work-Class", "fnlwgt", "Education", "Education-Num", 
            "Marital-Status", "Occupation", "Relationship", "Race", "Sex", 
            "Capital-gain", "Capital-loss", "Hours-per-week", "Native-Country", "Earnings-Raw"])
    adult.dropna(how='all', inplace=True)
    adult.columns
    adult['Hours-per-week'].describe()
    adult['Education-Num'].median()
    adult['Work-Class'].unique()
    adult['LongHours'] = adult['Hours-per-week'] > 40
    X = adult[['Age', 'Education-Num', 'Capital-gain', 'Capital-loss', 'Hours-per-week']].values
    y = (adult['Earnings-Raw'] == ' >50K').values
    transformer = SelectKBest(score_func=chi2, k=3)
    Xt_chi2 = transformer.fit_transform(X, y)
    #print(transformer.scores_)
    transformer = SelectKBest(score_func=multivariate_presonr, k=3)
    Xt_pearson = transformer.fit_transform(X, y)
    #print(transformer.scores_)
    clf = DecisionTreeClassifier(random_state=14)
    scores_chi2 = cross_val_score(clf, Xt_chi2, y, scoring='accuracy')
    scores_pearson = cross_val_score(clf, Xt_pearson, y, scoring='accuracy')
    print('Chi2性能:{0:.3f}, Pearson性能:{1:.3f}'.format(scores_chi2.mean(), scores_pearson.mean()))

    import os
    import pandas as pd
    import numpy as np
    from collections import defaultdict
    from sklearn.decomposition import PCA
    from sklearn.model_selection import cross_val_score
    from sklearn.tree import DecisionTreeClassifier
    from matplotlib import pyplot as plt
    
    def convert_number(x):
    try:                       
        return float(x)
    except ValueError:        
        return np.nan
    
    data_folder = os.path.join(os.getcwd(), 'advertisements')
    data_filename = os.path.join(data_folder, 'ad.data')
    converters = defaultdict(convert_number)
    converters[1558] = lambda x: 1 if x.strip() == "ad." else 0
    ads = pd.read_csv(data_filename, header=None, converters=converters, low_memory=False)
    ads = ads.replace('?', np.nan)
    ads = ads.replace(' ?', np.nan)
    ads = ads.replace('   ?', np.nan)
    ads = ads.replace('     ?', np.nan)
    ads = ads.replace(np.nan, 0)
    #print(ads[:5])
    X = ads.drop(1558, axis=1).values
    y = ads[1558]
    pca = PCA(n_components=5)
    Xd = pca.fit_transform(X)
    #Xd.shape
    np.set_printoptions(precision=3, suppress=True)
    pca.explained_variance_ratio_
    clf = DecisionTreeClassifier(random_state=14)
    scores_reduced = cross_val_score(clf, Xd, y, scoring='accuracy')
    print('正确率:{:.4f}'.format(np.mean(scores_reduced)))
    classes = set(y)
    colors = {'red', 'green'}
    for cur_class, color in zip(classes, colors):
    mask = (y == cur_class).values
    plt.scatter(Xd[mask, 0], Xd[mask, 1], marker='o', color=color, label=int(cur_class))
    plt.legend()
    plt.show()