Kaggle泰坦尼克号生存预测挑战——数据分析代码实现

    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt
    import seaborn as sns
    import warnings
    warnings.filterwarnings('ignore')
    seed =2020

 加载数据方法 ：pd.read_csv(),pd.read_table() 
    
    * pd.read_csv()：读取以‘，’分割的文件到DataFrame,用于读取csv文件(csv用逗号符分隔字符段) 
    * pd.read_table()：读取以‘\t’分割的文件到DataFrame,用于读取tsv文件(tsv用制表符分隔字符段) 
    * 实质上两个方法都是通用的，函数中参数sep可以选定分隔符的类型

 处理大型文件或内存不足时，采用分块读取方式： 
    
    * 使用参数chunksize指定文件块的大小（用于迭代）

 更多常用参数 ![在这里插入图片描述]()

    ##加载数据  不建议把特征名转成中文，在画图时可能出现乱码
    train_df = pd.read_csv('data_train.csv')
    test_df = pd.read_csv('data_test.csv')
    ##数据预览 查看前5行的数据
    train_df.head()

    test_df.head()

    train_df.info()

    train_df.describe()

    test_df.info()

    test_df.describe()

    ### 可以根据自己的需求封装一个函数
    def _data_info(data,categorical_features):
        print('number of train examples = {}'.format(data.shape[0]))
        print('number of train Shape = {}'.format(data.shape))
        print('Features={}'.format(data.columns))
        print('\n--------输出类别特征的种类--------')
        for i in categorical_features:
            if i in list(data.columns):
                print("train："+i+":",list(data[i].unique()))
        print('\n--------缺失值--------')
        missing = data.isnull().sum()
        missing = missing[missing > 0]
        print(missing)
        missing.sort_values(inplace=True)
        missing.plot.bar()
        plt.show()
    def data_info(data_train,data_test,categorical_features):
        print('--------训练集基本概况--------')
        _data_info(data_train,categorical_features)
        print('\n\n--------测试集基本概况--------')
        _data_info(data_test,categorical_features)

 Survived (标签): 取值{0,1},对应未幸存和幸存 
    2. Pclass: 取值{1,2,3},对应船舱级别 
    3. Sex: 取值{male,female},对应性别 
    4. Cabin: 船舱号 
    5. Embarked:取值{S,C,Q}。对应登船港口

    data_info(train_df,test_df,['Survived','Pclass','Sex','Cabin','Embarked','SibSp','Parch'])

 训练集缺失值：

2. 测试集缺失值：418

    #将数据合并一起处理,添加一个train特征用于区分训练集和测试集
    train_df['train'] = 1
    test_df['train'] = 0
    data_df = pd.concat([train_df,test_df],sort=True).reset_index(drop=True)
    ## 删除PassengerId特征
    data_df.drop('PassengerId',inplace=True,axis=1)
    
    ## 先将非数字的类别特征数字化
    from sklearn import preprocessing
    ler_sex = preprocessing.LabelEncoder()
    ler_sex.fit(data_df['Sex'])
    data_df['Sex'] = ler_sex.transform(data_df['Sex'])

    data_df['Embarked'].fillna(data_df['Embarked'].mode()[0],inplace=True)
    ## 填充完Embarker后，先将非数字的类别特征数字化
    ler_Embarked = preprocessing.LabelEncoder()
    ler_Embarked.fit(data_df['Embarked'])
    data_df['Embarked'] = ler_Embarked.transform(data_df['Embarked'])

    abs(data_df.corr()['Age']).sort_values(ascending=False)

    Age         1.000000
    Pclass      0.408106
    SibSp       0.243699
    Fare        0.178740
    Parch       0.150917
    Embarked    0.080195
    Survived    0.077221
    Sex         0.063645
    train       0.018528

    y = data_df['Age']
    plt.figure(1)
    plt.title('Distribution of Age')
    sns.distplot(y, kde=True)
    
    ## 不同性别的年龄分布，可以看出他们的分布趋于相同
    plt.figure(2);
    Age_Sex0 = data_df.loc[data_df['Sex']==0,'Age']
    Age_Sex1 = data_df.loc[data_df['Sex']==1,'Age']
    plt.title('Distribution of Age in Sex');
    plt.legend(['Sex0','Sex1']);
    sns.distplot(Age_Sex0, kde=True);
    sns.distplot(Age_Sex1, kde=True);

    Age_p1 = data_df.loc[data_df['Pclass']==1,'Age']
    Age_p2 = data_df.loc[data_df['Pclass']==2,'Age']
    Age_p3 = data_df.loc[data_df['Pclass']==3,'Age']
    sns.distplot(Age_p1,kde=True,color='b')
    sns.distplot(Age_p2,kde=True,color='green')
    sns.distplot(Age_p3,kde=True,color='grey')
    plt.title('Distribution of Age in Pclass')
    plt.legend(['p1','p2','p3'])

    Age_Pclass = data_df.groupby([ 'Pclass']).median()['Age']
    for pclass in range(1, 4):
        print('Median age of Pclass {}: {}'.format(pclass,Age_Pclass [pclass]))
    print('Median age of all passengers: {}'.format(data_df['Age'].median()))
    
    # 根据Pclass填充Age值
    data_df['Age'] = data_df.groupby(['Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))

    #查看Fare缺失的样本
    data_df[data_df['Fare'].isnull()]

    ## Pclass对价格的影响很大
    abs(data_df.corr()['Fare']).sort_values(ascending=False)

    Fare        1.000000
    Pclass      0.558629
    Survived    0.257307
    Embarked    0.238005
    Parch       0.221539
    Age         0.202512
    Sex         0.185523
    SibSp       0.160238
    train       0.030831

    ## 聚合数据属性
    print(data_df.groupby(['Pclass', 'Parch','SibSp','Embarked']).Fare.max()[3][0][0][0])#18.7875
    print(data_df.groupby(['Pclass', 'Parch','SibSp','Embarked']).Fare.min()[3][0][0][0])#4.0125
    print(data_df.groupby(['Pclass', 'Parch','SibSp','Embarked']).Fare.median()[3][0][0][0])#7.2292
    print(data_df.groupby(['Pclass', 'Parch','SibSp','Embarked']).Fare.mean()[3][0][0][0])#7.923984210526318
    ## 选择中位数进行填充
    data_df['Fare'].fillna(data_df.groupby(['Pclass', 'Parch','SibSp','Embarked'])['Fare'].median()[3][0][0][0],inplace=True)

    data_df.drop('Cabin',inplace=True,axis=1)

    #从data_df得到训练集
    train_data = data_df[data_df.train==1]
    train_data['Survived'] = train_df['Survived']
    train_data.drop('train',axis=1,inplace=True)
    #从data_df得到测试训练集
    test_data = data_df[data_df.train==0]
    test_data.drop(['Survived','train'],axis=1,inplace=True)

    train_data.corr()

    ### 从幸存和性别成负相关程度较大
    ### 从幸存和Pclass 成负相关程度较大
    ### 从幸存和Fare 成负相关程度较大
    train_data.corr()['Survived'].sort_values(ascending=False)

    Survived    1.000000
    Fare        0.257307
    Parch       0.081629
    SibSp      -0.035322
    Age        -0.046230
    Embarked   -0.167675
    Pclass     -0.338481
    Sex        -0.543351

    plt.figure( figsize=(10, 10))
    plt.title('Train Set Correlation HeatMap ',y=1,size=16)
    sns.heatmap(train_data.corr(),square = True,  vmax=0.7,annot=True,cmap='Accent')

    plt.bar(['Not Survived','Survived'],train_data['Survived'].value_counts().values)
    plt.title('Train_Set_Survived')

    test_data.corr()

    plt.figure( figsize=(10, 10))
    plt.title('Test Set Correlation HeatMap ',y=1,size=16)
    sns.heatmap(test_data.corr(),square = True,  vmax=0.7,annot=True,cmap='Accent')

    continue_features = ['Age', 'Fare']
    survived = train_data['Survived'] == 1
    
    fig, axs = plt.subplots(ncols=2, nrows=2, figsize=(20, 20))
    plt.subplots_adjust(right=1.5)
    
    for i, feature in enumerate(continue_features):    
        sns.distplot(train_data[~survived][feature], label='Not Survived', hist=True, color='#e74c3c', ax=axs[0][i])
        sns.distplot(train_data[survived][feature], label='Survived', hist=True, color='#2ecc71', ax=axs[0][i])
        
        sns.distplot(train_data[feature], label='Training Set', hist=False, color='#e74c3c', ax=axs[1][i])
        sns.distplot(test_data[feature], label='Test Set', hist=False, color='#2ecc71', ax=axs[1][i])
        
        axs[0][i].set_xlabel('')
        axs[1][i].set_xlabel('')
        
        for j in range(2):        
            axs[i][j].tick_params(axis='x', labelsize=20)
            axs[i][j].tick_params(axis='y', labelsize=20)
        
        axs[0][i].legend(loc='upper right', prop={'size': 20})
        axs[1][i].legend(loc='upper right', prop={'size': 20})
        axs[0][i].set_title('Distribution of Survival in {}'.format(feature), size=20, y=1.05)
    
    axs[1][0].set_title('Distribution of {} Feature'.format('Age'), size=20, y=1.05)
    axs[1][1].set_title('Distribution of {} Feature'.format('Fare'), size=20, y=1.05)
            
    plt.show()

    Categorical_features = ['Embarked', 'Parch','SibSp','Sex', 'Pclass']
    
    fig, axs = plt.subplots(ncols=2, nrows=3, figsize=(20, 20))
    plt.subplots_adjust(right=1.5, top=1.25)
    
    for i, feature in enumerate(Categorical_features, 1):    
        plt.subplot(2, 3, i)
        sns.countplot(x=feature, hue='Survived', data=train_data)
        
        plt.tick_params(axis='x', labelsize=20)
        plt.tick_params(axis='y', labelsize=20)
        
        plt.xlabel('{}'.format(feature), size=20, labelpad=15)
        plt.ylabel('Passenger Count', size=20, labelpad=15)    
        plt.legend(['Not Survived', 'Survived'], loc='upper center')
        plt.title('Count of Survival in {} Feature'.format(feature), size=20, y=1.05)
    plt.show()
    
    
    fig, axs = plt.subplots(ncols=2, nrows=3, figsize=(15, 15))
    plt.subplots_adjust(right=1.5, top=1.25)
    for i, feature in enumerate(Categorical_features, 1):    
        plt.subplot(2, 3, i)
        sns.pointplot(feature,y='Survived',data=train_data)
        
        plt.tick_params(axis='x', labelsize=20)
        plt.tick_params(axis='y', labelsize=20)
        
        plt.xlabel('{}'.format(feature), size=20, labelpad=15)
        plt.ylabel('Passenger Count', size=20, labelpad=15)    
        plt.title('Rate of Survival in {} Feature'.format(feature), size=20, y=1.05)
    plt.show()

    train_data.to_csv('./train.csv',index=False)
    test_data.to_csv('./test.csv',index=False)