NLP新闻文本分类学习赛 - Task4 基于fastText的文本分类

    将整篇文档的词及n-gram向量叠加平均得到文档向量，然后使用文档向量做softmax多分类。

    pip install fasttext

    import fasttext
    model = fasttext.train_supervised(input_fine, lr=1.0, 
                            dim=100, ws=5, epoch=25, minCount=1, 
                            wordNgrams=2, loss='ns', bucket=2000000, 
                            thread=12, lrUpdateRate=100, t=1e-4, 
                            label='__label__', verbose=2)

    input_file                 训练文件路径（必须）
    output                     输出文件路径
    label_prefix               标签前缀 default __label__
    lr                         学习率 default 0.1
    lr_update_rate             学习率更新速率 default 100
    dim                        词向量维度 default 100
    ws                         上下文窗口大小 default 5
    epoch                      epochs 数量 default 5
    min_count                  最低词频，过滤词频低的词 default 5
    word_ngrams                n-gram 设置 default 1
    loss                       损失函数 {ns,hs,softmax,ova}
    minn                       最小字符长度 default 3，用于设定word-n-gram
    maxn                       最大字符长度 default 6, 用于设定word-g-gram
    thread                     线程数量 default 12
    t                          采样阈值 default 0.0001
    silent                     禁用 c++ 扩展日志输出 default 1
    encoding                   指定 input_file 编码 default utf-8
    verbose                    日志显示，0不显示，1显示进度条，2显示详细信息
    pretrained_vectors         指定使用已有的词向量 .vec 文件 default None

    import pandas as pd
    from sklearn.metrics import f1_score
    import fasttext
    
    # 转换为FastText需要的格式
    train_df = pd.read_csv('TRAIN_DATA/train_set.csv', sep='\t', nrows=15000)
    train_df['label_ft'] = '__label__' + train_df['label'].astype(str)
    train_df[['text','label_ft']].iloc[:-5000].to_csv('fasttext_train.csv', index=None, header=None, sep='\t')
    
    
    model = fasttext.train_supervised('fasttext_train.csv', lr=1.0, wordNgrams=2,
                                             verbose=2, minCount=1, epoch=25, loss='ova')
    x=train_df.iloc[-5000:]['text']
    y=train_df['label']
    val_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[-5000:]['text']]
    val_y0 = train_df['label'].values[-5000:]
    val_y1 = val_y0.astype(str)
    print(f1_score(train_df['label'].values[-5000:].astype(str), val_pred, average='macro'))
    // 0.8845029863261721

    import pandas as pd
    from sklearn.model_selection import StratifiedKFold
    
    print('starting K10 cross-validation data split:')
    train_df = pd.read_csv('TRAIN_DATA/train_set.csv', sep='\t') 
    
    skf = StratifiedKFold(n_splits=10) //分层采样，确保分出来的训练集和测试集中各类别样本的比例与原始数据集中相同。
    
    for n_fold, (tr_idx, val_idx) in enumerate(skf.split(train_df['text'],train_df['label'])):
    	print(f'the {n_fold} data split ...')
    	tr_x, tr_y, val_x, val_y = train_df['text'].iloc[tr_idx], train_df['label'][tr_idx], train_df['text'].iloc[val_idx], train_df['label'][val_idx]
    	tr_y = '__label__' + tr_y.astype(str)
    	traindata = pd.DataFrame(list(zip(tr_x.values, tr_y.values)))
    	traindata.to_csv(f'fasttext_skf10_datasplit/train_split{n_fold}.csv', index=None, header=['text', 'label_ft'], sep='\t')
    
    	testdata = pd.DataFrame(list(zip(val_x.values, val_y.values)))
    	testdata.to_csv(f'fasttext_skf10_datasplit/test_split{n_fold}.csv', index=None, header=['text', 'label'], sep='\t')

    print('starting K10 cross-validation training:')
    val_f1=[]
    
    for n_fold in range(10):
    
    	model = fasttext.train_supervised(f'fasttext_skf10_datasplit/train_split{n_fold}.csv', lr=1.0, wordNgrams=2,
    	                                  verbose=2, minCount=1, epoch=25, loss='softmax')
    
    	val_df = pd.read_csv(f'fasttext_skf10_datasplit/test_split{n_fold}.csv', sep='\t')
    
    	val_pred = [model.predict(x)[0][0].split('__')[-1] for x in val_df['text']]
    	val_f1.append(f1_score(val_df['label'].values.astype(str), val_pred, average='macro'))
    	print(f'the f1_score of {n_fold} training is:', val_f1[n_fold])
    
    print()
    print('The average f1_score is', sum(val_f1)/len(val_f1))

    The average f1_score is 0.9303210475140224