import numpy as npimport pandas as pdimport matplotlib.pyplot as pltfrom sklearn.model_selection import train_test_splitimport seaborn as snsimport scipyfrom collections import Counter

df_train = pd.read_csv(train_path, sep='\t')df_test = pd.read_csv(test_path, sep='\t')

df_train.head(), len(df_train)

print(len(df_train['text'][0]), type(df_train['text'][0]))df_train.head()

np.vectorize(function)(array)

def split_df(df_row):    return len(str(df_row).split())len_dist = np.vectorize(split_df)(df_train['text'])len_test_dist = np.vectorize(split_df)(df_test['text'])

print(pd.Series(len_dist).describe())

print(pd.Series(len_test_dist).describe())

fig, ax = plt.subplots(1,1,figsize=(12,6))
ax = plt.hist(x=len_dist, bins=100)ax = plt.hist(x=len_test_dist, bins=100)
plt.xlim([0, max(max(len_dist), max(len_test_dist))])plt.xlabel("length of sample")plt.ylabel("number of sample")plt.legend(['train_len','test_len'])
plt.show()

plt.figure(figsize=(15,5))ax = sns.distplot(len_dist, bins=100)ax = sns.distplot(len_test_dist, bins=100)plt.xlim([0, max(max(len_dist), max(len_test_dist))])plt.xlabel("length of sample")plt.ylabel("prob of sample")plt.legend(['train_len','test_len'])

import scipyscipy.stats.ks_2samp(len_dist, len_test_dist)Ks_2sampResult(statistic=0.004049999999999998, pvalue=0.5279614323123156)

log_len_dist = np.log(1+len_dist)log_len_test_dist = np.log(1+len_test_dist)plt.figure(figsize=(15,5))ax = sns.distplot(log_len_dist)ax = sns.distplot(log_len_test_dist)plt.xlabel("log length of sample")plt.ylabel("prob of log")plt.legend(['train_len','test_len'])

_, lognormal_ks_pvalue = scipy.stats.kstest(rvs=log_len_dist, cdf='norm')print('P value is ', lognormal_ks_pvalue)
P value is  0.0

trans_data, lam = scipy.stats.boxcox(len_dist+1)scipy.stats.normaltest(trans_data)NormaltestResult(statistic=1347.793358118494, pvalue=2.1398873511704724e-293)

df_train['text_split'] = df_train['text'].apply(lambda x:x.split())df_train['len'] = df_train['text'].apply(lambda x:len(x.split()))df_train['first_char'] = df_train['text_split'].apply(lambda x:x[0])df_train['last_char'] = df_train['text_split'].apply(lambda x:x[-1])df_train['most_freq'] = df_train['text_split'].apply(lambda x:np.argmax(np.bincount(x)))df_train.head()

df_train_info = pd.DataFrame(columns=['count','len_mean','len_std','len_min','len_max','freq_fc','freq_lc','freq_freq'])for name, group in df_train.groupby('label'):    count = len(group) # 該類(lèi)別新聞數(shù)    len_mean = np.mean(group['len']) # 該類(lèi)別長(zhǎng)度平均值    len_std = np.std(group['len']) # 長(zhǎng)度標(biāo)準(zhǔn)差    len_min = np.min(group['len']) # 最短的新聞長(zhǎng)度    len_max = np.max(group['len']) # 最長(zhǎng)的新聞長(zhǎng)度    freq_fc = np.argmax(np.bincount(group['first_char'])) # 最頻繁出現(xiàn)的首詞    freq_lc = np.argmax(np.bincount(group['last_char'])) # 最頻繁出現(xiàn)的末詞    freq_freq = np.argmax(np.bincount(group['most_freq'])) # 該類(lèi)別最頻繁出現(xiàn)的詞    df_train_info.loc[name] = [count,len_mean,len_std,len_min,len_max,freq_fc,freq_lc,freq_freq]df_train_info

label_2_index_dict = {'科技': 0, '股票': 1, '體育': 2, '娛樂(lè)': 3, '時(shí)政': 4, '社會(huì)': 5, '教育': 6, '財(cái)經(jīng)': 7, '家居': 8, '游戲': 9, '房產(chǎn)': 10, '時(shí)尚': 11, '彩票': 12, '星座': 13}index_2_label_dict = {v:k for k,v in label_2_index_dict.items()}
plt.figure()plt.bar(x=range(14), height=np.bincount(df_train['label']))plt.xlabel("label")plt.ylabel("number of sample")plt.xticks(range(14), list(index_2_label_dict.values()), fontproperties=zhfont, rotation=60)plt.show()

df_train['len'] = df_train['text'].apply(lambda x: len(x.split()))plt.figure()ax = sns.catplot(x='label', y='len', data=df_train, kind='strip')plt.xticks(range(14), list(index_2_label_dict.values()), fontproperties=zhfont, rotation=60)

# 內(nèi)存警告！?。](méi)有8G內(nèi)存不要運(yùn)行該代碼all_lines = ' '.join(list(df_train['text']))word_count = Counter(all_lines.split(" "))word_count = sorted(word_count.items(), key=lambda d:d[1], reverse=True)
print(len(word_count))# 6869
print(word_count[0])# ('3750', 7482224)
print(word_count[-1])# ('3133', 1)

%%timedf_train['text_unique'] = df_train['text'].apply(lambda x: ' '.join(list(set(x.split(' ')))))all_lines = ' '.join(list(df_train['text_unique']))word_count = Counter(all_lines.split(" "))word_count = sorted(word_count.items(), key=lambda d:int(d[1]), reverse=True)# 打印整個(gè)訓(xùn)練集中覆蓋率前5的詞for i in range(5):    print("{} occurs {} times, {}%".format(word_count[i][0], word_count[i][1], (word_count[i][1]/200000)*100))