文本分析——分詞、統(tǒng)計(jì)詞頻、詞云

TNFE 發(fā)布于2019-07-30 15:34 / 1707人閱讀

摘要：導(dǎo)入包導(dǎo)入文件和數(shù)據(jù)數(shù)據(jù)預(yù)處理分詞并去除停用詞算法該類(lèi)會(huì)將文本中的詞語(yǔ)轉(zhuǎn)換為詞頻矩陣，矩陣元素表示詞在類(lèi)文本下的詞頻該類(lèi)會(huì)統(tǒng)計(jì)每個(gè)詞語(yǔ)的權(quán)值第一個(gè)是計(jì)算，第二個(gè)是將文本轉(zhuǎn)為詞頻矩陣獲取詞袋模型中的所有詞語(yǔ)將矩陣抽取出來(lái)，元素表示

導(dǎo)入包

import os
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
from pandas import Series, DataFrame  

import string
import re
import jieba
import jieba.analyse
import datetime
from wordcloud import WordCloud, ImageColorGenerator
import codecs

導(dǎo)入文件和數(shù)據(jù)

gongdan = pd.read_excel("Gongdan.xlsx")

數(shù)據(jù)預(yù)處理

gongdan["content"] = [str(i) for i in gongdan["content"]]
gongdan["content"] = ["".join(re.findall(u"[u4e00-u9fff]+", i)) for i in gongdan["content"]]
indexs = list(gongdan["content"][pd.isnull(gongdan["content"])].index)
gongdan = gongdan.drop(indexs)
indexs = list(gongdan["content"][gongdan["content"]==""].index)
gongdan = gongdan.drop(indexs)

content = gongdan["content"]

cont = "".join(content)
cont = "".join(re.findall(u"[u4e00-u9fa5]+", cont))

分詞并去除停用詞

stopwords = set()
fr = codecs.open("stopwords.txt", "r", "utf-8")
for word in fr:
   stopwords.add(str(word).strip())
fr.close()

jieba.load_userdict("dict.txt")
text = list(jieba.cut(cont, cut_all=False, HMM=True))
text = list(filter(lambda x: x not in stopwords, text))
text = [str(i) for i in text if i != " "]

Tfidf 算法

from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer  
from sklearn.feature_extraction.text import CountVectorizer  

test = " ".join(text)
tlist = []
tlist.append(test)

vectorizer=CountVectorizer()#該類(lèi)會(huì)將文本中的詞語(yǔ)轉(zhuǎn)換為詞頻矩陣，矩陣元素a[i][j] 表示j詞在i類(lèi)文本下的詞頻
transformer = TfidfTransformer()#該類(lèi)會(huì)統(tǒng)計(jì)每個(gè)詞語(yǔ)的tf-idf權(quán)值
tfidf = transformer.fit_transform(vectorizer.fit_transform(tlist))  #第一個(gè)fit_transform是計(jì)算tf-idf，第二個(gè)fit_transform是將文本轉(zhuǎn)為詞頻矩陣  

word=vectorizer.get_feature_names()#獲取詞袋模型中的所有詞語(yǔ)  
weight=tfidf.toarray()#將tf-idf矩陣抽取出來(lái)，元素a[i][j]表示j詞在i類(lèi)文本中的tf-idf權(quán)重  
tfidf_list = {}
for i in range(len(weight)):#打印每類(lèi)文本的tf-idf詞語(yǔ)權(quán)重，第一個(gè)for遍歷所有文本，第二個(gè)for便利某一類(lèi)文本下的詞語(yǔ)權(quán)重  
    for j in range(len(word)):  
        tfidf_list[word[j]] = weight[i][j]

詞云

font_path = "yahei.ttf"

from PIL import Image
back_coloring = np.array(Image.open("circle.jpg"))

wc = WordCloud(font_path=font_path,  # 設(shè)置字體
               background_color="white",  # 背景顏色
               max_words=60,  # 詞云顯示的最大詞數(shù)
               mask=back_coloring,  # 設(shè)置背景圖片
               stopwords=stopwords,
               max_font_size=100,  # 字體最大值
               random_state=42,
               width=1000, height=860, margin=2,# 設(shè)置圖片默認(rèn)的大小,但是如果使用背景圖片的話,那么保存的圖片大小將會(huì)按照其大小保存,margin為詞語(yǔ)邊緣距離
#               prefer_horizontal=1,
               )

wc.generate_from_frequencies(tfidf_list)

plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.figure()
wc.to_file("w.png")

# create coloring from image
image_colors = ImageColorGenerator(back_coloring)
# recolor wordcloud and show
# we could also give color_func=image_colors directly in the constructor
plt.imshow(wc.recolor(color_func=image_colors), interpolation="bilinear")
plt.axis("off")
plt.figure()
plt.imshow(back_coloring, cmap=plt.cm.gray, interpolation="bilinear")
plt.axis("off")
plt.show()