續

承接上一篇文章，爬蟲、整理分析方法都已經在前面文章教學過，本篇文章就直接進入正題，進行分析。

各組別文字分析(全文)

def tokenize(sentence):
    terms = []
    if pd.notnull(sentence):
        for term in jieba.cut(sentence):
            term = term.lower()
            if term not in stops:
                terms.append(term)
    return terms

for g in set(df['group']):
    print(g)
    df_content = df[df['group'] == g][['text_content']]
    df_content['processed'] = df_content['text_content'].apply(tokenize)
    total_terms = []
    for terms in df_content['processed']:
        total_terms.extend(terms)
        
    wordcloud = WordCloud(font_path="simsun.ttf", background_color='white')
    wordcloud.generate_from_frequencies(frequencies=Counter(total_terms))
    plt.figure(figsize=(10, 10))
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
    plt.savefig(os.path.join('day19_pic', 'worldcloud_' + g + '_chinese'), bbox_inches='tight', pad_inches=0)
    plt.show()

各組別文字分析(英文)

for g in set(df['group']):
    print(g)
    df_content = df[df['group'] == g][['text_content']]
    df_content['processed'] = df_content['text_content'].apply(tokenize)
    english_terms = []
    for terms in df_content['processed']:
        for term in terms:
            match_eng = re.match(r'[a-z]+', term)
            if match_eng != None and match_eng.group(0) == term:
                english_terms.append(term)
            
    wordcloud = WordCloud(font_path="simsun.ttf", background_color='white')
    wordcloud.generate_from_frequencies(frequencies=Counter(english_terms))
    plt.figure(figsize=(10, 10))
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
    plt.savefig(os.path.join('day19_pic', 'worldcloud_' + g + '_English'), bbox_inches='tight', pad_inches=0)
    plt.show()

各組別程式語言出現比例

1. 整理出各組程式語言出現的次數

group_lang_count = {}
for g in set(df['group']):
    print(g)
    df_content = df[df['group'] == g][['text_content']]
    df_content['processed'] = df_content['text_content'].apply(tokenize)
    total_terms = []
    for terms in df_content['processed']:
        for term in terms:
            match_eng = re.match(r'[a-z]+', term)
            if match_eng != None and match_eng.group(0) == term:
                total_terms.append(term)

    langs = ["nodejs", "node", "reactjs", "react", "js", 
             "python", "javascript", "ruby", 'java', 'c', 
             'c#', 'angularjs', 'angular', 'typescript', 
             'd3', 'd3js', 'sql', 'html', 'css', 'jquery',
            'go', 'vue', 'vuejs', 'r']
    
    langs = sorted(langs)

    lang_count = {}
    for lang in langs:
        count = 0
        if lang in total_terms:
            count = Counter(total_terms).get(lang)
        lang_count[lang] = count
    
    print(lang_count)
    group_lang_count[g] = lang_count
    
df_lang = pd.DataFrame(list(group_lang_count.values()), index=group_lang_count.keys())
df_lang

2. 畫圖

for g in set(df['group']):
    df_lang.T[g].plot(kind='pie', autopct='%.2f', title=g, fontsize=10, )
    plt.title(g, fontsize=20)
    plt.savefig(os.path.join('day19_pic', 'lang_pie_' + g), bbox_inches='tight', pad_inches=0)
    plt.show()

文章長度排名

文章讚數排名

文章觀看數排名

備註

上篇文章在這裡
code在這裡