Mezzanine の Blog に 関連記事 のレコメンド表示を組み込んでみる

# -*- coding: utf-8 -*-
from __future__ import print_function
from django.core.management.base import BaseCommand
from mezzanine.blog.models import BlogPost
from BeautifulSoup import BeautifulSoup
from markdown import markdown
import MeCab
import HTMLParser
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction import text
import numpy as np
from django.db import transaction
from logging import getLogger

logger = getLogger(__name__)

additional_stop_words = frozenset([
    'xxxx',
    'xxxxx',
    'xxxxxx',
    'xxxxxxxx',
    'xxxxxxxxx',
    'xxxxxxxxxx',
    'xxxxxxxxxxxx',
    'xxxxxxxxxxxxxx',
    'xxxxxxxxxxxxxxx',
    'xxxxxxxxxxxxxxxxxxxxxxxx',
    'xxxxxxxxxxxxxxxxxxxxxxxxx',
])

stop_words = text.ENGLISH_STOP_WORDS.union(additional_stop_words)


class Command(BaseCommand):
    help = "My recommend blog post command."

    def handle(self, *args, **options):
        logger.info(__name__ + " START")

        documents = []
        blogs = []

        # 1. BlogPostから、本文を取得、markdownから本文を抽出
        for blog_post in BlogPost.objects.published():
            source = blog_post.content
            html = markdown(source)
            text = ''.join(BeautifulSoup(html).findAll(text=True))
            htmlParser = HTMLParser.HTMLParser()
            unescaped_text = htmlParser.unescape(text)

            documents.append(str(unescaped_text.encode('utf-8')))
            blogs.append(blog_post)

        # 2. TfidfVectorizer を使って、ベクトル化、Cos類似度を求める
        cs_array = cosine_similarity(self.__vecs_array(documents),
                                     self.__vecs_array(documents))

        # 3. 関連投稿の全削除、Cos類似度の高い上位5件を登録
        with transaction.atomic():
            # 3-1. 関連投稿の全削除
            for blog in blogs:
                blog.related_posts.clear()

            # 3-2. Cos類似度の高い上位5件を登録
            for i, cs_item in enumerate(cs_array):
                blog = blogs[i]
                cs_dic = {}
                for j, cs in enumerate(cs_item):
                    if round(cs - 1.0, 5) != 0:
                        cs_dic[blogs[j]] = cs
                index = 0
                for k, v in sorted(cs_dic.items(), key=lambda x: x[1],
                                   reverse=True):
                    blog.related_posts.add(k)
                    index += 1
                    if index == 5:
                        break
        logger.info(__name__ + " END")

    def __vecs_array(self, documents):
        docs = np.array(documents)
        vectorizer = TfidfVectorizer(
            analyzer=self.__get_words,
            stop_words="|",
            min_df=1,
            token_pattern='(?u)\\b\\w+\\b')
        vecs = vectorizer.fit_transform(docs)

        return vecs.toarray()

    def __get_words(self, text):
        out_words = []
        # mecab-ipadic-neologd をカスタム辞書として指定
        tagger = MeCab.Tagger("-Ochasen -d /usr/lib64/mecab/dic/mecab-ipadic-neologd")
        tagger.parse('')
        node = tagger.parseToNode(text)

        while node:
            word_type = node.feature.split(",")[0]
            if word_type in ["名詞"]:
                word = node.surface.decode('utf-8')
                # stopwordを弾く
                if word not in stop_words:
                    out_words.append(word)
            node = node.next
        return out_words