[Python] scikit-learnで学ぶパーセプトロンによる文書分類入門

import argparse
import csv
import os
import pickle

import pandas as pd
import MeCab
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

def read_file(dir_name):
    titles = []
    for file_name in os.listdir(dir_name):
        path = os.path.join(dir_name, file_name)
        with open(path) as f:
            _ = next(f)  # url
            _ = next(f)  # date
            title = next(f)
            titles.append(title.strip())
    return titles

def read_data(dir_name, tagger):
    categories = []
    titles = []
    for dir_or_file in os.listdir(dir_name):
        path = os.path.join(dir_name, dir_or_file)
        if os.path.isdir(path):
            titles_of_category = read_file(path)
            categories += [dir_or_file for _ in range(len(titles_of_category))]
            titles += [tokenize(tagger, t) for t in titles_of_category]

    df = pd.DataFrame({'category': categories, 'title': titles})
    return df

def tokenize(tagger, text):
    result = tagger.parse(text).strip()
    return result

parser = argparse.ArgumentParser()
parser.add_argument('--dir')
parser.add_argument('--out', default='data')
args = parser.parse_args()

tagger = MeCab.Tagger('-Owakati')

df = read_data(args.dir, tagger)

# 学習データとテストデータに分割
train, test = train_test_split(df)

vectorizer = TfidfVectorizer(
    input='content')
# 素性ベクトル変換器を構築し、学習データに適用
train_titles = train['title']
train_X = vectorizer.fit_transform(train_titles)

# テストデータに素性ベクトル変換器を適用
test_titles = test['title']
test_X = vectorizer.transform(test_titles)

label_encoder = LabelEncoder()
# ラベルをIDへ変換する変換器を構築し、学習データに適用
train_categories = train['category']
train_y = label_encoder.fit_transform(train_categories)

# 構築したラベル変換器をテストデータに適用
test_categories = test['category']
test_y = label_encoder.transform(test_categories)

if not os.path.exists(args.out):
    os.makedirs(args.out)

train_file = os.path.join(args.out, 'train.pickle')
test_file = os.path.join(args.out, 'test.pickle')
vectorizer_file = os.path.join(args.out, 'vectorizer.pickle')
label_encoder_file = os.path.join(args.out, 'label_encoder.pickle')

with open(train_file, 'wb') as f:
    pickle.dump([train_X, train_y], f)

with open(test_file, 'wb') as f:
    pickle.dump([test_X, test_y], f)

with open(vectorizer_file, 'wb') as f:
    pickle.dump(vectorizer, f)

with open(label_encoder_file, 'wb') as f:
    pickle.dump(label_encoder, f)

import argparse
import pickle
import os

from sklearn.linear_model import Perceptron
from sklearn.metrics import precision_recall_fscore_support

parser = argparse.ArgumentParser()
parser.add_argument('--dir', default='data')
args = parser.parse_args()

train_file = os.path.join(args.dir, 'train.pickle')
with open(train_file, 'rb') as f:
    train_X, train_y = pickle.load(f)
    print('train', train_X.shape, train_y.shape)

test_file = os.path.join(args.dir, 'test.pickle')
with open(test_file, 'rb') as f:
    test_X, test_y = pickle.load(f)
    print('test:', test_X.shape, test_y.shape)

model = Perceptron(
    penalty='l2',
    shuffle=True,
    verbose=2)

# パーセプトロンを学習
model.fit(train_X, train_y)
# テストデータに対して予測
test_y_pred = model.predict(test_X)

# テストデータにおける適合率、再現率、F値を算出
precision, recall, fscore, _ = precision_recall_fscore_support(
    test_y,
    test_y_pred,
    average='micro')

print('Precision:', precision)
print('Recall:', recall)
print('F-score:', fscore)

model_file = os.path.join(args.dir, 'model.pickle')
with open(model_file, 'wb') as f:
    pickle.dump(model, f)

import argparse
import os
import pickle

import MeCab

import preprocess  # この記事の前処理用プログラム

parser = argparse.ArgumentParser()
parser.add_argument('--dir', default='data')
args = parser.parse_args()

model_file = os.path.join(args.dir, 'model.pickle')
with open(model_file, 'rb') as f:
    model = pickle.load(f)

label_encoder_file = os.path.join(args.dir, 'label_encoder.pickle')
with open(label_encoder_file, 'rb') as f:
    label_encoder = pickle.load(f)

vectorizer_file = os.path.join(args.dir, 'vectorizer.pickle')
with open(vectorizer_file, 'rb') as f:
    vectorizer = pickle.load(f)

tagger = MeCab.Tagger('-Owakati')

while True:
    text = input()
    tokenized = preprocess.tokenize(tagger, text)

    x = vectorizer.transform([tokenized])
    y = model.predict(x)
    # ラベルをIDから対応する文字に変換
    label = label_encoder.inverse_transform(y)[0]
    print('Tokenized:', tokenized)
    print('Label:', label)