Skip to content

Instantly share code, notes, and snippets.

@yatakeke

yatakeke/buld_lstm_model.ipynb

Created November 28, 2019 08:59
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save yatakeke/4aff24488b11cb091896beb9c4bc53dc to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save yatakeke/4aff24488b11cb091896beb9c4bc53dc to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
buld_lstm_model.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Reference"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- LSTMで文章分類を実装してみた\n",
" - https://qiita.com/m__k/items/841950a57a0d7ff05506\n",
"\n",
"- LSTMの概要\n",
" - https://qiita.com/KojiOhki/items/89cd7b69a8a6239d67ca"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"from glob import glob\n",
"import os\n",
"import linecache\n",
"import MeCab\n",
"import pandas as pd\n",
"from datetime import datetime as dt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# dataset"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"categories = []\n",
"text_path = '../text/'\n",
"for directory in os.listdir(text_path):\n",
" if directory.startswith('.'):\n",
" continue\n",
" if os.path.isdir(text_path + directory):\n",
" categories.append(directory)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'movie-enter': 0,\n",
" 'it-life-hack': 1,\n",
" 'kaden-channel': 2,\n",
" 'topic-news': 3,\n",
" 'livedoor-homme': 4,\n",
" 'peachy': 5,\n",
" 'sports-watch': 6,\n",
" 'dokujo-tsushin': 7,\n",
" 'smax': 8}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat2idx = {}\n",
"\n",
"for cat in categories:\n",
" if cat in cat2idx: continue\n",
" cat2idx[cat] = len(cat2idx)\n",
"\n",
"cat2idx"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"datasets = pd.DataFrame([], columns=['Title', 'Category'])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"for cat in categories:\n",
" dir_path = text_path + directory\n",
" if directory.startswith('.') or not os.path.isdir(dir_path): continue\n",
" \n",
" path = '../text/' + cat + '/*.txt'\n",
" files = glob(path)\n",
" \n",
" for text in files:\n",
" title = linecache.getline(text, 3)\n",
" s = pd.Series([title, cat], index=datasets.columns)\n",
" datasets = datasets.append(s, ignore_index=True)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"datasets.to_csv('../datasets.csv', index=None)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"datasets"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Utils"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"def get_timestamp():\n",
" dt_obj = dt.now()\n",
" return dt_obj.strftime(\"%Y/%m/%d, %H:%M:%S\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"ORIGIN_FORM = '原形'\n",
"options = [\n",
" \"-d\", \"/usr/local/lib/mecab/dic/mecab-ipadic-neologd/\",\n",
" \"-U\", \"%M\\\\t未知語,*,*,*,*,*,*,*,*\\\\n\"\n",
"]\n",
"\n",
"mecab_options = ' '.join(options)\n",
"mecab = MeCab.Tagger(mecab_options)\n",
"\n",
"columns = \"表層形,品詞,品詞細分類1,品詞細分類2,品詞細分類3,活用型,活用形,原形,読み,発音\".split(',')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def tokenize(text) -> pd.DataFrame:\n",
" text = text.replace(',', '')\n",
" token = mecab.parse(text).replace('\\t', ',')\n",
" token = token.replace('\\nEOS\\n', '')\n",
" token = [line.split(',') for line in token.split('\\n')]\n",
" df = pd.DataFrame(token, columns=columns)\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>表層形</th>\n",
" <th>品詞</th>\n",
" <th>品詞細分類1</th>\n",
" <th>品詞細分類2</th>\n",
" <th>品詞細分類3</th>\n",
" <th>活用型</th>\n",
" <th>活用形</th>\n",
" <th>原形</th>\n",
" <th>読み</th>\n",
" <th>発音</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>『</td>\n",
" <td>記号</td>\n",
" <td>括弧開</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>『</td>\n",
" <td>『</td>\n",
" <td>『</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>ベルセルク</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>BERSERK</td>\n",
" <td>ベルセルク</td>\n",
" <td>ベルセルク</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>』</td>\n",
" <td>記号</td>\n",
" <td>括弧閉</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>』</td>\n",
" <td>』</td>\n",
" <td>』</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2</td>\n",
" <td>未知語</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>作</td>\n",
" <td>名詞</td>\n",
" <td>接尾</td>\n",
" <td>助数詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>作</td>\n",
" <td>サク</td>\n",
" <td>サク</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>目</td>\n",
" <td>名詞</td>\n",
" <td>接尾</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>目</td>\n",
" <td>メ</td>\n",
" <td>メ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>、</td>\n",
" <td>記号</td>\n",
" <td>読点</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>最愛</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>最愛</td>\n",
" <td>サイアイ</td>\n",
" <td>サイアイ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>の</td>\n",
" <td>助詞</td>\n",
" <td>連体化</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>の</td>\n",
" <td>ノ</td>\n",
" <td>ノ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>友</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>友</td>\n",
" <td>トモ</td>\n",
" <td>トモ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>と</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>と</td>\n",
" <td>ト</td>\n",
" <td>ト</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>の</td>\n",
" <td>助詞</td>\n",
" <td>連体化</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>の</td>\n",
" <td>ノ</td>\n",
" <td>ノ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>決闘</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>決闘</td>\n",
" <td>ケットウ</td>\n",
" <td>ケットー</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>と</td>\n",
" <td>助詞</td>\n",
" <td>並立助詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>と</td>\n",
" <td>ト</td>\n",
" <td>ト</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>別れ</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>別れ</td>\n",
" <td>ワカレ</td>\n",
" <td>ワカレ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>を</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>を</td>\n",
" <td>ヲ</td>\n",
" <td>ヲ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>描く</td>\n",
" <td>動詞</td>\n",
" <td>自立</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>五段・カ行イ音便</td>\n",
" <td>基本形</td>\n",
" <td>描く</td>\n",
" <td>エガク</td>\n",
" <td>エガク</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 表層形 品詞 品詞細分類1 品詞細分類2 品詞細分類3 活用型 活用形 原形 読み 発音\n",
"0 『 記号 括弧開 * * * * 『 『 『\n",
"1 ベルセルク 名詞 固有名詞 一般 * * * BERSERK ベルセルク ベルセルク\n",
"2 』 記号 括弧閉 * * * * 』 』 』\n",
"3 2 未知語 * * * * * * * *\n",
"4 作 名詞 接尾 助数詞 * * * 作 サク サク\n",
"5 目 名詞 接尾 一般 * * * 目 メ メ\n",
"6 、 記号 読点 * * * * 、 、 、\n",
"7 最愛 名詞 一般 * * * * 最愛 サイアイ サイアイ\n",
"8 の 助詞 連体化 * * * * の ノ ノ\n",
"9 友 名詞 一般 * * * * 友 トモ トモ\n",
"10 と 助詞 格助詞 一般 * * * と ト ト\n",
"11 の 助詞 連体化 * * * * の ノ ノ\n",
"12 決闘 名詞 サ変接続 * * * * 決闘 ケットウ ケットー\n",
"13 と 助詞 並立助詞 * * * * と ト ト\n",
"14 別れ 名詞 一般 * * * * 別れ ワカレ ワカレ\n",
"15 を 助詞 格助詞 一般 * * * を ヲ ヲ\n",
"16 描く 動詞 自立 * * 五段・カ行イ音便 基本形 描く エガク エガク"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display(tokenize(datasets['Title'].loc[9]))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def cat2tensor(cat):\n",
" return torch.tensor([cat2idx[cat]], dtype=torch.long)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor([1])\n"
]
}
],
"source": [
"print(cat2tensor('it-life-hack'))"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"def sentence2idx(sentence, vocab):\n",
" df_token = tokenize(sentence)\n",
" original_forms = list(df_token[ORIGIN_FORM])\n",
" idxs = [vocab[w] for w in original_forms]\n",
" return torch.tensor(idxs, dtype=torch.long)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"sample_sentence = datasets['Title'].loc[3]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Vocabulary"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>表層形</th>\n",
" <th>品詞</th>\n",
" <th>品詞細分類1</th>\n",
" <th>品詞細分類2</th>\n",
" <th>品詞細分類3</th>\n",
" <th>活用型</th>\n",
" <th>活用形</th>\n",
" <th>原形</th>\n",
" <th>読み</th>\n",
" <th>発音</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>【</td>\n",
" <td>記号</td>\n",
" <td>括弧開</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>【</td>\n",
" <td>【</td>\n",
" <td>【</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>DVD</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>DVD</td>\n",
" <td>ディーブイディー</td>\n",
" <td>ディーブイディー</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>エンター</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>エンター</td>\n",
" <td>エンター</td>\n",
" <td>エンター</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>！</td>\n",
" <td>記号</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>！</td>\n",
" <td>！</td>\n",
" <td>！</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>】</td>\n",
" <td>記号</td>\n",
" <td>括弧閉</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>】</td>\n",
" <td>】</td>\n",
" <td>】</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>誘拐犯</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>誘拐犯</td>\n",
" <td>ユウカイハン</td>\n",
" <td>ユーカイハン</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>に</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>に</td>\n",
" <td>ニ</td>\n",
" <td>ニ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>育て</td>\n",
" <td>動詞</td>\n",
" <td>自立</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>一段</td>\n",
" <td>未然形</td>\n",
" <td>育てる</td>\n",
" <td>ソダテ</td>\n",
" <td>ソダテ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>られ</td>\n",
" <td>動詞</td>\n",
" <td>接尾</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>一段</td>\n",
" <td>連用形</td>\n",
" <td>られる</td>\n",
" <td>ラレ</td>\n",
" <td>ラレ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>た</td>\n",
" <td>助動詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>特殊・タ</td>\n",
" <td>基本形</td>\n",
" <td>た</td>\n",
" <td>タ</td>\n",
" <td>タ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>女</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>女</td>\n",
" <td>オンナ</td>\n",
" <td>オンナ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>が</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>が</td>\n",
" <td>ガ</td>\n",
" <td>ガ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>目</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>目</td>\n",
" <td>メ</td>\n",
" <td>メ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>に</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>に</td>\n",
" <td>ニ</td>\n",
" <td>ニ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>し</td>\n",
" <td>動詞</td>\n",
" <td>自立</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>サ変・スル</td>\n",
" <td>連用形</td>\n",
" <td>する</td>\n",
" <td>シ</td>\n",
" <td>シ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>た</td>\n",
" <td>助動詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>特殊・タ</td>\n",
" <td>基本形</td>\n",
" <td>た</td>\n",
" <td>タ</td>\n",
" <td>タ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>真実</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>真実</td>\n",
" <td>シンジツ</td>\n",
" <td>シンジツ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>は</td>\n",
" <td>助詞</td>\n",
" <td>係助詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>は</td>\n",
" <td>ハ</td>\n",
" <td>ワ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>、</td>\n",
" <td>記号</td>\n",
" <td>読点</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>孤独</td>\n",
" <td>名詞</td>\n",
" <td>形容動詞語幹</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>孤独</td>\n",
" <td>コドク</td>\n",
" <td>コドク</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>か</td>\n",
" <td>助詞</td>\n",
" <td>副助詞／並立助詞／終助詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>か</td>\n",
" <td>カ</td>\n",
" <td>カ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>幸福</td>\n",
" <td>名詞</td>\n",
" <td>形容動詞語幹</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>幸福</td>\n",
" <td>コウフク</td>\n",
" <td>コーフク</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>か</td>\n",
" <td>助詞</td>\n",
" <td>副助詞／並立助詞／終助詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>か</td>\n",
" <td>カ</td>\n",
" <td>カ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>藤原竜也</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>人名</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>藤原竜也</td>\n",
" <td>フジワラタツヤ</td>\n",
" <td>フジワラタツヤ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>、</td>\n",
" <td>記号</td>\n",
" <td>読点</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>中学生</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>中学生</td>\n",
" <td>チュウガクセイ</td>\n",
" <td>チューガクセイ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>とともに</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>連語</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>とともに</td>\n",
" <td>トトモニ</td>\n",
" <td>トトモニ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>ロケット打ち上げ</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>ロケット打ち上げ</td>\n",
" <td>ロケットウチアゲ</td>\n",
" <td>ロケットウチアゲ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>に</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>に</td>\n",
" <td>ニ</td>\n",
" <td>ニ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>成功</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>成功</td>\n",
" <td>セイコウ</td>\n",
" <td>セイコー</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125972</th>\n",
" <td>更新</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>更新</td>\n",
" <td>コウシン</td>\n",
" <td>コーシン</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125973</th>\n",
" <td>を</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>を</td>\n",
" <td>ヲ</td>\n",
" <td>ヲ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125974</th>\n",
" <td>提供開始</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>提供開始</td>\n",
" <td>テイキョウカイシ</td>\n",
" <td>テイキョーカイシ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125975</th>\n",
" <td>NTTドコモ</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>NTTドコモ</td>\n",
" <td>エヌティティドコモ</td>\n",
" <td>エヌティティドコモ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125976</th>\n",
" <td>、</td>\n",
" <td>記号</td>\n",
" <td>読点</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" <td>、</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125977</th>\n",
" <td>公式</td>\n",
" <td>名詞</td>\n",
" <td>形容動詞語幹</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>公式</td>\n",
" <td>コウシキ</td>\n",
" <td>コーシキ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125978</th>\n",
" <td>オンラインショップ</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>オンラインショップ</td>\n",
" <td>オンラインショップ</td>\n",
" <td>オンラインショップ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125979</th>\n",
" <td>で</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>で</td>\n",
" <td>デ</td>\n",
" <td>デ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125980</th>\n",
" <td>も</td>\n",
" <td>助詞</td>\n",
" <td>係助詞</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>も</td>\n",
" <td>モ</td>\n",
" <td>モ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125981</th>\n",
" <td>端末</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>端末</td>\n",
" <td>タンマツ</td>\n",
" <td>タンマツ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125982</th>\n",
" <td>複数</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>複数</td>\n",
" <td>フクスウ</td>\n",
" <td>フクスー</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125983</th>\n",
" <td>台</td>\n",
" <td>名詞</td>\n",
" <td>接尾</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>台</td>\n",
" <td>ダイ</td>\n",
" <td>ダイ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125984</th>\n",
" <td>購入</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>購入</td>\n",
" <td>コウニュウ</td>\n",
" <td>コーニュー</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125985</th>\n",
" <td>で</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>で</td>\n",
" <td>デ</td>\n",
" <td>デ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125986</th>\n",
" <td>最大</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>最大</td>\n",
" <td>サイダイ</td>\n",
" <td>サイダイ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125987</th>\n",
" <td>10500円</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>10500円</td>\n",
" <td>イチマンゴヒャクエン</td>\n",
" <td>イチマンゴヒャクエン</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125988</th>\n",
" <td>／</td>\n",
" <td>記号</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>／</td>\n",
" <td>／</td>\n",
" <td>／</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125989</th>\n",
" <td>台</td>\n",
" <td>名詞</td>\n",
" <td>固有名詞</td>\n",
" <td>地域</td>\n",
" <td>国</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>台</td>\n",
" <td>タイ</td>\n",
" <td>タイ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125990</th>\n",
" <td>の</td>\n",
" <td>助詞</td>\n",
" <td>連体化</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>の</td>\n",
" <td>ノ</td>\n",
" <td>ノ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125991</th>\n",
" <td>割り引き</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>割り引き</td>\n",
" <td>ワリビキ</td>\n",
" <td>ワリビキ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125992</th>\n",
" <td>が</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>が</td>\n",
" <td>ガ</td>\n",
" <td>ガ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125993</th>\n",
" <td>受け</td>\n",
" <td>動詞</td>\n",
" <td>自立</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>一段</td>\n",
" <td>未然形</td>\n",
" <td>受ける</td>\n",
" <td>ウケ</td>\n",
" <td>ウケ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125994</th>\n",
" <td>られる</td>\n",
" <td>動詞</td>\n",
" <td>接尾</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>一段</td>\n",
" <td>基本形</td>\n",
" <td>られる</td>\n",
" <td>ラレル</td>\n",
" <td>ラレル</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125995</th>\n",
" <td>「</td>\n",
" <td>記号</td>\n",
" <td>括弧開</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>「</td>\n",
" <td>「</td>\n",
" <td>「</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125996</th>\n",
" <td>家族</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>家族</td>\n",
" <td>カゾク</td>\n",
" <td>カゾク</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125997</th>\n",
" <td>セット</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>セット</td>\n",
" <td>セット</td>\n",
" <td>セット</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125998</th>\n",
" <td>割</td>\n",
" <td>名詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>割</td>\n",
" <td>ワリ</td>\n",
" <td>ワリ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125999</th>\n",
" <td>」</td>\n",
" <td>記号</td>\n",
" <td>括弧閉</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>」</td>\n",
" <td>」</td>\n",
" <td>」</td>\n",
" </tr>\n",
" <tr>\n",
" <th>126000</th>\n",
" <td>を</td>\n",
" <td>助詞</td>\n",
" <td>格助詞</td>\n",
" <td>一般</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>を</td>\n",
" <td>ヲ</td>\n",
" <td>ヲ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>126001</th>\n",
" <td>開始</td>\n",
" <td>名詞</td>\n",
" <td>サ変接続</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>*</td>\n",
" <td>開始</td>\n",
" <td>カイシ</td>\n",
" <td>カイシ</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>126002 rows × 10 columns</p>\n",
"</div>"
],
"text/plain": [
" 表層形 品詞 品詞細分類1 品詞細分類2 品詞細分類3 活用型 活用形 原形 \\\n",
"0 【 記号 括弧開 * * * * 【 \n",
"1 DVD 名詞 固有名詞 一般 * * * DVD \n",
"2 エンター 名詞 固有名詞 一般 * * * エンター \n",
"3 ！ 記号 一般 * * * * ！ \n",
"4 】 記号 括弧閉 * * * * 】 \n",
"5 誘拐犯 名詞 固有名詞 一般 * * * 誘拐犯 \n",
"6 に 助詞 格助詞 一般 * * * に \n",
"7 育て 動詞 自立 * * 一段 未然形 育てる \n",
"8 られ 動詞 接尾 * * 一段 連用形 られる \n",
"9 た 助動詞 * * * 特殊・タ 基本形 た \n",
"10 女 名詞 一般 * * * * 女 \n",
"11 が 助詞 格助詞 一般 * * * が \n",
"12 目 名詞 一般 * * * * 目 \n",
"13 に 助詞 格助詞 一般 * * * に \n",
"14 し 動詞 自立 * * サ変・スル 連用形 する \n",
"15 た 助動詞 * * * 特殊・タ 基本形 た \n",
"16 真実 名詞 一般 * * * * 真実 \n",
"17 は 助詞 係助詞 * * * * は \n",
"18 、 記号 読点 * * * * 、 \n",
"19 孤独 名詞 形容動詞語幹 * * * * 孤独 \n",
"20 か 助詞 副助詞／並立助詞／終助詞 * * * * か \n",
"21 幸福 名詞 形容動詞語幹 * * * * 幸福 \n",
"22 か 助詞 副助詞／並立助詞／終助詞 * * * * か \n",
"23 藤原竜也 名詞 固有名詞 人名 一般 * * 藤原竜也 \n",
"24 、 記号 読点 * * * * 、 \n",
"25 中学生 名詞 一般 * * * * 中学生 \n",
"26 とともに 助詞 格助詞 連語 * * * とともに \n",
"27 ロケット打ち上げ 名詞 固有名詞 一般 * * * ロケット打ち上げ \n",
"28 に 助詞 格助詞 一般 * * * に \n",
"29 成功 名詞 サ変接続 * * * * 成功 \n",
"... ... ... ... ... ... ... ... ... \n",
"125972 更新 名詞 サ変接続 * * * * 更新 \n",
"125973 を 助詞 格助詞 一般 * * * を \n",
"125974 提供開始 名詞 固有名詞 一般 * * * 提供開始 \n",
"125975 NTTドコモ 名詞 固有名詞 一般 * * * NTTドコモ \n",
"125976 、 記号 読点 * * * * 、 \n",
"125977 公式 名詞 形容動詞語幹 * * * * 公式 \n",
"125978 オンラインショップ 名詞 固有名詞 一般 * * * オンラインショップ \n",
"125979 で 助詞 格助詞 一般 * * * で \n",
"125980 も 助詞 係助詞 * * * * も \n",
"125981 端末 名詞 一般 * * * * 端末 \n",
"125982 複数 名詞 一般 * * * * 複数 \n",
"125983 台 名詞 接尾 一般 * * * 台 \n",
"125984 購入 名詞 サ変接続 * * * * 購入 \n",
"125985 で 助詞 格助詞 一般 * * * で \n",
"125986 最大 名詞 一般 * * * * 最大 \n",
"125987 10500円 名詞 固有名詞 一般 * * * 10500円 \n",
"125988 ／ 記号 一般 * * * * ／ \n",
"125989 台 名詞 固有名詞 地域 国 * * 台 \n",
"125990 の 助詞 連体化 * * * * の \n",
"125991 割り引き 名詞 サ変接続 * * * * 割り引き \n",
"125992 が 助詞 格助詞 一般 * * * が \n",
"125993 受け 動詞 自立 * * 一段 未然形 受ける \n",
"125994 られる 動詞 接尾 * * 一段 基本形 られる \n",
"125995 「 記号 括弧開 * * * * 「 \n",
"125996 家族 名詞 一般 * * * * 家族 \n",
"125997 セット 名詞 一般 * * * * セット \n",
"125998 割 名詞 一般 * * * * 割 \n",
"125999 」 記号 括弧閉 * * * * 」 \n",
"126000 を 助詞 格助詞 一般 * * * を \n",
"126001 開始 名詞 サ変接続 * * * * 開始 \n",
"\n",
" 読み 発音 \n",
"0 【 【 \n",
"1 ディーブイディー ディーブイディー \n",
"2 エンター エンター \n",
"3 ！ ！ \n",
"4 】 】 \n",
"5 ユウカイハン ユーカイハン \n",
"6 ニ ニ \n",
"7 ソダテ ソダテ \n",
"8 ラレ ラレ \n",
"9 タ タ \n",
"10 オンナ オンナ \n",
"11 ガ ガ \n",
"12 メ メ \n",
"13 ニ ニ \n",
"14 シ シ \n",
"15 タ タ \n",
"16 シンジツ シンジツ \n",
"17 ハ ワ \n",
"18 、 、 \n",
"19 コドク コドク \n",
"20 カ カ \n",
"21 コウフク コーフク \n",
"22 カ カ \n",
"23 フジワラタツヤ フジワラタツヤ \n",
"24 、 、 \n",
"25 チュウガクセイ チューガクセイ \n",
"26 トトモニ トトモニ \n",
"27 ロケットウチアゲ ロケットウチアゲ \n",
"28 ニ ニ \n",
"29 セイコウ セイコー \n",
"... ... ... \n",
"125972 コウシン コーシン \n",
"125973 ヲ ヲ \n",
"125974 テイキョウカイシ テイキョーカイシ \n",
"125975 エヌティティドコモ エヌティティドコモ \n",
"125976 、 、 \n",
"125977 コウシキ コーシキ \n",
"125978 オンラインショップ オンラインショップ \n",
"125979 デ デ \n",
"125980 モ モ \n",
"125981 タンマツ タンマツ \n",
"125982 フクスウ フクスー \n",
"125983 ダイ ダイ \n",
"125984 コウニュウ コーニュー \n",
"125985 デ デ \n",
"125986 サイダイ サイダイ \n",
"125987 イチマンゴヒャクエン イチマンゴヒャクエン \n",
"125988 ／ ／ \n",
"125989 タイ タイ \n",
"125990 ノ ノ \n",
"125991 ワリビキ ワリビキ \n",
"125992 ガ ガ \n",
"125993 ウケ ウケ \n",
"125994 ラレル ラレル \n",
"125995 「 「 \n",
"125996 カゾク カゾク \n",
"125997 セット セット \n",
"125998 ワリ ワリ \n",
"125999 」 」 \n",
"126000 ヲ ヲ \n",
"126001 カイシ カイシ \n",
"\n",
"[126002 rows x 10 columns]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfs = []\n",
"for title in datasets['Title']:\n",
" df_token = tokenize(title)\n",
" dfs.append(df_token)\n",
" \n",
"df_tokens = pd.concat([*dfs], ignore_index=True)\n",
"df_tokens"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"vocabulary = {}\n",
"for idx, row in df_tokens.iterrows():\n",
" vocabulary.setdefault(row[ORIGIN_FORM], len(vocabulary))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'【': 0,\n",
" 'DVD': 1,\n",
" 'エンター': 2,\n",
" '！': 3,\n",
" '】': 4,\n",
" '誘拐犯': 5,\n",
" 'に': 6,\n",
" '育てる': 7,\n",
" 'られる': 8,\n",
" 'た': 9,\n",
" '女': 10,\n",
" 'が': 11,\n",
" '目': 12,\n",
" 'する': 13,\n",
" '真実': 14,\n",
" 'は': 15,\n",
" '、': 16,\n",
" '孤独': 17,\n",
" 'か': 18,\n",
" '幸福': 19,\n",
" '藤原竜也': 20,\n",
" '中学生': 21,\n",
" 'とともに': 22,\n",
" 'ロケット打ち上げ': 23,\n",
" '成功': 24,\n",
" '『': 25,\n",
" '戦火の馬': 26,\n",
" '』': 27,\n",
" 'ロイヤル': 28,\n",
" '・': 29,\n",
" 'プレミア': 30,\n",
" 'ウィリアム王子': 31,\n",
" '＆': 32,\n",
" 'キャサリン妃': 33,\n",
" '出席': 34,\n",
" '香里奈': 35,\n",
" '女子高生': 36,\n",
" '100人': 37,\n",
" 'の': 38,\n",
" 'ガチンコ!': 39,\n",
" '質問': 40,\n",
" '回答': 41,\n",
" '「': 42,\n",
" 'ラーメン': 43,\n",
" 'も': 44,\n",
" '食べる': 45,\n",
" '」': 46,\n",
" 'YU-G': 47,\n",
" '前': 48,\n",
" '立ちはだかる': 49,\n",
" 'JOY': 50,\n",
" '僕': 51,\n",
" 'AKB': 52,\n",
" '高橋みなみ': 53,\n",
" 'を': 54,\n",
" '守る': 55,\n",
" 'インタビュ': 56,\n",
" '*': 57,\n",
" 'マット・ボマー': 58,\n",
" '（': 59,\n",
" 'ニール・キャフリー': 60,\n",
" '役': 61,\n",
" '）': 62,\n",
" 'インタビュー': 63,\n",
" '：': 64,\n",
" '杏': 65,\n",
" '子供': 66,\n",
" 'って': 67,\n",
" '本当に': 68,\n",
" 'ストレート': 69,\n",
" 'だ': 70,\n",
" 'もの': 71,\n",
" '持つ': 72,\n",
" 'て': 73,\n",
" 'いる': 74,\n",
" 'BeeTV': 75,\n",
" 'モテキ': 76,\n",
" '500万': 77,\n",
" 'DL': 78,\n",
" '突破': 79,\n",
" '久保': 80,\n",
" 'ミツロウ': 81,\n",
" '書き下ろす': 82,\n",
" '特製': 83,\n",
" '待': 84,\n",
" 'プレゼント': 85,\n",
" '夏': 86,\n",
" 'プール': 87,\n",
" 'ピラニア': 88,\n",
" 'リア充': 89,\n",
" '爆発': 90,\n",
" 'ピラニアリターンズ': 91,\n",
" 'BERSERK': 92,\n",
" '作': 93,\n",
" '最愛': 94,\n",
" '友': 95,\n",
" 'と': 96,\n",
" '決闘': 97,\n",
" '別れ': 98,\n",
" '描く': 99,\n",
" 'Vellselk': 100,\n",
" 'パート2': 101,\n",
" '振り返る': 102,\n",
" '岩永洋昭': 103,\n",
" 'キロカロリー': 104,\n",
" 'くらい': 105,\n",
" '消費': 106,\n",
" '小森純': 107,\n",
" '恐怖': 108,\n",
" 'メイク': 109,\n",
" 'で': 110,\n",
" '夫': 111,\n",
" '“': 112,\n",
" '浮気': 113,\n",
" '宣言': 114,\n",
" '”': 115,\n",
" '暴露': 116,\n",
" '編集部': 117,\n",
" '的': 118,\n",
" '映画批評': 119,\n",
" '水谷豊': 120,\n",
" '相棒': 121,\n",
" '座敷わらし': 122,\n",
" '懐かしい': 123,\n",
" 'ゲームブック': 124,\n",
" 'ネット': 125,\n",
" '再現': 126,\n",
" '鷹': 127,\n",
" '団': 128,\n",
" 'ガッツ': 129,\n",
" 'なる': 130,\n",
" '攻略': 131,\n",
" 'ポチたま': 132,\n",
" '人気': 133,\n",
" 'コーナー': 134,\n",
" 'SMAP': 135,\n",
" '香取慎吾': 136,\n",
" '広末涼子': 137,\n",
" '映画化': 138,\n",
" '工藤静香': 139,\n",
" 'ジョン・カーター': 140,\n",
" '絵画': 141,\n",
" 'コンクール': 142,\n",
" '作品': 143,\n",
" '発表': 144,\n",
" 'ローガン・ラーマン': 145,\n",
" 'ガブリエラ・ワイルド': 146,\n",
" 'カメラ': 147,\n",
" '回る': 148,\n",
" 'ない': 149,\n",
" '友達': 150,\n",
" '即完': 151,\n",
" '必至': 152,\n",
" 'アベンジャーズ': 153,\n",
" 'BE@RBRICK': 154,\n",
" '付き': 155,\n",
" '鑑賞券': 156,\n",
" '限定発売': 157,\n",
" '本格': 158,\n",
" 'パフォーマンス': 159,\n",
" '約束': 160,\n",
" '\\u3000': 161,\n",
" '新た': 162,\n",
" '聖歌': 163,\n",
" '隊': 164,\n",
" '感動': 165,\n",
" '呼ぶ': 166,\n",
" '20万人': 167,\n",
" '以上': 168,\n",
" '動員': 169,\n",
" 'AKB48': 170,\n",
" '第2弾': 171,\n",
" 'ドキュメンタリー映画': 172,\n",
" '早い': 173,\n",
" 'DVD化': 174,\n",
" '沢尻エリカ': 175,\n",
" '約': 176,\n",
" '4年': 177,\n",
" '半': 178,\n",
" 'ぶり': 179,\n",
" '女優': 180,\n",
" '復帰': 181,\n",
" 'いい女': 182,\n",
" '何': 183,\n",
" 'う': 184,\n",
" '？': 185,\n",
" '峰不二子': 186,\n",
" '勝る': 187,\n",
" '劣る': 188,\n",
" 'スタイル': 189,\n",
" '抜群': 190,\n",
" 'イイ': 191,\n",
" '映画': 192,\n",
" 'デビュー': 193,\n",
" '神': 194,\n",
" '失格': 195,\n",
" '男': 196,\n",
" '明かす': 197,\n",
" '仲': 198,\n",
" '悪い': 199,\n",
" '家族': 200,\n",
" '仮面ライダー': 201,\n",
" 'スーパー戦隊': 202,\n",
" '大人の事情': 203,\n",
" '有言実行': 204,\n",
" 'ジュード・ロウ': 205,\n",
" '自信': 206,\n",
" '引っ提げる': 207,\n",
" '来日': 208,\n",
" '決定': 209,\n",
" '週末': 210,\n",
" 'まとめ': 211,\n",
" '読み': 212,\n",
" '映画館': 213,\n",
" '特集': 214,\n",
" 'スタート': 215,\n",
" '電撃発表': 216,\n",
" 'ブルース・ウィリス': 217,\n",
" '育毛剤': 218,\n",
" 'プロデュース': 219,\n",
" '長編アニメーション': 220,\n",
" '賞': 221,\n",
" 'ジョニー・デップ': 222,\n",
" 'ランゴ': 223,\n",
" '米国': 224,\n",
" '版': 225,\n",
" 'ガリレオ': 226,\n",
" '美人すぎる': 227,\n",
" 'スパイ': 228,\n",
" '今': 229,\n",
" '海外ドラマ': 230,\n",
" 'いちおう': 231,\n",
" '劇場': 232,\n",
" '第2話': 233,\n",
" 'パパ': 234,\n",
" 'いつ': 235,\n",
" 'でも': 236,\n",
" '娘。': 237,\n",
" 'PAPARAZZI': 238,\n",
" '巻': 239,\n",
" '最強': 240,\n",
" '歴代': 241,\n",
" '最速': 242,\n",
" 'avatar': 243,\n",
" 'ハリポタ': 244,\n",
" '越える': 245,\n",
" '集結': 246,\n",
" 'ポスター': 247,\n",
" '画像': 248,\n",
" '世界初': 249,\n",
" '解禁': 250,\n",
" '栗山千明': 251,\n",
" 'チョコ': 252,\n",
" 'あげる': 253,\n",
" 'ドキドキ': 254,\n",
" 'ます': 255,\n",
" 'ケロロ軍曹': 256,\n",
" '貞子': 257,\n",
" 'コラボ': 258,\n",
" 'マット・デイモン': 259,\n",
" '優しい': 260,\n",
" '眼差し': 261,\n",
" '幸せへのキセキ': 262,\n",
" '公開': 263,\n",
" '麻生久美子': 264,\n",
" '大泉洋': 265,\n",
" '主演': 266,\n",
" '結成': 267,\n",
" '終了': 268,\n",
" 'キーファー・サザーランド': 269,\n",
" '話題': 270,\n",
" '日本': 271,\n",
" '上陸': 272,\n",
" '／': 273,\n",
" 'タッチ': 274,\n",
" '特別': 275,\n",
" '試写': 276,\n",
" '会': 277,\n",
" 'ご': 278,\n",
" '招待': 279,\n",
" '役者': 280,\n",
" 'ため': 281,\n",
" '生まれる': 282,\n",
" '来る': 283,\n",
" '普段': 284,\n",
" '三枚目': 285,\n",
" '素顔': 286,\n",
" '夢': 287,\n",
" '追う': 288,\n",
" '諦める': 289,\n",
" 'させる': 290,\n",
" '親': 291,\n",
" '気持ち': 292,\n",
" '元気': 293,\n",
" '東北': 294,\n",
" '秋田': 295,\n",
" '編': 296,\n",
" '自然': 297,\n",
" '大いなる': 298,\n",
" '力': 299,\n",
" '家族の絆': 300,\n",
" '噛みしめる': 301,\n",
" '日本版': 302,\n",
" 'EMINEM': 303,\n",
" 'HIPHOP': 304,\n",
" '界': 305,\n",
" '頂点': 306,\n",
" '問題作': 307,\n",
" 'イメージ': 308,\n",
" '楽曲': 309,\n",
" '参戦': 310,\n",
" 'ミッキー': 311,\n",
" 'お': 312,\n",
" '出迎え': 313,\n",
" 'Disney Dream': 314,\n",
" 'シアター': 315,\n",
" '仲間達': 316,\n",
" '開催中': 317,\n",
" 'ロード・オブ・ザ・リング': 318,\n",
" 'ファン': 319,\n",
" '必見': 320,\n",
" 'ついに': 321,\n",
" '壮大': 322,\n",
" '冒険': 323,\n",
" '始まる': 324,\n",
" 'アーティスト': 325,\n",
" 'ミシェル・アザナヴィシウス': 326,\n",
" '監督賞': 327,\n",
" '受賞': 328,\n",
" '第2部': 329,\n",
" '前作': 330,\n",
" '超える': 331,\n",
" 'エロス': 332,\n",
" 'VioLence': 333,\n",
" '注目': 334,\n",
" '韓国': 335,\n",
" '最高視聴率': 336,\n",
" '記録': 337,\n",
" 'ドラマ': 338,\n",
" '家庭料理': 339,\n",
" 'タイアップ': 340,\n",
" '今年': 341,\n",
" '最初': 342,\n",
" 'パン': 343,\n",
" '日': 344,\n",
" '最': 345,\n",
" '幸': 346,\n",
" '最高': 347,\n",
" 'メニュー': 348,\n",
" '1番': 349,\n",
" '誰': 350,\n",
" '北野武': 351,\n",
" '監督': 352,\n",
" '最新作': 353,\n",
" 'アウトレイジビヨンド': 354,\n",
" '試写会': 355,\n",
" 'スクリーン': 356,\n",
" 'から': 357,\n",
" '飛び出す': 358,\n",
" '貞子3D': 359,\n",
" '特報映像': 360,\n",
" '脚本': 361,\n",
" 'セリフ': 362,\n",
" '恋人': 363,\n",
" '同士': 364,\n",
" 'ある': 365,\n",
" '1日': 366,\n",
" '即興': 367,\n",
" '新感覚': 368,\n",
" 'ドラゴンボールZ': 369,\n",
" '17年ぶり': 370,\n",
" '新作': 371,\n",
" '劇場版': 372,\n",
" '世界': 373,\n",
" '興奮': 374,\n",
" '素敵': 375,\n",
" 'タレント': 376,\n",
" '井戸田': 377,\n",
" '思う': 378,\n",
" 'こと': 379,\n",
" 'ちゃんと伝える': 380,\n",
" '大事': 381,\n",
" '最も': 382,\n",
" '美しい': 383,\n",
" 'セクシー': 384,\n",
" '美魔女': 385,\n",
" 'ダークナイトライジング': 386,\n",
" '特典': 387,\n",
" 'グッズ': 388,\n",
" '前売り券': 389,\n",
" 'すごい': 390,\n",
" '15歳': 391,\n",
" '天才': 392,\n",
" '空手': 393,\n",
" '美少女': 394,\n",
" 'デビュー作': 395,\n",
" '見る': 396,\n",
" 'もらえる': 397,\n",
" 'ワクワク': 398,\n",
" 'どきどき': 399,\n",
" '宿る': 400,\n",
" '新曲': 401,\n",
" '披露': 402,\n",
" '宇宙戦艦ヤマト2199': 403,\n",
" 'TVシリーズ': 404,\n",
" '上映': 405,\n",
" 'ジェームズ・キャメロン': 406,\n",
" 'タイタニック3D': 407,\n",
" '山田孝之': 408,\n",
" '小西真奈美': 409,\n",
" '撮影': 410,\n",
" 'モノ': 411,\n",
" '映る': 412,\n",
" '込む': 413,\n",
" '伝説': 414,\n",
" 'ホラー映画': 415,\n",
" '蘇る': 416,\n",
" 'フライトナイト/恐怖の夜': 417,\n",
" '予告映像': 418,\n",
" 'ヘルプ~心がつなぐストーリー~': 419,\n",
" 'オクタヴィア・スペンサー': 420,\n",
" '初': 421,\n",
" '社': 422,\n",
" '競合': 423,\n",
" 'はやぶさ': 424,\n",
" 'Twitter': 425,\n",
" '総選挙': 426,\n",
" 'そっくりさん': 427,\n",
" '誰だ': 428,\n",
" '鉄の女': 429,\n",
" 'メリル・ストリープ': 430,\n",
" '29年ぶり': 431,\n",
" '3度目': 432,\n",
" 'オスカー': 433,\n",
" 'タイタンの逆襲': 434,\n",
" 'ビックリマン': 435,\n",
" '凄い': 436,\n",
" 'さ': 437,\n",
" '再確認': 438,\n",
" 'グラドル': 439,\n",
" '領域': 440,\n",
" '挑戦': 441,\n",
" 'おかもとまり': 442,\n",
" '私': 443,\n",
" '枕': 444,\n",
" '営業': 445,\n",
" 'です': 446,\n",
" '最終章': 447,\n",
" 'なんて': 448,\n",
" '言う': 449,\n",
" '続編': 450,\n",
" '恋しい': 451,\n",
" '林家三平': 452,\n",
" '国分佐智子': 453,\n",
" '出演': 454,\n",
" 'カーズ2': 455,\n",
" 'CM': 456,\n",
" 'ラブラブ': 457,\n",
" 'メイキング映像': 458,\n",
" '先行公開': 459,\n",
" '万': 460,\n",
" '再生': 461,\n",
" 'すぎる': 462,\n",
" '登場': 463,\n",
" '美女': 464,\n",
" 'たち': 465,\n",
" 'ドレス': 466,\n",
" '競演': 467,\n",
" '東京国際映画祭': 468,\n",
" '透け透け': 469,\n",
" 'シースルー!?': 470,\n",
" '衣装': 471,\n",
" '着る': 472,\n",
" 'セクシー女優': 473,\n",
" '恋': 474,\n",
" '吊り橋理論': 475,\n",
" '試す': 476,\n",
" '史上': 477,\n",
" '叫': 478,\n",
" '3D': 479,\n",
" 'アトラクション': 480,\n",
" 'ホラー': 481,\n",
" 'ファイナル・デッドブリッジ': 482,\n",
" 'チェック': 483,\n",
" 'TIME/タイム': 484,\n",
" '3億': 485,\n",
" '5000万円': 486,\n",
" '1位': 487,\n",
" '篠田麻里子': 488,\n",
" '宝石': 489,\n",
" '奇跡': 490,\n",
" '同額': 491,\n",
" '三池': 492,\n",
" '一命': 493,\n",
" 'について': 494,\n",
" '満島ひかり': 495,\n",
" 'いろんな': 496,\n",
" '意味': 497,\n",
" '限界': 498,\n",
" 'まで': 499,\n",
" 'やる': 500,\n",
" '小雪': 501,\n",
" '椎名桔平': 502,\n",
" '子育て': 503,\n",
" '指導': 504,\n",
" 'ナイナイ岡村': 505,\n",
" '自分': 506,\n",
" '破談': 507,\n",
" 'お見合い': 508,\n",
" '番組': 509,\n",
" '応援': 510,\n",
" '倉科カナ': 511,\n",
" 'セイ': 512,\n",
" 'シ': 513,\n",
" 'かける': 514,\n",
" '日韓': 515,\n",
" 'バトル': 516,\n",
" '挑む': 517,\n",
" 'TIGER&BUNNY': 518,\n",
" '特別企画': 519,\n",
" 'キング': 520,\n",
" 'HERO': 521,\n",
" '木下優樹菜': 522,\n",
" '本当': 523,\n",
" '強い': 524,\n",
" '貫き通す': 525,\n",
" '人': 526,\n",
" '全身': 527,\n",
" '整形': 528,\n",
" 'エリカ様': 529,\n",
" '大胆': 530,\n",
" 'ランジェリー': 531,\n",
" '姿': 532,\n",
" '冴える': 533,\n",
" '中年': 534,\n",
" 'ちょいわるおやじ': 535,\n",
" '変身': 536,\n",
" '数': 537,\n",
" 'だけ': 538,\n",
" '異なる': 539,\n",
" '幸せ': 540,\n",
" '形': 541,\n",
" '顔': 542,\n",
" '剛力彩芽': 543,\n",
" 'きもの': 544,\n",
" '魅力': 545,\n",
" 'たくさん': 546,\n",
" '方': 547,\n",
" '超': 548,\n",
" '応募': 549,\n",
" '殺到': 550,\n",
" 'サーバー': 551,\n",
" 'パンク': 552,\n",
" '2011年': 553,\n",
" '親子': 554,\n",
" '絆': 555,\n",
" '感じる': 556,\n",
" '小栗旬': 557,\n",
" '使える': 558,\n",
" '若者': 559,\n",
" 'キツツキと雨': 560,\n",
" 'バック・トゥ・ザ・フューチャー': 561,\n",
" 'デロリアン': 562,\n",
" '台': 563,\n",
" '限定': 564,\n",
" '販売': 565,\n",
" '活動休止': 566,\n",
" '原因': 567,\n",
" '人気漫画': 568,\n",
" 'クローバー': 569,\n",
" '連続ドラマ': 570,\n",
" '化': 571,\n",
" '片腕': 572,\n",
" '奪う': 573,\n",
" 'れる': 574,\n",
" '少女': 575,\n",
" '五体': 576,\n",
" '満足': 577,\n",
" '演じる': 578,\n",
" '方法': 579,\n",
" '岩手': 580,\n",
" '河童': 581,\n",
" 'ホント': 582,\n",
" '何者': 583,\n",
" '大島優子': 584,\n",
" '闇金': 585,\n",
" '転落': 586,\n",
" '闇金ウシジマくん': 587,\n",
" 'ドラゴン・タトゥーの女': 588,\n",
" '3日間': 589,\n",
" '獲得': 590,\n",
" 'ジャガー': 591,\n",
" 'トラ': 592,\n",
" '対決': 593,\n",
" '奇跡体験!アンビリバボー': 594,\n",
" '驚く': 595,\n",
" '得る': 596,\n",
" '体験': 597,\n",
" '中国武術': 598,\n",
" '源流': 599,\n",
" '少林拳': 600,\n",
" '秘密': 601,\n",
" '迫る': 602,\n",
" '3つ': 603,\n",
" '極意': 604,\n",
" '原作者': 605,\n",
" '唸る': 606,\n",
" 'せる': 607,\n",
" '仕上がり': 608,\n",
" '期待': 609,\n",
" 'るろうに剣心': 610,\n",
" '舞台挨拶': 611,\n",
" '2人': 612,\n",
" '会見': 613,\n",
" 'EXILE': 614,\n",
" 'ダンス': 615,\n",
" '真木よう子': 616,\n",
" '池脇千鶴': 617,\n",
" '選ぶ': 618,\n",
" 'どの': 619,\n",
" 'タイプ': 620,\n",
" '＜': 621,\n",
" '11月19日': 622,\n",
" '号': 623,\n",
" '＞': 624,\n",
" '優等生': 625,\n",
" '自慰': 626,\n",
" '行為': 627,\n",
" 'レズビアン': 628,\n",
" 'セックス': 629,\n",
" '禁断': 630,\n",
" '実験': 631,\n",
" 'フラン': 632,\n",
" '犬': 633,\n",
" '7つ': 634,\n",
" '明らか': 635,\n",
" '山形': 636,\n",
" '田舎': 637,\n",
" 'のに': 638,\n",
" 'しまう': 639,\n",
" '理由': 640,\n",
" 'アメリカ政府': 641,\n",
" '極秘': 642,\n",
" '輸送': 643,\n",
" '話し手': 644,\n",
" 'いける': 645,\n",
" 'シリーズ最新作': 646,\n",
" '世界各国': 647,\n",
" '好': 648,\n",
" '発進': 649,\n",
" '大阪': 650,\n",
" '商人': 651,\n",
" '好き': 652,\n",
" '続く': 653,\n",
" 'グリフィス': 654,\n",
" 'とも': 655,\n",
" 'ローラ': 656,\n",
" '泊': 657,\n",
" '200万円': 658,\n",
" '高級': 659,\n",
" 'ホテル': 660,\n",
" '高田純次': 661,\n",
" 'テキ': 662,\n",
" 'トー': 663,\n",
" '大小': 664,\n",
" '逆転': 665,\n",
" '生物': 666,\n",
" '仰天': 667,\n",
" 'センター・オブ・ジ・アース': 668,\n",
" '神秘の島': 669,\n",
" 'ロサンゼルス': 670,\n",
" 'トワイライト': 671,\n",
" '現象': 672,\n",
" '5日間': 673,\n",
" 'テント': 674,\n",
" '泊まり込む': 675,\n",
" '処刑人': 676,\n",
" 'Underworld': 677,\n",
" 'シリーズ': 678,\n",
" '第4弾': 679,\n",
" '敵': 680,\n",
" '人類': 681,\n",
" '秋': 682,\n",
" 'デートムービー': 683,\n",
" '大本命': 684,\n",
" 'COCO': 685,\n",
" 'スター・ウォーズ': 686,\n",
" '前売': 687,\n",
" '券': 688,\n",
" '長蛇': 689,\n",
" '列': 690,\n",
" '3倍': 691,\n",
" '売上': 692,\n",
" '売り切れ': 693,\n",
" '弁護士': 694,\n",
" 'ELIGH': 695,\n",
" 'プロデューサー': 696,\n",
" 'ジョージ・マイケル': 697,\n",
" '語る': 698,\n",
" '第6話': 699,\n",
" 'バレンタイン': 700,\n",
" 'ちょこっと': 701,\n",
" '欲しい': 702,\n",
" '女優魂': 703,\n",
" 'テスト': 704,\n",
" '全裸': 705,\n",
" '鳥越俊太郎': 706,\n",
" '全国': 707,\n",
" '希望': 708,\n",
" '人間': 709,\n",
" 'ボクシング': 710,\n",
" '死に絶える': 711,\n",
" '時代': 712,\n",
" 'ロボット': 713,\n",
" '格闘技': 714,\n",
" 'へ': 715,\n",
" '大統領': 716,\n",
" 'ニセモノ': 717,\n",
" 'G.I.ジョー': 718,\n",
" '特別映像': 719,\n",
" '満足度': 720,\n",
" '第': 721,\n",
" '制止': 722,\n",
" '騒動': 723,\n",
" '機動戦士ガンダムUC': 724,\n",
" '主題歌': 725,\n",
" '新人': 726,\n",
" '異例': 727,\n",
" '大': 728,\n",
" '抜擢': 729,\n",
" '野球カード': 730,\n",
" 'ボーイッシュ': 731,\n",
" '恋愛': 732,\n",
" 'トーク': 733,\n",
" 'ロックバンド': 734,\n",
" 'ピンク・フロイド': 735,\n",
" '爆音': 736,\n",
" '楽しむ': 737,\n",
" '鬼気': 738,\n",
" '婚活': 739,\n",
" '演技': 740,\n",
" '圧巻': 741,\n",
" '鍵泥棒のメソッド': 742,\n",
" '予告編': 743,\n",
" 'ぐっさん': 744,\n",
" '中村昌也': 745,\n",
" '吹っ飛ぶ': 746,\n",
" 'き': 747,\n",
" '暴れ': 748,\n",
" '4人': 749,\n",
" 'がかり': 750,\n",
" '止める': 751,\n",
" 'ほど': 752,\n",
" '乱闘': 753,\n",
" 'ミラ・ジョヴォヴィッチ': 754,\n",
" '連続': 755,\n",
" 'レイプ犯': 756,\n",
" '狙う': 757,\n",
" '食欲': 758,\n",
" 'そそる': 759,\n",
" '深夜食堂': 760,\n",
" '絶品': 761,\n",
" '自負': 762,\n",
" '無限': 763,\n",
" '野心': 764,\n",
" '果て': 765,\n",
" '追い求める': 766,\n",
" 'ここ': 767,\n",
" 'どう': 768,\n",
" '破滅': 769,\n",
" 'いく': 770,\n",
" '特報': 771,\n",
" '柴咲コウ': 772,\n",
" 'しばる': 773,\n",
" 'ちゃう': 774,\n",
" 'もったいない': 775,\n",
" 'ASIENCE': 776,\n",
" '曲': 777,\n",
" '発売': 778,\n",
" '桑田佳祐': 779,\n",
" '歌': 780,\n",
" '夢の扉': 781,\n",
" '開く': 782,\n",
" '推理作家': 783,\n",
" 'VS': 784,\n",
" '小説': 785,\n",
" '模倣犯': 786,\n",
" '命': 787,\n",
" '賭ける': 788,\n",
" '頭脳': 789,\n",
" 'ゲーム': 790,\n",
" '推理作家ポー最期の5日間': 791,\n",
" 'レオ': 792,\n",
" '様': 793,\n",
" '老人': 794,\n",
" 'さえ': 795,\n",
" '気付く': 796,\n",
" '観': 797,\n",
" 'キャラクター': 798,\n",
" '史上最高': 799,\n",
" '武藤敬司': 800,\n",
" '闘魂三銃士': 801,\n",
" 'ルーニー・マーラ': 802,\n",
" '誤解': 803,\n",
" '経験': 804,\n",
" '下半期': 805,\n",
" 'アワード': 806,\n",
" 'Vol.4': 807,\n",
" 'やっぱり': 808,\n",
" 'ジェニファー・アニストン': 809,\n",
" 'YOSHIKI': 810,\n",
" 'ゴールデン・グローブ賞': 811,\n",
" '授賞式': 812,\n",
" 'テーマ曲': 813,\n",
" '感無量': 814,\n",
" 'BIO HAZARD': 815,\n",
" '三銃士': 816,\n",
" 'ドイツ': 817,\n",
" '三浦友和': 818,\n",
" '百恵': 819,\n",
" 'さん': 820,\n",
" '夫婦関係': 821,\n",
" '告白': 822,\n",
" '映画祭': 823,\n",
" '沸騰': 824,\n",
" 'インドネシア': 825,\n",
" '壮絶': 826,\n",
" 'アクション': 827,\n",
" 'ザ・レイド': 828,\n",
" '3週': 829,\n",
" 'ディズニー': 830,\n",
" '達成': 831,\n",
" 'ブランド': 832,\n",
" '商品': 833,\n",
" '鈴木亜美': 834,\n",
" '一新': 835,\n",
" '大人': 836,\n",
" '色気': 837,\n",
" 'たっぷり': 838,\n",
" 'コールガール': 839,\n",
" '第12話': 840,\n",
" 'ミニサイズ': 841,\n",
" '体内': 842,\n",
" '行く': 843,\n",
" 'ぞ': 844,\n",
" '女性アイドル': 845,\n",
" '一堂': 846,\n",
" '会す': 847,\n",
" '究極': 848,\n",
" '杉良太郎': 849,\n",
" 'いじめ': 850,\n",
" 'とか': 851,\n",
" '自殺': 852,\n",
" '教育': 853,\n",
" '全然': 854,\n",
" 'できる': 855,\n",
" 'ので': 856,\n",
" 'を通じて': 857,\n",
" 'たい': 858,\n",
" '制服姿': 859,\n",
" '大政絢': 860,\n",
" 'や': 861,\n",
" 'ナルニア': 862,\n",
" '王子': 863,\n",
" 'など': 864,\n",
" '直撃': 865,\n",
" 'Volume.1': 866,\n",
" '共存': 867,\n",
" '操り人形': 868,\n",
" 'ハリウッド': 869,\n",
" '殿堂入り': 870,\n",
" '巨大': 871,\n",
" '都市': 872,\n",
" 'ニューヨーク': 873,\n",
" '各所': 874,\n",
" '出現': 875,\n",
" 'どこでもドア': 876,\n",
" 'コント': 877,\n",
" '決め手': 878,\n",
" '優香': 879,\n",
" '長編': 880,\n",
" 'アニメ映画': 881,\n",
" '声優': 882,\n",
" '触発': 883,\n",
" 'スノーホワイト': 884,\n",
" 'デジタル': 885,\n",
" '西田敏行': 886,\n",
" '思わず': 887,\n",
" '涙ぐむ': 888,\n",
" '父': 889,\n",
" '手紙': 890,\n",
" 'ピラミッド': 891,\n",
" '地球': 892,\n",
" 'ID': 893,\n",
" 'iTunes': 894,\n",
" 'ダウンロード': 895,\n",
" 'ダークナイト': 896,\n",
" 'ライジング!': 897,\n",
" '宮崎あおい': 898,\n",
" 'いろいろ': 899,\n",
" '夫婦': 900,\n",
" '映画界': 901,\n",
" '生きる伝説': 902,\n",
" '起源': 903,\n",
" '最多': 904,\n",
" '3冠': 905,\n",
" '火水': 906,\n",
" '原作': 907,\n",
" '古谷実': 908,\n",
" '下ろす': 909,\n",
" 'イラスト': 910,\n",
" '人種': 911,\n",
" '問題': 912,\n",
" 'タブー': 913,\n",
" 'Blu-Ray': 914,\n",
" 'セット': 915,\n",
" '最先端': 916,\n",
" '全長': 917,\n",
" '500m': 918,\n",
" '魔物': 919,\n",
" '助演女優賞': 920,\n",
" 'ヘルプ': 921,\n",
" '〜': 922,\n",
" '心': 923,\n",
" 'つなぐ': 924,\n",
" 'ストーリー': 925,\n",
" '悪夢': 926,\n",
" 'プロメテウス': 927,\n",
" '著名人': 928,\n",
" '絶賛': 929,\n",
" 'コメント': 930,\n",
" '続々': 931,\n",
" 'によって': 932,\n",
" '定める': 933,\n",
" '運命': 934,\n",
" '抗う': 935,\n",
" '恋人たち': 936,\n",
" 'どん底': 937,\n",
" '支える': 938,\n",
" 'もつ': 939,\n",
" 'ヒロイン': 940,\n",
" '多く': 941,\n",
" '女性': 942,\n",
" '共感': 943,\n",
" 'FBI': 944,\n",
" '行動': 945,\n",
" '分析': 946,\n",
" '課': 947,\n",
" '学ぶ': 948,\n",
" 'クリミナル・マインド': 949,\n",
" '流': 950,\n",
" 'トラブル': 951,\n",
" '生き抜く': 952,\n",
" 'ビジネス': 953,\n",
" '術': 954,\n",
" '若手': 955,\n",
" 'ゆがめる': 956,\n",
" '三': 957,\n",
" '七': 958,\n",
" '拍子': 959,\n",
" 'SCREAM': 960,\n",
" '絶叫': 961,\n",
" '映像': 962,\n",
" '高校時代': 963,\n",
" 'より': 964,\n",
" '遊ぶ': 965,\n",
" '夢中': 966,\n",
" 'ハート': 967,\n",
" '寞寞': 968,\n",
" '未公開シーン': 969,\n",
" '収録': 970,\n",
" 'BD': 971,\n",
" '発売決定': 972,\n",
" 'せっかく': 973,\n",
" '動画': 974,\n",
" 'はしゃぐ': 975,\n",
" '過ぎ': 976,\n",
" 'しっちゃかめっちゃか': 977,\n",
" '掟破り': 978,\n",
" '本人': 979,\n",
" 'そっくり': 980,\n",
" '世界中': 981,\n",
" 'マーガレット・サッチャー鉄の女の涙': 982,\n",
" 'メイクアップ賞': 983,\n",
" '勝つ': 984,\n",
" '第4話': 985,\n",
" '謹賀新年': 986,\n",
" 'サイコロ': 987,\n",
" 'サイ': 988,\n",
" '中川翔子': 989,\n",
" 'ジャッキー': 990,\n",
" '吹き替え': 991,\n",
" 'キャスト': 992,\n",
" '高田里穂': 993,\n",
" 'コスプレ': 994,\n",
" '願望': 995,\n",
" 'Volume.6': 996,\n",
" '鈴木勝大': 997,\n",
" '馬場': 998,\n",
" '良馬': 999,\n",
" ...}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vocabulary"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"class LSTMClassifier(nn.Module):\n",
" \n",
" def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):\n",
" \n",
" super(LSTMClassifier, self).__init__()\n",
" # define dimension of hidden layer, which is related to output\n",
" self.hidden_dim = hidden_dim\n",
" # define word_embedding\n",
" self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
" # define lstm layer\n",
" self.lstm = nn.LSTM(embedding_dim, hidden_dim)\n",
" # network for softmax layer to get output from lstm through linear layer\n",
" self.hidden2tag = nn.Linear(hidden_dim, tagset_size)\n",
" # dim => direction to transform output into probabillity\n",
" self.softmax = nn.LogSoftmax(dim=1)\n",
" \n",
" \n",
" def forward(self, sentence):\n",
" \n",
" # convert sentence into embeds\n",
" embeds = self.word_embeddings(sentence)\n",
" # transform embeds into 3d tensor\n",
" # use second return value\n",
" _, lstm_out = self.lstm(embeds.view(len(sentence), 1, -1))\n",
" # adjust size\n",
" tag_space = self.hidden2tag(lstm_out[0].view(-1, self.hidden_dim))\n",
" # return probability by softmax\n",
" tag_scores = self.softmax(tag_space)\n",
" \n",
" return tag_scores"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# training"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"import torch.optim as optim"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
"train, test = train_test_split(datasets, train_size=0.7)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(5163, 2213)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(train), len(test)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"EMBEDDING_DIM = 10\n",
"HIDDEN_DIM = 128\n",
"VOCAB_SIZE = len(vocabulary)\n",
"TAG_SIZE = len(categories)\n",
"model = LSTMClassifier(EMBEDDING_DIM, HIDDEN_DIM, VOCAB_SIZE, TAG_SIZE)\n",
"\n",
"loss_function = nn.NLLLoss()\n",
"optimizer = optim.SGD(model.parameters(), lr=0.01)"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2019/11/28, 17:42:56] START\n",
"[2019/11/28, 17:43:29] Epoch: 0 \t loss: 4832.0912630565\n",
"[2019/11/28, 17:44:02] Epoch: 1 \t loss: 4467.138694204681\n",
"[2019/11/28, 17:44:36] Epoch: 2 \t loss: 4098.260502435376\n",
"[2019/11/28, 17:45:09] Epoch: 3 \t loss: 3820.2763732786407\n",
"[2019/11/28, 17:45:41] Epoch: 4 \t loss: 3479.227999129622\n",
"[2019/11/28, 17:46:14] Epoch: 5 \t loss: 3173.5345642968277\n",
"[2019/11/28, 17:46:47] Epoch: 6 \t loss: 2886.67537934858\n",
"[2019/11/28, 17:47:21] Epoch: 7 \t loss: 2553.3790440822772\n",
"[2019/11/28, 17:47:54] Epoch: 8 \t loss: 2236.5872594857847\n",
"[2019/11/28, 17:48:27] Epoch: 9 \t loss: 1911.2969607356215\n",
"[2019/11/28, 17:49:00] Epoch: 10 \t loss: 1720.673903518108\n",
"[2019/11/28, 17:49:33] Epoch: 11 \t loss: 1430.2289244721273\n",
"[2019/11/28, 17:50:06] Epoch: 12 \t loss: 1267.3478920268794\n",
"[2019/11/28, 17:50:40] Epoch: 13 \t loss: 846.1975001558892\n",
"[2019/11/28, 17:51:13] Epoch: 14 \t loss: 609.5649736204631\n",
"[2019/11/28, 17:51:46] Epoch: 15 \t loss: 689.5234388912285\n",
"[2019/11/28, 17:52:19] Epoch: 16 \t loss: 637.552605238169\n",
"[2019/11/28, 17:52:54] Epoch: 17 \t loss: 514.207423426353\n",
"[2019/11/28, 17:53:27] Epoch: 18 \t loss: 299.237089747968\n",
"[2019/11/28, 17:54:00] Epoch: 19 \t loss: 282.0658448503776\n",
"[2019/11/28, 17:54:00] End\n"
]
}
],
"source": [
"losses = []\n",
"print(\"[{}] \".format(get_timestamp()), 'START')\n",
"for epoch in range(20):\n",
" all_loss = 0\n",
" for _, title, cat in train.itertuples():\n",
" \n",
" model.zero_grad()\n",
" \n",
" inputs = sentence2idx(title, vocabulary)\n",
" \n",
" out = model(inputs)\n",
" \n",
" answer = cat2tensor(cat)\n",
" \n",
" loss = loss_function(out, answer)\n",
" \n",
" loss.backward()\n",
" \n",
" optimizer.step()\n",
" \n",
" all_loss += loss.item()\n",
" \n",
" losses.append(all_loss)\n",
" print(\"[{}] \".format(get_timestamp()), \"Epoch: \", epoch, \"\\t\", \"loss: \", all_loss)\n",
" \n",
"print(\"[{}] \".format(get_timestamp()), 'End')"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xd8VGW+x/HPL4WEUFIgQKhBKYoUgdBRUVcFlxVsiFgQURTrru696l3vrnf37lruru6qCIIUC3ZFUNFd7AoEKVIFISAttNACIYGQ8Nw/5uBGTEyAZM5k5vt+veY1Z57zTOY3J5P55pzznHPMOYeIiESeKL8LEBERfygAREQilAJARCRCKQBERCKUAkBEJEIpAEREIpQCQEQkQikAREQiVIUCwMzWm9kyM1tsZgu8thQzm2Vma7z7ZK/dzOxJM8sys6Vm1qXEzxnu9V9jZsOr5i2JiEhFWEWOBDaz9UCGc25nibbHgN3OuUfM7H4g2Tl3n5ldDNwJXAz0AP7hnOthZinAAiADcMBCoKtzbk9Zr1u/fn2Xnp5+wm9ORCQSLVy4cKdzLrW8fjEn8RqDgH7e9PPAZ8B9XvsLLpAsmWaWZGZpXt9ZzrndAGY2C+gPvFLWC6Snp7NgwYKTKFFEJPKY2YaK9KvoPgAH/MvMFprZKK+toXNuqze9DWjoTTcBNpV47mavrax2ERHxQUXXAPo657LNrAEwy8xWlZzpnHNmVilnlfMCZhRA8+bNK+NHiohIKSq0BuCcy/budwDTgO7Adm/TDt79Dq97NtCsxNObem1ltR/7WuOdcxnOuYzU1HI3YYmIyAkqNwDMrJaZ1Tk6DVwILAdmAEdH8gwHpnvTM4DrvdFAPYFcb1PRP4ELzSzZGzF0odcmIiI+qMgmoIbANDM72v9l59yHZjYfeN3MRgIbgCFe/5kERgBlAfnACADn3G4z+xMw3+v3x6M7hEVEJPgqNAzULxkZGU6jgEREjo+ZLXTOZZTXT0cCi4hEqLAMgILCYh6asYLc/MN+lyIiErLCMgBWbMnl5XkbuWZiJnvzC/0uR0QkJIVlAGSkp/Ds9V1ZvT2PYRPmseeAQkBE5FhhGQAA57ZtwITrM8jKyWPYc/PYrRAQEfmRsA0AgHPapDJxeAbrcvIYNiGTXXmH/C5JRCRkhHUAAJzVOpVJN3Rj/a4DDJswj50KARERIAICAKBPq/pMGt6NDbsPcPX4THL2KwRERCIiAAB6t6rPlBHd2byngKsnZLJj/0G/SxIR8VXEBABAz1PqMWVEN7bsLWDo+Ey271MIiEjkiqgAAOhxSj2ev7E723MPMnR8JttyFQIiEpkiLgAAuqWn8MLI7uTsP8TQ8XPZmlvgd0kiIkEXkQEA0LVFIAR25RUydHwmW/YqBEQkskRsAAB0aZ7MCyO7szuvkKvGz2Xznny/SxIRCZqIDgCAzs2TeemmHuTmH2bo+Ew27VYIiEhkiPgAAOjULImpN/Vk/8EihYCIRAwFgKdD00Sm3tSDvEOBENi4SyEgIuFNAVBC+yaJvHxzD/ILi7hq/Fw27Drgd0kiIlVGAXCMMxon8vLNPTl4uJih4zMVAiISthQApTg9ra5CQETCngKgDAoBEQl3CoCfoRAQkXCmACiHQkBEwpUCoAIUAiISjhQAFXR6Wl2m3qQQEJHwoQA4Du0aKwREJHwoAI5TyRC4WiEgItWYAuAEHA2BAoWAiFRjCoATpBAQkepOAXASFAIiUp0pAE7S0RDIVwiISDWjAKgE7RrX5WWFgIhUMwqASqIQEJHqRgFQiY4NgTXb9/tdkohImRQAlexoCBQWOy55ejZvLdzsd0kiIqWqcACYWbSZfWNm73mPW5rZPDPLMrPXzKyG1x7nPc7y5qeX+BkPeO3fmdlFlf1mQkW7xnWZeVdfOjVL5N43lvCfby6hoLDY77JERH7keNYA7gZWlnj8KPCEc64VsAcY6bWPBPZ47U94/TCzdsBQ4AygP/CMmUWfXPmhq0HdeF4a2YM7z2vFGws3M3jMbLJ25PldlojIDyoUAGbWFPgl8Jz32IDzgDe9Ls8Dg73pQd5jvPnne/0HAa865w45574HsoDulfEmQlVMdBT3XtiW50d0JyfvEJc8/RXTF2f7XZaICFDxNYC/A/8JHPEe1wP2OueKvMebgSbedBNgE4A3P9fr/0N7Kc/5gZmNMrMFZrYgJyfnON5K6Dq7TSoz7zqL9o0TufvVxTzw9jIOHtYmIRHxV7kBYGYDgR3OuYVBqAfn3HjnXIZzLiM1NTUYLxkUjRLjefnmHozudyqvfL2RS5+Zw/c7NVRURPxTkTWAPsAlZrYeeJXApp9/AElmFuP1aQoc3baRDTQD8OYnArtKtpfynIgQEx3Fff1PY/IN3diaW8DAJ7/k3SVb/C5LRCJUuQHgnHvAOdfUOZdOYCfuJ865a4BPgSu8bsOB6d70DO8x3vxPnHPOax/qjRJqCbQGvq60d1KNnHtaA2bedRZtG9Xhzle+4cF3tElIRILvZI4DuA+4x8yyCGzjn+i1TwTqee33APcDOOdWAK8D3wIfArc75yL2W69xUk1eu6UXo84+hZcyN3LFuDk6elhEgsoC/5yHpoyMDLdgwQK/y6hyH327nXvfWMKRI47HrujIgA5pfpckItWYmS10zmWU109HAoeAX7RryHt39uWUBrUZPXURD81YwaGiiF05EpEgUQCEiGYpCbxxSy9u7NOSKXPWM2TcXLbmFvhdloiEMQVACKkRE8Xvf9WOcdd2ZW3OAa4YO5e1OTp6WESqhgIgBPVv34hXRwUuPD9k3FyWZ+f6XZKIhCEFQIhq3ySRN27tRXxsNEPHZ5K5bpffJYlImFEAhLBTUmvz5uhepCXGc/2kr5n17Xa/SxKRMKIACHFpiTV5/ZZenJ5Wl1tfWqjrC4hIpVEAVAPJtWow9aYe9DwlhXvfWMJzX67zuyQRCQMKgGqidlwMk27oxoD2jfjf91fy139+RygfxCcioU8BUI3ExUTz9LAuDO3WjKc/zeK/py+n+IhCQEROTEz5XSSUREcZD1/WgaSEGoz7fC178w/z+JAzqRGjLBeR46MAqIbMjPsHnEZyQiwPf7CKfQeLGHdtFxJq6NcpIhWnfxursVvOOZVHL+/AV2tyuPa5eezNL/S7JBGpRhQA1dxV3ZrzzDVdWJ69j6uezWTHvoN+lyQi1YQCIAz0b5/G5BHd2Lwnn8t1XQERqSAFQJjo06o+L9/ck7yDRVwxbi4rt+7zuyQRCXEKgDDSqVkSb9zai2gzhjw7l7lrdf4gESmbAiDMtGpQhzdH96Jh3XiumziPl+dt9LskEQlRCoAw1DQ5gbdv603f1vX5r2nLeGjGCoqKj/hdloiEGAVAmKobH8vE4d24qW/gCmMjpswnN/+w32WJSAhRAISx6CjjwYHteOzyjmSu28Wlz8xmna4wJiIeBUAEGNKtGS/f3JO9BYcZPGY2X67J8bskEQkBCoAI0S09hem39yEtsSY3TJ7P83PW62yiIhFOARBBmqUk8NZtvTm3bQP+MGMFv3tnOYe1c1gkYikAIkztuBjGX9eV0f1O5eV5G7lu4jz2HNA5hEQikQIgAkVFGff1P40nrurEoo17GTRmNmu27/e7LBEJMgVABLu0c1NeHdWT/MJiLn1mDp+u2uF3SSISRAqACNeleTIz7uhDi3oJ3Pj8fCZ8sU47h0UihAJAaJxUkzdu7UX/Mxrx55kr+Y83l3KoqNjvskSkiikABICEGjGMGdaFu89vzZsLNzNswjxy9h/yuywRqUIKAPlBVJTxmwva8PSwzqzYksvAp75k4YbdfpclIlVEASA/MbBjY94e3Yf42GiuejaTKbO/134BkTCkAJBStWtclxl39KVf21Qeevdb7n51MfmFRX6XJSKVSAEgZUqsGcv46zL4j4va8t7SLQweo5PJiYSTcgPAzOLN7GszW2JmK8zsf7z2lmY2z8yyzOw1M6vhtcd5j7O8+eklftYDXvt3ZnZRVb0pqTxRUcbt57bihRt7sDOvkEuens2Hy7f6XZaIVIKKrAEcAs5zznUCzgT6m1lP4FHgCedcK2APMNLrPxLY47U/4fXDzNoBQ4EzgP7AM2YWXZlvRqpO39b1effOvpyaWotbX1rEwzNX6iIzItVcuQHgAo6u98d6NwecB7zptT8PDPamB3mP8eafb2bmtb/qnDvknPseyAK6V8q7kKBoklST12/txbU9m/PsF+u4dqKGiopUZxXaB2Bm0Wa2GNgBzALWAnudc0f3Cm4GmnjTTYBNAN78XKBeyfZSniPVRFxMNP87uAOPD+nE4k17NVRUpBqrUAA454qdc2cCTQn8135aVRVkZqPMbIGZLcjJ0YVLQtVlXZpqqKhINXdco4Ccc3uBT4FeQJKZxXizmgLZ3nQ20AzAm58I7CrZXspzSr7GeOdchnMuIzU19XjKkyDTUFGR6q0io4BSzSzJm64JXACsJBAEV3jdhgPTvekZ3mO8+Z+4wL+GM4Ch3iihlkBr4OvKeiPiDw0VFam+KrIGkAZ8amZLgfnALOfce8B9wD1mlkVgG/9Er/9EoJ7Xfg9wP4BzbgXwOvAt8CFwu3NOZxwLA6UPFd3md1kiUg4L5e22GRkZbsGCBX6XIcche28Bt720kCWbc7nngjbceV4rAoPARCRYzGyhcy6jvH46Elgq1dGhopd1acLjs1bz39OXU3wkdP/JEIlkMeV3ETk+cTHR/O3KTjSoE8+4z9eyK6+QJ646k/hYHfcnEkoUAFIlzIz7B5xGap04/vTet+w+8DUThmdQNz7W79JExKNNQFKlRvZtyT+GnsmijXu46tlMduw76HdJIuJRAEiVG3RmEyYO78aGXQe4bOwcDRMVCREKAAmKs9uk8uqonhQUFnPFuLks3rTX75JEIp4CQIKmY9Mk3hzdm1px0QybkMnnq3WqDxE/KQAkqFrWr8Vbo3uTXq8WI6fMZ9o3m/0uSSRiKQAk6BrUiee1W3rSLT2F37y2hAlfrPO7JJGIpAAQX9SJj2XKjd34ZYc0/jxzJX+ZuZIjOmBMJKh0HID4Ji4mmiev7ky92jUY/8U6cvYf4rErOhIbrf9LRIJBASC+io4y/ueSM2hQJ46//ms1uw4UMvaaLtSK00dTpKrpXy3xnZlxx3mteeSyDny1JodhEzLZladLTYpUNQWAhIyh3Zvz7HUZrNq2nyvGzWXT7ny/SxIJawoACSkXtGvI1Jt6sPtAIYPHzGbRxj1+lyQSthQAEnIy0lN4+7be1IqL4erxmcxcttXvkkTCkgJAQtKpqbWZdltv2jdJ5Lapixj3+VpddF6kkikAJGTVqx3H1Jt6MLBjGo98sIr/mraMw8VH/C5LJGxorJ2EtPjYaJ4c2pn0erV4+tMsNu8pYMw1XXRdAZFKoDUACXlRUcZvL2rLY1d0ZO7aXVwxdg6b92iEkMjJUgBItTEkoxkv3NidrbkHGTxmDkt0SmmRk6IAkGqld6v6TLutN/GxUVw1fi4fLt/md0ki1ZYCQKqdVg3q8M7tfTg9rS6jpy5kwhfrNEJI5AQoAKRaql87jldu7snF7QNnE33wneUUaYSQyHHRKCCptuJjo3nq6s40r5fA2M/WsnlPAU8P60wdjRASqRCtAUi1FhVl3Nf/tMCJ5LJ2cuW4uWzZW+B3WSLVggJAwsLQ7s2ZMqIb2XsKGDxmNss25/pdkkjIUwBI2DirdSpv3dab2Ogohjw7l4++3e53SSIhTQEgYaVNwzpMu703rRvWZtSLC3gpc4PfJYmELAWAhJ0GdeJ5dVRP+rVtwIPvLOexD1dpmKhIKRQAEpYSasQw/rquXN29Oc98tpZ7Xl9CYZGGiYqUpGGgErZioqP4y6XtaZpck//753fs2H+Qsdd21YnkRDxaA5CwZmbcfm4r/nZlJ+at282QcXPZmqthoiKgAJAIcXnXpkwe0Y3Newq4dMwcVm3b53dJIr5TAEjEOKt1Kq/f0guH48qxc5mTtdPvkkR8VW4AmFkzM/vUzL41sxVmdrfXnmJms8xsjXef7LWbmT1pZllmttTMupT4WcO9/mvMbHjVvS2R0rVrXJe3b+tDo8R4hk/+mumLs/0uScQ3FVkDKALudc61A3oCt5tZO+B+4GPnXGvgY+8xwACgtXcbBYyFQGAAfwB6AN2BPxwNDZFgapJUkzdv7U2X5snc/epixn6m6w1LZCo3AJxzW51zi7zp/cBKoAkwCHje6/Y8MNibHgS84AIygSQzSwMuAmY553Y75/YAs4D+lfpuRCooMSGWF0Z251edGvPoh6v4/fQVFB9RCEhkOa5hoGaWDnQG5gENnXNbvVnbgIbedBNgU4mnbfbaymo/9jVGEVhzoHnz5sdTnshxiYuJ5h9XnUnjpHie/Xwd2/Yd5MmhnalZI9rv0kSCosI7gc2sNvAW8Gvn3I+GULjA+nOl/PvknBvvnMtwzmWkpqZWxo8UKVNUlPHAgNP5n0vO4KOV27l6Qia78g75XZZIUFQoAMwslsCX/1Tn3Nte83Zv0w7e/Q6vPRtoVuLpTb22stpFfDe8dzpjr+nKyq37uHzsHNbvPOB3SSJVriKjgAyYCKx0zj1eYtYM4OhInuHA9BLt13ujgXoCud6mon8CF5pZsrfz90KvTSQk9G/fiJdv7kluwWEuGzuHf63Ypp3DEtYqsgbQB7gOOM/MFnu3i4FHgAvMbA3wC+8xwExgHZAFTABuA3DO7Qb+BMz3bn/02kRCRtcWybw1ujfJCbGMenEhg5+Zw5drchQEEpYslD/YGRkZbsGCBX6XIRGoqPgIby/K5h8fryF7bwHdW6bw2wvb0r1lit+liZTLzBY65zLK7acAECnboaJiXpu/iac+ySJn/yHObpPKvRe0oVOzJL9LEymTAkCkEhUUFvNi5nrGfraWPfmHuaBdQ+69sA2nNarrd2kiP6EAEKkCeYeKmPzV94z/ch15h4oY2LExv/5Fa05Nre13aSI/UACIVKG9+YVM+HIdk2ev5+DhYi7v0pS7zm9Ns5QEv0sTUQCIBMPOvEOM/WwtL2ZuwDnHVd2aced5rWlYN97v0iSCKQBEgmhb7kGe+mQNr83fRHSUcV3PFtx5XmsSE3T1MQm+igaArgcgUgkaJcbz50s78Olv+zGwY2Mmzf6egU9/yYotuX6XJlImBYBIJWqWksDfhnTirdG9OVzkuOyZOUz7ZrPfZYmUSgEgUgU6N0/m3Tv7cmazJH7z2hIemrGCwqIjfpcl8iMKAJEqklonjqk39eCmvi2ZMmc9wyZksmPfQb/LEvmBAkCkCsVER/HgwHY8dXVnVmzZxy+f+or563UKLAkNCgCRIPhVp8a8c3sfatWI5urxmUyZ/b1OMCe+UwCIBEnbRnWYfkdf+rVN5aF3v+We15dQUFjsd1kSwRQAIkGUWDOW8ddlcM8FbXhncTaXjZ3Dxl35fpclEUoBIBJkUVHGXee3ZtIN3cjek8+vnv6KT7/bUf4TRSqZAkDEJ+e2bcC7d/YlLTGeG6fM56mP13DkiPYLSPAoAER81KJeLabd1odBnRrzt1mrGfXiAvYdPOx3WRIhFAAiPqtZI5onrjqTh37Vjs++y2HQ07P5btt+v8uSCKAAEAkBZsYNfVry8s092X+wiMFjZjP2s7Xk5mttQKqOAkAkhHRvmcL7d/UlIz2ZRz9cRa9HPuYP05ezfucBv0uTMKTTQYuEqOXZuUya/T3vLtlC0RHH+ac1ZGTflvQ8JQUz87s8CWG6HoBImNix7yAvZm7gpcwN7Mk/zBmN63Jjn5b8qlNjasRoJV5+SgEgEmYOHi5m2jfZTPzqe7J25JFaJ47hvVowrEcLUmrV8Ls8CSEKAJEw5Zzj89U5TJq9ni9W5xAXE8VlXZoysm86rRrU8bs8CQEVDYCYYBQjIpXHzOjXtgH92jZg9fb9TPrqe95atJlXvt7IOW1SGdm3JWe1rq/9BFIurQGIhIFdeYeYOm8jL8zdwM68Q7RpWJs/X9qBbukpfpcmPtA1gUUiSL3acdx1fmtm338uf72yEwcPH+HGKfN1QJn8LAWASBiJi4nmiq5NeWVUT2rGRjNi8tds11XIpAwKAJEw1CSpJpNu6EZuwWFumDyfvENFfpckIUgBIBKm2jdJZMw1XVi9fT+3TV3E4WJdlF5+TAEgEsb6tW3Anwe354vVOTw4bbkuQyk/omGgImFuaPfmZO8t4KlPsmiaXJM7z2/td0kSIhQAIhHgngvakL2ngL/NWk3jpJpc3rWp3yVJCFAAiEQAM+ORyzuybd9B7ntrKY0S4+nTqr7fZYnPyt0HYGaTzGyHmS0v0ZZiZrPMbI13n+y1m5k9aWZZZrbUzLqUeM5wr/8aMxteNW9HRMpSIyaKcdd15dTU2tz64kIdIyAV2gk8Beh/TNv9wMfOudbAx95jgAFAa+82ChgLgcAA/gD0ALoDfzgaGiISPHXjY5k8ohsJcdHcMPlrtuXqGIFIVm4AOOe+AHYf0zwIeN6bfh4YXKL9BReQCSSZWRpwETDLObfbObcHmMVPQ0VEgqCxd4zAvoLDjJiiYwQi2YkOA23onNvqTW8DGnrTTYBNJfpt9trKahcRH5zROJFnru3K6u37Gf3SQh0jEKFO+jgAFxhYXGmDi81slJktMLMFOTk5lfVjReQY57RJ5eFLO/Dlmp38btoyHSMQgU40ALZ7m3bw7nd47dlAsxL9mnptZbX/hHNuvHMuwzmXkZqaeoLliUhFDOnWjLvOb83rCzbz1CdZfpcjQXaiATADODqSZzgwvUT79d5ooJ5Arrep6J/AhWaW7O38vdBrExGf/eYXrbmsSxMen7WaNxdu9rscCaJyjwMws1eAfkB9M9tMYDTPI8DrZjYS2AAM8brPBC4GsoB8YASAc263mf0JmO/1+6Nz7tgdyyLiAzPjkcs6sn3fQe5/aylpOkYgYuiCMCICwL6Dh7ly7Fy27C3gjdG9OK1RXb9LkhOkC8KIyHEpeYzAiMnzyd5b4HdJUsUUACLyg8ZJNZl8Q3f2HyxiwN+/4LX5GzU6KIwpAETkR9o1rsuMO/pwelpd7ntrGcMmzGP9zgN+lyVVQAEgIj9xSmptXrm5Jw9f1oHlW3K56O9fMO7ztRTpgLGwogAQkVJFRRlXd2/OR/ecQ7+2qTzywSoGjZnN8uxcv0uTSqIAEJGf1bBuPM9el8G4a7uwY/8hBo2ZzcMfrKSgsNjv0uQkKQBEpEL6t0/jo9+cw5Vdm/Ls5+vo/48vmJO10++y5CToOAAROW5z1u7kv95exvpd+QzJaMrvLm5HYkLsSf/cfQcPs2TTXpZuzqVGdBQt6iWQXr8WzVMSiI+NroTKI0NFjwNQAIjICTl4uJi/f7SGCV+uIzmhBn8cdAYD2jfCzCr0/KLiI6zensfiTXv5ZuMevtm0l7U5eZT1lZSWGB8IhHq1SK9fi/R6CbSoV4sW9RJIqKGLG5akABCRoFiencv9by9lefY+LmjXkD8Nak+jxPif9Nux7yCLNu794Qt/WXYu+d5+hOSEWDo3T+bMZkl0bp5Ex6ZJOOfYsCuf9bsO/Oh+w64D7Mwr/NHPblAnjnQvDNLrB+67tkgmLbFmUJZBqFEAiEjQFBUfYdLs73l81mpio6K4b8BpnNaojvdlH/jC3+JdfSw22miXVvdHX/jNUxIqvOYAgU1FG0uGw85/h8SO/Yd+6JfRIpmLO6RxcYe0UkMpXCkARCToNuw6wANvL2PO2l0/tDVNrul90Qe+8M9oXLdKt+cfOFTE9zsP8Nl3O3hv6VZWedc+7pb+7zBoWDe8w0ABICK+cM7x8codOODMZkmk1onztZ61OXnMXLqV95cFwsAMurVI4eIOjRgQpmGgABAROUbWjjxmLtvK+0u38t32f4fBLzumMaB9IxqESRgoAEREfkbWjv28v3Qb7y/bwurteYEwSE9hYMc0+rdvRIM61TcMFAAiIhW0Zvt+3l+2lZnLtv4QBgPaN+KBAafTLCXB7/KOmwJAROQErN6+n3e+yWby7PUccY5bzjmV0eecSs0a1edANAWAiMhJ2LK3gIc/WMW7S7bQODGeBy4+nYEd045ruKpfdEUwEZGT0DipJk9d3ZnXb+lFUkIN7nzlG64an8m3W/b5XVqlUQCIiPyM7i1TePfOvvzl0g6s2b6fgU99ye+mLWP3gcLynxziFAAiIuWIjjKG9WjOZ789l+t7pfPq/E2c+9fPmDL7+2p9kRwFgIhIBSUmxPLQJWfwwd1n0aFJIg+9+y2/fPKrantabAWAiMhxatOwDi+O7M6z13Ul/3ARw56bx60vLmTT7ny/SzsuOoeqiMgJMDMuOqMR57RJ5bkv1zHm07V88t0Obj37FG7td2q1OEW1hoGKiFSCrbkFPPLBKqYv3kJaYjwD2qdROy6ahLgYatWIJqFGDLXijrmvEUNCXDS1asQQHxtVaUNMdRyAiIgPFqzfzV9mrmT19jwOFBaVeYGbY5lBQuy/A+MXpzfkwYHtTqiGigZA6K+jiIhUIxnpKbx9Wx8gcGbUg4ePcKCwiPxDxYH7wiIOHCr+0X1+YTEHCovJP1QUuC8sIi2p6i9mowAQEakiZkbNGtGB00jU9ruan9IoIBGRCKUAEBGJUAoAEZEIpQAQEYlQCgARkQilABARiVAKABGRCKUAEBGJUCF9KggzywE2nMSPqA+E8nlaVd/JUX0nR/WdnFCur4VzLrW8TiEdACfLzBZU5HwYflF9J0f1nRzVd3JCvb6K0CYgEZEIpQAQEYlQ4R4A4/0uoByq7+SovpOj+k5OqNdXrrDeByAiImUL9zUAEREpQ7UPADPrb2bfmVmWmd1fyvw4M3vNmz/PzNKDWFszM/vUzL41sxVmdncpffqZWa6ZLfZuvw9WfSVqWG9my7zX/8kl2CzgSW8ZLjWzLkGqq22J5bLYzPaZ2a+P6RP05Wdmk8xsh5ktL9GWYmazzGyNd59cxnOHe33WmNnwINb3f2a2yvv9TTOzpDKe+7OfhSqs7yEzyy7xe7y4jOf+7N97Fdb3Wona1pvZ4jKeW+XLr1I556rtDYgG1gKnADWAJUC7Y/rcBoxP7tZkAAAD8klEQVTzpocCrwWxvjSgizddB1hdSn39gPd8Xo7rgfo/M/9i4APAgJ7APJ9+19sIjG/2dfkBZwNdgOUl2h4D7vem7wceLeV5KcA67z7Zm04OUn0XAjHe9KOl1VeRz0IV1vcQ8NsKfAZ+9u+9quo7Zv7fgN/7tfwq81bd1wC6A1nOuXXOuULgVWDQMX0GAc97028C51tlXXm5HM65rc65Rd70fmAl0CQYr13JBgEvuIBMIMnM0oJcw/nAWufcyRwYWCmcc18Au49pLvk5ex4YXMpTLwJmOed2O+f2ALOA/sGozzn3L+dckfcwE2ha2a9bUWUsv4qoyN/7Sfu5+rzvjiHAK5X9un6o7gHQBNhU4vFmfvoF+0Mf7w8gF6gXlOpK8DY9dQbmlTK7l5ktMbMPzOyMoBYW4IB/mdlCMxtVyvyKLOeqNpSy/+j8Xn4ADZ1zW73pbUDDUvqEwnIEuJHAGl1pyvssVKU7vE1Uk8rYhBYKy+8sYLtzbk0Z8/1cfsetugdAtWBmtYG3gF875/YdM3sRgc0anYCngHeCXR/Q1znXBRgA3G5mZ/tQQ5nMrAZwCfBGKbNDYfn9iAtsCwjJ4XVm9jugCJhaRhe/PgtjgVOBM4GtBDazhKKr+fn//kP6b+lY1T0AsoFmJR439dpK7WNmMUAisCso1QVeM5bAl/9U59zbx853zu1zzuV50zOBWDOrH6z6vNfN9u53ANMIrGqXVJHlXJUGAIucc9uPnREKy8+z/ehmMe9+Ryl9fF2OZnYDMBC4xgupn6jAZ6FKOOe2O+eKnXNHgAllvK7fyy8GuAx4raw+fi2/E1XdA2A+0NrMWnr/JQ4FZhzTZwZwdLTFFcAnZX34K5u3vXAisNI593gZfRod3SdhZt0J/E6CGVC1zKzO0WkCOwuXH9NtBnC9NxqoJ5BbYnNHMJT5X5ffy6+Ekp+z4cD0Uvr8E7jQzJK9TRwXem1Vzsz6A/8JXOKcyy+jT0U+C1VVX8l9SpeW8boV+XuvSr8AVjnnNpc208/ld8L83gt9sjcCI1RWExgd8Duv7Y8EPugA8QQ2HWQBXwOnBLG2vgQ2BSwFFnu3i4FbgVu9PncAKwiMaMgEegd5+Z3ivfYSr46jy7BkjQaM8ZbxMiAjiPXVIvCFnliizdflRyCMtgKHCWyHHklgv9LHwBrgIyDF65sBPFfiuTd6n8UsYEQQ68sisP386Ofw6Mi4xsDMn/ssBKm+F73P1lICX+ppx9bnPf7J33sw6vPapxz93JXoG/TlV5k3HQksIhKhqvsmIBEROUEKABGRCKUAEBGJUAoAEZEIpQAQEYlQCgARkQilABARiVAKABGRCPX/uuFjDXEJdG8AAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"import matplotlib.pyplot as plt\n",
"plt.plot(losses)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment