tomonari-masada

tomonari-masada

tomonari-masada

tomonari-masada

from pyknp import Juman
import torch
from pytorch_transformers import *

config = BertConfig.from_json_file('Japanese_L-12_H-768_A-12_E-30_BPE/bert_config.json')
model = BertForMaskedLM.from_pretrained('Japanese_L-12_H-768_A-12_E-30_BPE/pytorch_model.bin',
                                        config=config)
tokenizer = BertTokenizer('Japanese_L-12_H-768_A-12_E-30_BPE/vocab.txt',
                          do_lower_case=False, do_basic_tokenize=False)

tomonari-masada

tomonari-masada

# -*- coding: utf-8 -*-
from lxml import etree
import os
import sys
from io import TextIOWrapper
from nltk.tokenize import RegexpTokenizer

#
# USAGE:
#

tomonari-masada

import sys
import torch
import torch.nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

tomonari-masada

import io, sys, math, random
import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn
from torch.autograd import Variable
from torch import optim

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

tomonari-masada

import torch
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d

torch.manual_seed(102)
np.random.seed(22)

fig = plt.figure()

tomonari-masada

import sys
import io
import MeCab
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
m = MeCab.Tagger()
fp = open('a1.txt', encoding='utf8')
wseq = list()
for line in fp:
    for ol in m.parse(line.strip()).split('\n'):
        if len(ol.split()) > 1: