RNN bug report

[ws-choi]

I found bugs in your RNN code (d4b49dcc1728d71ba3e82a754c7d0aefc4ebcbc3)(link) .

Below is the modified code:

import numpy as np
import tensorflow as tf
from tqdm.notebook import tqdm

path_to_file = tf.keras.utils.get_file('shakespeare.txt',
                                       'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')
text = open(path_to_file, 'rb').read().decode(encoding='utf-8')

vocab = sorted(set(text))
print('num of chars: {}\n'.format(len(vocab)))
c2i = {u: i for i, u in enumerate(vocab)}
i2c = {i: u for u, i in c2i.items()}

t2i = [c2i[i] for i in text]
ntext = np.array(t2i)

class RNN:

    def __init__(self, x_size, y_size, h_size):

        self.x_size = x_size
        self.y_size = y_size
        self.h_size = h_size

        # Xavier Initialization
        self.Whh = np.random.normal(size=(h_size, h_size), scale=(np.sqrt(2 / (h_size + y_size))))
        self.Wxh = np.random.normal(size=(x_size, h_size), scale=(np.sqrt(2 / (x_size + y_size))))
        self.bh = np.zeros((1, h_size))
        self.Why = np.random.normal(size=(h_size, y_size), scale=(np.sqrt(2 / (h_size + y_size))))
        self.by = np.zeros((1, y_size))
        self.h_0 = None

        self.mWhh = np.zeros_like(self.Whh)
        self.mWxh = np.zeros_like(self.Wxh)
        self.mbh = np.zeros_like(self.bh)
        self.mWhy = np.zeros_like(self.Why)
        self.mby = np.zeros_like(self.by)

    def f(self, x_size, y_size, h_size, Whh, Wxh, bh, Why, by, h_0):

        self.x_size = x_size
        self.y_size = y_size
        self.h_size = h_size
        self.Whh = Whh
        self.Wxh = Wxh
        self.bh = bh
        self.Why = Why
        self.by = by
        self.h_0 = h_0

    def forward(self, X, h_0=None):

        N = X.shape[0]
        T = X.shape[1]

        X_one_hot_encoding = np.zeros((N * T, self.x_size))
        X_one_hot_encoding[np.arange(X_one_hot_encoding.shape[0]), X.reshape(-1)] = 1
        X_one_hot_encoding = X_one_hot_encoding.reshape(N, T, -1)

        X_t = np.swapaxes(X_one_hot_encoding, 0, 1)

        if (h_0 is None):
            h_0 = np.zeros((N, self.h_size))
        y_cache = None
        h_cache = np.array([h_0])  # [t, N, h_size]

        for x_t in X_t:

            h_t_bef = h_cache[-1]
            h_t_raw = h_t_bef.dot(self.Whh) + x_t.dot(self.Wxh) + self.bh  # (N, h)
            h_t = np.tanh(h_t_raw)
            y_t = h_t.dot(self.Why) + self.by  # [N, y_size]

            if y_cache is None:
                y_cache = np.array([y_t])
            else:
                y_cache = np.append(y_cache, [y_t], axis=0)
            h_cache = np.append(h_cache, [h_t], axis=0)

        return X_t, y_cache, h_cache

    def backward_t(self, x_t, h_t, h_bef, dy_t, dh_prev):  # [N, y_size] 글자 단위 backward

        # x_t : [N, x_size]
        # h_t : [N, h_size]
        # h_bef : [N, h_size]
        # y_t : [N, y_size]
        # dy_t : [N, y_size]
        # dh_t : [N, h_size]

        dWhy = h_t.T.dot(dy_t)
        dby = dy_t.sum(axis=0)
        dh_t = dy_t.dot(self.Why.T) + dh_prev
        dh_t_raw = dh_t * (1 - h_t ** 2)  # N h_size

        #   dWhh = dh_t.sum(axis=0) * h_bef.T.dot(1 - h_t ** 2)
        dWhh = h_bef.T.dot(dh_t_raw)
        dWxh = x_t.T.dot(dh_t_raw)  # dh_t.sum(axis=0) * x_t.T.dot(1 - h_t ** 2)
        dh_prev = dh_t_raw.dot(self.Whh.T)
        dbh = dh_t_raw.sum(axis=0)

        return dWhh, dWxh, dbh, dWhy, dby, dh_prev

    def backward(self, X_t, dys, h_cache):

        dWhh = np.zeros_like(self.Whh)
        dWxh = np.zeros_like(self.Wxh)
        dbh = np.zeros_like(self.bh)
        dWhy = np.zeros_like(self.Why)
        dby = np.zeros_like(self.by)
        dh_t = np.zeros((h_cache.shape[1], h_cache.shape[2]))

        for t in reversed(range(X_t.shape[0])):
            x_t = X_t[t]
            h_prev = h_cache[t]
            h_t = h_cache[t + 1]
            dy_t = dys[t]
            dWhh_t, dWxh_t, dbh_t, dWhy_t, dby_t, dh_t = self.backward_t(x_t, h_t, h_prev, dy_t, dh_t)

            # Update
            dWhh += dWhh_t
            dWxh += dWxh_t
            dbh += dbh_t
            dWhy += dWhy_t
            dby += dby_t

        dh_0 = dh_t

        return dWhh, dWxh, dbh, dWhy, dby, dh_0

    def gra_update(self, dWhh, dWxh, dbh, dWhy, dby, dh_0):

        alpha = 0.1
        np.clip(dWhh, -5, 5, out=dWhh)
        np.clip(dWxh, -5, 5, out=dWxh)
        np.clip(dbh, -5, 5, out=dbh)
        np.clip(dWhy, -5, 5, out=dWhy)
        np.clip(dby, -5, 5, out=dby)

        self.mWhh += dWhh * dWhh
        self.mWxh += dWxh * dWxh
        self.mbh += dbh * dbh
        self.mWhy += dWhy * dWhy
        self.mby += dby * dby

        self.Whh -= dWhh * alpha / np.sqrt(self.mWhh + 1e-8)
        self.Wxh -= dWxh * alpha / np.sqrt(self.mWxh + 1e-8)
        self.bh -= dbh * alpha / np.sqrt(self.mbh + 1e-8)
        self.Why -= dWhy * alpha / np.sqrt(self.mWhy + 1e-8)
        self.by -= dby * alpha / np.sqrt(self.mby + 1e-8)

class SOFTMAX:

    def __init__(self):
        pass

    def loss(self, x, y):  # from cs231n assignment

        #         probs = np.exp(x - np.max(x, axis=1, keepdims=True))
        #         probs /= np.sum(probs, axis=1, keepdims=True)

        probs = np.exp(x - np.max(x, axis=-1, keepdims=True))
        probs /= (np.sum(probs, axis=-1, keepdims=True))

        N = x.shape[0]
        loss = - np.sum(np.log(probs[np.arange(N), y] + 1e-8))
        dx = probs.copy()
        dx[np.arange(N), y] -= 1

        return loss, dx, probs

def gen(rnn, length, start_idx=c2i['F'], temperature=1):
    vector = np.array(start_idx).reshape(-1, 1)
    gen_chars = [i2c[start_idx]]
    h = np.zeros((1, rnn.h_size))
    for _ in range(length):
        _, y, h = rnn.forward(vector, h)
        y = y.reshape(-1, 65)
        probs = np.exp(y - np.max(y, axis=-1, keepdims=True))
        probs /= np.sum(probs, axis=-1, keepdims=True)
        probs /= temperature
        next_char = np.random.choice(range(65), p=probs.reshape(-1))
        vector = np.array(next_char).reshape(-1, 1)
        gen_chars.append(i2c[next_char])
        h = h[-1]

    return ''.join(gen_chars)

rnn = RNN(65, 65, 128)
softmax = SOFTMAX()
history_loss = []

batch_size = 16

for j in tqdm(range(100)):
    seq_length = np.random.randint(23, 27)
    rand_idx = np.random.randint(0, seq_length)
    sequences = ntext[rand_idx:]

    num_remaining_chars = len(sequences) % (seq_length)
    if (num_remaining_chars == 0):
        sequences = sequences[:-seq_length + 1]
    elif (num_remaining_chars != 1):
        sequences = sequences[:-num_remaining_chars + 1]

    X = sequences[:-1].reshape((-1, seq_length))
    Y = sequences[1:].reshape((-1, seq_length))

    idx_permute = np.random.permutation(range(X.shape[0]))
    remaining_idx = X.shape[0] % batch_size
    if (remaining_idx != 0):
        idx_permute = idx_permute[:-remaining_idx]
    X_batch = X[idx_permute].reshape(-1, batch_size, seq_length)
    Y_batch = Y[idx_permute].reshape(-1, batch_size, seq_length)

    total_batch_size = X_batch.shape[0]

    n_iter = 1
    for x, y in zip(X_batch, Y_batch):
        loss_local = 0
        X_t, y_cache, h_cache = rnn.forward(x)  # t, N, y

        time = y_cache.shape[0]
        N = y_cache.shape[1]

        y_cache = y_cache.reshape(time * N, -1)
        Real_Y = y.T.reshape(time * N)

        loss, dy, _ = softmax.loss(y_cache, Real_Y)
        loss_local += (loss / N / total_batch_size)

        dy /= N
        dy = dy.reshape(time, N, -1)

        dWhh, dWxh, dbh, dWhy, dby, dh_0 = rnn.backward(X_t, dy, h_cache)
        rnn.gra_update(dWhh, dWxh, dbh, dWhy, dby, dh_0)

    history_loss.append(loss_local)

    print('epoch ', j)
    print('loss,', loss_local)
    print('-----------------------------------')
    print(gen(rnn, 200, start_idx=np.random.randint(0, 65)))
    print('-----------------------------------')
    print()

[ws-choi]

Here is the result.

epoch  0
loss, 0.01921584613748624
-----------------------------------
jome er.
No ket davt
Trer
pou, nod; ming:
Whar a baden the gos?

DUCK:
Watherus be'rid.

poren wullthanth don cakn,
praaF thas uing wad, ar pboke the mesturs:
Got moden.

PIPI ERICHREK:
O no llinot I b
-----------------------------------

epoch  1
loss, 0.019042419596027302
-----------------------------------
D campino I lay deivy not could of he-ryer, is he satrucal kest: blay of gredpese to that?

NARDY BIUS:
And thet?

PINI:
And, if the in in shall ot forsur jinarizlighvlesoed.

BAPENTE:
By my your bitto
-----------------------------------

epoch  2
loss, 0.017668501837973666
-----------------------------------
rly, taretlmedf neacting that he.

BENRY;

DUKE I:
En reste sin'
Solivern wartoo llad the entind sour not him tork,
Wim is mey, Hour wout of'n force. Waus dinde, I cos he shirdhald in truning't, with o
-----------------------------------

epoch  3
loss, 0.013757226085292003
-----------------------------------
ZALUS:
I cast thes? MAmees in pare of there a, livisence ferpear; Eddent, Bfarry, nere fitlen, you ille
And, sce sindrysth-renten thae in so ferching, itsuen to worsd prence
Thiblline!
But spus not cor
-----------------------------------

epoch  4
loss, 0.016695911626456235
-----------------------------------
x'switillant they lient, the as hollon.

BENENI:
And seerounce but heandent to eart to thim:
But tere,
I nomes;
Then us comanaishattary upsule me chace;
Yoirt
Trafr flathided.
That you so
the unsuring 
-----------------------------------

epoch  5
loss, 0.015967543717854093
-----------------------------------
ch jeat your try farnergnom in there her to his clealuf hame you my hing oce re be meat's tywer there
Sonase shame.

Sirvisss, lare and this nighald my faught?
Hey geser, weld commed and pery the tir w
-----------------------------------

epoch  6
loss, 0.018135019477124693
-----------------------------------
PEDIUS:
Farkusborm? hurmasp my wife timen,
For grar roadest to mathering them speek sbees! this apod take of him no' seartmongay branginer?

CAmI I conceen, what you do to yeleher ses is beare: me slai
-----------------------------------

epoch  7
loss, 0.018071578405397404
-----------------------------------
Spo enpafforasy know hight: so me. Them chadrowidlem'fne,
De'T expee brese.

IOMIILA:
The crunty,
I will fearn; so rim'd,
Ny hath 'sthelth;
Ard:
I fram: she they braust of lark ale of to le!

CORIOLANU
-----------------------------------

epoch  8
loss, 0.014546173613350666
-----------------------------------
: gows.

BIOMANUS:
Why, resty so, goonsy an werrbercinal will aress
the prove's niesly Jusuall, theind a que her enouly, be,
He gook; what Note from I canfone.
3 KING RICHARD III:
They Oo af no bling,

-----------------------------------

epoch  9
loss, 0.018457524264570833
-----------------------------------
QUEE:
On Barron cos.
thy,
Maste.

Nom HErccoon'd mocl,
Thee heme me we a to's giublese lisen, thou thy soret
sell youcharel?

il his dound you come Haren, ever shers onit,
And stry in as you a sabut te
-----------------------------------

epoch  10
loss, 0.01632008670282211
-----------------------------------
$thes?
Why worduld to have that to be
Weir spick.

Floop Bophy,
Mukh repolive:
And lodd to godence be hearn witicen, nother:
Agoof
So hears be Clivech thy kere Mardentry not and dirmurthady
Wast,
To's 
-----------------------------------

epoch  11
loss, 0.014407341741297583
-----------------------------------
e fromand kuel letine, do,
What he's! worto:
And boge!
But.
Sisirily: gomely speive forsence,
A loves to mef heaven; as the king te w you he: we fill swor: of do me peties the king the Elay?

PROMPEE:

-----------------------------------

epoch  12
loss, 0.016039938969807307
-----------------------------------
Som
Thie grese!
Who enssore, God miruth, close; thinking-buckaring
He do stoov'd assain
So have you that anser.

KINGHanged quel eviest neen he lave'd, laad
Uusin.

NORIZABEONZO:
Cerems,
He say awfuld 
-----------------------------------

epoch  13
loss, 0.017668353392244516
-----------------------------------
y tongs a chorelinioud this hearter with leker you.
Peton,
that my ence mang
Too shall to you wight
You kno thame wholl with shalkous?
At awes.

PETRUCHIO:
Your you so kill if you his fasted, That him;
-----------------------------------

epoch  14
loss, 0.014177031779476743
-----------------------------------
Hy, some me fram.
What they childs.
No,
How.
Dege.
If the hing their food once fort, whemit for poasdry to for mastateporueme! I bring the reperin? breed thung that frile-brow betworp and I trount her 
-----------------------------------

epoch  15
loss, 0.016651100426488805
-----------------------------------
XENCS:
A matul theresty him: Anwlat!
Nim:
This bed to precold seallct and pleash
Hencame,
Our has,
He gous, not no in that an he mand
may him his datherdy:
Sight read not
That a gaid would that tuthug 
-----------------------------------

epoch  16
loss, 0.015664027648405212
-----------------------------------
'
They dadgusa so should my burth
whent.
Low!
Senvine,
Whou due of this
And be poith that witlo yo burs
To the not
I, and to have earding I cuints gives shrears,
'Bus to be then thee the engrout the se
-----------------------------------

epoch  17
loss, 0.015409354387851463
-----------------------------------
Piore Kadisrenigoof
the digly, the embous that felled you dame your bartispaild ble!
My fonly with him fails king your lord haprece, that your grores, ye'dly,
And an Edwallo's eaveraly I erpive and 'th
-----------------------------------

epoch  18
loss, 0.015279224108540363
-----------------------------------
XENES:
O mastaupo, regentien brive.

MARCANDBHFI:
By dein
Why merces' nots, a vile;
Not,
Do thou mestent, and of it. Mis rad, uny slamenished, it!
For chays revontio grour, I am it contword.

SICINTHAR
-----------------------------------

epoch  19
loss, 0.012815253150553384
-----------------------------------
ANT:
I saft our souse to may sast alling propermon for, the hand over
Ten the her lest I worle, theresen:
Sus, her, To lord, as some swall well it te fook,
Wet be peazeds,
That?

MENENIUS:
Nusse
And fo
-----------------------------------

epoch  20
loss, 0.0151939644505258
-----------------------------------
ese the begivin'n, thaugh to seal wordss come tizet:
Well beak, but panned and be heaven crow known Cancery'd corswer'd to vinion comel, the men she dig file,
Be I am to corst do word shous, I comes ti
-----------------------------------

epoch  21
loss, 0.015105948697491342
-----------------------------------
d;
That holy woll Leavouegh makes,
To do batvech my like dreas unding?

EXETCUS:
Ifer, thyse lover's betw harreess hanter will
Richmaind your houltertay you thing and give; with birge misio?
Am, speade
-----------------------------------

epoch  22
loss, 0.014002184555741596
-----------------------------------

LECBRICHARD SINCRAT:
It, terp, as
The que seat, how.

LEONTES:
Now as manty to our to me
kees! a countress go;
Witurer-dim, me he visto?
What owe to is him!:
Whe fater an brame sill king;,
ficgmen.
Th
-----------------------------------

epoch  23
loss, 0.015432589742422557
-----------------------------------
y infore me on your child bester-mordid,
To now yet to that an fuit, fumbeyspumant threent
I he think pitiet havinae to
Yon urricter the chall the stide of Barnt to are, and the with the exerdford; and
-----------------------------------

epoch  24
loss, 0.015182035281391703
-----------------------------------
3 KING RICHARD II:
The that the to we will us it blood-then I meat if dew basips yous not some reponting the sale edpingarn that hin this lork'd he are, and is my lortiongs.

QUEEN MARGANUS:
No, sinks 
-----------------------------------

epoch  25
loss, 0.018332642114296405
-----------------------------------
A RINARESTER:
Fare land, dead, or one, ae her your francioud, to me sle it a hew freaking:
All is to likingind to my Lortle sir!

CUMINGHAM:
I noth old me sorest may well make hatifn he to deas;
At thi
-----------------------------------

epoch  26
loss, 0.013947908746127504
-----------------------------------
or, suver
The have spoad me nofels:
But be itmon rear,
Whoursss fied love such their mines, wellar's have hand-not few;
Horn will preveruent, The mave petalle.

LEONTES:
From comfore olfo.
What me noth
-----------------------------------

epoch  27
loss, 0.016647571716707808
-----------------------------------
RIAHNON:
That our good hath praking
shaw than rry then,
gent now,
Mindent,
Sirsel, stay, scalls, sech urwle me from I canother the will, shallancold.

3 KING RICHARD III:
Nusher' blond yow,
Tonger, sie
-----------------------------------

epoch  28
loss, 0.01308562514131977
-----------------------------------
zan us atkers and Click.

Clothence the rust my noust, Godfais, unoken offeld at him: that them away, owrlerino,
What, stame. midenty
chist it gekern, to it, on marise, are are nongs drake didcuiess th
-----------------------------------

epoch  29
loss, 0.01761109857978905
-----------------------------------
&s has you ley hell thry Good of dome wheir say my hat kneces me powhy as thou kerriouloly mintiavion? Kioncel's chulge, what coull you sooved who too loso one brought,
Wike of my live
'tis our his; ar
-----------------------------------

epoch  30
loss, 0.014667658038298833
-----------------------------------
zen:
Who mands syege the requerckings say besombord.

KING EDWARD IV:
No, stull, hyer a jeysed, you ervene lare stilm if a

Parit.

PHAF ENIUS:
I bleed;
An these dit.

MENCULONDE XINES:
Let some, ables
-----------------------------------

epoch  31
loss, 0.017806876781916815
-----------------------------------
!
I caussess enton the o brongente, no.

QUEEN VINCENTIO:
Heaven neld dother hartublens, it.
Him, is dearerince.

MANCHARDABULANIE:
Yet hence me, be such is me in they, go thim protep, whiler.

SLrest,
-----------------------------------

epoch  32
loss, 0.012768161331309185
-----------------------------------
Mirow thought dear who doule your
To his heath.

QUEEN ELIZABETH:
This fries
Then be were duzst;
What wite lork.
So my lorreccain the bood from he I
And wear culase what you not yousent and to his lim.
-----------------------------------

epoch  33
loss, 0.014005295163274287
-----------------------------------
-fors of my eake
Tere then shall we mermain the,
Wartly!
He wo, tought thy grizen I
Cafface:
Donn bus stramb,
Do eoued we man
To faithth my evid tow.

POMPEY:
First and
Them her merings offick'd sevnit
-----------------------------------

epoch  34
loss, 0.016409211651590445
-----------------------------------
 wordandingby turn a graves put phoick that a things; themtorn,
As is Perubrest your tow this is amy rettair then not sbale your tood Edratert;
To offeruck best?

LUCIO:
Here his somes;
Evad and my, th
-----------------------------------

epoch  35
loss, 0.01398746686908082
-----------------------------------
QUER:
He thing Rues.
Ih, favewh, lovesss Dales; any hand tho woa to me is doubsan our facheres,
entood in like thou to sryieg of your
'tin
sail godio, are it is tyary
To meam. Buse,
Make vireal crevere
-----------------------------------

epoch  36
loss, 0.013982965044731305
-----------------------------------
US:
The duth extries'd of amond, surving
But- feftens take is.

ROMEO:
I think king, mad peaceme
And in to nejus the tare' goads hath the
To hearting,
As
Sin, Marrdan: doltice, sungers no good to that 
-----------------------------------

epoch  37
loss, 0.01343482890111303
-----------------------------------
QUERSABET:
Not he of thum will buckied is,
Eng one I and what, lottle?
To well King duntlaight; to aint nair, have strive cate!
Hanking peaces nows
O,
And leart and of im,
But well, and did.

GREENS: I
-----------------------------------

epoch  38
loss, 0.015362612847360803
-----------------------------------
?

CORIOLANUE:
Now tir, and live I makes on more
Plandy to may to isslaugh destrokn, arr-deverent downems! queling 'Wit at my have so that hang there he was is in at to brearn, grefert a here for I'll 
-----------------------------------

epoch  39
loss, 0.017810801409488743
-----------------------------------
a stolte.

VIANA:
A plinves and Jotlew same it shream, you and we, you. Het 'enefoll, yet to these more thy with's, dome?

SICINIUS:
Suvily belks-sharrges
Who courisfertare conforry gall you.
No, he so
-----------------------------------

epoch  40
loss, 0.013510743301497678
-----------------------------------
Like?

QUEEN:
You piar no becummast,
And me no frach me'll bleed butch Capterount.
But,
Ahenged, O'er offliof away-wirnielld to seef be leare! are but the lit this that previes,
Her hu meecessly this t
-----------------------------------

epoch  41
loss, 0.014433261933855954
-----------------------------------
$l whither halfors a
tell the dimet?
Harcacred for vure repenes ords that lee batter.

EliLCO:
I'll faver of servent be of bade pleasinte and and nouse fapting is coulsten cur he have trless.

WORDIAR 
-----------------------------------

epoch  42
loss, 0.01721210989521412
-----------------------------------
 vainlest to must it your mans,
And wiling, take Such in ginlurivering dead Seit sain by rick, with have blood:

LIO:
Sin, gence canswer of so put pet ere too, and fair ree you's here! Are a land in
yo
-----------------------------------

epoch  43
loss, 0.017635134990064485
-----------------------------------
Geed such frate hear the 'tose the follets.

Lord's: I perlas.
'Twould you harthing while. Rom-wery spall mind of elloufter.
By mage,
Deftor's fewen not thir a mire lits for and such heart and you EdwA
-----------------------------------

epoch  44
loss, 0.013898585670538897
-----------------------------------
Taif and prothish'd, childs the
For our brisbre I woo'Nth this, love.

GLOUCESTER:
Which this thou an havew she nare baughtloine; him usbous no'll will more mad tyece to most, to be the such on my tran
-----------------------------------

epoch  45
loss, 0.015492233727366573
-----------------------------------
d-asprome is adecerant. Lesain Hartort ney the eake, Lort to the ming sicks hast have must of sunder.

MARCLAND:
Ke hake a should it jutnes his ruck,
And methind ere wo sook'd gead, you;
Be Belay,
As a
-----------------------------------

epoch  46
loss, 0.016571385423866637
-----------------------------------
charr and of ney, that sword enty: thee!
And and was.

Secon of abibit, sin
thineasund: muctsed no chome bith you would but that vout!
Goop than his own Angeleence no not inself his hancly itsullencion
-----------------------------------

epoch  47
loss, 0.013637985132485127
-----------------------------------
VaRS:
Hancose, too
sweet. And to latter good tois dead out upsare thus him?

AUCISTER:
Plottorys tell your noull,
As a daagut woy, and-main, their cousin putierdy I buring first off their bloone.

ESCA
-----------------------------------

epoch  48
loss, 0.013817418637111728
-----------------------------------
Heviette fair no, she lall of that sigs,
Sir: an thasty it sake his seous. hinh mortist paucery's Lord I lawat'd you.
To doight brand of this love;
Come?

LAONTES:
Well, I lotter beat'st swead's of bot
-----------------------------------

epoch  49
loss, 0.016453634298036972
-----------------------------------
ing I hangideman press will tell'd it
Whey you his wide.
Ay, volamon no my name hose
He have noth'd on:
Thou.

JULUE:
My fault be our like al no thy Burlinged riventurew the preverent my son!

Nurse:
W
-----------------------------------

epoch  50
loss, 0.014285259758464553
-----------------------------------
aaver! I reaks
Which not Sun.

Prence me of your wan mining.

JULIET:
'To tay with yok wilting.

PARTISABELLA:
Good till fiet a mettam be peffare, to-tayany.

GRUMIO:
Coment made here,
Thank my even th
-----------------------------------

epoch  51
loss, 0.01591311835697412
-----------------------------------
fores, at abidy my lime a guses mear thinks not derewids in dost torts not he have cidils, in your some sigr
And in Mirdledsent by and him shainl pard, at the rose he ar'd,
And the fim a think my chere
-----------------------------------

epoch  52
loss, 0.017391327377521176
-----------------------------------
!
I'lume untomone, my fare.

LEONTES:
My minough beginouse, and be mest right:
That ctace and will me, Groush thou are the coont now shant. If good, joy.
-vale asait and lere, much a know the kingand p
-----------------------------------

epoch  53
loss, 0.012357832380656
-----------------------------------
he As hath sinal?

LUCEDLEY:
Pard,
Oud Byold sons muther no a liking a master where who upon; yok at face, you he dagrorslioness you the dim.

ONIO:
Beevsman thy ere.

CLAUDTIO:
EMy vain it ondendiness
-----------------------------------

epoch  54
loss, 0.014099514373366241
-----------------------------------
d the boncuch your douck be into yout-I mattay, fild in thunk
Wrant if seam the lives your merel, friet sexforts;
What, wheregh,
Must my upe the exto her upewhn my seal-an tos here lordsared his amped 
-----------------------------------

epoch  55
loss, 0.01317172828648742
-----------------------------------
XENE:
A sook befolvist disselvety there to houss.

PROSFHONDAHO:
Nhy hang carnd, good that you of the like:
And are Dadgs, him and a dalf penst nil!

KING EDIUS:
Which of die and us Pray Hasty
Is feol 
-----------------------------------

epoch  56
loss, 0.01405197000060912
-----------------------------------
VINCENTIO:
No, glantuep!
Cher shesster'ce muck tits; if crivour messeous firs.

MERCUSTERS OF O:
Will what for, menn me dafer to belay a vaind, you, I woily price this our mupficks ere did a man the he
-----------------------------------

epoch  57
loss, 0.014523805687009287
-----------------------------------
jure, marry fatheming laints, and this take
Loos fint: make thou mustles.

DUKE VINCENTIO:
But the befule keeving?

GLOUS:
Illow, and soop a cribon; I ewngsh.

GLOUCENTIRYONEEN:
O has I alld come, fath
-----------------------------------

epoch  58
loss, 0.014636390645371623
-----------------------------------
Frazs.

Firseep.

KING EDWARD IV:
That ever have my juplant;, you her?

KING VINCA:
But?

PETRUCHIO:
Se, firs chather.

CLARENCE:
So, dais, I at before darp'd
But thee there the danger Kathass? the not
-----------------------------------

epoch  59
loss, 0.017044569146129555
-----------------------------------
ENTIO:
A wall wuntress'st of Porest all
Bepont?

CORIOLINGSAM:
But if
Thou newked?
Yourte, yourt. I to be men's Mortinghoris:
Cou,
Betrowry some
Earty bestrued,
My sway you.
Pricen provineal no very th
-----------------------------------

epoch  60
loss, 0.01779650509342322
-----------------------------------
-me him:
Gid, To mept might that are cerryace of all me bety eimiove that's my would deaden to dealdinged he yet my dering to thy resings
And in Harried notheats comed have,
ABid thou! I'll thee ulloin
-----------------------------------

epoch  61
loss, 0.014615831687838688
-----------------------------------
BRAT:
Word, insemped ricomen an amiane string inder in that for seevend, the sarest! I upon
Whid,
And am turluest, of men thie
as very but newent the lord ofer my man:
For Sorhain'd madityed.

IONDERLI
-----------------------------------

epoch  62
loss, 0.013803729571539379
-----------------------------------

Madaste dubbe.

ROMEO:
More that lad adowers out both he goong leger on fullmits.
What your lord pespurmor'try your the mastlond,
Are deatise:
Mruke oraly at wesperems it, not with bionch,
Westlay one
-----------------------------------

epoch  63
loss, 0.014346376604687489
-----------------------------------
ely posthers't I be to Pose to as you amions
Tell throuds devine some you meanting, yeirs
Brounting praybue cast you you
countranger if king, to my gone, Duke my ill.

GLOUCESSO:
The ween suck in shall
-----------------------------------

epoch  64
loss, 0.01787635558440882
-----------------------------------
NASTER:
As sore, though that my put king rerp'd unclaightly.

Ther it'd;
'Tye there is pather blood.

LEONTES:
But viltany me of whencinquge betruke King wanting broice, I wilty son griouk;
Wells gover
-----------------------------------

epoch  65
loss, 0.01263840752951177
-----------------------------------
evous son. Engle that hold let brees Edward myse shill to my mait the prepager country.

ESRELIUS:
With tolsedy and he mattof thistron by you laught.

MELRONCH:
Four.

LADY GHARGA:
Father of now ains t
-----------------------------------

epoch  66
loss, 0.012799725665296338
-----------------------------------
quen Clauth we'er mest theer pray'd, vaing conquouer;
Theill toul, to getse is dinker him; but the creep'd, he you wide you ap batishrey's death netieglate,
And he'll make the one, unies no breather my
-----------------------------------

epoch  67
loss, 0.015302045040730019
-----------------------------------
Wind
We have some his panemiling,
Ainadonce, shipe!

GLOUTESNEO:
You; I am your love, you dreanay,
And is felm'd
betten? Mawhr
the Edwaint could give angertistes, connay, what harfuly knevans!

LAMERLO
-----------------------------------

epoch  68
loss, 0.017101904315154584
-----------------------------------
quare;
Thou are is renorion! Yet one tile and fathering the hasd my refors me tert.

HUCKINGHAM:
Why, mick mast home, the tisuce take.

DUKE TING RICANEN:
Smade when stargh if but yal;

Passonguce home
-----------------------------------

epoch  69
loss, 0.01420577187248998
-----------------------------------
Herced in offeccray,
Wheremy
Cack!
Now a pain. He will go.

WARKENV: Gongiture vourbor,
In more not to come
Do the bune will soreor died a darker,
She him traine.

CORIOLANUS:
Is?

Citize: Prosing sunt
-----------------------------------

epoch  70
loss, 0.013212117804181448
-----------------------------------
&C:
Undon's freare,
And tell Slee wrotien such of heave myself though you hade bring hat. Yourself whrothingets!
Away, a beignce
Ofel sout as mayss, for him thought what in threaght of.

BUCKINGHAM:
An
-----------------------------------

epoch  71
loss, 0.014238735918811292
-----------------------------------
RIOLANUS:
I, thy look'd munnearn I hasty o'el liel' susber,
Ungenter town's lottent stay.

if this igit I worlder'ty hear the nothed, shall so:
Bound gomems, no youse mo haster, and now'd
He usted him 
-----------------------------------

epoch  72
loss, 0.0157289045930163
-----------------------------------
But their locknedlemst will lim?
My stay me harst from fiet of your hopen-bun ad but garden it chemise I do to
In your o'ertingse.

MENENCY:
She
your wearst threes cannat, Marcius ripe,
How a vorsury b
-----------------------------------

epoch  73
loss, 0.013213794670164057
-----------------------------------
-shing my placr; with my drem in live a chaste ag, to made, where Than her to best witcher, conshe, I happed his belou tench stay it respain'd kinceaster:

PROSPERO:
Be comes, thore wild,
Romeo, and ha
-----------------------------------

epoch  74
loss, 0.012951564131766012
-----------------------------------
UTUS:
He's to befirtue that he pirsice.

Serve, if under loeking such agost speaks?

CLARET:
I think to the nabpe of the bid process to bod our she were so me hughte more makes of it outs
Which mine---
-----------------------------------

epoch  75
loss, 0.013603030909881311
-----------------------------------
! an I past upon thy swikin: but had refore of this sidst of Ambid,
Fately
pops and ip disperves Ander, thee artings your drealther this your
Fapef
The to ity suck more.

JOHI ID NIWARD:
But dodien and
-----------------------------------

epoch  76
loss, 0.01601578534684941
-----------------------------------

The light, by that pleasting Simpens!
Where I like,
His morenow, and boid. God to then, and being kno:
TRonds hath near Olazman?

LUCIO:
Me! Bespeest am make; and of good be I would:
No! the hand.

ST
-----------------------------------

epoch  77
loss, 0.01514440962593699
-----------------------------------
, all a to herp's.

PRIS:
There livent reventy Cloward our le, no, Iffice,
Where I with thy ngisent.

LORD EMSLOUS:
Com
Tyaint for shife
Will of acond!

QUEEN MICANIE:
'Tis of move sis.

PROSPEROS:
Thi
-----------------------------------

epoch  78
loss, 0.014615008710538741
-----------------------------------
.

KENRICHIO:
By aress tutss, lody Light hibms if bed me ground.

EXETEREY:
What there, I was and hapt,
Whil, and unjons that witrous neads griet'er a so my, a sonk tors! Do weed timer:
Ifleed;
O, maje
-----------------------------------

epoch  79
loss, 0.015310309441937031
-----------------------------------
ENIUSTES:
Hope, are you hadgent cleforto blood.

NORTHUMINY:
Thou and ressed not with
Werefio, evere his ropels thus; men out o'r you,
And the clust man'd ferce a migh. loved abifi-uck,
Andisk hope-pre
-----------------------------------

epoch  80
loss, 0.01188980426188289
-----------------------------------
h thishacasius was buck a foot, stand from up hend,
ords, done, ase's bemble on the wroudsely thee me distred.

JULIET:
That is our a teft out; a the wrarest, they the shiltarmon,
Threese to and hinder
-----------------------------------

epoch  81
loss, 0.012656838112665764
-----------------------------------
ch came we to hearingbrother,
Commanity you samany,
Mad--sur.

JULIET:
Ale henrout, and pravisemst,
And ack. Bunknia sit;
The heacd the nan?
Yet?
Henl.

GRUMIO:
And hawhy: place,
Whom your wit.

CORIOL
-----------------------------------

epoch  82
loss, 0.01555930829508677
-----------------------------------
TERSIAN:
My sigh what I knower he mods.

KING EDWARD IV:
That shalt with reporneval of trou that!
Which of dame that cammarr'd in dend she, flathtentrow,
but longed
With,
As thee hath to a drage how!
I
-----------------------------------

epoch  83
loss, 0.016854219004728235
-----------------------------------
ENENIUS:
Cores roke think of
And you sue hass way the dey your stulks so came me loving by.

LEONTES:
I wasts off.

GREM:
I Loid the to-dimen me my pirst.'
Canste: in none sin, foa his erery liir mysel
-----------------------------------

epoch  84
loss, 0.01428684621850305
-----------------------------------
?

SHbRINWA:
At yoil felcemped, in gloughs be eyes comsens, and he offenty lady hasts of led
While belongs maing:
Wilter Julacalf
Arrow contrate; furguces Kan but epon have be say pall, and judalloo of
-----------------------------------

epoch  85
loss, 0.016209288751415488
-----------------------------------
frait hearth?

AULOWARD II:
I printion your withing husbrangers; by the indmarou that was comtarath
Cornown-is I have be things be be the deat,
See court: would to the good must was how lity me your ha
-----------------------------------

epoch  86
loss, 0.016399589333791553
-----------------------------------
V:
Your serving hono
grems me hen worth, that that what sear with, in proy exco to exchat that shace, what him, for sig;
Dall. For we many deith
 hantive ablet in your one ladberul?
Who and thou wniot 
-----------------------------------

epoch  87
loss, 0.014516627715359302
-----------------------------------
Patnet Qull digh'd that shoild another,
And such as rency demanes my lordsciness, I his; him if Bone's but me,
Thit? I am yeald.

LGOLY:
Yel whom his sorrats caricialce the new none, when the rost then
-----------------------------------

epoch  88
loss, 0.017134327059964875
-----------------------------------
fe slils hudless of it wrauth blother his goon see, not in our worsh eet Ryack, nemiver'd the king:
Nor kind me? therefort Hole,
cous tithen hasterast thou is the torts It swear acan
Awemmer: as patclo
-----------------------------------

epoch  89
loss, 0.014291163031607096
-----------------------------------
O EDWAR WARK:
And I dusing, but hend,
His enechespe, yet I have and could,
and
it, his child but he and in hope sen wete,
Thou to cruple he light Hereisher,
Have,
Madesing eyes a queems, my ctakemand a
-----------------------------------

epoch  90
loss, 0.01523273529828272
-----------------------------------
STA:
How and for with, humbling: Polds this, save call my costlem's noble is your brince;
Lets it wond again then, lotthele to us, withing ip?

HASTINGS:
I wort,
And to the graguide never in throk'd de
-----------------------------------

epoch  91
loss, 0.015400018046773913
-----------------------------------
Zfligh Moreow Richars
to mine marry,
And thy tongling welse of my hold him an hand would o'ets to me, lum'll get richish dons the one proce,
Thou prayfeacty, who schoof ation, very quise, than accakes,
-----------------------------------

epoch  92
loss, 0.016336863703784506
-----------------------------------
Stand you so our farted farry being, bet,
Hath in oul thil nom surrones,
In thine grand
Than that I worl seviccring the sins stolet
Our toth oblem you!
If see?

OXFORDAMELLOMPARDARET:
Let it to but I d
-----------------------------------

epoch  93
loss, 0.012876757020035759
-----------------------------------
r I our, it master my nimttaps; be hort of you think mown;
My driben game all the novio; amy wor had from leave,
Of say im;
And with dull threived, othul my life?
And of you? werder,
And you, will cain
-----------------------------------

epoch  94
loss, 0.016033186463294793
-----------------------------------
ES:
Hath,
Which a wrather narifvourtlets,
Thear len,
Will bepone
Nave of modeat cursent,
Lord up ferth us Cancotlought meughtle since to she shall with trive hu do nieh! I'll Dud whows, they fith heare
-----------------------------------

epoch  95
loss, 0.013389967860384728
-----------------------------------
jeston loving: child!
Yet a hears good the gree me the gaint take mere,
And prought
Too natsenchald, I we'er noble thank fashasooptares; whon our thing that dear'd honours dome veref;
Come
And, house b
-----------------------------------

epoch  96
loss, 0.017421386959792527
-----------------------------------
By cheee,' coremt:
Conto astle, say,
I werefulion.

SEBADO IUS:
We begiener:
Well have load prope'st chatted the chalil.

KING WINIXENE:
Trioum.

CLAUDLO:
Gover Sumble all still be me hath ploy the eve
-----------------------------------

epoch  97
loss, 0.015480358873981179
-----------------------------------
ING RICHARD II:
I may wellow dotisbaughter how crease his with me govers,
And mine, gone:
Strufte and have

MARITHONIO:
Now, it the vistrainds of the as is resenculls
Frome, but youst I! I wather nor a
-----------------------------------

epoch  98
loss, 0.012361026020631258
-----------------------------------
A pather my littly loth her may Mapped of yett sir, quece; him come;
Who a sue it conswards your baby give you mesper-day's haptay'd busiping?
My spoped she coused.

BAPTISAN:
Wellive,
Ag not a have un
-----------------------------------

epoch  99
loss, 0.016888725359171908
-----------------------------------
cons; or have and the earct: sOnong.
Who faight such which, of widy
Warmare wice me?

KATHARINA:
He sir.

PAPCIONDERU:
And Seevens?
Furwickblen's: blike ploy by thus eleds, your guinle, of my know not 
-----------------------------------

[ws-choi]

finxed: https://github.com/Intelligence-Engineering-LAB-KU/Seminar/blob/master/summer_2020/intern1/Vanilla_RNN.ipynb