Boston Pricing Frame work will not save network with dill

[paperstsoap]

I have been playing around with different things with the neupy library, so this network is arbitrary in purpose, however models the overall set up of the real ones. This code was just to get the dill dump/load figured out. The code is

import dill
import csv
import numpy as np
from sklearn import datasets, preprocessing
from sklearn.cross_validation import train_test_split
from neupy import algorithms, layers
from neupy.functions import rmsle

np.random.seed(0)

#variables
EPOCHS = 200
HIDDENLAYER = 17
TRAIN = 0.7
ROUND = 2
STEP = 0.003
TOL = 0.02

with open('binary_conversion_dataset_input_2.csv','r') as dest1_f:

    data_iter = csv.reader(dest1_f, delimiter = ',', quotechar = '"')
    data = [data for data in data_iter]
    data_array1 = np.asarray(data, dtype = float)
    hitmiss_in = data_array1 #loads entire dataset from excel csv file

with open('binary_conversion_dataset_target_2.csv','r') as dest2_f:
    data_iter = csv.reader(dest2_f, delimiter = ',', quotechar = '"')
    data = [data for data in data_iter]
    data_array2 = np.asarray(data, dtype = float)
    hitmiss_target = data_array2 #loads entire dataset from excel csv file
    hitmiss_input = hitmiss_in[:,:]
    hitmiss_target = hitmiss_target[:,:]
    hitmiss_predict = [0.53, 0.80, 0.40, 0.20, 0.07]

#####break target set into singles
hitmiss_target1a = hitmiss_target[:,0]
hitmiss_target1b = hitmiss_target[:,1]
hitmiss_target1c = hitmiss_target[:,2]
hitmiss_target1d = hitmiss_target[:,3]
hitmiss_target1e = hitmiss_target[:,4]

################################################Neural Network for hit miss

x_train, x_test, y_train, y_test = train_test_split(
    hitmiss_input, hitmiss_target1a, train_size=TRAIN
)

cgnet = algorithms.ConjugateGradient(
    connection=[
        layers.TanhLayer(5),
        layers.TanhLayer(HIDDENLAYER),
        layers.OutputLayer(1),
    ],
    search_method='golden',
    tol = TOL,
    step = STEP,
    show_epoch=25,
    optimizations=[algorithms.LinearSearch],
)

cgnet.train(x_train, y_train, x_test, y_test, epochs=EPOCHS)
hitmiss_final_A = cgnet.predict(hitmiss_predict).round(ROUND)

with open('network-storage.dill', 'w') as net:
    dill.dump(cgnet, net)

print(hitmiss_final_A)

The entire traceback errors produced are:

Traceback (most recent call last):

File "C:\Python27\save network script.py", line 74, in <module>

dill.dump(cgnet, net)

File "C:\Python27\lib\site-packages\dill\dill.py", line 115, in dump

pik.dump(obj)

File "C:\Python27\lib\pickle.py", line 224, in dump

self.save(obj)

File "C:\Python27\lib\pickle.py", line 331, in save

self.save_reduce(obj=obj, *rv)

File "C:\Python27\lib\pickle.py", line 401, in save_reduce

save(args)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\pickle.py", line 548, in save_tuple

save(element)

File "C:\Python27\lib\pickle.py", line 331, in save

self.save_reduce(obj=obj, *rv)

File "C:\Python27\lib\pickle.py", line 419, in save_reduce

save(state)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 440, in save_module_dict

StockPickler.save_dict(pickler, obj)

File "C:\Python27\lib\pickle.py", line 649, in save_dict

self._batch_setitems(obj.iteritems())

File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems

save(v)

File "C:\Python27\lib\pickle.py", line 331, in save

self.save_reduce(obj=obj, *rv)

File "C:\Python27\lib\pickle.py", line 419, in save_reduce

save(state)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 440, in save_module_dict

StockPickler.save_dict(pickler, obj)

File "C:\Python27\lib\pickle.py", line 649, in save_dict

self._batch_setitems(obj.iteritems())

File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems

save(v)

File "C:\Python27\lib\pickle.py", line 331, in save

self.save_reduce(obj=obj, *rv)

File "C:\Python27\lib\pickle.py", line 419, in save_reduce

save(state)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 440, in save_module_dict

StockPickler.save_dict(pickler, obj)

File "C:\Python27\lib\pickle.py", line 649, in save_dict

self._batch_setitems(obj.iteritems())

File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems

save(v)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 526, in save_functor

obj.keywords), obj=obj)

File "C:\Python27\lib\pickle.py", line 401, in save_reduce

save(args)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\pickle.py", line 562, in save_tuple

save(element)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 418, in save_function

obj.func_closure), obj=obj)

File "C:\Python27\lib\pickle.py", line 401, in save_reduce

save(args)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\pickle.py", line 562, in save_tuple

save(element)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\pickle.py", line 548, in save_tuple

save(element)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 584, in save_cell

pickler.save_reduce(_create_cell, (obj.cell_contents,), obj=obj)

File "C:\Python27\lib\pickle.py", line 401, in save_reduce

save(args)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\pickle.py", line 548, in save_tuple

save(element)

File "C:\Python27\lib\pickle.py", line 286, in save

f(self, obj) # Call unbound method with explicit self

File "C:\Python27\lib\site-packages\dill\dill.py", line 418, in save_function

obj.func_closure), obj=obj)

File "C:\Python27\lib\pickle.py", line 405, in save_reduce

self.memoize(obj)

File "C:\Python27\lib\pickle.py", line 244, in memoize

assert id(obj) not in self.memo

AssertionError

[itdxer]

What NeuPy version do you use? It looks a bit old, I've already changed some part of the library. The one thing that I think can cause this problem is the file mode. Dill will write a binary string, so you need to specify it in the code. Use 'wb' instead of 'w'

with open('network-storage.dill', 'wb') as net:

It's also important to read the network file as a binary.

with open('network-storage.dill', 'rb') as net:
    cgnet = dill.load(net)

If it's not help, I will suggest upgrading the NeuPy to the version 0.2.1 and try to run this code. I've modified your code a bit to make it possible to run it on my machine.

[itdxer]

@paperstsoap Did you solve your issue?

[paperstsoap]

I upgraded to the latest version of neupy and theano, and now I am getting a error that there is no nose_parameterized module. I attempted to install that module but have not been successful at fixing that issue yet. Are you aware of any Windows bugs with respect to that?

Sent from my iPhone

On May 8, 2016, at 12:53 PM, Yurii Shevchuk notifications@github.com wrote:

@paperstsoap Did you solve your issue?

— You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[itdxer]

That's odd. In Stackoveflow I found solution for the problem that looks similar to your, you can try to set up different protocol - http://stackoverflow.com/questions/4426981/python-pickle-dumps-assertionerror. dill works with python's pickle library, so the parameters are the same

dill.dump(obj, file_, protocol=pickle.HIGHEST_PROTOCOL)

[paperstsoap]

I've been tied up this weekend with Mother's Day but I plan on putting in a solid effort for a the next few days to try to get this figured out. I will definitely keep you updated.

Sent from my iPhone

On May 8, 2016, at 3:43 PM, Yurii Shevchuk notifications@github.com wrote:

That's odd. In Stackoveflow I found solution for the problem that looks similar to your, you can try to set up different protocol - http://stackoverflow.com/questions/4426981/python-pickle-dumps-assertionerror. dill works with python's pickle library, so the parameters are the same

dill.dump(obj, file_, protocol=pickle.HIGHEST_PROTOCOL) — You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[paperstsoap]

I have solved the problem. I updated to the latest version of neupy, which starting giving me the nose_parameterized module not found error. The issue was that the older neupy version (say 0.1.0) was still active in another location. Once I recycled that file and deleted it everything now works correctly. I guess it was pulling from the older version's path even though I installed the new one.

Thank you for all your help and quick response! I have just started messing with ANN and love your setup.

-Dave

Sent from my iPhone

On May 8, 2016, at 3:43 PM, Yurii Shevchuk notifications@github.com wrote:

That's odd. In Stackoveflow I found solution for the problem that looks similar to your, you can try to set up different protocol - http://stackoverflow.com/questions/4426981/python-pickle-dumps-assertionerror. dill works with python's pickle library, so the parameters are the same

dill.dump(obj, file_, protocol=pickle.HIGHEST_PROTOCOL) — You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[itdxer]

Great!

[paperstsoap]

Since the updated version uses Theano, have you ever tried using the neupy module with multiple GPUs?

-Dave

Sent from my iPhone

On May 9, 2016, at 1:43 PM, Yurii Shevchuk notifications@github.com wrote:

Great!

— You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[itdxer]

No, just with a single one.

[paperstsoap]

Do you have any sample code for doing that?

Sent from my iPhone

On May 22, 2016, at 6:02 AM, Yurii Shevchuk notifications@github.com wrote:

No, just with a single one.

— You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[itdxer]

You can try this one https://github.com/itdxer/neupy/blob/master/examples/gd/cifar10_cnn.py But you need to set up proper configurations for Theano

[paperstsoap]

Thank you for passing on the information, I really appreciate it. Would you be able to give a brief description of what it should be doing, so that I know it working properly when I try to run it?

Thank you, Dave

Sent from my iPhone

On May 22, 2016, at 11:14 AM, Yurii Shevchuk notifications@github.com wrote:

You can try this one https://github.com/itdxer/neupy/blob/master/examples/gd/cifar10_cnn.py

— You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub

[itdxer]

Sorry, I didn't get it. Did you mean "What you need to do to make it possible to run NeuPy with multiple GPUs?"

[paperstsoap]

Sorry if I was unclear. I asked if you had any sample code for using the neupy library with a gpu. You said that you did and sent me a link to a neupy sample code that use a gpu. I was wondering if I could get a brief description of what that sample code does so that I could understand what it does so when I try to use it, I know it is working correctly. The sample code provided looks like it does something with images, and uses the relu and softmax hidden layers, all of which I have limited knowledge on. I hope this make more sense, but if it does not I would be happy to clarify any confusion.

Thank you,

Dave

Sent from my iPhone

On May 29, 2016, at 7:42 AM, Yurii Shevchuk notifications@github.com wrote:

Sorry, I didn't get it. Do you mean "What you need to do to make it possible to run NeuPy with multiple GPUs?"

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[itdxer]

To make it possible to run any Theano based code on GPU you need to have preinstalled all GPU drivers and dependencies. If you have everything that you need you will be able to run code on GPU. The one important thing that you needed to add is a Theano flag that defines your GPU. It can be something like this:

$ THEANO_FLAGS=device=gpu python examples/gd/cifar10_cnn.py

The example contains convolutional neural network that trains on the CIFAR10 dataset. To test the GPU you don't need to understand network's structure inside of the script.

On GPU epoch learning time should be faster. This script works on my laptop 13 minutes per epoch, and on AWS GPU it took ~1.5 minutes per epoch or even less.

[paperstsoap]

Thank you again, this is exactly the information I was unsure about. I have a GeForce GT 550M with Cuda built into my laptop that I want to get my feet wet with. I appreciate you taking the time to answer my questions.

Thank you,

Dave

Sent from my iPhone

On May 30, 2016, at 4:49 AM, Yurii Shevchuk notifications@github.com wrote:

To make it possible to run any Theano based code on GPU you need to have preinstalled all GPU drivers and dependencies. If you have everything that you need you will be able to run code on GPU. The one important thing that you needed to add is a Theano flag that defines you GPU. It can be something like this:

$ THEANO_FLAGS=device=gpu python examples/gd/cifar10_cnn.py The example contains convolutional neural network that trains on the CIFAR10 dataset. To test the GPU you don't need to understand network's structure inside of the script.

On GPU epoch learning time should be faster. This script works on my laptop 13 minutes per epoch, and on AWS GPU it took ~1.5 minutes per epoch or even less.

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[paperstsoap]

I have been using neupy over the past 6 months for different projects with great success. My current project requires whole integers for the application. Do you have any frameworks or recommendations for this type of problem? I have a golden search network right now that gets really close, but just not precise enough for the whole integer needed.

Thank you,

Dave

Sent from my iPhone

On May 30, 2016, at 11:34 AM, David Edelen david.edelen3@me.com wrote:

Thank you again, this is exactly the information I was unsure about. I have a GeForce GT 550M with Cuda built into my laptop that I want to get my feet wet with. I appreciate you taking the time to answer my questions.

Thank you,

Dave

Sent from my iPhone

On May 30, 2016, at 4:49 AM, Yurii Shevchuk notifications@github.com wrote:

To make it possible to run any Theano based code on GPU you need to have preinstalled all GPU drivers and dependencies. If you have everything that you need you will be able to run code on GPU. The one important thing that you needed to add is a Theano flag that defines you GPU. It can be something like this:

$ THEANO_FLAGS=device=gpu python examples/gd/cifar10_cnn.py The example contains convolutional neural network that trains on the CIFAR10 dataset. To test the GPU you don't need to understand network's structure inside of the script.

On GPU epoch learning time should be faster. This script works on my laptop 13 minutes per epoch, and on AWS GPU it took ~1.5 minutes per epoch or even less.

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[itdxer]

Hi @paperstsoap, Do you need to have an integer number as an output from your network?

[paperstsoap]

Yes, that's correct.

Sent from my iPhone

On Jul 26, 2016, at 4:59 AM, Yurii Shevchuk notifications@github.com wrote:

Hi @paperstsoap, Do you need to have an integer number as an output from your network?

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[itdxer]

I think that's fine that you have float numbers as an output. Network should give you result which should be very close to integer that you expect to see. You can simply round your value to the closest integer.

[paperstsoap]

In real world testing I am getting say 59 when I need 60 due to the rounding. It is training to 0.0009 RMS and is often correct, but more often it is slightly 1 higher or lower.

Sent from my iPhone

On Jul 26, 2016, at 7:56 AM, Yurii Shevchuk notifications@github.com wrote:

I think that's fine that you have float numbers as an output. Network should give you close result to to integer that you expect to see. You can just simply round your value to the closest integer.

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[itdxer]

That's a normal case when you use regression. In addition you have very small RMS which means network has been trained successfully. You need to explore your results and understand the main source of problems from your data.

[itdxer]

I think it also can be useful to check the residuals between raw output from the network (without rounding) and target values. Based on it you can try to figure out how to make a better way to transform raw output from the network. Of course, it's possible only in case if your residuals aren't normally distributed with mean located near the 0.

[paperstsoap]

I'll explore that further. I'm currently using a sigmoid function, is there another function that might be better? I've read that the relu is popular for whole integer outputs, but have not used it yet.

Sent from my iPhone

On Jul 26, 2016, at 8:37 AM, Yurii Shevchuk notifications@github.com wrote:

I think it also can be useful to check the residuals between raw output from the network (without rounding) and target values. Based on it you can try to figure out how to make a better way to transform raw output from the network. Of course, it's possible only in case if your residuals aren't normally distributed with mean located near the 0.

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.

[itdxer]

One of the main advantages of ReLu over the Sigmoid function is that it's less likely to vanish the gradient (vanishing means that values in gradient become very close to 0). Vanishing makes training slower. I'm not sure that ReLu will help you to fix this issue.

[paperstsoap]

It seems that there is a heavy bias towards 0 in the dataset, so I'm not sure that anything can be done about that.

Sent from my iPhone

On Jul 26, 2016, at 8:55 AM, Yurii Shevchuk notifications@github.com wrote:

One of the main advantages of ReLu over the Sigmoid function is that it's less likely to vanish the gradient (vanishing means that values in gradient become very close to 0). Vanishing makes training slower. I'm not sure that ReLu will help you to fix this issue.

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub, or mute the thread.