Open schultz77 opened 6 months ago
I am trying to load the mnist data without tensorflow but i am have save errors with reshape:
def load_mnist():
base_url = 'http://yann.lecun.com/exdb/mnist/'
files = ['train-images-idx3-ubyte.gz', 'train-labels-idx1-ubyte.gz',
't10k-images-idx3-ubyte.gz', 't10k-labels-idx1-ubyte.gz']
data = {}
# Download and load training and testing data
for file in files:
url = base_url + file
filename = os.path.join('data', file)
if not os.path.exists('data'):
os.makedirs('data')
if not os.path.exists(filename):
print('Downloading', file, '...')
r = requests.get(url)
with open(filename, 'wb') as f:
f.write(r.content)
with gzip.open(filename, 'rb') as f:
if file.endswith('images-idx3-ubyte.gz'):
# data[file] = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 28, 28)
data[file] = np.frombuffer(f.read(), np.uint8, offset=16)
elif file.endswith('labels-idx1-ubyte.gz'):
data[file] = np.frombuffer(f.read(), np.uint8, offset=8)
return data
# Load MNIST dataset
mnist = load_mnist()
# Extract training data and labels
x_train, y_train = mnist['train-images-idx3-ubyte.gz'], mnist['train-labels-idx1-ubyte.gz']
x_test, y_test = mnist['t10k-images-idx3-ubyte.gz'], mnist['t10k-labels-idx1-ubyte.gz']
Do you have any hint? Thx!
It runs using this approach. However, I am not 100% sure:
import numpy as np
# from keras.datasets import mnist
# from keras.utils import to_categorical
from sklearn.datasets import fetch_openml
from dense import Dense
from convolutional import Convolutional
from reshape import Reshape
from activations import Sigmoid
from losses import binary_cross_entropy, binary_cross_entropy_prime
from network import train, predict
def to_one_hot(y_value):
num_classes = np.max(y_value) + 1
one_hot_matrix = np.eye(num_classes)[y_value.reshape(-1)]
return one_hot_matrix.reshape(list(y_value.shape) + [num_classes])
def preprocess_data(x_value, y_value, limit):
zero_indices = np.where(y_value.values == 0)[0][:limit]
one_indices = np.where(y_value.values == 1)[0][:limit]
all_indices = np.hstack((zero_indices, one_indices))
all_indices = np.random.permutation(all_indices)
x_value = x_value.iloc[all_indices].values.astype("float32") / 255
x_value = x_value.reshape(len(x_value), 1, 28, 28)
y_value = y_value.iloc[all_indices].values
y_value = to_one_hot(y_value)
y_value = y_value.reshape(len(y_value), 2, 1)
return x_value, y_value
# Load MNIST data from OpenML
mnist = fetch_openml('mnist_784')
# Separate features and labels
x_data, y_data = mnist['data'], mnist['target']
# Convert string labels to integers
y_data = y_data.astype(int)
# Preprocess the data
x_train, y_train = preprocess_data(x_data[:60000], y_data[:60000], 100)
x_test, y_test = preprocess_data(x_data[60000:], y_data[60000:], 100)
# Print shapes of training and testing data and labels
print("Training data shape:", x_train.shape)
print("Training labels shape:", y_train.shape)
print("Testing data shape:", x_test.shape)
print("Testing labels shape:", y_test.shape)
Hi, Amazing tutorial again. Unfortunately i am facing some problems with the code
my system:
in order to get free of errors i had to modify the code: from:
from keras.utils import np_utils
to:
from keras.utils import to_categorical
After running it i've got following error:
After installing tensorflow and running the script:
Could you please support me with this? i am really looking forward to debug the script to get deeper understanding of it.
Thank You in advance!