Helper package with multiple U-Net implementations in Keras as well as useful utility tools helpful when working with image semantic segmentation tasks. This library and underlying tools come from multiple projects I performed working on semantic segmentation tasks
MIT License
339
stars
108
forks
source link
logits and labels must have the same first dimension #54
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import string
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
# Download NLTK data
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
# Load data
with open('data.txt', 'r', encoding='utf-8') as f:
raw_data = f.read()
# Preprocess data
def preprocess(data):
# Tokenize data
tokens = nltk.word_tokenize(data)
# Lowercase all words
tokens = [word.lower() for word in tokens]
# Remove stopwords and punctuation
stop_words = set(stopwords.words('english'))
tokens = [word for word in tokens if word not in stop_words and word not in string.punctuation]
# Lemmatize words
lemmatizer = WordNetLemmatizer()
tokens = [lemmatizer.lemmatize(word) for word in tokens]
return tokens
# Preprocess data
processed_data = [preprocess(qa) for qa in raw_data.split('\n')]
# Set parameters
vocab_size = len(processed_data)
embedding_dim = 64
max_length = 5
trunc_type='pre'
padding_type='pre'
oov_tok = "<OOV>"
training_size = len(processed_data)
# Create tokenizer
tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)
tokenizer.fit_on_texts(processed_data)
word_index = tokenizer.word_index
# Create sequences
sequences = tokenizer.texts_to_sequences(processed_data)
padded_sequences = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)
# Create training data
training_data = padded_sequences[:training_size]
training_labels = padded_sequences[:training_size]
# Build model
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv1D(64, 5, activation='relu'),
tf.keras.layers.MaxPooling1D(pool_size=2, strides=2, padding='same'),
tf.keras.layers.LSTM(64),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(vocab_size, activation='softmax')
])
# Compile model
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# Train model
num_epochs = 50
history = model.fit(training_data, training_labels, epochs=num_epochs, verbose=2)
# Define function to predict answer
def predict_answer(model, tokenizer, question):
# Preprocess question
question = preprocess(question)
# Convert question to sequence
sequence = tokenizer.texts_to_sequences([question])
# Pad sequence
padded_sequence = pad_sequences(sequence, maxlen=max_length, padding=padding_type, truncating=trunc_type)
# Predict answer
pred = model.predict(padded_sequence)[0]
# Get index of highest probability
idx = np.argmax(pred)
# Get answer
answer = tokenizer.index_word[idx]
return answer
# Start chatbot
while True:
question = input('You: ')
answer = predict_answer(model, tokenizer, question)
print('Chatbot:', answer)
I get the error
logits and labels must have the same first dimension, got logits shape [32,50] and labels shape [160]
Logits first dimension is equal to training data, but labels first dimension is the product of training data and max_length. When i change max_length to 1, i get this error;
ValueError: One of the dimensions in the output is <= 0 due to downsampling in conv1d. Consider increasing the input size. Received input shape [None, 1, 64] which would produce output shape with a zero or negative value in a dimension.
when i change it to 0, i get this;
Negative dimension size caused by subtracting 5 from 0 for '{{node sequential/conv1d/Conv1D}} = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], explicit_paddings=[], padding="VALID", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true](sequential/conv1d/Conv1D/ExpandDims, sequential/conv1d/Conv1D/ExpandDims_1)' with input shapes: [?,1,0,64], [1,5,64,64].
Using the following code;
I get the error
Logits first dimension is equal to training data, but labels first dimension is the product of training data and max_length. When i change max_length to 1, i get this error;
when i change it to 0, i get this;
Is this an easy fix?
I'm using tensorflow 2.11.0 and keras 2.11.0