search

abdurrahman1828 / Team_SIOL

MIT License
0 stars 0 forks source link

Evaluation Update #1

Open sushant097 opened 1 day ago

sushant097 commented 1 day ago

We got following issue while running your project for MSU AI club Campus VIsion Challenge:

Custom code for evaluation: 

import os
import sys

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import tensorflow as tf
from sklearn.metrics import (
    accuracy_score,
    classification_report,
    confusion_matrix,
    f1_score,
    log_loss,
    precision_score,
    recall_score,
)
from tensorflow.keras.applications import Xception
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input
from tensorflow.keras.models import Model
from tensorflow.keras.preprocessing.image import ImageDataGenerator

class Logger:
    """Log print statements to a file and the console."""

    def __init__(self, log_file):
        self.console = sys.stdout
        self.file = open(log_file, "w")

    def write(self, message):
        self.console.write(message)
        self.file.write(message)

    def flush(self):
        self.console.flush()
        self.file.flush()

def evaluate_model(test_dir, weights_path, output_dir):
    # Create output directory
    os.makedirs(output_dir, exist_ok=True)

    # Set up logging to file
    log_file = os.path.join(output_dir, "analysis.txt")  # Log in the output directory
    sys.stdout = Logger(log_file)  # Redirect standard output to log file and console

    print(f"Logging to {log_file}")

    print(f"Test directory exists: {os.path.exists(test_dir)}")
    print(f"Weights file exists: {os.path.exists(weights_path)}")

    # Load the model with weights
    input_shape = (224, 224, 3)
    inputs = Input(shape=input_shape)
    data_augmentation = tf.keras.Sequential(
        [tf.keras.layers.RandomRotation(0.1), tf.keras.layers.RandomFlip("vertical")]
    )
    augmented_inputs = data_augmentation(inputs)
    base_model = Xception(
        weights="imagenet", include_top=False, input_tensor=augmented_inputs
    )
    base_model.trainable = True
    x = GlobalAveragePooling2D()(base_model.output)
    outputs = Dense(10, activation="softmax")(x)
    model = Model(inputs=inputs, outputs=outputs)
    model.compile(
        optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
        loss="sparse_categorical_crossentropy",
        metrics=["accuracy"],
    )

    model.load_weights(weights_path)
    print(f"Model loaded from {weights_path}")

    # Set up ImageDataGenerator for the test dataset
    test_datagen = ImageDataGenerator()
    test_generator = test_datagen.flow_from_directory(
        test_dir,
        target_size=(224, 224),
        batch_size=16,
        class_mode="categorical",
        shuffle=False,
    )

    print(f"Test dataset loaded from {test_dir}")

    # True labels and predictions
    y_true = test_generator.classes
    y_true_one_hot = tf.keras.utils.to_categorical(
        y_true, num_classes=len(test_generator.class_indices)
    )
    y_pred_probs = model.predict(test_generator)
    y_pred = np.argmax(y_pred_probs, axis=1)

    # Class names
    class_names = list(test_generator.class_indices.keys())
    print(f"Classes: {class_names}")

    # Calculate performance metrics
    accuracy = accuracy_score(y_true, y_pred)
    precision = precision_score(y_true, y_pred, average="weighted")
    recall = recall_score(y_true, y_pred, average="weighted")
    f1 = f1_score(y_true, y_pred, average="weighted")
    logloss = log_loss(y_true_one_hot, y_pred_probs)

    print(f"Accuracy: {accuracy:.4f}")
    print(f"Precision: {precision:.4f}")
    print(f"Recall: {recall:.4f}")
    print(f"F1-Score: {f1:.4f}")
    print(f"Log Loss: {logloss:.4f}")

    # Save metrics to a file
    metrics_file = os.path.join(output_dir, "metrics.txt")
    with open(metrics_file, "w") as f:
        f.write(f"Accuracy: {accuracy:.4f}\n")
        f.write(f"Precision: {precision:.4f}\n")
        f.write(f"Recall: {recall:.4f}\n")
        f.write(f"F1-Score: {f1:.4f}\n")
        f.write(f"Log Loss: {logloss:.4f}\n")

    print(f"Metrics saved in {metrics_file}")

    # Confusion Matrix
    cm = confusion_matrix(y_true, y_pred)
    plt.figure(figsize=(10, 8))
    sns.heatmap(
        cm,
        annot=True,
        fmt="d",
        xticklabels=class_names,
        yticklabels=class_names,
        cmap="Blues",
    )
    plt.title("Confusion Matrix")
    plt.xlabel("Predicted")
    plt.ylabel("True")
    confusion_matrix_file = os.path.join(output_dir, "confusion_matrix.png")
    plt.savefig(confusion_matrix_file)
    plt.close()

    print(f"Confusion matrix saved in {confusion_matrix_file}")

    # Save classification report
    report = classification_report(y_true, y_pred, target_names=class_names)
    classification_report_file = os.path.join(output_dir, "classification_report.txt")
    with open(classification_report_file, "w") as f:
        f.write(report)

    print(f"Classification report saved in {classification_report_file}")
    print(f"Evaluation complete. Results saved in {output_dir}")

if __name__ == "__main__":
    # Define paths for the single model and dataset
    test_dir = "../private-dataset/"  # Path to the private dataset
    weights_path = "model/best_model.weights.h5"  # Path to the model weights
    output_dir = "evaluation_results"  # Directory to save the evaluation results

    evaluate_model(test_dir, weights_path, output_dir)

The complete project structure looks like this:

├── evaluation_results/          # Output folder for evaluation results
│   ├── analysis.txt             # Logs and detailed evaluation output
│   
├── images/                      # Folder for storing auxiliary images or test files
├── model/                       # Pre-trained model weights and related files
│   ├── best_model.weights.h5    # Pre-trained model for evaluation
│   ├── dummy.txt                # Placeholder or dummy file
├── venv/                        # Python virtual environment
├── ../private_dataset/          # Private dataset with building images
│   ├── Butler Hall/             # Subfolder for Butler Hall images
│   ├── Carpenter Hall/          # Subfolder for Carpenter Hall images
│   ├── Lee Hall/                # Subfolder for Lee Hall images
│   ├── McCain Hall/             # Subfolder for McCain Hall images
│   ├── McCool Hall/             # Subfolder for McCool Hall images
│   ├── Old Main/                # Subfolder for Old Main images
│   ├── Simrall Hall/            # Subfolder for Simrall Hall images
│   ├── Student Union/           # Subfolder for Student Union images
│   ├── Swalm Hall/              # Subfolder for Swalm Hall images
│   ├── Walker Hall/             # Subfolder for Walker Hall images
├── evaluate_model.py            # Script for evaluating the model on test datasets
├── evaluate-private-testset.py  # Additional evaluation script for private datasets
├── LICENSE                      # License file
├── nas.py                       # Script related to neural architecture search (if applicable)
├── README.md                    # Readme file (current document)
├── requirements.txt             # Python dependencies for the project
├── source-analysis.txt          # Source code analysis or related notes
├── train.py                     # Script for training the model

Output we got:

python evaluate-private-testset.py 
2024-11-20 11:57:46.944248: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-11-20 11:57:46.944608: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.
2024-11-20 11:57:46.946313: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.
2024-11-20 11:57:46.951346: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1732125466.959952 3378840 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1732125466.962375 3378840 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2024-11-20 11:57:46.970892: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
Logging to evaluation_results/analysis.txt
Test directory exists: True
Weights file exists: True
W0000 00:00:1732125467.646311 3378840 gpu_device.cc:2344] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
/home/sg/AI-Club/Evaluation-Challenge/Evaluated-Submission/Team_SIOL/venv/lib/python3.10/site-packages/keras/src/saving/saving_lib.py:719: UserWarning: Skipping variable loading for optimizer 'adam', because it has 2 variables whereas the saved optimizer has 0 variables. 
  saveable.load_own_variables(weights_store.get(inner_path))
Traceback (most recent call last):
  File "/home/sg/AI-Club/Evaluation-Challenge/Evaluated-Submission/Team_SIOL/evaluate-private-testset.py", line 159, in <module>
    evaluate_model(test_dir, weights_path, output_dir)
  File "/home/sg/AI-Club/Evaluation-Challenge/Evaluated-Submission/Team_SIOL/evaluate-private-testset.py", line 72, in evaluate_model
    model.load_weights(weights_path)
  File "/home/sg/AI-Club/Evaluation-Challenge/Evaluated-Submission/Team_SIOL/venv/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py", line 122, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "/home/sg/AI-Club/Evaluation-Challenge/Evaluated-Submission/Team_SIOL/venv/lib/python3.10/site-packages/keras/src/saving/saving_lib.py", line 593, in _raise_loading_failure
    raise ValueError(msg)
ValueError: A total of 81 objects could not be loaded. Example error message for object <Conv2D name=block1_conv1, built=True>:

Layer 'block1_conv1' expected 1 variables, but received 0 variables during loading. Expected: ['kernel']

List of objects that could not be loaded:
[<Conv2D name=block1_conv1, built=True>, <BatchNormalization name=block1_conv1_bn, built=True>, <Conv2D name=block1_conv2, built=True>, <BatchNormalization name=block1_conv2_bn, built=True>, <SeparableConv2D name=block2_sepconv1, built=True>, <BatchNormalization name=block2_sepconv1_bn, built=True>, <SeparableConv2D name=block2_sepconv2, built=True>, <BatchNormalization name=block2_sepconv2_bn, built=True>, <Conv2D name=conv2d, built=True>, <BatchNormalization name=batch_normalization, built=True>, <SeparableConv2D name=block3_sepconv1, built=True>, <BatchNormalization name=block3_sepconv1_bn, built=True>, <SeparableConv2D name=block3_sepconv2, built=True>, <BatchNormalization name=block3_sepconv2_bn, built=True>, <Conv2D name=conv2d_1, built=True>, <BatchNormalization name=batch_normalization_1, built=True>, <SeparableConv2D name=block4_sepconv1, built=True>, <BatchNormalization name=block4_sepconv1_bn, built=True>, <SeparableConv2D name=block4_sepconv2, built=True>, <BatchNormalization name=block4_sepconv2_bn, built=True>, <Conv2D name=conv2d_2, built=True>, <BatchNormalization name=batch_normalization_2, built=True>, <SeparableConv2D name=block5_sepconv1, built=True>, <BatchNormalization name=block5_sepconv1_bn, built=True>, <SeparableConv2D name=block5_sepconv2, built=True>, <BatchNormalization name=block5_sepconv2_bn, built=True>, <SeparableConv2D name=block5_sepconv3, built=True>, <BatchNormalization name=block5_sepconv3_bn, built=True>, <SeparableConv2D name=block6_sepconv1, built=True>, <BatchNormalization name=block6_sepconv1_bn, built=True>, <SeparableConv2D name=block6_sepconv2, built=True>, <BatchNormalization name=block6_sepconv2_bn, built=True>, <SeparableConv2D name=block6_sepconv3, built=True>, <BatchNormalization name=block6_sepconv3_bn, built=True>, <SeparableConv2D name=block7_sepconv1, built=True>, <BatchNormalization name=block7_sepconv1_bn, built=True>, <SeparableConv2D name=block7_sepconv2, built=True>, <BatchNormalization name=block7_sepconv2_bn, built=True>, <SeparableConv2D name=block7_sepconv3, built=True>, <BatchNormalization name=block7_sepconv3_bn, built=True>, <SeparableConv2D name=block8_sepconv1, built=True>, <BatchNormalization name=block8_sepconv1_bn, built=True>, <SeparableConv2D name=block8_sepconv2, built=True>, <BatchNormalization name=block8_sepconv2_bn, built=True>, <SeparableConv2D name=block8_sepconv3, built=True>, <BatchNormalization name=block8_sepconv3_bn, built=True>, <SeparableConv2D name=block9_sepconv1, built=True>, <BatchNormalization name=block9_sepconv1_bn, built=True>, <SeparableConv2D name=block9_sepconv2, built=True>, <BatchNormalization name=block9_sepconv2_bn, built=True>, <SeparableConv2D name=block9_sepconv3, built=True>, <BatchNormalization name=block9_sepconv3_bn, built=True>, <SeparableConv2D name=block10_sepconv1, built=True>, <BatchNormalization name=block10_sepconv1_bn, built=True>, <SeparableConv2D name=block10_sepconv2, built=True>, <BatchNormalization name=block10_sepconv2_bn, built=True>, <SeparableConv2D name=block10_sepconv3, built=True>, <BatchNormalization name=block10_sepconv3_bn, built=True>, <SeparableConv2D name=block11_sepconv1, built=True>, <BatchNormalization name=block11_sepconv1_bn, built=True>, <SeparableConv2D name=block11_sepconv2, built=True>, <BatchNormalization name=block11_sepconv2_bn, built=True>, <SeparableConv2D name=block11_sepconv3, built=True>, <BatchNormalization name=block11_sepconv3_bn, built=True>, <SeparableConv2D name=block12_sepconv1, built=True>, <BatchNormalization name=block12_sepconv1_bn, built=True>, <SeparableConv2D name=block12_sepconv2, built=True>, <BatchNormalization name=block12_sepconv2_bn, built=True>, <SeparableConv2D name=block12_sepconv3, built=True>, <BatchNormalization name=block12_sepconv3_bn, built=True>, <SeparableConv2D name=block13_sepconv1, built=True>, <BatchNormalization name=block13_sepconv1_bn, built=True>, <SeparableConv2D name=block13_sepconv2, built=True>, <BatchNormalization name=block13_sepconv2_bn, built=True>, <Conv2D name=conv2d_3, built=True>, <BatchNormalization name=batch_normalization_3, built=True>, <SeparableConv2D name=block14_sepconv1, built=True>, <BatchNormalization name=block14_sepconv1_bn, built=True>, <SeparableConv2D name=block14_sepconv2, built=True>, <BatchNormalization name=block14_sepconv2_bn, built=True>, <Dense name=dense, built=True>]

We also tried this:

python evaluate_model.py --test_dir ../private-dataset/ --weights_path model/best_model.weights.h5

But got same error as above.

Let us know ASAP how to resolve this error so we can consider your project for evaluation.

FYI: We tried several times to resolve this error and couldn't. We think it is an error in your model architecture or implementation.

sushant097 commented 1 day ago

@abdurrahman1828 see the above issue.

abdurrahman1828 commented 1 day ago

Thank you for reaching out. We have updated the requirements.txt file with the exact package version. We have also updated the installation process in the readme file. With this, we are able to run the code in our windows system with GPU support. However, we are not super familiar with generating requirements files and virtual environment creation with pip. Therefore, we showed how we installed it using conda. Please let us know if you face any issues. We are very excited to be one of the participants in this interesting competition and willing to do anything to go forward.

Here is a screenshot of the evaluation code implemented. Thank you again.

Screenshot 2024-11-20 at 5 05 41 PM