Snakefile : combined Snakefile for both Process txt and Process apsimx is ignoring the latter which is the second rule

[DininduSenanayake]

We can get two separate Snakefiles to execute the two main rules on their own without any issues but the combined one is ignoring the second rule rule process_apsimx_files:

import os
from glob import glob

config = {
    "apptainer_bind": "/agr/scratch,/agr/persist",
    "apptainer_image": "/agr/persist/projects/2024_apsim_improvements/apsim-simulations/container/apsim-2024.09.7579.0.aimg",
    "excluded_txt_files": ["ExampleConfig.txt"],
    "excluded_apsimx_files": ["2023-10-09_MasterSoilApsimLibrary.apsimx", "LargerExample.apsimx"],
    "max_consecutive_failures": 10,
    "slurm_logdir": "slurmlogs"
}

# Gather input files
txt_files = [f for f in glob("*.txt") if f not in config["excluded_txt_files"]]
apsimx_files = [f for f in glob("*.apsimx") if f not in config["excluded_apsimx_files"]]

# Define all rule
rule all:
    input:
        config["slurm_logdir"],
        "txt_files_processed",
        expand("{file}.processed", file=[os.path.splitext(f)[0] for f in apsimx_files])

# Rule to create the log directory
rule create_logdir:
    output:
        directory(config["slurm_logdir"])
    shell:
        "mkdir -p {output}"

# Rule to process txt files
rule process_txt_files:
    input:
        txt_files = txt_files
    output:
        "txt_files_processed"
    params:
        logfile = lambda wildcards, output: f"{config['slurm_logdir']}/txt_files_%j.out"
    resources:
        mem_mb = 8000,
        time = "16:00:00"
    threads: 4
    shell:
        """
        #SBATCH --output={params.logfile}

        module load Apptainer
        export APPTAINER_BIND="{config[apptainer_bind]}"
        export APPTAINER_CMD="apptainer exec {config[apptainer_image]}"

        mkdir -p FAILED
        consecutive_failures=0
        max_consecutive_failures={config[max_consecutive_failures]}

        for file in {input.txt_files}; do
            if $APPTAINER_CMD Models --cpu-count {threads} --apply "$file"; then
                echo "Successfully processed $file"
                consecutive_failures=0
            else
                echo "Failed to process $file"
                mv "$file" FAILED/
                ((consecutive_failures++))

                if [ $consecutive_failures -ge $max_consecutive_failures ]; then
                    echo "Error: $max_consecutive_failures consecutive failures reached. Terminating job." >&2
                    exit 1
                fi
            fi
        done

        touch {output}
        """

# Rule to process apsimx files
rule process_apsimx_files:
    input:
        apsimx = "{file}.apsimx",
        db = "{file}.db"
    output:
        "{file}.processed"
    params:
        logfile = lambda wildcards, output: f"{config['slurm_logdir']}/{wildcards.file}_%j.out"
    resources:
        mem_mb = 8000,
        time = "00:10:00"
    threads: 12
    shell:
        """
        #SBATCH --output={params.logfile}

        module load Apptainer
        export APPTAINER_BIND="{config[apptainer_bind]}"
        apptainer exec {config[apptainer_image]} Models {input.apsimx}
        touch {output}
        """