Deletion Glitch that Autocorrects?

[EmbeddedDevops1]

In order to test out Cover Agent in "intelligent mode" (see #124) I've created a higher level script with some additional instructions.

Here's my setup:

• Cloned the PR Agent repository
• Installed all necessary pip packages
• Create an empty file called tests/unittest/test_single_file.py

I now run this script at the root of the repo:

import os
import sys
import subprocess
from cover_agent.CoverageProcessor import CoverageProcessor  # Assuming CoverageProcessor is in cover_agent

# Add the parent directory to the sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

from cover_agent.CoverAgent import CoverAgent

# Path to the initial coverage report
INITIAL_COVERAGE_REPORT_PATH = "coverage.xml"
# Path to the single test file to be used for all source files
TEST_FILE_PATH = "tests/unittest/test_single_file.py"
SOURCE_DIRECTORY = "pr_agent"

class Args:
    def __init__(self, source_file_path, test_file_path):
        self.source_file_path = source_file_path
        self.test_file_path = test_file_path
        self.test_file_output_path = ""
        self.code_coverage_report_path = "coverage.xml"
        self.test_command = f"coverage run -m pytest --cov={SOURCE_DIRECTORY} --cov-report=xml --cov-report=term --log-cli-level=INFO --timeout=30"
        self.test_command_dir = os.getcwd()
        self.included_files = None
        self.coverage_type = "cobertura"
        self.report_filepath = "test_results.html"
        self.desired_coverage = 70
        self.max_iterations = 3
        self.additional_instructions = ""
        self.model = "gpt-4o"
        self.api_base = "http://localhost:11434"
        self.prompt_only = False
        self.strict_coverage = False
        self.additional_instructions = f"You are receiving an ALMOST empty test file. This means you need to import the source file's class in addition to the source directory {SOURCE_DIRECTORY}. \
            For example, if you are testing MyClass.py and the class name is MyClass then you must add the language `from {SOURCE_DIRECTORY}.MyClass import MyClass`. \
            Another example: `agent/pr_agent.py` where the class name is `PRAgent` would become `from {SOURCE_DIRECTORY}.PRAgent import PRAgent`. \
            The `from {SOURCE_DIRECTORY}.MyClass` is extremely important. If you do not include from {SOURCE_DIRECTORY}.MyClass the test will not run."

def run_initial_coverage_command():
    # Run the initial coverage command to generate the initial coverage report
    initial_coverage_command = f"coverage run -m pytest --cov={SOURCE_DIRECTORY} --cov-report=xml:{INITIAL_COVERAGE_REPORT_PATH}"
    subprocess.run(initial_coverage_command, shell=True, check=True)

def extract_source_files_from_coverage_report(coverage_report_path):
    # Use CoverageProcessor to parse the coverage report and extract the list of source files
    coverage_processor = CoverageProcessor(coverage_report_path, "", "cobertura")
    coverage_data = coverage_processor.parse_coverage_report_cobertura()

    # Extract the list of source files from the coverage data
    source_files = list(coverage_data.keys())
    return source_files

if __name__ == "__main__":
    # Step 1: Run the initial coverage command
    run_initial_coverage_command()

    # Step 2: Extract the list of source files from the initial coverage report
    source_files = extract_source_files_from_coverage_report(INITIAL_COVERAGE_REPORT_PATH)

    # Keep track of total token count for cost monitoring
    total_input_token_count = 0
    total_output_token_count = 0

    # Step 3: Iterate through the list of source files and run CoverAgent
    for source_file in source_files:
        # Skip files names "__init__.py"
        if source_file == "__init__.py":
            continue

        # Join SOURCE_DIRECTORY with source_file
        source_file = os.path.join(SOURCE_DIRECTORY, source_file)

        # Validate that the source file exists
        if not os.path.exists(source_file):
            raise FileNotFoundError(f"Source file not found: {source_file}")

        # Print a banner for the current source file
        banner = f"Testing source file: {source_file}"
        print("*" * len(banner))
        print(banner)
        print("*" * len(banner))

        args = Args(source_file, TEST_FILE_PATH)
        agent = CoverAgent(args)
        agent.run()

        total_input_token_count += agent.test_gen.total_input_token_count
        total_output_token_count += agent.test_gen.total_output_token_count

    print(f"Total input token count: {total_input_token_count}, total output token count: {total_output_token_count}")

I've been seeing some strange ghost deletions (and then recoveries) with some of the initially generated tests. Here is a video that demonstrates the issue but anyone should be able to repeat it themself by running the above script and observing tests/unittest/test_single_file.py in real-time.