search

beeware / briefcase

Tools to support converting a Python project into a standalone native application.
https://briefcase.readthedocs.io/
BSD 3-Clause "New" or "Revised" License
2.48k stars 352 forks source link

Test suite doesn't run on Windows 11 #1805

Closed MarieRoald closed 1 month ago

MarieRoald commented 1 month ago

Describe the bug

Many tests that use the capsys fixture fail on Windows 11. It seems like it's because sys.__stdout__.isatty() evaluates to True when we run the tests with pytest, since the following (horrible) workaround makes everything pass:

import sys

@pytest.fixture(autouse=True)
def monkeypatch_stdout(monkeypatch):
    def false(*args, **kwargs): return False
    monkeypatch.setattr(sys.__stdout__, "isatty", false)

Steps to reproduce

  1. Use a Windows 11 machine
  2. Clone Briefcase and install it in a clean virtual environment
  3. Run pytest
  4. See error

Expected behavior

All tests should pass

Screenshots

No response

Environment

Logs

[...]

_______________________________________________________________________ test_variant_with_size ________________________________________________________________________

create_command = <tests.commands.create.conftest.DummyCreateCommand object at 0x000001EBA9515C10>
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_variant_with_size0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA9517950>

    def test_variant_with_size(create_command, tmp_path, capsys):
        """If the app specifies a variant with a size, the sized variant is used."""
        create_command.tools.shutil = mock.MagicMock(spec_set=shutil)

        # Create the source image
        source_file = tmp_path / "base_path/input/original-3742.png"
        source_file.parent.mkdir(parents=True, exist_ok=True)
        with source_file.open("w", encoding="utf-8") as f:
            f.write("image")

        # Try to install the image
        out_path = tmp_path / "output.png"
        create_command.install_image(
            "sample image",
            source={
                "round": "input/original",
            },
            variant="round",
            size="3742",
            target=out_path,
        )

        # The right message was written to output
        expected = (
            "Installing input/original-3742.png as 3742px round sample image... started\n"
            "Installing input/original-3742.png as 3742px round sample image... done\n\n"
        )
>       assert capsys.readouterr().out == expected
E       AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
E
E         Skipping 57 identical leading characters in diff, use -v to show
E         +  image... done
E         -  image... started
E         - Installing input/original-3742.png as 3742px round sample image... done

tests\commands\create\test_install_image.py:280: AssertionError
________________________________________________________________________ test_unsized_variant _________________________________________________________________________ 

create_command = <tests.commands.create.conftest.DummyCreateCommand object at 0x000001EBA92A7CE0>
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_unsized_variant0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA92A7950>

    def test_unsized_variant(create_command, tmp_path, capsys):
        """If the app specifies an unsized variant, it is used."""
        create_command.tools.shutil = mock.MagicMock(spec_set=shutil)

        # Create the source image
        source_file = tmp_path / "base_path/input/original.png"
        source_file.parent.mkdir(parents=True, exist_ok=True)
        with source_file.open("w", encoding="utf-8") as f:
            f.write("image")

        # Try to install the image
        # Unsized variants are an annoying edge case; they get the *variant*
        # as the *size*.
        out_path = tmp_path / "output.png"
        create_command.install_image(
            "sample image",
            source={
                "round": "input/original",
            },
            variant=None,
            size="round",
            target=out_path,
        )

        # The right message was written to output
        expected = (
            "Installing input/original.png as round sample image... started\n"
            "Installing input/original.png as round sample image... done\n\n"
        )
>       assert capsys.readouterr().out == expected
E       AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
E
E         Skipping 45 identical leading characters in diff, use -v to show
E         +  image... done
E         -  image... started
E         - Installing input/original.png as round sample image... done

tests\commands\create\test_install_image.py:378: AssertionError
_________________________________________________________________________ test_sign_app[True] _________________________________________________________________________

dummy_command = <tests.platforms.macOS.app.test_signing.DummySigningCommand object at 0x000001EBA9BF2C00>
first_app_with_binaries = <com.example.first-app v0.0.1 AppConfig>, verbose = True
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_sign_app_True_0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA9C96C30>

    @pytest.mark.parametrize("verbose", [True, False])
    def test_sign_app(dummy_command, first_app_with_binaries, verbose, tmp_path, capsys):
        """An app bundle can be signed."""
        if verbose:
            dummy_command.logger.verbosity = LogLevel.VERBOSE

        # Sign the app
        dummy_command.sign_app(
            first_app_with_binaries, identity="Sekrit identity (DEADBEEF)"
        )

        # A request has been made to sign all the so and dylib files
        # This acts as a test of the discovery process:
        # * It discovers frameworks
        # * It discovers apps
        # * It discovers Mach-O binaries in various forms and guises
        # * It *doesn't* discover directories
        # * It *doesn't* discover non-Mach-O binaries
        # * It traverses in "depth first" order
        app_path = (
            tmp_path
            / "base_path"
            / "build"
            / "first-app"
            / "macos"
            / "app"
            / "First App.app"
        )
        lib_path = app_path / "Contents/Resources/app_packages"
        frameworks_path = app_path / "Contents/Frameworks"
        dummy_command.tools.subprocess.run.assert_has_calls(
            [
                sign_call(tmp_path, lib_path / "subfolder/second_so.so"),
                sign_call(tmp_path, lib_path / "subfolder/second_dylib.dylib"),
                sign_call(tmp_path, lib_path / "special.binary"),
                sign_call(tmp_path, lib_path / "other_binary"),
                sign_call(tmp_path, lib_path / "first_so.so"),
                sign_call(tmp_path, lib_path / "first_dylib.dylib"),
                sign_call(tmp_path, lib_path / "Extras.app/Contents/MacOS/Extras"),
                sign_call(tmp_path, lib_path / "Extras.app"),
                sign_call(
                    tmp_path,
                    frameworks_path / "Extras.framework/Resources/extras.dylib",
                ),
                sign_call(tmp_path, frameworks_path / "Extras.framework"),
                sign_call(tmp_path, app_path),
            ],
            any_order=True,
        )

        # Also check that files are not signed after their parent directory has been
        # signed. Reduce the files mentions in the calls to the dummy command
        # to a list of path objects, then ensure that the call to sign any given file
        # does not occur *after* it's parent directory.
        sign_targets = [
            Path(call.args[0][1]) for call in dummy_command.tools.subprocess.run.mock_calls
        ]

        parents = set()
        for path in sign_targets:
            # Check parent of path is not in parents
            assert path.parent not in parents
            parents.add(path)

        # Output only happens if in debug mode.
        output = capsys.readouterr().out
        if sys.platform == "win32":
            # In practice, we won't ever actually run signing on win32; but to ensure test
            # coverage we need to. However, win32 doesn't handle executable permissions
            # the same as linux/unix, `unknown.binary` is identified as a signing target.
            # We ignore this discrepancy for testing purposes.
>           assert len(output.strip("\n").split("\n")) == (12 if verbose else 1)
E           AssertionError: assert 13 == 12
E            +  where 13 = len(['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...])
E            +    where ['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...] = <built-in method split of str object at 0x000001EBA79D8970>('\n')    
E            +      where <built-in method split of str object at 0x000001EBA79D8970> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...ng build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00'.split  
E            +        where 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...ng build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00' = <built-in method strip of str object at 0x000001EBA79DCFE0>('\n')
E            +          where <built-in method strip of str object at 0x000001EBA79DCFE0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build... build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00\n'.strip

tests\platforms\macOS\app\test_signing.py:555: AssertionError
__________________________________________________________________ test_sign_app_with_failure[True] ___________________________________________________________________

dummy_command = <tests.platforms.macOS.app.test_signing.DummySigningCommand object at 0x000001EBA92BA0C0>
first_app_with_binaries = <com.example.first-app v0.0.1 AppConfig>, verbose = True, capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA95409E0>

    @pytest.mark.parametrize("verbose", [True, False])
    def test_sign_app_with_failure(dummy_command, first_app_with_binaries, verbose, capsys):
        """If signing a single file in the app fails, the error is surfaced."""
        if verbose:
            dummy_command.logger.verbosity = LogLevel.VERBOSE

        # Sign the app. Signing first_dylib.dylib will fail.
        def _codesign(args, **kwargs):
            if Path(args[1]).name == "first_dylib.dylib":
                raise subprocess.CalledProcessError(
                    returncode=1, cmd=args, stderr=f"{args[1]}: Unknown error"
                )

        dummy_command.tools.subprocess.run.side_effect = _codesign

        # The invocation will raise an error; however, we can't predict exactly which
        # file will raise an error.
        with pytest.raises(
            BriefcaseCommandError, match=r"Unable to code sign .*first_dylib\.dylib"
        ):
            dummy_command.sign_app(
                first_app_with_binaries, identity="Sekrit identity (DEADBEEF)"
            )

        # There has been at least 1 call to sign files. We can't know how many are
        # actually signed, as threads are involved.
        dummy_command.tools.subprocess.run.call_count > 0

        # Output only happens if in debug mode.
        output = capsys.readouterr().out
        if sys.platform == "win32":
            # In practice, we won't ever actually run signing on win32; but to ensure test
            # coverage we need to. However, win32 doesn't handle executable permissions
            # the same as linux/unix, `unknown.binary` is identified as a signing target.
            # We ignore this discrepancy for testing purposes.
>           assert len(output.strip("\n").split("\n")) == (7 if verbose else 1)
E           AssertionError: assert 8 == 7
E            +  where 8 = len(['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...])
E            +    where ['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...] = <built-in method split of str object at 0x000001EBA62FF0E0>('\n')    
E            +      where <built-in method split of str object at 0x000001EBA62FF0E0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...nts\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01'.split  
E            +        where 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...nts\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01' = <built-in method strip of str object at 0x000001EBA62FDDF0>('\n')
E            +          where <built-in method strip of str object at 0x000001EBA62FDDF0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...s\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01\n'.strip

tests\platforms\macOS\app\test_signing.py:595: AssertionError
======================================================================= short test summary info =======================================================================
FAILED tests/commands/create/test_cleanup_app_content.py::test_no_cleanup[True] - AssertionError: assert 4 == 5
FAILED tests/commands/create/test_cleanup_app_content.py::test_no_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_cleanup[True] - AssertionError: assert 4 == 5
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_all_files_in_dir_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_all_files_in_dir_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_glob_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_glob_cleanup[True] - AssertionError: assert 6 == 7
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_deep_glob_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_deep_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_template_glob_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_template_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_non_existent_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_non_existent_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_install_image.py::test_no_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'    
FAILED tests/commands/create/test_install_image.py::test_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'       
FAILED tests/commands/create/test_install_image.py::test_variant_with_no_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
FAILED tests/commands/create/test_install_image.py::test_variant_with_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'    
FAILED tests/commands/create/test_install_image.py::test_unsized_variant - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'      
FAILED tests/platforms/macOS/app/test_signing.py::test_sign_app[True] - AssertionError: assert 13 == 12
FAILED tests/platforms/macOS/app/test_signing.py::test_sign_app_with_failure[True] - AssertionError: assert 8 == 7
======================================================= 25 failed, 2828 passed, 91 skipped in 131.64s (0:02:11) =======================================================

Additional context

No response

jdgsmallwood commented 1 month ago

Looking at this at PyCon US 2024