search

scientific-python / pytest-doctestplus

Pytest plugin providing advanced doctest features
Other
94 stars 38 forks source link

CI: ci job failures failures with py311-test-pytest74 and py311-test-pytest73 #265

Closed bsipocz closed 1 week ago

bsipocz commented 1 week ago

I cannot yet reproduce these locally on osx, and none of the other version combinations produces the same issue:


py311-test-pytest73: commands[0] /home/runner/work/pytest-doctestplus/pytest-doctestplus/.tmp/py311-test-pytest73> pip freeze
alabaster==1.0.0
babel==2.16.0
certifi==2024.8.30
charset-normalizer==3.3.2
docutils==0.21.2
idna==3.10
imagesize==1.4.1
iniconfig==2.0.0
Jinja2==3.1.4
MarkupSafe==2.1.5
numpy==2.1.1
packaging==24.1
pluggy==1.5.0
Pygments==2.18.0
pytest==7.3.2
pytest-doctestplus @ file:///home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/.tmp/package/1/pytest_doctestplus-1.2.2.dev33%2Bgb2577bd.tar.gz#sha256=338e67e6f1c672c6fb34c8944f754892f1a2b565fe24ec5fb10365888c138e98
pytest-remotedata==0.4.1
requests==2.32.3
snowballstemmer==2.2.0
Sphinx==8.0.2
sphinxcontrib-applehelp==2.0.0
sphinxcontrib-devhelp==2.0.0
sphinxcontrib-htmlhelp==2.1.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==2.0.0
sphinxcontrib-serializinghtml==2.0.0
urllib3==2.2.3
py311-test-pytest73: exit 0 (0.23 seconds) /home/runner/work/pytest-doctestplus/pytest-doctestplus/.tmp/py311-test-pytest73> pip freeze pid=1870
py311-test-pytest73: commands[1] /home/runner/work/pytest-doctestplus/pytest-doctestplus/.tmp/py311-test-pytest73> pytest /home/runner/work/pytest-doctestplus/pytest-doctestplus/tests --ignore=/home/runner/work/pytest-doctestplus/pytest-doctestplus/tests/docs/skip_some_remote_data.rst --doctest-plus --doctest-rst --remote-data
============================= test session starts ==============================
platform linux -- Python 3.11.10, pytest-7.3.2, pluggy-1.5.0
cachedir: .tox/py311-test-pytest73/.pytest_cache
rootdir: /home/runner/work/pytest-doctestplus/pytest-doctestplus
configfile: setup.cfg
plugins: remotedata-0.4.1, doctestplus-1.2.2.dev33+gb2577bd
collected 57 items

../../tests/test_doctestplus.py ..................x..................... [ 70%]
....F.....                                                               [ 87%]
../../tests/test_utils.py ..                                             [ 91%]
../../tests/docs/skip_all.rst s                                          [ 92%]
../../tests/docs/skip_some.rst .                                         [ 94%]
../../tests/python/doctests.py ...                                       [100%]

=================================== FAILURES ===================================
__________________________________ test_ufunc __________________________________

testdir = <Testdir local('/tmp/pytest-of-runner/pytest-0/test_ufunc0')>

    @pytest.mark.xfail(
            python_version() in ('3.11.9', '3.12.3'),
            reason='broken by https://github.com/python/cpython/pull/115440')
    def test_ufunc(testdir):
        pytest.importorskip('numpy')

        # Create and build example module
        testdir.makepyfile(module1="""
            def foo():
                '''A doctest...

                >>> foo()
                1
                '''
                return 1
            """)
        testdir.makepyfile(module2="""
            import functools
            from _module2 import foo, bar, bat as _bat

            def wrap_func(func):
                @functools.wraps(func)
                def wrapper(*args, **kwargs):
                    return func(*args, **kwargs)
                return wrapper

            bat = wrap_func(_bat)
            """)
        testdir.makepyfile(setup="""
            from setuptools import setup, Extension
            import numpy as np

            ext = Extension('_module2', ['_module2.c'],
                            extra_compile_args=['-std=c99'],
                            include_dirs=[np.get_include()])
            setup(name='example', py_modules=['module1', 'module2'], ext_modules=[ext])
            """)
        testdir.makefile('.c', _module2=r"""
            #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION

            #include <numpy/arrayobject.h>
            #include <numpy/ufuncobject.h>
            #include <Python.h>

            static double ufunc_inner(double a, double b)
            {
                return a + b;
            }

            static void ufunc_loop(
                char **args,
                const npy_intp *dimensions,
                const npy_intp *steps,
                void *NPY_UNUSED(data)
            ) {
                const npy_intp n = dimensions[0];
                for (npy_intp i = 0; i < n; i ++)
                {
                    *(double *) &args[2][i * steps[2]] = ufunc_inner(
                    *(double *) &args[0][i * steps[0]],
                    *(double *) &args[1][i * steps[1]]);
                }
            }

            static PyUFuncGenericFunction ufunc_loops[] = {ufunc_loop};
            static char ufunc_types[] = {NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE};
            static void *ufunc_data[] = {NULL};
            static const char ufunc_docstring[] = ">>> foo(1, 2)\n3.0";

            static PyModuleDef moduledef = {
                .m_base = PyModuleDef_HEAD_INIT,
                .m_name = "_module2",
                .m_size = -1
            };

            PyMODINIT_FUNC PyInit__module2(void)
            {
                import_array();
                import_ufunc();

                PyObject *module = PyModule_Create(&moduledef);
                if (!module)
                    return NULL;

                /* Add a ufunc _with_ a docstring. */
                PyObject *foo = PyUFunc_FromFuncAndData(
                    ufunc_loops, ufunc_data, ufunc_types, 1, 2, 1, PyUFunc_None,
                    "foo", ufunc_docstring, 0);
                if (!foo)
                {
                    Py_DECREF(module);
                    return NULL;
                }
                if (PyModule_AddObject(module, "foo", foo) < 0)
                {
                    Py_DECREF(foo);
                    Py_DECREF(module);
                    return NULL;
                }

                /* Add a ufunc _without_ a docstring. */
                PyObject *bar = PyUFunc_FromFuncAndData(
                    ufunc_loops, ufunc_data, ufunc_types, 1, 2, 1, PyUFunc_None,
                    "bar", NULL, 0);
                if (!bar)
                {
                    Py_DECREF(module);
                    return NULL;
                }
                if (PyModule_AddObject(module, "bar", bar) < 0)
                {
                    Py_DECREF(bar);
                    Py_DECREF(module);
                    return NULL;
                }

                /* Add another ufunc _without_ a docstring. */
                PyObject *bat = PyUFunc_FromFuncAndData(
                    ufunc_loops, ufunc_data, ufunc_types, 1, 2, 1, PyUFunc_None,
                    "bat", NULL, 0);
                if (!bat)
                {
                    Py_DECREF(module);
                    return NULL;
                }
                if (PyModule_AddObject(module, "bat", bat) < 0)
                {
                    Py_DECREF(bat);
                    Py_DECREF(module);
                    return NULL;
                }

                return module;
            }
            """)
        testdir.run(sys.executable, 'setup.py', 'build')
        build_dir, = glob.glob(str(testdir.tmpdir / 'build/lib.*'))

        result = testdir.inline_run(build_dir, '--doctest-plus', '--doctest-modules')
        result.assertoutcome(passed=1, failed=0)

        result = testdir.inline_run(build_dir, '--doctest-plus', '--doctest-modules', '--doctest-ufunc')
>       result.assertoutcome(passed=2, failed=0)
E       AssertionError: ([], [], [<CollectReport 'build/lib.linux-x86_64-cpython-311/module2.py' lenresult=0 outcome='failed'>])
E       assert {'failed': 1,... 'skipped': 0} == {'failed': 0,... 'skipped': 0}
E         Omitting 1 identical items, use -vv to show
E         Differing items:
E         {'failed': 1} != {'failed': 0}
E         {'passed': 0} != {'passed': 2}
E         Use -v to get more diff

/home/runner/work/pytest-doctestplus/pytest-doctestplus/tests/test_doctestplus.py:1292: AssertionError
----------------------------- Captured stdout call -----------------------------
running: /home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/py311-test-pytest73/bin/python setup.py build
     in: /tmp/pytest-of-runner/pytest-0/test_ufunc0
running build
running build_py
creating build/lib.linux-x86_64-cpython-311
copying module1.py -> build/lib.linux-x86_64-cpython-311
copying module2.py -> build/lib.linux-x86_64-cpython-311
running build_ext
building '_module2' extension
creating build/temp.linux-x86_64-cpython-311
gcc -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/py311-test-pytest73/lib/python3.11/site-packages/numpy/_core/include -I/home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/py311-test-pytest73/include -I/opt/hostedtoolcache/Python/3.11.10/x64/include/python3.11 -c _module2.c -o build/temp.linux-x86_64-cpython-311/_module2.o -std=c99
gcc -shared -Wl,--rpath=/opt/hostedtoolcache/Python/3.11.10/x64/lib -Wl,--rpath=/opt/hostedtoolcache/Python/3.11.10/x64/lib build/temp.linux-x86_64-cpython-311/_module2.o -L/opt/hostedtoolcache/Python/3.11.10/x64/lib -o build/lib.linux-x86_64-cpython-311/_module2.cpython-311-x86_64-linux-gnu.so
============================= test session starts ==============================
platform linux -- Python 3.11.10, pytest-7.3.2, pluggy-1.5.0
rootdir: /tmp/pytest-of-runner/pytest-0/test_ufunc0
plugins: remotedata-0.4.1, doctestplus-1.2.2.dev33+gb2577bd
collected 1 item

build/lib.linux-x86_64-cpython-311/module1.py .                          [100%]

============================== 1 passed in 0.01s ===============================
============================= test session starts ==============================
platform linux -- Python 3.11.10, pytest-7.3.2, pluggy-1.5.0
rootdir: /tmp/pytest-of-runner/pytest-0/test_ufunc0
plugins: remotedata-0.4.1, doctestplus-1.2.2.dev33+gb2577bd
collected 1 item / 1 error

==================================== ERRORS ====================================
________ ERROR collecting build/lib.linux-x86_64-cpython-311/module2.py ________
/home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/py311-test-pytest73/lib/python3.11/site-packages/pytest_doctestplus/plugin.py:298: in collect
    for test in finder.find(module):
/home/runner/work/pytest-doctestplus/pytest-doctestplus/.tox/py311-test-pytest73/lib/python3.11/site-packages/pytest_doctestplus/plugin.py:847: in find
    tests += doctest.DocTestFinder.find(
/opt/hostedtoolcache/Python/3.11.10/x64/lib/python3.11/doctest.py:942: in find
    self._find(tests, obj, name, module, source_lines, globs, {})
/opt/hostedtoolcache/Python/3.11.10/x64/lib/python3.11/doctest.py:1004: in _find
    test = self._get_test(obj, name, module, globs, source_lines)
/opt/hostedtoolcache/Python/3.11.10/x64/lib/python3.11/doctest.py:1072: in _get_test
    lineno = self._find_lineno(obj, source_lines)
/opt/hostedtoolcache/Python/3.11.10/x64/lib/python3.11/doctest.py:1121: in _find_lineno
    obj = inspect.unwrap(obj).__code__
E   AttributeError: 'numpy.ufunc' object has no attribute '__code__'
=========================== short test summary info ============================
ERROR build/lib.linux-x86_64-cpython-311/module2.py - AttributeError: 'numpy....
!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!
=============================== 1 error in 0.12s ===============================
=========================== short test summary info ============================
FAILED ../../tests/test_doctestplus.py::test_ufunc - AssertionError: ([], [],...
============== 1 failed, 54 passed, 1 skipped, 1 xfailed in 4.11s ==============
py311-test-pytest73: exit 1 (4.78 seconds) /home/runner/work/pytest-doctestplus/pytest-doctestplus/.tmp/py311-test-pytest73> pytest /home/runner/work/pytest-doctestplus/pytest-doctestplus/tests --ignore=/home/runner/work/pytest-doctestplus/pytest-doctestplus/tests/docs/skip_some_remote_data.rst --doctest-plus --doctest-rst --remote-data pid=1880
  py311-test-pytest73: FAIL code 1 (21.04=setup[16.03]+cmd[0.23,4.78] seconds)
  evaluation failed :( (21.21 seconds)
Error: Process completed with exit code 1.
bsipocz commented 1 week ago

Ahh, the difference is in the cpython version, and the test above should be xfail, but somehow it's now errors out.