Getting error during a data migration that occurs during initial test setup "sqlite3.OperationalError: no such column"

[ntouran]

I'm trying out pytest-django coming from a traditional manage.py test workflow. After configuring, when I run pytest all my tests are discovered properly, but they all fail with the same error during migrations. Exception shown below.

Basically, the system is crashing while running a custom data migration that accesses a database column that was once defined, but then later removed (and migrated). This is typically handled in Django by using historical models, as in documented in the django docs.

But when running pytest-django it seems that the historical model system isn't working right?

Note that I am using django-simple-history for the HistoricalParameter model.

My data migration that's crashing looks like this:

def pack_value_and_units_into_json(apps, schema_editor):
    Parameter = apps.get_model("plant", "Parameter")
    HistoricalParameter = apps.get_model("plant", "HistoricalParameter")
    for row in HistoricalParameter.objects.all():
        newvalue = json.dumps([{"units": row.units, "value": [row.value]}])
        row.value = newvalue
        row.save(update_fields=["value"])

    for row in Parameter.objects.all():
        newvalue = json.dumps([{"units": row.units, "value": [row.value]}])
        row.value = newvalue
        row.save(update_fields=["value"])

class Migration(migrations.Migration):

    dependencies = [
        ("plant", "0019_requirement_siblings"),
    ]

    operations = [
        migrations.RunPython(
            pack_value_and_units_into_json, reverse_code=migrations.RunPython.noop
        ),
        migrations.AlterField(
            model_name="historicalparameter",
            name="value",
            field=models.JSONField(
                blank=True,
                null=True,
            ),
        ),
        migrations.AlterField(
            model_name="parameter",
            name="value",
            field=models.JSONField(
                blank=True,
                null=True,
            ),
        ),
        migrations.RemoveField(
            model_name="historicalparameter",
            name="units",
        ),
        migrations.RemoveField(
            model_name="parameter",
            name="units",
        ),
    ]

The error is:

(atom39) C:\Users\ntouran\codes\atom\atom>pytest -x -v
===================================================================== test session starts ======================================================================
platform win32 -- Python 3.9.9, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 -- C:\Users\ntouran\codes\venvs\atom39\Scripts\python.exe
cachedir: .pytest_cache
django: settings: atom.settings_local (from ini)
rootdir: C:\Users\ntouran\codes\atom\atom, configfile: pytest.ini
plugins: django-4.5.2
collected 93 items

accounts/tests/test_accounts.py::TestEmail::test_email ERROR                                                                                              [  1%]

============================================================================ ERRORS ============================================================================
____________________________________________________________ ERROR at setup of TestEmail.test_email ____________________________________________________________

self = <django.db.backends.utils.CursorWrapper object at 0x0000019DCAF43A00>
sql = 'SELECT "plant_historicalparameter"."id", "plant_historicalparameter"."name", "plant_historicalparameter"."description...icalparameter" ORDER BY "plant_historicalparameter"."history_date" DESC, "plant_historicalparameter"."history_id" DESC'
params = ()
ignored_wrapper_args = (False, {'connection': <django.db.backends.sqlite3.base.DatabaseWrapper object at 0x0000019DC7CF0CA0>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x0000019DCAF43A00>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)

..\..\venvs\atom39\lib\site-packages\django\db\backends\utils.py:84:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <django.db.backends.sqlite3.base.SQLiteCursorWrapper object at 0x0000019DCB346280>
query = 'SELECT "plant_historicalparameter"."id", "plant_historicalparameter"."name", "plant_historicalparameter"."description...icalparameter" ORDER BY "plant_historicalparameter"."history_date" DESC, "plant_historicalparameter"."history_id" DESC'
params = ()

    def execute(self, query, params=None):
        if params is None:
            return Database.Cursor.execute(self, query)
        query = self.convert_query(query)
>       return Database.Cursor.execute(self, query, params)
E       sqlite3.OperationalError: no such column: plant_historicalparameter.units

..\..\venvs\atom39\lib\site-packages\django\db\backends\sqlite3\base.py:423: OperationalError

The above exception was the direct cause of the following exception:

request = <SubRequest '_django_setup_unittest' for <TestCaseFunction test_email>>
django_db_blocker = <pytest_django.plugin._DatabaseBlocker object at 0x0000019DB5717370>

    @pytest.fixture(autouse=True, scope="class")
    def _django_setup_unittest(
        request,
        django_db_blocker: "_DatabaseBlocker",
    ) -> Generator[None, None, None]:
        """Setup a django unittest, internal to pytest-django."""
        if not django_settings_is_configured() or not is_django_unittest(request):
            yield
            return

        # Fix/patch pytest.
        # Before pytest 5.4: https://github.com/pytest-dev/pytest/issues/5991
        # After pytest 5.4: https://github.com/pytest-dev/pytest-django/issues/824
        from _pytest.unittest import TestCaseFunction
        original_runtest = TestCaseFunction.runtest

        def non_debugging_runtest(self) -> None:
            self._testcase(result=self)

        try:
            TestCaseFunction.runtest = non_debugging_runtest  # type: ignore[assignment]

>           request.getfixturevalue("django_db_setup")

..\..\venvs\atom39\lib\site-packages\pytest_django\plugin.py:490:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
..\..\venvs\atom39\lib\site-packages\pytest_django\fixtures.py:122: in django_db_setup
    db_cfg = setup_databases(
..\..\venvs\atom39\lib\site-packages\django\test\utils.py:179: in setup_databases
    connection.creation.create_test_db(
..\..\venvs\atom39\lib\site-packages\django\db\backends\base\creation.py:74: in create_test_db
    call_command(
..\..\venvs\atom39\lib\site-packages\django\core\management\__init__.py:181: in call_command
    return command.execute(*args, **defaults)
..\..\venvs\atom39\lib\site-packages\django\core\management\base.py:398: in execute
    output = self.handle(*args, **options)
..\..\venvs\atom39\lib\site-packages\django\core\management\base.py:89: in wrapped
    res = handle_func(*args, **kwargs)
..\..\venvs\atom39\lib\site-packages\django\core\management\commands\migrate.py:244: in handle
    post_migrate_state = executor.migrate(
..\..\venvs\atom39\lib\site-packages\django\db\migrations\executor.py:117: in migrate
    state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
..\..\venvs\atom39\lib\site-packages\django\db\migrations\executor.py:147: in _migrate_all_forwards
    state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
..\..\venvs\atom39\lib\site-packages\django\db\migrations\executor.py:227: in apply_migration
    state = migration.apply(state, schema_editor)
..\..\venvs\atom39\lib\site-packages\django\db\migrations\migration.py:126: in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
..\..\venvs\atom39\lib\site-packages\django\db\migrations\operations\special.py:190: in database_forwards
    self.code(from_state.apps, schema_editor)
plant\migrations\0020_auto_20210830_1340.py:19: in pack_value_and_units_into_json
    for row in HistoricalParameter.objects.all():
..\..\venvs\atom39\lib\site-packages\django\db\models\query.py:280: in __iter__
    self._fetch_all()
..\..\venvs\atom39\lib\site-packages\django\db\models\query.py:1324: in _fetch_all
    self._result_cache = list(self._iterable_class(self))
..\..\venvs\atom39\lib\site-packages\django\db\models\query.py:51: in __iter__
    results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
..\..\venvs\atom39\lib\site-packages\django\db\models\sql\compiler.py:1175: in execute_sql
    cursor.execute(sql, params)
..\..\venvs\atom39\lib\site-packages\django\db\backends\utils.py:66: in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
..\..\venvs\atom39\lib\site-packages\django\db\backends\utils.py:75: in _execute_with_wrappers
    return executor(sql, params, many, context)
..\..\venvs\atom39\lib\site-packages\django\db\backends\utils.py:84: in _execute
    return self.cursor.execute(sql, params)
..\..\venvs\atom39\lib\site-packages\django\db\utils.py:90: in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
..\..\venvs\atom39\lib\site-packages\django\db\backends\utils.py:84: in _execute
    return self.cursor.execute(sql, params)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <django.db.backends.sqlite3.base.SQLiteCursorWrapper object at 0x0000019DCB346280>
query = 'SELECT "plant_historicalparameter"."id", "plant_historicalparameter"."name", "plant_historicalparameter"."description...icalparameter" ORDER BY "plant_historicalparameter"."history_date" DESC, "plant_historicalparameter"."history_id" DESC'
params = ()

    def execute(self, query, params=None):
        if params is None:
            return Database.Cursor.execute(self, query)
        query = self.convert_query(query)
>       return Database.Cursor.execute(self, query, params)
E       django.db.utils.OperationalError: no such column: plant_historicalparameter.units

..\..\venvs\atom39\lib\site-packages\django\db\backends\sqlite3\base.py:423: OperationalError
-------------------------------------------------------------------- Captured stderr setup ---------------------------------------------------------------------
Creating test database for alias 'default'...
Creating test database for alias 'test'...

[gador]

I am getting a similar error on django 4.1 but not on 4.0.7. Which django version do you use?

[ntouran]

I'm way back on 3.2.14

[gador]

ok, I have found my issue to be a regression in django4 (https://code.djangoproject.com/ticket/33899)