ERROR at setup

[djarecka]

I sometimes get the error when running test in docker (docker-compose run --rm django tox -e test dandiapi/api/tests/test_version.py -v). Any idea why this might happen?

.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:84: OperationalError
_____________________________________________ ERROR at setup of test_version_rest_delete_draft_admin _____________________________________________

self = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
sql = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL', params = []
ignored_wrapper_args = (False, {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               psycopg2.errors.DeadlockDetected: deadlock detected
E               DETAIL:  Process 3825 waits for AccessExclusiveLock on relation 29927 of database 29546; blocked by process 3829.
E               Process 3829 waits for AccessShareLock on relation 29869 of database 29546; blocked by process 3825.
E               HINT:  See server log for query details.

ignored_wrapper_args = (False, {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>})
params     = []
self       = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'

.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:84: DeadlockDetected

The above exception was the direct cause of the following exception:

request = <SubRequest 'django_db_setup' for <Function test_version_next_published_version_nosave>>, django_test_environment = None
django_db_blocker = <pytest_django.plugin._DatabaseBlocker object at 0x7f65595f9df0>, django_db_use_migrations = True, django_db_keepdb = False
django_db_createdb = False, django_db_modify_db_settings = None
    @pytest.fixture(scope="session")
    def django_db_setup(
        request,
        django_test_environment: None,
        django_db_blocker,
        django_db_use_migrations: bool,
        django_db_keepdb: bool,
        django_db_createdb: bool,
        django_db_modify_db_settings: None,
    ) -> None:
        """Top level fixture to ensure test databases are available"""
        from django.test.utils import setup_databases, teardown_databases

        setup_databases_args = {}

        if not django_db_use_migrations:
            _disable_native_migrations()

        if django_db_keepdb and not django_db_createdb:
            setup_databases_args["keepdb"] = True

        with django_db_blocker.unblock():
>           db_cfg = setup_databases(
                verbosity=request.config.option.verbose,
                interactive=False,
                **setup_databases_args
            )

django_db_blocker = <pytest_django.plugin._DatabaseBlocker object at 0x7f65595f9df0>
django_db_createdb = False
django_db_keepdb = False
django_db_modify_db_settings = None
django_db_use_migrations = True
django_test_environment = None
request    = <SubRequest 'django_db_setup' for <Function test_version_next_published_version_nosave>>
setup_databases = <function setup_databases at 0x7f6557016af0>
setup_databases_args = {}
teardown_databases = <function teardown_databases at 0x7f6557016ca0>

.tox/test/lib/python3.8/site-packages/pytest_django/fixtures.py:117: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/test/lib/python3.8/site-packages/django/test/utils.py:179: in setup_databases
    connection.creation.create_test_db(
        alias      = 'default'
        aliases    = ['default']
        connection = <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>
        db_name    = 'django'
        debug_sql  = False
        first_alias = 'default'
        interactive = False
        keepdb     = False
        kwargs     = {}
        mirrored_aliases = {}
        old_names  = [(<django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'django', True)]
        parallel   = 0
        test_databases = {('postgres', 5432, 'django.db.backends.postgresql', 'test_django'): ('django', ['default'])}
        time_keeper = <django.test.utils.NullTimeKeeper object at 0x7f6554e20f40>
        verbosity  = 1
.tox/test/lib/python3.8/site-packages/django/db/backends/base/creation.py:74: in create_test_db
    call_command(
        action     = 'Creating'
        autoclobber = True
        call_command = <function call_command at 0x7f6559150310>
        keepdb     = False
        self       = <django.db.backends.postgresql.creation.DatabaseCreation object at 0x7f6556ed4f10>
        serialize  = True
        test_database_name = 'test_django'
        verbosity  = 1
.tox/test/lib/python3.8/site-packages/django/core/management/__init__.py:181: in call_command
    return command.execute(*args, **defaults)
        app_name   = 'django.core'
        arg_options = {'database': 'default', 'interactive': False, 'run_syncdb': True, 'verbosity': 0}
        args       = ()
        command    = <django.core.management.commands.migrate.Command object at 0x7f6554e20fa0>
        command_name = 'migrate'
        defaults   = {'app_label': None, 'check_unapplied': False, 'configuration': None, 'database': 'default', ...}
        dest_parameters = {'app_label', 'check_unapplied', 'configuration', 'database', 'fake', 'fake_initial', ...}
        get_actions = <function call_command.<locals>.get_actions at 0x7f6554f92310>
        mutually_exclusive_required_options = set()
        opt        = _StoreAction(option_strings=['--configuration'], dest='configuration', nargs=None, const=None, default=None, type=None...g. "Development". If this isn\'t provided, the  DJANGO_CONFIGURATION environment variable will be used.', metavar=None)
        opt_mapping = {'check': 'check_unapplied', 'configuration': 'configuration', 'database': 'database', 'fake': 'fake', ...}
        options    = {'database': 'default', 'interactive': False, 'run_syncdb': True, 'verbosity': 0}
        parse_args = []
        parser     = CommandParser(prog=' migrate', usage=None, description='Updates database schema. Manages both apps with migrations and....', formatter_class=<class 'django.core.management.base.DjangoHelpFormatter'>, conflict_handler='error', add_help=True)
        parser_actions = [_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, cho...rgs=0, const=True, default=False, type=None, choices=None, help='Raise on CommandError exceptions', metavar=None), ...]
        stealth_options = {'stderr', 'stdout'}
        unknown_options = set()
        valid_options = {'app_label', 'check', 'check_unapplied', 'configuration', 'database', 'fake', ...}
.tox/test/lib/python3.8/site-packages/django/core/management/base.py:398: in execute
    output = self.handle(*args, **options)
        args       = ()
        options    = {'app_label': None, 'check_unapplied': False, 'configuration': None, 'database': 'default', ...}
        self       = <django.core.management.commands.migrate.Command object at 0x7f6554e20fa0>
.tox/test/lib/python3.8/site-packages/django/core/management/base.py:89: in wrapped
    res = handle_func(*args, **kwargs)
        args       = (<django.core.management.commands.migrate.Command object at 0x7f6554e20fa0>,)
        handle_func = <function Command.handle at 0x7f6554f8cf70>
        kwargs     = {'app_label': None, 'check_unapplied': False, 'configuration': None, 'database': 'default', ...}
        saved_locale = 'en-us'
        translation = <module 'django.utils.translation' from '/opt/django-project/.tox/test/lib/python3.8/site-packages/django/utils/translation/__init__.py'>
.tox/test/lib/python3.8/site-packages/django/core/management/commands/migrate.py:244: in handle
    post_migrate_state = executor.migrate(
        app_config = <AppConfig: github>
        args       = ()
        conflicts  = {}
        connection = <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>
        database   = 'default'
        executor   = <django.db.migrations.executor.MigrationExecutor object at 0x7f6554f83fa0>
        exit_dry   = False
        fake       = False
        fake_initial = False
        options    = {'app_label': None, 'check_unapplied': False, 'configuration': None, 'database': 'default', ...}
        plan       = [(<Migration contenttypes.0001_initial>, False), (<Migration auth.0001_initial>, False), (<Migration account.0001_init...ength>, False), (<Migration admin.0001_initial>, False), (<Migration admin.0002_logentry_remove_auto_add>, False), ...]
        pre_migrate_apps = <django.db.migrations.state.StateApps object at 0x7f6554f14e50>
        pre_migrate_state = <django.db.migrations.state.ProjectState object at 0x7f6554eb4430>
        run_syncdb = {'_allauth_support', 'allauth', 'corsheaders', 'django_extensions', 'django_filters', 'drf_yasg', ...}
        self       = <django.core.management.commands.migrate.Command object at 0x7f6554e20fa0>
        target_app_labels_only = True
        targets    = [('account', '0002_email_max_length'), ('admin', '0003_logentry_add_action_flag_choices'), ('api', '0018_alter_validat...user_first_name_max_length'), ('authtoken', '0003_tokenproxy'), ('contenttypes', '0002_remove_content_type_name'), ...]
.tox/test/lib/python3.8/site-packages/django/db/migrations/executor.py:117: in migrate
    state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
        all_backwards = False
        all_forwards = True
        fake       = False
        fake_initial = False
        full_plan  = [(<Migration contenttypes.0001_initial>, False), (<Migration auth.0001_initial>, False), (<Migration account.0001_init...ength>, False), (<Migration admin.0001_initial>, False), (<Migration admin.0002_logentry_remove_auto_add>, False), ...]
        plan       = [(<Migration contenttypes.0001_initial>, False), (<Migration auth.0001_initial>, False), (<Migration account.0001_init...ength>, False), (<Migration admin.0001_initial>, False), (<Migration admin.0002_logentry_remove_auto_add>, False), ...]
        self       = <django.db.migrations.executor.MigrationExecutor object at 0x7f6554f83fa0>
        state      = <django.db.migrations.state.ProjectState object at 0x7f6554eb4490>
        targets    = [('account', '0002_email_max_length'), ('admin', '0003_logentry_add_action_flag_choices'), ('api', '0018_alter_validat...user_first_name_max_length'), ('authtoken', '0003_tokenproxy'), ('contenttypes', '0002_remove_content_type_name'), ...]
.tox/test/lib/python3.8/site-packages/django/db/migrations/executor.py:147: in _migrate_all_forwards
    state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
        _          = False
        fake       = False
        fake_initial = False
        full_plan  = [(<Migration contenttypes.0001_initial>, False), (<Migration auth.0001_initial>, False), (<Migration account.0001_init...ength>, False), (<Migration admin.0001_initial>, False), (<Migration admin.0002_logentry_remove_auto_add>, False), ...]
        migration  = <Migration api.0018_alter_validation_errors>
        migrations_to_run = {<Migration auth.0002_alter_permission_name_max_length>, <Migration authtoken.0003_tokenproxy>, <Migration sessions.00....0002_remove_content_type_name>, <Migration authtoken.0002_auto_20160226_1747>, <Migration guardian.0001_initial>, ...}
        plan       = [(<Migration contenttypes.0001_initial>, False), (<Migration auth.0001_initial>, False), (<Migration account.0001_init...ength>, False), (<Migration admin.0001_initial>, False), (<Migration admin.0002_logentry_remove_auto_add>, False), ...]
        self       = <django.db.migrations.executor.MigrationExecutor object at 0x7f6554f83fa0>
        state      = <django.db.migrations.state.ProjectState object at 0x7f6554eb4490>
.tox/test/lib/python3.8/site-packages/django/db/migrations/executor.py:227: in apply_migration
    state = migration.apply(state, schema_editor)
        fake       = False
        fake_initial = False
        migration  = <Migration api.0018_alter_validation_errors>
        migration_recorded = False
        schema_editor = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        self       = <django.db.migrations.executor.MigrationExecutor object at 0x7f6554f83fa0>
        state      = <django.db.migrations.state.ProjectState object at 0x7f6554eb4490>
.tox/test/lib/python3.8/site-packages/django/db/migrations/migration.py:126: in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
        atomic_operation = False
        collect_sql = False
        old_state  = <django.db.migrations.state.ProjectState object at 0x7f6554ec05b0>
        operation  = <AlterField  model_name='version', name='validation_errors', field=<django.db.models.fields.json.JSONField>>
        project_state = <django.db.migrations.state.ProjectState object at 0x7f6554eb4490>
        schema_editor = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        self       = <Migration api.0018_alter_validation_errors>
.tox/test/lib/python3.8/site-packages/django/db/migrations/operations/fields.py:244: in database_forwards
    schema_editor.alter_field(from_model, from_field, to_field)
        app_label  = 'api'
        from_field = <django.db.models.fields.json.JSONField: validation_errors>
        from_model = <class '__fake__.Version'>
        from_state = <django.db.migrations.state.ProjectState object at 0x7f6554ec05b0>
        schema_editor = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        self       = <AlterField  model_name='version', name='validation_errors', field=<django.db.models.fields.json.JSONField>>
        to_field   = <django.db.models.fields.json.JSONField: validation_errors>
        to_model   = <class '__fake__.Version'>
        to_state   = <django.db.migrations.state.ProjectState object at 0x7f6554eb4490>
.tox/test/lib/python3.8/site-packages/django/db/backends/base/schema.py:608: in alter_field
    self._alter_field(model, old_field, new_field, old_type, new_type,
        model      = <class '__fake__.Version'>
        new_db_params = {'check': None, 'type': 'jsonb'}
        new_field  = <django.db.models.fields.json.JSONField: validation_errors>
        new_type   = 'jsonb'
        old_db_params = {'check': None, 'type': 'jsonb'}
        old_field  = <django.db.models.fields.json.JSONField: validation_errors>
        old_type   = 'jsonb'
        self       = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        strict     = False
.tox/test/lib/python3.8/site-packages/django/db/backends/postgresql/schema.py:196: in _alter_field
    super()._alter_field(
        __class__  = <class 'django.db.backends.postgresql.schema.DatabaseSchemaEditor'>
        model      = <class '__fake__.Version'>
        new_db_params = {'check': None, 'type': 'jsonb'}
        new_field  = <django.db.models.fields.json.JSONField: validation_errors>
        new_type   = 'jsonb'
        old_db_params = {'check': None, 'type': 'jsonb'}
        old_field  = <django.db.models.fields.json.JSONField: validation_errors>
        old_type   = 'jsonb'
        self       = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        strict     = False
.tox/test/lib/python3.8/site-packages/django/db/backends/base/schema.py:765: in _alter_field
    self.execute(
        actions    = [('ALTER COLUMN "validation_errors" DROP NOT NULL', [])]
        drop_foreign_keys = False
        fks_dropped = set()
        four_way_default_alteration = False
        fragment   = ('ALTER COLUMN "validation_errors" DROP NOT NULL', [])
        model      = <class '__fake__.Version'>
        needs_database_default = False
        new_collation = None
        new_db_params = {'check': None, 'type': 'jsonb'}
        new_field  = <django.db.models.fields.json.JSONField: validation_errors>
        new_type   = 'jsonb'
        null_actions = [('ALTER COLUMN "validation_errors" DROP NOT NULL', [])]
        old_collation = None
        old_db_params = {'check': None, 'type': 'jsonb'}
        old_field  = <django.db.models.fields.json.JSONField: validation_errors>
        old_type   = 'jsonb'
        params     = []
        post_actions = []
        self       = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        sql        = 'ALTER COLUMN "validation_errors" DROP NOT NULL'
        strict     = False
.tox/test/lib/python3.8/site-packages/django/db/backends/base/schema.py:145: in execute
    cursor.execute(sql, params)
        cursor     = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
        params     = []
        self       = <django.db.backends.postgresql.schema.DatabaseSchemaEditor object at 0x7f6554b5f220>
        sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'
.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:66: in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
        params     = []
        self       = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
        sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'
.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:75: in _execute_with_wrappers
    return executor(sql, params, many, context)
        context    = {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>}
        executor   = <bound method CursorWrapper._execute of <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>>
        many       = False
        params     = []
        self       = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
        sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'
.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:84: in _execute
    return self.cursor.execute(sql, params)
        ignored_wrapper_args = (False, {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>})
        params     = []
        self       = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
        sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'
.tox/test/lib/python3.8/site-packages/django/db/utils.py:90: in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
        db_exc_type = <class 'psycopg2.OperationalError'>
        dj_exc_type = <class 'django.db.utils.OperationalError'>
        dj_exc_value = OperationalError('deadlock detected\nDETAIL:  Process 3825 waits for AccessExclusiveLock on relation 29927 of database...ssShareLock on relation 29869 of database 29546; blocked by process 3825.\nHINT:  See server log for query details.\n')
        exc_type   = <class 'psycopg2.errors.DeadlockDetected'>
        exc_value  = DeadlockDetected('deadlock detected\nDETAIL:  Process 3825 waits for AccessExclusiveLock on relation 29927 of database...ssShareLock on relation 29869 of database 29546; blocked by process 3825.\nHINT:  See server log for query details.\n')
        self       = <django.db.utils.DatabaseErrorWrapper object at 0x7f6554f14fd0>
        traceback  = <traceback object at 0x7f6554c4f200>
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
sql = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL', params = []
ignored_wrapper_args = (False, {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               django.db.utils.OperationalError: deadlock detected
E               DETAIL:  Process 3825 waits for AccessExclusiveLock on relation 29927 of database 29546; blocked by process 3829.
E               Process 3829 waits for AccessShareLock on relation 29869 of database 29546; blocked by process 3825.
E               HINT:  See server log for query details.

ignored_wrapper_args = (False, {'connection': <django.db.backends.postgresql.base.DatabaseWrapper object at 0x7f6556ecea30>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>})
params     = []
self       = <django.db.backends.utils.CursorWrapper object at 0x7f6554c7cc10>
sql        = 'ALTER TABLE "api_version" ALTER COLUMN "validation_errors" DROP NOT NULL'

.tox/test/lib/python3.8/site-packages/django/db/backends/utils.py:84: OperationalError

[djarecka]

fyi. I'm having issue with repeating this, but it might have happened when I was trying different tox options and had two tests run in parallel

[waxlamp]

Since you can't reproduce, I'll close the issue, but if you do run into this again, please go ahead and re-open.