llazzaro / django-scheduler

A calendaring app for Django.
BSD 3-Clause "New" or "Revised" License
1.3k stars 393 forks source link

django-scheduler > 0.9.3 migrate throws error on MySQL #517

Closed EsGeh closed 3 years ago

EsGeh commented 3 years ago

Hello devs,

after upgrading from version 0.9.3 to any later version, i get an error when trying to migrate. This is obviously related to the migration 0013 and MySQ...

Output when running migrations:

$ ./manage.py migrate
[...]
Running migrations:
  Applying schedule.0013_auto_20210502_2303...Traceback (most recent call last):
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/mysql/base.py", line 73, in execute
    return self.cursor.execute(query, args)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/cursors.py", line 206, in execute
    res = self._query(query)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/cursors.py", line 319, in _query
    db.query(q)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/connections.py", line 259, in query
    _mysql.connection.query(self, query)
MySQLdb._exceptions.OperationalError: (1833, "Cannot change column 'event_ptr_id': used in a foreign key constraint 'event_calendar_event_eventinfo_id_50769baf_fk_event_cal' of table 'sgtest1.event_calendar_eventinfo_organizer'")

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "./manage.py", line 44, in <module>
    main()
  File "./manage.py", line 40, in main
    execute_from_command_line(sys.argv)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/__init__.py", line 419, in execute_from_command_line
    utility.execute()
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/__init__.py", line 413, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/base.py", line 354, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/base.py", line 398, in execute
    output = self.handle(*args, **options)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/base.py", line 89, in wrapped
    res = handle_func(*args, **kwargs)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/core/management/commands/migrate.py", line 244, in handle
    post_migrate_state = executor.migrate(
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/migrations/executor.py", line 117, in migrate
    state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards
    state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/migrations/executor.py", line 227, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/migrations/migration.py", line 126, in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/migrations/operations/fields.py", line 244, in database_forwards
    schema_editor.alter_field(from_model, from_field, to_field)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/base/schema.py", line 608, in alter_field
    self._alter_field(model, old_field, new_field, old_type, new_type,
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/base/schema.py", line 830, in _alter_field
    self.execute(
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/base/schema.py", line 145, in execute
    cursor.execute(sql, params)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/utils.py", line 66, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/django/db/backends/mysql/base.py", line 73, in execute
    return self.cursor.execute(query, args)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/cursors.py", line 206, in execute
    res = self._query(query)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/cursors.py", line 319, in _query
    db.query(q)
  File "/home/sgtest1/calendar/env/lib64/python3.8/site-packages/MySQLdb/connections.py", line 259, in query
    _mysql.connection.query(self, query)
django.db.utils.OperationalError: (1833, "Cannot change column 'event_ptr_id': used in a foreign key constraint 'event_calendar_event_eventinfo_id_50769baf_fk_event_cal' of table 'sgtest1.event_calendar_eventinfo_organizer'")
Traceback (most recent call last):
  File "./scripts/init.py", line 87, in <module>
    exec_cmd(
  File "./scripts/init.py", line 59, in exec_cmd
    subprocess.run(
  File "/usr/lib64/python3.8/subprocess.py", line 516, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['./manage.py', 'migrate', '--skip-checks']' returned non-zero exit status 1.

System information:

$ mysql --version
mysql  Ver 15.1 Distrib 10.3.31-MariaDB, for Linux (x86_64) using readline 5.1
$ pip freeze
[...]
mysqlclient==2.0.3
sqlparse==0.4.2
uWSGI==2.0.19.1
llazzaro commented 3 years ago

Hi! I will try to take a look into it this week. Thanks for reporting this issue

llazzaro commented 3 years ago

Hi, I'm not able to reproduce this bug. I think you have a model with a foreign key to a scheduler model, can you confirm this?

If you have a model with a foreign key to one scheduler model, you will need to change the type of the foreign key. Please check migration 0013

EsGeh commented 3 years ago

I think you have a model with a foreign key to a scheduler model, can you confirm this?

Yes, definitely. This is in my models.py:

import schedule.models as schedule
...
class EventInfo(schedule.Event):
    ...(other fields)...
    event_ptr = models.OneToOneField(
            schedule.Event,
            on_delete=models.CASCADE,
            parent_link = True,
            primary_key=True,
            related_name="event_info"
    )

(Side note: By the way, if there is a better method to extend django-schedulers base classes, I'd be happy for a hint.)

If you have a model with a foreign key to one scheduler model, you will need to change the type of the foreign key.

What does that mean? What other types of foreign keys are there?

llazzaro commented 3 years ago

Hi, In migration 0013 all models id were upgraded to BigAutoField. When you use FK in MySQL, both columns must be the same type. In your case, your models are using different column type for the event_id (to the one which is an fk to scheduler model). Our migration changed the type and this broke your extension.

I think you are extending scheduler the proper way, this was an unexpected issue.

Hope it helps.

EsGeh commented 3 years ago

Fixing this issue and migrating the database according to the changes introduced by django-scheduler > 0.9.3 was a nightmare. Finally I was able to fix this issue for my project. I am going to share my approach here in the hope of helping others to save time:

Given you have a model that depends on a Model in django-scheduler (python import: schedule):

import schedule.models as schedule
...
class EventInfo(schedule.Event):
    ...(other fields)...
    event_ptr = models.OneToOneField(
            schedule.Event,
            on_delete=models.CASCADE,
            parent_link = True,
            primary_key=True,
            related_name="event_info"
    )

We shall introduce 2 migrations:

your_app/migrations/0020_cut_foreign_keys.py:

from django.db import migrations, models
import django.db.models.deletion

def copy_to_backup(apps, schema_editor):
    EventInfo = apps.get_model('your_app', 'EventInfo')
    EventInfoBackup = apps.get_model('event_calendar', 'EventInfoBackup')
    for x in EventInfo.objects.all():
        copy = EventInfoBackup(
                event_ptr = x.pk,
                # ...other fields...
        )
        # for every ManyToManyField field "m2mField":
        # copy.m2mField.set(
        #         x.m2mField.all()
        # )
        copy.save()
    for x in EventInfo.objects.all():
        x.delete(
                keep_parents = True
        )

def copy_to_backup_reverse(apps, schema_editor):
    EventInfo = apps.get_model('your_app', 'EventInfo')
    EventInfoBackup = apps.get_model('your_app', 'EventInfoBackup')
    EventBase = apps.get_model( 'schedule', 'Event' )
    for copy in EventInfoBackup.objects.all():
        event = EventBase.objects.get(
                pk = copy.event_ptr
        )
        x = EventInfo(
                event_ptr = event,
                start = event.start,
                end = event.end,
                title = event.title,
                description = event.description,
                creator = event.creator,
                created_on = event.created_on,
                updated_on = event.updated_on,
                rule = event.rule,
                end_recurring_period = event.end_recurring_period,
                calendar = event.calendar,
                color_event = event.color_event,
        )
        x.event_ptr = event
        # for every field in EventInfo:
        # x.field = copy.field
        x.save()
        # for every ManyToManyField in EventInfo:
        x.m2mField.set(
                copy.m2mField.all()
        )
        x.save()

class Migration(migrations.Migration):

    dependencies = [
        ('you_app', '0019_...'), # <-- previous migration
        ('schedule', '0012_auto_20191025_1852'),
    ]

    run_before = [
        ('schedule', '0013_auto_20210502_2303'),
    ]

    operations = [
        migrations.CreateModel(
            name='EventInfoBackup',
            fields=[
                ('event_ptr', models.IntegerField(primary_key=True)),
                # ...other fields...
            ],
        ),
        # Copy EventInfos to backup
        migrations.RunPython(
            code = copy_to_backup,
            reverse_code = copy_to_backup_reverse
        ),
        migrations.DeleteModel(
            name = 'EventInfo',
        )
    ]

your_app/migrations/0021_repair_foreign_keys.py:

from django.db import migrations, models
import django.db.models.deletion

def copy_from_backup(apps, schema_editor):
    EventInfo = apps.get_model('your_app', 'EventInfo')
    EventInfoBackup = apps.get_model('your_app', 'EventInfoBackup')
    EventBase = apps.get_model( 'schedule', 'Event' )
    for copy in EventInfoBackup.objects.all():
        event = EventBase.objects.get(
                pk = copy.event_ptr
        )
        x = EventInfo(
                event_ptr = event,
                start = event.start,
                end = event.end,
                title = event.title,
                description = event.description,
                creator = event.creator,
                created_on = event.created_on,
                updated_on = event.updated_on,
                rule = event.rule,
                end_recurring_period = event.end_recurring_period,
                calendar = event.calendar,
                color_event = event.color_event,
        )
        # for every field "field" in EventInfo:
        x.field = copy.field
        x.event_ptr = event
        x.save()
        # for every ManyToManyField "m2mField":
        # x.m2mField.set(
        #         copy.m2mField.all()
        # )
        x.save()

def copy_from_backup_reverse(apps, schema_editor):
    EventInfo = apps.get_model('your_app', 'EventInfo')
    EventInfoBackup = apps.get_model('your_app', 'EventInfoBackup')
    for x in EventInfo.objects.all():
        copy = EventInfoBackup(
                event_ptr = x.pk
                # ...for all fields "field" except ManyToManyFields...
                # field = x.field,
        )
        # for every ManyToManyField "m2mField":
        # copy.m2mField.set(
        #         x.m2mField.all()
        # )
        copy.save()
    for x in EventInfo.objects.all():
        x.delete(
                keep_parents = True
        )

class Migration(migrations.Migration):

    dependencies = [
        ('your_app', '0020_cut_foreign_key'),
        ('schedule', '0014_use_autofields_for_pk'),
    ]

    operations = [
        migrations.CreateModel(
            name='EventInfo',
            fields=[
                ('event_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='event_info', serialize=False, to='schedule.event')),
                # ...other fields in EventInfo...
            ],
            bases=('schedule.event',),
        ),
        # Copy EventInfos from backup
        migrations.RunPython(
            code = copy_from_backup,
            reverse_code = copy_from_backup_reverse
        ),
        migrations.DeleteModel(
            name='EventInfoBackup',
        ),
    ]
llazzaro commented 3 years ago

Thanks for providing that information! I'm closing this issue. I will take a look to the open PR.