jayvynl / django-clickhouse-backend

Django clickhouse database backend.
MIT License
130 stars 21 forks source link

[BUG] partition by single expression raise TypeError #50

Closed jayvynl closed 1 year ago

jayvynl commented 1 year ago

Describe the bug partition via toYYYYMMDD raise TypeError: Value after * must be an iterable, not toYYYYMMDD.

To Reproduce

from clickhouse_backend import models

class Event(models.ClickhouseModel):
    ip = models.GenericIPAddressField(default='::')
    ipv4 = models.IPv4Field(default="127.0.0.1")
    ip_nullable = models.GenericIPAddressField(null=True)
    port = models.UInt16Field(default=0)
    protocol = models.StringField(default='', low_cardinality=True)
    content = models.JSONField()
    timestamp = models.DateTime64Field()

    class Meta:
        engine = models.ReplacingMergeTree(
            order_by=['id'],
            partition_by=models.toYYYYMMDD('timestamp')
        )

Exception raised when migrate.

Expected behavior No exception.

Versions

jayvynl commented 1 year ago

temporary, set engine as list type:

partition_by=[models.toYYYYMMDD('timestamp')]
Song-yes commented 1 year ago

hello,Is this question solved? I'm having the same issue. And, I used your method, but it still didn't work, and I still reported an error: (venv) PS D:\source\drf_api> python manage.py makemigrations No changes detected (venv) PS D:\source\drf_api> python manage.py migrate Operations to perform: Apply all migrations: admin, api01, auth, contenttypes, sessions Running migrations: Applying api01.0004_sptj_user_delete_mymodel_sptj_user_port_range...Traceback (most recent call last): File "D:\source\drf_api\manage.py", line 22, in <module> main() File "D:\source\drf_api\manage.py", line 18, in main execute_from_command_line(sys.argv) File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\__init__.py", line 442, in execute_from_command_line utility.execute() File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\__init__.py", line 436, in execute self.fetch_command(subcommand).run_from_argv(self.argv) File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\base.py", line 412, in run_from_argv self.execute(*args, **cmd_options) File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\base.py", line 458, in execute output = self.handle(*args, **options) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\base.py", line 106, in wrapper res = handle_func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\django\core\management\commands\migrate.py", line 356, in handle post_migrate_state = executor.migrate( ^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\django\db\migrations\executor.py", line 135, in migrate state = self._migrate_all_forwards( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\django\db\migrations\executor.py", line 167, in _migrate_all_forwards state = self.apply_migration( ^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\django\db\migrations\executor.py", line 252, in apply_migration state = migration.apply(state, schema_editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\clickhouse_backend\patch\migrations.py", line 170, in apply operation.database_forwards( File "D:\source\drf_api\venv\Lib\site-packages\django\db\migrations\operations\models.py", line 96, in database_forwards schema_editor.create_model(model) File "D:\source\drf_api\venv\Lib\site-packages\django\db\backends\base\schema.py", line 448, in create_model sql, params = self.table_sql(model) ^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\clickhouse_backend\backend\schema.py", line 144, in table_sql "extra": " ".join(extra_parts), ^^^^^^^^^^^^^^^^^^^^^ File "D:\source\drf_api\venv\Lib\site-packages\clickhouse_backend\backend\schema.py", line 258, in _model_extra_sql yield "PARTITION BY (%s)" % self._get_expression(model, *partition_by) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: Value after * must be an iterable, not Func `class sptj_user(models.ClickhouseModel): class Action(IntegerChoices): PASS = 1 DROP = 2 ALERT = 3 user_id = models.UInt64Field(primary_key=True, verbose_name="用户ID") user_name = models.StringField(max_length=50, verbose_name="用户名称") user_mail = models.StringField(max_length=50, null=True, verbose_name="用户邮箱") user_mobile = models.StringField(max_length=20, verbose_name="用户手机") login_passwd = models.StringField(max_length=30, verbose_name="登录密码") login_time = models.DateTime64Field(default=timezone.now) reg_date = models.DateTimeField(auto_now_add=30, verbose_name="注册时间") modify_time = models.DateTimeField(auto_now=True, verbose_name="修改时间") user_status = models.StringField(max_length=100, verbose_name="用户状态") action = models.EnumField(choices=Action.choices, default=Action.PASS)

class Meta:
    db_table = "sptj_user"
    verbose_name = "用户表"
    ordering = ["-user_id"]
    engine = models.ReplacingMergeTree(
        order_by=["user_id"],
        # partition_by=Func("login_time", function="toYYYYMMDD"),
        partition_by=[models.toYYYYMMDD('login_time')],
        index_granularity=8192,
        index_granularity_bytes=1 << 20,
        enable_mixed_granularity_parts=1,
    )`

My Versions

ClickHouse server version 23.6.2.18 (official build). Python 3.11.0 clickhouse-driver 0.2.6 clickhouse-pool 0.5.3 Django 4.2.5 django-clickhouse-backend 1.1.1 djangorestframework 3.14.0

jayvynl commented 1 year ago

@Song-yes delete your migration files, and run makemigration again.

Song-yes commented 1 year ago

I see, thank you for your guidance!