drop partition DDL should not be synchronized immediately

[Jayjlchan]

What did you do?

drop partition DDL should not be synchronized immediately. For example:

-- source and target table tab2 metadata
CREATE TABLE sbtest.tab2 (
    id INT NOT NULL,
    date DATE NOT NULL,
    value VARCHAR(50),
    PRIMARY KEY (id, date)
)
PARTITION BY RANGE (YEAR(date)) (
    PARTITION p0 VALUES LESS THAN (2000),
    PARTITION p1 VALUES LESS THAN (2005),
    PARTITION p2 VALUES LESS THAN (2010),
    PARTITION pmax VALUES LESS THAN MAXVALUE
);

-- mysql1 (source1) 
mysql> ALTER TABLE sbtest.tab2 drop partition pmax;
Query OK, 0 rows affected (0.02 sec)
Records: 0  Duplicates: 0  Warnings: 0

-- tidb (target) , drop partition DDL synchronized immediately
mysql> show create table tab2;

CREATE TABLE `tab2` (
  `id` int(11) NOT NULL,
  `date` date NOT NULL,
  `value` varchar(50) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`id`,`date`) /*T![clustered_index] CLUSTERED */
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
PARTITION BY RANGE (YEAR(`date`))
(PARTITION `p0` VALUES LESS THAN (2000),
 PARTITION `p1` VALUES LESS THAN (2005),
 PARTITION `p2` VALUES LESS THAN (2010))

 -- mysql2 (source2)
 mysql> insert into sbtest.tab2  values(2,'2023-01-01','a');
Query OK, 1 row affected (0.00 sec)

-- tidb (target), DM reports an error and the data is not synchronized
mysql> select * from tab2;
Empty set (0.00 sec)

What did you expect to see?

-- tidb (target)
mysql> select * from tab2;
+----+------------+-------+
| id | date       | value |
+----+------------+-------+
|  2 | 2023-01-01 | a     |
+----+------------+-------+
1 row in set (0.00 sec)

What did you see instead?

-- tidb (target), DM reports an error and the data is not synchronized
mysql> select * from tab2;
Empty set (0.00 sec)

Versions of the cluster

DM version (run dmctl -V or dm-worker -V or dm-master -V):

6.5.0

Upstream MySQL/MariaDB server version:

MySQL 5.7.19

Downstream TiDB cluster version (execute SELECT tidb_version(); in a MySQL client):

6.5.0

How did you deploy DM: tiup or manually?

tiup

Other interesting information (system version, hardware config, etc):

>
>

current status of DM cluster (execute query-status <task-name> in dmctl)

» query-status test650
{
    "result": true,
    "msg": "",
    "sources": [
        {
            "result": true,
            "msg": "",
            "sourceStatus": {
                "source": "mysql-replica-3307",
                "worker": "dm-172.16.201.206-8264",
                "result": null,
                "relayStatus": null
            },
            "subTaskStatus": [
                {
                    "name": "test650",
                    "stage": "Running",
                    "unit": "Sync",
                    "result": null,
                    "unresolvedDDLLockID": "",
                    "sync": {
                        "totalEvents": "18",
                        "totalTps": "0",
                        "recentTps": "0",
                        "masterBinlog": "(bin.000001, 131254)",
                        "masterBinlogGtid": "8bf2dbc3-b1c1-11ed-ae6a-fa163e3df2cf:1-566",
                        "syncerBinlog": "(bin.000001, 131254)",
                        "syncerBinlogGtid": "8bf2dbc3-b1c1-11ed-ae6a-fa163e3df2cf:1-566",
                        "blockingDDLs": [
                        ],
                        "unresolvedGroups": [
                        ],
                        "synced": true,
                        "binlogType": "remote",
                        "secondsBehindMaster": "0",
                        "blockDDLOwner": "",
                        "conflictMsg": "",
                        "totalRows": "18",
                        "totalRps": "0",
                        "recentRps": "0"
                    },
                    "validation": null
                }
            ]
        },
        {
            "result": true,
            "msg": "",
            "sourceStatus": {
                "source": "mysql-replica-3308",
                "worker": "dm-172.16.201.186-8264",
                "result": null,
                "relayStatus": null
            },
            "subTaskStatus": [
                {
                    "name": "test650",
                    "stage": "Paused",
                    "unit": "Sync",
                    "result": {
                        "isCanceled": false,
                        "errors": [
                            {
                                "ErrCode": 10006,
                                "ErrClass": "database",
                                "ErrScope": "not-set",
                                "ErrLevel": "high",
                                "Message": "startLocation: [position: (bin.000001, 108766), gtid-set: 2bb857c6-b1c3-11ed-983c-fa163e3df2cf:1-474], endLocation: [position: (bin.000001, 108811), gtid-set: 2bb857c6-b1c3-11ed-983c-fa163e3df2cf:1-474]: execute statement failed: REPLACE INTO `sbtest_sbtest`.`tab2` (`id`,`date`,`value`) VALUES (?,?,?)",
                                "RawCause": "Error 1526: Table has no partition for value 2023",
                                "Workaround": ""
                            }
                        ],
                        "detail": null
                    },
                    "unresolvedDDLLockID": "",
                    "sync": {
                        "totalEvents": "22",
                        "totalTps": "0",
                        "recentTps": "0",
                        "masterBinlog": "(bin.000001, 108842)",
                        "masterBinlogGtid": "2bb857c6-b1c3-11ed-983c-fa163e3df2cf:1-474",
                        "syncerBinlog": "(bin.000001, 108569)",
                        "syncerBinlogGtid": "2bb857c6-b1c3-11ed-983c-fa163e3df2cf:1-473",
                        "blockingDDLs": [
                        ],
                        "unresolvedGroups": [
                        ],
                        "synced": false,
                        "binlogType": "remote",
                        "secondsBehindMaster": "0",
                        "blockDDLOwner": "",
                        "conflictMsg": "",
                        "totalRows": "22",
                        "totalRps": "0",
                        "recentRps": "0"
                    },
                    "validation": null
                }
            ]
        }
    ]
}

[lance6716]

please check if you use shard-mode: "pessimistic" in your task configuration file

[Jayjlchan]

shard-mode: "optimistic"

[fubinzh]

/remove-type bug