geek-frio
commented
1 year ago The check failed code is here:
const PUBLISH_SPLITS_QUERY: &str = r#"
-- Select the splits to update, regardless of their state.
-- The left join make it possible to identify the splits that do not exist.
WITH input_splits AS (
SELECT input_splits.split_id, input_splits.expected_split_state, splits.actual_split_state
FROM (
SELECT split_id, 'Staged' AS expected_split_state
FROM UNNEST($3) AS staged_splits(split_id)
UNION
SELECT split_id, 'Published' AS expected_split_state
FROM UNNEST($4) AS published_splits(split_id)
) input_splits
LEFT JOIN (
SELECT split_id, split_state AS actual_split_state
FROM splits
WHERE
index_uid = $1
AND (split_id = ANY($3) OR split_id = ANY($4))
FOR UPDATE
) AS splits
USING (split_id)
),
-- Update the index metadata with the new checkpoint.
updated_index_metadata AS (
UPDATE indexes
SET
index_metadata_json = $2
WHERE
index_uid = $1
AND NOT EXISTS (
SELECT 1
FROM input_splits
WHERE
actual_split_state != expected_split_state
)
),
-- Publish the staged splits and mark the published splits for deletion.
updated_splits AS (
UPDATE splits
SET
split_state = CASE split_state
WHEN 'Staged' THEN 'Published'
ELSE 'MarkedForDeletion'
END,
update_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'),
publish_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
FROM input_splits
WHERE
splits.index_uid = $1
AND splits.split_id = input_splits.split_id
AND NOT EXISTS (
SELECT 1
FROM input_splits
WHERE
actual_split_state != expected_split_state
)
)
-- Report the outcome of the update query.
SELECT
COUNT(1) FILTER (WHERE actual_split_state = 'Staged' AND expected_split_state = 'Staged'),
COUNT(1) FILTER (WHERE actual_split_state = 'Published' AND expected_split_state = 'Published'),
COALESCE(ARRAY_AGG(split_id) FILTER (WHERE actual_split_state IS NULL), ARRAY[]::TEXT[]),
COALESCE(ARRAY_AGG(split_id) FILTER (WHERE actual_split_state != 'Staged' AND expected_split_state = 'Staged'), ARRAY[]::TEXT[]),
COALESCE(ARRAY_AGG(split_id) FILTER (WHERE actual_split_state != 'Published' AND expected_split_state = 'Published'), ARRAY[]::TEXT[])
FROM input_splits
"#;
let (
num_published_splits,
num_marked_splits,
not_found_split_ids,
not_staged_split_ids,
not_marked_split_ids,
): (i64, i64, Vec<String>, Vec<String>, Vec<String>) =
sqlx::query_as(PUBLISH_SPLITS_QUERY)
.bind(index_uid.to_string())
.bind(index_metadata_json)
.bind(staged_split_ids)
.bind(replaced_split_ids)
.fetch_one(tx)
.await
.map_err(|error| convert_sqlx_err(index_uid.index_id(), error))?;
if !not_found_split_ids.is_empty() {
return Err(MetastoreError::SplitsDoNotExist {
split_ids: not_found_split_ids,
});
}
if !not_staged_split_ids.is_empty() {
return Err(MetastoreError::SplitsNotStaged {
split_ids: not_staged_split_ids,
});
}
if !not_marked_split_ids.is_empty() {
return Err(MetastoreError::SplitsNotDeletable {
split_ids: not_marked_split_ids,
});
}Expected split status is Published, but after first merge publish operation, the split status is already changed to MarkedDelete status.
fulmicoton
fmassot
Describe the bug We encountered the following exception error in our production environment:
After analyzing the logs, I found that PlanMerge operation was triggered twice, which ultimately caused the failure of the execution of the publish_split method for the second time. This is because one of the splits in the replaced_split has already been marked as MarkedDelete, which doesn't match the expected Published state. For example, this split（01H569VFSJZ9D7FT2NPV7J6JYZ） appeared in both PlanMerge occurrences.
This ultimately led to the failure of the subsequent publish_splits operation. After analyzing the code, I found that the triggering of merge operations occurred in two actions: one is the automatically scheduled PlanMerge, and the other is the triggering after the creation of NewSplit. Is it possible that the same split was selected twice for PlanMerge, resulting in this issue?
My quickwit version is v0.6.2.