issues
search
databrickslabs
/
ucx
Your best companion for upgrading to Unity Catalog. UCX will guide you, the Databricks customer, through the process of upgrading your account, groups, workspaces, jobs etc. to Unity Catalog.
Other
196
stars
70
forks
source link
Test failure: `test_migrate_view`
#2085
Open
github-actions[bot]
opened
1 week ago
github-actions[bot]
commented
1 week ago
❌ test_migrate_view: databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. (1m14.69s)
``` databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. [gw2] linux -- Python 3.10.14 /home/runner/work/ucx/ucx/.venv/bin/python 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema hive_metastore.ucx_s29ky: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_tedby: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_tedby 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=
, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tedby', metastore_id=None, name='ucx_tedby', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location='dbfs:/user/hive/warehouse/ucx_s29ky/ucx_tedby', table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition=None, view_dependencies=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_tm2ju: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_tm2ju 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tm2ju', metastore_id=None, name='ucx_tm2ju', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', view_dependencies=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_twwlh: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_twwlh 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_twwlh', metastore_id=None, name='ucx_twwlh', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tm2ju', view_dependencies=None) 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added catalog fixture: CatalogInfo(browse_only=False, catalog_type=
, comment='', connection_name=None, created_at=1720163110317, created_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs', isolation_mode=
, metastore_id='8952c1e3-b265-4adf-98c3-6f755e2e1453', name='ucx_cbrcs', options=None, owner='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', properties=None, provider_name=None, provisioning_info=None, securable_kind=
, securable_type='CATALOG', share_name=None, storage_location=None, storage_root=None, updated_at=1720163110317, updated_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d') 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema ucx_cbrcs.ucx_s29ky: https://DATABRICKS_HOST/explore/data/ucx_cbrcs/ucx_s29ky 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='ucx_cbrcs', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 INFO [tests.integration.hive_metastore.test_migrate] dst_catalog=ucx_cbrcs, managed_table=hive_metastore.ucx_s29ky.ucx_tedby 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema hive_metastore.ucx_s1hsm: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s1hsm 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s1hsm', metastore_id=None, name='ucx_s1hsm', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] crawling new batch for tables 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky] listing tables 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_tedby] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_tm2ju] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_twwlh] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.view3] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] found 4 new records for tables 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 DEBUG [databricks.labs.ucx.hive_metastore.table_migrate] Migrating managed table hive_metastore.ucx_s29ky.ucx_tedby to ucx_cbrcs.ucx_s29ky.ucx_tedby using SQL query: CREATE TABLE IF NOT EXISTS ucx_cbrcs.ucx_s29ky.ucx_tedby DEEP CLONE hive_metastore.ucx_s29ky.ucx_tedby; 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 INFO [databricks.labs.ucx.hive_metastore.mapping] The intended target for hive_metastore.ucx_s29ky.ucx_tedby, ucx_cbrcs.ucx_s29ky.ucx_tedby, already exists. 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 WARNING [databricks.labs.ucx.hive_metastore.migration_status] Catalog ucx_cuvbn no longer exists. Skipping checking its migration status. 07:05 WARNING [databricks.labs.ucx.hive_metastore.migration_status] Catalog ucx_czfoa no longer exists. Skipping checking its migration status. 07:05 INFO [databricks.labs.ucx.hive_metastore.migration_status] ucx_s29ky.ucx_tedby is set as migrated 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:06 ERROR [databricks.labs.blueprint.parallel] migrate views(ViewToMigrate(src=Table(catalog='hive_metastore', database='ucx_s29ky', name='ucx_tm2ju', object_type='VIEW', table_format='UNKNOWN', location=None, view_text='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', upgraded_to=None, storage_properties='{serialization.format: 1}', is_partitioned=False), rule=Rule(workspace_name='workspace', catalog_name='ucx_cbrcs', src_schema='ucx_s29ky', dst_schema='ucx_s29ky', src_table='ucx_tm2ju', dst_table='ucx_tm2ju', recon_tolerance_percent=0, compare_rows=False)), []) task failed: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. Traceback (most recent call last): File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/blueprint/parallel.py", line 158, in inner return func(*args, **kwargs), None File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 200, in _migrate_view if self._view_can_be_migrated(src_view): File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 208, in _view_can_be_migrated if not self.index_full_refresh().get(table.schema, table.name): File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 65, in index_full_refresh self._migration_status_refresher.reset() File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/framework/crawlers.py", line 55, in reset self._exec(f"DELETE FROM {self.full_name}") File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/backends.py", line 197, in execute self._sql.execute(sql, catalog=catalog, schema=schema) File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/core.py", line 243, in execute self._raise_if_needed(status) File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/core.py", line 480, in _raise_if_needed raise error_class(error_message) databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:06 INFO [databricks.labs.ucx.hive_metastore.migration_status] ucx_s29ky.ucx_tedby is set as migrated 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:06 DEBUG [databricks.labs.ucx.hive_metastore.table_migrate] Migrating view hive_metastore.ucx_s29ky.view3 to using SQL query: CREATE VIEW IF NOT EXISTS ucx_cbrcs.ucx_s29ky.view3 (col1, col2) TBLPROPERTIES ('transient_lastDdlTime'='1720163109') AS SELECT * FROM ucx_cbrcs.ucx_s29ky.ucx_tedby 07:06 ERROR [databricks.labs.blueprint.parallel] More than half 'migrate views' tasks failed: 50% results available (1/2). Took 0:00:13.210795 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema hive_metastore.ucx_s29ky: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_tedby: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_tedby 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=
, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tedby', metastore_id=None, name='ucx_tedby', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location='dbfs:/user/hive/warehouse/ucx_s29ky/ucx_tedby', table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition=None, view_dependencies=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_tm2ju: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_tm2ju 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tm2ju', metastore_id=None, name='ucx_tm2ju', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', view_dependencies=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_twwlh: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_twwlh 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_twwlh', metastore_id=None, name='ucx_twwlh', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tm2ju', view_dependencies=None) 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added catalog fixture: CatalogInfo(browse_only=False, catalog_type=
, comment='', connection_name=None, created_at=1720163110317, created_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs', isolation_mode=
, metastore_id='8952c1e3-b265-4adf-98c3-6f755e2e1453', name='ucx_cbrcs', options=None, owner='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', properties=None, provider_name=None, provisioning_info=None, securable_kind=
, securable_type='CATALOG', share_name=None, storage_location=None, storage_root=None, updated_at=1720163110317, updated_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d') 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema ucx_cbrcs.ucx_s29ky: https://DATABRICKS_HOST/explore/data/ucx_cbrcs/ucx_s29ky 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='ucx_cbrcs', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 INFO [tests.integration.hive_metastore.test_migrate] dst_catalog=ucx_cbrcs, managed_table=hive_metastore.ucx_s29ky.ucx_tedby 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema hive_metastore.ucx_s1hsm: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s1hsm 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s1hsm', metastore_id=None, name='ucx_s1hsm', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] crawling new batch for tables 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky] listing tables 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_tedby] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_tm2ju] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.ucx_twwlh] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.hive_metastore.tables] [hive_metastore.ucx_s29ky.view3] fetching table metadata 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] found 4 new records for tables 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 DEBUG [databricks.labs.ucx.hive_metastore.table_migrate] Migrating managed table hive_metastore.ucx_s29ky.ucx_tedby to ucx_cbrcs.ucx_s29ky.ucx_tedby using SQL query: CREATE TABLE IF NOT EXISTS ucx_cbrcs.ucx_s29ky.ucx_tedby DEEP CLONE hive_metastore.ucx_s29ky.ucx_tedby; 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 INFO [databricks.labs.ucx.hive_metastore.mapping] The intended target for hive_metastore.ucx_s29ky.ucx_tedby, ucx_cbrcs.ucx_s29ky.ucx_tedby, already exists. 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:05 WARNING [databricks.labs.ucx.hive_metastore.migration_status] Catalog ucx_cuvbn no longer exists. Skipping checking its migration status. 07:05 WARNING [databricks.labs.ucx.hive_metastore.migration_status] Catalog ucx_czfoa no longer exists. Skipping checking its migration status. 07:05 INFO [databricks.labs.ucx.hive_metastore.migration_status] ucx_s29ky.ucx_tedby is set as migrated 07:05 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:06 ERROR [databricks.labs.blueprint.parallel] migrate views(ViewToMigrate(src=Table(catalog='hive_metastore', database='ucx_s29ky', name='ucx_tm2ju', object_type='VIEW', table_format='UNKNOWN', location=None, view_text='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', upgraded_to=None, storage_properties='{serialization.format: 1}', is_partitioned=False), rule=Rule(workspace_name='workspace', catalog_name='ucx_cbrcs', src_schema='ucx_s29ky', dst_schema='ucx_s29ky', src_table='ucx_tm2ju', dst_table='ucx_tm2ju', recon_tolerance_percent=0, compare_rows=False)), []) task failed: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. Traceback (most recent call last): File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/blueprint/parallel.py", line 158, in inner return func(*args, **kwargs), None File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 200, in _migrate_view if self._view_can_be_migrated(src_view): File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 208, in _view_can_be_migrated if not self.index_full_refresh().get(table.schema, table.name): File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/hive_metastore/table_migrate.py", line 65, in index_full_refresh self._migration_status_refresher.reset() File "/home/runner/work/ucx/ucx/src/databricks/labs/ucx/framework/crawlers.py", line 55, in reset self._exec(f"DELETE FROM {self.full_name}") File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/backends.py", line 197, in execute self._sql.execute(sql, catalog=catalog, schema=schema) File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/core.py", line 243, in execute self._raise_if_needed(status) File "/home/runner/work/ucx/ucx/.venv/lib/python3.10/site-packages/databricks/labs/lsql/core.py", line 480, in _raise_if_needed raise error_class(error_message) databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] crawling new batch for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.tables] fetching tables inventory 07:06 INFO [databricks.labs.ucx.hive_metastore.migration_status] ucx_s29ky.ucx_tedby is set as migrated 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] found 4 new records for migration_status 07:06 DEBUG [databricks.labs.ucx.framework.crawlers] [hive_metastore.ucx_s1hsm.migration_status] fetching migration_status inventory 07:06 DEBUG [databricks.labs.ucx.hive_metastore.table_migrate] Migrating view hive_metastore.ucx_s29ky.view3 to using SQL query: CREATE VIEW IF NOT EXISTS ucx_cbrcs.ucx_s29ky.view3 (col1, col2) TBLPROPERTIES ('transient_lastDdlTime'='1720163109') AS SELECT * FROM ucx_cbrcs.ucx_s29ky.ucx_tedby 07:06 ERROR [databricks.labs.blueprint.parallel] More than half 'migrate views' tasks failed: 50% results available (1/2). Took 0:00:13.210795 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] clearing 1 catalog fixtures 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing catalog fixture: CatalogInfo(browse_only=False, catalog_type=
, comment='', connection_name=None, created_at=1720163110317, created_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs', isolation_mode=
, metastore_id='8952c1e3-b265-4adf-98c3-6f755e2e1453', name='ucx_cbrcs', options=None, owner='0a330eb5-dd51-4d97-b6e4-c474356b1d5d', properties=None, provider_name=None, provisioning_info=None, securable_kind=
, securable_type='CATALOG', share_name=None, storage_location=None, storage_root=None, updated_at=1720163110317, updated_by='0a330eb5-dd51-4d97-b6e4-c474356b1d5d') 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] clearing 0 workspace group fixtures 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] clearing 0 table fixtures 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] clearing 3 table fixtures 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=
, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tedby', metastore_id=None, name='ucx_tedby', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location='dbfs:/user/hive/warehouse/ucx_s29ky/ucx_tedby', table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition=None, view_dependencies=None) 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tm2ju', metastore_id=None, name='ucx_tm2ju', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', view_dependencies=None) 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] ignoring error while table TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_tm2ju', metastore_id=None, name='ucx_tm2ju', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tedby', view_dependencies=None) teardown: [WRONG_COMMAND_FOR_OBJECT_TYPE] The operation DROP TABLE requires a EXTERNAL or MANAGED. But hive_metastore.ucx_s29ky.ucx_tm2ju is a VIEW. Use DROP VIEW instead. SQLSTATE: 42809 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_twwlh', metastore_id=None, name='ucx_twwlh', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tm2ju', view_dependencies=None) 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] ignoring error while table TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=None, deleted_at=None, delta_runtime_properties_kvpairs=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, encryption_details=None, full_name='hive_metastore.ucx_s29ky.ucx_twwlh', metastore_id=None, name='ucx_twwlh', owner=None, pipeline_id=None, properties={'RemoveAfter': '2024070508'}, row_filter=None, schema_name='ucx_s29ky', sql_path=None, storage_credential_name=None, storage_location=None, table_constraints=None, table_id=None, table_type=
, updated_at=None, updated_by=None, view_definition='SELECT * FROM hive_metastore.ucx_s29ky.ucx_tm2ju', view_dependencies=None) teardown: [WRONG_COMMAND_FOR_OBJECT_TYPE] The operation DROP TABLE requires a EXTERNAL or MANAGED. But hive_metastore.ucx_s29ky.ucx_twwlh is a VIEW. Use DROP VIEW instead. SQLSTATE: 42809 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] clearing 3 schema fixtures 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing schema fixture: SchemaInfo(browse_only=None, catalog_name='ucx_cbrcs', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='ucx_cbrcs.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:06 DEBUG [databricks.labs.ucx.mixins.fixtures] removing schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s1hsm', metastore_id=None, name='ucx_s1hsm', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) [gw2] linux -- Python 3.10.14 /home/runner/work/ucx/ucx/.venv/bin/python ```
Running from
nightly #112
❌ test_migrate_view: databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. (1m14.69s)
``` databricks.sdk.errors.platform.BadRequest: This transaction attempted to read one or more files that were deleted (for example part-00000-817adf46-a46e-45d0-9aea-385540395048.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. Conflicting commit: {"timestamp":1720163161784,"userId":"481119220561874","userName":"0a330eb5-dd51-4d97-b6e4-c474356b1d5d","operation":"DELETE","operationParameters":{"predicate":["true"]},"readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1723","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"8","numDeletionVectorsUpdated":"0","numDeletedRows":"4","scanTimeMs":"8","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"tags":{"noRowsCopied":"true","delta.rowTracking.preserved":"false","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.1.x-photon-scala2.12","txnId":"c045b7d1-7e34-4fad-8881-3fbd72363c2b"} Refer to https://docs.microsoft.com/CLOUD_ENV/databricks/delta/concurrency-control for more details. [gw2] linux -- Python 3.10.14 /home/runner/work/ucx/ucx/.venv/bin/python 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Schema hive_metastore.ucx_s29ky: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added schema fixture: SchemaInfo(browse_only=None, catalog_name='hive_metastore', catalog_type=None, comment=None, created_at=None, created_by=None, effective_predictive_optimization_flag=None, enable_predictive_optimization=None, full_name='hive_metastore.ucx_s29ky', metastore_id=None, name='ucx_s29ky', owner=None, properties=None, schema_id=None, storage_location=None, storage_root=None, updated_at=None, updated_by=None) 07:05 INFO [databricks.labs.ucx.mixins.fixtures] Table hive_metastore.ucx_s29ky.ucx_tedby: https://DATABRICKS_HOST/explore/data/hive_metastore/ucx_s29ky/ucx_tedby 07:05 DEBUG [databricks.labs.ucx.mixins.fixtures] added table fixture: TableInfo(access_point=None, browse_only=None, catalog_name='hive_metastore', columns=None, comment=None, created_at=None, created_by=None, data_access_configuration_id=None, data_source_format=Running from nightly #112