Closed yeqing2020 closed 6 months ago
I was having unstable connection to kusto with kqlmagic -- failure rate is extremely high. With the same query I could get 100% success rate if I run it directly in Kusto.
Restarted kennel, add -!se and generate bug report:
{ "Kqlmagic_last_execution": { "args": { "cell": null, "line": " -!se -query experiment_id_query", "override_connection": null, "override_options": null, "override_query_properties": null, "override_result_set": null, "override_vars": null }, "connection": "TestMetrics_Prod@...._westus2", "log": [ "Kqlmagic_core::execute - input: \n\rline: -!se -query experiment_id_query\n\rcell:\n\rNone", "Kqlmagic_core::execute - parsed_queries: [{'connection_string': '', 'query': '\nExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal\n', 'options': {'auto_dataframe': False, 'short_errors': False, 'feedback': True, 'show_conn_info': None, 'columns_to_local_vars': False, 'show_query_time': False, 'show_query': False, 'show_query_link': False, 'query_link_destination': 'Kusto.WebExplorer', 'enable_suppress_result': True, 'plotly_fs_includejs': False, 'popup_window': False, 'auto_limit': 0, 'display_limit': None, 'timeout': None, 'prettytable_style': 'DEFAULT', 'last_raw_result_var': '_kql_rawresult', 'table_package': 'prettytable', 'plot_package': 'plotly', 'dsn_filename': 'odbc.ini', 'validate_connection_string': True, 'auto_popup_schema': True, 'json_display': 'formatted', 'schema_json_display': 'auto', 'palette_desaturation': 1.0, 'params_dict': None, 'palette_name': 'tab10', 'cache': None, 'use_cache': None, 'temp_folder_name': 'temp_files', 'cache_folder_name': 'cache_files', 'export_folder_name': 'exported_files', 'add_kql_ref_to_help': True, 'add_schema_to_help': True, 'enable_add_items_to_help': True, 'notebook_app': 'visualstudiocode', 'debug': False, 'check_magic_version': True, 'show_what_new': True, 'show_init_banner': True, 'is_kernel_intializtion': False, 'warn_missing_dependencies': True, 'warn_missing_env_variables': True, 'allow_single_line_cell': True, 'allow_py_comments_before_cell': True, 'kqlmagic_kernel': False, 'extras_require': 'default', 'test_notebook_app': 'none', 'cloud': 'public', 'enable_sso': False, 'sso_db_gc_interval': 168, 'auth_use_http_client': False, 'try_azcli_login': False, 'try_azcli_login_by_profile': False, 'try_vscode_login': False, 'try_azcli_login_subscription': None, 'try_token': None, 'try_msi': None, 'request_id_tag': None, 'request_app_tag': None, 'request_user_tag': None, 'request_user_agent_tag': None, 'request_cache_max_age': 0, 'device_code_login_notification': 'auto', 'device_code_notification_email': '', 'save_as': None, 'save_to': None, 'query_properties': {}, 'palette_colors': 10, 'palette_reverse': False, 'popup_schema': False, 'display_id': False, 'display_handlers': {'acquire_token': None}, 'popup_interaction': 'auto', 'temp_files_server': 'auto', 'temp_files_server_address': 'http://127.0.0.1:49589', 'kernel_location': 'local', 'kernel_id': 'v2-5404JMa8IncvBvt8', 'notebook_service_address': None, 'dynamic_to_dataframe': 'object', 'temp_folder_location': 'user_dir', 'plotly_layout': None, 'auth_token_warnings': False, 'enable_curly_brackets_params': False, 'nop': False, 'assign_var': None, 'cursor_var': None, 'is_magic': True, 'code_auth_interactive_mode': 'device_code'}, 'command': {}, 'last_query': True}]", "Account(s) exists in cache, probably with token too. Let's try.", "Account(s) exists in cache, probably with token too. Let's try.", "KustoClient::execute - POST request - url: https://....westus2.kusto.windows.net/v2/rest/query, headers: {'User-Agent': 'Kqlmagic/0.1.114.post22', 'x-ms-client-version': 'Kqlmagic.Python.Client:0.1.114.post22', 'x-ms-client-request-id': 'Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', 'x-ms-app': 'Kqlmagic;visualstudiocode', 'Authorization': '...', 'Fed': 'True', 'Cache-Control': 'no-cache'}, payload: {'db': 'TestMetrics_Prod', 'csl': 'ExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal'}, timeout: None", "POST Request: https://.........westus2.kusto.windows.net/v2/rest/query, params: None, payload: {'db': 'TestMetrics_Prod', 'csl': 'ExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal'}, headers: {'User-Agent': 'Kqlmagic/0.1.114.post22', 'x-ms-client-version': 'Kqlmagic.Python.Client:0.1.114.post22', 'x-ms-client-request-id': 'Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', 'x-ms-app': 'Kqlmagic;visualstudiocode', 'Authorization': 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IlhSdmtvOFA3QTNVYVdTblU3Yk05blQwTWpoQSIsImtp.......', 'Fed': 'True', 'Cache-Control': 'no-cache', 'Content-Type': 'application/json'}", "POST Response: 400 b\"General_BadRequest: Request is invalid and cannot be executed.\r\nError details:\r\nClientRequestId='Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', ActivityId='41a6cc0f-bba7-48b9-bbd6-b36b3222eba1', Timestamp='2024-03-18T17:14:00.4438221Z'.\"", "KustoClient::execute - response - status: 400, headers: {'Date': 'Mon, 18 Mar 2024 17:14:00 GMT', 'Content-Type': 'text/plain', 'Server': 'Kestrel', 'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Transfer-Encoding': 'chunked', 'Content-Encoding': 'gzip', 'Expires': '-1', 'Vary': 'Accept-Encoding', 'Strict-Transport-Security': 'max-age=2592000; includeSubDomains', 'x-ms-client-request-id': 'Kqlmagic.execute...'}, payload: General_BadRequest: Request is invalid and cannot be executed.\r\nError details:\r\nClientRequestId='Kqlmagic.execute.../AzureDataExplorer', ActivityId='...', Timestamp='2024-03-18T17:14:00.4438221Z'." ], "query": { "request": { "endpoint": "https://..........westus2.kusto.windows.net/v2/rest/query", ... "branch": "", "name": "CPython", "version": "3.11.0" } }
You need to check with the Kusto team. looks like something is not working properly in your cluster.
I was having unstable connection to kusto with kqlmagic -- failure rate is extremely high. With the same query I could get 100% success rate if I run it directly in Kusto.
Restarted kennel, add -!se and generate bug report:
{ "Kqlmagic_last_execution": { "args": { "cell": null, "line": " -!se -query experiment_id_query", "override_connection": null, "override_options": null, "override_query_properties": null, "override_result_set": null, "override_vars": null }, "connection": "TestMetrics_Prod@...._westus2", "log": [ "Kqlmagic_core::execute - input: \n\rline: -!se -query experiment_id_query\n\rcell:\n\rNone", "Kqlmagic_core::execute - parsed_queries: [{'connection_string': '', 'query': '\nExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal\n', 'options': {'auto_dataframe': False, 'short_errors': False, 'feedback': True, 'show_conn_info': None, 'columns_to_local_vars': False, 'show_query_time': False, 'show_query': False, 'show_query_link': False, 'query_link_destination': 'Kusto.WebExplorer', 'enable_suppress_result': True, 'plotly_fs_includejs': False, 'popup_window': False, 'auto_limit': 0, 'display_limit': None, 'timeout': None, 'prettytable_style': 'DEFAULT', 'last_raw_result_var': '_kql_rawresult', 'table_package': 'prettytable', 'plot_package': 'plotly', 'dsn_filename': 'odbc.ini', 'validate_connection_string': True, 'auto_popup_schema': True, 'json_display': 'formatted', 'schema_json_display': 'auto', 'palette_desaturation': 1.0, 'params_dict': None, 'palette_name': 'tab10', 'cache': None, 'use_cache': None, 'temp_folder_name': 'temp_files', 'cache_folder_name': 'cache_files', 'export_folder_name': 'exported_files', 'add_kql_ref_to_help': True, 'add_schema_to_help': True, 'enable_add_items_to_help': True, 'notebook_app': 'visualstudiocode', 'debug': False, 'check_magic_version': True, 'show_what_new': True, 'show_init_banner': True, 'is_kernel_intializtion': False, 'warn_missing_dependencies': True, 'warn_missing_env_variables': True, 'allow_single_line_cell': True, 'allow_py_comments_before_cell': True, 'kqlmagic_kernel': False, 'extras_require': 'default', 'test_notebook_app': 'none', 'cloud': 'public', 'enable_sso': False, 'sso_db_gc_interval': 168, 'auth_use_http_client': False, 'try_azcli_login': False, 'try_azcli_login_by_profile': False, 'try_vscode_login': False, 'try_azcli_login_subscription': None, 'try_token': None, 'try_msi': None, 'request_id_tag': None, 'request_app_tag': None, 'request_user_tag': None, 'request_user_agent_tag': None, 'request_cache_max_age': 0, 'device_code_login_notification': 'auto', 'device_code_notification_email': '', 'save_as': None, 'save_to': None, 'query_properties': {}, 'palette_colors': 10, 'palette_reverse': False, 'popup_schema': False, 'display_id': False, 'display_handlers': {'acquire_token': None}, 'popup_interaction': 'auto', 'temp_files_server': 'auto', 'temp_files_server_address': 'http://127.0.0.1:49589', 'kernel_location': 'local', 'kernel_id': 'v2-5404JMa8IncvBvt8', 'notebook_service_address': None, 'dynamic_to_dataframe': 'object', 'temp_folder_location': 'user_dir', 'plotly_layout': None, 'auth_token_warnings': False, 'enable_curly_brackets_params': False, 'nop': False, 'assign_var': None, 'cursor_var': None, 'is_magic': True, 'code_auth_interactive_mode': 'device_code'}, 'command': {}, 'last_query': True}]", "Account(s) exists in cache, probably with token too. Let's try.", "Account(s) exists in cache, probably with token too. Let's try.", "KustoClient::execute - POST request - url: https://....westus2.kusto.windows.net/v2/rest/query, headers: {'User-Agent': 'Kqlmagic/0.1.114.post22', 'x-ms-client-version': 'Kqlmagic.Python.Client:0.1.114.post22', 'x-ms-client-request-id': 'Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', 'x-ms-app': 'Kqlmagic;visualstudiocode', 'Authorization': '...', 'Fed': 'True', 'Cache-Control': 'no-cache'}, payload: {'db': 'TestMetrics_Prod', 'csl': 'ExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal'}, timeout: None", "POST Request: https://.........westus2.kusto.windows.net/v2/rest/query, params: None, payload: {'db': 'TestMetrics_Prod', 'csl': 'ExecutionTelemetryIngestionProd01\n| extend experiment = customDimensions.experimentn\n| extend experimentId = tostring(experiment.id)\n| extend executionGoal = tostring(experiment.metadata.executionGoal)\n| extend experimentStatus = tostring(experiment.status)\n| where executionGoal in (\'local-disk-single-0312\', \'local-disk-multi-0313\')\n| where experimentStatus in (\"Succeeded\", \"InProgress\")\n| distinct executionGoal, experimentId, experimentStatus\n| order by executionGoal'}, headers: {'User-Agent': 'Kqlmagic/0.1.114.post22', 'x-ms-client-version': 'Kqlmagic.Python.Client:0.1.114.post22', 'x-ms-client-request-id': 'Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', 'x-ms-app': 'Kqlmagic;visualstudiocode', 'Authorization': 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IlhSdmtvOFA3QTNVYVdTblU3Yk05blQwTWpoQSIsImtp.......', 'Fed': 'True', 'Cache-Control': 'no-cache', 'Content-Type': 'application/json'}", "POST Response: 400 b\"General_BadRequest: Request is invalid and cannot be executed.\r\nError details:\r\nClientRequestId='Kqlmagic.execute;02ffc0ef-5c23-4801-82fb-e5975dfabb6f/39c93ea0-580e-49b2-8d6e-8187ec8862ab/AzureDataExplorer', ActivityId='41a6cc0f-bba7-48b9-bbd6-b36b3222eba1', Timestamp='2024-03-18T17:14:00.4438221Z'.\"", "KustoClient::execute - response - status: 400, headers: {'Date': 'Mon, 18 Mar 2024 17:14:00 GMT', 'Content-Type': 'text/plain', 'Server': 'Kestrel', 'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Transfer-Encoding': 'chunked', 'Content-Encoding': 'gzip', 'Expires': '-1', 'Vary': 'Accept-Encoding', 'Strict-Transport-Security': 'max-age=2592000; includeSubDomains', 'x-ms-client-request-id': 'Kqlmagic.execute...'}, payload: General_BadRequest: Request is invalid and cannot be executed.\r\nError details:\r\nClientRequestId='Kqlmagic.execute.../AzureDataExplorer', ActivityId='...', Timestamp='2024-03-18T17:14:00.4438221Z'." ], "query": { "request": { "endpoint": "https://..........westus2.kusto.windows.net/v2/rest/query", ... "branch": "", "name": "CPython", "version": "3.11.0" } }