Added a unit test to illustrate the problem: our errors don't pickle correctly. TODO: fix
Failing tests before:
/mnt/workdisk/iamroot/llm-foundry/tests/utils/test_exceptions.py::test_exception_serialization failed: tests/utils/test_exceptions.py:72: in test_exception_serialization
raise AssertionError(
E AssertionError: Failed to serialize/deserialize the following exceptions: dict_keys(['MissingHuggingFaceURLSplitError', 'NotEnoughDatasetSamplesError', 'UnknownExampleTypeError', 'NotEnoughChatDataError', 'ConsecutiveRepeatedChatRolesError', 'InvalidLastChatMessageRoleError', 'IncorrectMessageKeyQuantityError', 'InvalidRoleError', 'InvalidContentTypeError', 'InvalidPromptTypeError', 'InvalidResponseTypeError', 'InvalidPromptResponseKeysError', 'InvalidFileExtensionError', 'UnableToProcessPromptResponseError', 'ClusterDoesNotExistError', 'FailedToCreateSQLConnectionError', 'FailedToConnectToDatabricksError', 'InputFolderMissingDataError', 'OutputFolderNotEmptyError', 'MisconfiguredHfDatasetError', 'RunTimeoutError'])
E
E {
E "MissingHuggingFaceURLSplitError": "MissingHuggingFaceURLSplitError.__init__() takes 1 positional argument but 2 were given",
E "NotEnoughDatasetSamplesError": "NotEnoughDatasetSamplesError.__init__() missing 5 required positional arguments: 'split', 'dataloader_batch_size', 'world_size', 'full_dataset_size', and 'minimum_dataset_size'",
E "UnknownExampleTypeError": "assert '\"Found keys ...'messages\\'}\"' == '\"Found keys ...'messages\\'}\"'\n - \"Found keys Found keys my_keys in dataset. Unknown example type. For prompt and response finetuning, the valid prompt keys are {'prompt'} and the valid response keys are {'response', 'completion'}. For chat finetuning, the allowed keys are {'messages'} in dataset. Unknown example type. For prompt and response finetuning, the valid prompt keys are {'prompt'} and the valid response keys are {'response', 'completion'}. For chat finetuning, the allowed keys are {'messages'}\"\n + \"Found keys my_keys in dataset. Unknown example type. For prompt and response finetuning, the val...\n \n ...Full output truncated (1 line hidden), use '-vv' to show",
E "NotEnoughChatDataError": "NotEnoughChatDataError.__init__() takes 1 positional argument but 2 were given",
E "ConsecutiveRepeatedChatRolesError": "assert 'Conversation...onsecutively.' == 'Conversation...onsecutively.'\n Skipping 34 identical leading characters in diff, use -v to show\n - but found Conversation roles must alternate but found role repeated consecutively. repeated consecutively.\n + but found role repeated consecutively.",
E "InvalidLastChatMessageRoleError": "InvalidLastChatMessageRoleError.__init__() missing 1 required positional argument: 'expected_roles'",
E "IncorrectMessageKeyQuantityError": "assert 'Expected 2 k..., but found 2' == 'Expected 2 k... but found 39'\n - Expected 2 keys in message, but found 39\n ? ^^\n + Expected 2 keys in message, but found 2\n ? ^",
E "InvalidRoleError": "InvalidRoleError.__init__() missing 1 required positional argument: 'valid_roles'",
E "InvalidContentTypeError": "assert 'Expected con...d typing.Dict' == 'Expected con...d typing.Dict'\n Skipping 33 identical leading characters in diff, use -v to show\n - but found Expected content to be a string, but found typing.Dict\n + but found typing.Dict",
E "InvalidPromptTypeError": "assert 'Expected pro...d typing.Dict' == 'Expected pro...d typing.Dict'\n - Expected prompt to be a string, but found Expected prompt to be a string, but found typing.Dict\n + Expected prompt to be a string, but found typing.Dict",
E "InvalidResponseTypeError": "assert 'Expected res...d typing.Dict' == 'Expected res...d typing.Dict'\n Skipping 34 identical leading characters in diff, use -v to show\n - but found Expected response to be a string, but found typing.Dict\n + but found typing.Dict",
E "InvalidPromptResponseKeysError": "InvalidPromptResponseKeysError.__init__() missing 1 required positional argument: 'example'",
E "InvalidFileExtensionError": "InvalidFileExtensionError.__init__() missing 1 required positional argument: 'valid_extensions'",
E "UnableToProcessPromptResponseError": "assert \"Unable to ex...: 'response'}\" == \"Unable to ex...: 'response'}\"\n - Unable to extract prompt/response from Unable to extract prompt/response from {'prompt': 'response'}\n ? ---------------------------------------\n + Unable to extract prompt/response from {'prompt': 'response'}",
E "ClusterDoesNotExistError": "assert 'Cluster with...nd try again!' == 'Cluster with...nd try again!'\n - Cluster with id Cluster with id cluster_name does not exist. Check cluster id and try again! does not exist. Check cluster id and try again!\n + Cluster with id cluster_name does not exist. Check cluster id and try again!",
E "FailedToCreateSQLConnectionError": "FailedToCreateSQLConnectionError.__init__() takes 1 positional argument but 2 were given",
E "FailedToConnectToDatabricksError": "FailedToConnectToDatabricksError.__init__() takes 1 positional argument but 2 were given",
E "InputFolderMissingDataError": "assert 'No text file...nd at folder.' == 'No text file...d at folder..'\n - No text files were found at No text files were found at folder..\n + No text files were found at folder.",
E "OutputFolderNotEmptyError": "assert 'folder is no...it and retry.' == 'folder is no...it and retry.'\n Skipping 46 identical leading characters in diff, use -v to show\n - and retry. is not empty. Please remove or empty it and retry.\n + and retry.",
E "MisconfiguredHfDatasetError": "MisconfiguredHfDatasetError.__init__() missing 1 required positional argument: 'split'",
E "RunTimeoutError": "assert 'Run timed ou... 100 seconds.' == 'Run timed ou...nds. seconds.'\n - Run timed out after Run timed out after 100 seconds. seconds.\n + Run timed out after 100 seconds."
E }
Failing tests before:
Failing tests after: