Introduce support for reading single files in LocalStorageClient

lucasgml commented 2 weeks ago

The method read_blobs() from LocalStorageclient returned an error if blob_path pointed to an file. This PR aims to add support for reading single files without changing current support for reading directories.

Also including support for filter_predicate in both read_blobs() and list_blobs() for LocalStorageClient()

github-actions[bot] commented 2 weeks ago

Coverage Report

File	Stmts	Miss	Cover	Missing
adapta/connectors/service_bus
__init__.py	1	1	0%	18
_connector.py	17	17	0%	19–61
adapta/logs
_async_logger.py	81	3	96%	55, 78–79
_base.py	55	6	89%	35, 41, 44, 94–97
_internal_logger.py	109	6	94%	276–286
adapta/logs/handlers
datadog_api_handler.py	109	31	72%	89, 106–113, 124, 136–152, 161–195, 204, 210, 238
adapta/metrics/providers
datadog_provider.py	43	43	0%	19–147
adapta/ml
__init__.py	1	1	0%	19
_model.py	10	10	0%	17–42
adapta/ml/mlflow
__init__.py	2	2	0%	17–18
_client.py	42	42	0%	19–142
_functions.py	42	42	0%	17–111
adapta/process_communication
_models.py	33	6	82%	90–96
adapta/security/clients
__init__.py	26	12	54%	27–28, 34–35, 41–42, 48–49, 53–54, 58–59
_azure_client.py	73	50	32%	42, 55–65, 75–78, 81, 84–86, 95–153, 156, 159–197
adapta/security/clients/aws
_aws_client.py	38	17	55%	37–40, 47, 57, 63, 75, 100–108, 114–117
_aws_credentials.py	73	30	59%	60–79, 83, 87, 91, 95, 99, 108–112, 116, 120, 124, 128, 132
adapta/security/clients/hashicorp_vault
hashicorp_vault_client.py	31	4	87%	46, 87, 91, 95
kubernetes_client.py	21	5	76%	45–48, 67–68
oidc_client.py	45	20	56%	33–62, 80–83, 92
token_client.py	17	7	59%	42–45, 52–53, 56, 59
adapta/storage/blob
azure_storage_client.py	118	50	58%	71–78, 88, 95–96, 99–105, 127–128, 131–156, 159, 175–188, 196–200, 219, 240–242, 251, 267–271, 281–283, 286–305, 312
local_storage_client.py	54	7	87%	42, 67, 74, 87, 101, 104, 107
s3_storage_client.py	117	77	34%	57, 61, 68–80, 91–96, 117–125, 133–134, 148–165, 181–185, 203–212, 222–243, 254–275
adapta/storage/cache
redis_cache.py	37	37	0%	19–107
adapta/storage/database/v2
azure_sql.py	34	34	0%	21–140
odbc.py	73	73	0%	21–219
snowflake_sql.py	69	69	0%	6–228
trino_sql.py	39	39	0%	21–127
adapta/storage/database/v2/models
__init__.py	1	1	0%	19
_models.py	11	11	0%	20–54
adapta/storage/database/v3
azure_sql.py	32	20	38%	55, 70, 84–122, 132
odbc.py	72	11	85%	94–104, 114, 125–126, 140, 149–155, 182
snowflake_sql.py	77	23	70%	64–78, 93–95, 106–115, 142, 178, 193–196
trino_sql.py	38	38	0%	20–119
adapta/storage/delta_lake/v2
_functions.py	68	45	34%	73–106, 153–168, 210–294
adapta/storage/delta_lake/v3
_functions.py	66	12	82%	72, 156, 161, 163, 165, 221, 231, 241–251, 257, 292
adapta/storage/distributed_object_store/v2/datastax_astra
_models.py	19	9	53%	54–59, 62, 65, 68
astra_client.py	196	140	29%	40–43, 113–137, 143–184, 190–191, 197–198, 201, 211, 223–224, 233, 273–361, 369, 393–520, 530, 541–556, 579–594, 616–632, 656–673
adapta/storage/distributed_object_store/v3/datastax_astra
_model_mappers.py	159	22	86%	64, 130, 136, 138, 142, 148, 152–153, 171–174, 205, 262–272, 357, 398–410, 442
_models.py	19	9	53%	53–58, 61, 64, 67
astra_client.py	139	68	51%	35–38, 133–174, 180–181, 187–188, 191, 201, 213–214, 223, 267–353, 361, 373, 384–403, 424–440, 462–482, 506–517
adapta/storage/models
astra.py	35	10	71%	37, 40, 44, 59–61, 64, 70–73
aws.py	36	11	69%	36–39, 46–49, 57–59, 82, 91
azure.py	60	17	72%	32, 36–40, 47, 67, 70–71, 74–75, 89–93, 100, 113, 120, 123–124, 127
filter_expression.py	125	5	96%	55, 183–184, 238, 329
hive.py	57	25	56%	36, 40, 43, 91–98, 110, 113–114, 123–163, 166
local.py	21	4	81%	31, 35, 38, 50
adapta/storage/query_enabled_store
_models.py	61	10	84%	76, 138–141, 147–150, 156–157, 163
_qes_astra.py	57	13	77%	65–66, 78, 83–96, 106–109
_qes_delta.py	39	4	90%	33, 60, 72, 80
adapta/storage/secrets
azure_secret_client.py	20	20	0%	19–66
adapta/utils
_common.py	95	15	84%	36–37, 70–83, 94, 122, 142, 162, 245
concurrent_task_runner.py	27	1	96%	109
adapta/utils/data_structures
_functions.py	34	1	97%	134
adapta/utils/decorators
_logging.py	41	1	98%	32
_rate_limit.py	25	1	96%	58
adapta/utils/python_typing
_functions.py	11	3	73%	7–9, 22
tests
test_filtering_api.py	32	2	94%	197–198
test_python_typing_functions.py	11	2	82%	38–39
test_utils.py	166	1	99%	371
test_vault_client.py	80	18	78%	33–35, 40–42, 47–51, 56–57, 62–66
TOTAL	4485	1314	71%

Tests	Skipped	Failures	Errors	Time
207	6 :zzz:	0 :x:	0 :fire:	1m 3s :stopwatch:

github-actions[bot] commented 2 weeks ago

Coverage Report

File	Stmts	Miss	Cover	Missing
adapta/connectors/service_bus
__init__.py	1	1	0%	18
_connector.py	17	17	0%	19–61
adapta/logs
_async_logger.py	81	3	96%	55, 78–79
_base.py	55	6	89%	35, 41, 44, 94–97
_internal_logger.py	109	6	94%	276–286
adapta/logs/handlers
datadog_api_handler.py	109	31	72%	89, 106–113, 124, 136–152, 161–195, 204, 210, 238
adapta/metrics/providers
datadog_provider.py	43	43	0%	19–147
adapta/ml
__init__.py	1	1	0%	19
_model.py	10	10	0%	17–42
adapta/ml/mlflow
__init__.py	2	2	0%	17–18
_client.py	42	42	0%	19–142
_functions.py	42	42	0%	17–111
adapta/process_communication
_models.py	33	6	82%	90–96
adapta/security/clients
__init__.py	26	12	54%	27–28, 34–35, 41–42, 48–49, 53–54, 58–59
_azure_client.py	73	50	32%	42, 55–65, 75–78, 81, 84–86, 95–153, 156, 159–197
adapta/security/clients/aws
_aws_client.py	38	17	55%	37–40, 47, 57, 63, 75, 100–108, 114–117
_aws_credentials.py	73	30	59%	60–79, 83, 87, 91, 95, 99, 108–112, 116, 120, 124, 128, 132
adapta/security/clients/hashicorp_vault
hashicorp_vault_client.py	31	4	87%	46, 87, 91, 95
kubernetes_client.py	21	5	76%	45–48, 67–68
oidc_client.py	45	20	56%	33–62, 80–83, 92
token_client.py	17	7	59%	42–45, 52–53, 56, 59
adapta/storage/blob
azure_storage_client.py	119	51	57%	71–78, 88, 95–96, 99–105, 127–128, 131–156, 159, 175–188, 196–200, 219, 240–242, 251, 267–271, 281–283, 286–305, 312
local_storage_client.py	54	7	87%	42, 67, 74, 87, 101, 104, 107
s3_storage_client.py	117	77	34%	57, 61, 68–80, 91–96, 117–125, 133–134, 148–165, 181–185, 203–212, 222–243, 254–275
adapta/storage/cache
redis_cache.py	37	37	0%	19–107
adapta/storage/database/v2
azure_sql.py	34	34	0%	21–140
odbc.py	73	73	0%	21–219
snowflake_sql.py	69	69	0%	6–228
trino_sql.py	39	39	0%	21–127
adapta/storage/database/v2/models
__init__.py	1	1	0%	19
_models.py	11	11	0%	20–54
adapta/storage/database/v3
azure_sql.py	32	20	38%	55, 70, 84–122, 132
odbc.py	72	11	85%	94–104, 114, 125–126, 140, 149–155, 182
snowflake_sql.py	77	23	70%	64–78, 93–95, 106–115, 142, 178, 193–196
trino_sql.py	38	38	0%	20–119
adapta/storage/delta_lake/v2
_functions.py	68	45	34%	73–106, 153–168, 210–294
adapta/storage/delta_lake/v3
_functions.py	66	12	82%	72, 156, 161, 163, 165, 221, 231, 241–251, 257, 292
adapta/storage/distributed_object_store/v2/datastax_astra
_models.py	19	9	53%	54–59, 62, 65, 68
astra_client.py	196	140	29%	40–43, 113–137, 143–184, 190–191, 197–198, 201, 211, 223–224, 233, 273–361, 369, 393–520, 530, 541–556, 579–594, 616–632, 656–673
adapta/storage/distributed_object_store/v3/datastax_astra
_model_mappers.py	159	22	86%	64, 130, 136, 138, 142, 148, 152–153, 171–174, 205, 262–272, 357, 398–410, 442
_models.py	19	9	53%	53–58, 61, 64, 67
astra_client.py	139	68	51%	35–38, 133–174, 180–181, 187–188, 191, 201, 213–214, 223, 267–353, 361, 373, 384–403, 424–440, 462–482, 506–517
adapta/storage/models
astra.py	35	10	71%	37, 40, 44, 59–61, 64, 70–73
aws.py	36	11	69%	36–39, 46–49, 57–59, 82, 91
azure.py	60	17	72%	32, 36–40, 47, 67, 70–71, 74–75, 89–93, 100, 113, 120, 123–124, 127
filter_expression.py	125	5	96%	55, 183–184, 238, 329
hive.py	57	25	56%	36, 40, 43, 91–98, 110, 113–114, 123–163, 166
local.py	21	4	81%	31, 35, 38, 50
adapta/storage/query_enabled_store
_models.py	61	10	84%	76, 138–141, 147–150, 156–157, 163
_qes_astra.py	57	13	77%	65–66, 78, 83–96, 106–109
_qes_delta.py	39	4	90%	33, 60, 72, 80
adapta/storage/secrets
azure_secret_client.py	20	20	0%	19–66
adapta/utils
_common.py	95	15	84%	36–37, 70–83, 94, 122, 142, 162, 245
concurrent_task_runner.py	27	1	96%	109
adapta/utils/data_structures
_functions.py	34	1	97%	134
adapta/utils/decorators
_logging.py	41	1	98%	32
_rate_limit.py	25	1	96%	58
adapta/utils/python_typing
_functions.py	11	1	91%	24
tests
test_filtering_api.py	32	2	94%	197–198
test_utils.py	166	1	99%	371
test_vault_client.py	80	18	78%	33–35, 40–42, 47–51, 56–57, 62–66
TOTAL	4487	1311	71%

Tests	Skipped	Failures	Errors	Time
207	5 :zzz:	0 :x:	0 :fire:	1m 2s :stopwatch:

SneaksAndData / adapta

Introduce support for reading single files in LocalStorageClient #476