RAG Data Ingestion Benchmark

[Danidite]

Implements the RAG Data Ingestion Benchmark, closes #321

[github-actions[bot]]

Coverage Report

File	Stmts	Miss	Cover	Missing
caribou/common
utils.py	37	8	78%	43–58
caribou/common/models/remote_client
aws_remote_client.py	524	71	86%	43, 66, 68, 108, 154, 219, 222, 258, 275–276, 326–327, 365, 376, 386, 419–421, 429–430, 449–451, 470, 473, 497, 510–511, 514, 520, 560–562, 600–602, 620–621, 662, 668, 678, 682, 693–694, 702–703, 712, 716, 727, 737–739, 759, 770–772, 792, 807–809, 824–829, 857–858, 862, 878–882, 888–889, 904–905
remote_client_factory.py	21	3	86%	21–23
caribou/data_collector/components
data_collector.py	18	1	94%	23
data_exporter.py	34	2	94%	49, 56
caribou/data_collector/components/carbon
carbon_collector.py	21	2	90%	34–35
carbon_retriever.py	134	18	87%	25, 59, 93, 135–144, 148, 153, 160, 168–169, 215, 246, 249, 256
caribou/data_collector/components/performance
performance_retriever.py	38	6	84%	44, 50–51, 56–59
caribou/data_collector/components/provider
provider_collector.py	24	2	92%	43–44
provider_retriever.py	297	53	82%	24, 32, 34, 47, 59, 77, 84, 100, 132–156, 187–285, 303–308, 340, 348, 350, 352, 354, 356, 358, 360, 436, 450, 560, 653
caribou/data_collector/components/workflow
workflow_collector.py	25	1	96%	34
workflow_retriever.py	305	109	64%	216–257, 280, 330–337, 339, 371–414, 436, 467–526, 533–563, 566–583, 587–604, 615, 676, 698
caribou/data_collector/utils/latency_retriever
aws_latency_retriever.py	74	9	88%	59, 63, 65, 67, 70, 74, 76, 78, 81
latency_retriever.py	6	1	83%	8
caribou/deployment/client
caribou.py	16	1	94%	22
caribou_workflow.py	465	135	71%	54, 70–71, 80–83, 129, 201–223, 267–268, 282, 315–326, 347–354, 380–391, 434–454, 562, 578, 581, 636–662, 667–683, 705–715, 793, 796, 798, 800, 802, 804, 831, 866–881, 916–917, 927–931, 961, 996, 1004, 1016–1096, 1109–1138, 1172–1181, 1212, 1245, 1273–1282, 1302–1320, 1340, 1353–1359, 1365
caribou/deployment/client/remote_cli
remote_cli_handler.py	91	91	0%	1–146
caribou/deployment/common/config
config.py	83	1	99%	104
caribou/deployment/common/deploy
deployer.py	140	16	89%	43, 102–109, 199, 240–242, 249, 256, 267
deployment_packager.py	237	47	80%	31, 34, 39, 63–70, 73–84, 90, 96, 101, 106, 128, 156, 169, 182, 190, 251–253, 262, 275, 328–343, 353, 355, 357, 360
executor.py	53	4	92%	32–33, 66, 68
workflow_builder.py	171	13	92%	43, 59, 146, 178, 186, 189, 191, 196, 205, 208, 210, 324, 330
caribou/deployment/common/deploy/models
function.py	42	2	95%	42, 53
function_instance.py	12	2	83%	16, 22
iam_role.py	34	3	91%	19–21
instructions.py	18	2	89%	17, 26
resource.py	18	1	94%	25
workflow.py	194	32	84%	39, 44, 67, 77, 99, 120, 166, 171, 206, 219–231, 272, 281, 289, 293, 298–327
caribou/deployment/common/deploy_instructions
aws_deploy_instructions.py	24	2	92%	37, 83
deploy_instructions.py	54	9	83%	57, 95, 145, 151, 164, 168, 172, 176, 182
caribou/deployment/common/factories
deploy_instruction_factory.py	18	3	83%	20–22
deployer_factory.py	106	16	85%	23–26, 40, 51–52, 81, 89, 98, 107, 118, 125–128
caribou/deployment/server
re_deployment_server.py	61	5	92%	36, 48, 51, 58, 107
caribou/deployment_solver
workflow_config.py	75	11	85%	24, 26, 32–33, 54–56, 61, 79, 82, 107
caribou/deployment_solver/deployment_algorithms
coarse_grained_deployment_algorithm.py	29	2	93%	33, 38
deployment_algorithm.py	100	11	89%	54, 95, 111, 124, 138, 155, 190–196, 225
fine_grained_deployment_algorithm.py	29	2	93%	44, 49
stochastic_heuristic_deployment_algorithm.py	102	15	85%	20–28, 54, 57, 83, 98, 109, 112, 117, 123–126, 136, 153, 180, 187
caribou/deployment_solver/deployment_input
input_manager.py	199	55	72%	126, 132, 135, 194, 200, 246–254, 276–282, 291–294, 304–307, 320–339, 352–373, 379–387, 441–442, 476–487, 493–495, 501
caribou/deployment_solver/deployment_input/components/calculators
carbon_calculator.py	96	4	96%	124–128, 214, 261
cost_calculator.py	70	1	99%	163
runtime_calculator.py	109	11	90%	130–145, 152, 201, 209, 262, 295
caribou/deployment_solver/deployment_input/components/loaders
carbon_loader.py	21	3	86%	22, 43, 46
datacenter_loader.py	49	5	90%	106–111, 114
performance_loader.py	18	1	94%	38
region_viability_loader.py	17	2	88%	15, 32
workflow_loader.py	147	26	82%	86, 102–112, 122, 145, 165, 183, 193, 210, 264–276, 292, 364–369, 392, 430–432, 437
caribou/deployment_solver/deployment_metrics_calculator
deployment_metrics_calculator.py	40	1	98%	49
go_deployment_metrics_calculator.py	54	32	41%	21, 25–28, 32–34, 48–57, 60, 71–83, 86–90, 93–95, 98–99
simple_deployment_metrics_calculator.py	129	17	87%	125–142, 170–171, 203–204, 220–223, 235–236, 251
caribou/deployment_solver/deployment_metrics_calculator/models
workflow_instance.py	194	82	58%	84–95, 107, 141–144, 183–186, 194–196, 203–227, 238–269, 275–289, 308–441, 520
caribou/deployment_solver/models
indexer.py	17	1	94%	25
caribou/deployment_solver/ranker
ranker.py	62	4	94%	14, 26–27, 95
caribou/endpoint
client.py	141	21	85%	50, 60, 110, 120, 142–145, 156, 187, 203, 205, 243–247, 252–253, 257–260
caribou/monitors
deployment_manager.py	135	18	87%	65, 71, 82, 101, 123–128, 164–169, 184, 207, 222, 228, 263, 283
monitor.py	8	1	88%	12
caribou/syncers
log_sync_workflow.py	521	290	44%	108, 163–173, 178, 186, 193, 203, 210, 248–254, 256, 258, 260, 262, 264, 268, 272, 274, 276, 279–285, 288, 292, 302–369, 378–385, 395–463, 472–475, 480–495, 506–597, 610–664, 674–700, 705–722, 727–744, 758, 765–788, 797–798, 808, 813–819, 846–847, 849–850, 855–856, 890–897, 901, 938, 940, 945, 979, 983–986, 994–1004, 1007–1013, 1016–1027, 1030
log_syncer.py	42	1	98%	46
caribou/syncers/components
execution_data.py	74	4	95%	76, 86, 129, 137
workflow_run_sample.py	83	6	93%	136, 145–149
TOTAL	6655	1298	80%

Tests	Skipped	Failures	Errors	Time
665	0 :zzz:	0 :x:	0 :fire:	23.270s :stopwatch: