Adding the additional storage providers to the monitoring configuration

There appears to be an npe in the test-environment backend if the config is created and then updated at the very least with Azure but I believe it occurs for other providers as well.

Create:

2023 Nov 15 13:52:10 INFO test.env test/test_drivers.go:212 Config changed old="cortexWorkloads:{targets:{key:\"all\" value:{replicas:1}}} cortexConfig:{limits:{ingestion_rate:600000 ingestion_rate_strategy:\"local\" ingestion_burst_size:1000000 accept_ha_samples:false ha_cluster_label:\"cluster\" ha_replica_label:\"__replica__\" ha_max_clusters:0 max_label_name_length:1024 max_label_value_length:2048 max_label_names_per_series:30 max_labels_size_bytes:0 max_metadata_length:1024 reject_old_samples:false reject_old_samples_max_age:{seconds:1209600} creation_grace_period:{seconds:600} enforce_metadata_metric_name:true enforce_metric_name:true ingestion_tenant_shard_size:0 max_exemplars:0 max_series_per_query:100000 max_series_per_user:5000000 max_series_per_metric:50000 max_global_series_per_user:0 max_global_series_per_metric:0 max_metadata_per_user:8000 max_metadata_per_metric:10 max_global_metadata_per_user:0 max_global_metadata_per_metric:0 out_of_order_time_window:{} max_fetched_chunks_per_query:2000000 max_fetched_series_per_query:0 max_fetched_chunk_bytes_per_query:0 max_fetched_data_bytes_per_query:0 max_query_lookback:{} max_query_length:{} max_query_parallelism:14 max_cache_freshness:{seconds:60} max_queriers_per_tenant:0 max_outstanding_requests_per_tenant:100 ruler_evaluation_delay_duration:{} ruler_tenant_shard_size:0 ruler_max_rules_per_rule_group:0 ruler_max_rule_groups_per_tenant:0 store_gateway_tenant_shard_size:0 max_downloaded_bytes_per_request:0 compactor_blocks_retention_period:{seconds:2592000} compactor_tenant_shard_size:0 alertmanager_receivers_firewall_block_private_addresses:false alertmanager_notification_rate_limit:0 alertmanager_max_config_size_bytes:0 alertmanager_max_templates_count:0 alertmanager_max_template_size_bytes:0 alertmanager_max_dispatcher_aggregation_groups:0 alertmanager_max_alerts_count:0 alertmanager_max_alerts_size_bytes:0} runtime_config:{multi_kv_config:{} ingester_limits:{}} compactor:{block_ranges:{seconds:7200} block_ranges:{seconds:43200} block_ranges:{seconds:86400} block_sync_concurrency:20 meta_sync_concurrency:20 consistency_delay:{} compaction_interval:{seconds:3600} compaction_retries:3 compaction_concurrency:1 cleanup_interval:{seconds:900} cleanup_concurrency:20 deletion_delay:{seconds:43200} tenant_cleanup_delay:{seconds:21600} skip_blocks_with_out_of_order_chunks_enabled:false block_files_concurrency:10 blocks_fetch_concurrency:3 block_deletion_marks_migration_enabled:false block_visit_marker_timeout:{seconds:300} block_visit_marker_file_update_interval:{seconds:60} accept_malformed_index:false} querier:{max_concurrent:20 timeout:{seconds:120} iterators:false batch_iterators:true ingester_streaming:true ingester_metadata_streaming:false max_samples:50000000 query_ingesters_within:{} query_store_for_labels_enabled:false per_step_stats_enabled:false query_store_after:{} max_query_into_future:{seconds:600} default_evaluation_interval:{seconds:60} lookback_delta:{seconds:300} shuffle_sharding_ingesters_lookback_period:{} thanos_engine:false} storage:{backend:\"azure\" s3:{insecure:false signature_version:\"v4\" bucket_lookup_type:\"auto\" sse:{} http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} gcs:{} azure:{account_name:\"c\" account_key:\"d\" container_name:\"b\" endpoint_suffix:\"a\" max_retries:20 msi_resource:\"\" user_assigned_id:\"\" http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} swift:{auth_version:0 max_retries:3 connect_timeout:{seconds:10} request_timeout:{seconds:5}} filesystem:{}} log_level:\"warn\"} grafana:{enabled:false version:\"latest\"}" new="enabled:true cortexWorkloads:{targets:{key:\"all\" value:{replicas:1}}} cortexConfig:{limits:{ingestion_rate:600000 ingestion_rate_strategy:\"local\" ingestion_burst_size:1000000 accept_ha_samples:false ha_cluster_label:\"cluster\" ha_replica_label:\"__replica__\" ha_max_clusters:0 max_label_name_length:1024 max_label_value_length:2048 max_label_names_per_series:30 max_labels_size_bytes:0 max_metadata_length:1024 reject_old_samples:false reject_old_samples_max_age:{seconds:1209600} creation_grace_period:{seconds:600} enforce_metadata_metric_name:true enforce_metric_name:true ingestion_tenant_shard_size:0 max_exemplars:0 max_series_per_query:100000 max_series_per_user:5000000 max_series_per_metric:50000 max_global_series_per_user:0 max_global_series_per_metric:0 max_metadata_per_user:8000 max_metadata_per_metric:10 max_global_metadata_per_user:0 max_global_metadata_per_metric:0 out_of_order_time_window:{} max_fetched_chunks_per_query:2000000 max_fetched_series_per_query:0 max_fetched_chunk_bytes_per_query:0 max_fetched_data_bytes_per_query:0 max_query_lookback:{} max_query_length:{} max_query_parallelism:14 max_cache_freshness:{seconds:60} max_queriers_per_tenant:0 max_outstanding_requests_per_tenant:100 ruler_evaluation_delay_duration:{} ruler_tenant_shard_size:0 ruler_max_rules_per_rule_group:0 ruler_max_rule_groups_per_tenant:0 store_gateway_tenant_shard_size:0 max_downloaded_bytes_per_request:0 compactor_blocks_retention_period:{seconds:2592000} compactor_tenant_shard_size:0 alertmanager_receivers_firewall_block_private_addresses:false alertmanager_notification_rate_limit:0 alertmanager_max_config_size_bytes:0 alertmanager_max_templates_count:0 alertmanager_max_template_size_bytes:0 alertmanager_max_dispatcher_aggregation_groups:0 alertmanager_max_alerts_count:0 alertmanager_max_alerts_size_bytes:0} runtime_config:{multi_kv_config:{} ingester_limits:{}} compactor:{block_ranges:{seconds:7200} block_ranges:{seconds:43200} block_ranges:{seconds:86400} block_sync_concurrency:20 meta_sync_concurrency:20 consistency_delay:{} compaction_interval:{seconds:3600} compaction_retries:3 compaction_concurrency:1 cleanup_interval:{seconds:900} cleanup_concurrency:20 deletion_delay:{seconds:43200} tenant_cleanup_delay:{seconds:21600} skip_blocks_with_out_of_order_chunks_enabled:false block_files_concurrency:10 blocks_fetch_concurrency:3 block_deletion_marks_migration_enabled:false block_visit_marker_timeout:{seconds:300} block_visit_marker_file_update_interval:{seconds:60} accept_malformed_index:false} querier:{max_concurrent:20 timeout:{seconds:120} iterators:false batch_iterators:true ingester_streaming:true ingester_metadata_streaming:false max_samples:50000000 query_ingesters_within:{} query_store_for_labels_enabled:false per_step_stats_enabled:false query_store_after:{} max_query_into_future:{seconds:600} default_evaluation_interval:{seconds:60} lookback_delta:{seconds:300} shuffle_sharding_ingesters_lookback_period:{} thanos_engine:false} storage:{backend:\"azure\" s3:{insecure:false signature_version:\"v4\" bucket_lookup_type:\"auto\" sse:{} http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} gcs:{} azure:{account_name:\"c\" account_key:\"d\" container_name:\"b\" endpoint_suffix:\"a\" max_retries:20 msi_resource:\"\" user_assigned_id:\"\" http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} swift:{auth_version:0 max_retries:3 connect_timeout:{seconds:10} request_timeout:{seconds:5}} filesystem:{}} log_level:\"warn\"} grafana:{enabled:false version:\"latest\"}"

Update:

2023 Nov 15 13:52:25 INFO test.env test/test_drivers.go:212 Config changed old="enabled:true cortexWorkloads:{targets:{key:\"all\" value:{replicas:1}}} cortexConfig:{limits:{ingestion_rate:600000 ingestion_rate_strategy:\"local\" ingestion_burst_size:1000000 accept_ha_samples:false ha_cluster_label:\"cluster\" ha_replica_label:\"__replica__\" ha_max_clusters:0 max_label_name_length:1024 max_label_value_length:2048 max_label_names_per_series:30 max_labels_size_bytes:0 max_metadata_length:1024 reject_old_samples:false reject_old_samples_max_age:{seconds:1209600} creation_grace_period:{seconds:600} enforce_metadata_metric_name:true enforce_metric_name:true ingestion_tenant_shard_size:0 max_exemplars:0 max_series_per_query:100000 max_series_per_user:5000000 max_series_per_metric:50000 max_global_series_per_user:0 max_global_series_per_metric:0 max_metadata_per_user:8000 max_metadata_per_metric:10 max_global_metadata_per_user:0 max_global_metadata_per_metric:0 out_of_order_time_window:{} max_fetched_chunks_per_query:2000000 max_fetched_series_per_query:0 max_fetched_chunk_bytes_per_query:0 max_fetched_data_bytes_per_query:0 max_query_lookback:{} max_query_length:{} max_query_parallelism:14 max_cache_freshness:{seconds:60} max_queriers_per_tenant:0 max_outstanding_requests_per_tenant:100 ruler_evaluation_delay_duration:{} ruler_tenant_shard_size:0 ruler_max_rules_per_rule_group:0 ruler_max_rule_groups_per_tenant:0 store_gateway_tenant_shard_size:0 max_downloaded_bytes_per_request:0 compactor_blocks_retention_period:{seconds:2592000} compactor_tenant_shard_size:0 alertmanager_receivers_firewall_block_private_addresses:false alertmanager_notification_rate_limit:0 alertmanager_max_config_size_bytes:0 alertmanager_max_templates_count:0 alertmanager_max_template_size_bytes:0 alertmanager_max_dispatcher_aggregation_groups:0 alertmanager_max_alerts_count:0 alertmanager_max_alerts_size_bytes:0} runtime_config:{multi_kv_config:{} ingester_limits:{}} compactor:{block_ranges:{seconds:7200} block_ranges:{seconds:43200} block_ranges:{seconds:86400} block_sync_concurrency:20 meta_sync_concurrency:20 consistency_delay:{} compaction_interval:{seconds:3600} compaction_retries:3 compaction_concurrency:1 cleanup_interval:{seconds:900} cleanup_concurrency:20 deletion_delay:{seconds:43200} tenant_cleanup_delay:{seconds:21600} skip_blocks_with_out_of_order_chunks_enabled:false block_files_concurrency:10 blocks_fetch_concurrency:3 block_deletion_marks_migration_enabled:false block_visit_marker_timeout:{seconds:300} block_visit_marker_file_update_interval:{seconds:60} accept_malformed_index:false} querier:{max_concurrent:20 timeout:{seconds:120} iterators:false batch_iterators:true ingester_streaming:true ingester_metadata_streaming:false max_samples:50000000 query_ingesters_within:{} query_store_for_labels_enabled:false per_step_stats_enabled:false query_store_after:{} max_query_into_future:{seconds:600} default_evaluation_interval:{seconds:60} lookback_delta:{seconds:300} shuffle_sharding_ingesters_lookback_period:{} thanos_engine:false} storage:{backend:\"azure\" s3:{insecure:false signature_version:\"v4\" bucket_lookup_type:\"auto\" sse:{} http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} gcs:{} azure:{account_name:\"c\" account_key:\"d\" container_name:\"b\" endpoint_suffix:\"a\" max_retries:20 msi_resource:\"\" user_assigned_id:\"\" http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} swift:{auth_version:0 max_retries:3 connect_timeout:{seconds:10} request_timeout:{seconds:5}} filesystem:{}} log_level:\"warn\"} grafana:{enabled:false version:\"latest\"}" new="enabled:true cortexWorkloads:{targets:{key:\"all\" value:{replicas:1}}} cortexConfig:{limits:{ingestion_rate:600000 ingestion_rate_strategy:\"local\" ingestion_burst_size:1000000 accept_ha_samples:false ha_cluster_label:\"cluster\" ha_replica_label:\"__replica__\" ha_max_clusters:0 max_label_name_length:1024 max_label_value_length:2048 max_label_names_per_series:30 max_labels_size_bytes:0 max_metadata_length:1024 reject_old_samples:false reject_old_samples_max_age:{seconds:1209600} creation_grace_period:{seconds:600} enforce_metadata_metric_name:true enforce_metric_name:true ingestion_tenant_shard_size:0 max_exemplars:0 max_series_per_query:100000 max_series_per_user:5000000 max_series_per_metric:50000 max_global_series_per_user:0 max_global_series_per_metric:0 max_metadata_per_user:8000 max_metadata_per_metric:10 max_global_metadata_per_user:0 max_global_metadata_per_metric:0 out_of_order_time_window:{} max_fetched_chunks_per_query:2000000 max_fetched_series_per_query:0 max_fetched_chunk_bytes_per_query:0 max_fetched_data_bytes_per_query:0 max_query_lookback:{} max_query_length:{} max_query_parallelism:14 max_cache_freshness:{seconds:60} max_queriers_per_tenant:0 max_outstanding_requests_per_tenant:100 ruler_evaluation_delay_duration:{} ruler_tenant_shard_size:0 ruler_max_rules_per_rule_group:0 ruler_max_rule_groups_per_tenant:0 store_gateway_tenant_shard_size:0 max_downloaded_bytes_per_request:0 compactor_blocks_retention_period:{seconds:2592000} compactor_tenant_shard_size:0 alertmanager_receivers_firewall_block_private_addresses:false alertmanager_notification_rate_limit:0 alertmanager_max_config_size_bytes:0 alertmanager_max_templates_count:0 alertmanager_max_template_size_bytes:0 alertmanager_max_dispatcher_aggregation_groups:0 alertmanager_max_alerts_count:0 alertmanager_max_alerts_size_bytes:0} runtime_config:{multi_kv_config:{} ingester_limits:{}} compactor:{block_ranges:{seconds:7200} block_ranges:{seconds:43200} block_ranges:{seconds:86400} block_sync_concurrency:20 meta_sync_concurrency:20 consistency_delay:{} compaction_interval:{seconds:3600} compaction_retries:3 compaction_concurrency:1 cleanup_interval:{seconds:900} cleanup_concurrency:20 deletion_delay:{seconds:43200} tenant_cleanup_delay:{seconds:21600} skip_blocks_with_out_of_order_chunks_enabled:false block_files_concurrency:10 blocks_fetch_concurrency:3 block_deletion_marks_migration_enabled:false block_visit_marker_timeout:{seconds:300} block_visit_marker_file_update_interval:{seconds:60} accept_malformed_index:false} querier:{max_concurrent:20 timeout:{seconds:120} iterators:false batch_iterators:true ingester_streaming:true ingester_metadata_streaming:false max_samples:50000000 query_ingesters_within:{} query_store_for_labels_enabled:false per_step_stats_enabled:false query_store_after:{} max_query_into_future:{seconds:600} default_evaluation_interval:{seconds:60} lookback_delta:{seconds:300} shuffle_sharding_ingesters_lookback_period:{} thanos_engine:false} storage:{backend:\"azure\" s3:{insecure:false signature_version:\"v4\" bucket_lookup_type:\"auto\" sse:{} http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} gcs:{} azure:{account_name:\"c\" account_key:\"d\" container_name:\"b\" endpoint_suffix:\"a\" max_retries:20 msi_resource:\"\" user_assigned_id:\"\" http:{idle_conn_timeout:{seconds:90} response_header_timeout:{seconds:120} insecure_skip_verify:false tls_handshake_timeout:{seconds:10} expect_continue_timeout:{seconds:1} max_idle_connections:100 max_idle_connections_per_host:100 max_connections_per_host:0}} swift:{auth_version:0 max_retries:3 connect_timeout:{seconds:10} request_timeout:{seconds:5}} filesystem:{}} log_level:\"warn\"} grafana:{enabled:false version:\"latest\"}"
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0x2c5cba8]

rancher / opni

Adding the additional storage providers to the monitoring configuration #1851