Scalable configuration options

[obfuscurity]

Offer a choice of scaling configurations tuned for common instance sizes.

[obfuscurity]

A carbon.conf I've been using for benchmarking on an EC2 i2.4xlarge with a 400GB io2 20k IOPS EBS volume.

[relay]
USER = carbon
LOG_LISTENER_CONNECTIONS = False
RELAY_METHOD = consistent-hashing
REPLICATION_FACTOR = 1
DESTINATIONS = 127.0.0.1:2104:1, 127.0.0.1:2204:2, 127.0.0.1:2304:3, 127.0.0.1:2404:4, 127.0.0.1:2504:5, 127.0.0.1:2604:6, 127.0.0.1:2704:7, 127.0.0.1:2804:8
MAX_DATAPOINTS_PER_MESSAGE = 500
MAX_QUEUE_SIZE = 10000
USE_FLOW_CONTROL = True

[relay:1]
LINE_RECEIVER_PORT = 2113
PICKLE_RECEIVER_PORT = 2114

[relay:2]
LINE_RECEIVER_PORT = 2213
PICKLE_RECEIVER_PORT = 2214

[relay:3]
LINE_RECEIVER_PORT = 2313
PICKLE_RECEIVER_PORT = 2314

[relay:4]
LINE_RECEIVER_PORT = 2413
PICKLE_RECEIVER_PORT = 2414

[cache]
USER = carbon
CACHE_WRITE_STRATEGY = sorted
MAX_CACHE_SIZE = 1600000
USE_FLOW_CONTROL = True
WHISPER_FALLOCATE_CREATE = True
MAX_CREATES_PER_MINUTE = 12000
MAX_UPDATES_PER_SECOND = 200
USE_INSECURE_UNPICKLER = False
LOG_CACHE_HITS = False
LOG_CACHE_QUEUE_SORTS = False
LOG_LISTENER_CONNECTIONS = False
LOG_UPDATES = False
ENABLE_LOGROTATION = False
WHISPER_AUTOFLUSH = False

[cache:1]
LINE_RECEIVER_PORT = 2103
PICKLE_RECEIVER_PORT = 2104
CACHE_QUERY_PORT = 7102

[cache:2]
LINE_RECEIVER_PORT = 2203
PICKLE_RECEIVER_PORT = 2204
CACHE_QUERY_PORT = 7202

[cache:3]
LINE_RECEIVER_PORT = 2303
PICKLE_RECEIVER_PORT = 2304
CACHE_QUERY_PORT = 7302

[cache:4]
LINE_RECEIVER_PORT = 2403
PICKLE_RECEIVER_PORT = 2404
CACHE_QUERY_PORT = 7402

[cache:5]
LINE_RECEIVER_PORT = 2503
PICKLE_RECEIVER_PORT = 2504
CACHE_QUERY_PORT = 7502

[cache:6]
LINE_RECEIVER_PORT = 2603
PICKLE_RECEIVER_PORT = 2604
CACHE_QUERY_PORT = 7602

[cache:7]
LINE_RECEIVER_PORT = 2703
PICKLE_RECEIVER_PORT = 2704
CACHE_QUERY_PORT = 7702

[cache:8]
LINE_RECEIVER_PORT = 2803
PICKLE_RECEIVER_PORT = 2804
CACHE_QUERY_PORT = 7802

Tested with the following Haggar command:

$ GOPATH=~/gocode ./gocode/bin/haggar -agents=300 -metrics=2000 -carbon="x.x.x.x:2003"

Resulting in 60k/sec metricsReceived and 38 pointsPerUpdate. The plateaus shown below represent the full volume of agents X metrics after ramping up one agent every 10s.

[obfuscurity]

And the associated haproxy.cfg:

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon
    maxconn 8192
    pidfile /var/run/haproxy.pid

defaults
    balance roundrobin
    log global
    mode    tcp
    retries 3
    option  redispatch
    contimeout 5000
    clitimeout 50000
    srvtimeout 50000

# plaintext listener
listen carbon_relay_2003 0.0.0.0:2003
    server carbon_relay_2113 127.0.0.1:2113 check maxconn 1024
    server carbon_relay_2213 127.0.0.1:2213 check maxconn 1024
    server carbon_relay_2313 127.0.0.1:2313 check maxconn 1024
    server carbon_relay_2413 127.0.0.1:2413 check maxconn 1024

# pickle listener
listen carbon_relay_2004 0.0.0.0:2004
    server carbon_relay_2114 127.0.0.1:2114 check maxconn 1024
    server carbon_relay_2214 127.0.0.1:2214 check maxconn 1024
    server carbon_relay_2314 127.0.0.1:2314 check maxconn 1024
    server carbon_relay_2414 127.0.0.1:2414 check maxconn 1024

[obfuscurity]

Forgot to mention the storage-schemas.conf:

[collectd]
pattern = ^collectd\.
retentions = 10s:1w, 60s:1y

[haggar]
pattern = ^haggar\.
retentions = 10s:1d, 60s:1w, 1h:1y

[default]
pattern = .*
retentions = 60s:1y