search

douban / Kenshin

Kenshin: A time-series database alternative to Graphite Whisper with 40x improvement in IOPS
Apache License 2.0
206 stars 24 forks source link

what's role of relay dispatch ? some data point missing in the graph when dispatch also busy! #15

Open luckywarrior opened 7 years ago

luckywarrior commented 7 years ago

what's role of relay dispatch ? some data point missing in the graph when dispatch also busy!

@camper42 would you share some configurations of your online system about kenshin ?especially rurouni.conf and storage-schemas.conf and relay.conf . If possible, please let me know,thanks!

camper42 commented 7 years ago

storage-schemas.conf

[metric-test]
pattern = .*metric_test.*
xFilesFactor = 20
aggregationMethod = average
retentions = 10s:12h,60s:2d
cacheRetention = 900s
metricsPerFile = 8

[rurouni-stats]
pattern = ^rurouni\.
xFilesFactor = 20
retentions = 60s:2d,300s:7d,15m:25w,12h:5y
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[carbon-stats]
pattern = ^carbon\.
xFilesFactor = 20
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:2y
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[stats-counters-count]
pattern = stats\.counters\..*\.count$
xFilesFactor = 20
retentions = 12h:5y
aggregationMethod = sum
metricsPerFile = 8
cacheRetention = 600s

[stats-timers-upper]
pattern = stats\.timers\..*\.upper[^.]*
aggregationMethod = max
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
metricsPerFile = 8
cacheRetention = 600s

[stats-timers-mean]
pattern = stats\.timers\..*\.mean[^.]*
aggregationMethod = average
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
metricsPerFile = 8
cacheRetention = 600s

[stats-timers-lower]
pattern = stats\.timers\..*\.lower
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = min
metricsPerFile = 8
cacheRetention = 600s

[stats-timers-count]
pattern = stats\.timers\..*\.count[^.]*
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[nagios-gauges]
pattern = stats\.gauges\.nagios\.[^.]*\.[^.]*
retentions = 10s:12h,300s:7d,1h:1y,12h:5y
xFilesFactor = 20
aggregationMethod = max
metricsPerFile = 8
cacheRetention = 600s

[bandwidth]
pattern = .*\.bandwidth\.rate$
retentions = 10s:12h,60s:2d,300s:7d,15m:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[stats-counters-web]
pattern = stats\.counters\.web\..*\.rate$
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 620s

[diamond]
pattern = ^servers\..*
retentions = 20s:24h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 700s

[switch]
pattern = ^switch\..*
retentions = 15s:24h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[mfs]
pattern = ^mfs\..*
retentions = 60s:2d,1h:4w,12h:2y
xFilesFactor = 1
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 600s

[icinga2]
pattern = ^icinga2\..*
retentions = 60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 900s

[marathon]
pattern = ^marathon\..*
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 800s

[default]
pattern = .*
retentions = 10s:12h,60s:2d,300s:7d,15m:25w,12h:5y
xFilesFactor = 20
aggregationMethod = average
metricsPerFile = 8
cacheRetention = 620s
camper42 commented 7 years ago

relay.conf

cluster online
    fnv1a_sh
        server0:10300=0
        server1:10300=0
        server2:10300=0
        ...
        server0:10315=15
        server1:10315=15
        server2:10315=15
    ;

match <bad metric name regex>
    send to blackhole
    stop
    ;

match *
    send to online
    ;
camper42 commented 7 years ago

rurouni.conf

[cache]
LOCAL_DATA_DIR = /data/kenshin/storage/data
LOCAL_LINK_DIR = /data/kenshin/storage/link
CONF_DIR = /data/kenshin/conf
LOG_DIR = /data/kenshin/storage/log
PID_DIR = /data/kenshin/storage/run

LINE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
CACHE_QUERY_INTERFACE = 0.0.0.0

LOG_UPDATES = False
MAX_CREATES_PER_MINUTE = 1000
NUM_ALL_INSTANCE = 16

[cache:0]
LINE_RECEIVER_PORT = 10300
PICKLE_RECEIVER_PORT = 10400
CACHE_QUERY_PORT = 10500

...

[cache:15]
LINE_RECEIVER_PORT = 10315
PICKLE_RECEIVER_PORT = 10415
CACHE_QUERY_PORT = 10515