0001

[AmitKumarDas]

// https://dave.cheney.net/high-performance-go-workshop/gophercon-2019.html
//
// variance - https://github.com/golang/perf/tree/master/cmd/benchstat
// 

[AmitKumarDas]

// https://dave.cheney.net/2015/10/09/padding-is-hard
//
// - https://github.com/mdempsky/maligned

[AmitKumarDas]

// https://golang.org/doc/effective_go#leaky_buffer
//
// freelist - avoid allocating & freeing buffers continuously

// The default clauses in the select statements
// execute when no other case is ready, meaning
// that the selects never block.
//
// This implementation builds a leaky bucket 
// free list in just a few lines, relying on the
// buffered channel and the garbage collector for 
// bookkeeping.

var freeList = make(chan *Buffer, 100)
var serverChan = make(chan *Buffer)

func client() {
  for {
    var b *Buffer
    // grab a buffer if available; allocate if not.
    select {
    case b = <-freeList:
      // got one; nothing more to do
    default:
      // none free, so allocate a new one
      b = new(Buffer)
    }

    load(b)              // read next message from the net
    serverChan <- b      // send to server
  }
}

func server() {
  for {
    b := <-serverChan    // wait for work
    process(b)

    // reuse buffer if there's room
    select {
    case freeList <- b:
      // buffer on free list; nothing more to do
    default:
      // free list full, just carry on
    }
  }
}

[AmitKumarDas]

// https://blog.questionable.services/article/using-buffer-pools-with-go/

buf := new(bytes.Buffer)

// write to the buffer first so we can catch the error
// however this is wasteful
err := template.ExecuteTemplate(buf, "forms/create.html", user)
// or err := json.NewEncoder(buf).Encode(value)
if err != nil {
    return err
}

buf.WriteTo(w)

// In this case, we create and then implicitly
// throw away a temporary buffer when the 
// function exits. This is wasteful, since we 
// need a buffer on every request. We give more
// work to garbage collector (GC).

[AmitKumarDas]

// https://github.com/matryer/vice/blob/master/docs/design-patterns.md
//
// input & output channel to the same function
// context to signal cancellation
// for ever loop with context cancellation to break out
// for ever + select clause
// defer cancel() -- to cancel the context
// <- time.After(...) -- time as channel -- inside select clause -- inside for ever loop

[AmitKumarDas]

// https://github.com/matryer/vice/blob/master/vicetest/test.go
//
// interface tests ~ specification test
// dedicated package that deals with testing the library
// testing for plugins

[AmitKumarDas]

// https://github.com/matryer/vice/blob/master/example/greeter/client/main.go
//
// context + interrupt signal
// defer signal & cancel
// send & receive in different goroutines
// signal interrupt leads to context cancellation that unblocks <- ctx.Done()
// read from stdin & push into channel

[AmitKumarDas]

// https://github.com/matryer/vice/blob/master/transport.go
//
// interface is outside
// specific implementations are inside dedicated packages
// interface deals with channels
// stop vs done -- they mean different things
// ErrChan as an interface method
// custom error struct

[AmitKumarDas]

// https://github.com/matryer/vice/blob/master/backoff/backoff.go
//
// default backoff interval, max backoff interval 
// max calls
// every retry doubles the default backoff
// max backoff & max calls can be 0 i.e. unlimited retries if needed

[AmitKumarDas]

// https://github.com/matryer/vice/tree/master/queues
// 
// nats, nsq, redis, rabbitmq, sqs // design

// https://github.com/matryer/vice/tree/master/queues/nats
//
// nats.go, nats_test.go, options.go -- 3 files define the feature
// options struct as well as option function
// sensible defaults design i.e. New(...options)

// make sure Transport satisfies vice.Transport interface
// This interface is defined somewhere the outside this package 
// check use of nil instead of &Transport{}
// check use of same names 
// names are same but one is interface & other is struct
var _ vice.Transport = (*Transport)(nil)

[AmitKumarDas]

// https://github.com/d4l3k/messagediff
//
// -- WIP --

[AmitKumarDas]

// https://github.com/r3labs/diff
//
// -- WIP --

[AmitKumarDas]

// https://github.com/matryer/is 
//
// testing
// use of T as interface & I as struct
// struct composes interface
// New(T) // constructor takes Interface
// I does not implement T
// I makes use of one of the T's public methods based on how I is initialised
// Check the use of function as well as method based constructor **
// isNil(...) // use of reflect
// areEqual(a, b interface{})
// load comments, arguments, color **

[AmitKumarDas]

// https://github.com/banzaicloud/bank-vaults
//
// -- WIP --

[AmitKumarDas]

// https://github.com/banzaicloud/istio-operator
//
// -- WIP --

[AmitKumarDas]

// https://github.com/banzaicloud/kafka-operator
//
// -- WIP --

[AmitKumarDas]

// https://github.com/banzaicloud/logging-operator
//
// -- WIP --

[AmitKumarDas]

// https://github.com/banzaicloud/hpa-operator
// 
// -- WIP --

[AmitKumarDas]

// kubectl get prometheus --all-namespaces -oyaml
//
// Check the External URL
// Check if Admin API is enabled
// Check if AlertManager is configured to Prometheus
// Check the Retention Hours
// Check the Storage i.e. PV capacity

[AmitKumarDas]

// kubectl get alertmanagers --all-namespaces -oyaml
//
// Check External URL
// Check Retention Hours

[AmitKumarDas]

// CRDs
//
// -- monitoring --
// kubectl get crd podmonitors.monitoring.coreos.com
// kubectl get crd prometheuses.monitoring.coreos.com
// kubectl get crd prometheusrules.monitoring.coreos.com
// kubectl get crd servicemonitors.monitoring.coreos.com
// kubectl get crd alertmanagerconfigs.monitoring.coreos.com
// kubectl get crd alertmanagers.monitoring.coreos.com

[AmitKumarDas]

// https://d2iq.com/blog/running-kind-inside-a-kubernetes-cluster-for-continuous-integration
//
// MTU problem
// PID 1 problem
// Cgroup Mounts
// IP Tables

[AmitKumarDas]

// https://github.com/moby/buildkit/blob/master/examples/kubernetes/create-certs.sh
//
// setting up mTLS
// uses https://github.com/FiloSottile/mkcert
// creates daemon & client serts
// use kubectl to create generic Kubernetes secrets with --dry-run

[AmitKumarDas]

// https://rancher.com/blog/2020/custom-monitoring
//
// secret contains alert manager config
// AlertManager's config sits in a Secret object -- TIL
// base64 encode / decode

[AmitKumarDas]

// https://github.com/moby/buildkit/tree/master/examples/kubernetes
//
// rootless mode image
// - uses non-root user
// - does not require securityContext.priviledged
// - may not work on some host kernels

[AmitKumarDas]

// https://github.com/moby/buildkit/blob/master/docs/rootless.md
//
// use of /dev/fuse
// use of seccomp, apparmor
// masks for /prco mount
// isolate network namespace from host

[AmitKumarDas]

// https://banzaicloud.com/blog/kurun-port-forward/
//
// developer experience
// fast iteration, containers, registry, ci
// vs. https://github.com/google/ko
// https://github.com/banzaicloud/bank-vaults/blob/master/hack/acceptance-test.sh // testing **

// https://github.com/banzaicloud/kurun
//
// fastest way to run go file in targeted K8s cluster
// -- pass multiple .go files
// -- pass multiple args for above .go files
// -- no need to have go in your workstation but just kubectl
//
// link a locally running application / service to a K8s cluster & still make it accessible
// -- via inlets
// -- i.e. push a local running application into a remote K8s
// -- very useful for rapid development of K8s admission webhooks **
// -- avoids image build & push & then test in K8s

// https://github.com/banzaicloud/kurun/blob/master/kurun.go
//
// -- kubeconfig or in-cluster etc.
// -- os/exec, bytes.NewBuffer(nil), stdop, stderr
// -- kubectl get current context
// -- k8s needs are handled with default or set values e.g. cpu, mem, namespace, serviceaccount
//
// -- interrupt & done signals within goroutine
// -- cobra cli
// -- [][]byte

[AmitKumarDas]

// https://github.com/banzaicloud/hollowtrees
//

[AmitKumarDas]

// https://github.com/banzaicloud/pke

[AmitKumarDas]

// https://github.com/banzaicloud/imps
//
// image pull secrets controller

[AmitKumarDas]

// https://github.com/banzaicloud/operator-tools

[AmitKumarDas]

// https://github.com/banzaicloud/jwt-to-rbac

[AmitKumarDas]

// https://github.com/banzaicloud/dast-operator

[AmitKumarDas]

// https://github.com/open-policy-agent/cert-controller

[AmitKumarDas]

// https://github.com/cruise-automation/rbacsync 

[AmitKumarDas]

// https://github.com/cruise-automation/k-rail 
//
// policy, rbac

[AmitKumarDas]

// https://github.com/FairwindsOps/rbac-manager

[AmitKumarDas]

// https://github.com/FairwindsOps/rbac-lookup

[AmitKumarDas]

// https://github.com/banzaicloud/nodepool-labels-operator

[AmitKumarDas]

// https://github.com/banzaicloud/helm-s3

[AmitKumarDas]

// https://github.com/banzaicloud/hpa-operator

[AmitKumarDas]

// https://github.com/banzaicloud/cloudinfo

[AmitKumarDas]

// https://github.com/inlets/inlets

[AmitKumarDas]

// https://github.com/moby/buildkit

[AmitKumarDas]

// https://github.com/banzaicloud/kurun/blob/master/.goreleaser.yml

// https://github.com/banzaicloud/kurun/blob/master/.licensei.toml

[AmitKumarDas]

- record: service:response_latency_ms_bucket:rate2m
   expr: sum by (namespace, label_app_svc) (rate(response_latency_ms_bucket{direction="inbound"}[2m]) * on (namespace, pod) group_left(label_app_svc) kube_pod_labels{label_app_svc!=""})

// irate uses the two last data points of data points while rate uses the
// first and the last data point of a time range.

[AmitKumarDas]

// external DNS
// -- owner
// -- hostedzone
// -- github.com/miekg/dns
// external-dns.alpha.kubernetes.io/hostname
// service type LoadBalancer
// https proxy
// -- socks5

// github.com/miekg/dns
//

[AmitKumarDas]

// https://build.thebeat.co/dns-caching-for-kubernetes-fdd89c38c095
// https://pracucci.com/kubernetes-dns-resolution-ndots-options-and-why-it-may-affect-application-performances.html
// https://build.thebeat.co/an-elasticache-migration-story-9090a524b3f8
// https://build.thebeat.co/preparing-before-the-storm-migrating-our-monolith-into-kubernetes-581611c10ae6
// https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/
// https://build.thebeat.co/yet-another-kubernetes-dns-latency-story-2a1c00ebbb8d
// https://www.weave.works/blog/racy-conntrack-and-dns-lookup-timeouts
// https://github.com/kubernetes/kubernetes/issues/56903
// https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html#vpc-dns-limits
// https://github.com/kubernetes/kops/pull/9917
//
// NodeLocalDNSCache
// -- improves Cluster DNS performance by running a DNS caching agent as a DaemonSet
// -- additional iptables rules 
// -- DNS requests of all individual pods of the cluster will get redirected to the local node coreDNS pod
// -- that is part of the node local DNS cache daemonset
// -- Pods <== UDP ==>NodeLocalDNSCache <== TCP ==> Upstream DNS Server
//
// The “force_tcp” flag inside each zone’s configuration 
// -- will force the local coreDNS to reach the upstream server of each zone
// -- using TCP protocol if it doesn’t have a fresh response in its cache

[AmitKumarDas]

// TCP Dump or UDP Dump
//
// We were witnessing almost 40K DNS requests in 2 minutes
// Those are ~20K DNS requests per minute generated from one pod
// On scaling up to 300 pods at full load in each market
// resulting in ~6Million DNS Requests per minute hitting each Market’s k8s CoreDNS service

// The Ndots problem
//
// As our monolith did a lot of external traffic, for each connection established
// (or more specifically, for each name resolved) there were 5 DNS queries before
// the name was correctly resolved. This happens because it first goes through 
// the 4 local search domains, and then actually issues an absolute name resolution 
// query.
//
// TCPDump
// 1) 2282 A? db-monolith-reader.beat.monolith.svc.cluster.local.
// 2) 2282 A? db-monolith-reader.beat.svc.cluster.local.
// 3) 2282 A? db-monolith-reader.beat.cluster.local.
// 4) 2282 A? db-monolith-reader.beat.eu-west-1.compute.internal.
// 5) 2282 A? db-monolith-reader.beat. 

// By enabling ndots:1 we have managed to reduce the DNS requests to almost a quarter (¼) .
// Distribution of requests in the course of 2 minutes.
//
// 2519 A? db-monolith-reader.beat.
// 2512 A? kafka-monolith.beat.
//
// Summing up to ~10K Requests in 2 mins. That is a 75% Decrease!

[AmitKumarDas]

// Unbound is a validating, recursive, caching DNS resolver
//
// Rigorously Audited
// -- https://ostif.org/our-audit-of-unbound-dns-by-x41-d-sec-full-results/

// Initial tests were performed on a single monolith pod
// By installing unbound locally, 
// and manually editing /etc/resolv.conf to point to the local dns cache (unbound)
//
// Distribution of DNS Requests, as generated from one monolith pod 
// in the course of 2 minutes, with unbound local cache enabled, and ndots:1 :
//
// 24 A? db-monolith-reader.beat.
// 23 A? kafka-monolith.beat.
//
// Summing up to a total ~500 DNS Requests in 2 mins
// Compared to the ndots:1 configuration (Phase 1), that is a ~95% decrease

// setup a configmap with unbound DNS configuration
// mount this configmap to the target pods
// attach ubnound as sidecar to the target pods
// override coredns deployment spec with dnsPolicy set to None
// override coredns deployment spec with following dnsConfig

spec:  
  containers:
  dnsPolicy: None
  dnsConfig:
    nameservers:
    - 127.0.0.1    
    options:
    - name: ndots
    value: "1"
    searches:
    - mypod.svc.cluster.local

# kubectl get cm -n kube-system coredns -oyaml
data:
  Corefile: |
    .:53 {
        errors
        health
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }

[AmitKumarDas]

// Prometheus Cardinality
//
// https://www.robustperception.io/cardinality-is-key
// https://promcon.io/2019-munich/slides/containing-your-cardinality.pdf
// https://devops.stackexchange.com/questions/8189/why-is-prometheus-not-a-good-choice-for-data-with-high-cardinality
// https://github.com/thought-machine/prometheus-cardinality-exporter
// https://karlstoney.com/2020/02/25/federated-prometheus-to-reduce-metric-cardinality/
// https://blog.freshtracks.io/bomb-squad-automatic-detection-and-suppression-of-prometheus-cardinality-explosions-62ca8e02fa32
// https://grafana.com/blog/2021/05/28/how-to-alert-on-high-cardinality-data-with-grafana-loki/