Skip to content

Add configurable Kubernetes API client QPS and burst rate limits #1281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mcornea wants to merge 1 commit into
base: main
Choose a base branch
from
+12 −0
Open

Add configurable Kubernetes API client QPS and burst rate limits #1281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/node-termination-handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ func main() {
if err != nil {
log.Fatal().Err(err).Msgf("retreiving cluster config")
}
clusterConfig.QPS = float32(nthConfig.KubeApiQps)
clusterConfig.Burst = nthConfig.KubeApiBurst
clientset, err := kubernetes.NewForConfig(clusterConfig)
if err != nil {
log.Fatal().Err(err).Msgf("creating new clientset with config: %v", err)
Expand Down
10 changes: 10 additions & 0 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ const (
podTerminationGracePeriodDefault = -1
nodeTerminationGracePeriodConfigKey = "NODE_TERMINATION_GRACE_PERIOD"
nodeTerminationGracePeriodDefault = 120
kubeApiQpsConfigKey = "KUBE_API_QPS"
kubeApiQpsDefault = 5
kubeApiBurstConfigKey = "KUBE_API_BURST"
kubeApiBurstDefault = 10
webhookURLConfigKey = "WEBHOOK_URL"
webhookURLDefault = ""
webhookProxyConfigKey = "WEBHOOK_PROXY"
Expand Down Expand Up @@ -135,6 +139,8 @@ type Config struct {
KubernetesServicePort string
PodTerminationGracePeriod int
NodeTerminationGracePeriod int
KubeApiQps int
KubeApiBurst int
WebhookURL string
WebhookHeaders string
WebhookTemplate string
Expand Down Expand Up @@ -203,6 +209,8 @@ func ParseCliArgs() (config Config, err error) {
flag.IntVar(&gracePeriod, "grace-period", getIntEnv(gracePeriodConfigKey, podTerminationGracePeriodDefault), "[DEPRECATED] * Use pod-termination-grace-period instead * Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.")
flag.IntVar(&config.PodTerminationGracePeriod, "pod-termination-grace-period", getIntEnv(podTerminationGracePeriodConfigKey, podTerminationGracePeriodDefault), "Period of time in seconds given to each POD to terminate gracefully. If negative, the default value specified in the pod will be used.")
flag.IntVar(&config.NodeTerminationGracePeriod, "node-termination-grace-period", getIntEnv(nodeTerminationGracePeriodConfigKey, nodeTerminationGracePeriodDefault), "Period of time in seconds given to each NODE to terminate gracefully. Node draining will be scheduled based on this value to optimize the amount of compute time, but still safely drain the node before an event.")
flag.IntVar(&config.KubeApiQps, "kube-api-qps", getIntEnv(kubeApiQpsConfigKey, kubeApiQpsDefault), "QPS rate limit for the Kubernetes API client shared by all workers. Increase when running with high WORKERS count to avoid client-side throttling during correlated spot interruptions.")
flag.IntVar(&config.KubeApiBurst, "kube-api-burst", getIntEnv(kubeApiBurstConfigKey, kubeApiBurstDefault), "Burst rate limit for the Kubernetes API client. Should be >= kube-api-qps.")
flag.StringVar(&config.WebhookURL, "webhook-url", getEnv(webhookURLConfigKey, webhookURLDefault), "If specified, posts event data to URL upon instance interruption action.")
flag.StringVar(&config.WebhookProxy, "webhook-proxy", getEnv(webhookProxyConfigKey, webhookProxyDefault), "If specified, uses the HTTP(S) proxy to send webhooks. Example: --webhook-url='tcp://<ip-or-dns-to-proxy>:<port>'")
flag.StringVar(&config.WebhookHeaders, "webhook-headers", getEnv(webhookHeadersConfigKey, webhookHeadersDefault), "If specified, replaces the default webhook headers.")
Expand Down Expand Up @@ -351,6 +359,8 @@ func (c Config) PrintJsonConfigArgs() {
Bool("ignore_daemon_sets", c.IgnoreDaemonSets).
Int("pod_termination_grace_period", c.PodTerminationGracePeriod).
Int("node_termination_grace_period", c.NodeTerminationGracePeriod).
Int("kube_api_qps", c.KubeApiQps).
Int("kube_api_burst", c.KubeApiBurst).
Bool("enable_scheduled_event_draining", c.EnableScheduledEventDraining).
Bool("enable_spot_interruption_draining", c.EnableSpotInterruptionDraining).
Bool("enable_sqs_termination_draining", c.EnableSQSTerminationDraining).
Expand Down