From 0e342803a1dba337a49b5858964ed44e90399e19 Mon Sep 17 00:00:00 2001 From: Andre Medeiros Date: Mon, 31 Oct 2022 18:04:01 -0400 Subject: [PATCH] don't go crazy on emitting stats --- internal/worker/notifications.go | 7 +++++-- render.yaml | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/worker/notifications.go b/internal/worker/notifications.go index 0a4846f..616309c 100644 --- a/internal/worker/notifications.go +++ b/internal/worker/notifications.go @@ -29,6 +29,8 @@ const ( usernameMentionNotificationTitleFormat = "Mention in \u201c%s\u201d" ) +var notificationTags = []string{"queue:notifications"} + type notificationsWorker struct { context.Context @@ -138,7 +140,8 @@ func (nc *notificationsConsumer) Consume(delivery rmq.Delivery) { now := time.Now() defer func() { elapsed := time.Now().Sub(now).Milliseconds() - _ = nc.statsd.Histogram("apollo.consumer.runtime", float64(elapsed), []string{"queue:notifications"}, 0.1) + _ = nc.statsd.Histogram("apollo.consumer.runtime", float64(elapsed), notificationTags, 0.1) + _ = nc.statsd.Incr("apollo.consumer.executions", notificationTags, 0.1) }() id := delivery.Payload() @@ -147,7 +150,7 @@ func (nc *notificationsConsumer) Consume(delivery rmq.Delivery) { // Measure queue latency ttl := nc.redis.PTTL(ctx, key).Val() age := (domain.NotificationCheckTimeout - ttl) - _ = nc.statsd.Histogram("apollo.dequeue.latency", float64(age.Milliseconds()), []string{"queue:notifications"}, 0.1) + _ = nc.statsd.Histogram("apollo.dequeue.latency", float64(age.Milliseconds()), notificationTags, 0.1) defer func() { if err := nc.redis.Del(ctx, key).Err(); err != nil { diff --git a/render.yaml b/render.yaml index 7ac0064..765196f 100644 --- a/render.yaml +++ b/render.yaml @@ -87,11 +87,11 @@ services: - key: BUGSNAG_METADATA_QUEUE value: notifications scaling: - minInstances: 20 + minInstances: 30 maxInstances: 50 targetCPUPercent: 60 buildCommand: go install github.com/bugsnag/panic-monitor@latest && go build ./cmd/apollo - startCommand: panic-monitor ./apollo worker --queue notifications --consumers 512 + startCommand: panic-monitor ./apollo worker --queue notifications --consumers 1024 # Stuck Notifications Checker - type: worker