mirrors/apollo-backend
2
0
Fork 0
mirror of https://github.com/christianselig/apollo-backend synced 2024-11-22 11:47:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

tweak params

Browse source
This commit is contained in:
Andre Medeiros 2021-07-13 20:09:44 -04:00
parent 47ee2a6e52
commit 55c45592ac
6 changed files with 26 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Procfile
View file

@ -1,3 +1,3 @@
web: apollo api web: apollo api
scheduler: apollo scheduler scheduler: apollo scheduler
worker-notifications: apollo worker --queue notifications --multiplier 12 worker-notifications: apollo worker --queue notifications --multiplier 32

1
internal/api/api.go
View file

@ -28,6 +28,7 @@ func NewAPI(ctx context.Context, logger *logrus.Logger, statsd *statsd.Client, d
os.Getenv("REDDIT_CLIENT_ID"), os.Getenv("REDDIT_CLIENT_ID"),
os.Getenv("REDDIT_CLIENT_SECRET"), os.Getenv("REDDIT_CLIENT_SECRET"),
statsd, statsd,
16,
) )
models := data.NewModels(ctx, db) models := data.NewModels(ctx, db)

4
internal/cmd/worker.go
View file

@ -62,8 +62,8 @@ func WorkerCmd(ctx context.Context) *cobra.Command {
consumers := runtime.NumCPU() * multiplier consumers := runtime.NumCPU() * multiplier
worker := workerFn(logger, statsd, db, redis, queue) worker := workerFn(logger, statsd, db, redis, queue, consumers)
worker.Start(consumers) worker.Start()
<-ctx.Done() <-ctx.Done()

9
internal/reddit/client.go
View file

@ -50,7 +50,7 @@ func PostIDFromContext(context string) string {
return "" return ""
} }
func NewClient(id, secret string, statsd *statsd.Client) *Client { func NewClient(id, secret string, statsd *statsd.Client, connLimit int) *Client {
tracer := &httptrace.ClientTrace{ tracer := &httptrace.ClientTrace{
GotConn: func(info httptrace.GotConnInfo) { GotConn: func(info httptrace.GotConnInfo) {
if info.Reused { if info.Reused {
@ -66,9 +66,10 @@ func NewClient(id, secret string, statsd *statsd.Client) *Client {
} }
t := http.DefaultTransport.(*http.Transport).Clone() t := http.DefaultTransport.(*http.Transport).Clone()
t.MaxIdleConns = 128 t.MaxIdleConns = connLimit / 4
t.MaxConnsPerHost = 512 t.MaxConnsPerHost = connLimit
t.MaxIdleConnsPerHost = 128 t.MaxIdleConnsPerHost = connLimit / 4
t.IdleConnTimeout = 10 * time.Second
client := &http.Client{Transport: t} client := &http.Client{Transport: t}

27
internal/worker/notifications.go
View file

@ -28,20 +28,22 @@ const (
) )
type notificationsWorker struct { type notificationsWorker struct {
logger *logrus.Logger logger *logrus.Logger
statsd *statsd.Client statsd *statsd.Client
db *pgxpool.Pool db *pgxpool.Pool
redis *redis.Client redis *redis.Client
queue rmq.Connection queue rmq.Connection
reddit *reddit.Client reddit *reddit.Client
apns *token.Token apns *token.Token
consumers int
} }
func NewNotificationsWorker(logger *logrus.Logger, statsd *statsd.Client, db *pgxpool.Pool, redis *redis.Client, queue rmq.Connection) Worker { func NewNotificationsWorker(logger *logrus.Logger, statsd *statsd.Client, db *pgxpool.Pool, redis *redis.Client, queue rmq.Connection, consumers int) Worker {
reddit := reddit.NewClient( reddit := reddit.NewClient(
os.Getenv("REDDIT_CLIENT_ID"), os.Getenv("REDDIT_CLIENT_ID"),
os.Getenv("REDDIT_CLIENT_SECRET"), os.Getenv("REDDIT_CLIENT_SECRET"),
statsd, statsd,
consumers,
) )
var apns *token.Token var apns *token.Token
@ -66,20 +68,21 @@ func NewNotificationsWorker(logger *logrus.Logger, statsd *statsd.Client, db *pg
queue, queue,
reddit, reddit,
apns, apns,
consumers,
} }
} }
func (nw *notificationsWorker) Start(consumers int) error { func (nw *notificationsWorker) Start() error {
queue, err := nw.queue.OpenQueue("notifications") queue, err := nw.queue.OpenQueue("notifications")
if err != nil { if err != nil {
return err return err
} }
nw.logger.WithFields(logrus.Fields{ nw.logger.WithFields(logrus.Fields{
"numConsumers": consumers, "numConsumers": nw.consumers,
}).Info("starting up notifications worker") }).Info("starting up notifications worker")
prefetchLimit := int64(consumers * 2) prefetchLimit := int64(nw.consumers * 2)
if err := queue.StartConsuming(prefetchLimit, pollDuration); err != nil { if err := queue.StartConsuming(prefetchLimit, pollDuration); err != nil {
return err return err
@ -87,7 +90,7 @@ func (nw *notificationsWorker) Start(consumers int) error {
host, _ := os.Hostname() host, _ := os.Hostname()
for i := 0; i < consumers; i++ { for i := 0; i < nw.consumers; i++ {
name := fmt.Sprintf("consumer %s-%d", host, i) name := fmt.Sprintf("consumer %s-%d", host, i)
consumer := NewNotificationsConsumer(nw, i) consumer := NewNotificationsConsumer(nw, i)

5
internal/worker/worker.go
View file

@ -8,9 +8,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
type NewWorkerFn func(logger *logrus.Logger, statsd *statsd.Client, db *pgxpool.Pool, redis *redis.Client, queue rmq.Connection) Worker type NewWorkerFn func(logger *logrus.Logger, statsd *statsd.Client, db *pgxpool.Pool, redis *redis.Client, queue rmq.Connection, consumers int) Worker
type Worker interface { type Worker interface {
Start(int) error Start() error
Stop() Stop()
} }