mirror of
https://github.com/christianselig/apollo-backend
synced 2024-11-10 22:17:44 +00:00
rejig sql
This commit is contained in:
parent
9c70cee166
commit
d65bab7970
1 changed files with 35 additions and 9 deletions
|
@ -9,6 +9,7 @@ import (
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/DataDog/datadog-go/statsd"
|
||||||
"github.com/adjust/rmq/v4"
|
"github.com/adjust/rmq/v4"
|
||||||
"github.com/go-co-op/gocron"
|
"github.com/go-co-op/gocron"
|
||||||
"github.com/go-redis/redis/v8"
|
"github.com/go-redis/redis/v8"
|
||||||
|
@ -39,6 +40,13 @@ func main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
statsd, err := statsd.New("127.0.0.1:8125")
|
||||||
|
if err != nil {
|
||||||
|
logger.WithFields(logrus.Fields{
|
||||||
|
"err": err,
|
||||||
|
}).Error("failed to set up stats")
|
||||||
|
}
|
||||||
|
|
||||||
// Set up Postgres connection
|
// Set up Postgres connection
|
||||||
var pool *pgxpool.Pool
|
var pool *pgxpool.Pool
|
||||||
{
|
{
|
||||||
|
@ -89,7 +97,7 @@ func main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
s := gocron.NewScheduler(time.UTC)
|
s := gocron.NewScheduler(time.UTC)
|
||||||
s.Every(1).Second().Do(func() { enqueueAccounts(ctx, logger, pool, redisConn, notificationsQueue) })
|
s.Every(1).Second().Do(func() { enqueueAccounts(ctx, logger, statsd, pool, redisConn, notificationsQueue) })
|
||||||
s.StartAsync()
|
s.StartAsync()
|
||||||
|
|
||||||
signals := make(chan os.Signal, 1)
|
signals := make(chan os.Signal, 1)
|
||||||
|
@ -106,11 +114,16 @@ func main() {
|
||||||
s.Stop()
|
s.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
func enqueueAccounts(ctx context.Context, logger *logrus.Logger, pool *pgxpool.Pool, redisConn *redis.Client, queue rmq.Queue) {
|
func enqueueAccounts(ctx context.Context, logger *logrus.Logger, statsd *statsd.Client, pool *pgxpool.Pool, redisConn *redis.Client, queue rmq.Queue) {
|
||||||
now := float64(time.Now().UnixNano()/int64(time.Millisecond)) / 1000
|
now := float64(time.Now().UnixNano()/int64(time.Millisecond)) / 1000
|
||||||
|
|
||||||
start := time.Now().Unix()
|
// Start looking for accounts that were last checked at least 5 seconds ago
|
||||||
|
// and at most 6 seconds ago. Also look for accounts that haven't been checked
|
||||||
|
// in over a minute.
|
||||||
|
ts := time.Now().Unix()
|
||||||
|
start := ts - 6
|
||||||
end := start + 1
|
end := start + 1
|
||||||
|
expired := ts - 60
|
||||||
|
|
||||||
ids := []int64{}
|
ids := []int64{}
|
||||||
|
|
||||||
|
@ -120,15 +133,15 @@ func enqueueAccounts(ctx context.Context, logger *logrus.Logger, pool *pgxpool.P
|
||||||
SELECT id
|
SELECT id
|
||||||
FROM accounts
|
FROM accounts
|
||||||
WHERE
|
WHERE
|
||||||
(last_checked_at + 5) BETWEEN $1 and $2
|
last_checked_at BETWEEN $1 AND $2
|
||||||
OR last_checked_at + 60 <= $1
|
OR last_checked_at < $3
|
||||||
ORDER BY last_checked_at
|
ORDER BY last_checked_at
|
||||||
)
|
)
|
||||||
UPDATE accounts
|
UPDATE accounts
|
||||||
SET last_enqueued_at = $3
|
SET last_enqueued_at = $4
|
||||||
WHERE accounts.id IN(SELECT id FROM account)
|
WHERE accounts.id IN(SELECT id FROM account)
|
||||||
RETURNING accounts.id`
|
RETURNING accounts.id`
|
||||||
rows, err := tx.Query(ctx, stmt, start, end, now)
|
rows, err := tx.Query(ctx, stmt, start, end, expired, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -156,6 +169,7 @@ func enqueueAccounts(ctx context.Context, logger *logrus.Logger, pool *pgxpool.P
|
||||||
|
|
||||||
enqueued := 0
|
enqueued := 0
|
||||||
skipped := 0
|
skipped := 0
|
||||||
|
failed := 0
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
payload := fmt.Sprintf("%d", id)
|
payload := fmt.Sprintf("%d", id)
|
||||||
lockKey := fmt.Sprintf("locks:accounts:%s", payload)
|
lockKey := fmt.Sprintf("locks:accounts:%s", payload)
|
||||||
|
@ -178,13 +192,25 @@ func enqueueAccounts(ctx context.Context, logger *logrus.Logger, pool *pgxpool.P
|
||||||
}).Error("failed to lock account")
|
}).Error("failed to lock account")
|
||||||
}
|
}
|
||||||
|
|
||||||
enqueued++
|
if err = queue.Publish(payload); err != nil {
|
||||||
_ = queue.Publish(payload)
|
logger.WithFields(logrus.Fields{
|
||||||
|
"accountID": payload,
|
||||||
|
"err": err,
|
||||||
|
}).Error("failed to enqueue account")
|
||||||
|
failed++
|
||||||
|
} else {
|
||||||
|
enqueued++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
statsd.Histogram("apollo.queue.enqueued", float64(enqueued), []string{}, 1)
|
||||||
|
statsd.Histogram("apollo.queue.skipped", float64(skipped), []string{}, 1)
|
||||||
|
statsd.Histogram("apollo.queue.failed", float64(failed), []string{}, 1)
|
||||||
|
|
||||||
logger.WithFields(logrus.Fields{
|
logger.WithFields(logrus.Fields{
|
||||||
"count": enqueued,
|
"count": enqueued,
|
||||||
"skipped": skipped,
|
"skipped": skipped,
|
||||||
|
"failed": failed,
|
||||||
"start": start,
|
"start": start,
|
||||||
"end": end,
|
"end": end,
|
||||||
}).Info("done enqueueing account batch")
|
}).Info("done enqueueing account batch")
|
||||||
|
|
Loading…
Reference in a new issue