change locking around

cmd/apollo-scheduler/main.go

@@ -158,14 +158,23 @@ func enqueueAccounts(ctx context.Context, logger *logrus.Logger, pool *pgxpool.P
 	skipped := 0
 	for _, id := range ids {
 		payload := fmt.Sprintf("%d", id)
-		if _, err := redisConn.HGet(ctx, "locks:accounts", payload).Result(); err != redis.Nil {
+		lockKey := fmt.Sprintf("locks:accounts:%s", payload)
+
+		_, err := redisConn.Get(ctx, lockKey).Result()
+		if err == nil || err == redis.Nil {
 			skipped++
 			continue
+		} else {
+			logger.WithFields(logrus.Fields{
+				"lockKey": lockKey,
+				"err":     err,
+			}).Error("failed to check for account lock")
 		}
 
-		if err := redisConn.HSet(ctx, "locks:accounts", payload, true).Err(); err != nil {
+		if err := redisConn.SetEX(ctx, lockKey, true, 60*time.Second).Err(); err != nil {
 			logger.WithFields(logrus.Fields{
-				"err": err,
+				"lockKey": lockKey,
+				"err":     err,
 			}).Error("failed to lock account")
 		}
 

cmd/apollo-worker-notifications/main.go

@@ -185,7 +185,13 @@ func (c *Consumer) Consume(delivery rmq.Delivery) {
 	ctx := context.Background()
 
 	defer func() {
-		c.redis.HDel(ctx, "locks:accounts", delivery.Payload()).Err()
+		lockKey := fmt.Sprintf("locks:accounts:%s", delivery.Payload())
+		if err := c.redis.Del(ctx, lockKey).Err(); err != nil {
+			c.logger.WithFields(logrus.Fields{
+				"lockKey": lockKey,
+				"err":     err,
+			}).Error("failed to remove lock")
+		}
 	}()
 
 	c.logger.WithFields(logrus.Fields{
@@ -207,6 +213,7 @@ func (c *Consumer) Consume(delivery rmq.Delivery) {
 
 	stmt := `SELECT
 			id,
+			username,
 			access_token,
 			refresh_token,
 			expires_at,
@@ -217,6 +224,7 @@ func (c *Consumer) Consume(delivery rmq.Delivery) {
 	account := &data.Account{}
 	if err := c.pool.QueryRow(ctx, stmt, id).Scan(
 		&account.ID,
+		&account.Username,
 		&account.AccessToken,
 		&account.RefreshToken,
 		&account.ExpiresAt,