apollo-backend/internal/reddit/client.go

448 lines
11 KiB
Go
Raw Normal View History

2021-05-10 00:51:15 +00:00
package reddit
import (
2022-03-12 17:50:05 +00:00
"context"
2021-09-25 16:56:01 +00:00
"fmt"
2021-05-10 00:51:15 +00:00
"io/ioutil"
"net/http"
2021-07-08 02:19:02 +00:00
"net/http/httptrace"
2021-07-12 19:51:02 +00:00
"regexp"
2022-03-12 17:50:05 +00:00
"strconv"
2021-05-10 00:51:15 +00:00
"strings"
"time"
2021-07-08 02:19:02 +00:00
"github.com/DataDog/datadog-go/statsd"
2022-03-12 17:50:05 +00:00
"github.com/go-redis/redis/v8"
"github.com/valyala/fastjson"
2021-05-10 00:51:15 +00:00
)
2022-03-12 17:50:05 +00:00
const (
SkipRateLimiting = "<SKIP_RATE_LIMITING>"
RequestRemainingBuffer = 50
2022-03-12 18:15:59 +00:00
RateLimitRemainingHeader = "X-Ratelimit-Remaining"
RateLimitResetHeader = "X-Ratelimit-Reset"
2022-03-12 17:50:05 +00:00
)
2021-05-10 00:51:15 +00:00
type Client struct {
id string
secret string
client *http.Client
2021-07-08 02:19:02 +00:00
tracer *httptrace.ClientTrace
2021-07-15 14:51:34 +00:00
pool *fastjson.ParserPool
2021-08-14 17:42:28 +00:00
statsd statsd.ClientInterface
2022-03-12 17:50:05 +00:00
redis *redis.Client
2021-05-10 00:51:15 +00:00
}
2022-03-12 18:25:34 +00:00
type RateLimitingInfo struct {
Remaining float64
Reset int
Present bool
}
2021-10-28 14:57:09 +00:00
var backoffSchedule = []time.Duration{
4 * time.Second,
8 * time.Second,
16 * time.Second,
}
2021-07-10 18:51:42 +00:00
func SplitID(id string) (string, string) {
if parts := strings.Split(id, "_"); len(parts) == 2 {
return parts[0], parts[1]
}
return "", ""
}
2021-07-12 19:51:02 +00:00
func PostIDFromContext(context string) string {
exps := []*regexp.Regexp{
regexp.MustCompile(`\/r\/[^\/]*\/comments\/([^\/]*)\/.*`),
}
for _, exp := range exps {
matches := exp.FindStringSubmatch(context)
if len(matches) != 2 {
continue
}
return matches[1]
}
return ""
}
2022-03-12 17:50:05 +00:00
func NewClient(id, secret string, statsd statsd.ClientInterface, redis *redis.Client, connLimit int) *Client {
2021-07-08 02:19:02 +00:00
tracer := &httptrace.ClientTrace{
GotConn: func(info httptrace.GotConnInfo) {
2021-07-08 02:44:46 +00:00
if info.Reused {
2021-09-25 13:19:42 +00:00
_ = statsd.Incr("reddit.api.connections.reused", []string{}, 0.1)
2021-07-08 02:44:46 +00:00
if info.WasIdle {
idleTime := float64(int64(info.IdleTime) / int64(time.Millisecond))
2021-09-25 13:19:42 +00:00
_ = statsd.Histogram("reddit.api.connections.idle_time", idleTime, []string{}, 0.1)
2021-07-08 02:44:46 +00:00
}
2021-07-08 02:19:02 +00:00
} else {
2021-09-25 13:19:42 +00:00
_ = statsd.Incr("reddit.api.connections.created", []string{}, 0.1)
2021-07-08 02:19:02 +00:00
}
},
}
t := http.DefaultTransport.(*http.Transport).Clone()
2021-07-16 00:50:04 +00:00
t.MaxIdleConns = connLimit / 4 / 100
t.MaxConnsPerHost = connLimit / 100
t.MaxIdleConnsPerHost = connLimit / 4 / 100
2021-07-14 00:34:04 +00:00
t.IdleConnTimeout = 60 * time.Second
2021-07-14 00:16:38 +00:00
t.ResponseHeaderTimeout = 5 * time.Second
client := &http.Client{Transport: t}
2021-07-15 14:51:34 +00:00
pool := &fastjson.ParserPool{}
return &Client{
id,
secret,
client,
2021-07-08 02:19:02 +00:00
tracer,
2021-07-15 14:51:34 +00:00
pool,
2021-07-08 02:19:02 +00:00
statsd,
2022-03-12 17:50:05 +00:00
redis,
}
2021-05-10 00:51:15 +00:00
}
type AuthenticatedClient struct {
*Client
2022-03-12 17:50:05 +00:00
redditId string
2021-05-10 00:51:15 +00:00
refreshToken string
accessToken string
}
2022-03-12 17:50:05 +00:00
func (rc *Client) NewAuthenticatedClient(redditId, refreshToken, accessToken string) *AuthenticatedClient {
if redditId == "" {
panic("requires a redditId")
}
return &AuthenticatedClient{rc, redditId, refreshToken, accessToken}
2021-05-10 00:51:15 +00:00
}
2022-03-12 18:25:34 +00:00
func (rc *Client) doRequest(r *Request) ([]byte, *RateLimitingInfo, error) {
2021-05-10 00:51:15 +00:00
req, err := r.HTTPRequest()
if err != nil {
2022-03-12 18:25:34 +00:00
return nil, nil, err
2021-05-10 00:51:15 +00:00
}
2021-10-28 14:57:09 +00:00
req = req.WithContext(httptrace.WithClientTrace(req.Context(), rc.tracer))
2021-07-08 02:19:02 +00:00
2021-07-08 23:26:15 +00:00
start := time.Now()
2021-10-28 14:57:09 +00:00
resp, err := rc.client.Do(req)
_ = rc.statsd.Incr("reddit.api.calls", r.tags, 0.1)
_ = rc.statsd.Histogram("reddit.api.latency", float64(time.Since(start).Milliseconds()), r.tags, 0.1)
2021-07-08 23:26:15 +00:00
2021-05-10 00:51:15 +00:00
if err != nil {
2021-10-28 14:57:09 +00:00
_ = rc.statsd.Incr("reddit.api.errors", r.tags, 0.1)
2021-08-14 18:07:19 +00:00
if strings.Contains(err.Error(), "http2: timeout awaiting response headers") {
2022-03-12 18:25:34 +00:00
return nil, nil, ErrTimeout
2021-08-14 18:07:19 +00:00
}
2022-03-12 18:25:34 +00:00
return nil, nil, err
2021-05-10 00:51:15 +00:00
}
defer resp.Body.Close()
2021-05-10 00:51:15 +00:00
2022-03-12 18:25:34 +00:00
rli := &RateLimitingInfo{Present: false}
2022-03-12 18:45:50 +00:00
if resp.Header.Get(RateLimitRemainingHeader) != "" {
2022-03-12 18:25:34 +00:00
rli.Present = true
rli.Remaining, _ = strconv.ParseFloat(resp.Header.Get(RateLimitRemainingHeader), 64)
rli.Reset, _ = strconv.Atoi(resp.Header.Get(RateLimitResetHeader))
2022-03-12 17:50:05 +00:00
}
2021-10-28 14:57:09 +00:00
if resp.StatusCode != 200 {
_ = rc.statsd.Incr("reddit.api.errors", r.tags, 0.1)
2022-03-12 18:25:34 +00:00
return nil, rli, ServerError{resp.StatusCode}
2021-10-28 14:57:09 +00:00
}
2021-07-12 18:36:08 +00:00
bb, err := ioutil.ReadAll(resp.Body)
if err != nil {
2021-10-28 14:57:09 +00:00
_ = rc.statsd.Incr("reddit.api.errors", r.tags, 0.1)
2022-03-12 18:25:34 +00:00
return nil, rli, err
2021-07-12 18:36:08 +00:00
}
2022-03-12 18:25:34 +00:00
return bb, rli, nil
2021-10-28 14:57:09 +00:00
}
2021-07-15 14:51:34 +00:00
2021-10-28 14:57:09 +00:00
func (rac *AuthenticatedClient) request(r *Request, rh ResponseHandler, empty interface{}) (interface{}, error) {
2022-03-12 18:45:50 +00:00
if rac.isRateLimited() {
2022-03-12 17:50:05 +00:00
return nil, ErrRateLimited
}
2022-03-12 18:25:34 +00:00
bb, rli, err := rac.doRequest(r)
2022-03-12 17:50:05 +00:00
2021-10-28 14:57:09 +00:00
if err != nil && r.retry {
for _, backoff := range backoffSchedule {
done := make(chan struct{})
time.AfterFunc(backoff, func() {
_ = rac.statsd.Incr("reddit.api.retries", r.tags, 0.1)
2022-03-12 18:25:34 +00:00
bb, rli, err = rac.doRequest(r)
2021-10-28 14:57:09 +00:00
done <- struct{}{}
})
<-done
if err == nil {
break
}
}
}
if err != nil {
2021-09-25 13:19:42 +00:00
_ = rac.statsd.Incr("reddit.api.errors", r.tags, 0.1)
2021-10-28 14:57:09 +00:00
if strings.Contains(err.Error(), "http2: timeout awaiting response headers") {
return nil, ErrTimeout
}
return nil, err
2022-03-12 18:45:50 +00:00
} else {
rac.markRateLimited(rli)
2021-07-12 18:36:08 +00:00
}
if r.emptyResponseBytes > 0 && len(bb) == r.emptyResponseBytes {
return empty, nil
}
parser := rac.pool.Get()
defer rac.pool.Put(parser)
2021-07-15 15:51:04 +00:00
val, err := parser.ParseBytes(bb)
if err != nil {
return nil, err
}
return rh(val), nil
2021-05-10 00:51:15 +00:00
}
2022-03-12 18:45:50 +00:00
func (rac *AuthenticatedClient) isRateLimited() bool {
2022-03-12 17:50:05 +00:00
if rac.redditId == SkipRateLimiting {
2022-03-12 18:45:50 +00:00
return false
2022-03-12 17:50:05 +00:00
}
key := fmt.Sprintf("reddit:%s:ratelimited", rac.redditId)
2022-03-12 17:58:07 +00:00
_, err := rac.redis.Get(context.Background(), key).Result()
2022-03-12 18:45:50 +00:00
return err != redis.Nil
2022-03-12 17:50:05 +00:00
}
2022-03-12 18:45:50 +00:00
func (rac *AuthenticatedClient) markRateLimited(rli *RateLimitingInfo) error {
2022-03-12 17:50:05 +00:00
if rac.redditId == SkipRateLimiting {
return ErrRequiresRedditId
}
2022-03-12 18:45:50 +00:00
if !rli.Present {
return nil
}
if rli.Remaining > RequestRemainingBuffer {
return nil
}
_ = rac.statsd.Incr("reddit.api.ratelimit", nil, 0.1)
2022-03-12 17:50:05 +00:00
key := fmt.Sprintf("reddit:%s:ratelimited", rac.redditId)
2022-03-12 18:45:50 +00:00
duration := time.Duration(rli.Reset) * time.Second
_, err := rac.redis.SetEX(context.Background(), key, rli.Remaining, duration).Result()
2022-03-12 17:50:05 +00:00
return err
}
2021-06-24 02:19:43 +00:00
func (rac *AuthenticatedClient) RefreshTokens() (*RefreshTokenResponse, error) {
2021-05-10 00:51:15 +00:00
req := NewRequest(
2021-07-08 23:26:15 +00:00
WithTags([]string{"url:/api/v1/access_token"}),
2021-05-10 00:51:15 +00:00
WithMethod("POST"),
2021-07-15 22:47:11 +00:00
WithURL("https://www.reddit.com/api/v1/access_token"),
2021-05-10 00:51:15 +00:00
WithBody("grant_type", "refresh_token"),
WithBody("refresh_token", rac.refreshToken),
WithBasicAuth(rac.id, rac.secret),
)
rtr, err := rac.request(req, NewRefreshTokenResponse, nil)
2021-06-24 02:19:43 +00:00
if err != nil {
2021-08-14 17:42:28 +00:00
switch rerr := err.(type) {
case ServerError:
if rerr.StatusCode == 400 {
return nil, ErrOauthRevoked
}
}
2021-06-24 02:19:43 +00:00
return nil, err
}
ret := rtr.(*RefreshTokenResponse)
if ret.RefreshToken == "" {
ret.RefreshToken = rac.refreshToken
}
return ret, nil
2021-05-10 00:51:15 +00:00
}
2021-10-17 14:17:41 +00:00
func (rac *AuthenticatedClient) AboutInfo(fullname string, opts ...RequestOption) (*ListingResponse, error) {
opts = append([]RequestOption{
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL("https://oauth.reddit.com/api/info"),
WithQuery("id", fullname),
}, opts...)
req := NewRequest(opts...)
lr, err := rac.request(req, NewListingResponse, nil)
if err != nil {
return nil, err
}
return lr.(*ListingResponse), nil
}
2021-10-09 14:59:20 +00:00
func (rac *AuthenticatedClient) UserPosts(user string, opts ...RequestOption) (*ListingResponse, error) {
url := fmt.Sprintf("https://oauth.reddit.com/u/%s/submitted.json", user)
opts = append([]RequestOption{
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL(url),
}, opts...)
req := NewRequest(opts...)
lr, err := rac.request(req, NewListingResponse, nil)
if err != nil {
return nil, err
}
return lr.(*ListingResponse), nil
}
func (rac *AuthenticatedClient) UserAbout(user string, opts ...RequestOption) (*UserResponse, error) {
url := fmt.Sprintf("https://oauth.reddit.com/u/%s/about.json", user)
opts = append([]RequestOption{
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL(url),
}, opts...)
req := NewRequest(opts...)
ur, err := rac.request(req, NewUserResponse, nil)
if err != nil {
return nil, err
}
return ur.(*UserResponse), nil
}
2021-09-25 16:56:01 +00:00
func (rac *AuthenticatedClient) SubredditAbout(subreddit string, opts ...RequestOption) (*SubredditResponse, error) {
url := fmt.Sprintf("https://oauth.reddit.com/r/%s/about.json", subreddit)
opts = append([]RequestOption{
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL(url),
}, opts...)
req := NewRequest(opts...)
sr, err := rac.request(req, NewSubredditResponse, nil)
if err != nil {
return nil, err
}
return sr.(*SubredditResponse), nil
}
2021-09-25 17:05:05 +00:00
func (rac *AuthenticatedClient) subredditPosts(subreddit string, sort string, opts ...RequestOption) (*ListingResponse, error) {
url := fmt.Sprintf("https://oauth.reddit.com/r/%s/%s.json", subreddit, sort)
2021-09-25 16:56:01 +00:00
opts = append([]RequestOption{
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL(url),
}, opts...)
req := NewRequest(opts...)
2021-09-25 18:02:00 +00:00
lr, err := rac.request(req, NewListingResponse, nil)
2021-09-25 16:56:01 +00:00
if err != nil {
return nil, err
}
return lr.(*ListingResponse), nil
}
2021-09-25 17:05:05 +00:00
func (rac *AuthenticatedClient) SubredditHot(subreddit string, opts ...RequestOption) (*ListingResponse, error) {
return rac.subredditPosts(subreddit, "hot", opts...)
}
2021-10-10 15:51:42 +00:00
func (rac *AuthenticatedClient) SubredditTop(subreddit string, opts ...RequestOption) (*ListingResponse, error) {
return rac.subredditPosts(subreddit, "top", opts...)
}
2021-09-25 17:05:05 +00:00
func (rac *AuthenticatedClient) SubredditNew(subreddit string, opts ...RequestOption) (*ListingResponse, error) {
return rac.subredditPosts(subreddit, "new", opts...)
}
2021-07-15 14:51:34 +00:00
func (rac *AuthenticatedClient) MessageInbox(opts ...RequestOption) (*ListingResponse, error) {
opts = append([]RequestOption{
2021-07-08 23:55:14 +00:00
WithTags([]string{"url:/api/v1/message/inbox"}),
2021-05-10 00:51:15 +00:00
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL("https://oauth.reddit.com/message/inbox.json"),
WithEmptyResponseBytes(122),
2021-07-15 14:51:34 +00:00
}, opts...)
req := NewRequest(opts...)
2021-06-24 02:19:43 +00:00
lr, err := rac.request(req, NewListingResponse, EmptyListingResponse)
2021-07-15 00:52:51 +00:00
if err != nil {
2021-08-14 17:42:28 +00:00
switch rerr := err.(type) {
case ServerError:
if rerr.StatusCode == 403 {
return nil, ErrOauthRevoked
}
}
2021-07-15 00:52:51 +00:00
return nil, err
}
2021-07-15 15:51:04 +00:00
return lr.(*ListingResponse), nil
2021-05-10 00:51:15 +00:00
}
2021-07-15 14:51:34 +00:00
func (rac *AuthenticatedClient) MessageUnread(opts ...RequestOption) (*ListingResponse, error) {
opts = append([]RequestOption{
WithTags([]string{"url:/api/v1/message/unread"}),
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL("https://oauth.reddit.com/message/unread.json"),
WithEmptyResponseBytes(122),
2021-07-15 14:51:34 +00:00
}, opts...)
2021-07-15 14:51:34 +00:00
req := NewRequest(opts...)
lr, err := rac.request(req, NewListingResponse, EmptyListingResponse)
if err != nil {
2021-08-14 17:42:28 +00:00
switch rerr := err.(type) {
case ServerError:
if rerr.StatusCode == 403 {
return nil, ErrOauthRevoked
}
}
return nil, err
}
2021-07-15 15:51:04 +00:00
return lr.(*ListingResponse), nil
}
2021-05-10 00:51:15 +00:00
func (rac *AuthenticatedClient) Me() (*MeResponse, error) {
req := NewRequest(
2021-07-08 23:26:15 +00:00
WithTags([]string{"url:/api/v1/me"}),
2021-05-10 00:51:15 +00:00
WithMethod("GET"),
WithToken(rac.accessToken),
WithURL("https://oauth.reddit.com/api/v1/me"),
)
mr, err := rac.request(req, NewMeResponse, nil)
2021-05-10 00:51:15 +00:00
if err != nil {
2021-08-14 17:42:28 +00:00
switch rerr := err.(type) {
case ServerError:
if rerr.StatusCode == 403 {
return nil, ErrOauthRevoked
}
}
2021-05-10 00:51:15 +00:00
return nil, err
}
2021-07-15 15:51:04 +00:00
return mr.(*MeResponse), nil
2021-05-10 00:51:15 +00:00
}