From cc1d932781f35bea2fe8b8c469cbc796bfce645c Mon Sep 17 00:00:00 2001 From: Andre Medeiros Date: Tue, 13 Jul 2021 19:33:25 -0400 Subject: [PATCH] tweak default transport and try using message unread --- config/pgbouncer.ini | 358 ------------------------------- internal/reddit/client.go | 27 ++- internal/worker/notifications.go | 2 +- 3 files changed, 27 insertions(+), 360 deletions(-) delete mode 100644 config/pgbouncer.ini diff --git a/config/pgbouncer.ini b/config/pgbouncer.ini deleted file mode 100644 index d842f31..0000000 --- a/config/pgbouncer.ini +++ /dev/null @@ -1,358 +0,0 @@ -;;; -;;; PgBouncer configuration file -;;; - -;; database name = connect string -;; -;; connect string params: -;; dbname= host= port= user= password= auth_user= -;; client_encoding= datestyle= timezone= -;; pool_size= reserve_pool= max_db_connections= -;; pool_mode= connect_query= application_name= -[databases] -apollo = host=localhost dbname=apollo user=apollo -;; foodb over Unix socket -;foodb = - -;; redirect bardb to bazdb on localhost -;bardb = host=localhost dbname=bazdb - -;; access to dest database will go with single user -;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' - -;; use custom pool sizes -;nondefaultdb = pool_size=50 reserve_pool=10 - -;; use auth_user with auth_query if user not present in auth_file -;; auth_user must exist in auth_file -; foodb = auth_user=bar - -;; fallback connect string -;* = host=testserver - -;; User-specific configuration -[users] -;user1 = pool_mode=transaction max_user_connections=10 - -;; Configuration section -[pgbouncer] - -;;; -;;; Administrative settings -;;; - -logfile = /usr/local/var/log/pgbouncer.log -pidfile = /usr/local/var/run/pgbouncer.pid - -;;; -;;; Where to wait for clients -;;; - -;; IP address or * which means all IPs -listen_addr = localhost -listen_port = 6432 - -;; Unix socket is also used for -R. -;; On Debian it should be /var/run/postgresql -;unix_socket_dir = /tmp -;unix_socket_mode = 0777 -;unix_socket_group = - -;;; -;;; TLS settings for accepting clients -;;; - -;; disable, allow, require, verify-ca, verify-full -;client_tls_sslmode = disable - -;; Path to file that contains trusted CA certs -;client_tls_ca_file = - -;; Private key and cert to present to clients. -;; Required for accepting TLS connections from clients. -;client_tls_key_file = -;client_tls_cert_file = - -;; fast, normal, secure, legacy, -;client_tls_ciphers = fast - -;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 -;client_tls_protocols = secure - -;; none, auto, legacy -;client_tls_dheparams = auto - -;; none, auto, -;client_tls_ecdhcurve = auto - -;;; -;;; TLS settings for connecting to backend databases -;;; - -;; disable, allow, require, verify-ca, verify-full -;server_tls_sslmode = disable - -;; Path to that contains trusted CA certs -;server_tls_ca_file = - -;; Private key and cert to present to backend. -;; Needed only if backend server require client cert. -;server_tls_key_file = -;server_tls_cert_file = - -;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 -;server_tls_protocols = secure - -;; fast, normal, secure, legacy, -;server_tls_ciphers = fast - -;;; -;;; Authentication settings -;;; - -;; any, trust, plain, md5, cert, hba, pam -auth_type = any -;auth_file = /usr/local/etc/userlist.txt - -;; Path to HBA-style auth config -;auth_hba_file = - -;; Query to use to fetch password from database. Result -;; must have 2 columns - username and password hash. -;auth_query = SELECT usename, passwd FROM pg_shadow WHERE usename=$1 - -;;; -;;; Users allowed into database 'pgbouncer' -;;; - -;; comma-separated list of users who are allowed to change settings -;admin_users = user2, someadmin, otheradmin - -;; comma-separated list of users who are just allowed to use SHOW command -;stats_users = stats, root - -;;; -;;; Pooler personality questions -;;; - -;; When server connection is released back to pool: -;; session - after client disconnects (default) -;; transaction - after transaction finishes -;; statement - after statement finishes -;pool_mode = session - -;; Query for cleaning connection immediately after releasing from -;; client. No need to put ROLLBACK here, pgbouncer does not reuse -;; connections where transaction is left open. -;server_reset_query = DISCARD ALL - -;; Whether server_reset_query should run in all pooling modes. If it -;; is off, server_reset_query is used only for session-pooling. -;server_reset_query_always = 0 - -;; Comma-separated list of parameters to ignore when given in startup -;; packet. Newer JDBC versions require the extra_float_digits here. -;ignore_startup_parameters = extra_float_digits - -;; When taking idle server into use, this query is run first. -;server_check_query = select 1 - -;; If server was used more recently that this many seconds ago, -; skip the check query. Value 0 may or may not run in immediately. -;server_check_delay = 30 - -;; Close servers in session pooling mode after a RECONNECT, RELOAD, -;; etc. when they are idle instead of at the end of the session. -;server_fast_close = 0 - -;; Use as application_name on server. -;application_name_add_host = 0 - -;; Period for updating aggregated stats. -;stats_period = 60 - -;;; -;;; Connection limits -;;; - -;; Total number of clients that can connect -;max_client_conn = 100 - -;; Default pool size. 20 is good number when transaction pooling -;; is in use, in session pooling it needs to be the number of -;; max clients you want to handle at any moment -;default_pool_size = 20 - -;; Minimum number of server connections to keep in pool. -;min_pool_size = 0 - -; how many additional connection to allow in case of trouble -;reserve_pool_size = 0 - -;; If a clients needs to wait more than this many seconds, use reserve -;; pool. -;reserve_pool_timeout = 5 - -;; Maximum number of server connections for a database -;max_db_connections = 0 - -;; Maximum number of server connections for a user -;max_user_connections = 0 - -;; If off, then server connections are reused in LIFO manner -;server_round_robin = 0 - -;;; -;;; Logging -;;; - -;; Syslog settings -;syslog = 0 -;syslog_facility = daemon -;syslog_ident = pgbouncer - -;; log if client connects or server connection is made -;log_connections = 1 - -;; log if and why connection was closed -;log_disconnections = 1 - -;; log error messages pooler sends to clients -;log_pooler_errors = 1 - -;; write aggregated stats into log -;log_stats = 1 - -;; Logging verbosity. Same as -v switch on command line. -;verbose = 0 - -;;; -;;; Timeouts -;;; - -;; Close server connection if its been connected longer. -;server_lifetime = 3600 - -;; Close server connection if its not been used in this time. Allows -;; to clean unnecessary connections from pool after peak. -;server_idle_timeout = 600 - -;; Cancel connection attempt if server does not answer takes longer. -;server_connect_timeout = 15 - -;; If server login failed (server_connect_timeout or auth failure) -;; then wait this many second. -;server_login_retry = 15 - -;; Dangerous. Server connection is closed if query does not return in -;; this time. Should be used to survive network problems, _not_ as -;; statement_timeout. (default: 0) -;query_timeout = 0 - -;; Dangerous. Client connection is closed if the query is not -;; assigned to a server in this time. Should be used to limit the -;; number of queued queries in case of a database or network -;; failure. (default: 120) -;query_wait_timeout = 120 - -;; Dangerous. Client connection is closed if no activity in this -;; time. Should be used to survive network problems. (default: 0) -;client_idle_timeout = 0 - -;; Disconnect clients who have not managed to log in after connecting -;; in this many seconds. -;client_login_timeout = 60 - -;; Clean automatically created database entries (via "*") if they stay -;; unused in this many seconds. -; autodb_idle_timeout = 3600 - -;; Close connections which are in "IDLE in transaction" state longer -;; than this many seconds. -;idle_transaction_timeout = 0 - -;; How long SUSPEND/-R waits for buffer flush before closing -;; connection. -;suspend_timeout = 10 - -;;; -;;; Low-level tuning options -;;; - -;; buffer for streaming packets -;pkt_buf = 4096 - -;; man 2 listen -;listen_backlog = 128 - -;; Max number pkt_buf to process in one event loop. -;sbuf_loopcnt = 5 - -;; Maximum PostgreSQL protocol packet size. -;max_packet_size = 2147483647 - -;; Set SO_REUSEPORT socket option -;so_reuseport = 0 - -;; networking options, for info: man 7 tcp - -;; Linux: Notify program about new connection only if there is also -;; data received. (Seconds to wait.) On Linux the default is 45, on -;; other OS'es 0. -;tcp_defer_accept = 0 - -;; In-kernel buffer size (Linux default: 4096) -;tcp_socket_buffer = 0 - -;; whether tcp keepalive should be turned on (0/1) -;tcp_keepalive = 1 - -;; The following options are Linux-specific. They also require -;; tcp_keepalive=1. - -;; Count of keepalive packets -;tcp_keepcnt = 0 - -;; How long the connection can be idle before sending keepalive -;; packets -;tcp_keepidle = 0 - -;; The time between individual keepalive probes -;tcp_keepintvl = 0 - -;; How long may transmitted data remain unacknowledged before TCP -;; connection is closed (in milliseconds) -;tcp_user_timeout = 0 - -;; DNS lookup caching time -;dns_max_ttl = 15 - -;; DNS zone SOA lookup period -;dns_zone_check_period = 0 - -;; DNS negative result caching time -;dns_nxdomain_ttl = 15 - -;; Custom resolv.conf file, to set custom DNS servers or other options -;; (default: empty = use OS settings) -;resolv_conf = /etc/pgbouncer/resolv.conf - -;;; -;;; Random stuff -;;; - -;; Hackish security feature. Helps against SQL injection: when PQexec -;; is disabled, multi-statement cannot be made. -;disable_pqexec = 0 - -;; Config file to use for next RELOAD/SIGHUP -;; By default contains config file from command line. -;conffile - -;; Windows service name to register as. job_name is alias for -;; service_name, used by some Skytools scripts. -;service_name = pgbouncer -;job_name = pgbouncer - -;; Read additional config from other file -;%include /etc/pgbouncer/pgbouncer-other.ini diff --git a/internal/reddit/client.go b/internal/reddit/client.go index c53beaf..7821fe0 100644 --- a/internal/reddit/client.go +++ b/internal/reddit/client.go @@ -65,7 +65,12 @@ func NewClient(id, secret string, statsd *statsd.Client) *Client { }, } - client := &http.Client{} + t := http.DefaultTransport.(*http.Transport).Clone() + t.MaxIdleConns = 128 + t.MaxConnsPerHost = 512 + t.MaxIdleConnsPerHost = 128 + + client := &http.Client{Transport: t} parser := &fastjson.Parser{} @@ -168,6 +173,26 @@ func (rac *AuthenticatedClient) MessageInbox(from string) (*MessageListingRespon return mlr, nil } +func (rac *AuthenticatedClient) MessageUnread(from string) (*MessageListingResponse, error) { + req := NewRequest( + WithTags([]string{"url:/api/v1/message/unread"}), + WithMethod("GET"), + WithToken(rac.accessToken), + WithURL("https://oauth.reddit.com/message/unread.json"), + WithQuery("before", from), + ) + + body, err := rac.request(req) + + if err != nil { + return nil, err + } + + mlr := &MessageListingResponse{} + json.Unmarshal([]byte(body), mlr) + return mlr, nil +} + func (rac *AuthenticatedClient) Me() (*MeResponse, error) { req := NewRequest( WithTags([]string{"url:/api/v1/me"}), diff --git a/internal/worker/notifications.go b/internal/worker/notifications.go index 92171ee..546dbc0 100644 --- a/internal/worker/notifications.go +++ b/internal/worker/notifications.go @@ -245,7 +245,7 @@ func (nc *notificationsConsumer) Consume(delivery rmq.Delivery) { nc.logger.WithFields(logrus.Fields{ "accountID": id, }).Debug("fetching message inbox") - msgs, err := rac.MessageInbox(account.LastMessageID) + msgs, err := rac.MessageUnread(account.LastMessageID) if err != nil { nc.logger.WithFields(logrus.Fields{