From f4bdfefcec7ac43fa34d9b0747e03b3fd50e6408 Mon Sep 17 00:00:00 2001 From: Andre Medeiros Date: Mon, 5 Jul 2021 19:10:09 -0400 Subject: [PATCH] fix up bootstrap things --- Brewfile | 4 +- Brewfile.lock.json | 80 +++++++--- Procfile | 3 +- README.md | 1 - config/pgbouncer.ini | 358 +++++++++++++++++++++++++++++++++++++++++++ script/bootstrap | 12 ++ script/migrate | 12 +- 7 files changed, 432 insertions(+), 38 deletions(-) create mode 100644 config/pgbouncer.ini diff --git a/Brewfile b/Brewfile index 8c8ab9a..02c3ed2 100644 --- a/Brewfile +++ b/Brewfile @@ -1,9 +1,7 @@ tap 'contribsys/faktory' -brew 'faktory' brew 'foreman' brew 'golang' brew 'golang-migrate' brew 'postgres' -brew 'redis' - +brew 'pgbouncer' diff --git a/Brewfile.lock.json b/Brewfile.lock.json index 61d6ea2..725898c 100644 --- a/Brewfile.lock.json +++ b/Brewfile.lock.json @@ -2,59 +2,64 @@ "entries": { "brew": { "postgres": { - "version": "13.2_2", + "version": "13.3", "bottle": { "rebuild": 0, "root_url": "https://ghcr.io/v2/homebrew/core", "files": { "arm64_big_sur": { "cellar": "/opt/homebrew/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:5bb80b319443cc57a44a9a9a8507a5255728d2caf359ef7e931c79c2c833b900", - "sha256": "5bb80b319443cc57a44a9a9a8507a5255728d2caf359ef7e931c79c2c833b900" + "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:7c0e1b76d60b428facd521c729323221712d7f6d9954e21da389aeeb2c62348e", + "sha256": "7c0e1b76d60b428facd521c729323221712d7f6d9954e21da389aeeb2c62348e" }, "big_sur": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:ee460f1f8beb4d877121faf4ebcbf8069f86a8ef4ae05fa342bc2fcdde75bb47", - "sha256": "ee460f1f8beb4d877121faf4ebcbf8069f86a8ef4ae05fa342bc2fcdde75bb47" + "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:eaf28965ead970ecfb327b121ec6a07f0a4e39865797a1a0383605a17e5911e3", + "sha256": "eaf28965ead970ecfb327b121ec6a07f0a4e39865797a1a0383605a17e5911e3" }, "catalina": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:1f50565ab7a703d85ecc37179e8d461dc82de4e3e6266ba085dd19a0b95ed5db", - "sha256": "1f50565ab7a703d85ecc37179e8d461dc82de4e3e6266ba085dd19a0b95ed5db" + "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:74e946503c73cd0efc55ad4b373efbd8f4fb8a9e26a670b878c6db25794aea4a", + "sha256": "74e946503c73cd0efc55ad4b373efbd8f4fb8a9e26a670b878c6db25794aea4a" }, "mojave": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:c0a5828cb5caef09e7b4acd99b450b6d4ceb3a0d265a95b147e63e23fb6f7596", - "sha256": "c0a5828cb5caef09e7b4acd99b450b6d4ceb3a0d265a95b147e63e23fb6f7596" + "url": "https://ghcr.io/v2/homebrew/core/postgresql/blobs/sha256:36c7bde4788571e5b66ffe05b6174b62c69781d61c53c3ebcd9d278e8f148197", + "sha256": "36c7bde4788571e5b66ffe05b6174b62c69781d61c53c3ebcd9d278e8f148197" } } } }, "golang": { - "version": "1.16.3", + "version": "1.16.5", "bottle": { "rebuild": 0, "root_url": "https://ghcr.io/v2/homebrew/core", "files": { "arm64_big_sur": { "cellar": "/opt/homebrew/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:e7c1efdd09e951eb46d01a3200b01e7fa55ce285b75470051be7fef34f4233ce", - "sha256": "e7c1efdd09e951eb46d01a3200b01e7fa55ce285b75470051be7fef34f4233ce" + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:dde21eedfa67da23db70cf977ae82c0cadd5acf2a326cb91853ff54d0cf5886f", + "sha256": "dde21eedfa67da23db70cf977ae82c0cadd5acf2a326cb91853ff54d0cf5886f" }, "big_sur": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:ea37f33fd27369612a3e4e6db6adc46db0e8bdf6fac1332bf51bafaa66d43969", - "sha256": "ea37f33fd27369612a3e4e6db6adc46db0e8bdf6fac1332bf51bafaa66d43969" + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:416c5e2b7247c78482a5465f79d83c0240ee0c9098c8c7429f9c7af073402cc9", + "sha256": "416c5e2b7247c78482a5465f79d83c0240ee0c9098c8c7429f9c7af073402cc9" }, "catalina": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:69c28f5e60612801c66e51e93d32068f822b245ab83246cb6cb374572eb59e15", - "sha256": "69c28f5e60612801c66e51e93d32068f822b245ab83246cb6cb374572eb59e15" + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:8a7564fab7f715feed7506e3cc30f20295fd62914418fb636a5a4c4ca1fc7398", + "sha256": "8a7564fab7f715feed7506e3cc30f20295fd62914418fb636a5a4c4ca1fc7398" }, "mojave": { "cellar": "/usr/local/Cellar", - "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:bf1e90ed1680b8ee1acb49f2f99426c8a8ac3e49efd63c7f3b41e57e7214dd19", - "sha256": "bf1e90ed1680b8ee1acb49f2f99426c8a8ac3e49efd63c7f3b41e57e7214dd19" + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:a232e1f840525ab1e9411ba4edaa74c2bb73705e8e6feb7506649a7d608f9292", + "sha256": "a232e1f840525ab1e9411ba4edaa74c2bb73705e8e6feb7506649a7d608f9292" + }, + "x86_64_linux": { + "cellar": "/home/linuxbrew/.linuxbrew/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:1434dfa5cbe0fd0edb34eab477e156640f3f07599d33105958fe18b329bcfb7d", + "sha256": "1434dfa5cbe0fd0edb34eab477e156640f3f07599d33105958fe18b329bcfb7d" } } } @@ -154,6 +159,35 @@ } } } + }, + "pgbouncer": { + "version": "1.15.0", + "bottle": { + "rebuild": 1, + "root_url": "https://ghcr.io/v2/homebrew/core", + "files": { + "arm64_big_sur": { + "cellar": ":any", + "url": "https://ghcr.io/v2/homebrew/core/pgbouncer/blobs/sha256:8107249d240e1a53f6ae84587c08129acf5c294c4022f92d5f1c731ea6956ea3", + "sha256": "8107249d240e1a53f6ae84587c08129acf5c294c4022f92d5f1c731ea6956ea3" + }, + "big_sur": { + "cellar": ":any", + "url": "https://ghcr.io/v2/homebrew/core/pgbouncer/blobs/sha256:09f21ff3e7b2c125d793da2ba64110392227650ae8157ef987f041959af8fe7c", + "sha256": "09f21ff3e7b2c125d793da2ba64110392227650ae8157ef987f041959af8fe7c" + }, + "catalina": { + "cellar": ":any", + "url": "https://ghcr.io/v2/homebrew/core/pgbouncer/blobs/sha256:fad76f523bac43aaf7859fa0085ab7c6582f9d4aeb682e677db8f5acd9c4159a", + "sha256": "fad76f523bac43aaf7859fa0085ab7c6582f9d4aeb682e677db8f5acd9c4159a" + }, + "mojave": { + "cellar": ":any", + "url": "https://ghcr.io/v2/homebrew/core/pgbouncer/blobs/sha256:4187ceded551fad5801a26f790e61dd7d654acc675de73a1b4bf2858920d0734", + "sha256": "4187ceded551fad5801a26f790e61dd7d654acc675de73a1b4bf2858920d0734" + } + } + } } }, "tap": { @@ -165,12 +199,12 @@ "system": { "macos": { "big_sur": { - "HOMEBREW_VERSION": "3.1.5", + "HOMEBREW_VERSION": "3.2.1", "HOMEBREW_PREFIX": "/usr/local", - "Homebrew/homebrew-core": "17fb25009a4876e9369cb621e798b758eb0c7fe6", - "CLT": "12.5.0.0.1.1617976050", - "Xcode": "12.5", - "macOS": "11.3.1" + "Homebrew/homebrew-core": "4ba9cfde5eae5c7492a4eacebcefe72d57efd5fe", + "CLT": "12.5.0.22.11", + "Xcode": "12.5.1", + "macOS": "11.4" } } } diff --git a/Procfile b/Procfile index 2bc5498..decdd03 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,2 @@ postgres: postgres -D tmp/postgresql -server: go run github.com/andremedeiros/apollo/cmd/apollo-api -worker: go run github.com/andremedeiros/apollo/cmd/apollo-worker +pgbouncer: pgbouncer -q config/pgbouncer.ini diff --git a/README.md b/README.md index 92737f4..0f698a5 100644 --- a/README.md +++ b/README.md @@ -11,5 +11,4 @@ $ cp .env.example .env ```sh $ script/server -$ script/migrate ``` diff --git a/config/pgbouncer.ini b/config/pgbouncer.ini new file mode 100644 index 0000000..d842f31 --- /dev/null +++ b/config/pgbouncer.ini @@ -0,0 +1,358 @@ +;;; +;;; PgBouncer configuration file +;;; + +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= auth_user= +;; client_encoding= datestyle= timezone= +;; pool_size= reserve_pool= max_db_connections= +;; pool_mode= connect_query= application_name= +[databases] +apollo = host=localhost dbname=apollo user=apollo +;; foodb over Unix socket +;foodb = + +;; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +;; access to dest database will go with single user +;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +;; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +;; use auth_user with auth_query if user not present in auth_file +;; auth_user must exist in auth_file +; foodb = auth_user=bar + +;; fallback connect string +;* = host=testserver + +;; User-specific configuration +[users] +;user1 = pool_mode=transaction max_user_connections=10 + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +logfile = /usr/local/var/log/pgbouncer.log +pidfile = /usr/local/var/run/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +;; IP address or * which means all IPs +listen_addr = localhost +listen_port = 6432 + +;; Unix socket is also used for -R. +;; On Debian it should be /var/run/postgresql +;unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;client_tls_protocols = secure + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +;server_tls_sslmode = disable + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +;; any, trust, plain, md5, cert, hba, pam +auth_type = any +;auth_file = /usr/local/etc/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +;auth_query = SELECT usename, passwd FROM pg_shadow WHERE usename=$1 + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +;; comma-separated list of users who are allowed to change settings +;admin_users = user2, someadmin, otheradmin + +;; comma-separated list of users who are just allowed to use SHOW command +;stats_users = stats, root + +;;; +;;; Pooler personality questions +;;; + +;; When server connection is released back to pool: +;; session - after client disconnects (default) +;; transaction - after transaction finishes +;; statement - after statement finishes +;pool_mode = session + +;; Query for cleaning connection immediately after releasing from +;; client. No need to put ROLLBACK here, pgbouncer does not reuse +;; connections where transaction is left open. +;server_reset_query = DISCARD ALL + +;; Whether server_reset_query should run in all pooling modes. If it +;; is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +;; Comma-separated list of parameters to ignore when given in startup +;; packet. Newer JDBC versions require the extra_float_digits here. +;ignore_startup_parameters = extra_float_digits + +;; When taking idle server into use, this query is run first. +;server_check_query = select 1 + +;; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +;; Close servers in session pooling mode after a RECONNECT, RELOAD, +;; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;; Period for updating aggregated stats. +;stats_period = 60 + +;;; +;;; Connection limits +;;; + +;; Total number of clients that can connect +;max_client_conn = 100 + +;; Default pool size. 20 is good number when transaction pooling +;; is in use, in session pooling it needs to be the number of +;; max clients you want to handle at any moment +;default_pool_size = 20 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +;; If a clients needs to wait more than this many seconds, use reserve +;; pool. +;reserve_pool_timeout = 5 + +;; Maximum number of server connections for a database +;max_db_connections = 0 + +;; Maximum number of server connections for a user +;max_user_connections = 0 + +;; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +;; log if client connects or server connection is made +;log_connections = 1 + +;; log if and why connection was closed +;log_disconnections = 1 + +;; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; write aggregated stats into log +;log_stats = 1 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. Allows +;; to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return in +;; this time. Should be used to survive network problems, _not_ as +;; statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not +;; assigned to a server in this time. Should be used to limit the +;; number of queued queries in case of a database or network +;; failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this +;; time. Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they stay +;; unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; Close connections which are in "IDLE in transaction" state longer +;; than this many seconds. +;idle_transaction_timeout = 0 + +;; How long SUSPEND/-R waits for buffer flush before closing +;; connection. +;suspend_timeout = 10 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; Set SO_REUSEPORT socket option +;so_reuseport = 0 + +;; networking options, for info: man 7 tcp + +;; Linux: Notify program about new connection only if there is also +;; data received. (Seconds to wait.) On Linux the default is 45, on +;; other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. They also require +;; tcp_keepalive=1. + +;; Count of keepalive packets +;tcp_keepcnt = 0 + +;; How long the connection can be idle before sending keepalive +;; packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes +;tcp_keepintvl = 0 + +;; How long may transmitted data remain unacknowledged before TCP +;; connection is closed (in milliseconds) +;tcp_user_timeout = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;; Custom resolv.conf file, to set custom DNS servers or other options +;; (default: empty = use OS settings) +;resolv_conf = /etc/pgbouncer/resolv.conf + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL injection: when PQexec +;; is disabled, multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP +;; By default contains config file from command line. +;conffile + +;; Windows service name to register as. job_name is alias for +;; service_name, used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from other file +;%include /etc/pgbouncer/pgbouncer-other.ini diff --git a/script/bootstrap b/script/bootstrap index 874857f..649fc5c 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -12,6 +12,18 @@ brew bundle check >/dev/null 2>&1 || { [ -d "tmp/postgresql" ] || { echo "===> Setting up database..." initdb -D tmp/postgresql -U apollo + postgres -D tmp/postgresql & + + echo "===> Waiting for Postgres to finish starting up..." + while ! nc -z localhost 5432; do + sleep 0.1 # wait for 1/10 of the second before check again + done + + createdb apollo -U apollo + + script/migrate + + kill -INT `head -n1 tmp/postgresql/postmaster.pid` } go mod verify >/dev/null 2>&1 || { diff --git a/script/migrate b/script/migrate index d7dd14c..67428c3 100755 --- a/script/migrate +++ b/script/migrate @@ -4,13 +4,7 @@ set -e cd "$(dirname "$0")/.." -# ensure everything in the app is up to date. -script/bootstrap +DATABASE_URL=postgres://apollo:@localhost/apollo?sslmode=disable -[ -f ".env" ] && { - source .env - - echo "===> Running migrations..." - createdb -U apollo apollo || true - migrate -path=./migrations -database=$DATABASE_URL up -} +echo "===> Running migrations..." +migrate -path=./migrations -database=$DATABASE_URL up