From 8c0f2ed1667ba84ac1193c807b66e8bb22c5aa72 Mon Sep 17 00:00:00 2001 From: Will Rouesnel Date: Thu, 17 Nov 2016 00:58:25 +1100 Subject: [PATCH] Add postgres cluster replication smoke tests. --- .travis.yml | 7 + .../docker-postgres-replication/Dockerfile.p2 | 7 + tests/docker-postgres-replication/README.md | 11 ++ .../docker-compose.yml | 32 ++++ .../docker-entrypoint.sh | 137 ++++++++++++++++++ .../setup-replication.sh | 22 +++ tests/test-smoke | 89 ++++++++++-- 7 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 tests/docker-postgres-replication/Dockerfile.p2 create mode 100644 tests/docker-postgres-replication/README.md create mode 100644 tests/docker-postgres-replication/docker-compose.yml create mode 100755 tests/docker-postgres-replication/docker-entrypoint.sh create mode 100755 tests/docker-postgres-replication/setup-replication.sh diff --git a/.travis.yml b/.travis.yml index 8630d073..aeb1d89f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,13 @@ services: language: go go: - '1.7' +# Make sure we have p2 +before_install: +- wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 && + chmod +x /usr/local/bin/p2 +- wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 && + chmod +x /usr/local/bin/docker-compose + script: - make all - make docker diff --git a/tests/docker-postgres-replication/Dockerfile.p2 b/tests/docker-postgres-replication/Dockerfile.p2 new file mode 100644 index 00000000..e517a729 --- /dev/null +++ b/tests/docker-postgres-replication/Dockerfile.p2 @@ -0,0 +1,7 @@ +FROM postgres:{{VERSION}} +MAINTAINER Daniel Dent (https://www.danieldent.com) +ENV PG_MAX_WAL_SENDERS 8 +ENV PG_WAL_KEEP_SEGMENTS 8 +COPY setup-replication.sh /docker-entrypoint-initdb.d/ +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh diff --git a/tests/docker-postgres-replication/README.md b/tests/docker-postgres-replication/README.md new file mode 100644 index 00000000..86106b67 --- /dev/null +++ b/tests/docker-postgres-replication/README.md @@ -0,0 +1,11 @@ +# Replicated postgres cluster in docker. + +Upstream is forked from https://github.com/DanielDent/docker-postgres-replication + +My version lives at https://github.com/wrouesnel/docker-postgres-replication + +This very simple docker-compose file lets us stand up a replicated postgres +cluster so we can test streaming. + +# TODO: +Pull in p2 and template the Dockerfile so we can test multiple versions. diff --git a/tests/docker-postgres-replication/docker-compose.yml b/tests/docker-postgres-replication/docker-compose.yml new file mode 100644 index 00000000..81a66da1 --- /dev/null +++ b/tests/docker-postgres-replication/docker-compose.yml @@ -0,0 +1,32 @@ + +version: '2' + +services: + pg-master: + build: '.' + image: 'danieldent/postgres-replication' + restart: 'always' + environment: + POSTGRES_USER: 'postgres' + POSTGRES_PASSWORD: 'postgres' + PGDATA: '/var/lib/postgresql/data/pgdata' + volumes: + - '/var/lib/postgresql/data' + expose: + - '5432' + + pg-slave: + build: '.' + image: 'danieldent/postgres-replication' + restart: 'always' + environment: + POSTGRES_USER: 'postgres' + POSTGRES_PASSWORD: 'postgres' + PGDATA: '/var/lib/postgresql/data/pgdata' + REPLICATE_FROM: 'pg-master' + volumes: + - '/var/lib/postgresql/data' + expose: + - '5432' + links: + - 'pg-master' diff --git a/tests/docker-postgres-replication/docker-entrypoint.sh b/tests/docker-postgres-replication/docker-entrypoint.sh new file mode 100755 index 00000000..42502762 --- /dev/null +++ b/tests/docker-postgres-replication/docker-entrypoint.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# Backwards compatibility for old variable names (deprecated) +if [ "x$PGUSER" != "x" ]; then + POSTGRES_USER=$PGUSER +fi +if [ "x$PGPASSWORD" != "x" ]; then + POSTGRES_PASSWORD=$PGPASSWORD +fi + +# Forwards-compatibility for old variable names (pg_basebackup uses them) +if [ "x$PGPASSWORD" = "x" ]; then + export PGPASSWORD=$POSTGRES_PASSWORD +fi + +# Based on official postgres package's entrypoint script (https://hub.docker.com/_/postgres/) +# Modified to be able to set up a slave. The docker-entrypoint-initdb.d hook provided is inadequate. + +set -e + +if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" +fi + +if [ "$1" = 'postgres' ]; then + mkdir -p "$PGDATA" + chmod 700 "$PGDATA" + chown -R postgres "$PGDATA" + + mkdir -p /run/postgresql + chmod g+s /run/postgresql + chown -R postgres /run/postgresql + + # look specifically for PG_VERSION, as it is expected in the DB dir + if [ ! -s "$PGDATA/PG_VERSION" ]; then + if [ "x$REPLICATE_FROM" == "x" ]; then + eval "gosu postgres initdb $POSTGRES_INITDB_ARGS" + else + until ping -c 1 -W 1 ${REPLICATE_FROM} + do + echo "Waiting for master to ping..." + sleep 1s + done + until gosu postgres pg_basebackup -h ${REPLICATE_FROM} -D ${PGDATA} -U ${POSTGRES_USER} -vP -w + do + echo "Waiting for master to connect..." + sleep 1s + done + fi + + # check password first so we can output the warning before postgres + # messes it up + if [ ! -z "$POSTGRES_PASSWORD" ]; then + pass="PASSWORD '$POSTGRES_PASSWORD'" + authMethod=md5 + else + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOWARN' + **************************************************** + WARNING: No password has been set for the database. + This will allow anyone with access to the + Postgres port to access your database. In + Docker's default configuration, this is + effectively any other container on the same + system. + + Use "-e POSTGRES_PASSWORD=password" to set + it in "docker run". + **************************************************** + EOWARN + + pass= + authMethod=trust + fi + + if [ "x$REPLICATE_FROM" == "x" ]; then + + { echo; echo "host replication all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null + { echo; echo "host all all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null + + # internal start of server in order to allow set-up using psql-client + # does not listen on external TCP/IP and waits until start finishes + gosu postgres pg_ctl -D "$PGDATA" \ + -o "-c listen_addresses='localhost'" \ + -w start + + : ${POSTGRES_USER:=postgres} + : ${POSTGRES_DB:=$POSTGRES_USER} + export POSTGRES_USER POSTGRES_DB + + psql=( psql -v ON_ERROR_STOP=1 ) + + if [ "$POSTGRES_DB" != 'postgres' ]; then + "${psql[@]}" --username postgres <<-EOSQL + CREATE DATABASE "$POSTGRES_DB" ; + EOSQL + echo + fi + + if [ "$POSTGRES_USER" = 'postgres' ]; then + op='ALTER' + else + op='CREATE' + fi + "${psql[@]}" --username postgres <<-EOSQL + $op USER "$POSTGRES_USER" WITH SUPERUSER $pass ; + EOSQL + echo + + fi + + psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" ) + + echo + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.sql) echo "$0: running $f"; "${psql[@]}" < "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + if [ "x$REPLICATE_FROM" == "x" ]; then + gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop + fi + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + fi + + exec gosu postgres "$@" +fi + +exec "$@" diff --git a/tests/docker-postgres-replication/setup-replication.sh b/tests/docker-postgres-replication/setup-replication.sh new file mode 100755 index 00000000..460c5489 --- /dev/null +++ b/tests/docker-postgres-replication/setup-replication.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ "x$REPLICATE_FROM" == "x" ]; then + +cat >> ${PGDATA}/postgresql.conf < ${PGDATA}/recovery.conf < /dev/null 2>&1 " ; do echo "Waiting for postgres to start..." if [ $(( $(date +%s) - $WAIT_START )) -gt $TIMEOUT ]; then echo "Timed out waiting for postgres!" 1>&2 - docker logs $CONTAINER_NAME - docker kill $CONTAINER_NAME - docker rm $CONTAINER_NAME exit 1 fi sleep 1 @@ -34,14 +45,12 @@ smoketest_postgres() { DATA_SOURCE_NAME="postgresql://postgres:password@localhost:55432/?sslmode=disable" ./postgres_exporter & exporter_pid=$! + trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm $CONTAINER_NAME ; kill $exporter_pid" EXIT INT TERM local DAEMON_WAIT_START=$(date +%s) while ! nc -z localhost 9113 ; do echo "Waiting for exporter to start..." if [ $(( $(date +%s) - $WAIT_START )) -gt $TIMEOUT ]; then echo "Timed out waiting for exporter!" 1>&2 - docker logs $CONTAINER_NAME - docker kill $CONTAINER_NAME - docker rm $CONTAINER_NAME exit 1 fi sleep 1 @@ -51,15 +60,77 @@ smoketest_postgres() { if [ "$?" != "0" ]; then echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2 kill $exporter_pid - docker logs $CONTAINER_NAME - docker kill $CONTAINER_NAME - docker rm $CONTAINER_NAME exit 1 fi kill $exporter_pid docker kill $CONTAINER_NAME docker rm $CONTAINER_NAME + trap - EXIT INT TERM + + echo "Test replicated cluster..." + postgres_exporter=$(readlink -f ./postgres_exporter) + old_pwd=$(pwd) + cd docker-postgres-replication + + VERSION=$version p2 -t Dockerfile.p2 -o Dockerfile + if [ "$?" != "0" ]; then + echo "Templating failed" 1>&2 + exit 1 + fi + trap "docker-compose logs; docker-compose down ; docker-compose rm -v" EXIT INT TERM + docker-compose up -d --force-recreate --build + + master_container=$(docker-compose ps -q pg-master) + slave_container=$(docker-compose ps -q pg-slave) + master_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $master_container) + + local WAIT_START=$(date +%s) + while ! docker exec $master_container bash -c "psql -U postgres -c \"select 'running'\" > /dev/null 2>&1 " ; do + echo "Waiting for postgres master to start..." + if [ $(( $(date +%s) - $WAIT_START )) -gt $TIMEOUT ]; then + echo "Timed out waiting for postgres!" 1>&2 + exit 1 + fi + sleep 1 + done + + local WAIT_START=$(date +%s) + while ! docker exec $slave_container bash -c "psql -U postgres -c \"select 'running'\" > /dev/null 2>&1 " ; do + echo "Waiting for postgres master to start..." + if [ $(( $(date +%s) - $WAIT_START )) -gt $TIMEOUT ]; then + echo "Timed out waiting for postgres!" 1>&2 + exit 1 + fi + sleep 1 + done + + DATA_SOURCE_NAME="postgresql://postgres:password@$master_ip:5432/?sslmode=disable" $postgres_exporter & + exporter_pid=$! + trap "docker-compose logs; docker-compose down ; docker-compose rm -v ; kill $exporter_pid" EXIT INT TERM + local DAEMON_WAIT_START=$(date +%s) + while ! nc -z localhost 9113 ; do + echo "Waiting for exporter to start..." + if [ $(( $(date +%s) - $WAIT_START )) -gt $TIMEOUT ]; then + echo "Timed out waiting for exporter!" 1>&2 + exit 1 + fi + sleep 1 + done + + wget -q -O - http://localhost:9113/metrics 1> /dev/null + if [ "$?" != "0" ]; then + echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2 + exit 1 + fi + + kill $exporter_pid + + docker-compose down + docker-compose rm -v + trap - EXIT INT TERM + + cd $old_pwd } # Start pulling the docker images in advance