diff --git a/README.md b/README.md index 9269b939..b47c8112 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ You can additionally set: * `PACT_BROKER_SQL_LOG_WARN_DURATION` - optional, defaults to 5 seconds. Log the SQL for queries that take longer than this number of seconds. * `PACT_BROKER_DATABASE_MAX_CONNECTIONS` - optional, defaults to 4. The maximum size of the connection pool. There is no need to set this unless you notice particular connection contention issues. * `PACT_BROKER_DATABASE_POOL_TIMEOUT` - optional, 5 seconds by default. The number of seconds to wait if a connection cannot be acquired before raising an error. There is no need to set this unless you notice particular connection contention issues. + * `PACT_BROKER_DATABASE_CONNECT_MAX_RETRIES` - optional, defaults to 0. When running the Pact Broker Docker image experimentally using Docker Compose on a local development machine, the Broker application process may be ready before the database is available for connection, causing the application container to exit with an error. Setting the max retries to a non-zero number will allow it to retry the connection the configured number of times, waiting 3 seconds between attempts. ## Notes diff --git a/pact_broker/config.ru b/pact_broker/config.ru index bfdb516f..06202e24 100644 --- a/pact_broker/config.ru +++ b/pact_broker/config.ru @@ -12,7 +12,7 @@ dc.pact_broker_environment_variables.each{ |key, value| $logger.info "#{key}=#{v app = PactBroker::App.new do | config | config.logger = $logger - config.database_connection = create_database_connection_from_config(config.logger, dc.database_configuration) + config.database_connection = create_database_connection_from_config(config.logger, dc.database_configuration, dc.database_connect_max_retries) config.allow_missing_migration_files = true config.database_connection.timezone = :utc diff --git a/pact_broker/database_connection.rb b/pact_broker/database_connection.rb index bca049cf..a59dd43f 100644 --- a/pact_broker/database_connection.rb +++ b/pact_broker/database_connection.rb @@ -1,26 +1,44 @@ require 'sequel' require 'pact_broker/db/log_quietener' -def create_database_connection_from_config(logger, config) - ## - # Sequel by default does not test connections in its connection pool before - # handing them to a client. To enable connection testing you need to load the - # "connection_validator" extension like below. The connection validator - # extension is configurable, by default it only checks connections once per - # hour: - # - # http://sequel.rubyforge.org/rdoc-plugins/files/lib/sequel/extensions/connection_validator_rb.html - # - # - # A gotcha here is that it is not enough to enable the "connection_validator" - # extension, we also need to specify that we want to use the threaded connection - # pool, as noted in the documentation for the extension. - # - # -1 means that connections will be validated every time, which avoids errors - # when databases are restarted and connections are killed. This has a performance - # penalty, so consider increasing this timeout if building a frequently accessed service. +## +# Sequel by default does not test connections in its connection pool before +# handing them to a client. To enable connection testing you need to load the +# "connection_validator" extension like below. The connection validator +# extension is configurable, by default it only checks connections once per +# hour: +# +# http://sequel.rubyforge.org/rdoc-plugins/files/lib/sequel/extensions/connection_validator_rb.html +# +# +# A gotcha here is that it is not enough to enable the "connection_validator" +# extension, we also need to specify that we want to use the threaded connection +# pool, as noted in the documentation for the extension. +# +# -1 means that connections will be validated every time, which avoids errors +# when databases are restarted and connections are killed. This has a performance +# penalty, so consider increasing this timeout if building a frequently accessed service. + +def create_database_connection_from_config(logger, config, max_retries = 0) logger.info "Connecting to database with config: #{config.merge(password: "*****")}" - connection = Sequel.connect(config.merge(logger: PactBroker::DB::LogQuietener.new(logger))) + + tries = 0 + max_tries = max_retries + 1 + connection = nil + wait = 3 + + begin + connection = Sequel.connect(config.merge(logger: PactBroker::DB::LogQuietener.new(logger))) + rescue StandardError => e + if (tries += 1) < max_tries + logger.info "Error connecting to database (#{e.class}). Waiting #{wait} seconds and trying again. #{max_tries-tries} tries to go." + sleep wait + retry + else + raise e + end + end + logger.info "Connected to database #{config[:database]}" connection.extension(:connection_validator) connection.pool.connection_validation_timeout = -1 connection diff --git a/pact_broker/docker_configuration.rb b/pact_broker/docker_configuration.rb index 3e7752e0..11677497 100644 --- a/pact_broker/docker_configuration.rb +++ b/pact_broker/docker_configuration.rb @@ -45,6 +45,10 @@ def database_configuration ).compact end + def database_connect_max_retries + env_as_integer(:database_connect_max_retries, 0) + end + def base_equality_only_on_content_that_affects_verification_results if env_populated?(:base_equality_only_on_content_that_affects_verification_results) env(:base_equality_only_on_content_that_affects_verification_results) == 'true'