Re: BDR Error restarted

That's a lot of databases and explains why the logs were so busy.

For the purpose of testing it would be interesting to simplify this case
down, if you can, to the minimum required to reproduce the issue. It's
awfully hard to keep track of what's going on with this many concurrent
operations - especially since at the moment background workers don't
respect the %d format parameter in log_line_prefix .

The logs of the same time on the node you didn't restart would be useful
too, i.e. a matching pair.

The ideal is to determine what sequence of actions creates this situation
so we can reproduce what you're seeing.

On 21 January 2015 at 19:33, deans <dean(dot)song888(at)gmail(dot)com> wrote:

> Hi Andres,
>
> Any other conf should I list in addition to the bdr settings below?
>
> BDR Settings(replaced the real db name here):
>
> 1. on node 01, the replication src one:
> # Generic settings required for BDR
> max_replication_slots = 60
> max_wal_senders = 60
> wal_level = 'logical'
> track_commit_timestamp = on
> shared_preload_libraries = 'bdr'
> max_worker_processes = 100
>
> # These aren't required, but are useful for diagnosing problems
> #log_error_verbosity = verbose
> #log_min_messages = debug1
> #log_line_prefix = 'd=%d p=%p a=%a%q '
>
> # Useful options for playing with conflicts
> #bdr.default_apply_delay=2000 # milliseconds
> #bdr.log_conflicts_to_table=on
>
> bdr.connections =
>
> 'bdr02db1,bdr02db2,bdr02db3,bdr02db4,bdr02db5,bdr02db6,bdr02db7,bdr02db8,bdr02db9,bdr02db10,bdr02db11,bdr02db12,bdr02db13,bdr03db1,bdr03db2,bdr03db3,bdr03db4,bdr03db5,bdr03db6,bdr03db7,bdr03db8,bdr03db9,bdr03db10,bdr03db11,bdr03db12,bdr03db13'
>
> bdr.bdr02db1_dsn = 'dbname=db1 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db2_dsn = 'dbname=db2 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db3_dsn = 'dbname=db3 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db4_dsn = 'dbname=db4 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db5_dsn = 'dbname=db5 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db6_dsn = 'dbname=db6 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db7_dsn = 'dbname=db7 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db8_dsn = 'dbname=db8 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db8_dsn = 'dbname=db9 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db10_dsn = 'dbname=db10 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db11_dsn = 'dbname=db11 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db12_dsn = 'dbname=db12 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr02db13_dsn = 'dbname=db13 host=pgsql-bdr-cluster-02.ol.xxx.net
> user=postgres'
> bdr.bdr03db1_dsn = 'dbname=db1 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db2_dsn = 'dbname=db2 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db3_dsn = 'dbname=db3 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db4_dsn = 'dbname=db4 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db5_dsn = 'dbname=db5 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db6_dsn = 'dbname=db6 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db7_dsn = 'dbname=db7 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db8_dsn = 'dbname=db8 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db9_dsn = 'dbname=db9 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db10_dsn = 'dbname=db10 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db11_dsn = 'dbname=db11 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db12_dsn = 'dbname=db12 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db13_dsn = 'dbname=db13 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
>
> 2. on node 02
> # Generic settings required for BDR
> max_replication_slots = 60
> max_wal_senders = 60
> wal_level = 'logical'
> track_commit_timestamp = on
> shared_preload_libraries = 'bdr'
> max_worker_processes = 100
>
> # These aren't required, but are useful for diagnosing problems
> #log_error_verbosity = verbose
> #log_min_messages = debug1
> #log_line_prefix = 'd=%d p=%p a=%a%q '
>
> # Useful options for playing with conflicts
> #bdr.default_apply_delay=2000 # milliseconds
> #bdr.log_conflicts_to_table=on
>
> bdr.connections =
>
> 'bdr01db1,bdr01db2,bdr01db3,bdr01db4,bdr01db5,bdr01db6,bdr01db7,bdr01db8,bdr01db9,bdr01db10,bdr01db11,bdr01db12,bdr01db13,bdr03db1,bdr03db2,bdr03db3,bdr03db4,bdr03db5,bdr03db6,bdr03db7,bdr03db8,bdr03db9,bdr03db10,bdr03db11,bdr03db12,bdr03db13'
>
> bdr.bdr01db1_dsn = 'dbname=db1 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db2_dsn = 'dbname=db2 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db3_dsn = 'dbname=db3 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db4_dsn = 'dbname=db4 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db5_dsn = 'dbname=db5 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db6_dsn = 'dbname=db6 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db7_dsn = 'dbname=db7 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db8_dsn = 'dbname=db8 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db8_dsn = 'dbname=db9 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db10_dsn = 'dbname=db10 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db11_dsn = 'dbname=db11 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db12_dsn = 'dbname=db12 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db13_dsn = 'dbname=db13 host=pgsql-bdr-cluster-01.ol.xxx.net
> user=postgres'
> bdr.bdr01db1_dsn = 'dbname=db1 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db2_dsn = 'dbname=db2 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db3_dsn = 'dbname=db3 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db4_dsn = 'dbname=db4 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db5_dsn = 'dbname=db5 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db6_dsn = 'dbname=db6 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db7_dsn = 'dbname=db7 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db8_dsn = 'dbname=db8 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db9_dsn = 'dbname=db9 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db10_dsn = 'dbname=db10 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db11_dsn = 'dbname=db11 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db12_dsn = 'dbname=db12 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
> bdr.bdr03db13_dsn = 'dbname=db13 host=pgsql-bdr-cluster-03.ol.xxx.net
> user=postgres'
>
> bdr.bdr01db1_init_replica = on
> bdr.bbdr01db1_replica_local_dsn = 'dbname=db1 user=postgres'
> bdr.bdr01db2_init_replica = on
> bdr.bbdr01db2_replica_local_dsn = 'dbname=db2 user=postgres'
> bdr.bdr01db3_init_replica = on
> bdr.bbdr01db3_replica_local_dsn = 'dbname=db3 user=postgres'
> bdr.bdr01db4_init_replica = on
> bdr.bbdr01db4_replica_local_dsn = 'dbname=db4 user=postgres'
> bdr.bdr01db5_init_replica = on
> bdr.bbdr01db5_replica_local_dsn = 'dbname=db5 user=postgres'
> bdr.bdr01db6_init_replica = on
> bdr.bbdr01db6_replica_local_dsn = 'dbname=db6 user=postgres'
> bdr.bdr01db7_init_replica = on
> bdr.bbdr01db7_replica_local_dsn = 'dbname=db7 user=postgres'
> bdr.bdr01db8_init_replica = on
> bdr.bbdr01db8_replica_local_dsn = 'dbname=db8 user=postgres'
> bdr.bdr01db9_init_replica = on
> bdr.bbdr01db9_replica_local_dsn = 'dbname=db9 user=postgres'
> bdr.bdr01db10_init_replica = on
> bdr.bbdr01db10_replica_local_dsn = 'dbname=db10 user=postgres'
> bdr.bdr01db11_init_replica = on
> bdr.bbdr01db11_replica_local_dsn = 'dbname=db11 user=postgres'
> bdr.bdr01db12_init_replica = on
> bdr.bbdr01db12_replica_local_dsn = 'dbname=db12 user=postgres'
> bdr.bdr01db13_init_replica = on
> bdr.bbdr01db13_replica_local_dsn = 'dbname=db13 user=postgres'
>
>
>
> --
> View this message in context:
> http://postgresql.nabble.com/BDR-Error-restarted-tp5833139p5834859.html
> Sent from the PostgreSQL - general mailing list archive at Nabble.com.
>
>
> --
> Sent via pgsql-general mailing list (pgsql-general(at)postgresql(dot)org)
> To make changes to your subscription:
> http://www.postgresql.org/mailpref/pgsql-general
>

--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services