diff options
Diffstat (limited to 'src/repmgr/repmgr_msg.c')
| -rw-r--r-- | src/repmgr/repmgr_msg.c | 655 |
1 files changed, 602 insertions, 53 deletions
diff --git a/src/repmgr/repmgr_msg.c b/src/repmgr/repmgr_msg.c index 13537823..71cb2ada 100644 --- a/src/repmgr/repmgr_msg.c +++ b/src/repmgr/repmgr_msg.c @@ -1,7 +1,7 @@ /*- * See the file LICENSE for redistribution information. * - * Copyright (c) 2005, 2012 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015 Oracle and/or its affiliates. All rights reserved. * * $Id$ */ @@ -15,15 +15,19 @@ #include "dbinc_auto/repmgr_auto.h" static int dispatch_app_message __P((ENV *, REPMGR_MESSAGE *)); -static int finish_gmdb_update __P((ENV *, - DB_THREAD_INFO *, DBT *, u_int32_t, u_int32_t, __repmgr_member_args *)); +static int finish_gmdb_update __P((ENV *, DB_THREAD_INFO *, + DBT *, u_int32_t, u_int32_t, u_int32_t, __repmgr_member_args *)); static int incr_gm_version __P((ENV *, DB_THREAD_INFO *, DB_TXN *)); -static void marshal_site_data __P((ENV *, u_int32_t, u_int8_t *, DBT *)); +static void marshal_site_data __P((ENV *, + u_int32_t, u_int32_t, u_int8_t *, DBT *)); static void marshal_site_key __P((ENV *, repmgr_netaddr_t *, u_int8_t *, DBT *, __repmgr_member_args *)); static int message_loop __P((ENV *, REPMGR_RUNNABLE *)); +static int preferred_master_takeover __P((ENV*)); static int process_message __P((ENV*, DBT*, DBT*, int)); static int reject_fwd __P((ENV *, REPMGR_CONNECTION *)); +static int rejoin_connections(ENV *); +static int rejoin_deferred_election(ENV *); static int rescind_pending __P((ENV *, DB_THREAD_INFO *, int, u_int32_t, u_int32_t)); static int resolve_limbo_int __P((ENV *, DB_THREAD_INFO *)); @@ -33,9 +37,13 @@ static int send_permlsn_conn __P((ENV *, REPMGR_CONNECTION *, u_int32_t, DB_LSN *)); static int serve_join_request __P((ENV *, DB_THREAD_INFO *, REPMGR_MESSAGE *)); +static int serve_lsnhist_request __P((ENV *, DB_THREAD_INFO *, + REPMGR_MESSAGE *)); +static int serve_readonly_master_request __P((ENV *, REPMGR_MESSAGE *)); static int serve_remove_request __P((ENV *, DB_THREAD_INFO *, REPMGR_MESSAGE *)); static int serve_repmgr_request __P((ENV *, REPMGR_MESSAGE *)); +static int serve_restart_client_request __P((ENV *, REPMGR_MESSAGE *)); /* * Map one of the phase-1/provisional membership status values to its @@ -72,6 +80,7 @@ message_loop(env, th) REPMGR_RUNNABLE *th; { DB_REP *db_rep; + DB_THREAD_INFO *ip; REP *rep; REPMGR_MESSAGE *msg; REPMGR_CONNECTION *conn; @@ -83,6 +92,7 @@ message_loop(env, th) COMPQUIET(membership, 0); db_rep = env->rep_handle; rep = db_rep->region; + ENV_ENTER(env, ip); LOCK_MUTEX(db_rep->mutex); while ((ret = __repmgr_queue_get(env, &msg, th)) == 0) { incremented = FALSE; @@ -141,7 +151,21 @@ message_loop(env, th) * detect it without the need for application * activity. */ - ret = __rep_flush(env->dbenv); + ret = __rep_flush_int(env); + } else if (db_rep->prefmas_pending == master_switch && + IS_PREFMAS_MODE(env) && + FLD_ISSET(rep->config, REP_C_PREFMAS_MASTER) && + F_ISSET(rep, REP_F_CLIENT)) { + RPRINT(env, (env, DB_VERB_REPMGR_MISC, +"message_loop heartbeat preferred master switch")); + /* + * We are a preferred master site currently + * running as a client and we have finished + * syncing with the temporary master. It is + * now time to take over as master. + */ + db_rep->prefmas_pending = no_action; + ret = preferred_master_takeover(env); } else { /* * Use heartbeat message to initiate rerequest @@ -162,6 +186,12 @@ message_loop(env, th) db_rep->non_rep_th--; if (ret != 0) goto out; + if (db_rep->view_mismatch) { + __db_errx(env, DB_STR("3699", + "Site is not recorded as a view in the group membership database")); + ret = EINVAL; + goto out; + } } /* * A return of DB_REP_UNAVAIL from __repmgr_queue_get() merely means we @@ -171,6 +201,7 @@ message_loop(env, th) ret = 0; out: UNLOCK_MUTEX(db_rep->mutex); + ENV_LEAVE(env, ip); return (ret); } @@ -341,16 +372,45 @@ process_message(env, control, rec, eid) break; case DB_REP_DUPMASTER: - /* - * Initiate an election if we're configured to be using - * elections, but only if we're *NOT* using leases. When using - * leases, there is never any uncertainty over which site is the - * rightful master, and only the loser gets the DUPMASTER return - * code. - */ - if ((ret = __repmgr_become_client(env)) == 0 && + if (IS_PREFMAS_MODE(env) && + FLD_ISSET(rep->config, REP_C_PREFMAS_MASTER)) { + /* + * The preferred master site must restart as a master + * so that it sends out a NEWMASTER to help the client + * sync. It must force a role change so that it + * advances its gen even though it is already master. + * This is needed if there was a temporary master at + * a higher gen that is now restarting as a client. + * A client won't process messages from a master at + * a lower gen than its own. + */ + ret = __repmgr_repstart(env, DB_REP_MASTER, + REP_START_FORCE_ROLECHG); + } else if (IS_PREFMAS_MODE(env) && + FLD_ISSET(rep->config, REP_C_PREFMAS_CLIENT) && + (ret = __repmgr_become_client(env)) == 0) { + /* + * The preferred master client site must restart as + * client without any elections to enable the preferred + * master site to preserve its own transactions. It + * uses an election thread to repeatedly perform client + * startups so that it will perform its client sync + * when the preferred master's gen has caught up. + */ + LOCK_MUTEX(db_rep->mutex); + ret = __repmgr_init_election(env, + ELECT_F_CLIENT_RESTART); + UNLOCK_MUTEX(db_rep->mutex); + } else if ((ret = __repmgr_become_client(env)) == 0 && FLD_ISSET(rep->config, REP_C_LEASE | REP_C_ELECTIONS) == REP_C_ELECTIONS) { + /* + * Initiate an election if we're configured to be using + * elections, but only if we're *NOT* using leases. + * When using leases, there is never any uncertainty + * over which site is the rightful master, and only the + * loser gets the DUPMASTER return code. + */ LOCK_MUTEX(db_rep->mutex); ret = __repmgr_init_election(env, ELECT_F_IMMED); UNLOCK_MUTEX(db_rep->mutex); @@ -406,6 +466,14 @@ DB_TEST_RECOVERY_LABEL t_ret = __op_rep_exit(env); if (ret == ENOENT) ret = 0; + else if (ret == DB_DELETED && db_rep->demotion_pending) + /* + * If a demotion is in progress, we want to keep + * the repmgr threads instead of bowing out because + * they are needed when we rejoin the replication group + * immediately as a view. + */ + ret = 0; else if (ret == DB_DELETED) ret = __repmgr_bow_out(env); if (t_ret != 0 && ret == 0) @@ -428,8 +496,10 @@ __repmgr_handle_event(env, event, info) void *info; { DB_REP *db_rep; + REP *rep; db_rep = env->rep_handle; + rep = db_rep->region; if (db_rep->selector == NULL) { /* Repmgr is not in use, so all events go to application. */ @@ -457,9 +527,46 @@ __repmgr_handle_event(env, event, info) /* Application still needs to see this. */ break; + case DB_EVENT_REP_MASTER: + case DB_EVENT_REP_STARTUPDONE: + /* + * Detect a rare case where a dupmaster or incomplete gmdb + * operation has left the site's gmdb inconsistent with + * a view callback definition. The user would have correctly + * defined a view callback and called repmgr_start(), but the + * gmdb operation to update this site to a view would have been + * incomplete or rolled back. The site cannot operate in this + * inconsistent state, so set an indicator to cause a message + * thread to panic and terminate. + * + * The one exception is during a demotion to view, when + * this inconsistency is expected for a short time. + */ + if (IS_VALID_EID(db_rep->self_eid) && + PARTICIPANT_TO_VIEW(db_rep, + SITE_FROM_EID(db_rep->self_eid)) && + !db_rep->demotion_pending) + db_rep->view_mismatch = TRUE; + + /* + * In preferred master mode, when the preferred master site + * finishes synchronizing with the temporary master it must + * prepare to take over as master. This is detected by the + * next heartbeat in a message thread, where the takeover is + * actually performed. + */ + if (event == DB_EVENT_REP_STARTUPDONE && + IS_PREFMAS_MODE(env) && + FLD_ISSET(rep->config, REP_C_PREFMAS_MASTER)) { + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "startupdone set preferred master switch")); + db_rep->prefmas_pending = master_switch; + } + break; default: break; } + COMPQUIET(info, NULL); return (DB_EVENT_NOT_HANDLED); } @@ -504,7 +611,7 @@ send_permlsn(env, generation, lsn) */ policy = site->ack_policy > 0 ? site->ack_policy : rep->perm_policy; - if (policy == DB_REPMGR_ACKS_NONE || + if (IS_VIEW_SITE(env) || policy == DB_REPMGR_ACKS_NONE || (IS_PEER_POLICY(policy) && rep->priority == 0)) ack = FALSE; else @@ -614,26 +721,149 @@ send_permlsn_conn(env, conn, generation, lsn) return (ret); } +/* + * Perform the steps on the preferred master site to take over again as + * preferred master from a temporary master. This routine should only be + * called after the preferred master has restarted as a client and finished + * a client sync with the temporary master. + * + * This routine makes a best effort to wait until all temporary master + * transactions have been applied on this site before taking over. + */ +static int +preferred_master_takeover(env) + ENV *env; +{ + DB_LOG *dblp; + DB_REP *db_rep; + LOG *lp; + REP *rep; + DB_LSN last_ready_lsn, ready_lsn, sync_lsn; + u_long usec; + u_int32_t gen, max_tries, tries; + int ret, synced; + + dblp = env->lg_handle; + lp = dblp->reginfo.primary; + db_rep = env->rep_handle; + rep = db_rep->region; + gen = 0; + ZERO_LSN(sync_lsn); + ret = 0; + + if (!IS_PREFMAS_MODE(env)) + return (ret); + + /* + * Start by making the temporary master a readonly master so that we + * can know when we have applied all of its transactions on this + * site before taking over. + */ + if ((ret = __repmgr_make_site_readonly_master(env, + 1, &gen, &sync_lsn)) != 0) + return (ret); + DB_ASSERT(env, gen >= rep->gen); + + /* + * Make a best effort to wait until this site has all transactions + * from the temporary master. We want to preserve temporary master + * transactions, but we can't wait forever. If we exceed our wait, + * we restart this site as preferred master anyway. This may + * sacrifice some temporary master transactions in order to preserve + * repgroup write availability. + * + * We restart the number of tries each time we make progress in + * transactions applied, until either we apply through sync_lsn or + * we exceed max_tries without progress. + */ + if ((ret = __repmgr_prefmas_get_wait(env, &max_tries, &usec)) != 0) + return (ret); + tries = 0; + synced = 0; + ZERO_LSN(ready_lsn); + ZERO_LSN(last_ready_lsn); + while (!synced && tries < max_tries) { + __os_yield(env, 0, usec); + tries++; + /* + * lp->ready_lsn is the next LSN we expect to receive, + * which also indicates how much we've applied. sync_lsn + * is the lp->lsn (indicating the next log record expected) + * from the other site. + */ + MUTEX_LOCK(env, rep->mtx_clientdb); + ready_lsn = lp->ready_lsn; + MUTEX_UNLOCK(env, rep->mtx_clientdb); + if (gen == rep->gen && LOG_COMPARE(&ready_lsn, &sync_lsn) >= 0) + synced = 1; + else if (LOG_COMPARE(&ready_lsn, &last_ready_lsn) >= 0) { + /* We are making progress, restart number of tries. */ + last_ready_lsn = ready_lsn; + tries = 0; + } + } + + /* Restart the remote readonly temporary master as a client. */ + if ((ret = __repmgr_restart_site_as_client(env, 1)) != 0) + return (ret); + + /* Restart this site as the preferred master, waiting for + * REP_LOCKOUT_MSG. The NEWCLIENT message sent back from + * restarting the other site as client can briefly lock + * REP_LOCKOUT_MSG to do some cleanup. We don't want this + * to cause the rep_start_int() call to restart this site + * as master to return 0 without doing anything. + */ + ret = __repmgr_become_master(env, REP_START_WAIT_LOCKMSG); + return (ret); +} + static int serve_repmgr_request(env, msg) ENV *env; REPMGR_MESSAGE *msg; { - DB_THREAD_INFO *ip; + DB_REP *db_rep; DBT *dbt; + DB_THREAD_INFO *ip; REPMGR_CONNECTION *conn; + u_int32_t mtype; int ret, t_ret; - ENV_ENTER(env, ip); - switch (REPMGR_OWN_MSG_TYPE(msg->msg_hdr)) { + db_rep = env->rep_handle; + ENV_GET_THREAD_INFO(env, ip); + conn = msg->v.gmdb_msg.conn; + mtype = REPMGR_OWN_MSG_TYPE(msg->msg_hdr); + switch (mtype) { case REPMGR_JOIN_REQUEST: ret = serve_join_request(env, ip, msg); break; + case REPMGR_LSNHIST_REQUEST: + ret = serve_lsnhist_request(env, ip, msg); + break; + case REPMGR_READONLY_MASTER: + ret = serve_readonly_master_request(env, msg); + break; case REPMGR_REJOIN: RPRINT(env, (env, DB_VERB_REPMGR_MISC, "One try at rejoining group automatically")); if ((ret = __repmgr_join_group(env)) == DB_REP_UNAVAIL) ret = __repmgr_bow_out(env); + else if (ret == 0 && IS_PREFMAS_MODE(env)) { + /* + * For preferred master mode, we need to get + * a "regular" connection to the other site without + * calling an election prematurely here. + */ + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Establishing connections after rejoin")); + ret = rejoin_connections(env); + } else if (ret == 0 && db_rep->rejoin_pending) { + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Calling deferred election after rejoin")); + ret = rejoin_deferred_election(env); + } + db_rep->rejoin_pending = FALSE; break; case REPMGR_REMOVE_REQUEST: ret = serve_remove_request(env, ip, msg); @@ -641,23 +871,32 @@ serve_repmgr_request(env, msg) case REPMGR_RESOLVE_LIMBO: ret = resolve_limbo_wrapper(env, ip); break; + case REPMGR_RESTART_CLIENT: + ret = serve_restart_client_request(env, msg); + break; case REPMGR_SHARING: dbt = &msg->v.gmdb_msg.request; - ret = __repmgr_refresh_membership(env, dbt->data, dbt->size); + ret = __repmgr_refresh_membership(env, dbt->data, dbt->size, + (conn == NULL ? DB_REPMGR_VERSION : conn->version)); break; default: ret = __db_unknown_path(env, "serve_repmgr_request"); break; } - if ((conn = msg->v.gmdb_msg.conn) != NULL) { + if (conn != NULL) { + /* + * A site that removed itself may have already closed its + * connections. Do not return an error and panic if we + * can't close the one-shot GMDB connection for a remove + * request here. + */ if ((t_ret = __repmgr_close_connection(env, conn)) != 0 && - ret == 0) + ret == 0 && mtype != REPMGR_REMOVE_REQUEST) ret = t_ret; if ((t_ret = __repmgr_decr_conn_ref(env, conn)) != 0 && ret == 0) ret = t_ret; } - ENV_LEAVE(env, ip); return (ret); } @@ -674,8 +913,10 @@ serve_join_request(env, ip, msg) { DB_REP *db_rep; REPMGR_CONNECTION *conn; + REPMGR_SITE *site; DBT *dbt; __repmgr_site_info_args site_info; + __repmgr_v4site_info_args v4site_info; u_int8_t *buf; char *host; size_t len; @@ -686,9 +927,18 @@ serve_join_request(env, ip, msg) COMPQUIET(status, 0); conn = msg->v.gmdb_msg.conn; + DB_ASSERT(env, conn->version > 0 && conn->version <= DB_REPMGR_VERSION); dbt = &msg->v.gmdb_msg.request; - ret = __repmgr_site_info_unmarshal(env, - &site_info, dbt->data, dbt->size, NULL); + if (conn->version < 5) { + ret = __repmgr_v4site_info_unmarshal(env, + &v4site_info, dbt->data, dbt->size, NULL); + site_info.host = v4site_info.host; + site_info.port = v4site_info.port; + site_info.status = v4site_info.flags; + site_info.flags = 0; + } else + ret = __repmgr_site_info_unmarshal(env, + &site_info, dbt->data, dbt->size, NULL); host = site_info.host.data; host[site_info.host.size - 1] = '\0'; @@ -703,7 +953,23 @@ serve_join_request(env, ip, msg) LOCK_MUTEX(db_rep->mutex); if ((ret = __repmgr_find_site(env, host, site_info.port, &eid)) == 0) { DB_ASSERT(env, eid != db_rep->self_eid); - status = SITE_FROM_EID(eid)->membership; + site = SITE_FROM_EID(eid); + status = site->membership; + /* + * Remote site electability is usually exchanged when + * a connection is established, but when a new site + * joins the repgroup there is a brief gap between the + * join and the connection. Record electability for + * the joining site so that we are not overly conservative + * about the number of acks we require for a PERM + * transaction if the joining site is unelectable. + */ + if (FLD_ISSET(site_info.flags, SITE_JOIN_ELECTABLE)) { + F_SET(site, SITE_ELECTABLE); + FLD_CLR(site_info.flags, SITE_JOIN_ELECTABLE); + } else + F_CLR(site, SITE_ELECTABLE); + F_SET(site, SITE_HAS_PRIO); } UNLOCK_MUTEX(db_rep->mutex); if (ret != 0) @@ -712,7 +978,8 @@ serve_join_request(env, ip, msg) switch (status) { case 0: case SITE_ADDING: - ret = __repmgr_update_membership(env, ip, eid, SITE_ADDING); + ret = __repmgr_update_membership(env, ip, eid, SITE_ADDING, + site_info.flags); break; case SITE_PRESENT: /* Already in desired state. */ @@ -729,7 +996,7 @@ serve_join_request(env, ip, msg) goto err; LOCK_MUTEX(db_rep->mutex); - ret = __repmgr_marshal_member_list(env, &buf, &len); + ret = __repmgr_marshal_member_list(env, conn->version, &buf, &len); UNLOCK_MUTEX(db_rep->mutex); if (ret != 0) goto err; @@ -760,6 +1027,7 @@ serve_remove_request(env, ip, msg) REPMGR_SITE *site; DBT *dbt; __repmgr_site_info_args site_info; + __repmgr_v4site_info_args v4site_info; char *host; u_int32_t status, type; int eid, ret, t_ret; @@ -768,9 +1036,18 @@ serve_remove_request(env, ip, msg) db_rep = env->rep_handle; conn = msg->v.gmdb_msg.conn; + DB_ASSERT(env, conn->version > 0 && conn->version <= DB_REPMGR_VERSION); dbt = &msg->v.gmdb_msg.request; - ret = __repmgr_site_info_unmarshal(env, - &site_info, dbt->data, dbt->size, NULL); + if (conn->version < 5) { + ret = __repmgr_v4site_info_unmarshal(env, + &v4site_info, dbt->data, dbt->size, NULL); + site_info.host = v4site_info.host; + site_info.port = v4site_info.port; + site_info.status = v4site_info.flags; + site_info.flags = 0; + } else + ret = __repmgr_site_info_unmarshal(env, + &site_info, dbt->data, dbt->size, NULL); host = site_info.host.data; host[site_info.host.size - 1] = '\0'; @@ -810,7 +1087,8 @@ serve_remove_request(env, ip, msg) break; case SITE_PRESENT: case SITE_DELETING: - ret = __repmgr_update_membership(env, ip, eid, SITE_DELETING); + ret = __repmgr_update_membership(env, ip, eid, SITE_DELETING, + site_info.flags); break; default: ret = __db_unknown_path(env, "serve_remove_request"); @@ -829,7 +1107,175 @@ err: default: return (ret); } - return (__repmgr_send_sync_msg(env, conn, type, NULL, 0)); + /* + * It is possible when a site removes itself that by now it has + * already acted on the first GMDB update and closed its connections. + * Do not return an error and panic if we can't send the final + * status of the remove operation. + */ + if ((ret = __repmgr_send_sync_msg(env, conn, type, NULL, 0)) != 0) + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Problem sending remove site status message %d", ret)); + return (0); +} + +/* + * Serve the REPMGR_RESTART_CLIENT message by restarting this site as a + * client if it is not already a client. Always sends back a + * REPMGR_PREFMAS_SUCCESS message with an empty payload. + */ +static int +serve_restart_client_request(env, msg) + ENV *env; + REPMGR_MESSAGE *msg; +{ + DB_REP *db_rep; + REP * rep; + REPMGR_CONNECTION *conn; + int ret, t_ret; + + db_rep = env->rep_handle; + rep = db_rep->region; + ret = 0; + + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Serving restart_client request")); + conn = msg->v.gmdb_msg.conn; + DB_ASSERT(env, conn->version > 0 && conn->version <= DB_REPMGR_VERSION); + /* No need to read payload - it is just a dummy byte. */ + + if (IS_PREFMAS_MODE(env) && !F_ISSET(rep, REP_F_CLIENT)) + ret = __repmgr_become_client(env); + + if ((t_ret = __repmgr_send_sync_msg(env, conn, + REPMGR_PREFMAS_SUCCESS, NULL, 0)) != 0) + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Problem sending restart client success message %d", ret)); + + if (ret == 0 && t_ret != 0) + ret = t_ret; + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Request for restart_client returning %d", ret)); + return (ret); +} + +/* + * Serve the REPMGR_READONLY_MASTER message by turning this site into a + * readonly master. Always sends back a REPMGR_READONLY_RESPONSE message with + * a payload containing this site's gen and next LSN expected. If there are + * any errors, the gen is 0 and the next LSN is [0,0]. + */ +static int +serve_readonly_master_request(env, msg) + ENV *env; + REPMGR_MESSAGE *msg; +{ + REPMGR_CONNECTION *conn; + __repmgr_permlsn_args permlsn; + u_int8_t buf[__REPMGR_PERMLSN_SIZE]; + int ret, t_ret; + + ret = 0; + + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Serving readonly_master request")); + conn = msg->v.gmdb_msg.conn; + DB_ASSERT(env, conn->version > 0 && conn->version <= DB_REPMGR_VERSION); + /* No need to read payload - it is just a dummy byte. */ + + if (IS_PREFMAS_MODE(env)) + ret = __rep_become_readonly_master(env, + &permlsn.generation, &permlsn.lsn); + + __repmgr_permlsn_marshal(env, &permlsn, buf); + if ((t_ret = __repmgr_send_sync_msg(env, conn, + REPMGR_READONLY_RESPONSE, buf, __REPMGR_PERMLSN_SIZE)) != 0) + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Problem sending readonly response message %d", ret)); + if (ret == 0 && t_ret != 0) + ret = t_ret; + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Request for readonly_master returning %d", ret)); + return (ret); +} + +/* + * Serve the REPMGR_LSNHIST_REQUEST message by retrieving information from + * this site's LSN history database for the requested gen. If the requested + * gen exists at this site, sends back a REPMGR_LSNHIST_RESPONSE message + * containing the LSN and timestamp at the requested gen and the LSN for the + * next gen if that gen exists (next gen LSN is [0,0] if next gen doesn't + * yet exist at this site.) Sends back a PREFMAS_FAILURE message if the + * requested gen does not yet exist at this site or if there are any errors. + */ +static int +serve_lsnhist_request(env, ip, msg) + ENV *env; + DB_THREAD_INFO *ip; + REPMGR_MESSAGE *msg; +{ + REPMGR_CONNECTION *conn; + DBT *dbt; + __repmgr_lsnhist_match_args lsnhist_match; + __rep_lsn_hist_data_args lsnhist_data, next_lsnhist_data; + __rep_lsn_hist_key_args key; + u_int8_t match_buf[__REPMGR_LSNHIST_MATCH_SIZE]; + DB_LSN next_gen_lsn; + int ret, t_ret; + + RPRINT(env, (env, DB_VERB_REPMGR_MISC, "Serving lsnhist request")); + conn = msg->v.gmdb_msg.conn; + DB_ASSERT(env, conn->version > 0 && conn->version <= DB_REPMGR_VERSION); + /* Read lsn_hist_key incoming payload to get gen being requested. */ + dbt = &msg->v.gmdb_msg.request; + if ((ret = __rep_lsn_hist_key_unmarshal(env, + &key, dbt->data, dbt->size, NULL)) != 0) + return (ret); + if (key.version != REP_LSN_HISTORY_FMT_VERSION) { + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "serve_lsnhist_request version mismatch")); + return (0); + } + + /* + * There's no need to retry if we don't find an lsnhist record for + * requested gen. This site is either a temporary master or a client, + * which means that if it doesn't already have an lsnhist record at + * this gen, it is highly unlikely to get one in the near future. + */ + if ((ret = __rep_get_lsnhist_data(env, + ip, key.gen, &lsnhist_data)) == 0) { + + if ((t_ret = __rep_get_lsnhist_data(env, + ip, key.gen + 1, &next_lsnhist_data)) == 0) + next_gen_lsn = next_lsnhist_data.lsn; + else + ZERO_LSN(next_gen_lsn); + + lsnhist_match.lsn = lsnhist_data.lsn; + lsnhist_match.hist_sec = lsnhist_data.hist_sec; + lsnhist_match.hist_nsec = lsnhist_data.hist_nsec; + lsnhist_match.next_gen_lsn = next_gen_lsn; + __repmgr_lsnhist_match_marshal(env, &lsnhist_match, match_buf); + if ((t_ret = __repmgr_send_sync_msg(env, conn, + REPMGR_LSNHIST_RESPONSE, match_buf, + __REPMGR_LSNHIST_MATCH_SIZE)) != 0) + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Problem sending lsnhist response message %d", + ret)); + } else if ((t_ret = __repmgr_send_sync_msg(env, conn, + REPMGR_PREFMAS_FAILURE, NULL, 0)) != 0) + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Problem sending prefmas failure message %d", ret)); + + /* Do not return an error if LSN history record not found. */ + if (ret == DB_NOTFOUND) + ret = 0; + if (ret == 0 && t_ret != 0) + ret = t_ret; + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Request for lsnhist returning %d", ret)); + return (ret); } /* @@ -917,7 +1363,13 @@ resolve_limbo_int(env, ip) if (orig_status == SITE_PRESENT || orig_status == 0) goto out; - if (IS_ZERO_LSN(db_rep->limbo_failure)) + /* + * It is possible after an autotakeover on a master to have no + * limbo_failure LSN but to have a limbo_victim that was found + * in the gmdb that still needs to be resolved. + */ + if (IS_ZERO_LSN(db_rep->limbo_failure) && + !db_rep->limbo_resolution_needed) goto out; /* @@ -947,7 +1399,8 @@ resolve_limbo_int(env, ip) ip, NULL, &txn, DB_IGNORE_LEASE)) != 0) goto out; - marshal_site_data(env, orig_status, data_buf, &data_dbt); + marshal_site_data(env, + orig_status, site->gmdb_flags, data_buf, &data_dbt); ret = __db_put(db_rep->gmdb, ip, txn, &key_dbt, &data_dbt, 0); if ((t_ret = __db_txn_auto_resolve(env, txn, 0, ret)) != 0 && @@ -980,15 +1433,15 @@ resolve_limbo_int(env, ip) UNLOCK_MUTEX(db_rep->mutex); locked = FALSE; status = NEXT_STATUS(orig_status); - if ((ret = finish_gmdb_update(env, - ip, &key_dbt, orig_status, status, &logrec)) != 0) + if ((ret = finish_gmdb_update(env, ip, + &key_dbt, orig_status, status, site->gmdb_flags, &logrec)) != 0) goto out; /* Track modified membership status in our in-memory sites array. */ LOCK_MUTEX(db_rep->mutex); locked = TRUE; if ((ret = __repmgr_set_membership(env, - addr.host, addr.port, status)) != 0) + addr.host, addr.port, status, site->gmdb_flags)) != 0) goto out; __repmgr_set_sites(env); @@ -1005,14 +1458,15 @@ out: * status is inferred (ADDING -> PRESENT, or DELETING -> 0). * * PUBLIC: int __repmgr_update_membership __P((ENV *, - * PUBLIC: DB_THREAD_INFO *, int, u_int32_t)); + * PUBLIC: DB_THREAD_INFO *, int, u_int32_t, u_int32_t)); */ int -__repmgr_update_membership(env, ip, eid, pstatus) +__repmgr_update_membership(env, ip, eid, pstatus, site_flags) ENV *env; DB_THREAD_INFO *ip; int eid; u_int32_t pstatus; /* Provisional status. */ + u_int32_t site_flags; { DB_REP *db_rep; REPMGR_SITE *site; @@ -1092,7 +1546,7 @@ retry: * those seem even more confusing. */ if ((ret = __repmgr_set_membership(env, - addr.host, addr.port, pstatus)) != 0) + addr.host, addr.port, pstatus, site_flags)) != 0) goto err; __repmgr_set_sites(env); @@ -1108,7 +1562,7 @@ retry: if ((ret = __txn_begin(env, ip, NULL, &txn, DB_IGNORE_LEASE)) != 0) goto err; marshal_site_key(env, &addr, key_buf, &key_dbt, &logrec); - marshal_site_data(env, pstatus, status_buf, &data_dbt); + marshal_site_data(env, pstatus, site_flags, status_buf, &data_dbt); if ((ret = __db_put(db_rep->gmdb, ip, txn, &key_dbt, &data_dbt, 0)) != 0) goto err; @@ -1152,13 +1606,14 @@ retry: locked = FALSE; if ((ret = finish_gmdb_update(env, ip, - &key_dbt, pstatus, ult_status, &logrec)) != 0) + &key_dbt, pstatus, ult_status, site_flags, &logrec)) != 0) goto err; /* Track modified membership status in our in-memory sites array. */ LOCK_MUTEX(db_rep->mutex); locked = TRUE; - ret = __repmgr_set_membership(env, addr.host, addr.port, ult_status); + ret = __repmgr_set_membership(env, addr.host, addr.port, + ult_status, site_flags); __repmgr_set_sites(env); err: @@ -1173,7 +1628,7 @@ err: * that we keep in sync. */ (void)__repmgr_set_membership(env, - addr.host, addr.port, orig_status); + addr.host, addr.port, orig_status, site_flags); } if ((t_ret = __repmgr_cleanup_gmdb_op(env, do_close)) != 0 && ret == 0) @@ -1215,13 +1670,14 @@ retry: UNLOCK_MUTEX(db_rep->mutex); marshal_site_key(env, &addr, key_buf, &key_dbt, &logrec); - if ((ret = finish_gmdb_update(env, - ip, &key_dbt, cur_status, new_status, &logrec)) != 0) + if ((ret = finish_gmdb_update(env, ip, + &key_dbt, cur_status, new_status, site->gmdb_flags, &logrec)) != 0) goto err; /* Track modified membership status in our in-memory sites array. */ LOCK_MUTEX(db_rep->mutex); - ret = __repmgr_set_membership(env, addr.host, addr.port, new_status); + ret = __repmgr_set_membership(env, addr.host, addr.port, + new_status, site->gmdb_flags); __repmgr_set_sites(env); UNLOCK_MUTEX(db_rep->mutex); @@ -1301,11 +1757,11 @@ __repmgr_set_gm_version(env, ip, txn, version) * really deleted. */ static int -finish_gmdb_update(env, ip, key_dbt, prev_status, status, logrec) +finish_gmdb_update(env, ip, key_dbt, prev_status, status, flags, logrec) ENV *env; DB_THREAD_INFO *ip; DBT *key_dbt; - u_int32_t prev_status, status; + u_int32_t prev_status, status, flags; __repmgr_member_args *logrec; { DB_REP *db_rep; @@ -1324,7 +1780,7 @@ finish_gmdb_update(env, ip, key_dbt, prev_status, status, logrec) if (status == 0) ret = __db_del(db_rep->gmdb, ip, txn, key_dbt, 0); else { - marshal_site_data(env, status, data_buf, &data_dbt); + marshal_site_data(env, status, flags, data_buf, &data_dbt); ret = __db_put(db_rep->gmdb, ip, txn, key_dbt, &data_dbt, 0); } if (ret != 0) @@ -1617,16 +2073,18 @@ marshal_site_key(env, addr, buf, dbt, logrec) } static void -marshal_site_data(env, status, buf, dbt) +marshal_site_data(env, status, flags, buf, dbt) ENV *env; u_int32_t status; + u_int32_t flags; u_int8_t *buf; DBT *dbt; { - __repmgr_membership_data_args member_status; + __repmgr_membership_data_args member_data; - member_status.flags = status; - __repmgr_membership_data_marshal(env, &member_status, buf); + member_data.status = status; + member_data.flags = flags; + __repmgr_membership_data_marshal(env, &member_data, buf); DB_INIT_DBT(*dbt, buf, __REPMGR_MEMBERSHIP_DATA_SIZE); } @@ -1640,16 +2098,107 @@ __repmgr_set_sites(env) ENV *env; { DB_REP *db_rep; + REP *rep; int ret; u_int32_t n; u_int i; db_rep = env->rep_handle; + rep = db_rep->region; for (i = 0, n = 0; i < db_rep->site_cnt; i++) { - if (db_rep->sites[i].membership > 0) + /* + * Views do not count towards nsites because they cannot + * vote in elections, become master or contribute to + * durability. + */ + if (db_rep->sites[i].membership > 0 && + !FLD_ISSET(db_rep->sites[i].gmdb_flags, SITE_VIEW)) n++; } ret = __rep_set_nsites_int(env, n); DB_ASSERT(env, ret == 0); + if (FLD_ISSET(rep->config, + REP_C_PREFMAS_MASTER | REP_C_PREFMAS_CLIENT) && + rep->config_nsites > 2) + __db_errx(env, DB_STR("3701", + "More than two sites in preferred master replication group")); +} + +/* + * If a site is rejoining a 2-site repgroup with 2SITE_STRICT off + * and has a rejection because it needs to catch up with the latest + * group membership database, it cannot call an election right away + * because it would win with only its own vote and ignore an existing + * master in the repgroup. Instead, this routine is used to call the + * deferred election after the site has rejoined the repgroup successfully. + */ +static int +rejoin_deferred_election(env) + ENV *env; +{ + DB_REP *db_rep; + u_int32_t flags; + int eid, ret; + + db_rep = env->rep_handle; + LOCK_MUTEX(db_rep->mutex); + + /* + * First, retry all connections so that the election can communicate + * with the other sites. Normally there should only be one other + * site in the repgroup, but it is safest to retry all remote sites + * found in case the group membership changed while we were gone. + */ + FOR_EACH_REMOTE_SITE_INDEX(eid) { + if ((ret = + __repmgr_schedule_connection_attempt(env, eid, TRUE)) != 0) + break; + } + + /* + * Call an immediate, but not a fast, election because a fast + * election reduces the number of votes needed by 1. + */ + flags = ELECT_F_EVENT_NOTIFY; + if (FLD_ISSET(db_rep->region->config, REP_C_ELECTIONS)) + LF_SET(ELECT_F_IMMED); + else + RPRINT(env, (env, DB_VERB_REPMGR_MISC, + "Deferred rejoin election, but no elections")); + ret = __repmgr_init_election(env, flags); + + UNLOCK_MUTEX(db_rep->mutex); + return (ret); +} +/* + * If a site is rejoining a preferred master replication group and has a + * rejection because it needs to catch up with the latest group membership + * database, it needs to establish its "regular" connection to the other site + * so that it can proceed through the preferred master startup sequence. + */ +static int +rejoin_connections(env) + ENV *env; +{ + DB_REP *db_rep; + int eid, ret; + + db_rep = env->rep_handle; + ret = 0; + LOCK_MUTEX(db_rep->mutex); + + /* + * Retry all connections. Normally there should only be one other + * site in the repgroup, but it is safest to retry all remote sites + * found in case the group membership changed while we were gone. + */ + FOR_EACH_REMOTE_SITE_INDEX(eid) { + if ((ret = + __repmgr_schedule_connection_attempt(env, eid, TRUE)) != 0) + break; + } + + UNLOCK_MUTEX(db_rep->mutex); + return (ret); } |