Failover: Add simple controller failover

Fix BUG#47: Add simple controller failover to provide control plane
session redundancy.
Improved BUG#47: Add liveliness check of a peer using ECHO exchange
with non-zero XID.
Fix BUG#50: Change minimum configurable value of inactivity-timer for
OpenFlow control plane session to provide quick controller failover.
Change ofprotocol(8) man page to describe failover.

lib/rconn.c

@@ -131,6 +131,12 @@ struct rconn {
 #define MAX_MONITORS 8
     struct vconn *monitors[8];
     size_t n_monitors;
+
+    /* Protocol statistical informaition. */
+    struct ofpstat ofps_rcvd;
+    struct ofpstat ofps_sent;
+
+    uint32_t idle_echo_xid;
 };
 
 static unsigned int elapsed_in_this_state(const struct rconn *);
@@ -211,10 +217,15 @@ rconn_create(int probe_interval, int max_backoff)
     rc->questionable_connectivity = false;
     rc->last_questioned = time_now();
 
-    rc->probe_interval = probe_interval ? MAX(5, probe_interval) : 0;
+    rc->probe_interval = probe_interval ? MAX(1, probe_interval) : 0;
 
     rc->n_monitors = 0;
 
+    memset(&rc->ofps_rcvd, 0, sizeof(rc->ofps_rcvd));
+    memset(&rc->ofps_sent, 0, sizeof(rc->ofps_sent));
+
+    rc->idle_echo_xid = 0;
+
     return rc;
 }
 
@@ -474,7 +485,18 @@ rconn_recv(struct rconn *rc)
             rc->last_received = time_now();
             rc->packets_received++;
             if (rc->state == S_IDLE) {
-                state_transition(rc, S_ACTIVE);
+                /* Check liveliness of a peer. */
+                if (h->type == OFPT_ECHO_REPLY) {
+                    if (rc->idle_echo_xid == 0) {
+                        state_transition(rc, S_ACTIVE);
+                    } else {
+                        if (rc->idle_echo_xid == h->xid)
+                            state_transition(rc, S_ACTIVE);
+                        rc->idle_echo_xid = 0;
+                    }
+                } else {
+                    state_transition(rc, S_ACTIVE);
+                }
             }
             return buffer;
         } else if (error != EAGAIN) {
@@ -721,8 +743,11 @@ try_send(struct rconn *rc)
     int retval = 0;
     struct ofpbuf *next = rc->txq.head->next;
     int *n_queued = rc->txq.head->private;
+    ofpstat_inc_protocol_stat(&rc->ofps_sent, h);
+    rc->idle_echo_xid = h->xid;
     retval = vconn_send(rc->vconn, rc->txq.head);
     if (retval) {
+        rc->idle_echo_xid = 0;
         if (retval != EAGAIN) {
             disconnect(rc, retval);
         }
@@ -765,7 +790,7 @@ disconnect(struct rconn *rc, int error)
             rc->backoff = 1;
         } else {
             rc->backoff = MIN(rc->max_backoff, MAX(1, 2 * rc->backoff));
-            VLOG_INFO("%s: waiting %d seconds before reconnect\n",
+            VLOG_INFO("%s: waiting %d seconds before reconnect",
                       rc->name, rc->backoff);
         }
         rc->backoff_deadline = now + rc->backoff;

lib/vconn.c

@@ -980,7 +980,7 @@ make_echo_request(void)
     rq->version = OFP_VERSION;
     rq->type = OFPT_ECHO_REQUEST;
     rq->length = htons(sizeof *rq);
-    rq->xid = 0;
+    rq->xid = alloc_xid();
     return out;
 }
 

lib/vlog-modules.def

@@ -15,6 +15,7 @@ VLOG_MODULE(dpif)
 VLOG_MODULE(dpctl)
 VLOG_MODULE(executer)
 VLOG_MODULE(fail_open)
+VLOG_MODULE(failover)
 VLOG_MODULE(fault)
 VLOG_MODULE(flow)
 VLOG_MODULE(flow_end)

secchan/automake.mk

@@ -8,6 +8,8 @@ secchan_ofprotocol_SOURCES = \
 	secchan/executer.h \
 	secchan/fail-open.c \
 	secchan/fail-open.h \
+	secchan/failover.c \
+	secchan/failover.h \
 	secchan/flow-end.c \
 	secchan/flow-end.h \
 	secchan/in-band.c \

secchan/failover.c

@@ -0,0 +1,144 @@
+/*-
+ * Copyright (c) 2008, 2009
+ *      The Board of Trustees of The Leland Stanford Junior University
+ *
+ * We are making the OpenFlow specification and associated documentation
+ * (Software) available for public use and benefit with the expectation that
+ * others will use, modify and enhance the Software and contribute those
+ * enhancements back to the community. However, since we would like to make the
+ * Software available for broadest use, with as few restrictions as possible
+ * permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this Software to deal in the Software under the copyrights without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * The name and trademarks of copyright holder(s) may NOT be used in
+ * advertising or publicity pertaining to the Software or any derivatives
+ * without specific, written prior permission.
+ */
+
+#include <config.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "util.h"
+#include "rconn.h"
+#include "secchan.h"
+#include "status.h"
+#include "timeval.h"
+#include "sat-math.h"
+#include "failover.h"
+#define THIS_MODULE VLM_failover
+#include "vlog.h"
+
+struct failover_peer {
+	time_t epoch;
+};
+
+struct failover_context {
+	const struct settings *settings;
+	const struct secchan *secchan;
+	struct rconn *remote_rconn;
+	int index;
+	struct failover_peer *peers[MAX_CONTROLLERS];
+};
+
+static void failover_status_cb(struct status_reply *, void *);
+static bool is_timed_out(const struct failover_peer *, int);
+static void failover_periodic_cb(void *);
+
+static void
+failover_status_cb(struct status_reply *status_reply, void *context_)
+{
+	struct failover_context *context = context_;
+	int i;
+
+	status_reply_put(status_reply, "num-controllers=%d",
+			 context->settings->num_controllers);
+
+	for (i = 0; i < MAX_CONTROLLERS; ++i) {
+		if (context->settings->controller_names[i] == NULL)
+			continue;
+		status_reply_put(status_reply, "controller#%d=%s",
+				 i, context->settings->controller_names[i]);
+	}
+}
+
+static bool
+is_timed_out(const struct failover_peer *peer, int max_backoff)
+{
+	unsigned int sat_value = sat_add(peer->epoch, max_backoff);
+	return time_now() >= sat_value;
+}
+
+static void
+failover_periodic_cb(void *context_)
+{
+	struct failover_context *context = context_;
+	char *curr_peer = NULL;
+	char *prev_peer = NULL;
+
+	if (rconn_is_connected(context->remote_rconn))
+		return;
+
+	if (!is_timed_out(context->peers[context->index],
+			  context->settings->max_backoff)) {
+		return;
+	}
+
+	rconn_disconnect(context->remote_rconn);
+	prev_peer = (char *)context->settings->controller_names[context->index];
+	context->index = (context->index + 1)
+		% context->settings->num_controllers;
+	curr_peer = (char *)context->settings->controller_names[context->index];
+	rconn_connect(context->remote_rconn,
+		      context->settings->controller_names[context->index]);
+	context->peers[context->index]->epoch = time_now();
+	VLOG_INFO("Switching over to %s, from %s", curr_peer, prev_peer);
+}
+
+void
+failover_start(struct secchan *secchan, const struct settings *settings,
+	       struct switch_status *switch_status, struct rconn *remote_rconn)
+{
+	struct failover_context *context = NULL;
+	int i;
+	static struct hook_class failover_hook_class = {
+		NULL,		/* local_packet_cb */
+		NULL,		/* remote_packet_cb */
+		failover_periodic_cb,	/* periodic_cb */
+		NULL,		/* wait_cb */
+		NULL,		/* closing_cb */
+	};
+
+	context = xmalloc(sizeof(*context));
+	context->settings = settings;
+	context->secchan = secchan;
+	context->remote_rconn = remote_rconn;
+	context->index = 0;
+	for (i = 0; i < MAX_CONTROLLERS; ++i) {
+		context->peers[i] = NULL;
+		if (settings->controller_names[i] == NULL)
+			continue;
+		context->peers[i] = xmalloc(sizeof(struct failover_peer));
+		context->peers[i]->epoch = time_now();
+	}
+
+	switch_status_register_category(switch_status, "failover",
+					failover_status_cb, context);
+	add_hook(secchan, &failover_hook_class, context);
+}

secchan/failover.h

@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2008, 2009
+ *      The Board of Trustees of The Leland Stanford Junior University
+ *
+ * We are making the OpenFlow specification and associated documentation
+ * (Software) available for public use and benefit with the expectation that
+ * others will use, modify and enhance the Software and contribute those
+ * enhancements back to the community. However, since we would like to make the
+ * Software available for broadest use, with as few restrictions as possible
+ * permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this Software to deal in the Software under the copyrights without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * The name and trademarks of copyright holder(s) may NOT be used in
+ * advertising or publicity pertaining to the Software or any derivatives
+ * without specific, written prior permission.
+ */
+
+#ifndef FAILOVER_H_
+#define FAILOVER_H_ 1
+
+struct rconn;
+struct secchan;
+struct settings;
+struct switch_status;
+
+void failover_start(struct secchan *, const struct settings *,
+		    struct switch_status *, struct rconn *);
+
+#endif

secchan/ofprotocol.8.in

@@ -6,7 +6,7 @@ ofprotocol \- secure channel connecting an OpenFlow datapath to a controller
 
 .SH SYNOPSIS
 .B ofprotocol
-[\fIoptions\fR] \fIdatapath\fR [\fIcontroller\fR]
+[\fIoptions\fR] \fIdatapath\fR controller[,\fIcontroller\fR...]
 
 .SH DESCRIPTION
 The \fBofprotocol\fR program sets up a secure channel between a local
@@ -30,8 +30,9 @@ The \fIfile\fR argument must the same one specified on the
 \fBofdatapath\fR command line.
 
 .PP
-The optional \fIcontroller\fR argument specifies how to connect to
-the OpenFlow controller.  It takes one of the following forms:
+The optional \fIcontroller\fR argument specifies how to connect to 
+an OpenFlow controller. Up to four controllers may be specified, 
+using the following forms:
 
 .TP
 \fBssl:\fIhost\fR[\fB:\fIport\fR]
@@ -49,6 +50,10 @@ The specified TCP \fIport\fR (default: 6633) on the given remote
 The Unix domain server socket named \fIfile\fR.
 
 .PP
+If multiple controllers are specified, \fBofprotocol\fR will attempt to
+connect to a new controller when a controller connection fails, times
+out, or is closed, or when a controller stops responding to echo requests.
+
 If \fIcontroller\fR is omitted, \fBofprotocol\fR attempts to discover the
 location of the controller automatically (see below).
 
@@ -262,7 +267,7 @@ channel waits for a message to be received from the controller for
 controller.  After sending the inactivity probe, if no response is
 received for an additional \fIsecs\fR seconds, the secure channel
 assumes that the connection has been broken and attempts to reconnect.
-The default is 15 seconds, and the minimum value is 5 seconds.
+The default is 15 seconds, and the minimum value is 1 seconds.
 
 When fail-open mode is configured, changing the inactivity probe
 interval also changes the interval before entering fail-open mode (see

secchan/secchan.c

@@ -48,6 +48,7 @@
 #include "discovery.h"
 #include "executer.h"
 #include "fail-open.h"
+#include "failover.h"
 #include "fault.h"
 #include "in-band.h"
 #include "leak-checker.h"
@@ -159,7 +160,7 @@ main(int argc, char *argv[])
 
     /* Check datapath name, to try to catch command-line invocation errors. */
     if (strncmp(s.dp_name, "nl:", 3) && strncmp(s.dp_name, "unix:", 5)
-        && !s.controller_name) {
+        && !s.controller_names[0]) {
         VLOG_WARN("Controller not specified and datapath is not nl: or "
                   "unix:.  (Did you forget to specify the datapath?)");
     }
@@ -194,10 +195,10 @@ main(int argc, char *argv[])
 
     /* Connect to controller. */
     remote_rconn = rconn_create(s.probe_interval, s.max_backoff);
-    if (s.controller_name) {
-        retval = rconn_connect(remote_rconn, s.controller_name);
+    if (s.controller_names[0]) {
+        retval = rconn_connect(remote_rconn, s.controller_names[0]);
         if (retval == EAFNOSUPPORT) {
-            ofp_fatal(0, "No support for %s vconn", s.controller_name);
+            ofp_fatal(0, "No support for %s vconn", s.controller_names[0]);
         }
     }
     switch_status_register_category(switch_status, "remote",
@@ -225,6 +226,9 @@ main(int argc, char *argv[])
         fail_open_start(&secchan, &s, switch_status,
                         local_rconn, remote_rconn);
     }
+    if (s.num_controllers > 1) {
+        failover_start(&secchan, &s, switch_status, remote_rconn);
+    }
     if (s.rate_limit) {
         rate_limit_start(&secchan, &s, switch_status, remote_rconn);
     }
@@ -673,8 +677,8 @@ parse_options(int argc, char *argv[], struct settings *s)
 
         case OPT_INACTIVITY_PROBE:
             s->probe_interval = atoi(optarg);
-            if (s->probe_interval < 5) {
-                ofp_fatal(0, "--inactivity-probe argument must be at least 5");
+            if (s->probe_interval < 1) {
+                ofp_fatal(0, "--inactivity-probe argument must be at least 1");
             }
             break;
 
@@ -806,7 +810,23 @@ parse_options(int argc, char *argv[], struct settings *s)
 
     /* Local and remote vconns. */
     s->dp_name = argv[0];
-    s->controller_name = argc > 1 ? xstrdup(argv[1]) : NULL;
+    {
+        char *curr;
+        char *save;
+        int i;
+
+        s->num_controllers = 0;
+        for (i = 0; i < MAX_CONTROLLERS; ++i)
+            s->controller_names[i] = NULL;
+        if (argc > 1) {
+            for (curr = strtok_r(argv[1], ",,", &save), i = 0;
+                 curr && i < MAX_CONTROLLERS;
+                 curr = strtok_r(NULL, ",,", &save), ++i) {
+                s->controller_names[i] = xstrdup(curr);
+                ++s->num_controllers;
+            }
+        }
+    }
 
     /* Set accept_controller_regex. */
     if (!accept_re) {
@@ -823,7 +843,7 @@ parse_options(int argc, char *argv[], struct settings *s)
     s->accept_controller_re = accept_re;
 
     /* Mode of operation. */
-    s->discovery = s->controller_name == NULL;
+    s->discovery = s->controller_names[0] == NULL;
     if (s->discovery && !s->in_band) {
         ofp_fatal(0, "Cannot perform discovery with out-of-band control");
     }

secchan/secchan.h

@@ -51,6 +51,8 @@ enum fail_mode {
 /* Maximum number of management connection listeners. */
 #define MAX_MGMT 8
 
+#define MAX_CONTROLLERS 3
+
 /* Settings that may be configured by the user. */
 struct settings {
     /* Overall mode of operation. */
@@ -59,7 +61,8 @@ struct settings {
 
     /* Related vconns and network devices. */
     const char *dp_name;        /* Local datapath. */
-    const char *controller_name; /* Controller (if not discovery mode). */
+    int num_controllers;        /* Number of configured controllers. */
+    const char *controller_names[MAX_CONTROLLERS]; /* Controllers (if not discovery mode). */
     const char *listener_names[MAX_MGMT]; /* Listen for mgmt connections. */
     size_t n_listeners;          /* Number of mgmt connection listeners. */
     const char *monitor_name;   /* Listen for traffic monitor connections. */