diff --git a/Makefile.am b/Makefile.am index f804ae5..97ac1f0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -13,7 +13,7 @@ else distcheck-hook: endif -AM_CPPFLAGS = $(SSL_CFLAGS) +AM_CPPFLAGS = $(SSL_CFLAGS) -g AM_CPPFLAGS += -I $(top_srcdir)/include AM_CPPFLAGS += -I $(top_srcdir)/lib @@ -65,3 +65,4 @@ include tests/automake.mk include include/automake.mk include third-party/automake.mk include debian/automake.mk +include hw-lib/automake.mk diff --git a/acinclude.m4 b/acinclude.m4 index 8bd73d5..7ccfbaa 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -169,6 +169,94 @@ AC_DEFUN([OFP_CHECK_HWTABLES], done AC_SUBST(HW_TABLES)]) +dnl Checks for --enable-hw-lib and substitutes BUILD_HW_LIBS and plat name +AC_DEFUN([OFP_CHECK_HWLIBS], + [AC_ARG_ENABLE( + [hw-lib], + [AC_HELP_STRING([--enable-hw-lib=PLATFORM], + [Configure and build the specified externally supplied + hardware library: lb4g, t2ref, scorref or nf2])]) + case "${enable_hw_lib}" in # ( + yes) + AC_MSG_ERROR([--enable-hw-lib has a required argument]) + ;; # ( + ''|no) + hw_lib= + NF2=no + LB4G=no + T2REF=no + SCORREF=no + BUILD_HW_LIBS=no + ;; # ( + nf2) + NF2=yes + LB4G=no + T2REF=no + SCORREF=no + hw_lib=$enable_hw_lib + BUILD_HW_LIBS=yes + ;; # ( + lb4g) + NF2=no + LB4G=yes + T2REF=no + SCORREF=no + hw_lib=$enable_hw_lib + BUILD_HW_LIBS=yes + ;; # ( + t2ref) + NF2=no + LB4G=no + T2REF=yes + SCORREF=no + hw_lib=$enable_hw_lib + BUILD_HW_LIBS=yes + ;; # ( + scorref) + NF2=no + LB4G=no + SCORREF=yes + T2REF=no + hw_lib=$enable_hw_lib + BUILD_HW_LIBS=yes + ;; # ( + *) + AC_MSG_ERROR([--enable-hw-lib: Unknown platform: ${enable_hw_lib}]) + BUILD_HW_LIBS=no + ;; + esac + if test $BUILD_HW_LIBS = yes; then + if test -e "$srcdir/hw-lib/automake.mk"; then + : + else + AC_MSG_ERROR([cannot configure HW libraries without "hw-lib" directory]) + fi + AC_DEFINE([BUILD_HW_LIBS], [1], + [Whether the OpenFlow hardware libraries are available]) + fi + if test $NF2 = yes; then + AC_DEFINE([NF2], [1], + [Support NetFPGA platform]) + fi + if test $LB4G = yes; then + AC_DEFINE([LB4G], [1], + [Support Stanford-LB4G platform]) + fi + if test $T2REF = yes; then + AC_DEFINE([T2REF], [1], + [Support Broadcom 56634 reference platform]) + fi + if test $SCORREF = yes; then + AC_DEFINE([SCORREF], [1], + [Support Broadcom 56820 reference platform]) + fi + AM_CONDITIONAL([NF2], [test $NF2 = yes]) + AM_CONDITIONAL([LB4G], [test $LB4G = yes]) + AM_CONDITIONAL([T2REF], [test $T2REF = yes]) + AM_CONDITIONAL([SCORREF], [test $SCORREF = yes]) + AM_CONDITIONAL([BUILD_HW_LIBS], [test $BUILD_HW_LIBS = yes]) + AC_SUBST(HW_LIB)]) + dnl Checks for net/if_packet.h. AC_DEFUN([OFP_CHECK_IF_PACKET], [AC_CHECK_HEADER([net/if_packet.h], diff --git a/configure.ac b/configure.ac index 8b02fd2..13064f6 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,7 @@ fi OFP_CHECK_LIBOPENFLOW OFP_CHECK_IF_PACKET OFP_CHECK_HWTABLES +OFP_CHECK_HWLIBS AC_SYS_LARGEFILE AC_CHECK_FUNCS([strsignal]) diff --git a/hw-lib/automake.mk b/hw-lib/automake.mk new file mode 100644 index 0000000..c792b87 --- /dev/null +++ b/hw-lib/automake.mk @@ -0,0 +1,30 @@ +# +# Hardware library dependency file definitions. +# + +# Sample name +if LB4G +# +# Sample library +# +noinst_LIBRARIES += hw-lib/libskeleton.a + +hw_lib_libskeleton_a_SOURCES = \ + hw-lib/skeleton/of_hw_platform.h \ + hw-lib/skeleton/txrx.c \ + hw-lib/skeleton/txrx.h \ + hw-lib/skeleton/hw_drv.c \ + hw-lib/skeleton/hw_drv.h \ + hw-lib/skeleton/hw_flow.c \ + hw-lib/skeleton/hw_flow.h \ + hw-lib/skeleton/port.c \ + hw-lib/skeleton/port.h \ + hw-lib/skeleton/debug.h + +hw_lib_libskeleton_a_CPPFLAGS = $(AM_CPPFLAGS) $(OF_CPP_FLAGS) +hw_lib_libskeleton_a_CPPFLAGS += -I hw-lib/skeleton +hw_lib_libskeleton_a_CPPFLAGS += -I $(HW_SYSTEM)/include +hw_lib_libskeleton_a_CPPFLAGS += -DSAMPLE_PLAT +hw_lib_libskeleton_a_CPPFLAGS += -DSKELETON + +endif diff --git a/hw-lib/skeleton/debug.h b/hw-lib/skeleton/debug.h new file mode 100644 index 0000000..d6a8f8a --- /dev/null +++ b/hw-lib/skeleton/debug.h @@ -0,0 +1,106 @@ + +#ifndef OF_HW_DEBUG_H +#define OF_HW_DEBUG_H 1 + +#include +#include + +#if !defined(HWTABLE_NO_DEBUG) +extern int of_hw_debug; +#define dbg_send(mod, lvl, fmt, args...) \ + if (of_hw_debug >= (lvl)) fprintf(stderr, fmt, ##args) +#endif + +#define DBG_LVL_NONE -1 /* All output off */ +#define DBG_LVL_ERROR 0 /* Default value */ +#define DBG_LVL_ALWAYS 0 /* For requested dump output */ +#define DBG_LVL_WARN 1 +#define DBG_LVL_VERBOSE 2 +#define DBG_LVL_VVERB 3 /* Include success indications */ + +/* Sorry for the lazy syntax here. */ +#define DBG_CHECK(lvl) (((lvl) >= 0) && (of_hw_debug >= (lvl))) +#define DBG_ERROR(fmt, args...) dbg_send(0, DBG_LVL_ERROR, fmt, ##args) +#define DBG_ALWAYS(fmt, args...) dbg_send(0, DBG_LVL_ALWAYS, fmt, ##args) +#define DBG_WARN(fmt, args...) dbg_send(0, DBG_LVL_WARN, fmt, ##args) +#define DBG_VERBOSE(fmt, args...) dbg_send(0, DBG_LVL_VERBOSE, fmt, ##args) +#define DBG_VVERB(fmt, args...) dbg_send(0, DBG_LVL_VVERB, fmt, ##args) +#define DBG_NONE(fmt, args...) +/* Same as DEBUG_ALWAYS */ +#define DEBUGK(fmt, args...) dbg_send(0, DBG_LVL_ALWAYS, fmt, ##args) + +#define REPORT_ERROR(str) \ + DBG_ERROR("ERROR: %s:%d. %s\n", __FUNCTION__, __LINE__, str) + +/* Default debugging location string */ +#define ANNOUNCE_LOCATION DBG_VVERB("%s: %d\n", __FUNCTION__, __LINE__) +#define DBG_INCR(cnt) (++(cnt)) + +/* Should assert return? Not presently */ +#define ASSERT(cond) \ + if (!(cond)) DBG_ERROR("ASSERTION %s IN %s FAILED LINE %d\n", \ + #cond, __FUNCTION__, __LINE__) + +/* DEBUG for HW flow lists */ +#define HW_FLOW_MAGIC 0xba5eba11 +#define HW_FLOW_MAKE_VALID(hf) (hf)->magic = HW_FLOW_MAGIC +#define HW_FLOW_IS_VALID(hf) \ + (((hf) != NULL) && ((hf)->magic == HW_FLOW_MAGIC)) + +/* + * Carry out an operation and check for error, optionally returning + * from the calling routine. To avoid compiler + * warnings in routines with void return, we give two macros. + * + * WARNING: This check uses rv != 0 (not just rv < 0). + */ + +#define TRY(op, str) do { \ + int rv; \ + if (((rv = (op)) < 0)) { \ + DBG_ERROR("ERROR %d: %s\n", rv, str); \ + return rv; \ + } else { \ + DBG_NONE("%s: success\n", str); \ + } \ + } while (0) + +#define TRY_NR(op, str) do { \ + int rv; \ + if (((rv = (op)) != 0)) { \ + DBG_ERROR("ERROR %d: %s\n", rv, str); \ + } else { \ + DBG_NONE("%s: success\n", str); \ + } \ + } while (0) + +#else /* No debugging */ + +#define DBG_CHECK(lvl) 0 +#define ANNOUNCE_LOCATION +#define ACT_STRING(action) "" +#define DBG_INCR(cnt) +#define DBG_ERROR(fmt, args...) +#define DBG_ALWAYS(fmt, args...) +#define DBG_WARN(fmt, args...) +#define DBG_VERBOSE(fmt, args...) +#define DBG_VVERB(fmt, args...) +#define DBG_NONE(fmt, args...) +#define DEBUGK(fmt, args...) +#define REPORT_ERROR(str) + +#define ASSERT(cond) + +#define HW_FLOW_MAKE_VALID(hf) +#define HW_FLOW_IS_VALID(hf) 1 + +#define TRY(op, str) do { \ + int rv; \ + if (((rv = (op)) != 0)) return rv; \ + } while (0) + +#define TRY_NR(op, str) (void)op + +#endif /* !defined(HWTABLE_NO_DEBUG) */ + +#endif /* OF_HW_DEBUG_H */ diff --git a/hw-lib/skeleton/hw_drv.c b/hw-lib/skeleton/hw_drv.c new file mode 100644 index 0000000..0ecd590 --- /dev/null +++ b/hw-lib/skeleton/hw_drv.c @@ -0,0 +1,309 @@ +/* + * Skeleton hardware datapath, internal implemenation + * + */ +#include + +#include +#include + +#include "list.h" + +#include "os.h" +#include "debug.h" +#include "of_hw_platform.h" +#include "hw_drv.h" +#include "hw_flow.h" +#include "port.h" +#include "txrx.h" +#include "udatapath/switch-flow.h" + +/**************************************************************** + * + * Hardware resource management section + * + ****************************************************************/ + +static int global_init_done = 0; +int of_hw_drv_instances = 0; + +#if !defined(HWTABLE_NO_DEBUG) +int of_hw_debug = DBG_LVL_WARN; +#endif + +/* Internal initialization of HW datapath structures */ +static int +hw_drv_global_init(int init_hw) +{ + HW_DRV_MUTEX_INIT; + of_hw_ports_init(); + + if (init_hw) { /* Do hardware initialization */ + TRY(PLATFORM_INIT(), "PLATFORM INIT"); + TRY_NR(of_hw_fp_setup(), "global FP setup"); + } + + global_init_done = 1; + return 0; +} + +/* Skeleton capabilities structure */ +static of_hw_driver_caps_t of_hw_caps = { + .flags = , + .max_flows = , + .wc_supported = OFPFW_ALL, + .actions_supported = OFPAT_XXX, + .ofpc_flags = OFPC_XXX +}; + +/* Initialize the internal HW datapath structure */ +static int +hw_drv_int_init(of_hw_driver_int_t *hw_int, struct datapath *dp) +{ + static int dp_idx = 0; + + /* Fill in caps object */ + memcpy(&hw_int->hw_driver.caps, &of_hw_caps, sizeof(of_hw_caps)); + hw_int->dp = dp; + hw_int->dp_idx = dp_idx++; + + hw_int->n_flows = 0; + list_init(&hw_int->flows); + list_init(&hw_int->iter_flows); + hw_int->next_serial = 0; + + return 0; +} + +/**************************************************************** + * + * Driver functions + * + ****************************************************************/ + +/* + * of_hw_ioctl + * + * HW IOCTL function + */ + +static int +of_hw_ioctl(of_hw_driver_t *hw_drv, uint32_t op, void **io_param, + int *io_len UNUSED) +{ + int val; + of_hw_driver_int_t *hw_int; + int rv = 0; + + hw_int = (of_hw_driver_int_t *)hw_drv; + + switch (op) { + case OF_HW_IOCTL_TABLE_DEBUG_SET: + val = *(int *)(*io_param); + DBG_WARN("Changing table debug value to %d\n", val); + /* FIXME: Use HW driver for debug level? */ + hw_int->table_debug_level = val; + of_hw_debug = val; + break; + case OF_HW_IOCTL_PORT_DEBUG_SET: + val = *(int *)(*io_param); + DBG_WARN("Changing port debug value to %d\n", val); + hw_int->port_debug_level = val; + break; + case OF_HW_IOCTL_BYTE_PKT_CNTR_SET: + val = *(int *)(*io_param); + if ((val != OF_HW_CNTR_PACKETS) && + (val != OF_HW_CNTR_BYTES)) { + DBG_ERROR("Bad byte/pkt counters select value %d\n", val); + } else { + hw_int->stat_sel = val; + } + break; + default: + rv = -1; /* Change to UNSUPPORTED */ + break; + } + + return 0; +} + +static void +of_hw_table_destroy(struct sw_table *table) +{ + of_hw_driver_int_t *hw_int; + + hw_int = (of_hw_driver_int_t *)table; + if (hw_int != NULL && hw_int->dp != NULL) { + DBG_WARN("Table destroy called for dp idx %d\n", + hw_int->dp_idx); + } else { + DBG_ERROR("Table destroy called on NULL dp\n"); + } + + delete_of_hw_driver(&hw_int->hw_driver); +} + +static int +of_hw_table_iterate(struct sw_table *table, + const struct sw_flow_key *key, uint16_t out_port, + struct sw_table_position *position, + int (*callback)(struct sw_flow *flow, void *private), + void *private) +{ + + of_hw_driver_int_t *hw_int; + struct sw_flow *flow; + unsigned long start; + + hw_int = (of_hw_driver_int_t *)table; + if (hw_int != NULL) { + start = ~position->private[0]; + LIST_FOR_EACH (flow, struct sw_flow, iter_node, &hw_int->iter_flows) { + if (flow->serial <= start + && flow_matches_2wild(key, &flow->key) + && flow_has_out_port(flow, out_port)) { + int error; + + /* Update stats as that's what's generally used */ + TRY_NR(of_hw_sw_flow_stat_update(flow, NULL, false), + "sw flow stat update"); + error = callback(flow, private); + if (error) { + position->private[0] = ~(flow->serial - 1); + return error; + } + } + } + } else { + DBG_ERROR("Table iterate called on NULL driver or SW flow table\n"); + return -1; + } + + return 0; +} + +static const char *hw_drv_name = "HW FlowDriver"; + +static void +of_hw_sw_table_stats(struct sw_table *table, struct sw_table_stats *stats) +{ + of_hw_driver_int_t *hw_int; + unsigned long matched = 0; + unsigned long missed = 0; + + hw_int = (of_hw_driver_int_t *)table; + stats->name = hw_drv_name; + stats->wildcards = OFPFW_ALL; + stats->n_flows = hw_int->n_flows; + stats->max_flows = of_hw_caps.max_flows; + /* FIXME: Collect stats */ + TRY_NR(of_hw_table_stats_update(hw_int, &matched, &missed), + "of_hw_table_stats_update"); + stats->n_lookup = missed + matched; + stats->n_matched = matched; +} + +static int +of_hw_table_stats_get(of_hw_driver_t *hw_drv, struct ofp_table_stats *stats) +{ + DBG_WARN("TABLE STATS FIXME\n"); + return 0; +} + +/* + * Create and initialize a new hardware datapath object + */ + +of_hw_driver_t * +new_of_hw_driver(struct datapath *dp) +{ + struct sw_table *sw_tab; + of_hw_driver_t *hw_drv; + of_hw_driver_int_t *hw_int; + + if (!global_init_done) { + if (hw_drv_global_init(1) < 0) { + REPORT_ERROR("HW driver global init failed\n"); + return NULL; + } + } + hw_int = ALLOC(sizeof(*hw_int)); + if (hw_int == NULL) { + REPORT_ERROR("Could not allocate HW driver\n"); + return NULL; + } + memset(hw_int, 0, sizeof(*hw_int)); + + if (hw_drv_int_init(hw_int, dp) < 0) { + FREE(hw_int); + return NULL; + } + + + /* These all point to the same place */ + hw_drv = &hw_int->hw_driver; + sw_tab = &hw_drv->sw_table; + + sw_tab->n_lookup = 0; + sw_tab->n_matched = 0; + + /* Fill out the function pointers */ + sw_tab->lookup = of_hw_flow_lookup; + sw_tab->insert = of_hw_flow_install; + sw_tab->modify = of_hw_flow_modify; + sw_tab->delete = of_hw_flow_delete; + sw_tab->timeout = of_hw_flow_timeout; + + sw_tab->destroy = of_hw_table_destroy; + sw_tab->iterate = of_hw_table_iterate; + sw_tab->stats = of_hw_sw_table_stats; + + hw_drv->table_stats_get =of_hw_table_stats_get; + hw_drv->port_stats_get = of_hw_port_stats_get; + hw_drv->flow_stats_get = NULL; + hw_drv->aggregate_stats_get = NULL; + + hw_drv->port_add = of_hw_port_add; + hw_drv->port_remove = of_hw_port_remove; + hw_drv->port_link_get = of_hw_port_link_get; + hw_drv->port_enable_set = of_hw_port_enable_set; + hw_drv->port_enable_get = of_hw_port_enable_get; + hw_drv->port_queue_config = of_hw_port_queue_config; + hw_drv->port_queue_remove = of_hw_port_queue_remove; + hw_drv->port_change_register = of_hw_port_change_register; + + hw_drv->packet_send = of_hw_packet_send; + hw_drv->packet_receive_register = of_hw_packet_receive_register; + + hw_drv->ioctl = of_hw_ioctl; + + ++of_hw_drv_instances; + + return hw_drv; +} + +/* + * Deallocate a hardware datapath object + * If clear_hw is set, the HW structures related to the + * datapath are also cleared. + * + * In general, clear_hw should be set for now. + */ +void +delete_of_hw_driver(of_hw_driver_t *hw_drv) +{ + int idx; + of_hw_driver_int_t *hw_int; + + hw_int = (of_hw_driver_int_t *)hw_drv; + + /* Clear the port table of ownership for this dp */ + FOREACH_DP_PORT(idx, hw_drv) { + of_hw_ports[idx].owner = NULL; + } + + of_hw_flow_remove_all(hw_int); + + FREE(hw_drv); + --of_hw_drv_instances; +} diff --git a/hw-lib/skeleton/hw_drv.h b/hw-lib/skeleton/hw_drv.h new file mode 100644 index 0000000..7e6ecb0 --- /dev/null +++ b/hw-lib/skeleton/hw_drv.h @@ -0,0 +1,97 @@ +#ifndef OF_HW_DRV_H +#define OF_HW_DRV_H 1 + +#include +#include "of_hw_platform.h" +#include "list.h" + +/**************************************************************** + * + * High Level Design Notes + * + * Hardware tables are managed field_control_t and entry_control_t + * structures in hw_flow.h; these roughly mirror the HW structure + * and are preallocated. + * + * The glue between these and SW flows are provided by the HW flow + * objects (also in hw_flow.h) which are dynamically allocated and + * logically extend the SW flow object. + * + * A linear table of HW flow objects is maintained, mostly drawn + * from the existing table-linear.c implementation in udatapath. + * + */ + +/**************************************************************** + * Hardware independent port mapping macros + ****************************************************************/ + +/* Map ports using software data structures and DP check; independent of HW */ +#define OF_PORT_IN_DP(_hwdrv, _i) (((_i) < OF_HW_MAX_PORT) && \ + (of_hw_ports[_i].owner == (of_hw_driver_t *)(_hwdrv))) + +#define OF_PORT_TO_HW_PORT(_hwdrv, _i) \ + (OF_PORT_IN_DP(_hwdrv, _i) ? of_hw_ports[_i].port : -1) + +/* TBD: Define flow tracking objects */ + +/**************************************************************** + * + * Hardware datapath internal control structure + * Extends generic hardware datapath structure + * (which currently extends software table structure) + * + ****************************************************************/ + +/* Counts the number of instances */ +extern int of_hw_drv_instances; + +typedef struct of_hw_driver_int { + of_hw_driver_t hw_driver; + struct datapath *dp; /* Owning datapath */ + + /* Maintain a linked list of flows for tracking */ + struct list flows; /* The main list */ + struct list iter_flows; /* For iteration operation */ + unsigned long int next_serial; + + /* Callback function for received packets. */ + of_packet_in_f rx_handler; + void *rx_cookie; + + /* Callback function for port change notification */ + of_port_change_f port_change; + void *port_change_cookie; + + /* Lock object? */ + + // struct list sw_flows; + // struct list iter_sw_flows; + // struct sw_flow *iter_state; /* Place tracker for interrupted iterations */ + + /* Stats */ + uint32_t n_flows; /* Current number of flows in table */ + uint32_t n_inserts; /* Total inserts */ + uint32_t rx_pkt_alloc_failures; + uint32_t insert_errors; + + /* Configuration */ + /* Port bitmap of ports in DP by device */ + FIXME port_bitmap; + FIXME flood_bitmap; + int stat_sel; /* Packets or bytes */ + int dp_idx; /* In HW dp, but not SW dp */ + + int table_debug_level; + int port_debug_level; +} of_hw_driver_int_t; + +/* HW_DRV_MUTEX object ? */ + +#define HW_DRV_MUTEX_INIT /* TBD */ +#define HW_DRV_LOCK /* TBD */ +#define HW_DRV_UNLOCK /* TBD */ + +#define HW_INT(hw_drv) ((of_hw_driver_int_t *)hw_drv) + +#endif /* OF_HW_DRV_H */ diff --git a/hw-lib/skeleton/hw_flow.c b/hw-lib/skeleton/hw_flow.c new file mode 100644 index 0000000..2f9d6ec --- /dev/null +++ b/hw-lib/skeleton/hw_flow.c @@ -0,0 +1,708 @@ +/* + * Functions related to hardware flow processing; this is the glue + * and administration between upper level flow operations and lower + * level field processor operations. + */ + +#include +#include +#include + +#include "os.h" +#include "debug.h" +#include "hw_drv.h" +#include "port.h" +#include "hw_flow.h" +#include "udatapath/switch-flow.h" +#include "udatapath/datapath.h" +#include "lib/packets.h" + +static field_control_t field_control; + +/**************************************************************** + * + * SW/HW flow correlation book keeping routines + * + ****************************************************************/ + +/* Allocate and initialize a HW flow control structure. */ +static hw_flow_t * +hw_flow_create(struct sw_flow *flow) +{ + hw_flow_t *hw_flow; + + if ((hw_flow = ALLOC(sizeof(*hw_flow))) != NULL) { + memset(hw_flow, 0, sizeof(*hw_flow)); + HW_FLOW_MAKE_VALID(hw_flow); + hw_flow->flow = flow; + flow->private = (void *)hw_flow; + } + + return hw_flow; +} + +/* Remove an FP entry from a device */ +static int +HW_entry_remove(entry_control_t *entry) +{ + FIXME; + + return 0; +} + +/**************************************************************** + * + * Flow stats operations + * + ****************************************************************/ + +/* + * Read and convert the stats for an FP entry; maintains state + * of last counter value read and accumulates differences. + * + * Assumes mutex held. + */ +static int +entry_stat_update(entry_control_t *ent, int *used) +{ + unsigned long int cur_counter; + + FIXME_get_current_stats(cur_counter); + + ent->total_counter += cur_counter - ent->last_counter; + ent->last_counter = cur_counter; + + if (used != NULL) { + *used |= (ent->used_check != cur_counter); + ent->used_check = cur_counter; + } + + return 0; +} + +/* + * Get the stats for an OF (SW) flow object from HW tables + * NOTE: This will clear any existing value from the flow + * counters; if SW table counters need to be added in, that + * should be done after this calculation. + * + * If used is non-NULL, it will be filled with a boolean + * indication of whether the flow's counters have changed + * from their current state. + * + * Assumes mutex held. + * + * If force_sync is true, will sync SW counters with HW first + */ +int +of_hw_sw_flow_stat_update(struct sw_flow *flow, int *used, int force_sync) +{ + entry_control_t *ent; + hw_flow_t *hw_flow; + int idx; + + if (used != NULL) { + *used = false; + } + hw_flow = (hw_flow_t *)(flow->private); + if (!HW_FLOW_IS_VALID(hw_flow)) { + DBG_ERROR("BAD HW FLOW OBJECT %p for flow %p\n", hw_flow, flow); + return -1; + } + ASSERT(hw_flow->entry_count <= HW_FLOWS_PER_FLOW_MAX); + + if (force_sync) { + FIXME_sync_hw_stats(); + } + + /* FIXME: Will this accumulate and clear HW counters? */ + hw_flow->hw_byte_count = hw_flow->hw_packet_count = 0; + for (idx = 0; idx < hw_flow->entry_count; idx++) { + ent = hw_flow->entry_list[idx]; + + if (entry_stat_update(ent, used) < 0) { + DBG_WARN("Warning: could not get stats for flow\n"); + continue; + } + FIXME_update proper hw_flow counters; + } + flow->packet_count = hw_flow->hw_packet_count; + flow->byte_count = hw_flow->hw_byte_count; + + return 0; +} + +/* Read all the stats from the chip and update counters in flow tab */ +/* Index 0 holds default rule that indicates a missed count */ +/* ASSUMES LOCK HELD */ +int +of_hw_table_stats_update(of_hw_driver_int_t *hw_drv, unsigned long *matched, + unsigned long *missed) +{ + struct sw_flow *flow, *n; + of_hw_driver_int_t *hw_int; + + hw_int = (of_hw_driver_int_t *)hw_drv; + *matched = 0; + *missed = 0; + + if (STATS_BYTES(hw_int->stat_sel)) { /* Lookup count not supported */ + return -1; + } + + LIST_FOR_EACH_SAFE (flow, n, struct sw_flow, node, &hw_int->flows) { + if (flow->private != NULL) { + TRY_NR(of_hw_sw_flow_stat_update(flow, NULL, false), + "update flow stat"); + *matched += flow->packet_count; + } + } + + /* + * FIXME: To get missed stats, query entry 0 of HW table. + * This isn't really correct if there are entries in the SW datapath + * + * FIXME: If there are multiple data paths, should have one + * default entry qualifying on ports in that datapath with + * another entry dropping any packets on other ports. + */ + { + field_control_t *fc; + entry_control_t *ent; + + fc = &field_control; + ent = &fc->entry_list[0]; + ASSERT(ent->in_use); + TRY_NR(entry_stat_update(ent, NULL), "tab: entry_stat_update"); + *missed += ent->total_counter; + } + + return 0; +} + +/* + * Remove the FP entry or entries associated to a SW flow from the HW; + * + */ +static int +hw_flow_remove(struct sw_flow *flow) +{ + field_control_t *fc; + entry_control_t *ent; + hw_flow_t *hw_flow; + int idx; + +#if 0 /* Do not remove entries (debugging) */ + return 0; +#endif + + if (flow == NULL) { + return 0; + } + + hw_flow = (hw_flow_t *)(flow->private); + DBG_VERBOSE("Removing hw flow %p, flow %p\n", hw_flow, flow); + if (!HW_FLOW_IS_VALID(hw_flow)) { + return 0; + } + ASSERT(hw_flow->entry_count <= HW_FLOWS_PER_FLOW_MAX); + + TRY_NR(of_hw_sw_flow_stat_update(flow, NULL, true), + "rmv flow stat update"); + for (idx = 0; idx < hw_flow->entry_count; idx++) { + ent = hw_flow->entry_list[idx]; + ASSERT(ent != NULL); + + TRY_NR(HW_entry_remove(ent), "HW_entry_remove"); + hw_flow->entry_list[idx] = NULL; + } + FREE(hw_flow); + flow->private = NULL; + + return 0; +} + + +/**************************************************************** + * + * Important support routines related to FP setup, actions, flows + * + ****************************************************************/ + +/* ASSERT: of_port belongs to datapath */ +static void +add_dport_to_extra(of_hw_driver_int_t *hw_int, int of_port, + hw_flow_extra_t *extra) +{ + ASSERT(OF_PORT_IN_DP(hw_int, of_port)); + + extra->dest_port = MAP_OF_PORT_TO_HW_PORT(hw_int, of_port); + FIXME_update extra->dest_count if appropriate; +} + +/* Set up flood/all bitmaps + * NOTE: Assumes source port is not wildcarded. + */ +static void +mcast_bitmaps_set(of_hw_driver_int_t *hw_int, hw_flow_extra_t *extra, + int dest_port) +{ + FIXME; +} + +/* + * Validate actions for a flow relative to HW capabilities; + * + * Bool: true means supported, false not supported + * + * Accumulate additional info about the flow in extra. Specifically, + * this determines the output ports on each physical device related + * to the flow. + * + * FIXME: No-flood ports are currently not specified + */ + +static int +actions_supported_check(of_hw_driver_int_t *hw_int, + const struct sw_flow_key *key, + const struct ofp_action_header *actions, + size_t actions_len, + hw_flow_extra_t *extra) +{ + uint8_t *p; + int src_port; /* Port in host order */ + int dest_port; /* Port in host order */ + int action; + int src_is_wc = false; /* Is source wildcarded? */ + + ANNOUNCE_LOCATION; + ASSERT(extra != NULL); + + FIXME("probably needs work for your platform"); + + p = (uint8_t *)actions; + src_port = ntohs(key->flow.in_port); + + src_is_wc = key->wildcards & OFPFW_IN_PORT; + if (!src_is_wc) { + if (OF_PORT_IN_DP(hw_int, src_port)) { + extra->source_port = MAP_TO_HW_PORT(hw_int, src_port); + } else { + DBG_WARN("Source port %d not in DP %d\n", src_port, + hw_int->dp_idx); + return false; + } + } else { + extra->source_port = -1; + if (of_hw_drv_instances > 1) { + DBG_WARN("Source port WC not supported w/ multi-DP\n"); + return false; /* Not currently supported w/ multiple datapaths */ + } + /* NOTE/FIXME: Source wildcard with multiple output ports requires + * using a per-port hardware flow rule to allow filtering of + * the source port. (Currently not supported and checked below.) + */ + } + + while (actions_len > 0) { + struct ofp_action_header *ah = (struct ofp_action_header *)p; + struct ofp_action_output *oa = (struct ofp_action_output *)p; + size_t len = ntohs(ah->len); + + action = ntohs(ah->type); + DBG_VVERB("action chk %d\n", action); + /* Currently supported: output port(s) action */ + if ((action < 0) || (action > ACTION_LAST)) { + DBG_WARN("Unknown action id %d\n", action); + return false; + } + + if (action_map[action] == ACTION_NOT_SUPPORTED) { + DBG_WARN("Action %d not supported\n", action); + return false; + } + + /* Support front panel ports plus IN_PORT, ALL, FLOOD. */ + if ((action == OFPAT_OUTPUT) || (action == OFPAT_ENQUEUE)) { + /* FIXME: For now, using the fact that enqueue and output + * actions are the same at the start of the structure + */ + dest_port = ntohs(oa->port); + if (dest_port == OFPP_TABLE) { /* Unsupported */ + DBG_WARN("Warning: TABLE output action seen\n"); + return false; + } else if (OF_PORT_IN_DP(hw_int, dest_port)) { + add_dport_to_extra(hw_int, dest_port, extra); + } else if ((dest_port == OFPP_FLOOD) || (dest_port == OFPP_ALL)) { + mcast_bitmaps_set(hw_int, extra, dest_port); + } else if (dest_port == OFPP_IN_PORT) { + if (src_is_wc) { + DBG_WARN("Warning: IN_PORT action on source wildcard\n"); + return false; + } + add_dport_to_extra(hw_int, src_port, extra); + } else if (dest_port == OFPP_CONTROLLER) { + /* Controller/local are implemented with a "copy to CPU" + * action and a special reason code; Don't count as output port + */ + extra->local_reason |= CPU_REASON_TO_CONTROLLER; + } else if (dest_port == OFPP_LOCAL) { + extra->local_reason |= CPU_REASON_TO_LOCAL; + } else { /* NORMAL */ + // DBG_WARN("Output action to port 0x%x not supported\n", + // dest_port); + // return false; /* FIXME: Ignore bad ports for now */ + } + if (action == OFPAT_ENQUEUE) { + uint32 qid; + struct ofp_action_enqueue *ea; + ea = (struct ofp_action_enqueue *)p; + qid = ntohl(ea->queue_id); + if ((extra->cosq = of_hw_qid_find(dest_port, qid)) < 0) { + DBG_WARN("Warning: qid %d, port %d, map to cos failed\n", + qid, dest_port); + } + } + } + p += len; + actions_len -= len; + } + + if (src_is_wc && (extra->dest_count > 1)) { + DBG_WARN("Warning: multi dest w/ src wildcard\n"); + return false; + } + + DBG_VERBOSE("Actions supported\n"); + return true; +} + +/**************************************************************** + * + * FP Table management and manipulation routines + * + ****************************************************************/ + +/* + * Functions related to hardware flow table manipulation and maintenance + */ + +/* Set up field control structure; should be idempotent */ +static void +field_control_init(void) +{ + field_control_t *fc; + int idx; + + fc = &field_control; + fc->entry_count = 0; + for (idx = 0; idx < FIELD_ENTRY_MAX; idx++) { + fc->entry_list[idx].in_use = 0; + fc->entry_list[idx].hw_flow = NULL; + fc->entry_list[idx].index = idx; + fc->entry_list[idx].last_counter = 0; + fc->entry_list[idx].counter_last_check = 0; + fc->entry_list[idx].total_counter = 0; + } +} + +/* Alloc HW entry control structure, add qualification, actions and install */ +static int +hw_entry_install(of_hw_driver_int_t *hw_int, + struct sw_flow *flow, hw_flow_t *hw_flow, + hw_flow_extra_t *extra) +{ + + /* Install HW entry */ + FIXME; +} + +/* + * hw_flow_install + * + * Install HW table entries corresponding to the given SW flow + */ +static int +hw_flow_install(of_hw_driver_int_t *hw_int, struct sw_flow *flow, + hw_flow_extra_t *extra) +{ + hw_flow_t *hw_flow; + + /* Allocate the HW flow object */ + if ((hw_flow = hw_flow_create(flow)) == NULL) { + DBG_ERROR("failed to create hw flow struct\n"); + return -1; + } + + DBG_VERBOSE("hw flow install: sw %p. hw %p\n", flow, hw_flow); + TRY(hw_entry_install(hw_int, 0, flow, hw_flow, extra), + "local entry"); + + return 0; +} + +/**************************************************************** + * + * The driver APIs + * + ****************************************************************/ + +/* + * Install a flow. + * + * First, check that the flow is supported; simultaneously, build + * up the "extra" information needed by the hardware for its installation. + * This includes bitmaps of the ports to which the packet will be + * forwarded (on local and remote devices if appropriate). + * + * Then look to see if the flow should overwrite an existing entry. + * If so, just remove that entry. + * + * Then call hw_flow_install which does all the actual HW changes + * based on the flow and on extra. + */ + +/* NOTE: Returns 1 if flow installed, not normal error code */ +int +of_hw_flow_install(struct sw_table *sw_tab, struct sw_flow *flow) +{ + hw_flow_extra_t extra; + of_hw_driver_int_t *hw_int; + int hw_rc = 0; + int sw_insert_done = 0; + struct sw_flow *f; + + hw_int = (of_hw_driver_int_t *)sw_tab; + + DBG_VERBOSE("flow install %p\n", flow); + memset(&extra, 0, sizeof(extra)); + extra.cosq = -1; + + if (!actions_supported_check(hw_int, &flow->key, flow->sf_acts->actions, + flow->sf_acts->actions_len, &extra)) { + /* Unsupported actions */ + DBG_VERBOSE("HW install failed: Unsupported actions\n"); + return 0; + } + + /* LOCK; */ + /* Go through list looking for matching flows */ + LIST_FOR_EACH (f, struct sw_flow, node, &hw_int->flows) { + if (f->priority == flow->priority + && f->key.wildcards == flow->key.wildcards + && flow_matches_2wild(&f->key, &flow->key)) { + /* Just remove the HW flow; install other below */ + TRY_NR(hw_flow_remove(f), "hw_flow_remove for replace"); + flow->serial = f->serial; + list_replace(&flow->node, &f->node); + list_replace(&flow->iter_node, &f->iter_node); + sw_insert_done = 1; + flow_free(f); + break; + } + if (f->priority < flow->priority) { + break; + } + } + + /* ASSERT: sw_insert_done OR f points to insertion point for flow */ + + hw_rc = hw_flow_install(hw_int, flow, &extra); + + if (hw_rc < 0) { + hw_int->insert_errors++; + if (sw_insert_done) { /* Remove from SW list */ + list_remove(&flow->node); + list_remove(&flow->iter_node); + flow_free(flow); + hw_int->n_flows--; + } + } else { + hw_int->n_flows++; + hw_int->n_inserts++; + if (!sw_insert_done) { + flow->serial = hw_int->next_serial++; + list_insert(&f->node, &flow->node); + list_push_front(&hw_int->iter_flows, &flow->iter_node); + } + } + /* UNLOCK */ + + if (hw_rc < 0) { + DBG_WARN("Could not install flow in HW\n"); + if (sw_insert_done) { /* Remove from SW list */ + DBG_WARN("Removed matching flow but could not replace in HW\n"); + } + return 0; + } + + return 1; +} + +/* FIXME: Change API to return list of deleted flows? */ +/* Would make this easier */ +/* Remove a flow or flows from the HW and SW tracking tables */ +int +of_hw_flow_delete(struct datapath *dp, struct sw_table *sw_tab, + const struct sw_flow_key *key, uint16_t out_port, + uint16_t priority, int strict) +{ + of_hw_driver_int_t *hw_int = (of_hw_driver_int_t *)sw_tab; + struct sw_flow *flow, *n; + int count = 0; + struct list deleted; + + DBG_VERBOSE("delete: dp %p, idx %d, key %p, out 0x%x, prio %d, strict %d\n", + dp, hw_int->dp_idx, key, out_port, priority, strict); + + list_init(&deleted); + + /* LOCK; */ + LIST_FOR_EACH_SAFE (flow, n, struct sw_flow, node, &hw_int->flows) { + if (flow_matches_desc(&flow->key, key, strict) + && flow_has_out_port(flow, out_port) + && (!strict || (flow->priority == priority))) { + TRY_NR(hw_flow_remove(flow), "hw flow remove"); + list_remove(&flow->node); + list_remove(&flow->iter_node); + list_push_back(&deleted, &flow->node); + count++; + } + } + /* UNLOCK; */ + + /* Notify DP of deleted flows */ + LIST_FOR_EACH_SAFE (flow, n, struct sw_flow, node, &deleted) { + dp_send_flow_end(dp, flow, flow->reason); + list_remove(&flow->node); + flow_free(flow); + } + + return count; +} + + +/* + * Modify an existing flow + * + * We chicken out and just de-install/re-install in HW + */ +int +of_hw_flow_modify(struct sw_table *sw_tab, const struct sw_flow_key *key, + uint16_t priority, int strict, + const struct ofp_action_header *actions, size_t actions_len) +{ + of_hw_driver_int_t *hw_int = (of_hw_driver_int_t *)sw_tab; + struct sw_flow *flow; + int count = 0; + hw_flow_extra_t extra; + + memset(&extra, 0, sizeof(extra)); + extra.cosq = -1; +#if defined(PLATFORM_HAS_REMOTES) + extra.new_vid = -1; + extra.new_pcp = -1; +#endif + + if (!actions_supported_check(hw_int, &flow->key, actions, + actions_len, &extra)) { + /* Unsupported actions */ + DBG_VERBOSE("Mod actions-supported failed: Unsupported actions\n"); + return 0; + } + + /* LOCK; */ + LIST_FOR_EACH (flow, struct sw_flow, node, &hw_int->flows) { + if (flow_matches_desc(&flow->key, key, strict) + && (!strict || (flow->priority == priority))) { + /* Change the flow, de-install from HW and re-install */ + TRY_NR(hw_flow_remove(flow), "hw_flow_remove modify"); + flow_replace_acts(flow, actions, actions_len); + TRY_NR(hw_flow_install(hw_int, flow, &extra), + "hw_flow_install modify"); + /* FIXME: Clear stats on flow if updated? */ + count++; + } + } + /* UNLOCK; */ + + return count; +} + +/* + * Update stats + * Call sw tracking table timeout + * Iterate the deleted object list and call HW flow remove + */ +void +of_hw_flow_timeout(struct sw_table *sw_tab, struct list *deleted) +{ + of_hw_driver_int_t *hw_int = (of_hw_driver_int_t *)sw_tab; + struct sw_flow *flow, *n; + int used; + uint64_t now = time_msec(); + + /* LOCK; */ + /* FIXME */ + LIST_FOR_EACH_SAFE (flow, n, struct sw_flow, node, &hw_int->flows) { + if (of_hw_sw_flow_stat_update(flow, &used, false) == 0) { + if (used) { + flow->used = now; + } else { + if (flow_timeout(flow)) { + DBG_VERBOSE("Flow %p expired\n", flow); + list_remove(&flow->node); + list_remove(&flow->iter_node); + list_push_back(deleted, &flow->node); + ASSERT(flow->private != NULL); + TRY_NR(hw_flow_remove(flow), "hw flow remove, timeout"); + hw_int->n_flows--; + } + } + } + } + + /* UNLOCK; */ +} + +struct sw_flow * +of_hw_flow_lookup(struct sw_table *sw_tab, const struct sw_flow_key *key) +{ + of_hw_driver_int_t *hw_int = (of_hw_driver_int_t *)sw_tab; + struct sw_flow *flow; + + LIST_FOR_EACH (flow, struct sw_flow, node, &hw_int->flows) { + if (flow_matches_1wild(key, &flow->key)) + return flow; + } + return NULL; +} + +int +of_hw_flow_stats_get(of_hw_driver_t *hw_drv, struct ofp_match match, + struct ofp_flow_stats **stats, int *count) +{ + DBG_WARN("FIXME FLOW STATS GET\n"); + return 0; +} + +int +of_hw_aggregate_stats_get(struct ofp_match match, + struct ofp_aggregate_stats_reply *stats) +{ + DBG_WARN("FIXME AGGREGATE STATS GET\n"); + return 0; +} + +void +of_hw_flow_remove_all(of_hw_driver_int_t *hw_int) +{ + struct sw_flow *flow, *n; + + /* FIXME: Other de-init? Keep count of DPs? */ + LIST_FOR_EACH_SAFE (flow, n, struct sw_flow, node, &hw_int->flows) { + TRY_NR(hw_flow_remove(flow), "hw_flow_remove all"); + list_remove(&flow->node); + list_remove(&flow->iter_node); + } +} diff --git a/hw-lib/skeleton/hw_flow.h b/hw-lib/skeleton/hw_flow.h new file mode 100644 index 0000000..7c801c6 --- /dev/null +++ b/hw-lib/skeleton/hw_flow.h @@ -0,0 +1,111 @@ +#ifndef SAMPLE_PLATFORM_HW_FLOW_H +#define SAMPLE_PLATFORM_HW_FLOW_H 1 + +/* + * Hardware Flow operations: The glue between software flows and + * actual hardware table entries. + */ + +#include +#include + +#include "of_hw_platform.h" + +/* DEBUG for HW flow lists */ +#define HW_FLOW_MAGIC 0xba5eba11 +#define HW_FLOW_MAKE_VALID(hf) (hf)->magic = HW_FLOW_MAGIC +#define HW_FLOW_IS_VALID(hf) \ + (((hf) != NULL) && ((hf)->magic == HW_FLOW_MAGIC)) + +/* Some flows require multiple HW entries */ +#define HW_FLOWS_PER_FLOW_MAX (2) + +/* The glue between SW flows and HW entries */ +typedef struct hw_flow { + struct sw_flow *flow; /* Corresponding SW flow */ + of_hw_driver_t *hw_drv; /* DP for this flow; MAY BE NULL for internal */ + int entry_count; /* SW flow may require multiple HW table entries */ + entry_control_t *entry_list[HW_FLOWS_PER_FLOW_MAX]; + uint32 hw_packet_count; + uint32 hw_byte_count; + uint32 magic; /* DEBUG */ +} hw_flow_t; + +/* Extra info needed by hardware about flow entry; + * determined during supported check; all in host order + * + * For smac, dmac, vid and priority, if these are changed at ingress + * and a remote rule is necessary, the remote rule must have the + * rewrite values to match. This is indicated if value is not -1. + */ +typedef struct hw_flow_extra_s { + int source_port; /* If single source port, what is it */ + int dest_port; /* If one dest port, what is it */ + /* If multi dest ports, one pbmp per device */ + FIXME dports; + int dest_count; /* Drop (0), unicast (1), multicast (>1) */ + uint32 local_reason; /* Why forwarded to CPU */ + int cosq; /* For enqueue action */ +} hw_flow_extra_t; + +/* Driver functions */ +extern int of_hw_flow_install(struct sw_table *flowtab, struct sw_flow *flow); +extern int of_hw_flow_modify(struct sw_table *flowtab, + const struct sw_flow_key *key, uint16_t priority, int strict, + const struct ofp_action_header *actions, size_t actions_len); +extern int of_hw_flow_delete(struct datapath *dp, struct sw_table *flowtab, + const struct sw_flow_key *key, uint16_t out_port, + uint16_t priority, int strict); +extern void of_hw_flow_timeout(struct sw_table *flowtab, + struct list *deleted); +extern struct sw_flow *of_hw_flow_lookup(struct sw_table *flowtab, + const struct sw_flow_key *key); + + +extern int of_hw_sw_flow_stat_update(struct sw_flow *flow, int *used, + int force_sync); +extern int of_hw_flow_stats_get(of_hw_driver_t *hw_drv, struct ofp_match, + struct ofp_flow_stats **stats, int *count); +extern int of_hw_aggregate_stats_get(struct ofp_match, + struct ofp_aggregate_stats_reply *stats); + +/* Controls a single FP (HW table flow) entry */ +struct entry_control_s { + /* These must be persistent; update fp_entry_remove if you change this */ + int index; /* This entry-s index in field_control list */ + + int in_use; + hw_flow_t *hw_flow; + + /* + * HW specific info + */ + FIXME; + + unsigned long int used_check; /* Last counter; for checking if used */ + unsigned long int last_counter; /* pkts or bytes, see stat_sel above */ + unsigned long int counter_last_check; /* Track entry usage by cntr */ + unsigned long int total_counter; /* FIXME: u64? */ +}; + +/* Driver associated with a HW entry */ +#define HW_ENTRY_DRIVER(ent) ((ent)->hw_flow->hw_drv) + +/* FP control structure */ +struct field_control_s { + /* HW Specific info */ + FIXME; + int entry_count; /* How many entries in use */ + entry_control_t entry_list[FIELD_ENTRY_MAX]; /* Instantiation of entries */ +}; + +/* FIXME: Do we need reference counts on stat objects? */ + +extern int of_hw_fp_setup(void); +extern int of_hw_table_stats_update(of_hw_driver_int_t *hw_drv, + unsigned long *matched, unsigned long *missed); + +extern void of_hw_flow_remove_all(of_hw_driver_int_t *hw_int); + + +#endif /* SAMPLE_PLATFORM_HW_FLOW_H */ diff --git a/hw-lib/skeleton/of_hw_platform.h b/hw-lib/skeleton/of_hw_platform.h new file mode 100644 index 0000000..11a3364 --- /dev/null +++ b/hw-lib/skeleton/of_hw_platform.h @@ -0,0 +1,38 @@ +#ifndef HW_LIB_SKELETON_PLATFORM_H +#define HW_LIB_SKELETON_PLATFORM_H 1 + +#if defined(OF_HW_DP_MAIN) +#define P printf +#else +#define P printf +#endif + +/* + * General Hardware Switch platform defines + * + * Also includes platform specific definitions based on make defines + */ + +#define OF_HW_MAX_PORT 64 + +/* Reasons that a packet is being forwarded to the controller; */ +#define CPU_REASON_DEFAULT (1 << 0) +#define CPU_REASON_TO_CONTROLLER (1 << 1) +#define CPU_REASON_TO_LOCAL (1 << 2) + +#define EXACT_MATCH_PRIORITY TBD + +/**************************************************************** + * + * Platform specific defines and linkage + * + * Implicitly we have a "driver" for the board which includes + * hardware specific information including HW to OF port mapping + * + ****************************************************************/ + +#if defined(OF_SAMPLE_PLAT) +#include "sample_plat.h" +#endif + +#endif /* HW_LIB_SKELETON_PLATFORM_H */ diff --git a/hw-lib/skeleton/os.h b/hw-lib/skeleton/os.h new file mode 100644 index 0000000..268daa9 --- /dev/null +++ b/hw-lib/skeleton/os.h @@ -0,0 +1,29 @@ + +/* + * OS abstractions + */ + +#ifndef OF_HW_OS_H +#define OF_HW_OS_H 1 + +#include + +#define ALLOC(bytes) malloc(bytes) +#define FREE(ptr) free(ptr) + +#define PKT_ALLOC(bytes) TBD +#define PKT_FREE(ptr) TBD + +#define MUTEX_DECLARE(name) TBD +#define MUTEX_INIT(name) TBD +#define MUTEX_LOCK(name) TBD +#define MUTEX_UNLOCK(name) TBD + +#define TIME_NOW(time) TBD +#define TIME_DIFF(diff, early, late) TBD + +/* FIXME */ +#include +#define os_pkt_free(pkt) ofpbuf_delete(pkt) + +#endif /* OF_HW_OS_H */ diff --git a/hw-lib/skeleton/port.c b/hw-lib/skeleton/port.c new file mode 100644 index 0000000..7d8c5e3 --- /dev/null +++ b/hw-lib/skeleton/port.c @@ -0,0 +1,497 @@ +/* + * Port related HW datapath functions + */ + +#include +#include + +#include "xtoxll.h" +#include "hw_drv.h" +#include "of_hw_platform.h" +#include "port.h" +#include "debug.h" + +/* Single set of HW port objects managed here; indexed by OF port num */ +of_hw_port_t of_hw_ports[OF_HW_MAX_PORT]; + +/* Check if hw DP and port are valid and if port belongs to the DP */ +#define CHECK_PORT(_drv, _p) do { \ + if ((_drv) == NULL) return -1; \ + if (((_p) < 1) || ((_p) > OF_HW_MAX_PORT)) { \ + REPORT_ERROR("bad port number"); \ + return -1; \ + } \ + if (!(OF_PORT_IN_DP(((of_hw_driver_int_t *)(_drv)), (_p)))) { \ + REPORT_ERROR("DP does not own port"); \ + return -1; \ + } \ +} while (0) + + +/* + * Queue configuration section + * + * Port queue: Per port structure for queue related information + * Queue map: Indicated queue is mapped and what OF qid its mapped to as + * well as queue properties (min-bw). Indexed by OF port number; + * per port instance of array + * + */ + +typedef struct queue_map_s { + int in_use; + uint32_t qid; /* OF queue name */ + int min_bw; /* OF value, tenths of a percent */ +} queue_map_t; + +typedef struct port_queue_s { + int speed; /* In Mbps */ + queue_map_t queue_map[NUM_COS_QUEUES]; +} port_queue_t; + +static port_queue_t port_queue[OF_HW_MAX_PORT]; + +/* Return the cos for the queue matching qid if found; else -1. + * Assumes lock held + */ +static int +qid_find(int of_port, uint32_t qid) +{ + struct queue_map_s *qm; + int cosq; + + qm = port_queue[of_port].queue_map; + for (cosq = 0; cosq < NUM_COS_QUEUES; cosq++) { + if (qm[cosq].in_use && (qid == qm[cosq].qid)) { + return cosq; + } + } + + return -1; +} + +/* Return the cos for the queue matching qid if found; else -1. + * Assumes lock held + */ +int +of_hw_qid_find(int of_port, uint32_t qid) +{ + int cosq; + + /* LOCK */ + cosq = qid_find(of_port, qid); + /* UNLOCK */ + + return cosq; +} + +/* + * Map an OF qid to a COS queue index; if not present, add it if possible + * Assumes lock held + */ +static int +qid_find_add(int of_port, uint32_t qid) +{ + struct queue_map_s *qm; + int cosq; + + cosq = qid_find(of_port, qid); + if (cosq >= 0) { + return cosq; + } + + /* Not found; add queue */ + for (cosq = 0; cosq < NUM_COS_QUEUES; cosq++) { + qm = port_queue[of_port].queue_map; + if (!qm[cosq].in_use) { + qm[cosq].in_use = true; + qm[cosq].qid = qid; + return cosq; + } + } + + return -1; +} + +/* + * Port speed has changed; requires queue b/w to change + */ +static void +port_speed_change_reconfig(of_hw_driver_int_t *hw_int, int of_port) +{ + int cosq; + struct queue_map_s *qm; + int rv; + + /* LOCK */ + for (cosq = 0; cosq < NUM_COS_QUEUES; cosq++) { + qm = port_queue[of_port].queue_map; + if (qm[cosq].in_use) { + /* FIXME: Calc and program values for HW for min-bw */ + if (rv < 0) { + DBG_ERROR("ERROR: Set min BW for queue on port %d " + "link change\n", of_port); + } + } + } + /* UNLOCK */ +} + + +/* Internal handler for port link status changes */ + +/* FIXME */ +static void +of_hw_linkscan_handler(...) +{ + int of_port; + of_hw_driver_int_t *hw_int; + + /* Assumes speed, linkstatus from HW */ + /* Map port to internal port object */ + of_port = HW_MAP_TO_PORT(); + if ((of_port < 0) || (of_port > OF_HW_MAX_PORT)) { + return; + } + + /* LOCK */ + hw_int = (of_hw_driver_int_t *)(of_hw_ports[of_port].owner); + if (hw_int != NULL) { + if (port_queue[of_port].speed != speed) { + port_queue[of_port].speed = speed; + port_speed_change_reconfig(hw_int, of_port); + } + } + /* Record in local struct */ + of_hw_ports[of_port].link = linkstatus != 0; + if ((hw_int != NULL) && (hw_int->port_change != NULL)) { + hw_int->port_change(of_port, linkstatus != 0, + hw_int->port_change_cookie); + } + /* UNLOCK */ +} + +static int linkscan_registered; + +/* + * of_hw_ports_init + * Set up hardware port map + */ +void +of_hw_ports_init(void) +{ + int i; + + for (i = 0; i < OF_HW_MAX_PORT; i++) { + of_hw_ports[i].owner = NULL; + of_hw_ports[i].port = MAP_OF_PORT_TO_HW_PORT(i); + } + + /* FIXME: REGISTER FOR LINK CHANGE CALLBACK */ + /* Always register linkscan handler */ + linkscan_registered = 1; + + /* Get current link status for each port; ignore errors for bad ports */ + for (i = 0; i < OF_HW_MAX_PORT; i++) { + /* FIXME */ + HW_LINK_STATUS_GET(of_hw_ports[i].port, &of_hw_ports[i].link); + } +} + + +#define HTON64 htonll + +/* Get port stats into a standard openflow port stats structure */ +int +of_hw_port_stats_get(of_hw_driver_t *hw_drv, int of_port, + struct ofp_port_stats *stats) +{ + uint64 v1, v2; + + CHECK_PORT(hw_drv, of_port); + + +#if 0 /* FIXME; Get stats from HW */ + HW_STAT(of_port, snmpIfInNUcastPkts, &v1); + HW_STAT(of_port, snmpIfInUcastPkts, &v2); + v1 += v2; + stats->rx_packets = HTON64(v1); + HW_STAT(of_port, snmpIfOutNUcastPkts, &v1); + HW_STAT(of_port, snmpIfOutUcastPkts, &v2); + v1 += v2; + stats->tx_packets = HTON64(v1); + HW_STAT(of_port, snmpIfInOctets, &v1); + stats->rx_bytes = HTON64(v1); + HW_STAT(of_port, snmpIfOutOctets, &v1); + stats->tx_bytes = HTON64(v1); + HW_STAT(of_port, snmpIfInDiscards, &v1); + stats->rx_dropped = HTON64(v1); + HW_STAT(of_port, snmpIfOutDiscards, &v1); + stats->tx_dropped = HTON64(v1); + HW_STAT(of_port, snmpIfInErrors, &v1); + stats->rx_errors = HTON64(v1); + HW_STAT(of_port, snmpIfOutErrors, &v1); + stats->tx_errors = HTON64(v1); +#endif + + v1 = UINT64_C(0xffffffffffffffff); + stats->rx_frame_err = HTON64(v1); + stats->rx_over_err = HTON64(v1); + stats->rx_crc_err = HTON64(v1); + stats->collisions = HTON64(v1); + + return 0; +} + +/* + * port_add/remove(table, port) + * + * The indicated port has been added to/removed from the datapath + * Add also maps the of_port number to the hw_port indicated + * + * SPEC CHANGE: If of_port passed is less than 0, use the passed + * port name to determine the port number and attach to that value; + * + * Returns the of_port number or -1 on error + */ +int +of_hw_port_add(of_hw_driver_t *hw_drv, int of_port, const char *hw_name) +{ + int hw_idx; + of_hw_port_t *port_ctl; + of_hw_driver_int_t *hw_int; + + if (hw_drv == NULL) { + return -1; + } + + hw_idx = hw_port_name_to_index(hw_name); + if (of_port < 0) { + of_port = hw_idx + 1; + } else if (hw_idx + 1 != of_port) { + DBG_WARN("Add Port: OF port %d does not match name %s\n", + of_port, hw_name); + } + + if (of_port >= OF_HW_MAX_PORT) { + DBG_ERROR("Add Port: Bad port number %d\n", of_port); + return -1; + } + + hw_int = (of_hw_driver_int_t *)hw_drv; + port_ctl = &of_hw_ports[of_port]; + + HW_DRV_LOCK; + if (port_ctl->owner == NULL) { + port_ctl->owner = hw_drv; + /* FIXME: port bitmaps in HW structure */ + } else if (port_ctl->owner != hw_drv) { + DBG_ERROR("Add Port: OF port %d owned by other DP\n", of_port); + } else { + DBG_WARN("Add Port: OF port %d already added\n", of_port); + } + HW_DRV_UNLOCK; + + return of_port; +} + +int +of_hw_port_remove(of_hw_driver_t *hw_drv, of_port_t of_port) +{ + of_hw_port_t *port_ctl; + of_hw_driver_int_t *hw_int; + int rc = 0; + + if (hw_drv == NULL) { + return -1; + } + + if ((of_port < 1) || (of_port >= OF_HW_MAX_PORT)) { + DBG_ERROR("Remove Port: Bad port number %d\n", of_port); + return -1; + } + + hw_int = (of_hw_driver_int_t *)hw_drv; + port_ctl = &of_hw_ports[of_port]; + + HW_DRV_LOCK; + if (port_ctl->owner == NULL) { + DBG_WARN("Remove Port: OF port %d already free\n", of_port); + } else if (port_ctl->owner != hw_drv) { + DBG_ERROR("Remove Port: OF port %d owned by other DP\n", of_port); + rc = -1; + } else { + port_ctl->owner = NULL; + } + /* FIXME UPDATE HW port bitmaps in hw_int */ + HW_DRV_UNLOCK; + + return rc; +} + + +/* + * port_link_get(table, port) + * port_enable_set(table, port, enable) + * port_enable_get(table, port) + * + * Get/set the indicated properties of a port. Only real ports + * set with port_add are supported. + */ +int +of_hw_port_link_get(of_hw_driver_t *hw_drv, int of_port) +{ + int rc; + int link; + + CHECK_PORT(hw_drv, of_port); + + if (linkscan_registered) { + return of_hw_ports[of_port].link; + } + + if ((rc = HW_LINK_GET(of_hw_ports[of_port].port, &link)) < 0) { + DBG_ERROR("link_get: error %d port %d\n", rc, of_port); + /* Return link down on error */ + return 0; + } + + return link ? 1 : 0; +} + +int +of_hw_port_enable_set(of_hw_driver_t *hw_drv, int of_port, int enable) +{ + int rc; + + CHECK_PORT(hw_drv, of_port); + + if ((rc = HW_PORT_ENABLE_SET(of_hw_ports[of_port].port, enable)) < 0) { + DBG_ERROR("of_hw_port_enable_set: error %d port %d\n", + rc, of_port); + return rc; + } + + return 0; +} + +int +of_hw_port_enable_get(of_hw_driver_t *hw_drv, int of_port) +{ + int rc; + int enable; + + CHECK_PORT(hw_drv, of_port); + + if ((rc = HW_PORT_ENABLE_GET(of_hw_ports[of_port].port, &enable)) < 0) { + DBG_ERROR("of_hw_port_enable_get: error %d port %d\n", + rc, of_port); + return rc; + } + + return enable ? 1 : 0; +} + + +/* + * port_change_register + * + * Register a callback function to receive port change notifications + * from ports in this datapath; only one callback per datapath is + * supported. + */ +int +of_hw_port_change_register(of_hw_driver_t *hw_drv, of_port_change_f callback, + void *cookie) +{ + of_hw_driver_int_t *dp_int; + + dp_int = (of_hw_driver_int_t *)hw_drv; + dp_int->port_change = callback; + dp_int->port_change_cookie = cookie; + + return 0; +} + +/* + * Init COS queue setup. The queues number is fixed at 8. Deficit + * round robin is the discipline, initially with all equal weights. + * As queues are configured with min bandwidth reservations, the + * weights are adjusted to ensure the requested targets. + * + * For now, these are device generic; may need to specialize in the + * future. + */ + +int +of_hw_cos_setup(int num_cos) +{ + /* FIXME: Set up COS queues */ + + return 0; +} + +/* + * Add and/or configure an output queue on a port. + * + * If qid exists, update. If not, look for an unreferenced queue and + * set the qid to that value. + * + * Note that when a port state changes, may need to re-configure + * the bandwidth values for the port's queues. + */ + +int +of_hw_port_queue_config(of_hw_driver_t *hw_drv, int of_port, uint32_t qid, + int min_bw) /* In tenths of a percent */ +{ + int min_kbps; + of_hw_driver_int_t *hw_int; + int cosq; + struct queue_map_s *qm; + int rv; + + hw_int = (of_hw_driver_int_t *)hw_drv; + + CHECK_PORT(hw_int, of_port); + + /* LOCK */ + /* Get the current bandwidth of the port (cached?) */ + if ((cosq = qid_find_add(of_port, qid)) < 0) { + DBG_ERROR("Could not add queue: of port %d, qid %d\n", of_port, qid); + /* UNLOCK */ + return -1; + } + qm = &port_queue[of_port].queue_map[qid]; + qm->min_bw = min_bw; + + HW_SET_MIN_BW(...); + /* UNLOCK */ + + return (rv != 0) ? -1 : 0; +} + +/* Remove a queue from a port; this potentially affects the queue + * configuration otherwise we would not worry about it here. + * Return -1 if not found; 0 on success + */ + +int +of_hw_port_queue_remove(of_hw_driver_t *hw_drv, int of_port, uint32_t qid) +{ + int cosq; + + (void)hw_drv; + /* LOCK */ + if ((cosq = qid_find(of_port, qid)) == -1) { + /* UNLOCK */ + return -1; + } + + /* Do we need to update stats or DRR weights or anything? */ + + port_queue[of_port].queue_map[cosq].in_use = false; + + /* UNLOCK */ + return 0; +} diff --git a/hw-lib/skeleton/port.h b/hw-lib/skeleton/port.h new file mode 100644 index 0000000..dd2c082 --- /dev/null +++ b/hw-lib/skeleton/port.h @@ -0,0 +1,45 @@ +#ifndef OF_HW_PORT_H +#define OF_HW_PORT_H 1 + +#include +#include +#include "of_hw_platform.h" + +#define NUM_COS_QUEUES 8 + +/* Define port mapping object indexed by OF number */ +typedef struct of_hw_port_s { + of_hw_driver_t *owner; /* Owner of this port */ + int port; /* Physical port number on device */ + int link; /* Link state */ + /* TBD: Keep track of link state and don't transmit if down? */ + /* Add other physical info here as needed */ +} of_hw_port_t; + +extern of_hw_port_t of_hw_ports[OF_HW_MAX_PORT]; + +#define FOREACH_DP_PORT(_idx, _drv) \ + for (_idx = 0; _idx < OF_HW_MAX_PORT; _idx++) \ + if (of_hw_ports[_idx].owner == (of_hw_driver_t *)(_drv)) + +extern int of_hw_cos_setup(int num_cos); +extern int of_hw_qid_find(int of_port, uint32_t qid); + +extern void of_hw_ports_init(void); +extern int of_hw_port_stats_get(of_hw_driver_t *hw_drv, int of_port, + struct ofp_port_stats *stats); +extern int of_hw_port_add(of_hw_driver_t *hw_drv, int of_port, + const char *hw_name); +extern int of_hw_port_remove(of_hw_driver_t *hw_drv, of_port_t of_port); +extern int of_hw_port_link_get(of_hw_driver_t *hw_drv, int of_port); +extern int of_hw_port_enable_set(of_hw_driver_t *hw_drv, int of_port, + int enable); +extern int of_hw_port_enable_get(of_hw_driver_t *hw_drv, int of_port); +extern int of_hw_port_queue_config(of_hw_driver_t *hw_drv, int of_port, + uint32_t qid, int min_bw); /* In tenths of a percent */ +extern int of_hw_port_queue_remove(of_hw_driver_t *hw_drv, int of_port, + uint32_t qid); +extern int of_hw_port_change_register(of_hw_driver_t *hw_drv, + of_port_change_f callback, void *cookie); + +#endif /* OF_HW_PORT_H */ diff --git a/hw-lib/skeleton/sample_plat.c b/hw-lib/skeleton/sample_plat.c new file mode 100644 index 0000000..17f3747 --- /dev/null +++ b/hw-lib/skeleton/sample_plat.c @@ -0,0 +1,82 @@ +/* + * sample_plat.c + * + * FIXME: Standard confidential header + * + * $Id: $ + */ + +/* + * Sample hardware initialization functions + */ + +#include "debug.h" +#include "of_hw_platform.h" + +#if defined(SAMPLE_PLAT) + +#include "port.h" + +/* sample_plat_port_setup + * + * Set up linkscan and necessary spanning tree for ports + * Disable VLAN dropping + */ +static int +sample_plat_port_setup(...) +{ + + return 0; +} + +#if defined(OF_HW_DP_MAIN) && defined(SAMPLE_HW_PLAT) + +/* If defined, need to do init here */ + +/* + * sample_plat_pre_init + * + * Turn off everything that might cause problems during init + */ + +static int +sample_plat_pre_init(void) +{ + /* bring system down to known state */ + return 0; +} + +/* + */ +int +sample_plat_init(void) +{ + /* Init system */ + + /* First, clear out everything */ + sample_plat_pre_init(); + + /* ... */ + + sample_plat_port_setup(...); + + return 0; +} + +/* FIXME: Deal with initial FP setup */ + +#else /* Not stand alone init; Other code does init */ + +int +sample_plat_init(void) +{ + DBG_WARN("sample_plat_init\n"); + + TRY(sample_plat_port_setup(...), "sample plat init port setup"); + + return 0; +} + +#endif + +#endif /* SAMPLE_PLAT */ diff --git a/hw-lib/skeleton/sample_plat.h b/hw-lib/skeleton/sample_plat.h new file mode 100644 index 0000000..3d88eb0 --- /dev/null +++ b/hw-lib/skeleton/sample_plat.h @@ -0,0 +1,61 @@ +#ifndef SAMPLE_PLAT_H +#define SAMPLE_PLAT_H 1 + + +#include +#include +#include + +#include + +#define OF_HW_MAX_PORTS 4 + +static inline int +of_port_to_hw_port(int of_port) +{ + return of_port - 1; +} + + +/* Map port to OF port number */ +static inline int +of_hw_port_to_of_port(int port) +{ + if ((port < 0) || (port > 3)) { + return -1; + } + + return port + 1; +} + +#define _IS_DIGIT(c) ((c) >= '0' && (c) <= '9') + +/* Map name to hw port number: N => N-1 + ge0 => 0 where "ge" can be any string and 0 can be any number +*/ + +static inline int +hw_port_name_to_index(const char *name) +{ + + if (_IS_DIGIT(name[0])) { + /* Treat as 1-based, OF number */ + i = strtoul(&name[0], NULL, 10); + return i - 1; + } + + len = strlen(name); + for (idx = 0; idx < len && !_IS_DIGIT(name[idx]); idx++) ; + + if ((idx < len) && _IS_DIGIT(name[idx])) { + i = strtoul(&name[idx], NULL, 10); + return i; + } + + return -1; +} +#undef _IS_DIGIT + + + +#endif /* SAMPLE_PLAT_H */ diff --git a/hw-lib/skeleton/txrx.c b/hw-lib/skeleton/txrx.c new file mode 100644 index 0000000..950d839 --- /dev/null +++ b/hw-lib/skeleton/txrx.c @@ -0,0 +1,129 @@ +/* + * Transmit and receive related functions for hardware platforms + */ + +#include +#include "os.h" +#include "hw_drv.h" +#include "txrx.h" +#include "port.h" +#include "debug.h" +#include "of_hw_platform.h" + +static int pkt_count, failures, prev_success, free_count, error_free; +static void +tx_pkt_callback(...) +{ + of_packet_t *of_pkt = cookie; + + /* os_pkt_free(of_pkt->os_pkt); */ + FREE(of_pkt); + /* ... */ + ++free_count; +} + +/* + * tx_packet_send(table, of_port, pkt, flags) + * + * Send packet to an openflow port. + * + * Proposed flags: + * APPLY_FLOW_TABLE: If set, and if the hardware supports + * it, send the packet through the flow table with the source + * port being the local CPU port. (Would be nice to have + * a flexible source port indicated; could hide in flags...) + * + * Assumes buffer in pkt struct can be used for sending data + * + * + */ +int +of_hw_packet_send(of_hw_driver_t *hw_drv, int of_port, of_packet_t *pkt, + uint32_t flags) +{ + /* FIXME: Code to prepare and send pkt */ + return 0; +} + +/* Callback function registered with HW */ +static int +of_hw_rx(...) +{ + /* Sample RX handler */ + int of_port; + of_hw_driver_int_t *hw_drv; + of_packet_t *of_pkt; + int pkt_len; + + /* map receive port to of_port */ + /* map received packet to of_pkt */ + /* Call callback */ + + of_port = of_hw_port_to_of_port(receive_port); + if (of_port < 0) { + return hw_not_handled; + } + + /* LOCK */ + hw_drv = (of_hw_driver_int_t *)(of_hw_ports[of_port].owner); + if ((hw_drv == NULL) || (hw_drv->rx_handler == NULL)) { + /* UNLOCK */ + return hw_not_handled; + } + + of_pkt = ALLOC(sizeof(of_packet_t)); + if (of_pkt == NULL) { + ++hw_drv->rx_pkt_alloc_failures; + /* UNLOCK */ + return hw_not_handled; + } + pkt_len = pkt->tot_len; + + /* FIXME: FOR NOW, COPY DATA INTO NEW BUFFER; */ + of_pkt->data = ALLOC(pkt_len); + if (of_pkt->data == NULL) { + FREE(of_pkt); + /* UNLOCK */ + return hw_not_handled; + } + + /* Handle VLAN tagging if needed */ + /* Coy pkt data appropriately */ + + /* FIXME: Determine reason (dflt rule or directed) */ + /* FIXME: Return code interpretation? */ + /* OFPR_NO_MATCH, No matching flow. */ + /* OFPR_ACTION Action explicitly output to controller. */ + hw_drv->rx_handler(of_port, of_pkt, 0, hw_drv->rx_cookie); + /* UNLOCK */ + + return hw_handled; +} + +static int rx_registered = 0; + +/* + * packet_receive_register + * + * Register a callback function to receive packets from ports in + * this datapath + */ +int +of_hw_packet_receive_register(of_hw_driver_t *hw_drv, + of_packet_in_f callback, void *cookie) +{ + of_hw_driver_int_t *dp_int; + int rv; + + /* Register for link status changes */ + if (!rx_registered) { + /* Set up low level pkt receive for callback */ + rx_registered = 1; + } + + dp_int = (of_hw_driver_int_t *)hw_drv; + dp_int->rx_handler = callback; + dp_int->rx_cookie = cookie; + + return 0; +} diff --git a/hw-lib/skeleton/txrx.h b/hw-lib/skeleton/txrx.h new file mode 100644 index 0000000..1aef902 --- /dev/null +++ b/hw-lib/skeleton/txrx.h @@ -0,0 +1,12 @@ +#ifndef OF_HW_TXRX_H +#define OF_HW_TXRX_H 1 + +#include +#include + +extern int of_hw_packet_send(of_hw_driver_t *hw_drv, int of_port, + of_packet_t *pkt, uint32_t flags); +extern int of_hw_packet_receive_register(of_hw_driver_t *hw_drv, + of_packet_in_f callback, void *cookie); + +#endif /* OF_HW_TXRX_H */ diff --git a/include/openflow/of_hw_api.h b/include/openflow/of_hw_api.h new file mode 100644 index 0000000..2dca190 --- /dev/null +++ b/include/openflow/of_hw_api.h @@ -0,0 +1,313 @@ +/* Copyright (c) 2008, 2009, 2010 The Board of Trustees of The Leland Stanford + * Junior University + * + * We are making the OpenFlow specification and associated documentation + * (Software) available for public use and benefit with the expectation + * that others will use, modify and enhance the Software and contribute + * those enhancements back to the community. However, since we would + * like to make the Software available for broadest use, with as few + * restrictions as possible permission is hereby granted, free of + * charge, to any person obtaining a copy of this Software to deal in + * the Software under the copyrights without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * The name and trademarks of copyright holder(s) may NOT be used in + * advertising or publicity pertaining to the Software or any + * derivatives without specific, written prior permission. + */ + +#if !defined(OF_HW_API_H) +#define OF_HW_API_H + +/* + * OpenFlow hardware API definition + * + * This header file provides an abstraction of the flow table and + * port operations that can be used to build a driver for hardware + * that implements the OpenFlow protocol. + * + * Currently this driver depends (extends) the sw_table defined + * in the udatapath/table.h file. Hopefully that file will be + * moved up to library status to support kernel and userspace + * implementations. + * + */ + +#include +#include /* For sw_table */ + +/* REQUIRES: + * struct sw_table defined + * TBD: We could remove this restriction; it's mainly so that + * current chain.c operations can work. It also allows + * pointer coersion between the two types. + * + * Eventually, sw_table may be extended to include everything in + * this driver. + * + * pointer to struct datapath + */ + +/**************** basic types ****************/ + +typedef uint32_t of_port_t; +typedef struct of_hw_driver of_hw_driver_t; + +/**************** packet ****************/ + +/* The OpenFlow hardware packet abstraction */ +typedef void *os_pkt_t; /* OS representation of packet */ + +/* Requires monolithic packet data */ +typedef struct of_packet_s { + unsigned char *data; /* Pointer to packet data */ + int length; /* Length in bytes */ + os_pkt_t os_pkt; /* OS specific representation */ +} of_packet_t; + +/* Init an of_packet struct from an ofp_buffer struct */ +#define OF_PKT_INIT(pkt, ofp_buf) do { \ + (pkt)->data = (ofp_buf)->data; \ + (pkt)->length = (ofp_buf)->size; \ + (pkt)->os_pkt = (ofp_buf); \ + } while (0) + +/**************** callback protos ****************/ + +/* packet in callback function prototype */ +typedef int (*of_packet_in_f)(of_port_t port, + of_packet_t *packet, + int reason, + void *cookie); + +typedef void (*of_port_change_f)(of_port_t port, + int state, + void *cookie); + +/**************************************************************** + * + * Hardware Driver + * + ****************************************************************/ + +/* Hardware capabilities structure */ +typedef struct of_hw_driver_caps { + /* Proposed Flags: + * COUNT_PKTS_OR_BYTES Can count either pkts or bytes, not both + * INTERNAL_PRI Support internal priority mapping, and thus + * normal enqueuing action + * LOCAL_CPU_THRU_TABLE Can send packets from the CPU through + * the flow table + */ + uint32_t flags; + + /* Number of fully qualified flows supported (approx) */ + int max_flows; + uint32_t wc_supported; /* Bitmap of OFPFW_* supported wildcards */ + uint32_t actions_supported; /* Bitmap of OFPAT_* supported actions */ + uint32_t ofpc_flags; /* Bitmap of ofp_capabilities flags */ +} of_hw_driver_caps_t; + +enum of_hw_driver_flags { + OF_HW_DRV_COUNT_PKTS_OR_BYTES = 1 << 0, + OF_HW_DRV_INTERNAL_PRI = 1 << 1, + OF_HW_DRV_CPU_PKTS_THRU_TABLE = 1 << 2 +}; + +/**************** Constructor/Destructor ****************/ + +extern of_hw_driver_t *new_of_hw_driver(struct datapath *dp); +extern void delete_of_hw_driver(of_hw_driver_t *hw_drv); + +/* TBD: Add a HW/DP init function? */ + +/**************** HW DataPath Driver Structure ****************/ +/* Extends sw_table */ +struct of_hw_driver { + + /* + * Notes on sw_table inheritance: + * + * See above as well + * + * n_lookup and n_matched are not dynamically updated, but the + * call to table_stats_update should set them + */ + struct sw_table sw_table; + + /* HW datapath capabilities structure */ + of_hw_driver_caps_t caps; + + /* OPTIONAL + * init(table, flags) + * + * Initialize necessary hardware and software to run + * the switching table. Must be called prior to any other calls + * into the table (except maybe some ioctls?). + * + * Proposed flags include: + * BYTES/PACKETS: If COUNT_PKTS_OR_BYTES, which to count by default + * REATTACH: Inidicates HW was running, don't re-initialize HW + * + */ + int (*init)(of_hw_driver_t *hw_drv, uint32_t flags); + + /* + * table_stats_get(table, stats) + * port_stats_get(port, stats) + * flow_stats_get(flow_desc, stats) + * aggregate_stats_get(flow_desc, stats) + * + * Fill out the stats object(s) for this table/port/flow(s)/set of flows + * + * Returns 0 on success. + * + * For all but flow_stats, the routine fills out a pre-allocated + * stats structure. For flow stats, an array of stats is allocated + * by the called routine with *count elements. It must be freed by + * the caller. + * + * (Optional? If count is NULL for flow_stats_get, find a single + * match with exactly the given ofp_match.) + */ + int (*table_stats_get)(of_hw_driver_t *hw_drv, struct + ofp_table_stats *stats); + int (*port_stats_get)(of_hw_driver_t *hw_drv, int of_port, + struct ofp_port_stats *stats); + int (*flow_stats_get)(of_hw_driver_t *hw_drv, struct ofp_match, + struct ofp_flow_stats **stats, int *count); + int (*aggregate_stats_get)(struct ofp_match, + struct ofp_aggregate_stats_reply *stats); + + /* + * port_add/remove(table, port) + * + * The indicated port has been added to/removed from the datapath + * Add also maps the of_port number to the hw_port indicated + */ + int (*port_add)(of_hw_driver_t *hw_drv, int of_port, const char *hw_name); + int (*port_remove)(of_hw_driver_t *hw_drv, of_port_t port); + + /* + * port_link_get(table, port) + * port_enable_set(table, port, enable) + * port_enable_get(table, port) + * + * Get/set the indicated properties of a port. Only real ports + * set with port_add are supported. + */ + int (*port_link_get)(of_hw_driver_t *hw_drv, int of_port); + int (*port_enable_set)(of_hw_driver_t *hw_drv, int of_port, int enable); + int (*port_enable_get)(of_hw_driver_t *hw_drv, int of_port); + + /* + * port_queue_config(drv, port, qid, min-bw) + * port_queue_remove(drv, port, qid) + * + * Port queue control. Config will add the queue if not present + */ + int (*port_queue_config)(of_hw_driver_t *hw_drv, int of_port, + uint32_t qid, int min_bw); + int (*port_queue_remove)(of_hw_driver_t *hw_drv, int of_port, + uint32_t qid); + + /* + * port_change_register + * + * Register a callback function to receive port change notifications + * from ports in this datapath + */ + int (*port_change_register)(of_hw_driver_t *hw_drv, + of_port_change_f callback, void *cookie); + + /* + * packet_send(table, of_port, pkt, flags) + * + * Send packet to an openflow port. + * + * Proposed flags: + * APPLY_FLOW_TABLE: If set, and if the hardware supports + * it, send the packet through the flow table with the source + * port being the local CPU port. (Would be nice to have + * a flexible source port indicated; could hide in flags...) + * + * TBD: Owner of pkt and pkt->data after call; sync/async. + */ + int (*packet_send)(of_hw_driver_t *hw_drv, int of_port, of_packet_t *pkt, + uint32_t flags); + + /* + * packet_receive_register + * + * Register a callback function to receive packets from ports in + * this datapath + * + * TBD: Semantics for owning packets and return codes so indicating. + */ + int (*packet_receive_register)(of_hw_driver_t *hw_drv, + of_packet_in_f callback, void *cookie); + + /* OPTIONAL + * ioctl(table, request, io_param) + * + * Execute an ioctl on the table. A few ioctls are predefined, + * but most will be implementation specific. + * Returns 0 on success or an implementation specific other code. + * + * io_param is an input/output parameter whose value may be + * returned to the caller. + * io_len is the length of io_param in bytes. + * On input, the *io_param pointer may clobbered, so the caller must + * maintain it for deallocation if necessary. + * On output, when used -- which depends on the operation -- + * the *io_param is a pointer to a buffer allocated by the ioctl + * routine, but owned by the calling routine. + * + * Question: Should a full I/O buffer be supported? + * ioctl(table, op, in_buf, in_len, out_buf, out_len); or + * ioctl(table, op, io_buf, io_len); where buf/len set on output. + * + * Proposed operations: + * Set debug level + * Clear port/flow/table stats + * Select packet or byte counter collection + */ + int (*ioctl)(of_hw_driver_t *hw_drv, uint32_t op, void **io_param, + int *io_len); + +}; + +/**************** IOCTL values ****************/ + +enum of_hw_ioctl_e { + OF_HW_IOCTL_TABLE_DEBUG_SET = 1, + OF_HW_IOCTL_PORT_DEBUG_SET = 2, + OF_HW_IOCTL_BYTE_PKT_CNTR_SET = 3 +}; + +/* Values for OF_HW_IOCTL_BYTE_PKT_CNTR_SET */ +#define OF_HW_CNTR_PACKETS 0 +#define OF_HW_CNTR_BYTES 1 + +enum of_hw_error_e { + OF_HW_OKAY = 0, + OF_HW_ERROR = -1, + OF_HW_PORT_DOWN = -2 +}; + +#endif /* OF_HW_API_H */ diff --git a/udatapath/automake.mk b/udatapath/automake.mk index b485e90..b4b4d8e 100644 --- a/udatapath/automake.mk +++ b/udatapath/automake.mk @@ -1,3 +1,7 @@ +# +# Build udatapath as binary +# + bin_PROGRAMS += udatapath/ofdatapath man_MANS += udatapath/ofdatapath.8 @@ -22,6 +26,50 @@ udatapath_ofdatapath_SOURCES = \ udatapath/table-linear.c udatapath_ofdatapath_LDADD = lib/libopenflow.a $(SSL_LIBS) $(FAULT_LIBS) +udatapath_ofdatapath_CPPFLAGS = $(AM_CPPFLAGS) EXTRA_DIST += udatapath/ofdatapath.8.in DISTCLEANFILES += udatapath/ofdatapath.8 + +if BUILD_HW_LIBS + +# Options for each platform +if NF2 +udatapath_ofdatapath_LDADD += hw-lib/libnf2.a +udatapath_ofdatapath_CPPFLAGS += -DOF_HW_PLAT -DUSE_NETDEV -g +noinst_LIBRARIES += hw-lib/libnf2.a +endif + +endif + +if BUILD_HW_LIBS +# +# Build udatapath as a library +# + +noinst_LIBRARIES += udatapath/libudatapath.a + +udatapath_libudatapath_a_SOURCES = \ + udatapath/chain.c \ + udatapath/chain.h \ + udatapath/crc32.c \ + udatapath/crc32.h \ + udatapath/datapath.c \ + udatapath/datapath.h \ + udatapath/dp_act.c \ + udatapath/dp_act.h \ + udatapath/of_ext_msg.c \ + udatapath/of_ext_msg.h \ + udatapath/udatapath.c \ + udatapath/private-msg.c \ + udatapath/private-msg.h \ + udatapath/switch-flow.c \ + udatapath/switch-flow.h \ + udatapath/table.h \ + udatapath/table-hash.c \ + udatapath/table-linear.c + +udatapath_libudatapath_a_CPPFLAGS = $(AM_CPPFLAGS) +udatapath_libudatapath_a_CPPFLAGS += -DOF_HW_PLAT -DUDATAPATH_AS_LIB -g + +endif diff --git a/udatapath/chain.c b/udatapath/chain.c index a5db92f..b9dc39c 100644 --- a/udatapath/chain.c +++ b/udatapath/chain.c @@ -38,6 +38,11 @@ #include #include "switch-flow.h" #include "table.h" +#include "datapath.h" + +#if defined(OF_HW_PLAT) +#include +#endif #define THIS_MODULE VLM_chain #include "vlog.h" @@ -70,6 +75,13 @@ struct sw_chain *chain_create(struct datapath *dp) return NULL; chain->dp = dp; +#if defined(OF_HW_PLAT) + if (dp && dp->hw_drv) { + if (add_table(chain, (struct sw_table *)dp->hw_drv, 0) != 0) { + VLOG_ERR("Could not attach HW table to chain\n"); + } + } +#endif if (add_table(chain, table_hash2_create(0x1EDC6F41, TABLE_HASH_MAX_FLOWS, 0x741B8CD7, TABLE_HASH_MAX_FLOWS), 0) diff --git a/udatapath/datapath.c b/udatapath/datapath.c index ddc2ac2..5254b77 100644 --- a/udatapath/datapath.c +++ b/udatapath/datapath.c @@ -62,6 +62,73 @@ #define THIS_MODULE VLM_datapath #include "vlog.h" +#if defined(OF_HW_PLAT) +#include +#include +#endif + +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) +/* Queue to decouple receive packet thread from rconn control thread */ +/* Could make mutex per-DP */ +static pthread_mutex_t pkt_q_mutex = PTHREAD_MUTEX_INITIALIZER; +#define PKT_Q_LOCK pthread_mutex_lock(&pkt_q_mutex) +#define PKT_Q_UNLOCK pthread_mutex_unlock(&pkt_q_mutex) + +static void +enqueue_pkt(struct datapath *dp, struct ofpbuf *buffer, of_port_t port_no, + int reason) +{ + struct hw_pkt_q_entry *q_entry; + + if ((q_entry = xmalloc(sizeof(*q_entry))) == NULL) { + VLOG_WARN("Could not alloc q entry\n"); + /* FIXME: Dealloc buffer */ + return; + } + q_entry->buffer = buffer; + q_entry->next = NULL; + q_entry->port_no = port_no; + q_entry->reason = reason; + pthread_mutex_lock(&pkt_q_mutex); + if (dp->hw_pkt_list_head == NULL) { + dp->hw_pkt_list_head = q_entry; + } else { + dp->hw_pkt_list_tail->next = q_entry; + } + dp->hw_pkt_list_tail = q_entry; + pthread_mutex_unlock(&pkt_q_mutex); +} + +/* If queue non-empty, fill out params and return 1; else return 0 */ +static int +dequeue_pkt(struct datapath *dp, struct ofpbuf **buffer, of_port_t *port_no, + int *reason) +{ + struct hw_pkt_q_entry *q_entry; + int rv = 0; + + pthread_mutex_lock(&pkt_q_mutex); + q_entry = dp->hw_pkt_list_head; + if (dp->hw_pkt_list_head != NULL) { + dp->hw_pkt_list_head = dp->hw_pkt_list_head->next; + if (dp->hw_pkt_list_head == NULL) { + dp->hw_pkt_list_tail = NULL; + } + } + pthread_mutex_unlock(&pkt_q_mutex); + + if (q_entry != NULL) { + rv = 1; + *buffer = q_entry->buffer; + *port_no = q_entry->port_no; + *reason = q_entry->reason; + free(q_entry); + } + + return rv; +} +#endif + extern char mfr_desc; extern char hw_desc; extern char sw_desc; @@ -156,7 +223,7 @@ dp_lookup_queue(struct sw_port *p, uint32_t queue_id) struct sw_queue *q; LIST_FOR_EACH(q, struct sw_queue, node, &p->queue_list) { - if (q->queue_id == queue_id) { + if ((q != NULL) && (q->queue_id == queue_id)) { return q; } } @@ -175,6 +242,89 @@ gen_datapath_id(void) return eth_addr_to_uint64(ea); } +/* FIXME: Should not depend on udatapath_as_lib */ +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) && defined(UDATAPATH_AS_LIB) +/* + * Receive packet handling for hardware driver controlled ports + * + * FIXME: For now, call the pkt fwding directly; eventually may + * want to enqueue packets at this layer; at that point must + * make sure poll event is registered or timer kicked + */ +static int +hw_packet_in(of_port_t port_no, of_packet_t *packet, int reason, + void *cookie) +{ + struct sw_port *port; + struct ofpbuf *buffer = NULL; + struct datapath *dp = (struct datapath *)cookie; + const int headroom = 128 + 2; + const int hard_header = VLAN_ETH_HEADER_LEN; + const int tail_room = sizeof(uint32_t); /* For crc if needed later */ + + VLOG_INFO("dp rcv packet on port %d, size %d\n", + port_no, packet->length); + if ((port_no < 1) || port_no > DP_MAX_PORTS) { + VLOG_ERR("Bad receive port %d\n", port_no); + /* TODO increment error counter */ + return -1; + } + port = &dp->ports[port_no]; + if (!PORT_IN_USE(port)) { + VLOG_WARN("Receive port not active: %d\n", port_no); + return -1; + } + if (!IS_HW_PORT(port)) { + VLOG_ERR("Receive port not controlled by HW: %d\n", port_no); + return -1; + } + /* Note: We're really not counting these for port stats as they + * should be gotten directly from the HW */ + port->rx_packets++; + port->rx_bytes += packet->length; + /* For now, copy data into OFP buffer; eventually may steal packet + * from RX to avoid copy. As per dp_run, add headroom and offset bytes. + */ + buffer = ofpbuf_new(headroom + hard_header + packet->length + tail_room); + if (buffer == NULL) { + VLOG_WARN("Could not alloc ofpbuf on hw pkt in\n"); + fprintf(stderr, "Could not alloc ofpbuf on hw pkt in\n"); + } else { + buffer->data = (char*)buffer->data + headroom; + buffer->size = packet->length; + memcpy(buffer->data, packet->data, packet->length); + enqueue_pkt(dp, buffer, port_no, reason); + poll_immediate_wake(); + } + + return 0; +} +#endif + +#if defined(OF_HW_PLAT) +static int +dp_hw_drv_init(struct datapath *dp) +{ + dp->hw_pkt_list_head = NULL; + dp->hw_pkt_list_tail = NULL; + + dp->hw_drv = new_of_hw_driver(dp); + if (dp->hw_drv == NULL) { + VLOG_ERR("Could not create HW driver"); + return -1; + } +#if !defined(USE_NETDEV) + if (dp->hw_drv->packet_receive_register(dp->hw_drv, + hw_packet_in, dp) < 0) { + VLOG_ERR("Could not register with HW driver to receive pkts"); + } +#endif + + return 0; +} + +#endif + int dp_new(struct datapath **dp_, uint64_t dpid) { @@ -190,6 +340,10 @@ dp_new(struct datapath **dp_, uint64_t dpid) dp->listeners = NULL; dp->n_listeners = 0; dp->id = dpid <= UINT64_C(0xffffffffffff) ? dpid : gen_datapath_id(); +/* FIXME: Should not depend on udatapath_as_lib */ +#if defined(OF_HW_PLAT) && (defined(UDATAPATH_AS_LIB) || defined(USE_NETDEV)) + dp_hw_drv_init(dp); +#endif dp->chain = chain_create(dp); if (!dp->chain) { VLOG_ERR("could not create chain"); @@ -272,6 +426,7 @@ new_port(struct datapath *dp, struct sw_port *port, uint16_t port_no, list_init(&port->queue_list); port->dp = dp; + port->flags |= SWP_USED; port->netdev = netdev; port->port_no = port_no; port->num_queues = num_queues; @@ -283,6 +438,48 @@ new_port(struct datapath *dp, struct sw_port *port, uint16_t port_no, return 0; } +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) +int +dp_add_port(struct datapath *dp, const char *port_name, uint16_t num_queues) +{ + int port_no; + int rc = 0; + struct sw_port *port; + + fprintf(stderr, "Adding port %s. hw_drv is %p\n", port_name, dp->hw_drv); + if (dp->hw_drv && dp->hw_drv->port_add) { + port_no = dp->hw_drv->port_add(dp->hw_drv, -1, port_name); + if (port_no >= 0) { + port = &dp->ports[port_no]; + if (port->flags & SWP_USED) { + VLOG_ERR("HW port %s (%d) already created\n", + port_name, port_no); + rc = -1; + } else { + fprintf(stderr, "Adding HW port %s as OF port number %d\n", + port_name, port_no); + /* FIXME: Determine and record HW addr, etc */ + port->flags |= SWP_USED | SWP_HW_DRV_PORT; + port->dp = dp; + port->port_no = port_no; + list_init(&port->queue_list); + port->num_queues = num_queues; + strncpy(port->hw_name, port_name, sizeof(port->hw_name)); + list_push_back(&dp->port_list, &port->node); + send_port_status(port, OFPPR_ADD); + } + } else { + VLOG_ERR("Port %s not recognized by hardware driver", port_name); + rc = -1; + } + } else { + VLOG_ERR("No hardware driver support; can't add ports"); + rc = -1; + } + + return rc; +} +#else /* Not HW platform support */ int dp_add_port(struct datapath *dp, const char *netdev, uint16_t num_queues) { @@ -295,6 +492,7 @@ dp_add_port(struct datapath *dp, const char *netdev, uint16_t num_queues) } return EXFULL; } +#endif /* OF_HW_PLAT */ int dp_add_local_port(struct datapath *dp, const char *netdev, uint16_t num_queues) @@ -349,9 +547,27 @@ dp_run(struct datapath *dp) } poll_timer_wait(1000); +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) + { /* Process packets received from callback thread */ + struct ofpbuf *buffer; + of_port_t port_no; + int reason; + struct sw_port *p; + + while (dequeue_pkt(dp, &buffer, &port_no, &reason)) { + p = dp_lookup_port(dp, port_no); + /* FIXME: We're throwing away the reason that came from HW */ + fwd_port_input(dp, buffer, p); + } + } +#endif + LIST_FOR_EACH_SAFE (p, pn, struct sw_port, node, &dp->port_list) { int error; + if (IS_HW_PORT(p)) { + continue; + } if (!buffer) { /* Allocate buffer with some headroom to add headers in forwarding * to the controller or adding a vlan tag, plus an extra 2 bytes to @@ -512,6 +728,9 @@ dp_wait(struct datapath *dp) size_t i; LIST_FOR_EACH (p, struct sw_port, node, &dp->port_list) { + if (IS_HW_PORT(p)) { + continue; + } netdev_recv_wait(p->netdev); } LIST_FOR_EACH (r, struct remote, node, &dp->remotes) { @@ -529,7 +748,7 @@ static int output_all(struct datapath *dp, struct ofpbuf *buffer, int in_port, int flood) { struct sw_port *p; - int prev_port; + int prev_port; /* Buffer is cloned for multiple transmits */ prev_port = -1; LIST_FOR_EACH (p, struct sw_port, node, &dp->port_list) { @@ -563,6 +782,32 @@ output_packet(struct datapath *dp, struct ofpbuf *buffer, uint16_t out_port, q = NULL; p = dp_lookup_port(dp, out_port); + +/* FIXME: Needs update for queuing */ +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) + if ((p != NULL) && IS_HW_PORT(p)) { + if (dp && dp->hw_drv) { + if (dp->hw_drv->port_link_get(dp->hw_drv, p->port_no)) { + of_packet_t *pkt; + int rv; + + pkt = calloc(1, sizeof(*pkt)); + OF_PKT_INIT(pkt, buffer); + rv = dp->hw_drv->packet_send(dp->hw_drv, out_port, pkt, 0); + if ((rv < 0) && (rv != OF_HW_PORT_DOWN)) { + VLOG_ERR("Error %d sending pkt on HW port %d\n", + rv, out_port); + ofpbuf_delete(buffer); + free(pkt); + } + } + } + return; + } + + /* Fall through to software controlled ports if not HW port */ +#endif + if (p && p->netdev != NULL) { if (!(p->config & OFPPC_PORT_DOWN)) { /* avoid the queue lookup for best-effort traffic */ @@ -749,18 +994,41 @@ static void fill_port_desc(struct sw_port *p, struct ofp_phy_port *desc) { desc->port_no = htons(p->port_no); - strncpy((char *) desc->name, netdev_get_name(p->netdev), - sizeof desc->name); - desc->name[sizeof desc->name - 1] = '\0'; - memcpy(desc->hw_addr, netdev_get_etheraddr(p->netdev), ETH_ADDR_LEN); + if (IS_HW_PORT(p)) { +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) + of_hw_driver_t *hw_drv; + + hw_drv = p->dp->hw_drv; + strncpy((char *) desc->name, p->hw_name, sizeof desc->name); + desc->name[sizeof desc->name - 1] = '\0'; + /* Update local port state */ + if (hw_drv->port_link_get(hw_drv, p->port_no)) { + p->state &= ~OFPPS_LINK_DOWN; + } else { + p->state |= OFPPS_LINK_DOWN; + } + if (hw_drv->port_enable_get(hw_drv, p->port_no)) { + p->config &= ~OFPPC_PORT_DOWN; + } else { + p->config |= OFPPC_PORT_DOWN; + } + /* FIXME: Add current, supported and advertised features */ +#endif + } else if (p->netdev) { + strncpy((char *) desc->name, netdev_get_name(p->netdev), + sizeof desc->name); + desc->name[sizeof desc->name - 1] = '\0'; + memcpy(desc->hw_addr, netdev_get_etheraddr(p->netdev), ETH_ADDR_LEN); + desc->curr= htonl(netdev_get_features(p->netdev, + NETDEV_FEAT_CURRENT)); + desc->supported = htonl(netdev_get_features(p->netdev, + NETDEV_FEAT_SUPPORTED)); + desc->advertised = htonl(netdev_get_features(p->netdev, + NETDEV_FEAT_ADVERTISED)); + desc->peer = htonl(netdev_get_features(p->netdev, NETDEV_FEAT_PEER)); + } desc->config = htonl(p->config); desc->state = htonl(p->state); - desc->curr = htonl(netdev_get_features(p->netdev, NETDEV_FEAT_CURRENT)); - desc->supported = htonl(netdev_get_features(p->netdev, - NETDEV_FEAT_SUPPORTED)); - desc->advertised = htonl(netdev_get_features(p->netdev, - NETDEV_FEAT_ADVERTISED)); - desc->peer = htonl(netdev_get_features(p->netdev, NETDEV_FEAT_PEER)); } static void @@ -1453,10 +1721,10 @@ struct queue_stats_state { }; static int -port_stats_init(const void *body, int body_len, void **state) +port_stats_init(const void *body, int body_len UNUSED, void **state) { struct port_stats_state *s = xmalloc(sizeof *s); - struct ofp_port_stats_request *psr = body; + const struct ofp_port_stats_request *psr = body; s->start_port = 1; s->port_no = ntohs(psr->port_no); @@ -1465,25 +1733,57 @@ port_stats_init(const void *body, int body_len, void **state) } static void -dump_port_stats(struct sw_port *port, struct ofpbuf *buffer) +dump_port_stats(struct datapath *dp, struct sw_port *port, + struct ofpbuf *buffer) { struct ofp_port_stats *ops = ofpbuf_put_uninit(buffer, sizeof *ops); ops->port_no = htons(port->port_no); memset(ops->pad, 0, sizeof ops->pad); - ops->rx_packets = htonll(port->rx_packets); - ops->tx_packets = htonll(port->tx_packets); - ops->rx_bytes = htonll(port->rx_bytes); - ops->tx_bytes = htonll(port->tx_bytes); - ops->rx_dropped = htonll(-1); - ops->tx_dropped = htonll(port->tx_dropped); - ops->rx_errors = htonll(-1); - ops->tx_errors = htonll(-1); - ops->rx_frame_err = htonll(-1); - ops->rx_over_err = htonll(-1); - ops->rx_crc_err = htonll(-1); - ops->collisions = htonll(-1); + if (IS_HW_PORT(port)) { +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) + struct ofp_port_stats stats; + + memset(&stats, 0, sizeof(stats)); + if (dp->hw_drv->port_stats_get) { + if (dp->hw_drv->port_stats_get(dp->hw_drv, port->port_no, + &stats) < 0) { + VLOG_WARN("Error getting stats on port %d\n", port->port_no); + return; + } + } + ops->rx_packets = htonll(stats.rx_packets); + ops->tx_packets = htonll(stats.tx_packets); + ops->rx_bytes = htonll(stats.rx_bytes); + ops->tx_bytes = htonll(stats.tx_bytes); + ops->rx_dropped = htonll(stats.rx_dropped); + ops->tx_dropped = htonll(stats.tx_dropped); + ops->rx_errors = htonll(stats.rx_errors); + ops->tx_errors = htonll(stats.tx_errors); + ops->rx_frame_err = htonll(stats.rx_frame_err); + ops->rx_over_err = htonll(stats.rx_over_err); + ops->rx_crc_err = htonll(stats.rx_crc_err); + ops->collisions = htonll(stats.collisions); +#endif + } else { + ops->rx_packets = htonll(port->rx_packets); + ops->tx_packets = htonll(port->tx_packets); + ops->rx_bytes = htonll(port->rx_bytes); + ops->tx_bytes = htonll(port->tx_bytes); + ops->rx_dropped = htonll(-1); + ops->tx_dropped = htonll(port->tx_dropped); + ops->rx_errors = htonll(-1); + ops->tx_errors = htonll(-1); + ops->rx_frame_err = htonll(-1); + ops->rx_over_err = htonll(-1); + ops->rx_crc_err = htonll(-1); + ops->collisions = htonll(-1); + } } +/* Although this makes some of the motions of being called + * multiple times preserving state, it doesn't actually support + * that process; the for loop can never break early. + */ static int port_stats_dump(struct datapath *dp, void *state, struct ofpbuf *buffer) { @@ -1495,18 +1795,18 @@ static int port_stats_dump(struct datapath *dp, void *state, /* Dump statistics for all ports */ for (i = s->start_port; i < DP_MAX_PORTS; i++) { p = dp_lookup_port(dp, i); - if (p && p->netdev) { - dump_port_stats(p, buffer); + if (p && PORT_IN_USE(p)) { + dump_port_stats(dp, p, buffer); } } if (dp->local_port) { - dump_port_stats(dp->local_port, buffer); + dump_port_stats(dp, dp->local_port, buffer); } } else { /* Dump statistics for a single port */ p = dp_lookup_port(dp, s->port_no); - if (p && p->netdev) { - dump_port_stats(p, buffer); + if (p && PORT_IN_USE(p)) { + dump_port_stats(dp, p, buffer); } } diff --git a/udatapath/datapath.h b/udatapath/datapath.h index 977c8ec..c79c5ea 100644 --- a/udatapath/datapath.h +++ b/udatapath/datapath.h @@ -44,6 +44,11 @@ #include "list.h" #include "netdev.h" +/* FIXME: Can declare struct of_hw_driver instead */ +#if defined(OF_HW_PLAT) +#include +#endif + struct rconn; struct pvconn; struct sw_flow; @@ -62,11 +67,26 @@ struct sw_queue { uint16_t min_rate; }; +#define MAX_HW_NAME_LEN 32 +enum sw_port_flags { + SWP_USED = 1 << 0, /* Is port being used */ + SWP_HW_DRV_PORT = 1 << 1, /* Port controlled by HW driver */ +}; +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) +#define IS_HW_PORT(p) ((p)->flags & SWP_HW_DRV_PORT) +#else +#define IS_HW_PORT(p) 0 +#endif + +#define PORT_IN_USE(p) (((p) != NULL) && (p)->flags & SWP_USED) + struct sw_port { uint32_t config; /* Some subset of OFPPC_* flags. */ uint32_t state; /* Some subset of OFPPS_* flags. */ + uint32_t flags; /* SWP_* flags above */ struct datapath *dp; struct netdev *netdev; + char hw_name[OFP_MAX_PORT_NAME_LEN]; struct list node; /* Element in datapath.ports. */ unsigned long long int rx_packets, tx_packets; unsigned long long int rx_bytes, tx_bytes; @@ -78,6 +98,15 @@ struct sw_port { struct list queue_list; /* list of all queues for this port */ }; +#if defined(OF_HW_PLAT) +struct hw_pkt_q_entry { + struct ofpbuf *buffer; + struct hw_pkt_q_entry *next; + of_port_t port_no; + int reason; +}; +#endif + #define DP_MAX_PORTS 255 BUILD_ASSERT_DECL(DP_MAX_PORTS <= OFPP_MAX); @@ -105,6 +134,15 @@ struct datapath { struct sw_port ports[DP_MAX_PORTS]; struct sw_port *local_port; /* OFPP_LOCAL port, if any. */ struct list port_list; /* All ports, including local_port. */ + +#if defined(OF_HW_PLAT) + /* Although the chain maintains the pointer to the HW driver + * for flow operations, the datapath needs the port functions + * in the driver structure + */ + of_hw_driver_t *hw_drv; + struct hw_pkt_q_entry *hw_pkt_list_head, *hw_pkt_list_tail; +#endif }; int dp_new(struct datapath **, uint64_t dpid); @@ -124,4 +162,6 @@ void dp_output_control(struct datapath *, struct ofpbuf *, int in_port, struct sw_port * dp_lookup_port(struct datapath *, uint16_t); struct sw_queue * dp_lookup_queue(struct sw_port *, uint32_t); +int udatapath_cmd(int argc, char *argv[]); + #endif /* datapath.h */ diff --git a/udatapath/of_ext_msg.c b/udatapath/of_ext_msg.c index 680b9dd..36d587b 100644 --- a/udatapath/of_ext_msg.c +++ b/udatapath/of_ext_msg.c @@ -142,7 +142,7 @@ recv_of_exp_queue_modify(struct datapath *dp, struct ofp_packet_queue *opq; struct ofp_queue_prop_min_rate *mr; - int error; + int error = 0; uint16_t port_no; uint32_t queue_id; @@ -167,7 +167,7 @@ recv_of_exp_queue_modify(struct datapath *dp, queue_id = ntohl(opq->queue_id); p = dp_lookup_port(dp, port_no); - if (p->netdev){ + if (PORT_IN_USE(p)) { q = dp_lookup_queue(p, queue_id); if (q) { /* queue exists - modify it */ @@ -208,6 +208,18 @@ recv_of_exp_queue_modify(struct datapath *dp, VLOG_ERR("Failed to create/modify queue - port %d doesn't exist", port_no); } + if (!error) { + if (IS_HW_PORT(p)) { +#if defined(OF_HW_PLAT) && !defined(USE_NETDEV) + error = dp->hw_drv->port_queue_config(dp->hw_drv, port_no, + queue_id, ntohs(mr->rate)); + if (error < 0) { + VLOG_ERR("Failed to update HW port %d queue %d", + port_no, queue_id); + } +#endif + } + } } /** * Parses a set dp_desc message and uses it to set diff --git a/udatapath/switch-flow.h b/udatapath/switch-flow.h index f680468..374cc73 100644 --- a/udatapath/switch-flow.h +++ b/udatapath/switch-flow.h @@ -75,6 +75,8 @@ struct sw_flow { struct list node; struct list iter_node; unsigned long int serial; + + void *private; /* Cookie for tables */ }; int flow_matches_1wild(const struct sw_flow_key *, const struct sw_flow_key *); diff --git a/udatapath/table.h b/udatapath/table.h index 1d17c07..1312e3c 100644 --- a/udatapath/table.h +++ b/udatapath/table.h @@ -39,8 +39,8 @@ #include #include -#include "datapath.h" +struct datapath; /* Forward declaration for delete operation */ struct sw_flow; struct sw_flow_key; struct ofp_action_header; diff --git a/udatapath/udatapath.c b/udatapath/udatapath.c index 41e759f..0f0f769 100644 --- a/udatapath/udatapath.c +++ b/udatapath/udatapath.c @@ -55,6 +55,10 @@ #include "vconn-ssl.h" #include "vlog-socket.h" +#if defined(OF_HW_PLAT) +#include +#endif + #define THIS_MODULE VLM_udatapath #include "vlog.h" @@ -77,8 +81,26 @@ static uint16_t num_queues = NETDEV_MAX_QUEUES; static void add_ports(struct datapath *dp, char *port_list); +/* Need to treat this more generically */ +#if defined(UDATAPATH_AS_LIB) +#define OFP_FATAL(_er, _str, args...) do { \ + fprintf(stderr, _str, ## args); \ + return -1; \ + } while (0) +#else +#define OFP_FATAL(_er, _str, args...) ofp_fatal(_er, _str, ## args) +#endif + +#if !defined(UDATAPATH_AS_LIB) int main(int argc, char *argv[]) +{ + return udatapath_cmd(argc, argv); +} +#endif + +int +udatapath_cmd(int argc, char *argv[]) { int n_listeners; int error; @@ -92,7 +114,7 @@ main(int argc, char *argv[]) signal(SIGPIPE, SIG_IGN); if (argc - optind < 1) { - ofp_fatal(0, "at least one listener argument is required; " + OFP_FATAL(0, "at least one listener argument is required; " "use --help for usage"); } @@ -113,7 +135,7 @@ main(int argc, char *argv[]) } } if (!n_listeners) { - ofp_fatal(0, "could not listen for any connections"); + OFP_FATAL(0, "could not listen for any connections"); } if (port_list) { @@ -122,13 +144,13 @@ main(int argc, char *argv[]) if (local_port) { error = dp_add_local_port(dp, local_port, 0); if (error) { - ofp_fatal(error, "failed to add local port %s", local_port); + OFP_FATAL(error, "failed to add local port %s", local_port); } } error = vlog_server_listen(NULL, NULL); if (error) { - ofp_fatal(error, "could not listen for vlog connections"); + OFP_FATAL(error, "could not listen for vlog connections"); } die_if_already_running();