Controller: remove default '-v' flag!!

For the moment, we are deprecating using localhost. We may allow
it again in the future.

bin/mn

@@ -29,7 +29,7 @@ from mininet.node import ( Host, CPULimitedHost, Controller, OVSController,
                            DefaultController, NullController,
                            UserSwitch, OVSSwitch, OVSBridge,
                            IVSSwitch )
-from mininet.nodelib import LinuxBridge
+from mininet.nodelib import LinuxBridge, Server
 from mininet.link import Link, TCLink, OVSLink
 from mininet.topo import ( SingleSwitchTopo, LinearTopo,
                            SingleSwitchReversedTopo, MinimalTopo )
@@ -40,8 +40,7 @@ from mininet.util import buildTopo
 from functools import partial
 
 # Experimental! cluster edition prototype
-from mininet.examples.cluster import ( MininetCluster, RemoteHost,
-                                       RemoteOVSSwitch, RemoteLink,
+from mininet.examples.cluster import ( MininetCluster,
                                        SwitchBinPlacer, RandomPlacer,
                                        ClusterCleanup )
 from mininet.examples.clustercli import ClusterCLI
@@ -69,6 +68,7 @@ SWITCHES = { 'user': UserSwitch,
 
 HOSTDEF = 'proc'
 HOSTS = { 'proc': Host,
+          'server': Server,
           'rt': specialClass( CPULimitedHost, defaults=dict( sched='rt' ) ),
           'cfs': specialClass( CPULimitedHost, defaults=dict( sched='cfs' ) ) }
 
@@ -335,9 +335,7 @@ class MininetRunner( object ):
         Net = MininetWithControlNet if inNamespace else Mininet
         cli = ClusterCLI if cluster else CLI
         if cluster:
-            warn( '*** WARNING: Experimental cluster mode!\n'
-                  '*** Using RemoteHost, RemoteOVSSwitch, RemoteLink\n' )
-            host, switch, link = RemoteHost, RemoteOVSSwitch, RemoteLink
+            warn( '*** WARNING: Experimental cluster mode!\n' )
             Net = partial( MininetCluster, servers=servers,
                            placement=PLACEMENT[ self.options.placement ] )
 

examples/cluster-old.py

@@ -0,0 +1,918 @@
+#!/usr/bin/python
+
+"""
+cluster.py: prototyping/experimentation for distributed Mininet,
+            aka Mininet: Cluster Edition
+
+Author: Bob Lantz
+
+Core classes:
+
+    RemoteNode: a Node() running on a remote server
+    RemoteOVSSwitch(): an OVSSwitch() running on a remote server
+    RemoteLink: a Link() on a remote server
+    Tunnel: a Link() between a local Node() and a RemoteNode()
+
+These are largely interoperable with local objects.
+
+- One Mininet to rule them all
+
+It is important that the same topologies, APIs, and CLI can be used
+with minimal or no modification in both local and distributed environments.
+
+- Multiple placement models
+
+Placement should be as easy as possible. We should provide basic placement
+support and also allow for explicit placement.
+
+Questions:
+
+What is the basic communication mechanism?
+
+To start with? Probably a single multiplexed ssh connection between each
+pair of mininet servers that needs to communicate.
+
+How are tunnels created?
+
+We have several options including ssh, GRE, OF capsulator, socat, VDE, l2tp,
+etc..  It's not clear what the best one is.  For now, we use ssh tunnels since
+they are encrypted and semi-automatically shared.  We will probably want to
+support GRE as well because it's very easy to set up with OVS.
+
+How are tunnels destroyed?
+
+They are destroyed when the links are deleted in Mininet.stop()
+
+How does RemoteNode.popen() work?
+
+It opens a shared ssh connection to the remote server and attaches to
+the namespace using mnexec -a -g.
+
+Is there any value to using Paramiko vs. raw ssh?
+
+Maybe, but it doesn't seem to support L2 tunneling.
+
+Should we preflight the entire network, including all server-to-server
+connections?
+
+Yes! We don't yet do this with remote server-to-server connections yet.
+
+Should we multiplex the link ssh connections?
+
+Yes, this is done automatically with ControlMaster=auto.
+
+Note on ssh and DNS:
+Please add UseDNS: no to your /etc/ssh/sshd_config!!!
+
+Things to do:
+
+- asynchronous/pipelined/parallel startup
+- ssh debugging/profiling
+- make connections into real objects
+- support for other tunneling schemes
+- tests and benchmarks
+- hifi support (e.g. delay compensation)
+"""
+
+from mininet.node import Node, Host, OVSSwitch, Controller
+from mininet.link import Link, Intf
+from mininet.net import Mininet
+from mininet.topo import LinearTopo
+from mininet.topolib import TreeTopo
+from mininet.util import quietRun, errRun
+from mininet.examples.clustercli import CLI
+from mininet.log import setLogLevel, debug, info, error
+from mininet.clean import addCleanupCallback
+
+from signal import signal, SIGINT, SIG_IGN
+from subprocess import Popen, PIPE, STDOUT
+import os
+from random import randrange
+import sys
+import re
+from itertools import groupby
+from operator import attrgetter
+from distutils.version import StrictVersion
+
+
+def findUser():
+    "Try to return logged-in (usually non-root) user"
+    return (
+            # If we're running sudo
+            os.environ.get( 'SUDO_USER', False ) or
+            # Logged-in user (if we have a tty)
+            ( quietRun( 'who am i' ).split() or [ False ] )[ 0 ] or
+            # Give up and return effective user
+            quietRun( 'whoami' ).strip() )
+
+
+class ClusterCleanup( object ):
+    "Cleanup callback"
+
+    inited = False
+    serveruser = {}
+
+    @classmethod
+    def add( cls, server, user='' ):
+        "Add an entry to server: user dict"
+        if not cls.inited:
+            addCleanupCallback( cls.cleanup )
+        if not user:
+            user = findUser()
+        cls.serveruser[ server ] = user
+
+    @classmethod
+    def cleanup( cls ):
+        "Clean up"
+        info( '*** Cleaning up cluster\n' )
+        for server, user in cls.serveruser.iteritems():
+            if server == 'localhost':
+                # Handled by mininet.clean.cleanup()
+                continue
+            else:
+                cmd = [ 'su', user, '-c',
+                        'ssh %s@%s sudo mn -c' % ( user, server ) ]
+                info( cmd, '\n' )
+                info( quietRun( cmd ) )
+
+# BL note: so little code is required for remote nodes,
+# we will probably just want to update the main Node()
+# class to enable it for remote access! However, there
+# are a large number of potential failure conditions with
+# remote nodes which we may want to detect and handle.
+# Another interesting point is that we could put everything
+# in a mix-in class and easily add cluster mode to 2.0.
+
+class RemoteMixin( object ):
+
+    "A mix-in class to turn local nodes into remote nodes"
+
+    # ssh base command
+    # -q: don't print stupid diagnostic messages
+    # BatchMode yes: don't ask for password
+    # ForwardAgent yes: forward authentication credentials
+    sshbase = [ 'ssh', '-q',
+                '-o', 'BatchMode=yes',
+                '-o', 'ForwardAgent=yes', '-tt' ]
+
+    def __init__( self, name, server='localhost', user=None, serverIP=None,
+                  controlPath=True, splitInit=False, **kwargs):
+        """Instantiate a remote node
+           name: name of remote node
+           server: remote server (optional)
+           user: user on remote server (optional)
+           controlPath: specify shared ssh control path (optional)
+           splitInit: split initialization?
+           **kwargs: see Node()"""
+        # We connect to servers by IP address
+        self.server = server if server else 'localhost'
+        self.serverIP = ( serverIP if serverIP
+                          else self.findServerIP( self.server ) )
+        self.user = user if user else findUser()
+        ClusterCleanup.add( server=server, user=user )
+        if controlPath is True:
+            # Set a default control path for shared SSH connections
+            controlPath = '/tmp/mn-%r@%h:%p'
+        self.controlPath = controlPath
+        self.splitInit = splitInit
+        if self.user and self.server != 'localhost':
+            self.dest = '%s@%s' % ( self.user, self.serverIP )
+            self.sshcmd = [ 'sudo', '-E', '-u', self.user ] + self.sshbase
+            if self.controlPath:
+                self.sshcmd += [ '-o', 'ControlPath=' + self.controlPath,
+                                 '-o', 'ControlMaster=auto',
+                                 '-o', 'ControlPersist=' + '1' ]
+            self.sshcmd += [ self.dest ]
+            self.isRemote = True
+        else:
+            self.dest = None
+            self.sshcmd = []
+            self.isRemote = False
+        # Satisfy pylint
+        self.shell, self.pid = None, None
+        super( RemoteMixin, self ).__init__( name, **kwargs )
+
+    # Determine IP address of local host
+    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
+
+    @classmethod
+    def findServerIP( cls, server ):
+        "Return our server's IP address"
+        # First, check for an IP address
+        ipmatch = cls._ipMatchRegex.findall( server )
+        if ipmatch:
+            return ipmatch[ 0 ]
+        # Otherwise, look up remote server
+        output = quietRun( 'getent ahostsv4 %s' % server )
+        ips = cls._ipMatchRegex.findall( output )
+        ip = ips[ 0 ] if ips else None
+        return ip
+
+    # Command support via shell process in namespace
+    def startShell( self, *args, **kwargs ):
+        "Start a shell process for running commands"
+        if self.isRemote:
+            kwargs.update( mnopts='-cp' )
+        super( RemoteMixin, self ).startShell( *args, **kwargs )
+        # Optional split initialization
+        self.sendCmd( 'echo $$' )
+        if not self.splitInit:
+            self.finishInit()
+
+    def finishInit( self ):
+        "Wait for split initialization to complete"
+        self.pid = int( self.waitOutput() )
+
+    def rpopen( self, *cmd, **opts ):
+        "Return a Popen object on underlying server in root namespace"
+        params = { 'stdin': PIPE,
+                   'stdout': PIPE,
+                   'stderr': STDOUT,
+                   'sudo': True }
+        params.update( opts )
+        return self._popen( *cmd, **params )
+
+    def rcmd( self, *cmd, **opts):
+        """rcmd: run a command on underlying server
+           in root namespace
+           args: string or list of strings
+           returns: stdout and stderr"""
+        popen = self.rpopen( *cmd, **opts )
+        # print 'RCMD: POPEN:', popen
+        # These loops are tricky to get right.
+        # Once the process exits, we can read
+        # EOF twice if necessary.
+        result = ''
+        while True:
+            poll = popen.poll()
+            result += popen.stdout.read()
+            if poll is not None:
+                break
+        return result
+
+    @staticmethod
+    def _ignoreSignal():
+        "Detach from process group to ignore all signals"
+        os.setpgrp()
+
+    def _popen( self, cmd, sudo=True, tt=True, **params):
+        """Spawn a process on a remote node
+            cmd: remote command to run (list)
+            **params: parameters to Popen()
+            returns: Popen() object"""
+        if type( cmd ) is str:
+            cmd = cmd.split()
+        if self.isRemote:
+            if sudo:
+                cmd = [ 'sudo', '-E' ] + cmd
+            if tt:
+                cmd = self.sshcmd + cmd
+            else:
+                # Hack: remove -tt
+                sshcmd = list( self.sshcmd )
+                sshcmd.remove( '-tt' )
+                cmd = sshcmd + cmd
+        else:
+            if self.user and not sudo:
+                # Drop privileges
+                cmd = [ 'sudo', '-E', '-u', self.user ] + cmd
+        params.update( preexec_fn=self._ignoreSignal )
+        debug( '_popen', cmd, '\n' )
+        popen = super( RemoteMixin, self )._popen( cmd, **params )
+        return popen
+
+    def popen( self, *args, **kwargs ):
+        "Override: disable -tt"
+        return super( RemoteMixin, self).popen( *args, tt=False, **kwargs )
+
+    def addIntf( self, *args, **kwargs ):
+        "Override: use RemoteLink.moveIntf"
+        kwargs.update( moveIntfFn=RemoteLink.moveIntf )
+        return super( RemoteMixin, self).addIntf( *args, **kwargs )
+
+
+class RemoteNode( RemoteMixin, Node ):
+    "A node on a remote server"
+    pass
+
+
+class RemoteHost( RemoteNode ):
+    "A RemoteHost is simply a RemoteNode"
+    pass
+
+
+class RemoteOVSSwitch( RemoteMixin, OVSSwitch ):
+    "Remote instance of Open vSwitch"
+
+    OVSVersions = {}
+
+    def __init__( self, *args, **kwargs ):
+        # No batch startup yet
+        kwargs.update( batch=True )
+        super( RemoteOVSSwitch, self ).__init__( *args, **kwargs )
+
+    def isOldOVS( self ):
+        "Is remote switch using an old OVS version?"
+        cls = type( self )
+        if self.server not in cls.OVSVersions:
+            # pylint: disable=not-callable
+            vers = self.cmd( 'ovs-vsctl --version' )
+            # pylint: enable=not-callable
+            cls.OVSVersions[ self.server ] = re.findall(
+                r'\d+\.\d+', vers )[ 0 ]
+        return ( StrictVersion( cls.OVSVersions[ self.server ] ) <
+                 StrictVersion( '1.10' ) )
+
+    @classmethod
+    def batchStartup( cls, switches, **_kwargs ):
+        "Start up switches in per-server batches"
+        key = attrgetter( 'server' )
+        for server, switchGroup in groupby( sorted( switches, key=key ), key ):
+            info( '(%s)' % server )
+            group = tuple( switchGroup )
+            switch = group[ 0 ]
+            OVSSwitch.batchStartup( group, run=switch.cmd )
+        return switches
+
+    @classmethod
+    def batchShutdown( cls, switches, **_kwargs ):
+        "Stop switches in per-server batches"
+        key = attrgetter( 'server' )
+        for server, switchGroup in groupby( sorted( switches, key=key ), key ):
+            info( '(%s)' % server )
+            group = tuple( switchGroup )
+            switch = group[ 0 ]
+            OVSSwitch.batchShutdown( group, run=switch.rcmd )
+        return switches
+
+
+class RemoteLink( Link ):
+    "A RemoteLink is a link between nodes which may be on different servers"
+
+    def __init__( self, node1, node2, **kwargs ):
+        """Initialize a RemoteLink
+           see Link() for parameters"""
+        # Create links on remote node
+        self.node1 = node1
+        self.node2 = node2
+        self.tunnel = None
+        kwargs.setdefault( 'params1', {} )
+        kwargs.setdefault( 'params2', {} )
+        self.cmd = None  # satisfy pylint
+        Link.__init__( self, node1, node2, **kwargs )
+
+    def stop( self ):
+        "Stop this link"
+        if self.tunnel:
+            self.tunnel.terminate()
+            self.intf1.delete()
+            self.intf2.delete()
+        else:
+            Link.stop( self )
+        self.tunnel = None
+
+    def makeIntfPair( self, intfname1, intfname2, addr1=None, addr2=None,
+                      node1=None, node2=None, deleteIntfs=True   ):
+        """Create pair of interfaces
+            intfname1: name of interface 1
+            intfname2: name of interface 2
+            (override this method [and possibly delete()]
+            to change link type)"""
+        node1 = self.node1 if node1 is None else node1
+        node2 = self.node2 if node2 is None else node2
+        server1 = getattr( node1, 'server', 'localhost' )
+        server2 = getattr( node2, 'server', 'localhost' )
+        if server1 == server2:
+            # Link within same server
+            return Link.makeIntfPair( intfname1, intfname2, addr1, addr2,
+                                      node1, node2, deleteIntfs=deleteIntfs,
+                                      runCmd=None )
+        # Otherwise, make a tunnel
+        self.tunnel = self.makeTunnel( node1, node2, intfname1, intfname2,
+                                       addr1, addr2 )
+        return self.tunnel
+
+    @staticmethod
+    def moveIntf( intf, node, printError=True ):
+        """Move remote interface from root ns to node
+            intf: string, interface
+            dstNode: destination Node
+            srcNode: source Node or None (default) for root ns
+            printError: if true, print error"""
+        intf = str( intf )
+        cmd = 'ip link set %s netns %s' % ( intf, node.pid )
+        node.rcmd( cmd )
+        links = node.cmd( 'ip link show' )
+        if not ' %s:' % intf in links:
+            if printError:
+                error( '*** Error: RemoteLink.moveIntf: ' + intf +
+                       ' not successfully moved to ' + node.name + '\n' )
+            return False
+        return True
+
+    def makeTunnel( self, node1, node2, intfname1, intfname2,
+                    addr1=None, addr2=None ):
+        "Make a tunnel across switches on different servers"
+        # We should never try to create a tunnel to ourselves!
+        assert node1.server != 'localhost' or node2.server != 'localhost'
+        # And we can't ssh into this server remotely as 'localhost',
+        # so try again swappping node1 and node2
+        if node2.server == 'localhost':
+            return self.makeTunnel( node2, node1, intfname2, intfname1,
+                                    addr2, addr1 )
+        # 1. Create tap interfaces
+        for node in node1, node2:
+            # For now we are hard-wiring tap9, which we will rename
+            cmd = 'ip tuntap add dev tap9 mode tap user ' + node.user
+            result = node.rcmd( cmd )
+            if result:
+                raise Exception( 'error creating tap9 on %s: %s' %
+                                 ( node, result ) )
+        # 2. Create ssh tunnel between tap interfaces
+        # -n: close stdin
+        dest = '%s@%s' % ( node2.user, node2.serverIP )
+        cmd = [ 'ssh', '-n', '-o', 'Tunnel=Ethernet', '-w', '9:9',
+                dest, 'echo @' ]
+        self.cmd = cmd
+        tunnel = node1.rpopen( cmd, sudo=False )
+        # When we receive the character '@', it means that our
+        # tunnel should be set up
+        debug( 'Waiting for tunnel to come up...\n' )
+        ch = tunnel.stdout.read( 1 )
+        if ch != '@':
+            raise Exception( 'makeTunnel:\n',
+                             'Tunnel setup failed for',
+                             '%s:%s' % ( node1, node1.dest ), 'to',
+                             '%s:%s\n' % ( node2, node2.dest ),
+                             'command was:', cmd, '\n' )
+        # 3. Move interfaces if necessary
+        for node in node1, node2:
+            if not self.moveIntf( 'tap9', node ):
+                raise Exception( 'interface move failed on node %s' % node )
+        # 4. Rename tap interfaces to desired names
+        for node, intf, addr in ( ( node1, intfname1, addr1 ),
+                                  ( node2, intfname2, addr2 ) ):
+            if not addr:
+                result = node.cmd( 'ip link set tap9 name', intf )
+            else:
+                result = node.cmd( 'ip link set tap9 name', intf,
+                                   'address', addr )
+            if result:
+                raise Exception( 'error renaming %s: %s' % ( intf, result ) )
+        return tunnel
+
+    def status( self ):
+        "Detailed representation of link"
+        if self.tunnel:
+            if self.tunnel.poll() is not None:
+                status = "Tunnel EXITED %s" % self.tunnel.returncode
+            else:
+                status = "Tunnel Running (%s: %s)" % (
+                    self.tunnel.pid, self.cmd )
+        else:
+            status = "OK"
+        result = "%s %s" % ( Link.status( self ), status )
+        return result
+
+
+# Some simple placement algorithms for MininetCluster
+
+class Placer( object ):
+    "Node placement algorithm for MininetCluster"
+
+    def __init__( self, servers=None, nodes=None, hosts=None,
+                  switches=None, controllers=None, links=None ):
+        """Initialize placement object
+           servers: list of servers
+           nodes: list of all nodes
+           hosts: list of hosts
+           switches: list of switches
+           controllers: list of controllers
+           links: list of links
+           (all arguments are optional)
+           returns: server"""
+        self.servers = servers or []
+        self.nodes = nodes or []
+        self.hosts = hosts or []
+        self.switches = switches or []
+        self.controllers = controllers or []
+        self.links = links or []
+
+    def place( self, node ):
+        "Return server for a given node"
+        assert self, node  # satisfy pylint
+        # Default placement: run locally
+        return 'localhost'
+
+
+class RandomPlacer( Placer ):
+    "Random placement"
+    def place( self, nodename ):
+        """Random placement function
+            nodename: node name"""
+        assert nodename  # please pylint
+        # This may be slow with lots of servers
+        return self.servers[ randrange( 0, len( self.servers ) ) ]
+
+
+class RoundRobinPlacer( Placer ):
+    """Round-robin placement
+       Note this will usually result in cross-server links between
+       hosts and switches"""
+
+    def __init__( self, *args, **kwargs ):
+        Placer.__init__( self, *args, **kwargs )
+        self.next = 0
+
+    def place( self, nodename ):
+        """Round-robin placement function
+            nodename: node name"""
+        assert nodename  # please pylint
+        # This may be slow with lots of servers
+        server = self.servers[ self.next ]
+        self.next = ( self.next + 1 ) % len( self.servers )
+        return server
+
+
+class SwitchBinPlacer( Placer ):
+    """Place switches (and controllers) into evenly-sized bins,
+       and attempt to co-locate hosts and switches"""
+
+    def __init__( self, *args, **kwargs ):
+        Placer.__init__( self, *args, **kwargs )
+        # Easy lookup for servers and node sets
+        self.servdict = dict( enumerate( self.servers ) )
+        self.hset = frozenset( self.hosts )
+        self.sset = frozenset( self.switches )
+        self.cset = frozenset( self.controllers )
+        # Server and switch placement indices
+        self.placement = self.calculatePlacement()
+
+    @staticmethod
+    def bin( nodes, servers ):
+        "Distribute nodes evenly over servers"
+        # Calculate base bin size
+        nlen = len( nodes )
+        slen = len( servers )
+        # Basic bin size
+        quotient = int( nlen / slen )
+        binsizes = { server: quotient for server in servers }
+        # Distribute remainder
+        remainder = nlen % slen
+        for server in servers[ 0 : remainder ]:
+            binsizes[ server ] += 1
+        # Create binsize[ server ] tickets for each server
+        tickets = sum( [ binsizes[ server ] * [ server ]
+                         for server in servers ], [] )
+        # And assign one ticket to each node
+        return { node: ticket for node, ticket in zip( nodes, tickets ) }
+
+    def calculatePlacement( self ):
+        "Pre-calculate node placement"
+        placement = {}
+        # Create host-switch connectivity map,
+        # associating host with last switch that it's
+        # connected to
+        switchFor = {}
+        for src, dst in self.links:
+            if src in self.hset and dst in self.sset:
+                switchFor[ src ] = dst
+            if dst in self.hset and src in self.sset:
+                switchFor[ dst ] = src
+        # Place switches
+        placement = self.bin( self.switches, self.servers )
+        # Place controllers and merge into placement dict
+        placement.update( self.bin( self.controllers, self.servers ) )
+        # Co-locate hosts with their switches
+        for h in self.hosts:
+            if h in placement:
+                # Host is already placed - leave it there
+                continue
+            if h in switchFor:
+                placement[ h ] = placement[ switchFor[ h ] ]
+            else:
+                raise Exception(
+                        "SwitchBinPlacer: cannot place isolated host " + h )
+        return placement
+
+    def place( self, node ):
+        """Simple placement algorithm:
+           place switches into evenly sized bins,
+           and place hosts near their switches"""
+        return self.placement[ node ]
+
+
+class HostSwitchBinPlacer( Placer ):
+    """Place switches *and hosts* into evenly-sized bins
+       Note that this will usually result in cross-server
+       links between hosts and switches"""
+
+    def __init__( self, *args, **kwargs ):
+        Placer.__init__( self, *args, **kwargs )
+        # Calculate bin sizes
+        scount = len( self.servers )
+        self.hbin = max( int( len( self.hosts ) / scount ), 1 )
+        self.sbin = max( int( len( self.switches ) / scount ), 1 )
+        self.cbin = max( int( len( self.controllers ) / scount ), 1 )
+        info( 'scount:', scount )
+        info( 'bins:', self.hbin, self.sbin, self.cbin, '\n' )
+        self.servdict = dict( enumerate( self.servers ) )
+        self.hset = frozenset( self.hosts )
+        self.sset = frozenset( self.switches )
+        self.cset = frozenset( self.controllers )
+        self.hind, self.sind, self.cind = 0, 0, 0
+
+    def place( self, nodename ):
+        """Simple placement algorithm:
+            place nodes into evenly sized bins"""
+        # Place nodes into bins
+        if nodename in self.hset:
+            server = self.servdict[ self.hind / self.hbin ]
+            self.hind += 1
+        elif nodename in self.sset:
+            server = self.servdict[ self.sind / self.sbin ]
+            self.sind += 1
+        elif nodename in self.cset:
+            server = self.servdict[ self.cind / self.cbin ]
+            self.cind += 1
+        else:
+            info( 'warning: unknown node', nodename )
+            server = self.servdict[ 0 ]
+        return server
+
+
+# The MininetCluster class is not strictly necessary.
+# However, it has several purposes:
+# 1. To set up ssh connection sharing/multiplexing
+# 2. To pre-flight the system so that everything is more likely to work
+# 3. To allow connection/connectivity monitoring
+# 4. To support pluggable placement algorithms
+
+class MininetCluster( Mininet ):
+
+    "Cluster-enhanced version of Mininet class"
+
+    # Default ssh command
+    # BatchMode yes: don't ask for password
+    # ForwardAgent yes: forward authentication credentials
+    sshcmd = [ 'ssh', '-o', 'BatchMode=yes', '-o', 'ForwardAgent=yes' ]
+
+    def __init__( self, *args, **kwargs ):
+        """servers: a list of servers to use (note: include
+           localhost or None to use local system as well)
+           user: user name for server ssh
+           placement: Placer() subclass"""
+        params = { 'host': RemoteHost,
+                   'switch': RemoteOVSSwitch,
+                   'link': RemoteLink,
+                   'precheck': True }
+        params.update( kwargs )
+        servers = params.pop( 'servers', [ 'localhost' ] )
+        servers = [ s if s else 'localhost' for s in servers ]
+        self.servers = servers
+        self.serverIP = params.pop( 'serverIP', {} )
+        if not self.serverIP:
+            self.serverIP = { server: RemoteMixin.findServerIP( server )
+                              for server in self.servers }
+        self.user = params.pop( 'user', findUser() )
+        if params.pop( 'precheck' ):
+            self.precheck()
+        self.connections = {}
+        self.placement = params.pop( 'placement', SwitchBinPlacer )
+        # Make sure control directory exists
+        self.cdir = os.environ[ 'HOME' ] + '/.ssh/mn'
+        errRun( [ 'mkdir', '-p', self.cdir ] )
+        Mininet.__init__( self, *args, **params )
+
+    def popen( self, cmd ):
+        "Popen() for server connections"
+        assert self  # please pylint
+        old = signal( SIGINT, SIG_IGN )
+        conn = Popen( cmd, stdin=PIPE, stdout=PIPE, close_fds=True )
+        signal( SIGINT, old )
+        return conn
+
+    def baddLink( self, *args, **kwargs ):
+        "break addlink for testing"
+        pass
+
+    def precheck( self ):
+        """Pre-check to make sure connection works and that
+           we can call sudo without a password"""
+        result = 0
+        info( '*** Checking servers\n' )
+        for server in self.servers:
+            ip = self.serverIP[ server ]
+            if not server or server == 'localhost':
+                continue
+            info( server, '' )
+            dest = '%s@%s' % ( self.user, ip )
+            cmd = [ 'sudo', '-E', '-u', self.user ]
+            cmd += self.sshcmd + [ '-n', dest, 'sudo true' ]
+            debug( ' '.join( cmd ), '\n' )
+            _out, _err, code = errRun( cmd )
+            if code != 0:
+                error( '\nstartConnection: server connection check failed '
+                       'to %s using command:\n%s\n'
+                        % ( server, ' '.join( cmd ) ) )
+            result |= code
+        if result:
+            error( '*** Server precheck failed.\n'
+                   '*** Make sure that the above ssh command works'
+                   ' correctly.\n'
+                   '*** You may also need to run mn -c on all nodes, and/or\n'
+                   '*** use sudo -E.\n' )
+            sys.exit( 1 )
+        info( '\n' )
+
+    def modifiedaddHost( self, *args, **kwargs ):
+        "Slightly modify addHost"
+        assert self  # please pylint
+        kwargs[ 'splitInit' ] = True
+        return Mininet.addHost( *args, **kwargs )
+
+    def placeNodes( self ):
+        """Place nodes on servers (if they don't have a server), and
+           start shell processes"""
+        if not self.servers or not self.topo:
+            # No shirt, no shoes, no service
+            return
+        nodes = self.topo.nodes()
+        placer = self.placement( servers=self.servers,
+                                 nodes=self.topo.nodes(),
+                                 hosts=self.topo.hosts(),
+                                 switches=self.topo.switches(),
+                                 links=self.topo.links() )
+        for node in nodes:
+            config = self.topo.nodeInfo( node )
+            # keep local server name consistent accross nodes
+            if 'server' in config.keys() and config[ 'server' ] is None:
+                config[ 'server' ] = 'localhost'
+            server = config.setdefault( 'server', placer.place( node ) )
+            if server:
+                config.setdefault( 'serverIP', self.serverIP[ server ] )
+            info( '%s:%s ' % ( node, server ) )
+            key = ( None, server )
+            _dest, cfile, _conn = self.connections.get(
+                        key, ( None, None, None ) )
+            if cfile:
+                config.setdefault( 'controlPath', cfile )
+
+    def addController( self, *args, **kwargs ):
+        "Patch to update IP address to global IP address"
+        controller = Mininet.addController( self, *args, **kwargs )
+        # Update IP address for controller that may not be local
+        if ( isinstance( controller, Controller )
+             and controller.IP() == '127.0.0.1' ):
+            links = controller.cmd( 'ip link show' )
+            eth0 = re.findall( ' (.*eth0):', links )
+            if not eth0:
+                raise Exception( 'Cannot find IP address for controller eth0' )
+            Intf( eth0[ 0 ], node=controller ).updateIP()
+        return controller
+
+    def buildFromTopo( self, *args, **kwargs ):
+        "Start network"
+        info( '*** Placing nodes\n' )
+        self.placeNodes()
+        info( '\n' )
+        Mininet.buildFromTopo( self, *args, **kwargs )
+
+
+def testNsTunnels():
+    "Test tunnels between nodes in namespaces"
+    net = Mininet( host=RemoteHost, link=RemoteLink )
+    h1 = net.addHost( 'h1' )
+    h2 = net.addHost( 'h2', server='ubuntu2' )
+    net.addLink( h1, h2 )
+    net.start()
+    net.pingAll()
+    net.stop()
+
+# Manual topology creation with net.add*()
+#
+# This shows how node options may be used to manage
+# cluster placement using the net.add*() API
+
+def testRemoteNet( remote='ubuntu2' ):
+    "Test remote Node classes"
+    print '*** Remote Node Test'
+    net = Mininet( host=RemoteHost, switch=RemoteOVSSwitch,
+                   link=RemoteLink )
+    c0 = net.addController( 'c0' )
+    # Make sure controller knows its non-loopback address
+    Intf( 'eth0', node=c0 ).updateIP()
+    print "*** Creating local h1"
+    h1 = net.addHost( 'h1' )
+    print "*** Creating remote h2"
+    h2 = net.addHost( 'h2', server=remote )
+    print "*** Creating local s1"
+    s1 = net.addSwitch( 's1' )
+    print "*** Creating remote s2"
+    s2 = net.addSwitch( 's2', server=remote )
+    print "*** Adding links"
+    net.addLink( h1, s1 )
+    net.addLink( s1, s2 )
+    net.addLink( h2, s2 )
+    net.start()
+    print 'Mininet is running on', quietRun( 'hostname' ).strip()
+    for node in c0, h1, h2, s1, s2:
+        print 'Node', node, 'is running on', node.cmd( 'hostname' ).strip()
+    net.pingAll()
+    CLI( net )
+    net.stop()
+
+
+# High-level/Topo API example
+#
+# This shows how existing Mininet topologies may be used in cluster
+# mode by creating node placement functions and a controller which
+# can be accessed remotely. This implements a very compatible version
+# of cluster edition with a minimum of code!
+
+remoteHosts = [ 'h2' ]
+remoteSwitches = [ 's2' ]
+remoteServer = 'ubuntu2'
+
+def HostPlacer( name, *args, **params ):
+    "Custom Host() constructor which places hosts on servers"
+    if name in remoteHosts:
+        return RemoteHost( name, *args, server=remoteServer, **params )
+    else:
+        return Host( name, *args, **params )
+
+def SwitchPlacer( name, *args, **params ):
+    "Custom Switch() constructor which places switches on servers"
+    if name in remoteSwitches:
+        return RemoteOVSSwitch( name, *args, server=remoteServer, **params )
+    else:
+        return RemoteOVSSwitch( name, *args, **params )
+
+def ClusterController( *args, **kwargs):
+    "Custom Controller() constructor which updates its eth0 IP address"
+    controller = Controller( *args, **kwargs )
+    # Find out its IP address so that cluster switches can connect
+    Intf( 'eth0', node=controller ).updateIP()
+    return controller
+
+def testRemoteTopo():
+    "Test remote Node classes using Mininet()/Topo() API"
+    topo = LinearTopo( 2 )
+    net = Mininet( topo=topo, host=HostPlacer, switch=SwitchPlacer,
+                   link=RemoteLink, controller=ClusterController )
+    net.start()
+    net.pingAll()
+    net.stop()
+
+# Need to test backwards placement, where each host is on
+# a server other than its switch!! But seriously we could just
+# do random switch placement rather than completely random
+# host placement.
+
+def testRemoteSwitches():
+    "Test with local hosts and remote switches"
+    servers = [ 'localhost', 'ubuntu2']
+    topo = TreeTopo( depth=4, fanout=2 )
+    net = MininetCluster( topo=topo, servers=servers,
+                          placement=RoundRobinPlacer )
+    net.start()
+    net.pingAll()
+    net.stop()
+
+
+#
+# For testing and demo purposes it would be nice to draw the
+# network graph and color it based on server.
+
+# The MininetCluster() class integrates pluggable placement
+# functions, for maximum ease of use. MininetCluster() also
+# pre-flights and multiplexes server connections.
+
+def testMininetCluster():
+    "Test MininetCluster()"
+    servers = [ 'localhost', 'ubuntu2' ]
+    topo = TreeTopo( depth=3, fanout=3 )
+    net = MininetCluster( topo=topo, servers=servers,
+                          placement=SwitchBinPlacer )
+    net.start()
+    net.pingAll()
+    net.stop()
+
+def signalTest():
+    "Make sure hosts are robust to signals"
+    h = RemoteHost( 'h0', server='ubuntu1' )
+    h.shell.send_signal( SIGINT )
+    h.shell.poll()
+    if h.shell.returncode is None:
+        print 'OK: ', h, 'has not exited'
+    else:
+        print 'FAILURE:', h, 'exited with code', h.shell.returncode
+    h.stop()
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    # testRemoteTopo()
+    # testRemoteNet()
+    # testMininetCluster()
+    # testRemoteSwitches()
+    signalTest()

examples/clustercli.py

@@ -19,8 +19,7 @@ class ClusterCLI( CLI ):
                    'magenta', 'pink', 'grey', 'brown',
                    'white' ]
         slen, clen = len( seq ), len( colors )
-        reps = max( 1, slen / clen )
-        colors = colors * reps
+        colors = colors * ( slen / clen + 1 )
         colors = colors[ 0 : slen ]
         return colors
 

examples/fakecluster.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python
+
+"""
+fakecluster.py: a fake cluster for testing Mininet cluster edition!!!
+
+We are going to self-host Mininet by creating a virtual cluster
+for cluster edition.
+
+Note: ssh is kind of a mess - you end up having to do things
+like h1 sudo -E -u openflow ssh 10.2
+"""
+
+from mininet.net import Mininet
+from mininet.nodelib import LinuxBridge, Server
+from mininet.cli import CLI
+from mininet.topo import Topo, SingleSwitchTopo
+from mininet.log import setLogLevel, warn
+from mininet.util import errRun, quietRun
+from mininet.link import Link
+
+from functools import partial
+from sys import argv
+
+class MininetServer( Server ):
+    "A server (for nested Mininet) that runs ssh and ovs"
+
+    privateDirs = [ '/var/run/sshd', '/etc/openvswitch',
+                    '/var/run/openvswitch', '/var/log/openvswitch' ]
+
+    def __init__( self, *args, **kwargs ):
+        "Turn on ovs by default"
+        kwargs.setdefault( 'ovs', True )
+        super( MininetServer, self ).__init__( *args, **kwargs )
+
+    def config( self, **kwargs ):
+        """Configure/start sshd and other stuff
+           ovs: start Open vSwitch?"""
+        self.ovs = kwargs.get( 'ovs' )
+        super( MininetServer, self ).config( **kwargs )
+        if self.ovs:
+            self.service( 'openvswitch-switch start' )
+
+    def terminate( self, *args, **kwargs ):
+        "Shut down services and terminate server"
+        if self.ovs:
+            self.service( 'openvswitch-switch stop' )
+        super( MininetServer, self ).terminate( *args, **kwargs )
+
+
+class ServerLink( Link ):
+    def intfName( self, node, n ):
+        "Override to avoid destruction by cleanup!"
+        # This is kind of ugly... for some reason 'eth0' fails so
+        # we just use 'm1eth0'; however, this should nest reasonably.
+        return ( node.name + 'eth' + repr( n ) if isinstance( node, Server )
+                 else node.name + '-eth' + repr( n ) )
+    def makeIntfPair( self, *args, **kwargs ):
+        "Override to use quietRun"
+        kwargs.update( runCmd=quietRun )
+        super( ServerLink, self ).makeIntfPair( *args, **kwargs )
+
+class ClusterTopo( Topo ):
+    "Cluster topology: m1..mN"
+    def build( self, n ):
+        ms1 = self.addSwitch( 'ms1' )
+        for i in range( 1, n + 1 ):
+            h = self.addHost( 'm%d' % i )
+            self.addLink( h, ms1, cls=ServerLink )
+
+
+def test( serverCount ):
+    "Test this setup"
+    setLogLevel( 'info' )
+    topo = ClusterTopo( serverCount )
+    host = partial( MininetServer, ssh=True, ovs=True)
+    net = Mininet( topo=topo, host=host, switch=LinuxBridge, ipBase='10.0.1.0/24' )
+    MininetServer.updateHostsFiles( net.hosts )
+    # addNAT().configDefault() also connects root namespace to Mininet
+    net.addNAT().configDefault()
+    net.start()
+    CLI( net )
+    net.stop()
+
+hosts = { 'mn': MininetServer }
+links = { 'mn': ServerLink }
+
+if __name__ == '__main__':
+    n = 12 if len( argv ) != 2 else int( argv[ 1 ] )
+    test( n )

mininet/clean.py

@@ -25,15 +25,15 @@ def sh( cmd ):
 
 def killprocs( pattern ):
     "Reliably terminate processes matching a pattern (including args)"
-    sh( 'pkill -9 -f %s' % pattern )
-    # Make sure they are gone
     while True:
         try:
-            pids = co( [ 'pgrep', '-f', pattern ] )
+            pids = co( [ 'pgrep', '-f', pattern ] ).split( '\n' )
         except CalledProcessError:
-            pids = ''
+            pids = []
+        # Don't kill init
+        pids = [ pid for pid in pids if pid and pid != '1' ]
         if pids:
-            sh( 'pkill -9 -f %s' % pattern )
+            sh( "kill -9 %s" % ' '.join( pids ) )
             time.sleep( .5 )
         else:
             break

mininet/link.py

@@ -25,7 +25,7 @@ Link: basic link class for creating veth pairs
 """
 
 from mininet.log import info, error, debug
-from mininet.util import makeIntfPair
+from mininet.util import makeIntfPair, quietRun
 import mininet.node
 import re
 
@@ -452,7 +452,8 @@ class Link( object ):
 
     @classmethod
     def makeIntfPair( cls, intfname1, intfname2, addr1=None, addr2=None,
-                      node1=None, node2=None, deleteIntfs=True ):
+                      node1=None, node2=None, deleteIntfs=True,
+                      runCmd=quietRun ):
         """Create pair of interfaces
            intfname1: name for interface 1
            intfname2: name for interface 2
@@ -465,7 +466,7 @@ class Link( object ):
         # Leave this as a class method for now
         assert cls
         return makeIntfPair( intfname1, intfname2, addr1, addr2, node1, node2,
-                             deleteIntfs=deleteIntfs )
+                             deleteIntfs=deleteIntfs, runCmd=runCmd )
 
     def delete( self ):
         "Delete this link"

mininet/net.py

@@ -466,7 +466,7 @@ class Mininet( object ):
     def stopXterms( self ):
         "Kill each xterm."
         for term in self.terms:
-            os.kill( term.pid, signal.SIGKILL )
+            term.send_signal( signal.SIGKILL )
         cleanUpScreens()
 
     def staticArp( self ):

mininet/node.py

@@ -74,9 +74,9 @@ class Node( object ):
 
     portBase = 0  # Nodes always start with eth0/port0, even in OF 1.0
 
-    def __init__( self, name, inNamespace=True, **params ):
+    def __init__( self, name, **params ):
         """name: name of node
-           inNamespace: in network namespace?
+           ns: private namespaces to use ['net','mnt']
            privateDirs: list of private directory strings or tuples
            params: Node parameters (see config() for details)"""
 
@@ -85,7 +85,14 @@ class Node( object ):
 
         self.name = params.get( 'name', name )
         self.privateDirs = params.get( 'privateDirs', [] )
-        self.inNamespace = params.get( 'inNamespace', inNamespace )
+        self.overlayDirs = params.get( 'overlayDirs', [] )
+
+        # Support old inNamespace param
+        self.ns = params.get( 'ns', ( 'net', 'mnt' ) )
+        inNamespace = params.get( 'inNamespace', True )
+        if not inNamespace:
+            self.ns = []
+        self.inNamespace = 'net' in self.ns
 
         # Stash configuration parameters for future reference
         self.params = params
@@ -102,8 +109,9 @@ class Node( object ):
         self.waiting = False
         self.readbuf = ''
 
-        # Start command interpreter shell
+        # Start command interpreter shell and mount any local dirs
         self.startShell()
+        self.mountOverlayDirs()
         self.mountPrivateDirs()
 
     # File descriptor to node mapping support
@@ -120,17 +128,21 @@ class Node( object ):
         node = cls.outToNode.get( fd )
         return node or cls.inToNode.get( fd )
 
+    _marker = re.compile( chr( 1 ) + r'(\d+)\r\n' )
+
     # Command support via shell process in namespace
     def startShell( self, mnopts=None ):
         "Start a shell process for running commands"
         if self.shell:
             error( "%s: shell is already running\n" % self.name )
             return
-        # mnexec: (c)lose descriptors, (d)etach from tty,
-        # (p)rint pid, and run in (n)amespace
-        opts = '-cd' if mnopts is None else mnopts
-        if self.inNamespace:
-            opts += 'n'
+        # mnexec: (c)lose descriptors
+        # (p)rint pid, and run in (n)etwork and (m)ount namespace
+        opts = '-cdp' if mnopts is None else mnopts
+        # Handle additional namespaces if specified
+        nsmap = { 'pid': 'P', 'mnt': 'm', 'net': 'n', 'uts': 'u' }
+        chars = [ nsmap.get( ns, '' ) for ns in self.ns ]
+        opts += ''.join( chars )
         # bash -i: force interactive
         # -s: pass $* to shell, and make process easy to find in ps
         # prompt is set to sentinel chr( 127 )
@@ -157,17 +169,16 @@ class Node( object ):
         self.lastPid = None
         self.readbuf = ''
         # Wait for prompt
-        while True:
-            data = self.read( 1024 )
-            if data[ -1 ] == chr( 127 ):
-                break
-            self.pollOut.poll()
-        self.waiting = False
+        self.waiting = True
+        self.waitOutput()
+        if 'P' in opts:
+            assert self.lastPid is not None
+            self.pid = self.lastPid
         # +m: disable job control notification
         self.cmd( 'unset HISTFILE; stty -echo; set +m' )
 
     def mountPrivateDirs( self ):
-        "mount private directories"
+        "Mount private directories"
         # Avoid expanding a string into a list of chars
         assert not isinstance( self.privateDirs, basestring )
         for directory in self.privateDirs:
@@ -185,13 +196,61 @@ class Node( object ):
                 self.cmd( 'mount -n -t tmpfs tmpfs %s' % directory )
 
     def unmountPrivateDirs( self ):
-        "mount private directories"
+        "Unmount private and overlay directories"
         for directory in self.privateDirs:
             if isinstance( directory, tuple ):
                 self.cmd( 'umount ', directory[ 0 ] )
             else:
                 self.cmd( 'umount ', directory )
 
+    # XXX We should make overlayDirs as consistent as possible
+    # with privateDirs.
+
+    def _overlayFrom( self, entry ):
+        "Helper function: return mountpaint, overlay, tmpfs from entry"
+        if type( entry ) is str:
+            # '/mountpoint'
+            mountpoint, overlay = entry, None
+        elif len( entry ) is 1:
+            # [ '/mountpoint' ]
+            mountpoint, overlay = entry[ 0 ], None
+        else:
+            # [ '/mountpoint', '/overlay' ]
+            mountpoint, overlay = entry
+        tmpfs = None if overlay else '/tmp/%s/%s' % ( self, mountpoint )
+        return mountpoint, overlay, tmpfs
+
+    def mountOverlayDirs( self ):
+        """Mount overlay directories. Overlay directories are similar
+            to private directories except they are copy-on-write copies
+            of directories in the host file system.
+            overlayDirs is of the form ((mountpoint,overlaydir), ...)
+            much like privateDirs. If overlaydir doesn't exist, we
+            mount a tmpfs at the specified mount point."""
+        # Avoid expanding a string into a list of chars
+        assert not isinstance( self.overlayDirs, basestring )
+        for entry in self.overlayDirs:
+            mountpoint, overlay, tmpfs  = self._overlayFrom( entry )
+            # Create tmpfs if overlay dir is not specified
+            if not overlay:
+                overlay = tmpfs
+                self.cmd( 'mkdir -p', overlay )
+                self.cmd( 'mount -t tmpfs tmpfs', overlay )
+            # Mount overlay dir at mount point
+            self.cmd( ( 'mount -t overlayfs -o upperdir=%s,lowerdir=%s'
+                       ' overlayfs %s' ) % ( overlay, mountpoint, mountpoint ) )
+
+    def unmountOverlayDirs( self ):
+        "Unmount overlay directories"
+        for entry in self.overlayDirs:
+            mountpoint, overlay, tmpfs = self._overlayFrom( entry )
+            # Unfortunately these umounts can fail if the mount point
+            # is in use, possibly leaving tmpfs garbage in the root
+            # mount namespace / file system
+            self.cmd( 'umount', mountpoint )
+            if not overlay:
+                self.cmd( 'umount', tmpfs )
+
     def _popen( self, cmd, **params ):
         """Internal method: spawn and return a process
             cmd: command to run (list)
@@ -245,6 +304,7 @@ class Node( object ):
     def terminate( self ):
         "Send kill signal to Node and clean up after it."
         self.unmountPrivateDirs()
+        self.unmountOverlayDirs()
         if self.shell:
             if self.shell.poll() is None:
                 os.killpg( self.shell.pid, signal.SIGHUP )
@@ -293,10 +353,10 @@ class Node( object ):
         self.lastPid = None
         self.waiting = True
 
-    def sendInt( self, intr=chr( 3 ) ):
+    def sendInt( self, signal=signal.SIGINT ):
         "Interrupt running command."
-        debug( 'sendInt: writing chr(%d)\n' % ord( intr ) )
-        self.write( intr )
+        debug( "sending signal %d to pgrp %d" % ( signal, self.pid ) )
+        os.killpg( self.pid, signal )
 
     def monitor( self, timeoutms=None, findPid=True ):
         """Monitor and return the output of a command.
@@ -307,18 +367,18 @@ class Node( object ):
         data = self.read( 1024 )
         pidre = r'\[\d+\] \d+\r\n'
         # Look for PID
-        marker = chr( 1 ) + r'\d+\r\n'
         if findPid and chr( 1 ) in data:
             # suppress the job and PID of a backgrounded command
             if re.findall( pidre, data ):
                 data = re.sub( pidre, '', data )
             # Marker can be read in chunks; continue until all of it is read
-            while not re.findall( marker, data ):
+            while True:
+                markers = self._marker.findall( data )
+                if markers:
+                    self.lastPid = int( markers[ -1 ] )
+                    data = self._marker.sub( '', data )
+                    break
                 data += self.read( 1024 )
-            markers = re.findall( marker, data )
-            if markers:
-                self.lastPid = int( markers[ 0 ][ 1: ] )
-                data = re.sub( marker, '', data )
         # Look for sentinel/EOF
         if len( data ) > 0 and data[ -1 ] == chr( 127 ):
             self.waiting = False
@@ -384,6 +444,7 @@ class Node( object ):
         # Shell requires a string, not a list!
         if defaults.get( 'shell', False ):
             cmd = ' '.join( cmd )
+        debug( cmd, defaults )
         popen = self._popen( cmd, **defaults )
         return popen
 
@@ -858,7 +919,7 @@ class Switch( Node ):
         self.dpid = self.defaultDpid( dpid )
         self.opts = opts
         self.listenPort = listenPort
-        if not self.inNamespace:
+        if 'net' not in self.ns:
             self.controlIntf = Intf( 'lo', self, port=0 )
 
     def defaultDpid( self, dpid=None ):
@@ -1064,10 +1125,9 @@ class OVSSwitch( Switch ):
         version = quietRun( 'ovs-vsctl --version' )
         cls.OVSVersion = findall( r'\d+\.\d+', version )[ 0 ]
 
-    @classmethod
-    def isOldOVS( cls ):
+    def isOldOVS( self ):
         "Is OVS ersion < 1.10?"
-        return ( StrictVersion( cls.OVSVersion ) <
+        return ( StrictVersion( self.OVSVersion ) <
                  StrictVersion( '1.10' ) )
 
     def dpctl( self, *args ):
@@ -1152,7 +1212,7 @@ class OVSSwitch( Switch ):
         "Start up a new OVS OpenFlow switch using ovs-vsctl"
         if self.inNamespace:
             raise Exception(
-                'OVS kernel switch does not work in a namespace' )
+                'OVS kernel switch does not work in a network namespace' )
         int( self.dpid, 16 )  # DPID must be a hex string
         # Command to add interfaces
         intfs = ''.join( ' -- add-port %s %s' % ( self, intf ) +
@@ -1345,7 +1405,7 @@ class Controller( Node ):
        OpenFlow controller."""
 
     def __init__( self, name, inNamespace=False, command='controller',
-                  cargs='-v ptcp:%d', cdir=None, ip="127.0.0.1",
+                  cargs='ptcp:%d', cdir=None, ip="127.0.0.1",
                   port=6633, protocol='tcp', **params ):
         self.command = command
         self.cargs = cargs

mininet/nodelib.py

@@ -4,12 +4,13 @@ Node Library for Mininet
 This contains additional Node types which you may find to be useful.
 """
 
-from mininet.node import Node, Switch
+from mininet.node import Node, Host, Switch
 from mininet.log import info, warn
 from mininet.moduledeps import pathCheck
 from mininet.util import quietRun
 
 import re
+from tempfile import NamedTemporaryFile
 
 class LinuxBridge( Switch ):
     "Linux Bridge (with optional spanning tree)"
@@ -142,3 +143,86 @@ class NAT( Node ):
         # Put the forwarding state back to what it was
         self.cmd( 'sysctl net.ipv4.ip_forward=%s' % self.forwardState )
         super( NAT, self ).terminate()
+
+
+class Server( Host ):
+    """Run sshd in a net/mnt/pid/uts namespace, with private /etc/hosts
+       WARNING!!! KNOWN ISSUES:
+       - control-c does not work in Mininet CLI with pid namespace
+       - xterm does not work from Mininet CLI
+       We may be able to address these issues in the future."""
+
+    ns = [ 'net', 'mnt', 'pid', 'uts' ]
+    overlayDirs = [ '/etc', '/var/run', '/var/log' ]
+    privateDirs = [ '/var/run/sshd', ]
+
+    def __init__( self, *args, **kwargs ):
+        """Add overlay dirs and private dirs, and change permissions
+           ssh: run sshd? (True)"""
+        kwargs.setdefault( 'inNamespace', True )
+        kwargs.setdefault( 'ns', self.ns )
+        kwargs.setdefault( 'privateDirs', self.privateDirs )
+        kwargs.setdefault( 'overlayDirs', self.overlayDirs )
+        kwargs.setdefault( 'ssh', True )
+        super( Server, self ).__init__( *args, **kwargs )
+        # Change permissions, mainly for ssh
+        for pdir in self.privateDirs:
+            self.cmd( 'chown root:root', pdir )
+            self.cmd( 'chmod 755', pdir )
+
+    @staticmethod
+    def updateHostsFiles( servers, tmpdir='/tmp' ):
+        """Update local hosts files on a list of servers
+            servers: list of servers
+            tmpdir: tmp dir shared between mn and servers"""
+        # This scales as n^2, so for a large configuration it's
+        # more efficient to use a DNS server
+        for s in servers:
+            dirs = ( getattr( s, 'overlayDirs', [] ) +
+            getattr( s, 'privateDirs', [] ))
+            if '/etc' in dirs:
+                with NamedTemporaryFile( dir=tmpdir ) as tmpfile:
+                      tmpfile.write( '# Mininet hosts file\n' )
+                      tmpfile.write( '127.0.0.1 localhost %s\n' % s )
+                      for t  in servers:
+                        tmpfile.write( '%s %s\n' % ( t.IP(), t ) )
+                      tmpfile.flush()
+                      s.cmd( 'cp', tmpfile.name, '/etc/hosts' )
+            else:
+                warn( 'not updating hosts file on %s\n' % s )
+
+    def service( self, cmd ):
+        """Start or stop a service
+           usage: service( 'ssh stop' )"""
+        self.cmd( '/etc/init.d/%s' % cmd )
+
+    def motd( self ):
+        "Return login message as a string"
+        return 'Welcome to Mininet host %s at %s' % ( self, self.IP() )
+
+    def startSSH( self, motdPath='/var/run/motd.dynamic' ):
+        "Update motd, clear out utmp/wtmp/btmp, and start sshd"
+        # Note: /var/run and /var/log must be overlays!
+        assert ( '/var/run' in ( self.overlayDirs + self.privateDirs ) and
+                 '/var/log' in ( self.overlayDirs + self.privateDirs ) )
+        self.cmd( "echo  '%s' > %s" % ( self.motd(), motdPath ) )
+        self.cmd( 'truncate -s0 /var/run/utmp /var/log/wtmp* /var/log/btmp*' )
+        # sshd.pid should really be in /var/run/sshd instead of /var/run
+        self.cmd( 'rm /var/run/sshd.pid' )
+        self.cmd( '/etc/init.d/ssh start' )
+
+    def config( self, **kwargs ):
+        """Configure/start sshd and other stuff
+            ssh: start sshd? (True )"""
+        super( Server, self ).config( **kwargs )
+        self.ssh = kwargs.get( 'ssh' )
+        if self.ssh:
+            self.startSSH()
+        if 'uts' in self.ns:
+            self.cmd( 'hostname', self )
+
+    def terminate( self, *args, **kwargs ):
+        "Shut down services and terminate server"
+        if self.ssh:
+            self.service( 'ssh stop' )
+        super( Server, self ).terminate( *args, **kwargs )

mininet/term.py

@@ -5,12 +5,13 @@ Utility functions to run a terminal (connected via socat(1)) on each host.
 Requires socat(1) and xterm(1).
 Optionally uses gnome-terminal.
 """
-
-from os import environ
-
 from mininet.log import error
 from mininet.util import quietRun, errRun
 
+from os import environ, getpid, path
+from subprocess import Popen
+from tempfile import NamedTemporaryFile
+
 def tunnelX11( node, display=None):
     """Create an X11 tunnel from node:6000 to the root host
        display: display on root host (optional)
@@ -28,13 +29,64 @@ def tunnelX11( node, display=None):
         quietRun( 'xhost +si:localuser:root' )
         return display, None
     else:
-        # Create a tunnel for the TCP connection
+        # XXX Need to handle case where node is in UTS namespace
+        # in this case, we need to set XAUTHORITY to a private
+        # xauth file
+        if False and 'uts' in node.inNamespace and  not hasattr( node, 'xauthFile' ):
+            node.xauthFile = NamedTemporaryFile()
+            quietRun( 'xauth extract $DISPLAY | xauth -f %s merge' % node.xauthFile.name )
         port = 6000 + int( float( screen ) )
-        connection = r'TCP\:%s\:%s' % ( host, port )
-        cmd = [ "socat", "TCP-LISTEN:%d,fork,reuseaddr" % port,
-                "EXEC:'mnexec -a 1 socat STDIO %s'" % connection ]
+        # This can conflict if we are running nested Mininet
+        # in a pid namespace
+        socket = '/tmp/mininet.x11.%d' % getpid()
+        if not path.exists( socket ):
+            cmd = 'socat unix-listen:%s,fork tcp:localhost:%d' % ( socket, port  )
+            # Should be shut down when mn shuts down
+            tunnelX11.socket = Popen( cmd, shell=True )
+        # Create a tunnel for the TCP connection
+        cmd = 'socat tcp-listen:%d,fork,reuseaddr unix:%s' % ( port, socket )
+
     return 'localhost:' + screen, node.popen( cmd )
 
+"""
+
+With pid namespaces, we can't easily escape the pid jail using mnexec
+(or can we?)
+
+What we can do, however, is create a unix socket in /tmp which connects
+to our x server, and a tcp listener in the host that connects to our
+unix socket!
+
+We just have to make sure that we clean up our processes and sockets
+when we quit.
+
+If we're using UTS namespaces, then xauth will get confused because
+our hostname has changed. So, we use a private XAUTHORITY file per
+host. We need to initialize this file with the key of our X server;
+this is a bit hard to figure out because the usual xauth list $DISPLAY
+may fail even if xlib can figure out a backup cookie to use.
+
+But what about namespace conflicts? This could certainly be very
+annoying for nested mininet!! In this case, our nested mininet servers
+should have a private /tmp that they can use.... except that conflicts
+with the shared unix domain socket in /tmp! ;-p
+
+We could also use a random name for the socket, to avoid the namespace
+conflict, although it's not clear what to do for cleanup to avoid
+blasting this....
+
+Perhaps the best idea is to use a canonical name (mininet.x11) and if the
+first name fails, try a second name??
+
+Another idea is to create a tmp dir for Mininet based on the pid of the
+mn process.....
+
+recommendation: for now, use mininet.x11.1234 as the socket.
+
+"""
+
+
+
 def makeTerm( node, title='Node', term='xterm', display=None, cmd='bash'):
     """Create an X11 tunnel to the node and start up a terminal.
        node: Node object

mininet/util.py

@@ -177,18 +177,19 @@ def makeIntfPair( intf1, intf2, addr1=None, addr2=None, node1=None, node2=None,
         runCmd( 'ip link del ' + intf1 )
         runCmd2( 'ip link del ' + intf2 )
     # Create new pair
-    netns = 1 if not node2 else node2.pid
+    ns1 = 1 if not node1 else node1.pid
+    ns2 = 1 if not node2 else node2.pid
     if addr1 is None and addr2 is None:
-        cmdOutput = runCmd( 'ip link add name %s '
+        cmdOutput = runCmd( 'ip link add name %s netns %s '
                             'type veth peer name %s '
-                            'netns %s' % ( intf1, intf2, netns ) )
+                            'netns %s' % ( intf1, ns1, intf2, ns2 ) )
     else:
         cmdOutput = runCmd( 'ip link add name %s '
-                            'address %s '
+                            'address %s netns %s '
                             'type veth peer name %s '
                             'address %s '
                             'netns %s' %
-                            (  intf1, addr1, intf2, addr2, netns ) )
+                            (  intf1, addr1, ns1, intf2, addr2, ns2 ) )
     if cmdOutput:
         raise Exception( "Error creating interface pair (%s,%s): %s " %
                          ( intf1, intf2, cmdOutput ) )
@@ -507,21 +508,42 @@ def custom( cls, **params ):
     customized.__name__ = 'custom(%s,%s)' % ( cls, params )
     return customized
 
-def splitArgs( argstr ):
-    """Split argument string into usable python arguments
-       argstr: argument string with format fn,arg2,kw1=arg3...
-       returns: fn, args, kwargs"""
-    split = argstr.split( ',' )
-    fn = split[ 0 ]
-    params = split[ 1: ]
+
+
+def parseArgs( argstr ):
+    """Parse argument string
+       returns args, kwargs"""
+    # One step at a time: support param=[a,b,c]
+    paramre = r'\[[^\]]*\]|[^,\[\]]+'
+    paramre = r'\w+=\[[^\]]*\]|\w+\[^,\[\]]+' + paramre
+    params = re.findall( paramre, argstr )
+    # Parse lists
+    for i, arg in enumerate( params ):
+        if arg.startswith( '[' ) and arg.endswith( ']' ):
+            arg = arg[ 1 : -1 ]
+            print "recurse on", arg
+            params[ i ] = parseArgs( args )
     # Convert int and float args; removes the need for function
     # to be flexible with input arg formats.
     args = [ makeNumeric( s ) for s in params if '=' not in s ]
     kwargs = {}
     for s in [ p for p in params if '=' in p ]:
-        key, val = s.split( '=', 1 )
-        kwargs[ key ] = makeNumeric( val )
-    return fn, args, kwargs
+        key, arg = s.split( '=', 1 )
+        if arg.startswith( '[' ) and arg.endswith( ']' ):
+            arg = arg[ 1 : -1 ]
+            print 'recurse on', arg
+            arg = parseArgs( arg )[ 0 ]
+        else:
+            arg = makeNumeric( arg )
+        kwargs[ key ] = arg
+    return args, kwargs
+
+def splitArgs( argstr ):
+    """Split argument string into usable python arguments
+       argstr: argument string with format fn,arg2,kw1=arg3...
+       returns: fn, args, kwargs"""
+    args, kwargs = parseArgs( argstr )
+    return args[ 0 ], args[ 1: ], kwargs
 
 def customClass( classes, argStr ):
     """Return customized class based on argStr

mnexec.c

@@ -5,9 +5,9 @@
  *
  *  - closing all file descriptors except stdin/out/error
  *  - detaching from a controlling tty using setsid
- *  - running in network and mount namespaces
+ *  - running in network and other namespaces
  *  - printing out the pid of a process so we can identify it later
- *  - attaching to a namespace and cgroup
+ *  - attaching to namespace(s) and cgroup
  *  - setting RT scheduling
  *
  * Partially based on public domain setsid(1)
@@ -24,21 +24,26 @@
 #include <sched.h>
 #include <ctype.h>
 #include <sys/mount.h>
+#include <sys/wait.h>
 
 #if !defined(VERSION)
 #define VERSION "(devel)"
 #endif
 
+
 void usage(char *name)
 {
     printf("Execution utility for Mininet\n\n"
-           "Usage: %s [-cdnp] [-a pid] [-g group] [-r rtprio] cmd args...\n\n"
+           "Usage: %s [-cdmnPpu] [-a pid] [-g group] [-r rtprio] cmd args...\n\n"
            "Options:\n"
            "  -c: close all file descriptors except stdin/out/error\n"
            "  -d: detach from tty by calling setsid()\n"
-           "  -n: run in new network and mount namespaces\n"
+           "  -m: run in a new mount namespace\n"
+           "  -n: run in a new network namespace\n"
+           "  -P: run in a new pid namespace\n"
+           "  -u: run in a new UTS (ipc, hostname) namespace\n"
            "  -p: print ^A + pid\n"
-           "  -a pid: attach to pid's network and mount namespaces\n"
+           "  -a pid: attach to pid's namespaces\n"
            "  -g group: add to cgroup\n"
            "  -r rtprio: run with SCHED_RR (usually requires -g)\n"
            "  -v: print version\n",
@@ -51,6 +56,7 @@ int setns(int fd, int nstype)
     return syscall(__NR_setns, fd, nstype);
 }
 
+
 /* Validate alphanumeric path foo1/bar2/baz */
 void validate(char *path)
 {
@@ -63,6 +69,7 @@ void validate(char *path)
     }
 }
 
+
 /* Add our pid to cgroup */
 void cgroup(char *gname)
 {
@@ -92,107 +99,181 @@ void cgroup(char *gname)
     }
 }
 
+/* Attach to ns 'name' if present */
+int attachns( pid_t pid, char *name ) {
+    char path[PATH_MAX];
+    int nsid;
+
+    sprintf(path, "/proc/%d/ns/%s", pid, name) ;
+
+    if ((nsid = open(path, O_RDONLY)) < 0)
+        return nsid;
+
+    if (setns(nsid, 0) != 0) {
+        perror("setns");
+        return 1;
+    }
+
+    return 0;
+}
+
+/* Attach to pid's namespaces - returns true if pidns */
+int attach(int pid) {
+
+    char *cwd = get_current_dir_name();
+    char path[PATH_MAX];
+    int pidns = 0;
+
+    attachns(pid, "net");
+    attachns(pid, "uts");
+    if ( attachns(pid, "pid") == 0 )
+        pidns = 1;
+
+    if (attachns(pid, "mnt") != 0) {
+        /* Plan B: chroot into pid's root file system */
+        sprintf(path, "/proc/%d/root", pid);
+        if (chroot(path) < 0) {
+            perror(path);
+        }
+    }
+
+    /* chdir to correct working directory */
+    if (chdir(cwd) != 0) {
+        perror(cwd);
+    }
+
+    return pidns;
+}
+
+
 int main(int argc, char *argv[])
 {
     int c;
-    int fd;
-    char path[PATH_MAX];
-    int nsid;
-    int pid;
-    char *cwd = get_current_dir_name();
 
-    static struct sched_param sp;
-    while ((c = getopt(argc, argv, "+cdnpa:g:r:vh")) != -1)
+    /* Argument flags */
+    int flags = 0;
+    int closefds = 0;
+    int attachpid = 0;
+    char *cgrouparg = NULL;
+    int detachtty = 0;
+    int printpid = 0;
+    int rtprio = 0;
+    int pidns = 0;
+
+    while ((c = getopt(argc, argv, "+cdmnPpa:g:r:uvh")) != -1)
         switch(c) {
-        case 'c':
-            /* close file descriptors except stdin/out/error */
-            for (fd = getdtablesize(); fd > 2; fd--)
-                close(fd);
-            break;
-        case 'd':
-            /* detach from tty */
-            if (getpgrp() == getpid()) {
-                switch(fork()) {
-                    case -1:
-                        perror("fork");
-                        return 1;
-                    case 0:     /* child */
-                        break;
-                    default:    /* parent */
-                        return 0;
-                }
-            }
-            setsid();
-            break;
-        case 'n':
-            /* run in network and mount namespaces */
-            if (unshare(CLONE_NEWNET|CLONE_NEWNS) == -1) {
-                perror("unshare");
+            case 'c': closefds = 1; break;
+            case 'd':   detachtty = 1; break;
+            case 'm':   flags |= CLONE_NEWNS; break;
+            case 'n':   flags |= CLONE_NEWNET; break;
+            case 'p':   printpid = 1; break;
+            case 'P':   flags |= CLONE_NEWPID; break;
+            case 'a':   attachpid = atoi(optarg);break;
+            case 'g':   cgrouparg = optarg ; break;
+            case 'r':   rtprio = atoi(optarg); break;
+            case 'u':   flags |= CLONE_NEWUTS; break;
+            case 'v':   printf("%s\n", VERSION); exit(0);
+            case 'h':   usage(argv[0]); exit(0);
+            default:    usage(argv[0]); exit(1);
+    }
+
+    if (closefds) {
+        /* close file descriptors except stdin/out/error */
+        int fd;
+        for (fd = getdtablesize(); fd > 2; fd--) close(fd);
+    }
+
+    /* XXX We should not fork twice if we don't need to!! */
+    if (detachtty) {
+        /* detach from tty */
+        if (getpgrp() == getpid()) {
+            switch(fork()) {
+            case -1:
+                perror("fork");
                 return 1;
+            case 0:     /* child */
+                break;
+            default:    /* parent */
+                return 0;
             }
-            /* mount sysfs to pick up the new network namespace */
-            if (mount("sysfs", "/sys", "sysfs", MS_MGC_VAL, NULL) == -1) {
-                perror("mount");
-                return 1;
-            }
-            break;
-        case 'p':
-            /* print pid */
-            printf("\001%d\n", getpid());
-            fflush(stdout);
-            break;
-        case 'a':
-            /* Attach to pid's network namespace and mount namespace */
-            pid = atoi(optarg);
-            sprintf(path, "/proc/%d/ns/net", pid);
-            nsid = open(path, O_RDONLY);
-            if (nsid < 0) {
-                perror(path);
-                return 1;
-            }
-            if (setns(nsid, 0) != 0) {
-                perror("setns");
-                return 1;
-            }
-            /* Plan A: call setns() to attach to mount namespace */
-            sprintf(path, "/proc/%d/ns/mnt", pid);
-            nsid = open(path, O_RDONLY);
-            if (nsid < 0 || setns(nsid, 0) != 0) {
-                /* Plan B: chroot/chdir into pid's root file system */
-                sprintf(path, "/proc/%d/root", pid);
-                if (chroot(path) < 0) {
-                    perror(path);
-                    return 1;
-                }
-            }
-            /* chdir to correct working directory */
-            if (chdir(cwd) != 0) {
-                perror(cwd);
-                return 1;
-            }
-            break;
-        case 'g':
-            /* Attach to cgroup */
-            cgroup(optarg);
-            break;
-        case 'r':
-            /* Set RT scheduling priority */
-            sp.sched_priority = atoi(optarg);
-            if (sched_setscheduler(getpid(), SCHED_RR, &sp) < 0) {
-                perror("sched_setscheduler");
-                return 1;
-            }
-            break;
-        case 'v':
-            printf("%s\n", VERSION);
-            exit(0);
-        case 'h':
-            usage(argv[0]);
-            exit(0);
-        default:
-            usage(argv[0]);
-            exit(1);
         }
+        setsid();
+    }
+
+    if (attachpid) {
+        /* Attach to existing namespace(s) */
+      pidns = attach(attachpid);
+    }
+    else {
+        /* Create new namespace(s) */
+        if (unshare(flags) == -1) {
+            perror("unshare");
+            return 1;
+        }
+    }
+
+    /* Use a new process group so we can use killpg */
+    setpgid( 0, 0 );
+
+    if ( flags & CLONE_NEWPID || pidns ) {
+        /* For pid namespace, we need to fork and wait for child ;-( */
+        pid_t pid = fork();
+        switch(pid) {
+            int status;
+            case -1:
+                perror("fork");
+                return 1;
+            case 0:
+                /* child continues below */
+                break;
+            default:
+                /* We print the *child pid* if needed */
+                if (printpid) {
+                    printf("\001%d\n", pid);
+                    fflush(stdout);
+                }
+                /* Parent needs to wait for child and exit */
+                wait(&status);
+                return 0;
+        }
+    }
+
+    else if (printpid) {
+        /* If we're in a pid namespace, parent prints our pid instead */
+        printf("\001%d\n", getpid());
+        fflush(stdout);
+    }
+
+
+    if (flags & CLONE_NEWNS && !attachpid) {
+        /* Child remounts /proc for ps */
+        if (mount("proc", "/proc", "proc", MS_MGC_VAL, NULL) == -1) {
+            perror("mountproc");
+        }
+    }
+
+    if (cgrouparg) {
+        /* Attach to cgroup */
+        cgroup(cgrouparg);
+    }
+
+    if (flags & CLONE_NEWNET & CLONE_NEWNS) {
+        /* Mount sysfs to pick up the new network namespace */
+        if (mount("sysfs", "/sys", "sysfs", MS_MGC_VAL, NULL) == -1) {
+            perror("mount");
+            return 1;
+        }
+    }
+
+    if (rtprio != 0) {
+        /* Set RT scheduling priority */
+        static struct sched_param sp;
+        sp.sched_priority = atoi(optarg);
+        if (sched_setscheduler(getpid(), SCHED_RR, &sp) < 0) {
+            perror("sched_setscheduler");
+            return 1;
+        }
+    }
 
     if (optind < argc) {
         execvp(argv[optind], &argv[optind]);

util/clustersetup.sh

@@ -31,7 +31,7 @@ usage="./clustersetup.sh [ -p|h|c ] [ host1 ] [ host2 ] ...\n
         via ssh for mininet cluster edition. By default, we use a
         temporary ssh setup. An ssh directory is mounted over
         $USERDIR on each machine in the cluster.
-        
+
                 -h: display this help
                 -p: create a persistent ssh setup. This will add
                     new ssh keys and known_hosts to each nodes
@@ -40,13 +40,25 @@ usage="./clustersetup.sh [ -p|h|c ] [ host1 ] [ host2 ] ...\n
                     Any hosts taken as arguments will be cleaned
         "
 
+
 persistentSetup() {
     echo "***creating key pair"
-    ssh-keygen -t rsa -C "Cluster_Edition_Key" -f $USERDIR/cluster_key -N '' &> /dev/null
-    cat $USERDIR/cluster_key.pub >> $USERDIR/authorized_keys
+    if ! ssh-keygen -t rsa -C "Cluster_Edition_Key" -f $USERDIR/cluster_key -N ''; then
+        echo "key generation returned error" $?
+    fi
+
+    echo "***adding to authorized_keys"
+    if ! grep -f $USERDIR/cluster_key.pub $USERDIR/authorized_keys > /dev/null; then
+        cat $USERDIR/cluster_key.pub >> $USERDIR/authorized_keys
+    else
+        echo "$USERDIR/cluster_key.pub already in $USERDIR/authorized_keys"
+    fi
+
     echo "***configuring ssh"
-    echo "IdentityFile $USERDIR/cluster_key" >> $USERDIR/config
-    echo "IdentityFile $USERDIR/id_rsa" >> $USERDIR/config
+    if ! grep cluster_key $USERDIR/config; then
+        echo "IdentityFile $USERDIR/cluster_key" >> $USERDIR/config
+        echo "IdentityFile $USERDIR/id_rsa" >> $USERDIR/config
+    fi
 
     for host in $hosts; do
         echo "***copying public key to $host"
@@ -56,23 +68,30 @@ persistentSetup() {
         scp $USERDIR/cluster_key.pub $user@$host:$USERDIR
         echo "***configuring remote host"
         ssh -o ForwardAgent=yes  $user@$host "
-        echo 'IdentityFile $USERDIR/cluster_key' >> $USERDIR/config
-        echo 'IdentityFile $USERDIR/id_rsa' >> $USERDIR/config"
+        if ! grep cluster_key $USERDIR/config > /dev/null; then
+            echo 'IdentityFile $USERDIR/cluster_key' >> $USERDIR/config
+            echo 'IdentityFile $USERDIR/id_rsa' >> $USERDIR/config;
+        fi"
     done
 
     for host in $hosts; do
         echo "***copying known_hosts to $host"
         scp $USERDIR/known_hosts $user@$host:$USERDIR/cluster_known_hosts
+        echo "***appending"
         ssh $user@$host "
-        cat $USERDIR/cluster_known_hosts >> $USERDIR/known_hosts
+        if ! grep -f $USERDIR/cluster_known_hosts $USERDIR/known_hosts > /dev/null; then
+            cat $USERDIR/cluster_known_hosts >> $USERDIR/known_hosts;
+        fi;
         rm $USERDIR/cluster_known_hosts"
     done
+
+    echo "***done with persistent setup"
 }
 
 tempSetup() {
-    
+
     echo "***creating temporary ssh directory"
-    mkdir -p $SSHDIR 
+    mkdir -p $SSHDIR
     echo "***creating key pair"
     ssh-keygen -t rsa -C "Cluster_Edition_Key" -f $SSHDIR/id_rsa -N '' &> /dev/null
 
@@ -99,7 +118,7 @@ tempSetup() {
 }
 
 cleanup() {
-    
+
     for host in $hosts; do
     echo "***cleaning up $host"
     ssh $user@$host "sudo umount $USERDIR
@@ -145,9 +164,8 @@ if $showHelp; then
 fi
 
 for i in "$@"; do
-    output=$(getent ahostsv4 "$i")
-    if [ -z "$output" ]; then
-        echo '***WARNING: could not find hostname "$i"'
+    if ! getent ahostsv4 $i > /dev/null; then
+        echo "***WARNING: could not find hostname $i"
         echo ""
     else
         hosts+="$i "

util/m

@@ -40,5 +40,5 @@ if [ -d $rootdir -a -x $rootdir/bin/bash ]; then
     cmd="chroot $rootdir /bin/bash -c $cmd"
 fi
 
-cmd="exec sudo mnexec $cg -a $pid $cmd"
+cmd="exec sudo -E mnexec $cg -a $pid $cmd"
 eval $cmd