Clean up makeIntfPair to prepare for pid namespaces, chroot

Note that we can't actually use ip netns del to remove the namespaces
untill ALL of the processes are gone, necessitating this cleanup.

mininet/clean.py

@@ -10,24 +10,30 @@ It may also get rid of 'false positives', but hopefully
 nothing irreplaceable!
 """
 
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, STDOUT, check_output as co
+from sys import stdout, exit
+from time import sleep
 
-from mininet.log import info
+from mininet.log import info, error
 from mininet.term import cleanUpScreens
 
 def sh( cmd ):
-    "Print a command and send it to the shell"
+    "Run a command in the shell and return non-empty output lines"
     info( cmd + '\n' )
-    return Popen( [ '/bin/sh', '-c', cmd ], stdout=PIPE ).communicate()[ 0 ]
+    output = ( Popen( [ '/bin/sh', '-c', cmd ], stdout=PIPE )
+            .communicate()[ 0 ]
+            .strip()
+            .split( '\n' ) )
+    return [ s for s in output if s ]
 
 def cleanup():
     """Clean up junk which might be left over from old runs;
        do fast stuff before slow dp and link removal!"""
 
-    info("*** Removing excess controllers/ofprotocols/ofdatapaths/pings/noxes"
+    info( "*** Removing excess "
-         "\n")
+          "controllers/ofprotocols/ofdatapaths/pings/noxes\n" )
     zombies = 'controller ofprotocol ofdatapath ping nox_core lt-nox_core '
-    zombies += 'ovs-openflowd udpbwtest mnexec'
+    zombies += 'ovs-openflowd ovs-controller udpbwtest mnexec'
     # Note: real zombie processes can't actually be killed, since they
     # are already (un)dead. Then again,
     # you can't connect to them either, so they're mostly harmless.
@@ -43,21 +49,44 @@ def cleanup():
     cleanUpScreens()
 
     info( "*** Removing excess kernel datapaths\n" )
-    dps = sh( "ps ax | egrep -o 'dp[0-9]+' | sed 's/dp/nl:/'" ).split( '\n' )
+    dps = sh( "ps ax | egrep -o 'dp[0-9]+' | sed 's/dp/nl:/'" )
     for dp in dps:
-        if dp != '':
+        sh( 'dpctl deldp ' + dp )
-            sh( 'dpctl deldp ' + dp )
 
-    info( "***  Removing OVS datapaths" )
+    info( "*** Removing OVS datapaths\n" )
-    dps = sh("ovs-vsctl list-br").split( '\n' )
+    dps = sh("ovs-vsctl list-br")
     for dp in dps:
-        if dp:
+        sh( 'ovs-vsctl del-br ' + dp )
-            sh( 'ovs-vsctl del-br ' + dp )
+    if co( 'ovs-vsctl list-br', shell=True ):
+        raise Excpetion( "Error: could not remove all OVS datapaths" )
 
     info( "*** Removing all links of the pattern foo-ethX\n" )
-    links = sh( "ip link show | egrep -o '(\w+-eth\w+)'" ).split( '\n' )
+    links = sh( "ip link show | egrep -o '(\w+-eth\w+)'" )
     for link in links:
-        if link != '':
+        sh( "ip link del " + link )
-            sh( "ip link del " + link )
+    if sh( "ip link show | egrep -o '(\w+-eth\w+)'" ):
+        raise Exception( "Error could not remove stale links")
+
+    info( "*** Killing stale mininet node processes\n" )
+    sh( 'pkill -9 -f mininet:' )
+    # Make sure they are gone
+    while True:
+        try:
+            pids = co( 'pgrep -f mininet:'.split() )
+        except:
+            pids = ''
+        if pids:
+            sh( 'pkill -f 9 mininet:' )
+            sleep( .5 )
+        else:
+            break
+
+    info( "*** Removing stale namespaces\n" )
+    nses = sh( "ip netns list" )
+    for ns in nses:
+        sh( "ip netns del " + ns )
+    if co( "ip netns list", shell=True ):
+        error( "Error: could not remove all namespaces - exiting\n" )
+        exit( 1 )
 
     info( "*** Cleanup complete.\n" )

mininet/link.py

@@ -25,7 +25,7 @@ Link: basic link class for creating veth pairs
 """
 
 from mininet.log import info, error, debug
-from mininet.util import makeIntfPair
+from mininet.util import makeIntfPair, errFail, quietRun
 from time import sleep
 import re
 
@@ -109,7 +109,7 @@ class Intf( object ):
     def rename( self, newname ):
         "Rename interface"
         self.ifconfig( 'down' )
-        result = self.cmd( 'ip link set', self.name, 'name', newname )
+        result = self.cmd( 'ip link set dev', self.name, 'name', newname )
         self.name = newname
         self.ifconfig( 'up' )
         return result
@@ -347,7 +347,7 @@ class Link( object ):
         if not intfName2:
             intfName2 = self.intfName( node2, port2 )
 
-        self.makeIntfPair( intfName1, intfName2 )
+        self.makeIntfPair( intfName1, intfName2, node1, node2 )
 
         if not cls1:
             cls1 = intf
@@ -372,13 +372,24 @@ class Link( object ):
         return node.name + '-eth' + repr( n )
 
     @classmethod
-    def makeIntfPair( cls, intf1, intf2 ):
+    def makeIntfPair( cls, intf1, intf2, node1=None, node2=None ):
         """Create pair of interfaces
            intf1: name of interface 1
            intf2: name of interface 2
            (override this class method [and possibly delete()]
            to change link type)"""
-        makeIntfPair( intf1, intf2  )
+        # To be compatible with pid namespaces and chroot in the future
+        # we create links in the root namespace and then move
+        # the ends as needed.
+        # First, make sure there aren't stale links sitting around
+        quietRun( 'ip link delete %s type veth' % intf1 )
+        quietRun( 'ip link delete %s type veth' % intf2 )
+        cmd = 'ip link add %s type veth peer name %s' % ( intf1, intf2 )
+        if node2 and node2.inNamespace:
+            cmd += ' netns %s' % node2
+        errFail( cmd )
+        if node1 and node1.inNamespace:
+            errFail( 'ip link set dev %s netns %s' % ( intf1, node1 ) )
 
     def delete( self ):
         "Delete this link"
@@ -388,6 +399,7 @@ class Link( object ):
     def __str__( self ):
         return '%s<->%s' % ( self.intf1, self.intf2 )
 
+
 class TCLink( Link ):
     "Link with symmetric TC interfaces configured via opts"
     def __init__( self, node1, node2, port1=None, port2=None,

mininet/net.py

@@ -97,7 +97,7 @@ from mininet.cli import CLI
 from mininet.log import info, error, debug, output
 from mininet.node import Host, OVSKernelSwitch, Controller
 from mininet.link import Link, Intf
-from mininet.util import quietRun, fixLimits, numCores, ensureRoot
+from mininet.util import quietRun, errFail, fixLimits, numCores, ensureRoot
 from mininet.util import macColonHex, ipStr, ipParse, netParse, ipAdd
 from mininet.term import cleanUpScreens, makeTerms
 
@@ -152,7 +152,8 @@ class Mininet( object ):
         self.hosts = []
         self.switches = []
         self.controllers = []
-
+        self.links = []
+        
         self.nameToNode = {}  # name to Node (Host/Switch) objects
 
         self.terms = []  # list of spawned xterm processes
@@ -252,7 +253,9 @@ class Mininet( object ):
         defaults.update( params )
         if not cls:
             cls = self.link
-        return cls( node1, node2, **defaults )
+        link = cls( node1, node2, **defaults )
+        self.links.append( link )
+        return link
 
     def configHosts( self ):
         "Configure a set of hosts."
@@ -372,20 +375,34 @@ class Mininet( object ):
         if self.terms:
             info( '*** Stopping %i terms\n' % len( self.terms ) )
             self.stopXterms()
+        info( '*** Stopping %i controllers\n' % len( self.controllers ) )
+        for controller in self.controllers:
+            info( controller.name + ' ' )
+            controller.stop()
+        info( '\n' )
+        info( '*** Stopping %i switches\n' % len( self.switches ) )
+        for switch in self.switches:
+            info( switch.name + ' ' )
+            switch.stop( deleteIntfs=False )
+        info( '\n' )
+        info( '*** Removing links\n' )
+        for link in self.links:
+            info( '.' )
+            link.delete()
+        info( '\n*** Terminating switches\n' )
+        for switch in self.switches:
+            info( '.' )
+            switch.terminate()
+        info( '\n' )
         info( '*** Stopping %i hosts\n' % len( self.hosts ) )
         for host in self.hosts:
             info( host.name + ' ' )
             host.terminate()
         info( '\n' )
-        info( '*** Stopping %i switches\n' % len( self.switches ) )
+        nses = quietRun( 'ip netns list').strip().split()
-        for switch in self.switches:
+        info( '*** Removing namespaces' )
-            info( switch.name + ' ' )
+        for ns in nses:
-            switch.stop()
+            errFail( 'ip netns del ' + ns )
-        info( '\n' )
-        info( '*** Stopping %i controllers\n' % len( self.controllers ) )
-        for controller in self.controllers:
-            info( controller.name + ' ' )
-            controller.stop()
         info( '\n*** Done\n' )
 
     def run( self, test, *args, **kwargs ):

mininet/node.py

@@ -49,6 +49,8 @@ import re
 import signal
 import select
 from subprocess import Popen, PIPE, STDOUT
+from sys import stdout
+from time import sleep
 
 from mininet.log import info, error, warn, debug
 from mininet.util import ( quietRun, errRun, errFail, moveIntf, isShellBuiltin,
@@ -107,20 +109,26 @@ class Node( object ):
 
     # Command support via shell process in namespace
 
+    def _popen( self, *args, **kwargs ):
+        "Internal wrapper for Popen"
+        old = signal.signal( signal.SIGINT, signal.SIG_IGN )
+        result = Popen( *args, **kwargs )
+        signal.signal( signal.SIGINT, old )
+        return result
+
     def startShell( self ):
         "Start a shell process for running commands"
         if self.shell:
             error( "%s: shell is already running" )
             return
-        # mnexec: (c)lose descriptors, (d)etach from tty,
-        # (p)rint pid, and run in (n)amespace
-        opts = '-cdp'
-        if self.inNamespace:
-            opts += 'n'
         # bash -m: enable job control
         # -s: pass $* to shell, and make process easy to find in ps
-        cmd = [ 'mnexec', opts, 'bash', '-ms', 'mininet:' + self.name ]
+        cmd = [ 'bash', '-ms', 'mininet:' + self.name ]
-        self.shell = Popen( cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT,
+        if self.inNamespace:
+            quietRun( 'ip netns del ' + self.name )
+            errFail( 'ip netns add ' + self.name )
+            cmd = [ 'ip', 'netns', 'exec', self.name ] + cmd
+        self.shell = self._popen( cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT,
                             close_fds=True )
         self.stdin = self.shell.stdin
         self.stdout = self.shell.stdout
@@ -140,11 +148,14 @@ class Node( object ):
 
     def cleanup( self ):
         "Help python collect its garbage."
-        if not self.inNamespace:
+        if self.shell:
-            for intfName in self.intfNames():
+            self.shell.terminate()
-                if self.name in intfName:
+            self.shell.wait()
-                    quietRun( 'ip link del ' + intfName )
+            self.shell = None
-        self.shell = None
+        if self.inNamespace:
+            # Note: this will only work if there are no other
+            # processes running in namespaces, but we'll do it anyway
+            quietRun( 'ip netns del ' + self.name )
 
     # Subshell I/O, commands and control
 
@@ -312,7 +323,7 @@ class Node( object ):
         # Shell requires a string, not a list!
         if defaults.get( 'shell', False ):
             cmd = ' '.join( cmd )
-        return Popen( cmd, **defaults )
+        return self._popen( cmd, **defaults )
 
     def pexec( self, *args, **kwargs ):
         """Execute a command using popen
@@ -346,10 +357,8 @@ class Node( object ):
         self.ports[ intf ] = port
         self.nameToIntf[ intf.name ] = intf
         debug( '\n' )
+        assert intf.name in self.cmd( 'ip link show' )
         debug( 'added intf %s:%d to node %s\n' % ( intf, port, self.name ) )
-        if self.inNamespace:
-            debug( 'moving', intf, 'into namespace for', self.name, '\n' )
-            moveIntf( intf.name, self )
 
     def defaultIntf( self ):
         "Return interface for lowest port"
@@ -400,6 +409,7 @@ class Node( object ):
         for intf in self.intfs.values():
             intf.delete()
             info( '.' )
+            stdout.flush()
 
     # Routing support
 
@@ -835,11 +845,12 @@ class UserSwitch( Switch ):
                   ' --fail=closed ' + self.opts +
                   ' 1> ' + ofplog + ' 2>' + ofplog + ' &' )
 
-    def stop( self ):
+    def stop( self, deleteIntfs=True ):
         "Stop OpenFlow reference user datapath."
         self.cmd( 'kill %ofdatapath' )
         self.cmd( 'kill %ofprotocol' )
-        self.deleteIntfs()
+        if deleteIntfs:
+            self.deleteIntfs()
 
 
 class OVSLegacyKernelSwitch( Switch ):
@@ -886,11 +897,12 @@ class OVSLegacyKernelSwitch( Switch ):
                   ' 1>' + ofplog + ' 2>' + ofplog + '&' )
         self.execed = False
 
-    def stop( self ):
+    def stop( self, deleteIntfs=True ):
         "Terminate kernel datapath."
         quietRun( 'ovs-dpctl del-dp ' + self.dp )
         self.cmd( 'kill %ovs-openflowd' )
-        self.deleteIntfs()
+        if deleteIntfs:
+            self.deleteIntfs()
 
 
 class OVSSwitch( Switch ):
@@ -969,10 +981,11 @@ class OVSSwitch( Switch ):
             clist += ' ptcp:%s' % self.listenPort
         self.cmd( 'ovs-vsctl set-controller', self, clist )
 
-    def stop( self ):
+    def stop( self, deleteIntfs=True ):
-        "Terminate OVS switch."
+        "Stop OVS switch."
         self.cmd( 'ovs-vsctl del-br', self )
-        self.deleteIntfs()
+        if deleteIntfs:
+            self.deleteIntfs()
 
 OVSKernelSwitch = OVSSwitch
 

util/m

@@ -9,18 +9,6 @@ else
   host=$1
 fi
 
-pid=`ps ax | grep mininet:$host | grep bash | awk '{print $1};'`
-
-if echo $pid | grep -q ' '; then
-  echo "Error: found multiple mininet:$host processes"
-  exit 2
-fi
-
-if [ "$pid" == "" ]; then
-  echo "Could not find Mininet host $host"
-  exit 3
-fi
-
 if [ -z $2 ]; then
   cmd='bash'
 else
@@ -28,9 +16,11 @@ else
   cmd=$*
 fi
 
+# We could do this in this script, and we may want to eventually,
+# but for now we'll use mnexec to attach to the host's cgroup
 cgroup=/sys/fs/cgroup/cpu/$host
 if [ -d "$cgroup" ]; then
-  cg="-g $host"
+  cg="mnexec -g $host"
 fi
 
-exec sudo mnexec -a $pid $cg $cmd
+exec sudo ip netns exec $host $cg $cmd