Add .config

Any code in mn is not usable by other Python code.

Hence, move this code into util, so other scripts can use it.

.gitignore

@@ -1,3 +1,8 @@
 mnexec
 *.pyc
+*~
+\#*\#
 mininet.egg-info
+build/*
+dist/*
+

.pylint

@@ -25,9 +25,6 @@ ignore=CVS
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# Set the cache size for astng objects.
-cache-size=500
-
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
 load-plugins=
@@ -35,32 +32,23 @@ load-plugins=
 
 [MESSAGES CONTROL]
 
-# Enable only checker(s) with the given id(s). This option conflicts with the
-# disable-checker option
-#enable-checker=
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
 
-# Enable all checker(s) except those with the given id(s). This option
-# conflicts with the enable-checker option
-#disable-checker=
-
-# Enable all messages in the listed categories (IRCWEF).
-#enable-msg-cat=
-
-# Disable all messages in the listed categories (IRCWEF).
-disable-msg-cat=IR
-
-# Enable the message(s) with the given id(s).
-#enable-msg=
-
-# Disable the message(s) with the given id(s).
-disable-msg=W0704,C0103,W0231,E1102,W0511,W0142,R0902,R0903,R0904,R0913,R0914,R0801
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=W0704,C0103,W0231,E1102,W0511,W0142,R0902,R0903,R0904,R0913,R0914,R0801,I0011
 
 
 [REPORTS]
 
 # Set the output format. Available formats are text, parseable, colorized, msvs
 # (visual studio) and html
-output-format=text
+output-format=colorized
 
 # Include message's id in outpu
 include-ids=yes
@@ -276,7 +264,7 @@ int-import-graph=
 max-line-length=80
 
 # Maximum number of lines in a module
-max-module-lines=1000
+max-module-lines=1500
 
 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
 # tab).

Makefile

@@ -1,30 +1,39 @@
-all: codecheck test
-
-clean:
-	rm -rf build dist *.egg-info *.pyc mnexec bin/mnexec
-
 MININET = mininet/*.py
 TEST = mininet/test/*.py
 EXAMPLES = examples/*.py
 BIN = bin/mn
 PYSRC = $(MININET) $(TEST) $(EXAMPLES) $(BIN)
-
+MNEXEC = mnexec
 P8IGN = E251,E201,E302,E202
 
+all: codecheck test
+
+clean:
+	rm -rf build dist *.egg-info *.pyc $(MNEXEC)
+
 codecheck: $(PYSRC)
 	-echo "Running code check"
 	pyflakes $(PYSRC)
 	pylint --rcfile=.pylint $(PYSRC)
 	pep8 --repeat --ignore=$(P8IGN) $(PYSRC)
 
+errcheck: $(PYSRC)
+	-echo "Running check for errors only"
+	pyflakes $(PYSRC)
+	pylint -E --rcfile=.pylint $(PYSRC)
+
 test: $(MININET) $(TEST)
 	-echo "Running tests"
 	mininet/test/test_nets.py
 
-install: mnexec
-	cp mnexec bin/
+install: $(MNEXEC)
+	install $(MNEXEC) /usr/local/bin/
 	python setup.py install
 
+develop: $(MNEXEC)
+	install $(MNEXEC) /usr/local/bin/
+	python setup.py develop
+
 doc:
 	doxygen doxygen.cfg
 

bin/mn

@@ -18,13 +18,17 @@ import time
 
 from mininet.clean import cleanup
 from mininet.cli import CLI
-from mininet.log import lg, LEVELS, info
-from mininet.net import Mininet, init
-from mininet.node import KernelSwitch, Host, Controller, ControllerParams, NOX
-from mininet.node import RemoteController, UserSwitch, OVSKernelSwitch
+from mininet.log import lg, LEVELS, info, warn
+from mininet.net import Mininet, MininetWithControlNet
+from mininet.node import ( Host, CPULimitedHost, Controller, OVSController,
+                           NOX, RemoteController, UserSwitch, OVSKernelSwitch,
+                           OVSLegacyKernelSwitch )
+from mininet.link import Link, TCLink
 from mininet.topo import SingleSwitchTopo, LinearTopo, SingleSwitchReversedTopo
 from mininet.topolib import TreeTopo
-from mininet.util import makeNumeric
+from mininet.util import makeNumeric, custom, customConstructor, splitArgs
+from mininet.util import buildTopo
+
 
 # built in topologies, created only when run
 TOPODEF = 'minimal'
@@ -35,20 +39,26 @@ TOPOS = { 'minimal': lambda: SingleSwitchTopo( k=2 ),
          'tree': TreeTopo }
 
 SWITCHDEF = 'ovsk'
-SWITCHES = { 'kernel': KernelSwitch,
-            'user': UserSwitch,
-            'ovsk': OVSKernelSwitch }
+SWITCHES = { 'user': UserSwitch,
+            'ovsk': OVSKernelSwitch,
+             'ovsl': OVSLegacyKernelSwitch }
 
-HOSTDEF = 'process'
-HOSTS = { 'process': Host }
+HOSTDEF = 'proc'
+HOSTS = { 'proc': Host,
+          'rt': custom( CPULimitedHost, sched='rt' ),
+          'cfs': custom( CPULimitedHost, sched='cfs' ) }
 
 CONTROLLERDEF = 'ref'
-# a and b are the name and inNamespace params.
 CONTROLLERS = { 'ref': Controller,
-               'nox_dump': lambda name: NOX( name, 'packetdump' ),
-               'nox_pysw': lambda name: NOX( name, 'pyswitch' ),
-               'remote': lambda name: None,
-               'none': lambda name: None }
+                'ovsc': OVSController,
+                'nox': NOX,
+                'remote': RemoteController,
+                'none': lambda name: None }
+
+LINKDEF = 'default'
+LINKS = { 'default': Link,
+          'tc': TCLink }
+
 
 # optional tests to run
 TESTS = [ 'cli', 'build', 'pingall', 'pingpair', 'iperf', 'all', 'iperfudp',
@@ -57,25 +67,6 @@ TESTS = [ 'cli', 'build', 'pingall', 'pingpair', 'iperf', 'all', 'iperfudp',
 ALTSPELLING = { 'pingall': 'pingAll', 'pingpair': 'pingPair',
     'iperfudp': 'iperfUdp', 'iperfUDP': 'iperfUdp', 'prefixlen': 'prefixLen' }
 
-def buildTopo( topo ):
-    "Create topology from string with format (object, arg1, arg2,...)."
-    topo_split = topo.split( ',' )
-    topo_name = topo_split[ 0 ]
-    topo_params = topo_split[ 1: ]
-
-    # Convert int and float args; removes the need for every topology to
-    # be flexible with input arg formats.
-    topo_seq_params = [ s for s in topo_params if '=' not in s ]
-    topo_seq_params = [ makeNumeric( s ) for s in topo_seq_params ]
-    topo_kw_params = {}
-    for s in [ p for p in topo_params if '=' in p ]:
-        key, val = s.split( '=' )
-        topo_kw_params[ key ] = makeNumeric( val )
-
-    if topo_name not in TOPOS.keys():
-        raise Exception( 'Invalid topo_name %s' % topo_name )
-    return TOPOS[ topo_name ]( *topo_seq_params, **topo_kw_params )
-
 
 def addDictOption( opts, choicesDict, default, name, helpStr=None ):
     """Convenience function to add choices dicts to OptionParser.
@@ -88,10 +79,10 @@ def addDictOption( opts, choicesDict, default, name, helpStr=None ):
         raise Exception( 'Invalid  default %s for choices dict: %s' %
                         ( default, name ) )
     if not helpStr:
-        helpStr = '[' + ' '.join( choicesDict.keys() ) + ']'
+        helpStr = ( '|'.join( sorted( choicesDict.keys() ) ) +
+                   '[,param=value...]' )
     opts.add_option( '--' + name,
-                    type='choice',
-                    choices=choicesDict.keys(),
+                    type='string',
                     default = default,
                     help = helpStr )
 
@@ -124,11 +115,11 @@ class MininetRunner( object ):
 
     def parseCustomFile( self, fileName ):
         "Parse custom file and add params before parsing cmd-line options."
-        custom = {}
+        customs = {}
         if os.path.isfile( fileName ):
-            execfile( fileName, custom, custom )
-            for name in custom:
-                self.setCustom( name, custom[ name ] )
+            execfile( fileName, customs, customs )
+            for name, val in customs.iteritems():
+                self.setCustom( name, val )
         else:
             raise Exception( 'could not find custom file: %s' % fileName )
 
@@ -136,11 +127,10 @@ class MininetRunner( object ):
         """Parse command-line args and return options object.
            returns: opts parse options dict"""
         if '--custom' in sys.argv:
-            print "custom in sys.argv"
             index = sys.argv.index( '--custom' )
             if len( sys.argv ) > index + 1:
-                custom = sys.argv[ index + 1 ]
-                self.parseCustomFile( custom )
+                filename = sys.argv[ index + 1 ]
+                self.parseCustomFile( filename )
             else:
                 raise Exception( 'Custom file name not found' )
 
@@ -148,46 +138,43 @@ class MininetRunner( object ):
         addDictOption( opts, SWITCHES, SWITCHDEF, 'switch' )
         addDictOption( opts, HOSTS, HOSTDEF, 'host' )
         addDictOption( opts, CONTROLLERS, CONTROLLERDEF, 'controller' )
+        addDictOption( opts, LINKS, LINKDEF, 'link' )
+        addDictOption( opts, TOPOS, TOPODEF, 'topo' )
 
-        opts.add_option( '--topo', type='string', default=TOPODEF,
-                        help='[' + ' '.join( TOPOS.keys() ) + '],arg1,arg2,'
-                        '...argN')
         opts.add_option( '--clean', '-c', action='store_true',
                         default=False, help='clean and exit' )
         opts.add_option( '--custom', type='string', default=None,
                         help='read custom topo and node params from .py file' )
         opts.add_option( '--test', type='choice', choices=TESTS,
                         default=TESTS[ 0 ],
-                        help='[' + ' '.join( TESTS ) + ']' )
+                        help='|'.join( TESTS ) )
         opts.add_option( '--xterms', '-x', action='store_true',
                         default=False, help='spawn xterms for each node' )
+        opts.add_option( '--ipbase', '-i', type='string', default='10.0.0.0/8',
+                         help='base IP address for hosts' )
         opts.add_option( '--mac', action='store_true',
-                        default=False, help='set MACs equal to DPIDs' )
+                        default=False, help='automatically set host MACs' )
         opts.add_option( '--arp', action='store_true',
                         default=False, help='set all-pairs ARP entries' )
         opts.add_option( '--verbosity', '-v', type='choice',
                         choices=LEVELS.keys(), default = 'info',
-                        help = '[' + ' '.join( LEVELS.keys() ) + ']' )
-        opts.add_option( '--ip', type='string', default='127.0.0.1',
-                        help='[ip address as a dotted decimal string for a'
-                        'remote controller]' )
-        opts.add_option( '--port', type='int', default=6633,
-                        help='[port integer for a listening remote'
-                        ' controller]' )
+                        help = '|'.join( LEVELS.keys() )  )
         opts.add_option( '--innamespace', action='store_true',
                         default=False, help='sw and ctrl in namespace?' )
-        opts.add_option( '--listenport', type='int', default=6634,
-                        help='[base port for passive switch listening'
-                        ' controller]' )
+        opts.add_option( '--listenport', type='int', default=6635,
+                         help='base port for passive switch listening' )
         opts.add_option( '--nolistenport', action='store_true',
                         default=False, help="don't use passive listening port")
         opts.add_option( '--pre', type='string', default=None,
-                        help='[CLI script to run before tests]' )
+                        help='CLI script to run before tests' )
         opts.add_option( '--post', type='string', default=None,
-                        help='[CLI script to run after tests]' )
+                        help='CLI script to run after tests' )
         opts.add_option( '--prefixlen', type='int', default=8,
-                        help='[prefix length (e.g. /8) for automatic '
-                        'network configuration]' )
+                        help='prefix length (e.g. /8) for automatic '
+                        'network configuration' )
+        opts.add_option( '--pin', action='store_true',
+                         default=False, help="pin hosts to CPU cores "
+                         "(requires --host cfs or --host rt)" )
 
         self.options, self.args = opts.parse_args()
 
@@ -202,9 +189,6 @@ class MininetRunner( object ):
                     % self.options.verbosity )
         lg.setLogLevel( self.options.verbosity )
 
-        # validate environment setup
-        init()
-
     def begin( self ):
         "Create and run mininet."
 
@@ -214,35 +198,33 @@ class MininetRunner( object ):
 
         start = time.time()
 
-        topo = buildTopo( self.options.topo )
-        switch = SWITCHES[ self.options.switch ]
-        host = HOSTS[ self.options.host ]
-        controller = CONTROLLERS[ self.options.controller ]
-        if self.options.controller == 'remote':
-            controller = lambda a: RemoteController( a,
-                             defaultIP=self.options.ip,
-                             port=self.options.port )
+        topo = buildTopo( TOPOS, self.options.topo )
+        switch = customConstructor( SWITCHES, self.options.switch )
+        host = customConstructor( HOSTS, self.options.host )
+        controller = customConstructor( CONTROLLERS, self.options.controller )
+        link = customConstructor( LINKS, self.options.link )
 
         if self.validate:
             self.validate( self.options )
 
-        # We should clarify what this is actually for...
-        # It seems like it should be default values for the
-        # *data* network, so it may be misnamed.
-        controllerParams = ControllerParams( '10.0.0.0',
-            self.options.prefixlen)
-
         inNamespace = self.options.innamespace
+        Net = MininetWithControlNet if inNamespace else Mininet
+        ipBase = self.options.ipbase
         xterms = self.options.xterms
         mac = self.options.mac
         arp = self.options.arp
+        pin = self.options.pin
         listenPort = None
         if not self.options.nolistenport:
             listenPort = self.options.listenport
-        mn = Mininet( topo, switch, host, controller, controllerParams,
-                     inNamespace=inNamespace,
-                     xterms=xterms, autoSetMacs=mac,
-                     autoStaticArp=arp, listenPort=listenPort )
+        mn = Net( topo=topo,
+                  switch=switch, host=host, controller=controller,
+                  link=link,
+                  ipBase=ipBase,
+                  inNamespace=inNamespace,
+                  xterms=xterms, autoSetMacs=mac,
+                  autoStaticArp=arp, autoPinCpus=pin,
+                  listenPort=listenPort )
 
         if self.options.pre:
             CLI( mn, script=self.options.pre )

examples/README

@@ -6,12 +6,21 @@ Mininet's Python API.
 
 ---
 
+baresshd.py:
+
+This example uses Mininet's medium-level API to create an sshd
+process running in a namespace. Doesn't use OpenFlow.
+
 consoles.py:
 
 This example creates a grid of console windows, one for each node, 
 and allows interaction with and monitoring of each console, including
 graphical monitoring.
 
+controllers.py:
+
+This example creates a network and adds multiple controllers to it.
+
 emptynet.py:
 
 This example demonstrates creating an empty network (i.e. with no
@@ -26,16 +35,34 @@ miniedit.py:
 
 This example demonstrates creating a network via a graphical editor.
 
+multiping.py:
+
+This example demonstrates one method for 
+monitoring output from multiple hosts, using node.monitor().
+
+multipoll.py:
+
+This example demonstrates monitoring output files from multiple hosts.
+
 multitest.py:
 
 This example creates a network and runs multiple tests on it.
 
+popenpoll.py:
+
+This example demonstrates monitoring output from multiple hosts using
+the node.popen() interface (which returns Popen objects) and pmonitor().
+
 scratchnet.py, scratchnetuser.py:
 
 These two examples demonstrate how to create a network by using the lowest-
 level Mininet functions. Generally the higher-level API is easier to use,
 but scratchnet shows what is going on behind the scenes.
 
+simpleperf.py:
+
+A simple example of configuring network and CPU bandwidth limits.
+
 sshd.py:
 
 This example shows how to run an sshd process in each host, allowing
@@ -44,10 +71,10 @@ to an interface in the root namespace (generaly the control network
 already lives in the root namespace, so it does not need to be explicitly
 connected.)
 
-treeping64:
+treeping64.py:
 
 This example creates a 64-host tree network, and attempts to check full
-connectivity using ping, for three different switch/datapath types.
+connectivity using ping, for different switch/datapath types.
 
 tree1024.py:
 
@@ -55,7 +82,3 @@ This example attempts to create a 1024-host network, and then runs the
 CLI on it. It may run into scalability limits, depending on available
 memory and sysctl configuration (see INSTALL.)
 
-udpbwtest.py:
-
-This example shows how to run a test across an entire network, and monitor
-the output of a set of hosts in real time.

examples/baresshd.py

@@ -6,14 +6,17 @@ from mininet.node import Host
 
 print "*** Creating nodes"
 h1 = Host( 'h1' )
+
 root = Host( 'root', inNamespace=False )
 
 print "*** Creating links"
 h1.linkTo( root )
 
+print h1
+
 print "*** Configuring nodes"
-h1.setIP( h1.intfs[ 0 ], '10.0.0.1', 8 )
-root.setIP( root.intfs[ 0 ], '10.0.0.2', 8 )
+h1.setIP( '10.0.0.1', 8 )
+root.setIP( '10.0.0.2', 8 )
 
 print "*** Creating banner file"
 f = open( '/tmp/%s.banner' % h1.name, 'w' )

examples/consoles.py

@@ -107,8 +107,10 @@ class Console( Frame ):
         self.text.insert( 'end', text )
         self.text.mark_set( 'insert', 'end' )
         self.text.see( 'insert' )
+        outputHook = lambda x, y: True  # make pylint happier
         if self.outputHook:
-            self.outputHook( self, text )
+            outputHook = self.outputHook
+        outputHook( self, text )
 
     def handleKey( self, event ):
         "If it's an interactive command, send it to the node."
@@ -130,27 +132,22 @@ class Console( Frame ):
         self.sendCmd( cmd )
 
     # Callback ignores event
-    # pylint: disable-msg=W0613
-    def handleInt( self, event=None ):
+    def handleInt( self, _event=None ):
         "Handle control-c."
         self.node.sendInt()
-    # pylint: enable-msg=W0613
 
     def sendCmd( self, cmd ):
         "Send a command to our node."
         if not self.node.waiting:
             self.node.sendCmd( cmd )
 
-    # Callback ignores fds
-    # pylint: disable-msg=W0613
-    def handleReadable( self, fds, timeoutms=None ):
+    def handleReadable( self, _fds, timeoutms=None ):
         "Handle file readable event."
         data = self.node.monitor( timeoutms )
         self.append( data )
         if not self.node.waiting:
             # Print prompt
             self.append( self.prompt )
-    # pylint: enable-msg=W0613
 
     def waiting( self ):
         "Are we waiting for output?"
@@ -319,9 +316,7 @@ class ConsoleApp( Frame ):
 
         self.pack( expand=True, fill='both' )
 
-    # Update callback doesn't use console arg
-    # pylint: disable-msg=W0613
-    def updateGraph( self, console, output ):
+    def updateGraph( self, _console, output ):
         "Update our graph."
         m = re.search( r'(\d+) Mbits/sec', output )
         if not m:
@@ -332,7 +327,6 @@ class ConsoleApp( Frame ):
             self.graph.addBar( self.bw )
             self.bw = 0
             self.updates = 0
-    # pylint: enable-msg=W0613
 
     def setOutputHook( self, fn=None, consoles=None ):
         "Register fn as output hook [on specific consoles.]"

examples/cpu.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+
+"""
+cpu.py: test iperf bandwidth for varying cpu limtis
+"""
+
+from mininet.net import Mininet
+from mininet.node import CPULimitedHost
+from mininet.topolib import TreeTopo
+from mininet.util import custom
+from mininet.log import setLogLevel, output
+
+from time import sleep
+
+def waitListening(client, server, port):
+    "Wait until server is listening on port"
+    if not client.cmd('which telnet'):
+        raise Exception('Could not find telnet')
+    cmd = ('sh -c "echo A | telnet -e A %s %s"' %
+           (server.IP(), port))
+    while 'Connected' not in client.cmd(cmd):
+        output('waiting for', server,
+               'to listen on port', port, '\n')
+        sleep(.5)
+
+
+def bwtest( cpuLimits, period_us=100000, seconds=5 ):
+    """Example/test of link and CPU bandwidth limits
+       cpu: cpu limit as fraction of overall CPU time"""
+
+    topo = TreeTopo( depth=1, fanout=2 )
+
+    results = {}
+
+    for sched in 'rt', 'cfs':
+        print '*** Testing with', sched, 'bandwidth limiting'
+        for cpu in cpuLimits:
+            host = custom( CPULimitedHost, sched=sched,
+                           period_us=period_us,
+                           cpu=cpu )
+            net = Mininet( topo=topo, host=host )
+            net.start()
+            net.pingAll()
+            hosts = [ net.getNodeByName( h ) for h in topo.hosts() ]
+            client, server = hosts[ 0 ], hosts[ -1 ]
+            server.cmd( 'iperf -s -p 5001 &' )
+            waitListening( client, server, 5001 )
+            result = client.cmd( 'iperf -yc -t %s -c %s' % (
+                    seconds, server.IP() ) ).split( ',' )
+            bps = float( result[ -1 ] )
+            server.cmdPrint( 'kill %iperf' )
+            net.stop()
+            updated = results.get( sched, [] )
+            updated += [ ( cpu, bps ) ]
+            results[ sched ] = updated
+
+    return results
+
+
+def dump( results ):
+    "Dump results"
+
+    fmt = '%s\t%s\t%s'
+
+    print
+    print fmt % ( 'sched', 'cpu', 'client MB/s' )
+    print
+
+    for sched in sorted( results.keys() ):
+        entries = results[ sched ]
+        for cpu, bps in entries:
+            pct = '%.2f%%' % ( cpu * 100 )
+            mbps = bps / 1e6
+            print fmt % ( sched, pct, mbps )
+
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    limits = [ .45, .4, .3, .2, .1 ]
+    out = bwtest( limits )
+    dump( out )

examples/limit.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+"""
+limit.py: example of using link and CPU limits
+"""
+
+from mininet.net import Mininet
+from mininet.link import TCIntf
+from mininet.node import CPULimitedHost
+from mininet.topolib import TreeTopo
+from mininet.util import custom, quietRun
+from mininet.log import setLogLevel
+from time import sleep
+
+def testLinkLimit( net, bw ):
+    "Run bandwidth limit test"
+    print '*** Testing network %.2f Mbps bandwidth limit' % bw
+    net.iperf( )
+
+def testCpuLimit( net, cpu ):
+    "run CPU limit test"
+    pct = cpu * 100
+    print '*** Testing CPU %.0f%% bandwidth limit' % pct
+    h1, h2 = net.hosts
+    h1.cmd( 'while true; do a=1; done &' )
+    h2.cmd( 'while true; do a=1; done &' )
+    pid1 = h1.cmd( 'echo $!' ).strip()
+    pid2 = h2.cmd( 'echo $!' ).strip()
+    cmd = 'ps -p %s,%s -o pid,%%cpu,args' % ( pid1, pid2 )
+    # It's a shame that this is what pylint prefers
+    for _ in range( 5 ):
+        sleep( 1 )
+        print quietRun( cmd ).strip()
+    h1.cmd( 'kill %1')
+    h2.cmd( 'kill %1')
+
+def limit( bw=10, cpu=.4 ):
+    """Example/test of link and CPU bandwidth limits
+       bw: interface bandwidth limit in Mbps
+       cpu: cpu limit as fraction of overall CPU time"""
+    intf = custom( TCIntf, bw=bw )
+    myTopo = TreeTopo( depth=1, fanout=2 )
+    for sched in 'rt', 'cfs':
+        print '*** Testing with', sched, 'bandwidth limiting'
+        host = custom( CPULimitedHost, sched=sched, cpu=cpu )
+        net = Mininet( topo=myTopo, intf=intf, host=host )
+        net.start()
+        testLinkLimit( net, bw=bw )
+        testCpuLimit( net, cpu=cpu )
+        net.stop()
+
+def verySimpleLimit( bw=150 ):
+    "Absurdly simple limiting test"
+    intf = custom( TCIntf, bw=bw )
+    net = Mininet( intf=intf )
+    h1, h2 = net.addHost( 'h1' ), net.addHost( 'h2' )
+    net.addLink( h1, h2 )
+    net.start()
+    net.pingAll()
+    net.iperf()
+    h1.cmdPrint( 'tc -s qdisc ls dev', h1.defaultIntf() )
+    h2.cmdPrint( 'tc -d class show dev', h2.defaultIntf() )
+    h1.cmdPrint( 'tc -s qdisc ls dev', h1.defaultIntf() )
+    h2.cmdPrint( 'tc -d class show dev', h2.defaultIntf() )
+    net.stop()
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    limit()

examples/linearbandwidth.py

@@ -6,9 +6,9 @@ using both kernel and user datapaths.
 
 We construct a network of N hosts and N-1 switches, connected as follows:
 
-h1 <-> sN+1 <-> sN+2 .. sN+N-1
-       |        |       |
-       h2       h3      hN
+h1 <-> s1 <-> s2 .. sN-1
+       |       |    |
+       h2      h3   hN
 
 WARNING: by default, the reference controller only supports 16
 switches, so this test WILL NOT WORK unless you have recompiled
@@ -23,42 +23,40 @@ of switches, this example demonstrates:
 
 """
 
+from mininet.net import Mininet
+from mininet.node import UserSwitch, OVSKernelSwitch
+from mininet.topo import Topo
+from mininet.log import lg
+from mininet.util import irange
+
 import sys
 flush = sys.stdout.flush
 
-from mininet.net import init, Mininet
-# from mininet.node import KernelSwitch
-from mininet.node import UserSwitch, OVSKernelSwitch
-from mininet.topo import Topo, Node
-from mininet.log import lg
-
 class LinearTestTopo( Topo ):
     "Topology for a string of N hosts and N-1 switches."
 
-    def __init__( self, N ):
+    def __init__( self, N, **params ):
 
-        # Add default members to class.
-        super( LinearTestTopo, self ).__init__()
+        # Initialize topology
+        Topo.__init__( self, **params )
 
-        # Create switch and host nodes
-        hosts = range( 1, N + 1 )
-        switches = range( N + 1 , N + N )
-        for h in hosts:
-            self.add_node( h, Node( is_switch=False ) )
-        for s in switches:
-            self.add_node( s, Node( is_switch=True ) )
+        # Create switches and hosts
+        hosts = [ self.add_host( 'h%s' % h )
+                  for h in irange( 1, N ) ]
+        switches = [ self.add_switch( 's%s' % s )
+                     for s in irange( 1, N - 1 ) ]
 
         # Wire up switches
-        for s in switches[ :-1 ]:
-            self.add_edge( s, s + 1 )
+        last = None
+        for switch in switches:
+            if last:
+                self.add_link( last, switch )
+            last = switch
 
         # Wire up hosts
-        self.add_edge( hosts[ 0 ], switches[ 0 ] )
-        for h in hosts[ 1: ]:
-            self.add_edge( h, h + N - 1 )
-
-        # Consider all switches and hosts 'on'
-        self.enable_all()
+        self.add_link( hosts[ 0 ], switches[ 0 ] )
+        for host, switch in zip( hosts[ 1: ], switches ):
+            self.add_link( host, switch )
 
 
 def linearBandwidthTest( lengths ):
@@ -69,15 +67,16 @@ def linearBandwidthTest( lengths ):
     switchCount = max( lengths )
     hostCount = switchCount + 1
 
-    switches = {  # 'reference kernel': KernelSwitch,
-            'reference user': UserSwitch,
+    switches = { 'reference user': UserSwitch,
             'Open vSwitch kernel': OVSKernelSwitch }
 
+    topo = LinearTestTopo( hostCount )
+
     for datapath in switches.keys():
         print "*** testing", datapath, "datapath"
         Switch = switches[ datapath ]
         results[ datapath ] = []
-        net = Mininet( topo=LinearTestTopo( hostCount ), switch=Switch )
+        net = Mininet( topo=topo, switch=Switch )
         net.start()
         print "*** testing basic connectivity"
         for n in lengths:
@@ -106,7 +105,6 @@ def linearBandwidthTest( lengths ):
 
 if __name__ == '__main__':
     lg.setLogLevel( 'info' )
-    init()
     sizes = [ 1, 10, 20, 40, 60, 80, 100 ]
     print "*** Running linearBandwidthTest", sizes
     linearBandwidthTest( sizes  )

examples/miniedit.py

@@ -299,14 +299,11 @@ class MiniEdit( Frame ):
         # Delete from view
         self.canvas.delete( item )
 
-    # Callback ignores event
-    # pylint: disable-msg=W0613
-    def deleteSelection( self, event ):
+    def deleteSelection( self, _event ):
         "Delete the selected item."
         if self.selection is not None:
             self.deleteItem( self.selection )
         self.selectItem( None )
-    # pylint: enable-msg=W0613
 
     def nodeIcon( self, node, name ):
         "Create a new node icon."
@@ -350,14 +347,11 @@ class MiniEdit( Frame ):
         c = self.canvas
         c.coords( self.link, self.linkx, self.linky, x, y )
 
-    # Callback ignores event
-    # pylint: disable-msg=W0613
-    def releaseLink( self, event ):
+    def releaseLink( self, _event ):
         "Give up on the current link."
         if self.link is not None:
             self.canvas.delete( self.link )
         self.linkWidget = self.linkItem = self.link = None
-    # pylint: enable-msg=W0613
 
     # Generic node handlers
 
@@ -385,12 +379,9 @@ class MiniEdit( Frame ):
         "Select node on entry."
         self.selectNode( event )
 
-    # Callback ignores event
-    # pylint: disable-msg=W0613
-    def leaveNode( self, event ):
+    def leaveNode( self, _event ):
         "Restore old selection on exit."
         self.selectItem( self.lastSelection )
-    # pylint: enable-msg=W0613
 
     def clickNode( self, event ):
         "Node click handler."
@@ -454,23 +445,21 @@ class MiniEdit( Frame ):
         # Link bindings
         # Selection still needs a bit of work overall
         # Callbacks ignore event
-        # pylint: disable-msg=W0613
 
-        def select( event, link=self.link ):
+        def select( _event, link=self.link ):
             "Select item on mouse entry."
             self.selectItem( link )
 
-        def highlight( event, link=self.link ):
+        def highlight( _event, link=self.link ):
             "Highlight item on mouse entry."
             # self.selectItem( link )
             self.canvas.itemconfig( link, fill='green' )
 
-        def unhighlight( event, link=self.link ):
+        def unhighlight( _event, link=self.link ):
             "Unhighlight item on mouse exit."
             self.canvas.itemconfig( link, fill='blue' )
             # self.selectItem( None )
 
-        # pylint: disable-msg=W0613
         self.canvas.tag_bind( self.link, '<Enter>', highlight )
         self.canvas.tag_bind( self.link, '<Leave>', unhighlight )
         self.canvas.tag_bind( self.link, '<ButtonPress-1>', select )
@@ -602,7 +591,7 @@ class MiniEdit( Frame ):
         cleanUpScreens()
         self.net = None
 
-    def xterm( self, ignore=None ):
+    def xterm( self, _=None ):
         "Make an xterm when a button is pressed."
         if ( self.selection is None or
              self.net is None or

examples/multiping.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python
+
+"""
+multiping.py: monitor multiple sets of hosts using ping
+
+This demonstrates how one may send a simple shell script to
+multiple hosts and monitor their output interactively for a period=
+of time.
+"""
+
+from mininet.net import Mininet
+from mininet.node import Node
+from mininet.topo import SingleSwitchTopo
+from mininet.log import setLogLevel
+
+from select import poll, POLLIN
+from time import time
+
+def chunks( l, n ):
+    "Divide list l into chunks of size n - thanks Stackoverflow"
+    return [ l[ i : i + n ] for i in range( 0, len( l ), n ) ]
+
+def startpings( host, targetips ):
+    "Tell host to repeatedly ping targets"
+
+    targetips.append( '10.0.0.200' )
+
+    targetips = ' '.join( targetips )
+
+    # BL: Not sure why loopback intf isn't up!
+    host.cmd( 'ifconfig lo up' )
+
+    # Simple ping loop
+    cmd = ( 'while true; do '
+            ' for ip in %s; do ' % targetips +
+            '  echo -n %s "->" $ip ' % host.IP() +
+            '   `ping -c1 -w 1 $ip | grep packets` ;'
+            '  sleep 1;'
+            ' done; '
+            'done &' )
+
+    print ( '*** Host %s (%s) will be pinging ips: %s' %
+           ( host.name, host.IP(), targetips ) )
+
+    host.cmd( cmd )
+
+def multiping( netsize, chunksize, seconds):
+    "Ping subsets of size chunksize in net of size netsize"
+
+    # Create network and identify subnets
+    topo = SingleSwitchTopo( netsize )
+    net = Mininet( topo=topo )
+    net.start()
+    hosts = net.hosts
+    subnets = chunks( hosts, chunksize )
+
+    # Create polling object
+    fds = [ host.stdout.fileno() for host in hosts ]
+    poller = poll()
+    for fd in fds:
+        poller.register( fd, POLLIN )
+
+    # Start pings
+    for subnet in subnets:
+        ips = [ host.IP() for host in subnet ]
+        for host in subnet:
+            startpings( host, ips )
+
+    # Monitor output
+    endTime = time() + seconds
+    while time() < endTime:
+        readable = poller.poll(1000)
+        for fd, _mask in readable:
+            node = Node.outToNode[ fd ]
+            print '%s:' % node.name, node.monitor().strip()
+
+    # Stop pings
+    for host in hosts:
+        host.cmd( 'kill %while' )
+
+    net.stop()
+
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    multiping( netsize=20, chunksize=4, seconds=10 )

examples/multipoll.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+
+"""
+Simple example of sending output to multiple files and
+monitoring them
+"""
+
+from mininet.topo import SingleSwitchTopo
+from mininet.net import Mininet
+from mininet.log import setLogLevel
+
+from time import time
+from select import poll, POLLIN
+from subprocess import Popen, PIPE
+
+def monitorFiles( outfiles, seconds, timeoutms ):
+    "Monitor set of files and return [(host, line)...]"
+    devnull = open( '/dev/null', 'w' )
+    tails, fdToFile, fdToHost = {}, {}, {}
+    for h, outfile in outfiles.iteritems():
+        tail = Popen( [ 'tail', '-f', outfile ],
+                     stdout=PIPE, stderr=devnull )
+        fd = tail.stdout.fileno()
+        tails[ h ] = tail
+        fdToFile[ fd ] = tail.stdout
+        fdToHost[ fd ] = h
+    # Prepare to poll output files
+    readable = poll()
+    for t in tails.values():
+        readable.register( t.stdout.fileno(), POLLIN )
+    # Run until a set number of seconds have elapsed
+    endTime = time() + seconds
+    while time() < endTime:
+        fdlist = readable.poll(timeoutms)
+        if fdlist:
+            for fd, _flags in fdlist:
+                f = fdToFile[ fd ]
+                host = fdToHost[ fd ]
+                # Wait for a line of output
+                line = f.readline().strip()
+                yield host, line
+        else:
+            # If we timed out, return nothing
+            yield None, ''
+    for t in tails.values():
+        t.terminate()
+    devnull.close()  # Not really necessary
+
+
+def monitorTest( N=3, seconds=3 ):
+    "Run pings and monitor multiple hosts"
+    topo = SingleSwitchTopo( N )
+    net = Mininet( topo )
+    net.start()
+    hosts = net.hosts
+    print "Starting test..."
+    server = hosts[ 0 ]
+    outfiles, errfiles = {}, {}
+    for h in hosts:
+        # Create and/or erase output files
+        outfiles[ h ] = '/tmp/%s.out' % h.name
+        errfiles[ h ] = '/tmp/%s.err' % h.name
+        h.cmd( 'echo >', outfiles[ h ] )
+        h.cmd( 'echo >', errfiles[ h ] )
+        # Start pings
+        h.cmdPrint('ping', server.IP(),
+                   '>', outfiles[ h ],
+                   '2>', errfiles[ h ],
+                   '&' )
+    print "Monitoring output for", seconds, "seconds"
+    for h, line in monitorFiles( outfiles, seconds, timeoutms=500 ):
+        if h:
+            print '%s: %s' % ( h.name, line )
+    for h in hosts:
+        h.cmd('kill %ping')
+    net.stop()
+
+
+if __name__ == '__main__':
+    setLogLevel('info')
+    monitorTest()

examples/popen.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+"""
+This example monitors a number of hosts using host.popen() and
+pmonitor()
+"""
+
+from mininet.net import Mininet
+from mininet.node import CPULimitedHost
+from mininet.topo import SingleSwitchTopo
+from mininet.log import setLogLevel
+from mininet.util import custom, pmonitor
+
+def monitorhosts( hosts=5, sched='cfs' ):
+    "Start a bunch of pings and monitor them using popen"
+    mytopo = SingleSwitchTopo( hosts )
+    cpu = .5 / hosts
+    myhost = custom( CPULimitedHost, cpu=cpu, sched=sched )
+    net = Mininet( topo=mytopo, host=myhost )
+    net.start()
+    # Start a bunch of pings
+    popens = {}
+    last = net.hosts[ -1 ]
+    for host in net.hosts:
+        popens[ host ] = host.popen( "ping -c5 %s" % last.IP() )
+        last = host
+    # Monitor them and print output
+    for host, line in pmonitor( popens ):
+        if host:
+            print "<%s>: %s" % ( host.name, line.strip() )
+    # Done
+    net.stop()
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    monitorhosts( hosts=5 )

examples/popenpoll.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+
+"Monitor multiple hosts using popen()/pmonitor()"
+
+from mininet.net import Mininet
+from mininet.topo import SingleSwitchTopo
+from mininet.util import pmonitor
+from time import time
+from signal import SIGINT
+
+def pmonitorTest( N=3, seconds=10 ):
+    "Run pings and monitor multiple hosts using pmonitor"
+    topo = SingleSwitchTopo( N )
+    net = Mininet( topo )
+    net.start()
+    hosts = net.hosts
+    print "Starting test..."
+    server = hosts[ 0 ]
+    popens = {}
+    for h in hosts:
+        popens[ h ] = h.popen('ping', server.IP() )
+    print "Monitoring output for", seconds, "seconds"
+    endTime = time() + seconds
+    for h, line in pmonitor( popens, timeoutms=500 ):
+        if h:
+            print '%s: %s' % ( h.name, line ),
+        if time() >= endTime:
+            for p in popens.values():
+                p.send_signal( SIGINT )
+    net.stop()
+
+if __name__ == '__main__':
+    pmonitorTest()

examples/scratchnet.py

@@ -8,13 +8,16 @@ but it exposes the configuration details and allows customization.
 For most tasks, the higher-level API will be preferable.
 """
 
-from mininet.net import init
-from mininet.node import Node, OVSKernelSwitch
-from mininet.util import createLink
+from mininet.net import Mininet
+from mininet.node import Node
+from mininet.link import Link
 from mininet.log import setLogLevel, info
+from mininet.util import quietRun
 
-def scratchNet( cname='controller', cargs='ptcp:' ):
-    "Create network from scratch using kernel switch."
+from time import sleep
+
+def scratchNet( cname='controller', cargs='-v ptcp:' ):
+    "Create network from scratch using Open vSwitch."
 
     info( "*** Creating nodes\n" )
     controller = Node( 'c0', inNamespace=False )
@@ -23,36 +26,43 @@ def scratchNet( cname='controller', cargs='ptcp:' ):
     h1 = Node( 'h1' )
 
     info( "*** Creating links\n" )
-    createLink( node1=h0, node2=switch, port1=0, port2=0 )
-    createLink( node1=h1, node2=switch, port1=0, port2=1 )
+    Link( h0, switch )
+    Link( h1, switch )
 
     info( "*** Configuring hosts\n" )
-    h0.setIP( h0.intfs[ 0 ], '192.168.123.1', 24 )
-    h1.setIP( h1.intfs[ 0 ], '192.168.123.2', 24 )
+    h0.setIP( '192.168.123.1/24' )
+    h1.setIP( '192.168.123.2/24' )
     info( str( h0 ) + '\n' )
     info( str( h1 ) + '\n' )
 
-    info( "*** Starting network using Open vSwitch kernel datapath\n" )
+    info( "*** Starting network using Open vSwitch\n" )
     controller.cmd( cname + ' ' + cargs + '&' )
-    switch.cmd( 'ovs-dpctl del-dp dp0' )
-    switch.cmd( 'ovs-dpctl add-dp dp0' )
+    switch.cmd( 'ovs-vsctl del-br dp0' )
+    switch.cmd( 'ovs-vsctl add-br dp0' )
     for intf in switch.intfs.values():
-        print switch.cmd( 'ovs-dpctl add-if dp0 ' + intf )
-    print switch.cmd( 'ovs-openflowd dp0 tcp:127.0.0.1 &' )
+        print switch.cmd( 'ovs-vsctl add-port dp0 %s' % intf )
+
+    # Note: controller and switch are in root namespace, and we
+    # can connect via loopback interface
+    switch.cmd( 'ovs-vsctl set-controller dp0 tcp:127.0.0.1:6633' )
+
+    info( '*** Waiting for switch to connect to controller' )
+    while 'is_connected' not in quietRun( 'ovs-vsctl show' ):
+        sleep( 1 )
+        info( '.' )
+    info( '\n' )
 
     info( "*** Running test\n" )
     h0.cmdPrint( 'ping -c1 ' + h1.IP() )
 
     info( "*** Stopping network\n" )
     controller.cmd( 'kill %' + cname )
-    switch.cmd( 'ovs-dpctl del-dp dp0' )
-    switch.cmd( 'kill %ovs-openflowd' )
+    switch.cmd( 'ovs-vsctl del-br dp0' )
     switch.deleteIntfs()
     info( '\n' )
 
 if __name__ == '__main__':
     setLogLevel( 'info' )
     info( '*** Scratch network demo (kernel datapath)\n' )
-    OVSKernelSwitch.setup()
-    init()
+    Mininet.init()
     scratchNet()

examples/scratchnetuser.py

@@ -10,11 +10,16 @@ For most tasks, the higher-level API will be preferable.
 This version uses the user datapath and an explicit control network.
 """
 
-from mininet.net import init
+from mininet.net import Mininet
 from mininet.node import Node
-from mininet.util import createLink
+from mininet.link import Link
 from mininet.log import setLogLevel, info
 
+def linkIntfs( node1, node2 ):
+    "Create link from node1 to node2 and return intfs"
+    link = Link( node1, node2 )
+    return link.intf1, link.intf2
+
 def scratchNetUser( cname='controller', cargs='ptcp:' ):
     "Create network from scratch using user switch."
 
@@ -28,17 +33,17 @@ def scratchNetUser( cname='controller', cargs='ptcp:' ):
     switch = Node( 's0')
     h0 = Node( 'h0' )
     h1 = Node( 'h1' )
-    cintf, sintf = createLink( controller, switch )
-    h0intf, sintf1 = createLink( h0, switch )
-    h1intf, sintf2 = createLink( h1, switch )
+    cintf, sintf = linkIntfs( controller, switch )
+    h0intf, sintf1 = linkIntfs( h0, switch )
+    h1intf, sintf2 = linkIntfs( h1, switch )
 
     info( '*** Configuring control network\n' )
-    controller.setIP( cintf, '10.0.123.1', 24 )
-    switch.setIP( sintf, '10.0.123.2', 24 )
+    controller.setIP( '10.0.123.1/24', cintf )
+    switch.setIP( '10.0.123.2/24', sintf)
 
     info( '*** Configuring hosts\n' )
-    h0.setIP( h0intf, '192.168.123.1', 24 )
-    h1.setIP( h1intf, '192.168.123.2', 24 )
+    h0.setIP( '192.168.123.1/24', h0intf )
+    h1.setIP( '192.168.123.2/24', h1intf )
 
     info( '*** Network state:\n' )
     for node in controller, switch, h0, h1:
@@ -47,7 +52,7 @@ def scratchNetUser( cname='controller', cargs='ptcp:' ):
     info( '*** Starting controller and user datapath\n' )
     controller.cmd( cname + ' ' + cargs + '&' )
     switch.cmd( 'ifconfig lo 127.0.0.1' )
-    intfs = [ sintf1, sintf2 ]
+    intfs = [ str( i ) for i in sintf1, sintf2 ]
     switch.cmd( 'ofdatapath -i ' + ','.join( intfs ) + ' ptcp: &' )
     switch.cmd( 'ofprotocol tcp:' + controller.IP() + ' tcp:localhost &' )
 
@@ -64,5 +69,5 @@ def scratchNetUser( cname='controller', cargs='ptcp:' ):
 if __name__ == '__main__':
     setLogLevel( 'info' )
     info( '*** Scratch network demo (user datapath)\n' )
-    init()
+    Mininet.init()
     scratchNetUser()

examples/simpleperf.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+"""
+Simple example of setting network and CPU parameters
+"""
+
+from mininet.topo import Topo
+from mininet.net import Mininet
+from mininet.node import CPULimitedHost
+from mininet.link import TCLink
+from mininet.util import dumpNodeConnections
+from mininet.log import setLogLevel
+
+class SingleSwitchTopo(Topo):
+    "Single switch connected to n hosts."
+    def __init__(self, n=2, **opts):
+        Topo.__init__(self, **opts)
+        switch = self.add_switch('s1')
+        for h in range(n):
+            # Each host gets 50%/n of system CPU
+            host = self.add_host('h%s' % (h + 1),
+                                 cpu=.5 / n)
+            # 10 Mbps, 5ms delay, 10% loss
+            self.add_link(host, switch,
+                          bw=10, delay='5ms', loss=10, use_htb=True)
+
+def perfTest():
+    "Create network and run simple performance test"
+    topo = SingleSwitchTopo(n=4)
+    net = Mininet(topo=topo,
+                  host=CPULimitedHost, link=TCLink)
+    net.start()
+    print "Dumping host connections"
+    dumpNodeConnections(net.hosts)
+    print "Testing network connectivity"
+    net.pingAll()
+    print "Testing bandwidth between h1 and h4"
+    h1, h4 = net.getNodeByName('h1', 'h4')
+    net.iperf((h1, h4))
+    net.stop()
+
+if __name__ == '__main__':
+    setLogLevel('info')
+    perfTest()

examples/sshd.py

@@ -21,7 +21,7 @@ from mininet.cli import CLI
 from mininet.log import lg
 from mininet.node import Node, OVSKernelSwitch
 from mininet.topolib import TreeTopo
-from mininet.util import createLink
+from mininet.link import Link
 
 def TreeNet( depth=1, fanout=2, **kwargs ):
     "Convenience function for creating tree networks."
@@ -37,13 +37,13 @@ def connectToRootNS( network, switch, ip, prefixLen, routes ):
       routes: host networks to route to"""
     # Create a node in root namespace and link to switch 0
     root = Node( 'root', inNamespace=False )
-    intf = createLink( root, switch )[ 0 ]
-    root.setIP( intf, ip, prefixLen )
+    intf = Link( root, switch ).intf1
+    root.setIP( ip, prefixLen, intf )
     # Start network that now includes link to root namespace
     network.start()
     # Add routes from root ns to hosts
     for route in routes:
-        root.cmd( 'route add -net ' + route + ' dev ' + intf )
+        root.cmd( 'route add -net ' + route + ' dev ' + str( intf ) )
 
 def sshd( network, cmd='/usr/sbin/sshd', opts='-D' ):
     "Start a network, connect it to root ns, and run sshd on all hosts."

mininet/clean.py

@@ -45,6 +45,12 @@ def cleanup():
         if dp != '':
             sh( 'dpctl deldp ' + dp )
 
+    info( "***  Removing OVS datapaths" )
+    dps = sh("ovs-vsctl list-br").split( '\n' )
+    for dp in dps:
+        if dp:
+            sh( 'ovs-vsctl del-br ' + dp )
+
     info( "*** Removing all links of the pattern foo-ethX\n" )
     links = sh( "ip link show | egrep -o '(\w+-eth\w+)'" ).split( '\n' )
     for link in links:

mininet/cli.py

@@ -30,10 +30,11 @@ from cmd import Cmd
 from os import isatty
 from select import poll, POLLIN
 import sys
+import time
 
 from mininet.log import info, output, error
 from mininet.term import makeTerms
-from mininet.util import quietRun, isShellBuiltin
+from mininet.util import quietRun, isShellBuiltin, dumpNodeConnections
 
 class CLI( Cmd ):
     "Simple command-line interface to talk to nodes."
@@ -46,6 +47,9 @@ class CLI( Cmd ):
         self.nodemap = {}  # map names to Node objects
         for node in self.nodelist:
             self.nodemap[ node.name ] = node
+        # Local variable bindings for py command
+        self.locals = { 'net': mininet }
+        self.locals.update( self.nodemap )
         # Attempt to handle input
         self.stdin = stdin
         self.inPoller = poll()
@@ -77,7 +81,7 @@ class CLI( Cmd ):
     # Disable pylint "Unused argument: 'arg's'" messages, as well as
     # "method could be a function" warning, since each CLI function
     # must have the same interface
-    # pylint: disable-msg=W0613,R0201
+    # pylint: disable-msg=R0201
 
     helpStr = (
         'You may also send a command to a node using:\n'
@@ -104,21 +108,14 @@ class CLI( Cmd ):
         if line is '':
             output( self.helpStr )
 
-    def do_nodes( self, line ):
+    def do_nodes( self, _line ):
         "List all nodes."
         nodes = ' '.join( [ node.name for node in sorted( self.nodelist ) ] )
         output( 'available nodes are: \n%s\n' % nodes )
 
-    def do_net( self, line ):
+    def do_net( self, _line ):
         "List network connections."
-        for switch in self.mn.switches:
-            output( switch.name, '<->' )
-            for intf in switch.intfs.values():
-                # Ugly, but pylint wants it
-                name = switch.connection.get( intf,
-                    ( None, 'Unknown ' ) )[ 1 ]
-                output( ' %s' % name )
-            output( '\n' )
+        dumpNodeConnections( self.nodelist )
 
     def do_sh( self, line ):
         "Run an external shell command"
@@ -131,7 +128,7 @@ class CLI( Cmd ):
         """Evaluate a Python expression.
            Node names may be used, e.g.: h1.cmd('ls')"""
         try:
-            result = eval( line, globals(), self.nodemap )
+            result = eval( line, globals(), self.locals )
             if not result:
                 return
             elif isinstance( result, str ):
@@ -143,11 +140,11 @@ class CLI( Cmd ):
 
     # pylint: enable-msg=W0703
 
-    def do_pingall( self, line ):
+    def do_pingall( self, _line ):
         "Ping between all hosts."
         self.mn.pingAll()
 
-    def do_pingpair( self, line ):
+    def do_pingpair( self, _line ):
         "Ping between first two hosts, useful for testing."
         self.mn.pingPair()
 
@@ -191,16 +188,16 @@ class CLI( Cmd ):
             error( 'invalid number of args: iperfudp bw src dst\n' +
                    'bw examples: 10M\n' )
 
-    def do_intfs( self, line ):
+    def do_intfs( self, _line ):
         "List interfaces."
         for node in self.nodelist:
             output( '%s: %s\n' %
-                ( node.name, ' '.join( sorted( node.intfs.values() ) ) ) )
+                ( node.name, ','.join( node.intfNames() ) ) )
 
-    def do_dump( self, line ):
+    def do_dump( self, _line ):
         "Dump node info."
         for node in self.nodelist:
-            output( '%s\n' % node )
+            output( '%s\n' % repr( node ) )
 
     def do_link( self, line ):
         "Bring link(s) between two nodes up or down."
@@ -229,7 +226,7 @@ class CLI( Cmd ):
         "Spawn gnome-terminal(s) for the given node(s)."
         self.do_xterm( line, term='gterm' )
 
-    def do_exit( self, line ):
+    def do_exit( self, _line ):
         "Exit"
         return 'exited by user command'
 
@@ -275,16 +272,19 @@ class CLI( Cmd ):
     def do_dpctl( self, line ):
         "Run dpctl command on all switches."
         args = line.split()
-        if len(args) == 0:
+        if len(args) < 1:
             error( 'usage: dpctl command [arg1] [arg2] ...\n' )
             return
-        if not self.mn.listenPort:
-            error( "can't run dpctl w/no passive listening port\n")
-            return
         for sw in self.mn.switches:
             output( '*** ' + sw.name + ' ' + ('-' * 72) + '\n' )
-            output( sw.cmd( 'dpctl ' + ' '.join(args) +
-                            ' tcp:127.0.0.1:%i' % sw.listenPort ) )
+            output( sw.dpctl( *args ) )
+
+    def do_time( self, line ):
+        "Measure time taken for any command in Mininet."
+        start = time.time()
+        self.onecmd(line)
+        elapsed = time.time() - start
+        self.stdout.write("*** Elapsed time: %0.6f secs\n" % elapsed)
 
     def default( self, line ):
         """Called on an input line when the command prefix is not recognized.
@@ -293,6 +293,8 @@ class CLI( Cmd ):
         corresponding IP addrs."""
 
         first, args, line = self.parseline( line )
+        if not args:
+            return
         if args and len(args) > 0 and args[ -1 ] == '\n':
             args = args[ :-1 ]
         rest = args.split( ' ' )
@@ -311,7 +313,7 @@ class CLI( Cmd ):
         else:
             error( '*** Unknown command: %s\n' % first )
 
-    # pylint: enable-msg=W0613,R0201
+    # pylint: enable-msg=R0201
 
     def waitForNode( self, node ):
         "Wait for a node to finish, and  print its output."
@@ -319,8 +321,8 @@ class CLI( Cmd ):
         nodePoller = poll()
         nodePoller.register( node.stdout )
         bothPoller = poll()
-        bothPoller.register( self.stdin )
-        bothPoller.register( node.stdout )
+        bothPoller.register( self.stdin, POLLIN )
+        bothPoller.register( node.stdout, POLLIN )
         if self.isatty():
             # Buffer by character, so that interactive
             # commands sort of work

mininet/link.py

@@ -0,0 +1,390 @@
+"""
+link.py: interface and link abstractions for mininet
+
+It seems useful to bundle functionality for interfaces into a single
+class.
+
+Also it seems useful to enable the possibility of multiple flavors of
+links, including:
+
+- simple veth pairs
+- tunneled links
+- patchable links (which can be disconnected and reconnected via a patchbay)
+- link simulators (e.g. wireless)
+
+Basic division of labor:
+
+  Nodes: know how to execute commands
+  Intfs: know how to configure themselves
+  Links: know how to connect nodes together
+
+Intf: basic interface object that can configure itself
+TCIntf: interface with bandwidth limiting and delay via tc
+
+Link: basic link class for creating veth pairs
+"""
+
+from mininet.log import info, error, debug
+from mininet.util import makeIntfPair
+from time import sleep
+import re
+
+class Intf( object ):
+
+    "Basic interface object that can configure itself."
+
+    def __init__( self, name, node=None, port=None, link=None, **params ):
+        """name: interface name (e.g. h1-eth0)
+           node: owning node (where this intf most likely lives)
+           link: parent link if we're part of a link
+           other arguments are passed to config()"""
+        self.node = node
+        self.name = name
+        self.link = link
+        self.mac, self.ip, self.prefixLen = None, None, None
+        # Add to node (and move ourselves if necessary )
+        node.addIntf( self, port=port )
+        # Save params for future reference
+        self.params = params
+        self.config( **params )
+
+    def cmd( self, *args, **kwargs ):
+        "Run a command in our owning node"
+        return self.node.cmd( *args, **kwargs )
+
+    def ifconfig( self, *args ):
+        "Configure ourselves using ifconfig"
+        return self.cmd( 'ifconfig', self.name, *args )
+
+    def setIP( self, ipstr, prefixLen=None ):
+        """Set our IP address"""
+        # This is a sign that we should perhaps rethink our prefix
+        # mechanism and/or the way we specify IP addresses
+        if '/' in ipstr:
+            self.ip, self.prefixLen = ipstr.split( '/' )
+            return self.ifconfig( ipstr, 'up' )
+        else:
+            self.ip, self.prefixLen = ipstr, prefixLen
+            return self.ifconfig( '%s/%s' % ( ipstr, prefixLen ) )
+
+    def setMAC( self, macstr ):
+        """Set the MAC address for an interface.
+           macstr: MAC address as string"""
+        self.mac = macstr
+        return ( self.ifconfig( 'down' ) +
+                 self.ifconfig( 'hw', 'ether', macstr ) +
+                 self.ifconfig( 'up' ) )
+
+    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
+    _macMatchRegex = re.compile( r'..:..:..:..:..:..' )
+
+    def updateIP( self ):
+        "Return updated IP address based on ifconfig"
+        ifconfig = self.ifconfig()
+        ips = self._ipMatchRegex.findall( ifconfig )
+        self.ip = ips[ 0 ] if ips else None
+        return self.ip
+
+    def updateMAC( self ):
+        "Return updated MAC address based on ifconfig"
+        ifconfig = self.ifconfig()
+        macs = self._macMatchRegex.findall( ifconfig )
+        self.mac = macs[ 0 ] if macs else None
+        return self.mac
+
+    def IP( self ):
+        "Return IP address"
+        return self.ip
+
+    def MAC( self ):
+        "Return MAC address"
+        return self.mac
+
+    def isUp( self, setUp=False ):
+        "Return whether interface is up"
+        if setUp:
+            self.ifconfig( 'up' )
+        return "UP" in self.ifconfig()
+
+    def rename( self, newname ):
+        "Rename interface"
+        self.ifconfig( 'down' )
+        result = self.cmd( 'ip link set', self.name, 'name', newname )
+        self.name = newname
+        self.ifconfig( 'up' )
+        return result
+
+    # The reason why we configure things in this way is so
+    # That the parameters can be listed and documented in
+    # the config method.
+    # Dealing with subclasses and superclasses is slightly
+    # annoying, but at least the information is there!
+
+    def setParam( self, results, method, **param ):
+        """Internal method: configure a *single* parameter
+           results: dict of results to update
+           method: config method name
+           param: arg=value (ignore if value=None)
+           value may also be list or dict"""
+        name, value = param.items()[ 0 ]
+        f = getattr( self, method, None )
+        if not f or value is None:
+            return
+        if type( value ) is list:
+            result = f( *value )
+        elif type( value ) is dict:
+            result = f( **value )
+        else:
+            result = f( value )
+        results[ name ] = result
+        return result
+
+    def config( self, mac=None, ip=None, ifconfig=None,
+                up=True, **_params ):
+        """Configure Node according to (optional) parameters:
+           mac: MAC address
+           ip: IP address
+           ifconfig: arbitrary interface configuration
+           Subclasses should override this method and call
+           the parent class's config(**params)"""
+        # If we were overriding this method, we would call
+        # the superclass config method here as follows:
+        # r = Parent.config( **params )
+        r = {}
+        self.setParam( r, 'setMAC', mac=mac )
+        self.setParam( r, 'setIP', ip=ip )
+        self.setParam( r, 'isUp', up=up )
+        self.setParam( r, 'ifconfig', ifconfig=ifconfig )
+        self.updateIP()
+        self.updateMAC()
+        return r
+
+    def delete( self ):
+        "Delete interface"
+        self.cmd( 'ip link del ' + self.name )
+        # Does it help to sleep to let things run?
+        sleep( 0.001 )
+
+    def __repr__( self ):
+        return '<%s %s>' % ( self.__class__.__name__, self.name )
+
+    def __str__( self ):
+        return self.name
+
+
+class TCIntf( Intf ):
+    """Interface customized by tc (traffic control) utility
+       Allows specification of bandwidth limits (various methods)
+       as well as delay, loss and max queue length"""
+
+    def bwCmds( self, bw=None, speedup=0, use_hfsc=False, use_tbf=False,
+                enable_ecn=False, enable_red=False ):
+        "Return tc commands to set bandwidth"
+
+        cmds, parent = [], ' root '
+
+        if bw and ( bw < 0 or bw > 1000 ):
+            error( 'Bandwidth', bw, 'is outside range 0..1000 Mbps\n' )
+
+        elif bw is not None:
+            # BL: this seems a bit brittle...
+            if ( speedup > 0 and
+                 self.node.name[0:1] == 's' ):
+                bw = speedup
+            # This may not be correct - we should look more closely
+            # at the semantics of burst (and cburst) to make sure we
+            # are specifying the correct sizes. For now I have used
+            # the same settings we had in the mininet-hifi code.
+            if use_hfsc:
+                cmds += [ '%s qdisc add dev %s root handle 1:0 hfsc default 1',
+                          '%s class add dev %s parent 1:0 classid 1:1 hfsc sc '
+                          + 'rate %fMbit ul rate %fMbit' % ( bw, bw ) ]
+            elif use_tbf:
+                latency_us = 10 * 1500 * 8 / bw
+                cmds += ['%s qdisc add dev %s root handle 1: tbf ' +
+                        'rate %fMbit burst 15000 latency %fus' %
+                         ( bw, latency_us ) ]
+            else:
+                cmds += [ '%s qdisc add dev %s root handle 1:0 htb default 1',
+                         '%s class add dev %s parent 1:0 classid 1:1 htb ' +
+                         'rate %fMbit burst 15k' % bw ]
+            parent = ' parent 1:1 '
+
+            # ECN or RED
+            if enable_ecn:
+                cmds += [ '%s qdisc add dev %s' + parent +
+                          'handle 10: red limit 1000000 ' +
+                          'min 20000 max 25000 avpkt 1000 ' +
+                          'burst 20 ' +
+                          'bandwidth %fmbit probability 1 ecn' % bw ]
+                parent = ' parent 10: '
+            elif enable_red:
+                cmds += [ '%s qdisc add dev %s' + parent +
+                          'handle 10: red limit 1000000 ' +
+                          'min 20000 max 25000 avpkt 1000 ' +
+                          'burst 20 ' +
+                          'bandwidth %fmbit probability 1' % bw ]
+                parent = ' parent 10: '
+        return cmds, parent
+
+    @staticmethod
+    def delayCmds( parent, delay=None, loss=None,
+                   max_queue_size=None ):
+        "Internal method: return tc commands for delay and loss"
+        cmds = []
+        if delay and delay < 0:
+            error( 'Negative delay', delay, '\n' )
+        elif loss and ( loss < 0 or loss > 100 ):
+            error( 'Bad loss percentage', loss, '%%\n' )
+        else:
+            # Delay/loss/max queue size
+            netemargs = '%s%s%s' % (
+                'delay %s ' % delay if delay is not None else '',
+                'loss %d ' % loss if loss is not None else '',
+                'limit %d' % max_queue_size if max_queue_size is not None
+                 else '' )
+            if netemargs:
+                cmds = [ '%s qdisc add dev %s ' + parent +
+                         ' handle 10: netem ' +
+                          netemargs ]
+        return cmds
+
+    def tc( self, cmd, tc='tc' ):
+        "Execute tc command for our interface"
+        c = cmd % (tc, self)  # Add in tc command and our name
+        debug(" *** executing command: %s\n" % c)
+        return self.cmd( c )
+
+    def config( self, bw=None, delay=None, loss=None, disable_gro=True,
+                speedup=0, use_hfsc=False, use_tbf=False, enable_ecn=False,
+                enable_red=False, max_queue_size=None, **params ):
+        "Configure the port and set its properties."
+
+        result = Intf.config( self, **params)
+
+        # Disable GRO
+        if disable_gro:
+            self.cmd( 'ethtool -K %s gro off' % self )
+
+        # Optimization: return if nothing else to configure
+        # Question: what happens if we want to reset things?
+        if ( bw is None and not delay and not loss
+             and max_queue_size is None ):
+            return
+
+        # Clear existing configuration
+        cmds = [ '%s qdisc del dev %s root' ]
+
+        # Bandwidth limits via various methods
+        bwcmds, parent = self.bwCmds( bw=bw, speedup=speedup,
+                                 use_hfsc=use_hfsc, use_tbf=use_tbf,
+                                 enable_ecn=enable_ecn,
+                                 enable_red=enable_red )
+        cmds += bwcmds
+
+        # Delay/loss/max_queue_size using netem
+        cmds += self.delayCmds( delay=delay, loss=loss,
+                                 max_queue_size=max_queue_size,
+                                 parent=parent )
+
+        # Ugly but functional: display configuration info
+        stuff = ( ( [ '%.2fMbit' % bw ] if bw is not None else [] ) +
+                  ( [ '%s delay' % delay ] if delay is not None else [] ) +
+                  ( ['%d%% loss' % loss ] if loss is not None else [] ) +
+                  ( [ 'ECN' ] if enable_ecn  else [ 'RED' ]
+                    if enable_red else [] ) )
+        info( '(' + ' '.join( stuff ) + ') ' )
+
+        # Execute all the commands in our node
+        debug("at map stage w/cmds: %s\n" % cmds)
+        tcoutputs = [ self.tc(cmd) for cmd in cmds ]
+        debug( "cmds:", cmds, '\n' )
+        debug( "outputs:", tcoutputs, '\n' )
+        result[ 'tcoutputs'] = tcoutputs
+
+        return result
+
+
+class Link( object ):
+
+    """A basic link is just a veth pair.
+       Other types of links could be tunnels, link emulators, etc.."""
+
+    def __init__( self, node1, node2, port1=None, port2=None,
+                  intfName1=None, intfName2=None,
+                  intf=Intf, cls1=None, cls2=None, params1=None,
+                  params2=None ):
+        """Create veth link to another node, making two new interfaces.
+           node1: first node
+           node2: second node
+           port1: node1 port number (optional)
+           port2: node2 port number (optional)
+           intf: default interface class/constructor
+           cls1, cls2: optional interface-specific constructors
+           intfName1: node1 interface name (optional)
+           intfName2: node2  interface name (optional)
+           params1: parameters for interface 1
+           params2: parameters for interface 2"""
+        # This is a bit awkward; it seems that having everything in
+        # params would be more orthogonal, but being able to specify
+        # in-line arguments is more convenient!
+        if port1 is None:
+            port1 = node1.newPort()
+        if port2 is None:
+            port2 = node2.newPort()
+        if not intfName1:
+            intfName1 = self.intfName( node1, port1 )
+        if not intfName2:
+            intfName2 = self.intfName( node2, port2 )
+
+        self.makeIntfPair( intfName1, intfName2 )
+
+        if not cls1:
+            cls1 = intf
+        if not cls2:
+            cls2 = intf
+        if not params1:
+            params1 = {}
+        if not params2:
+            params2 = {}
+
+        intf1 = cls1( name=intfName1, node=node1, port=port1,
+                      link=self, **params1  )
+        intf2 = cls2( name=intfName2, node=node2, port=port2,
+                      link=self, **params2 )
+
+        # All we are is dust in the wind, and our two interfaces
+        self.intf1, self.intf2 = intf1, intf2
+
+    @classmethod
+    def intfName( cls, node, n ):
+        "Construct a canonical interface name node-ethN for interface n."
+        return node.name + '-eth' + repr( n )
+
+    @classmethod
+    def makeIntfPair( cls, intf1, intf2 ):
+        """Create pair of interfaces
+           intf1: name of interface 1
+           intf2: name of interface 2
+           (override this class method [and possibly delete()]
+           to change link type)"""
+        makeIntfPair( intf1, intf2  )
+
+    def delete( self ):
+        "Delete this link"
+        self.intf1.delete()
+        self.intf2.delete()
+
+    def __str__( self ):
+        return '%s<->%s' % ( self.intf1, self.intf2 )
+
+class TCLink( Link ):
+    "Link with symmetric TC interfaces configured via opts"
+    def __init__( self, node1, node2, port1=None, port2=None,
+                  intfName1=None, intfName2=None, **params ):
+        Link.__init__( self, node1, node2, port1=port1, port2=port2,
+                       intfName1=intfName1, intfName2=intfName2,
+                       cls1=TCIntf,
+                       cls2=TCIntf,
+                       params1=params,
+                       params2=params)

mininet/net.py

@@ -1,6 +1,6 @@
 """
 
-    Mininet: A simple networking testbed for OpenFlow!
+    Mininet: A simple networking testbed for OpenFlow/SDN!
 
 author: Bob Lantz (rlantz@cs.stanford.edu)
 author: Brandon Heller (brandonh@stanford.edu)
@@ -94,225 +94,223 @@ from time import sleep
 
 from mininet.cli import CLI
 from mininet.log import info, error, debug, output
-from mininet.node import Host, UserSwitch, OVSKernelSwitch, Controller
-from mininet.node import ControllerParams
-from mininet.util import quietRun, fixLimits
-from mininet.util import createLink, macColonHex, ipStr, ipParse
+from mininet.node import Host, OVSKernelSwitch, Controller
+from mininet.link import Link, Intf
+from mininet.util import quietRun, fixLimits, numCores
+from mininet.util import macColonHex, ipStr, ipParse, netParse, ipAdd
 from mininet.term import cleanUpScreens, makeTerms
 
 class Mininet( object ):
     "Network emulation with hosts spawned in network namespaces."
 
     def __init__( self, topo=None, switch=OVSKernelSwitch, host=Host,
-                 controller=Controller,
-                 cparams=ControllerParams( '10.0.0.0', 8 ),
-                 build=True, xterms=False, cleanup=False,
+                 controller=Controller, link=Link, intf=Intf,
+                 build=True, xterms=False, cleanup=False, ipBase='10.0.0.0/8',
                  inNamespace=False,
-                 autoSetMacs=False, autoStaticArp=False, listenPort=None ):
+                 autoSetMacs=False, autoStaticArp=False, autoPinCpus=False,
+                 listenPort=None ):
         """Create Mininet object.
            topo: Topo (topology) object or None
-           switch: Switch class
-           host: Host class
-           controller: Controller class
-           cparams: ControllerParams object
+           switch: default Switch class
+           host: default Host class/constructor
+           controller: default Controller class/constructor
+           link: default Link class/constructor
+           intf: default Intf class/constructor
+           ipBase: base IP address for hosts,
            build: build now from topo?
            xterms: if build now, spawn xterms?
            cleanup: if build now, cleanup before creating?
            inNamespace: spawn switches and controller in net namespaces?
-           autoSetMacs: set MAC addrs from topo?
+           autoSetMacs: set MAC addrs automatically like IP addresses?
            autoStaticArp: set all-pairs static MAC addrs?
+           autoPinCpus: pin hosts to (real) cores (requires CPULimitedHost)?
            listenPort: base listening port to open; will be incremented for
                each additional switch in the net if inNamespace=False"""
+        self.topo = topo
         self.switch = switch
         self.host = host
         self.controller = controller
-        self.cparams = cparams
-        self.topo = topo
+        self.link = link
+        self.intf = intf
+        self.ipBase = ipBase
+        self.ipBaseNum, self.prefixLen = netParse( self.ipBase )
+        self.nextIP = 1  # start for address allocation
         self.inNamespace = inNamespace
         self.xterms = xterms
         self.cleanup = cleanup
         self.autoSetMacs = autoSetMacs
         self.autoStaticArp = autoStaticArp
+        self.autoPinCpus = autoPinCpus
+        self.numCores = numCores()
+        self.nextCore = 0  # next core for pinning hosts to CPUs
         self.listenPort = listenPort
 
         self.hosts = []
         self.switches = []
         self.controllers = []
+
         self.nameToNode = {}  # name to Node (Host/Switch) objects
-        self.idToNode = {}  # dpid to Node (Host/Switch) objects
-        self.dps = 0  # number of created kernel datapaths
+
         self.terms = []  # list of spawned xterm processes
 
-        init()
-        switch.setup()
+        Mininet.init()  # Initialize Mininet if necessary
 
         self.built = False
         if topo and build:
             self.build()
 
-    def addHost( self, name, mac=None, ip=None ):
+    def addHost( self, name, cls=None, **params ):
         """Add host.
            name: name of host to add
-           mac: default MAC address for intf 0
-           ip: default IP address for intf 0
+           cls: custom host class/constructor (optional)
+           params: parameters for host
            returns: added host"""
-        host = self.host( name, defaultMAC=mac, defaultIP=ip )
-        self.hosts.append( host )
-        self.nameToNode[ name ] = host
-        return host
+        # Default IP and MAC addresses
+        defaults = { 'ip': ipAdd( self.nextIP,
+                                  ipBaseNum=self.ipBaseNum,
+                                  prefixLen=self.prefixLen ) }
+        if self.autoSetMacs:
+            defaults[ 'mac'] = macColonHex( self.nextIP )
+        if self.autoPinCpus:
+            defaults[ 'cores' ] = self.nextCore
+            self.nextCore = ( self.nextCore + 1 ) % self.numCores
+        self.nextIP += 1
+        defaults.update( params )
+        if not cls:
+            cls = self.host
+        h = cls( name, **defaults )
+        self.hosts.append( h )
+        self.nameToNode[ name ] = h
+        return h
 
-    def addSwitch( self, name, mac=None, ip=None ):
+    def addSwitch( self, name, cls=None, **params ):
         """Add switch.
            name: name of switch to add
-           mac: default MAC address for kernel/OVS switch intf 0
+           cls: custom switch class/constructor (optional)
            returns: added switch
-           side effect: increments the listenPort member variable."""
-        if self.switch == UserSwitch:
-            sw = self.switch( name, listenPort=self.listenPort,
-                defaultMAC=mac, defaultIP=ip, inNamespace=self.inNamespace )
-        else:
-            sw = self.switch( name, listenPort=self.listenPort,
-                defaultMAC=mac, defaultIP=ip, dp=self.dps,
-                inNamespace=self.inNamespace )
+           side effect: increments listenPort ivar ."""
+        defaults = { 'listenPort': self.listenPort,
+                     'inNamespace': self.inNamespace }
+        defaults.update( params )
+        if not cls:
+            cls = self.switch
+        sw = cls( name, **defaults )
         if not self.inNamespace and self.listenPort:
             self.listenPort += 1
-        self.dps += 1
         self.switches.append( sw )
         self.nameToNode[ name ] = sw
         return sw
 
-    def addController( self, name='c0', controller=None, **kwargs ):
+    def addController( self, name='c0', controller=None, **params ):
         """Add controller.
            controller: Controller class"""
         if not controller:
             controller = self.controller
-        controller_new = controller( name, **kwargs )
+        controller_new = controller( name, **params )
         if controller_new:  # allow controller-less setups
             self.controllers.append( controller_new )
             self.nameToNode[ name ] = controller_new
         return controller_new
 
-    # Control network support:
-    #
-    # Create an explicit control network. Currently this is only
-    # used by the user datapath configuration.
-    #
-    # Notes:
-    #
-    # 1. If the controller and switches are in the same (e.g. root)
-    #    namespace, they can just use the loopback connection.
-    #
-    # 2. If we can get unix domain sockets to work, we can use them
-    #    instead of an explicit control network.
-    #
-    # 3. Instead of routing, we could bridge or use 'in-band' control.
-    #
-    # 4. Even if we dispense with this in general, it could still be
-    #    useful for people who wish to simulate a separate control
-    #    network (since real networks may need one!)
+    # BL: is this better than just using nameToNode[] ?
+    # Should it have a better name?
+    def getNodeByName( self, *args ):
+        "Return node(s) with given name(s)"
+        if len( args ) == 1:
+            return self.nameToNode[ args[ 0 ] ]
+        return [ self.nameToNode[ n ] for n in args ]
 
-    def configureControlNetwork( self ):
-        "Configure control network."
-        self.configureRoutedControlNetwork()
+    def get( self, *args ):
+        "Convenience alias for getNodeByName"
+        return self.getNodeByName( *args )
 
-    # We still need to figure out the right way to pass
-    # in the control network location.
-
-    def configureRoutedControlNetwork( self, ip='192.168.123.1',
-        prefixLen=16 ):
-        """Configure a routed control network on controller and switches.
-           For use with the user datapath only right now.
-           """
-        controller = self.controllers[ 0 ]
-        info( controller.name + ' <->' )
-        cip = ip
-        snum = ipParse( ip )
-        for switch in self.switches:
-            info( ' ' + switch.name )
-            sintf, cintf = createLink( switch, controller )
-            snum += 1
-            while snum & 0xff in [ 0, 255 ]:
-                snum += 1
-            sip = ipStr( snum )
-            controller.setIP( cintf, cip, prefixLen )
-            switch.setIP( sintf, sip, prefixLen )
-            controller.setHostRoute( sip, cintf )
-            switch.setHostRoute( cip, sintf )
-        info( '\n' )
-        info( '*** Testing control network\n' )
-        while not controller.intfIsUp( cintf ):
-            info( '*** Waiting for', cintf, 'to come up\n' )
-            sleep( 1 )
-        for switch in self.switches:
-            while not switch.intfIsUp( sintf ):
-                info( '*** Waiting for', sintf, 'to come up\n' )
-                sleep( 1 )
-            if self.ping( hosts=[ switch, controller ] ) != 0:
-                error( '*** Error: control network test failed\n' )
-                exit( 1 )
-        info( '\n' )
+    def addLink( self, node1, node2, port1=None, port2=None,
+                 cls=None, **params ):
+        """"Add a link from node1 to node2
+            node1: source node
+            node2: dest node
+            port1: source port
+            port2: dest port
+            returns: link object"""
+        defaults = { 'port1': port1,
+                     'port2': port2,
+                     'intf': self.intf }
+        defaults.update( params )
+        if not cls:
+            cls = self.link
+        return cls( node1, node2, **defaults )
 
     def configHosts( self ):
         "Configure a set of hosts."
-        # params were: hosts, ips
         for host in self.hosts:
-            hintf = host.intfs[ 0 ]
-            host.setIP( hintf, host.defaultIP, self.cparams.prefixLen )
-            host.setDefaultRoute( hintf )
-            # You're low priority, dude!
-            quietRun( 'renice +18 -p ' + repr( host.pid ) )
             info( host.name + ' ' )
+            intf = host.defaultIntf()
+            if intf:
+                host.configDefault( defaultRoute=intf )
+            else:
+                # Don't configure nonexistent intf
+                host.configDefault( ip=None, mac=None )
+            # You're low priority, dude!
+            # BL: do we want to do this here or not?
+            # May not make sense if we have CPU lmiting...
+            # quietRun( 'renice +18 -p ' + repr( host.pid ) )
+            # This may not be the right place to do this, but
+            # it needs to be done somewhere.
+            host.cmd( 'ifconfig lo up' )
         info( '\n' )
 
-    def buildFromTopo( self, topo ):
+    def buildFromTopo( self, topo=None ):
         """Build mininet from a topology object
            At the end of this function, everything should be connected
            and up."""
 
-        def addNode( prefix, addMethod, nodeId ):
-            "Add a host or a switch."
-            name = prefix + topo.name( nodeId )
-            mac = macColonHex( nodeId ) if self.setMacs else None
-            ip = topo.ip( nodeId )
-            node = addMethod( name, mac=mac, ip=ip )
-            self.idToNode[ nodeId ] = node
-            info( name + ' ' )
-
         # Possibly we should clean up here and/or validate
         # the topo
         if self.cleanup:
             pass
 
-        info( '*** Adding controller\n' )
-        self.addController( 'c0' )
         info( '*** Creating network\n' )
+
+        if not self.controllers:
+            # Add a default controller
+            info( '*** Adding controller\n' )
+            self.addController( 'c0' )
+
         info( '*** Adding hosts:\n' )
-        for hostId in sorted( topo.hosts() ):
-            addNode( 'h', self.addHost, hostId )
+        for hostName in topo.hosts():
+            self.addHost( hostName, **topo.nodeInfo( hostName ) )
+            info( hostName + ' ' )
+
         info( '\n*** Adding switches:\n' )
-        for switchId in sorted( topo.switches() ):
-            addNode( 's', self.addSwitch, switchId )
+        for switchName in topo.switches():
+            self.addSwitch( switchName, **topo.nodeInfo( switchName) )
+            info( switchName + ' ' )
+
         info( '\n*** Adding links:\n' )
-        for srcId, dstId in sorted( topo.edges() ):
-            src, dst = self.idToNode[ srcId ], self.idToNode[ dstId ]
-            srcPort, dstPort = topo.port( srcId, dstId )
-            createLink( src, dst, srcPort, dstPort )
+        for srcName, dstName in topo.links(sort=True):
+            src, dst = self.nameToNode[ srcName ], self.nameToNode[ dstName ]
+            params = topo.linkInfo( srcName, dstName )
+            srcPort, dstPort = topo.port( srcName, dstName )
+            self.addLink( src, dst, srcPort, dstPort, **params )
             info( '(%s, %s) ' % ( src.name, dst.name ) )
+
         info( '\n' )
 
+    def configureControlNetwork( self ):
+        "Control net config hook: override in subclass"
+        raise Exception( 'configureControlNetwork: '
+               'should be overriden in subclass', self )
+
     def build( self ):
         "Build mininet."
         if self.topo:
             self.buildFromTopo( self.topo )
-        if self.inNamespace:
-            info( '*** Configuring control network\n' )
+        if ( self.inNamespace ):
             self.configureControlNetwork()
         info( '*** Configuring hosts\n' )
         self.configHosts()
         if self.xterms:
             self.startTerms()
-        if self.autoSetMacs:
-            self.setMacs()
         if self.autoStaticArp:
             self.staticArp()
         self.built = True
@@ -327,17 +325,10 @@ class Mininet( object ):
 
     def stopXterms( self ):
         "Kill each xterm."
-        # Kill xterms
         for term in self.terms:
             os.kill( term.pid, signal.SIGKILL )
         cleanUpScreens()
 
-    def setMacs( self ):
-        """Set MAC addrs to correspond to default MACs on hosts.
-           Assume that the host only has one interface."""
-        for host in self.hosts:
-            host.setMAC( host.intfs[ 0 ], host.defaultMAC )
-
     def staticArp( self ):
         "Add all-pairs ARP entries to remove the need to handle broadcast."
         for src in self.hosts:
@@ -365,18 +356,19 @@ class Mininet( object ):
             self.stopXterms()
         info( '*** Stopping %i hosts\n' % len( self.hosts ) )
         for host in self.hosts:
-            info( '%s ' % host.name )
+            info( host.name + ' ' )
             host.terminate()
         info( '\n' )
         info( '*** Stopping %i switches\n' % len( self.switches ) )
         for switch in self.switches:
-            info( switch.name )
+            info( switch.name + ' ' )
             switch.stop()
         info( '\n' )
         info( '*** Stopping %i controllers\n' % len( self.controllers ) )
         for controller in self.controllers:
+            info( controller.name + ' ' )
             controller.stop()
-        info( '*** Done\n' )
+        info( '\n*** Done\n' )
 
     def run( self, test, *args, **kwargs ):
         "Perform a complete start/test/stop cycle."
@@ -410,6 +402,9 @@ class Mininet( object ):
             if not ready and timeoutms >= 0:
                 yield None, None
 
+    # XXX These test methods should be moved out of this class.
+    # Probably we should create a tests.py for them
+
     @staticmethod
     def _parsePing( pingOutput ):
         "Parse ping output and return packets sent, received."
@@ -481,6 +476,8 @@ class Mininet( object ):
             error( 'could not parse iperf output: ' + iperfOutput )
             return ''
 
+    # XXX This should be cleaned up
+
     def iperf( self, hosts=None, l4Type='TCP', udpBw='10M' ):
         """Run iperf between two hosts.
            hosts: list of hosts; if None, uses opposite hosts
@@ -508,10 +505,11 @@ class Mininet( object ):
         servout = ''
         while server.lastPid is None:
             servout += server.monitor()
-        while 'Connected' not in client.cmd(
-            'sh -c "echo A | telnet -e A %s 5001"' % server.IP()):
-            output('waiting for iperf to start up')
-            sleep(.5)
+        if l4Type == 'TCP':
+            while 'Connected' not in client.cmd(
+                'sh -c "echo A | telnet -e A %s 5001"' % server.IP()):
+                output('waiting for iperf to start up...')
+                sleep(.5)
         cliout = client.cmd( iperfArgs + '-t 5 -c ' + server.IP() + ' ' +
                            bwArgs )
         debug( 'Client output: %s\n' % cliout )
@@ -524,6 +522,8 @@ class Mininet( object ):
         output( '*** Results: %s\n' % result )
         return result
 
+    # BL: I think this can be rewritten now that we have
+    # a real link class.
     def configLinkStatus( self, src, dst, status ):
         """Change status of src <-> dst links.
            src: node name
@@ -534,15 +534,18 @@ class Mininet( object ):
         elif dst not in self.nameToNode:
             error( 'dst not in network: %s\n' % dst )
         else:
-            srcNode, dstNode = self.nameToNode[ src ], self.nameToNode[ dst ]
-            connections = srcNode.connectionsTo( dstNode )
+            if type( src ) is str:
+                src = self.nameToNode[ src ]
+            if type( dst ) is str:
+                dst = self.nameToNode[ dst ]
+            connections = src.connectionsTo( dst )
             if len( connections ) == 0:
                 error( 'src and dst not connected: %s %s\n' % ( src, dst) )
             for srcIntf, dstIntf in connections:
-                result = srcNode.cmd( 'ifconfig', srcIntf, status )
+                result = srcIntf.ifconfig( status )
                 if result:
                     error( 'link src status change failed: %s\n' % result )
-                result = dstNode.cmd( 'ifconfig', dstIntf, status )
+                result = dstIntf.ifconfig( status )
                 if result:
                     error( 'link dst status change failed: %s\n' % result )
 
@@ -553,26 +556,89 @@ class Mininet( object ):
         self.stop()
         return result
 
+    inited = False
 
-# pylint thinks inited is unused
-# pylint: disable-msg=W0612
+    @classmethod
+    def init( cls ):
+        "Initialize Mininet"
+        if cls.inited:
+            return
+        if os.getuid() != 0:
+            # Note: this script must be run as root
+            # Probably we should only sudo when we need
+            # to as per Big Switch's patch
+            print "*** Mininet must run as root."
+            exit( 1 )
+        fixLimits()
+        cls.inited = True
 
-def init():
-    "Initialize Mininet."
-    if init.inited:
-        return
-    if os.getuid() != 0:
-        # Note: this script must be run as root
-        # Perhaps we should do so automatically!
-        print "*** Mininet must run as root."
-        exit( 1 )
-    # If which produces no output, then mnexec is not in the path.
-    # May want to loosen this to handle mnexec in the current dir.
-    if not quietRun( 'which mnexec' ):
-        raise Exception( "Could not find mnexec - check $PATH" )
-    fixLimits()
-    init.inited = True
 
-init.inited = False
+class MininetWithControlNet( Mininet ):
 
-# pylint: enable-msg=W0612
+    """Control network support:
+
+       Create an explicit control network. Currently this is only
+       used/usable with the user datapath.
+
+       Notes:
+
+       1. If the controller and switches are in the same (e.g. root)
+          namespace, they can just use the loopback connection.
+
+       2. If we can get unix domain sockets to work, we can use them
+          instead of an explicit control network.
+
+       3. Instead of routing, we could bridge or use 'in-band' control.
+
+       4. Even if we dispense with this in general, it could still be
+          useful for people who wish to simulate a separate control
+          network (since real networks may need one!)
+
+       5. Basically nobody ever used this code, so it has been moved
+          into its own class.
+
+       6. Ultimately we may wish to extend this to allow us to create a
+          control network which every node's control interface is
+          attached to."""
+
+    def configureControlNetwork( self ):
+        "Configure control network."
+        self.configureRoutedControlNetwork()
+
+    # We still need to figure out the right way to pass
+    # in the control network location.
+
+    def configureRoutedControlNetwork( self, ip='192.168.123.1',
+        prefixLen=16 ):
+        """Configure a routed control network on controller and switches.
+           For use with the user datapath only right now."""
+        controller = self.controllers[ 0 ]
+        info( controller.name + ' <->' )
+        cip = ip
+        snum = ipParse( ip )
+        for switch in self.switches:
+            info( ' ' + switch.name )
+            link = self.link( switch, controller, port1=0 )
+            sintf, cintf = link.intf1, link.intf2
+            switch.controlIntf = sintf
+            snum += 1
+            while snum & 0xff in [ 0, 255 ]:
+                snum += 1
+            sip = ipStr( snum )
+            cintf.setIP( cip, prefixLen )
+            sintf.setIP( sip, prefixLen )
+            controller.setHostRoute( sip, cintf )
+            switch.setHostRoute( cip, sintf )
+        info( '\n' )
+        info( '*** Testing control network\n' )
+        while not cintf.isUp():
+            info( '*** Waiting for', cintf, 'to come up\n' )
+            sleep( 1 )
+        for switch in self.switches:
+            while not sintf.isUp():
+                info( '*** Waiting for', sintf, 'to come up\n' )
+                sleep( 1 )
+            if self.ping( hosts=[ switch, controller ] ) != 0:
+                error( '*** Error: control network test failed\n' )
+                exit( 1 )
+        info( '\n' )

mininet/test/test_nets.py

@@ -5,16 +5,14 @@
 
 import unittest
 
-from mininet.net import init, Mininet
-from mininet.node import Host, Controller, ControllerParams
-# from mininet.node import KernelSwitch
+from mininet.net import Mininet
+from mininet.node import Host, Controller
 from mininet.node import UserSwitch, OVSKernelSwitch
 from mininet.topo import SingleSwitchTopo, LinearTopo
 from mininet.log import setLogLevel
 
 SWITCHES = { 'user': UserSwitch,
              'ovsk': OVSKernelSwitch,
-            # 'kernel': KernelSwitch
 }
 
 
@@ -23,21 +21,15 @@ class testSingleSwitch( unittest.TestCase ):
 
     def testMinimal( self ):
         "Ping test with both datapaths on minimal topology"
-        init()
         for switch in SWITCHES.values():
-            controllerParams = ControllerParams( '10.0.0.0', 8 )
-            mn = Mininet( SingleSwitchTopo(), switch, Host, Controller,
-                         controllerParams )
+            mn = Mininet( SingleSwitchTopo(), switch, Host, Controller )
             dropped = mn.run( mn.ping )
             self.assertEqual( dropped, 0 )
 
     def testSingle5( self ):
         "Ping test with both datapaths on 5-host single-switch topology"
-        init()
         for switch in SWITCHES.values():
-            controllerParams = ControllerParams( '10.0.0.0', 8 )
-            mn = Mininet( SingleSwitchTopo( k=5 ), switch, Host, Controller,
-                         controllerParams )
+            mn = Mininet( SingleSwitchTopo( k=5 ), switch, Host, Controller )
             dropped = mn.run( mn.ping )
             self.assertEqual( dropped, 0 )
 
@@ -47,15 +39,12 @@ class testLinear( unittest.TestCase ):
 
     def testLinear5( self ):
         "Ping test with both datapaths on a 5-switch topology"
-        init()
         for switch in SWITCHES.values():
-            controllerParams = ControllerParams( '10.0.0.0', 8 )
-            mn = Mininet( LinearTopo( k=5 ), switch, Host, Controller,
-                         controllerParams )
+            mn = Mininet( LinearTopo( k=5 ), switch, Host, Controller )
             dropped = mn.run( mn.ping )
             self.assertEqual( dropped, 0 )
 
 
 if __name__ == '__main__':
-    setLogLevel('warning')
+    setLogLevel( 'warning' )
     unittest.main()

mininet/topo.py

@@ -16,258 +16,125 @@ setup for testing, and can even be emulated with the Mininet package.
 # from networkx.classes.graph import Graph
 
 from networkx import Graph
-from mininet.node import SWITCH_PORT_BASE
-
-class NodeID(object):
-    '''Topo node identifier.'''
-
-    def __init__(self, dpid = None):
-        '''Init.
-
-        @param dpid dpid
-        '''
-        # DPID-compatible hashable identifier: opaque 64-bit unsigned int
-        self.dpid = dpid
-
-    def __str__(self):
-        '''String conversion.
-
-        @return str dpid as string
-        '''
-        return str(self.dpid)
-
-    def name_str(self):
-        '''Name conversion.
-
-        @return name name as string
-        '''
-        return str(self.dpid)
-
-    def ip_str(self):
-        '''Name conversion.
-
-        @return ip ip as string
-        '''
-        hi = (self.dpid & 0xff0000) >> 16
-        mid = (self.dpid & 0xff00) >> 8
-        lo = self.dpid & 0xff
-        return "10.%i.%i.%i" % (hi, mid, lo)
-
-
-class Node(object):
-    '''Node-specific vertex metadata for a Topo object.'''
-
-    def __init__(self, connected = False, admin_on = True,
-                 power_on = True, fault = False, is_switch = True):
-        '''Init.
-
-        @param connected actively connected to controller
-        @param admin_on administratively on or off
-        @param power_on powered on or off
-        @param fault fault seen on node
-        @param is_switch switch or host
-        '''
-        self.connected = connected
-        self.admin_on = admin_on
-        self.power_on = power_on
-        self.fault = fault
-        self.is_switch = is_switch
-
-
-class Edge(object):
-    '''Edge-specific metadata for a StructuredTopo graph.'''
-
-    def __init__(self, admin_on = True, power_on = True, fault = False):
-        '''Init.
-
-        @param admin_on administratively on or off; defaults to True
-        @param power_on powered on or off; defaults to True
-        @param fault fault seen on edge; defaults to False
-        '''
-        self.admin_on = admin_on
-        self.power_on = power_on
-        self.fault = fault
-
+from mininet.util import irange, natural, naturalSeq
 
 class Topo(object):
-    '''Data center network representation for structured multi-trees.'''
+    "Data center network representation for structured multi-trees."
 
-    def __init__(self):
-        '''Create Topo object.
-
-        '''
+    def __init__(self, hopts=None, sopts=None, lopts=None):
+        """Topo object:
+           hinfo: default host options
+           sopts: default switch options
+           lopts: default link options"""
         self.g = Graph()
-        self.node_info = {}  # dpids hash to Node objects
-        self.edge_info = {}  # (src_dpid, dst_dpid) tuples hash to Edge objects
+        self.node_info = {}
+        self.link_info = {}  # (src, dst) tuples hash to EdgeInfo objects
+        self.hopts = {} if hopts is None else hopts
+        self.sopts = {} if sopts is None else sopts
+        self.lopts = {} if lopts is None else lopts
         self.ports = {}  # ports[src][dst] is port on src that connects to dst
-        self.id_gen = NodeID  # class used to generate dpid
 
-    def add_node(self, dpid, node):
-        '''Add Node to graph.
+    def add_node(self, name, **opts):
+        """Add Node to graph.
+           name: name
+           opts: node options
+           returns: node name"""
+        self.g.add_node(name)
+        self.node_info[name] = opts
+        return name
 
-        @param dpid dpid
-        @param node Node object
-        '''
-        self.g.add_node(dpid)
-        self.node_info[dpid] = node
+    def add_host(self, name, **opts):
+        """Convenience method: Add host to graph.
+           name: host name
+           opts: host options
+           returns: host name"""
+        if not opts and self.hopts:
+            opts = self.hopts
+        return self.add_node(name, **opts)
 
-    def add_edge(self, src, dst, edge = None):
-        '''Add edge (Node, Node) to graph.
+    def add_switch(self, name, **opts):
+        """Convenience method: Add switch to graph.
+           name: switch name
+           opts: switch options
+           returns: switch name"""
+        if not opts and self.sopts:
+            opts = self.sopts
+        result = self.add_node(name, is_switch=True, **opts)
+        return result
 
-        @param src src dpid
-        @param dst dst dpid
-        @param edge Edge object
-        '''
-        src, dst = tuple(sorted([src, dst]))
-        self.g.add_edge(src, dst)
-        if not edge:
-            edge = Edge()
-        self.edge_info[(src, dst)] = edge
-        self.add_port(src, dst)
+    def add_link(self, node1, node2, port1=None, port2=None,
+                 **opts):
+        """node1, node2: nodes to link together
+           port1, port2: ports (optional)
+           opts: link options (optional)
+           returns: link info key"""
+        if not opts and self.lopts:
+            opts = self.lopts
+        self.add_port(node1, node2, port1, port2)
+        key = tuple(self.sorted([node1, node2]))
+        self.link_info[key] = opts
+        self.g.add_edge(*key)
+        return key
 
-    def add_port(self, src, dst):
+    def add_port(self, src, dst, sport=None, dport=None):
         '''Generate port mapping for new edge.
-
-        @param src source switch DPID
-        @param dst destination switch DPID
+        @param src source switch name
+        @param dst destination switch name
         '''
-        src_base = SWITCH_PORT_BASE if self.is_switch(src) else 0
-        dst_base = SWITCH_PORT_BASE if self.is_switch(dst) else 0
-        if src not in self.ports:
-            self.ports[src] = {}
-        if dst not in self.ports[src]:
-            # num outlinks
-            self.ports[src][dst] = len(self.ports[src]) + src_base
-        if dst not in self.ports:
-            self.ports[dst] = {}
-        if src not in self.ports[dst]:
-            # num outlinks
-            self.ports[dst][src] = len(self.ports[dst]) + dst_base
+        self.ports.setdefault(src, {})
+        self.ports.setdefault(dst, {})
+        # New port: number of outlinks + base
+        src_base = 1 if self.is_switch(src) else 0
+        dst_base = 1 if self.is_switch(dst) else 0
+        if sport is None:
+            sport = len(self.ports[src]) + src_base
+        if dport is None:
+            dport = len(self.ports[dst]) + dst_base
+        self.ports[src][dst] = sport
+        self.ports[dst][src] = dport
 
-    def node_enabled(self, dpid):
-        '''Is node connected, admin on, powered on, and fault-free?
-
-        @param dpid dpid
-
-        @return bool node is enabled
-        '''
-        ni = self.node_info[dpid]
-        return ni.connected and ni.admin_on and ni.power_on and not ni.fault
-
-    def nodes_enabled(self, dpids, enabled = True):
-        '''Return subset of enabled nodes
-
-        @param dpids list of dpids
-        @param enabled only return enabled nodes?
-
-        @return dpids filtered list of dpids
-        '''
-        if enabled:
-            return [n for n in dpids if self.node_enabled(n)]
+    def nodes(self, sort=True):
+        "Return nodes in graph"
+        if sort:
+            return self.sorted( self.g.nodes() )
         else:
-            return dpids
-
-    def nodes(self, enabled = True):
-        '''Return graph nodes.
-
-        @param enabled only return enabled nodes?
-
-        @return dpids list of dpids
-        '''
-        return self.nodes_enabled(self.g.nodes(), enabled)
-
-    def nodes_str(self, dpids):
-        '''Return string of custom-encoded nodes.
-
-        @param dpids list of dpids
-
-        @return str string
-        '''
-        return [str(self.id_gen(dpid = dpid)) for dpid in dpids]
+            return self.g.nodes()
 
     def is_switch(self, n):
         '''Returns true if node is a switch.'''
-        return self.node_info[n].is_switch
+        info = self.node_info[n]
+        return info and info.get('is_switch', False)
 
-    def switches(self, enabled = True):
+    def switches(self, sort=True):
         '''Return switches.
-
-        @param enabled only return enabled nodes?
-
+        sort: sort switches alphabetically
         @return dpids list of dpids
         '''
-        nodes = [n for n in self.g.nodes() if self.is_switch(n)]
-        return self.nodes_enabled(nodes, enabled)
+        return [n for n in self.nodes(sort) if self.is_switch(n)]
 
-    def hosts(self, enabled = True):
+    def hosts(self, sort=True):
         '''Return hosts.
-
-        @param enabled only return enabled nodes?
-
+        sort: sort hosts alphabetically
         @return dpids list of dpids
         '''
+        return [n for n in self.nodes(sort) if not self.is_switch(n)]
 
-        def is_host(n):
-            '''Returns true if node is a host.'''
-            return not self.node_info[n].is_switch
-
-        nodes = [n for n in self.g.nodes() if is_host(n)]
-        return self.nodes_enabled(nodes, enabled)
-
-    def edge_enabled(self, edge):
-        '''Is edge admin on, powered on, and fault-free?
-
-        @param edge (src, dst) dpid tuple
-
-        @return bool edge is enabled
+    def links(self, sort=True):
+        '''Return links.
+        sort: sort links alphabetically
+        @return links list of name pairs
         '''
-        src, dst = edge
-        src, dst = tuple(sorted([src, dst]))
-        ei = self.edge_info[tuple(sorted([src, dst]))]
-        return ei.admin_on and ei.power_on and not ei.fault
-
-    def edges_enabled(self, edges, enabled = True):
-        '''Return subset of enabled edges
-
-        @param edges list of edges
-        @param enabled only return enabled edges?
-
-        @return edges filtered list of edges
-        '''
-        if enabled:
-            return [e for e in edges if self.edge_enabled(e)]
+        if not sort:
+            return self.g.edges()
         else:
-            return edges
-
-    def edges(self, enabled = True):
-        '''Return edges.
-
-        @param enabled only return enabled edges?
-
-        @return edges list of dpid pairs
-        '''
-        return self.edges_enabled(self.g.edges(), enabled)
-
-    def edges_str(self, dpid_pairs):
-        '''Return string of custom-encoded node pairs.
-
-        @param dpid_pairs list of dpid pairs (src, dst)
-
-        @return str string
-        '''
-        edges = []
-        for pair in dpid_pairs:
-            src, dst = pair
-            src = str(self.id_gen(dpid = src))
-            dst = str(self.id_gen(dpid = dst))
-            edges.append((src, dst))
-        return edges
+            links = [tuple(self.sorted(e)) for e in self.g.edges()]
+            return sorted( links, key=naturalSeq )
 
     def port(self, src, dst):
         '''Get port number.
 
-        @param src source switch DPID
-        @param dst destination switch DPID
+        @param src source switch name
+        @param dst destination switch name
         @return tuple (src_port, dst_port):
             src_port: port on source switch leading to the destination switch
             dst_port: port on destination switch leading to the source switch
@@ -276,130 +143,88 @@ class Topo(object):
             assert dst in self.ports and src in self.ports[dst]
             return (self.ports[src][dst], self.ports[dst][src])
 
-    def enable_edges(self):
-        '''Enable all edges in the network graph.
+    def linkInfo( self, src, dst ):
+        "Return link metadata"
+        src, dst = self.sorted([src, dst])
+        return self.link_info[(src, dst)]
 
-        Set admin on, power on, and fault off.
-        '''
-        for e in self.g.edges():
-            src, dst = e
-            ei = self.edge_info[tuple(sorted([src, dst]))]
-            ei.admin_on = True
-            ei.power_on = True
-            ei.fault = False
+    def setlinkInfo( self, src, dst, info ):
+        "Set link metadata"
+        src, dst = self.sorted([src, dst])
+        self.link_info[(src, dst)] = info
 
-    def enable_nodes(self):
-        '''Enable all nodes in the network graph.
+    def nodeInfo( self, name ):
+        "Return metadata (dict) for node"
+        info = self.node_info[ name ]
+        return info if info is not None else {}
 
-        Set connected on, admin on, power on, and fault off.
-        '''
-        for node in self.g.nodes():
-            ni = self.node_info[node]
-            ni.connected = True
-            ni.admin_on = True
-            ni.power_on = True
-            ni.fault = False
-
-    def enable_all(self):
-        '''Enable all nodes and edges in the network graph.'''
-        self.enable_nodes()
-        self.enable_edges()
-
-    def name(self, dpid):
-        '''Get string name of node ID.
-
-        @param dpid DPID of host or switch
-        @return name_str string name with no dashes
-        '''
-        return self.id_gen(dpid = dpid).name_str()
-
-    def ip(self, dpid):
-        '''Get IP dotted-decimal string of node ID.
-
-        @param dpid DPID of host or switch
-        @return ip_str
-        '''
-        return self.id_gen(dpid = dpid).ip_str()
+    def setNodeInfo( self, name, info ):
+        "Set metadata (dict) for node"
+        self.node_info[ name ] = info
 
+    @staticmethod
+    def sorted( items ):
+        "Items sorted in natural (i.e. alphabetical) order"
+        return sorted(items, key=natural)
 
 class SingleSwitchTopo(Topo):
     '''Single switch connected to k hosts.'''
 
-    def __init__(self, k = 2, enable_all = True):
+    def __init__(self, k=2, **opts):
         '''Init.
 
         @param k number of hosts
         @param enable_all enables all nodes and switches?
         '''
-        super(SingleSwitchTopo, self).__init__()
+        super(SingleSwitchTopo, self).__init__(**opts)
 
         self.k = k
 
-        self.add_node(1, Node())
-        hosts = range(2, k + 2)
-        for h in hosts:
-            self.add_node(h, Node(is_switch = False))
-            self.add_edge(h, 1, Edge())
-
-        if enable_all:
-            self.enable_all()
+        switch = self.add_switch('s1')
+        for h in irange(1, k):
+            host = self.add_host('h%s' % h)
+            self.add_link(host, switch)
 
 
-class SingleSwitchReversedTopo(SingleSwitchTopo):
+class SingleSwitchReversedTopo(Topo):
     '''Single switch connected to k hosts, with reversed ports.
 
     The lowest-numbered host is connected to the highest-numbered port.
 
     Useful to verify that Mininet properly handles custom port numberings.
     '''
-
-    def port(self, src, dst):
-        '''Get port number.
-
-        @param src source switch DPID
-        @param dst destination switch DPID
-        @return tuple (src_port, dst_port):
-            src_port: port on source switch leading to the destination switch
-            dst_port: port on destination switch leading to the source switch
-        '''
-        if src == 1:
-            if dst in range(2, self.k + 2):
-                dst_index = dst - 2
-                highest = self.k - 1
-                return (highest - dst_index, 0)
-            else:
-                raise Exception('unexpected dst: %i' % dst)
-        elif src in range(2, self.k + 2):
-            if dst == 1:
-                raise Exception('unexpected dst: %i' % dst)
-            else:
-                src_index = src - 2
-                highest = self.k - 1
-                return (0, highest - src_index)
-
-
-class LinearTopo(Topo):
-    '''Linear topology of k switches, with one host per switch.'''
-
-    def __init__(self, k = 2, enable_all = True):
+    def __init__(self, k=2, **opts):
         '''Init.
 
-        @param k number of switches (and hosts too)
+        @param k number of hosts
         @param enable_all enables all nodes and switches?
         '''
-        super(LinearTopo, self).__init__()
+        super(SingleSwitchReversedTopo, self).__init__(**opts)
+        self.k = k
+        switch = self.add_switch('s1')
+        for h in irange(1, k):
+            host = self.add_host('h%s' % h)
+            self.add_link(host, switch,
+                          port1=0, port2=(k - h + 1))
+
+class LinearTopo(Topo):
+    "Linear topology of k switches, with one host per switch."
+
+    def __init__(self, k=2, **opts):
+        """Init.
+           k: number of switches (and hosts)
+           hconf: host configuration options
+           lconf: link configuration options"""
+
+        super(LinearTopo, self).__init__(**opts)
 
         self.k = k
 
-        switches = range(1, k + 1)
-        for s in switches:
-            h = s + k
-            self.add_node(s, Node())
-            self.add_node(h, Node(is_switch = False))
-            self.add_edge(s, h, Edge())
-        for s in switches:
-            if s != k:
-                self.add_edge(s, s + 1, Edge())
-
-        if enable_all:
-            self.enable_all()
+        lastSwitch = None
+        for i in irange(1, k):
+            host = self.add_host('h%s' % i)
+            switch = self.add_switch('s%s' % i)
+            self.add_link( host, switch)
+            if lastSwitch:
+                self.add_link( switch, lastSwitch)
+            lastSwitch = switch

mininet/topolib.py

@@ -1,6 +1,6 @@
 "Library of potentially useful topologies for Mininet"
 
-from mininet.topo import Topo, Node
+from mininet.topo import Topo
 from mininet.net import Mininet
 
 class TreeTopo( Topo ):
@@ -8,36 +8,27 @@ class TreeTopo( Topo ):
 
     def __init__( self, depth=1, fanout=2 ):
         super( TreeTopo, self ).__init__()
-        # Numbering:  h1..N, sN+1..M
-        hostCount = fanout ** depth
+        # Numbering:  h1..N, s1..M
         self.hostNum = 1
-        self.switchNum = hostCount + 1
+        self.switchNum = 1
         # Build topology
         self.addTree( depth, fanout )
-        # Consider all switches and hosts 'on'
-        self.enable_all()
-
-    # It is OK that i is "unused" in the for loop.
-    # pylint: disable-msg=W0612
 
     def addTree( self, depth, fanout ):
         """Add a subtree starting with node n.
            returns: last node added"""
         isSwitch = depth > 0
         if isSwitch:
-            num = self.switchNum
+            node = self.add_switch( 's%s' % self.switchNum )
             self.switchNum += 1
-        else:
-            num = self.hostNum
-            self.hostNum += 1
-        self.add_node( num, Node( is_switch=isSwitch ) )
-        if isSwitch:
-            for i in range( 0, fanout ):
+            for _ in range( fanout ):
                 child = self.addTree( depth - 1, fanout )
-                self.add_edge( num, child )
-        return num
+                self.add_link( node, child )
+        else:
+            node = self.add_host( 'h%s' % self.hostNum )
+            self.hostNum += 1
+        return node
 
-    # pylint: enable-msg=W0612
 
 def TreeNet( depth=1, fanout=2, **kwargs ):
     "Convenience function for creating tree networks."

mininet/util.py

@@ -1,11 +1,14 @@
 "Utility functions for Mininet."
 
+from mininet.log import output, info, error
+
 from time import sleep
 from resource import setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE
-import select
+from select import poll, POLLIN
 from subprocess import call, check_call, Popen, PIPE, STDOUT
-
-from mininet.log import error
+import re
+from fcntl import fcntl, F_GETFL, F_SETFL
+from os import O_NONBLOCK
 
 # Command execution support
 
@@ -22,7 +25,7 @@ def checkRun( cmd ):
 # pylint doesn't understand explicit type checking
 # pylint: disable-msg=E1103
 
-def quietRun( *cmd ):
+def oldQuietRun( *cmd ):
     """Run a command, routing stderr to stdout, and return the output.
        cmd: list of command params"""
     if len( cmd ) == 1:
@@ -33,22 +36,87 @@ def quietRun( *cmd ):
     # We can't use Popen.communicate() because it uses
     # select(), which can't handle
     # high file descriptor numbers! poll() can, however.
-    output = ''
-    readable = select.poll()
+    out = ''
+    readable = poll()
     readable.register( popen.stdout )
     while True:
         while readable.poll():
             data = popen.stdout.read( 1024 )
             if len( data ) == 0:
                 break
-            output += data
+            out += data
         popen.poll()
         if popen.returncode != None:
             break
-    return output
+    return out
+
+
+# This is a bit complicated, but it enables us to
+# monitor command output as it is happening
+
+def errRun( *cmd, **kwargs ):
+    """Run a command and return stdout, stderr and return code
+       cmd: string or list of command and args
+       stderr: STDOUT to merge stderr with stdout
+       shell: run command using shell
+       echo: monitor output to console"""
+    # Allow passing in a list or a string
+    if len( cmd ) == 1:
+        cmd = cmd[ 0 ]
+        if isinstance( cmd, str ):
+            cmd = cmd.split( ' ' )
+    cmd = [ str( arg ) for arg in cmd ]
+    # By default we separate stderr, don't run in a shell, and don't echo
+    stderr = kwargs.get( 'stderr', PIPE )
+    shell = kwargs.get( 'shell', False )
+    echo = kwargs.get( 'echo', False )
+    if echo:
+        # cmd goes to stderr, output goes to stdout
+        info( cmd, '\n' )
+    popen = Popen( cmd, stdout=PIPE, stderr=stderr, shell=shell )
+    # We use poll() because select() doesn't work with large fd numbers,
+    # and thus communicate() doesn't work either
+    out, err = '', ''
+    poller = poll()
+    poller.register( popen.stdout, POLLIN )
+    fdtofile = { popen.stdout.fileno(): popen.stdout }
+    outDone, errDone = False, True
+    if popen.stderr:
+        fdtofile[ popen.stderr.fileno() ] = popen.stderr
+        poller.register( popen.stderr, POLLIN )
+        errDone = False
+    while not outDone or not errDone:
+        readable = poller.poll()
+        for fd, _event in readable:
+            f = fdtofile[ fd ]
+            data = f.read( 1024 )
+            if echo:
+                output( data )
+            if f == popen.stdout:
+                out += data
+                if data == '':
+                    outDone = True
+            elif f == popen.stderr:
+                err += data
+                if data == '':
+                    errDone = True
+    returncode = popen.wait()
+    return out, err, returncode
+
+def errFail( *cmd, **kwargs ):
+    "Run a command using errRun and raise exception on nonzero exit"
+    out, err, ret = errRun( *cmd, **kwargs )
+    if ret:
+        raise Exception( "errFail: %s failed with return code %s: %s"
+                         % ( cmd, ret, err ) )
+    return out, err, ret
+
+def quietRun( cmd, **kwargs ):
+    "Run a command and return merged stdout and stderr"
+    return errRun( cmd, stderr=STDOUT, **kwargs )[ 0 ]
 
 # pylint: enable-msg=E1103
-# pylint: disable-msg=E1101,W0612
+# pylint: disable-msg=E1101
 
 def isShellBuiltin( cmd ):
     "Return True if cmd is a bash builtin."
@@ -61,7 +129,7 @@ def isShellBuiltin( cmd ):
 
 isShellBuiltin.builtIns = None
 
-# pylint: enable-msg=E1101,W0612
+# pylint: enable-msg=E1101
 
 # Interface management
 #
@@ -124,25 +192,41 @@ def moveIntf( intf, node, printError=False, retries=3, delaySecs=0.001 ):
        printError: if true, print error"""
     retry( retries, delaySecs, moveIntfNoRetry, intf, node, printError )
 
-def createLink( node1, node2, port1=None, port2=None ):
-    """Create a link between nodes, making an interface for each.
-       node1: Node object
-       node2: Node object
-       port1: node1 port number (optional)
-       port2: node2 port number (optional)
-       returns: intf1 name, intf2 name"""
-    return node1.linkTo( node2, port1, port2 )
+# Support for dumping network
 
+def dumpNodeConnections( nodes ):
+    "Dump connections to/from nodes."
+
+    def dumpConnections( node ):
+        "Helper function: dump connections to node"
+        for intf in node.intfList():
+            output( ' %s:' % intf )
+            if intf.link:
+                intfs = [ intf.link.intf1, intf.link.intf2 ]
+                intfs.remove( intf )
+                output( intfs[ 0 ] )
+            else:
+                output( ' ' )
+
+    for node in nodes:
+        output( node.name )
+        dumpConnections( node )
+        output( '\n' )
+
+def dumpNetConnections( net ):
+    "Dump connections in network"
+    nodes = net.controllers + net.switches + net.hosts
+    dumpNodeConnections( nodes )
 
 # IP and Mac address formatting and parsing
 
-def _colonHex( val, bytes ):
+def _colonHex( val, bytecount ):
     """Generate colon-hex string.
        val: input as unsigned int
-       bytes: number of bytes to convert
+       bytescount: number of bytes to convert
        returns: chStr colon-hex string"""
     pieces = []
-    for i in range( bytes - 1, -1, -1 ):
+    for i in range( bytecount - 1, -1, -1 ):
         piece = ( ( 0xff << ( i * 8 ) ) & val ) >> ( i * 8 )
         pieces.append( '%02x' % piece )
     chStr = ':'.join( pieces )
@@ -158,23 +242,44 @@ def ipStr( ip ):
     """Generate IP address string from an unsigned int.
        ip: unsigned int of form w << 24 | x << 16 | y << 8 | z
        returns: ip address string w.x.y.z, or 10.x.y.z if w==0"""
-    w = ( ip & 0xff000000 ) >> 24
+    w = ( ip >> 24 ) & 0xff
     w = 10 if w == 0 else w
-    x = ( ip & 0xff0000 ) >> 16
-    y = ( ip & 0xff00 ) >> 8
+    x = ( ip >> 16 ) & 0xff
+    y = ( ip >> 8 ) & 0xff
     z = ip & 0xff
     return "%i.%i.%i.%i" % ( w, x, y, z )
 
 def ipNum( w, x, y, z ):
-    """Generate unsigned int from components ofIP address
+    """Generate unsigned int from components of IP address
        returns: w << 24 | x << 16 | y << 8 | z"""
     return  ( w << 24 ) | ( x << 16 ) | ( y << 8 ) | z
 
+def ipAdd( i, prefixLen=8, ipBaseNum=0x0a000000 ):
+    """Return IP address string from ints
+       i: int to be added to ipbase
+       prefixLen: optional IP prefix length
+       ipBaseNum: option base IP address as int
+       returns IP address as string"""
+    # Ugly but functional
+    assert i < ( 1 << ( 32 - prefixLen ) )
+    mask = 0xffffffff ^ ( ( 1 << prefixLen ) - 1 )
+    ipnum = i + ( ipBaseNum & mask )
+    return ipStr( ipnum )
+
 def ipParse( ip ):
     "Parse an IP address and return an unsigned int."
     args = [ int( arg ) for arg in ip.split( '.' ) ]
     return ipNum( *args )
 
+def netParse( ipstr ):
+    """Parse an IP network specification, returning
+       address and prefix len as unsigned ints"""
+    prefixLen = 0
+    if '/' in ipstr:
+        ip, pf = ipstr.split( '/' )
+        prefixLen = int( pf )
+    return ipParse( ip ), prefixLen
+
 def checkInt( s ):
     "Check if input string is an int"
     try:
@@ -200,10 +305,149 @@ def makeNumeric( s ):
     else:
         return s
 
+# Popen support
+
+def pmonitor(popens, timeoutms=500, readline=True,
+             readmax=1024 ):
+    """Monitor dict of hosts to popen objects
+       a line at a time
+       timeoutms: timeout for poll()
+       readline: return single line of output
+       yields: host, line/output (if any)
+       terminates: when all EOFs received"""
+    poller = poll()
+    fdToHost = {}
+    for host, popen in popens.iteritems():
+        fd = popen.stdout.fileno()
+        fdToHost[ fd ] = host
+        poller.register( fd, POLLIN )
+        if not readline:
+            # Use non-blocking reads
+            flags = fcntl( fd, F_GETFL )
+            fcntl( fd, F_SETFL, flags | O_NONBLOCK )
+    while True:
+        fds = poller.poll( timeoutms )
+        if fds:
+            for fd, _event in fds:
+                host = fdToHost[ fd ]
+                popen = popens[ host ]
+                if readline:
+                    # Attempt to read a line of output
+                    # This blocks until we receive a newline!
+                    line = popen.stdout.readline()
+                else:
+                    line = popen.stdout.read( readmax )
+                yield host, line
+                # Check for EOF
+                if not line:
+                    popen.poll()
+                    if popen.returncode is not None:
+                        poller.unregister( fd )
+                        del popens[ host ]
+                        if not popens:
+                            return
+        else:
+            yield None, ''
 
 # Other stuff we use
 
 def fixLimits():
     "Fix ridiculously small resource limits."
-    setrlimit( RLIMIT_NPROC, ( 4096, 8192 ) )
-    setrlimit( RLIMIT_NOFILE, ( 16384, 32768 ) )
+    setrlimit( RLIMIT_NPROC, ( 8192, 8192 ) )
+    setrlimit( RLIMIT_NOFILE, ( 16384, 16384 ) )
+
+def mountCgroups():
+    "Make sure cgroups file system is mounted"
+    mounts = quietRun( 'mount' )
+    cgdir = '/sys/fs/cgroup'
+    csdir = cgdir + '/cpuset'
+    if 'cgroups on %s' % cgdir not in mounts:
+        raise Exception( "cgroups not mounted on " + cgdir )
+    if 'cpuset on %s' % csdir not in mounts:
+        errRun( 'mkdir -p ' + csdir )
+        errRun( 'mount -t cgroup -ocpuset cpuset ' + csdir )
+
+def natural( text ):
+    "To sort sanely/alphabetically: sorted( l, key=natural )"
+    def num( s ):
+        "Convert text segment to int if necessary"
+        return int( s ) if s.isdigit() else s
+    return [  num( s ) for s in re.split( r'(\d+)', text ) ]
+
+def naturalSeq( t ):
+    "Natural sort key function for sequences"
+    return [ natural( x ) for x in t ]
+
+def numCores():
+    "Returns number of CPU cores based on /proc/cpuinfo"
+    if hasattr( numCores, 'ncores' ):
+        return numCores.ncores
+    try:
+        numCores.ncores = int( quietRun('grep -c processor /proc/cpuinfo') )
+    except ValueError:
+        return 0
+    return numCores.ncores
+
+def irange(start, end):
+    """Inclusive range from start to end (vs. Python insanity.)
+       irange(1,5) -> 1, 2, 3, 4, 5"""
+    return range( start, end + 1 )
+
+def custom( cls, **params ):
+    "Returns customized constructor for class cls."
+    def customized( *args, **kwargs):
+        "Customized constructor"
+        kwargs.update( params )
+        return cls( *args, **kwargs )
+    return customized
+
+def splitArgs( argstr ):
+    """Split argument string into usable python arguments
+       argstr: argument string with format fn,arg2,kw1=arg3...
+       returns: fn, args, kwargs"""
+    split = argstr.split( ',' )
+    fn = split[ 0 ]
+    params = split[ 1: ]
+    # Convert int and float args; removes the need for function
+    # to be flexible with input arg formats.
+    args = [ makeNumeric( s ) for s in params if '=' not in s ]
+    kwargs = {}
+    for s in [ p for p in params if '=' in p ]:
+        key, val = s.split( '=' )
+        kwargs[ key ] = makeNumeric( val )
+    return fn, args, kwargs
+
+def customConstructor( constructors, argStr ):
+    """Return custom constructor based on argStr
+    
+    The args and key/val pairs in argsStr will be automatically applied
+    when the generated constructor is later used.
+    """
+    cname, newargs, kwargs = splitArgs( argStr )
+    constructor = constructors.get( cname, None )
+
+    if not constructor:
+        raise Exception( "error: %s is unknown - please specify one of %s" %
+               ( cname, constructors.keys() ) )
+
+    def customized( name, *args, **params ):
+        "Customized constructor, useful for Node, Link, and other classes"
+        params.update( kwargs )
+        if not newargs:
+            return constructor( name, *args, **params )
+        if args:
+            warn( 'warning: %s replacing %s with %s\n' % (
+                  constructor, args, newargs ) )
+        return constructor( name, *newargs, **params )
+
+    return customized
+
+def buildTopo( topos, topoStr ):
+    """Create topology from string with format (object, arg1, arg2,...).
+    
+    input topos is a dict of topo names to constructors, possibly w/args.
+    """
+    topo, args, kwargs = splitArgs( topoStr )
+    if topo not in topos:
+        raise Exception( 'Invalid topo name %s' % topo )
+    return topos[ topo ]( *args, **kwargs )

mnexec.c

@@ -7,6 +7,8 @@
  *  - detaching from a controlling tty using setsid
  *  - running in a network namespace
  *  - printing out the pid of a process so we can identify it later
+ *  - attaching to a namespace and cgroup
+ *  - setting RT scheduling
  *
  * Partially based on public domain setsid(1)
 */
@@ -14,23 +16,83 @@
 #include <stdio.h>
 #include <linux/sched.h>
 #include <unistd.h>
+#include <limits.h>
+#include <syscall.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <sched.h>
 
 void usage(char *name) 
 {
     printf("Execution utility for Mininet.\n"
-           "usage: %s [-cdnp]\n"
+           "usage: %s [-cdnp] [-a pid] [-g group] [-r rtprio] cmd args...\n"
            "-c: close all file descriptors except stdin/out/error\n"
            "-d: detach from tty by calling setsid()\n"
            "-n: run in new network namespace\n"
-           "-p: print ^A + pid\n", name);
+           "-p: print ^A + pid\n"
+           "-a pid: attach to pid's network namespace\n"
+           "-g group: add to cgroup\n"
+           "-r rtprio: run with SCHED_RR (usually requires -g)\n",
+           name);
+}
+
+
+int setns(int fd, int nstype)
+{
+	return syscall(308, fd, nstype);
+}
+
+/* Validate alphanumeric path foo1/bar2/baz */
+void validate(char *path)
+{
+    char *s;
+    for (s=path; *s; s++) {
+        if (!isalnum(*s) && *s != '/') {
+            fprintf(stderr, "invalid path: %s\n", path);
+            exit(1);
+        }
+    }
+}
+
+/* Add our pid to cgroup */
+int cgroup(char *gname)
+{
+    static char path[PATH_MAX];
+    static char *groups[] = {
+        "cpu", "cpuacct", "cpuset", NULL
+    };
+    char **gptr;
+    pid_t pid = getpid();
+    int count = 0;
+    validate(gname);
+    for (gptr = groups; *gptr; gptr++) {
+        FILE *f;
+        snprintf(path, PATH_MAX, "/sys/fs/cgroup/%s/%s/tasks",
+                 *gptr, gname);
+        f = fopen(path, "w");
+        if (f) {
+            count++;
+            fprintf(f, "%d\n", pid);
+            fclose(f);
+        }
+    }
+    if (!count) {
+        fprintf(stderr, "cgroup: could not add to cgroup %s\n",
+            gname);
+        exit(1);
+    }
 }
 
 int main(int argc, char *argv[])
 {
     char c;
     int fd;
-    
-    while ((c = getopt(argc, argv, "+cdnp")) != -1)
+    char path[PATH_MAX];
+    int nsid;
+    int pid;
+    static struct sched_param sp;
+    while ((c = getopt(argc, argv, "+cdnpa:g:r:")) != -1)
         switch(c) {
         case 'c':
             /* close file descriptors except stdin/out/error */
@@ -64,16 +126,42 @@ int main(int argc, char *argv[])
             printf("\001%d\n", getpid());
             fflush(stdout);
             break;
+        case 'a':
+            /* Attach to pid's network namespace */
+            pid = atoi(optarg);
+            sprintf(path, "/proc/%d/ns/net", pid );
+            nsid = open(path, O_RDONLY);
+            if (nsid < 0) {
+                perror(path);
+                return 1;
+            }
+            if (setns(nsid, 0) != 0) {
+                perror("setns");
+                return 1;
+            }
+            break;
+        case 'g':
+            /* Attach to cgroup */
+            cgroup(optarg);
+            break;
+        case 'r':
+            /* Set RT scheduling priority */
+            sp.sched_priority = atoi(optarg);
+            if (sched_setscheduler(getpid(), SCHED_RR, &sp) < 0) {
+                perror("sched_setscheduler");
+                return 1;
+            }
+            break;
         default:
             usage(argv[0]);
             break;
         }
 
     if (optind < argc) {
-		execvp(argv[optind], &argv[optind]);
-		perror(argv[optind]);
-		return 1;
-	}
+        execvp(argv[optind], &argv[optind]);
+        perror(argv[optind]);
+        return 1;
+    }
     
     usage(argv[0]);
 }

setup.py

@@ -5,8 +5,7 @@
 from setuptools import setup, find_packages
 from os.path import join
 
-scripts = [ join( 'bin', filename ) for filename in [ 
-    'mn', 'mnexec' ] ]
+scripts = [ join( 'bin', filename ) for filename in [ 'mn' ] ]
 
 modname = distname = 'mininet'
 

util/build-ovs-packages.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Attempt to build debian packages for OVS
+
+set -e  # exit on error
+set -u  # exit on undefined variable
+
+kvers=`uname -r`
+ksrc=/lib/modules/$kvers/build
+dist=`lsb_release -is | tr [A-Z] [a-z]`
+release=`lsb_release -rs`
+arch=`uname -m`
+buildsuffix=-2
+if [ "$arch" = "i686" ]; then arch=i386; fi
+if [ "$arch" = "x86_64" ]; then arch=amd64; fi
+
+overs=1.4.0
+ovs=openvswitch-$overs
+ovstgz=$ovs.tar.gz
+ovsurl=http://openvswitch.org/releases/$ovstgz
+
+install='sudo apt-get install -y'
+
+echo "*** Installing debian/ubuntu build system"
+ $install build-essential devscripts ubuntu-dev-tools debhelper dh-make
+ $install diff patch cdbs quilt gnupg fakeroot lintian pbuilder piuparts
+ $install module-assistant
+
+echo "*** Installing OVS dependencies"
+ $install pkg-config gcc make python-dev libssl-dev libtool
+ $install dkms ipsec-tools
+
+echo "*** Installing headers for $kvers"
+ $install linux-headers-$kvers
+
+echo "*** Retrieving OVS source"
+ wget -c $ovsurl
+ tar xzf $ovstgz
+ cd $ovs
+
+echo "*** Patching OVS source"
+ # Not sure why this fails, but off it goes!
+ sed -i -e 's/dh_strip/# dh_strip/' debian/rules
+ if [ "$release" = "10.04" ]; then
+  # Lucid doesn't seem to have all the packages for ovsdbmonitor
+  echo "*** Patching debian/rules to remove dh_python2"
+  sed -i -e 's/dh_python2/dh_pysupport/' debian/rules
+  echo "*** Not building ovsdbmonitor since it's too hard on 10.04"
+  mv debian/ovsdbmonitor.install debian/ovsdbmonitor.install.backup
+  sed -i -e 's/ovsdbmonitor.install/ovsdbmonitor.install.backup/' Makefile.in
+ else
+  # Install a bag of hurt for ovsdbmonitor
+  $install python-pyside.qtcore pyqt4-dev-tools python-twisted python-twisted-bin \
+   python-twisted-core python-twisted-conch python-anyjson python-zope.interface
+ fi
+ # init script was written to assume that commands complete
+ sed -i -e 's/^set -e/#set -e/' debian/openvswitch-controller.init
+
+echo "*** Building OVS user packages"
+ opts=--with-linux=/lib/modules/`uname -r`/build
+ fakeroot make -f debian/rules DATAPATH_CONFIGURE_OPTS=$opts binary
+
+echo "*** Building OVS datapath kernel module package"
+ # Still looking for the "right" way to do this...
+ sudo mkdir -p /usr/src/linux
+ ln -sf _debian/openvswitch.tar.gz .
+ sudo make -f debian/rules.modules KSRC=$ksrc KVERS=$kvers binary-modules
+
+echo "*** Built the following packages:"
+ cd ~
+ ls -l *deb
+
+archive=ovs-$overs-core-$dist-$release-$arch$buildsuffix.tar
+ovsbase='common pki switch brcompat controller datapath-dkms'
+echo "*** Packing up $ovsbase .debs into:"
+echo "    $archive"
+ pkgs=""
+ for component in $ovsbase; do
+  if echo $component | egrep 'dkms|pki'; then
+    # Architecture-independent packages
+    deb=(openvswitch-${component}_$overs*all.deb)
+  else
+    deb=(openvswitch-${component}_$overs*$arch.deb)
+  fi
+  pkgs="$pkgs $deb"
+ done
+ rm -rf $archive
+ tar cf $archive $pkgs
+
+echo "*** Contents of archive $archive:"
+ tar tf $archive
+
+echo "*** Done (hopefully)"
+

util/doxify.py

@@ -82,7 +82,7 @@ if __name__ == '__main__':
     fixLines( infile.readlines(), outfid )
     infile.close()
     os.close( outfid )
-    call( [ 'doxypy.py', outname ] )
+    call( [ 'doxypy', outname ] )
 
 
 

util/install.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 # Mininet install script for Ubuntu (and Debian Lenny)
 # Brandon Heller (brandonh@stanford.edu)
 
@@ -16,57 +17,78 @@ KERNEL_LOC=http://www.openflow.org/downloads/mininet
 DIST=Unknown
 RELEASE=Unknown
 CODENAME=Unknown
+ARCH=`uname -m`
+if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; fi
+if [ "$ARCH" = "i686" ]; then ARCH="i386"; fi
+
 test -e /etc/debian_version && DIST="Debian"
 grep Ubuntu /etc/lsb-release &> /dev/null && DIST="Ubuntu"
 if [ "$DIST" = "Ubuntu" ] || [ "$DIST" = "Debian" ]; then
-    sudo apt-get install -y lsb-release
+    install='sudo apt-get -y install'
+    remove='sudo apt-get -y remove'
+    pkginst='sudo dpkg -i'
+    # Prereqs for this script
+    if ! which lsb_release &> /dev/null; then
+        $install lsb-release
+    fi
+    if ! which bc &> /dev/null; then
+        $install bc
+    fi 
 fi
 if which lsb_release &> /dev/null; then
     DIST=`lsb_release -is`
     RELEASE=`lsb_release -rs`
     CODENAME=`lsb_release -cs`
 fi
-echo "Detected Linux distribution: $DIST $RELEASE $CODENAME"
+echo "Detected Linux distribution: $DIST $RELEASE $CODENAME $ARCH"
 
 # Kernel params
 
-if [ "$DIST" = "Debian" ]; then
-    KERNEL_NAME=2.6.33.1-mininet
-    KERNEL_HEADERS=linux-headers-${KERNEL_NAME}_${KERNEL_NAME}-10.00.Custom_i386.deb
-    KERNEL_IMAGE=linux-image-${KERNEL_NAME}_${KERNEL_NAME}-10.00.Custom_i386.deb
-elif [ "$DIST" = "Ubuntu" ]; then
+if [ "$DIST" = "Ubuntu" ]; then
     if [ "$RELEASE" = "10.04" ]; then
         KERNEL_NAME='3.0.0-15-generic'
     else
         KERNEL_NAME=`uname -r`
     fi
     KERNEL_HEADERS=linux-headers-${KERNEL_NAME}
+elif [ "$DIST" = "Debian" ] && [ "$ARCH" = "i386" ] && [ "$CODENAME" = "lenny" ]; then
+    KERNEL_NAME=2.6.33.1-mininet
+    KERNEL_HEADERS=linux-headers-${KERNEL_NAME}_${KERNEL_NAME}-10.00.Custom_i386.deb
+    KERNEL_IMAGE=linux-image-${KERNEL_NAME}_${KERNEL_NAME}-10.00.Custom_i386.deb
 else
-    echo "Install.sh currently only supports Ubuntu and Debian."
+    echo "Install.sh currently only supports Ubuntu and Debian Lenny i386."
     exit 1
 fi
 
+# More distribution info
+DIST_LC=`echo $DIST | tr [A-Z] [a-z]` # as lower case
 
 # Kernel Deb pkg to be removed:
-KERNEL_IMAGE_OLD=linux-image-2.6.26-2-686
+KERNEL_IMAGE_OLD=linux-image-2.6.26-33-generic
 
 DRIVERS_DIR=/lib/modules/${KERNEL_NAME}/kernel/drivers/net
 
-OVS_RELEASE=v1.2.2
+OVS_RELEASE=1.4.0
+OVS_PACKAGE_LOC=https://github.com/downloads/mininet/mininet
+OVS_BUILDSUFFIX=-ignore # was -2
+OVS_PACKAGE_NAME=ovs-$OVS_RELEASE-core-$DIST_LC-$RELEASE-$ARCH$OVS_BUILDSUFFIX.tar
 OVS_SRC=~/openvswitch
+OVS_TAG=v$OVS_RELEASE
 OVS_BUILD=$OVS_SRC/build-$KERNEL_NAME
 OVS_KMODS=($OVS_BUILD/datapath/linux/{openvswitch_mod.ko,brcompat_mod.ko})
 
 function kernel {
     echo "Install Mininet-compatible kernel if necessary"
     sudo apt-get update
-    if [ "$DIST" = "Debian" ]; then
+    if [ "$DIST" = "Ubuntu" ] &&  [ "$RELEASE" = "10.04" ]; then
+        $install linux-image-$KERNEL_NAME
+    elif [ "$DIST" = "Debian" ]; then
         # The easy approach: download pre-built linux-image and linux-headers packages:
         wget -c $KERNEL_LOC/$KERNEL_HEADERS
         wget -c $KERNEL_LOC/$KERNEL_IMAGE
 
         # Install custom linux headers and image:
-        sudo dpkg -i $KERNEL_IMAGE $KERNEL_HEADERS
+        $pkginst $KERNEL_IMAGE $KERNEL_HEADERS
 
         # The next two steps are to work around a bug in newer versions of
         # kernel-package, which fails to add initrd images with the latest kernels.
@@ -83,16 +105,15 @@ function kernel {
         # /boot/grub/menu.lst to set the default to the entry corresponding to the
         # kernel you just installed.
     fi
-    if [ "$DIST" = "Ubuntu" ] &&  [ "$RELEASE" = "10.04" ]; then
-        sudo apt-get -y install linux-image-$KERNEL_NAME
-    fi
 }
 
 function kernel_clean {
     echo "Cleaning kernel..."
 
     # To save disk space, remove previous kernel
-    sudo apt-get -y remove $KERNEL_IMAGE_OLD
+    if ! $remove $KERNEL_IMAGE_OLD; then
+        echo $KERNEL_IMAGE_OLD not installed.
+    fi
 
     # Also remove downloaded packages:
     rm -f ~/linux-headers-* ~/linux-image-*
@@ -101,8 +122,8 @@ function kernel_clean {
 # Install Mininet deps
 function mn_deps {
     echo "Installing Mininet dependencies"
-    sudo aptitude install -y gcc make screen psmisc xterm ssh iperf iproute \
-        python-setuptools python-networkx
+    $install gcc make screen psmisc xterm ssh iperf iproute \
+        python-setuptools python-networkx cgroup-bin ethtool
 
     if [ "$DIST" = "Ubuntu" ] && [ "$RELEASE" = "10.04" ]; then
         echo "Upgrading networkx to avoid deprecation warning"
@@ -124,16 +145,14 @@ function mn_deps {
 
 # The following will cause a full OF install, covering:
 # -user switch
-# -dissector
 # The instructions below are an abbreviated version from
 # http://www.openflowswitch.org/wk/index.php/Debian_Install
 # ... modified to use Debian Lenny rather than unstable.
 function of {
-    echo "Installing OpenFlow and its tools..."
-
+    echo "Installing OpenFlow reference implementation..."
     cd ~/
-    sudo apt-get install -y git-core automake m4 pkg-config libtool \
-		make libc6-dev autoconf autotools-dev gcc
+    $install git-core autoconf automake autotools-dev pkg-config \
+		make gcc libtool libc6-dev 
     git clone git://openflowswitch.org/openflow.git
     cd ~/openflow
 
@@ -146,19 +165,9 @@ function of {
     make
     sudo make install
 
-    # Install dissector:
-    sudo apt-get install -y wireshark libgtk2.0-dev
-    cd ~/openflow/utilities/wireshark_dissectors/openflow
-    make
-    sudo make install
-
-    # Copy coloring rules: OF is white-on-blue:
-    mkdir -p ~/.wireshark
-    cp ~/mininet/util/colorfilters ~/.wireshark
-
     # Remove avahi-daemon, which may cause unwanted discovery packets to be 
-	# sent during tests, near link status changes:
-    sudo apt-get remove -y avahi-daemon
+    # sent during tests, near link status changes:
+    $remove avahi-daemon
 
     # Disable IPv6.  Add to /etc/modprobe.d/blacklist:
     if [ "$DIST" = "Ubuntu" ]; then
@@ -167,46 +176,157 @@ function of {
         BLACKLIST=/etc/modprobe.d/blacklist
     fi
     sudo sh -c "echo 'blacklist net-pf-10\nblacklist ipv6' >> $BLACKLIST"
+    cd ~
 }
 
+function wireshark {
+    echo "Installing Wireshark dissector..."
+
+    sudo apt-get install -y wireshark libgtk2.0-dev
+
+    if [ "$DIST" = "Ubuntu" ] && [ "$RELEASE" != "10.04" ]; then
+        # Install newer version
+        sudo apt-get install -y scons mercurial libglib2.0-dev
+        sudo apt-get install -y libwiretap-dev libwireshark-dev
+        cd ~
+        hg clone https://bitbucket.org/barnstorm/of-dissector
+        cd of-dissector/src
+        export WIRESHARK=/usr/include/wireshark
+        scons
+        # libwireshark0/ on 11.04; libwireshark1/ on later
+        WSDIR=`ls -d /usr/lib/wireshark/libwireshark* | head -1`
+        WSPLUGDIR=$WSDIR/plugins/
+        sudo cp openflow.so $WSPLUGDIR
+        echo "Copied openflow plugin to $WSPLUGDIR"
+    else
+        # Install older version from reference source
+        cd ~/openflow/utilities/wireshark_dissectors/openflow
+        make
+        sudo make install
+    fi
+
+    # Copy coloring rules: OF is white-on-blue:
+    mkdir -p ~/.wireshark
+    cp ~/mininet/util/colorfilters ~/.wireshark
+}
+
+
 # Install Open vSwitch
 # Instructions derived from OVS INSTALL, INSTALL.OpenFlow and README files.
+
 function ovs {
     echo "Installing Open vSwitch..."
 
-    if [ "$DIST" = "Debian" ] && [ "$CODENAME" == "lenny" ]; then
-        sudo aptitude -y install pkg-config gcc make git-core python-dev libssl-dev
-        # Install Autoconf 2.63+ backport from Debian Backports repo:
-        # Instructions from http://backports.org/dokuwiki/doku.php?id=instructions
-        sudo su -c "echo 'deb http://www.backports.org/debian lenny-backports main contrib non-free' >> /etc/apt/sources.list"
-        sudo apt-get update
-        sudo apt-get -y --force-yes install debian-backports-keyring
-        sudo apt-get -y --force-yes -t lenny-backports install autoconf
+    # Required for module build/dkms install
+    $install $KERNEL_HEADERS
+
+    ovspresent=0
+
+    # First see if we have packages
+    # XXX wget -c seems to fail from github/amazon s3
+    cd /tmp
+    if wget $OVS_PACKAGE_LOC/$OVS_PACKAGE_NAME 2> /dev/null; then
+	$install patch dkms fakeroot python-argparse
+        tar xf $OVS_PACKAGE_NAME
+        orig=`tar tf $OVS_PACKAGE_NAME`
+        # Now install packages in reasonable dependency order
+        order='dkms common pki openvswitch-switch brcompat controller'
+        pkgs=""
+        for p in $order; do
+            pkg=`echo "$orig" | grep $p`
+	    # Annoyingly, things seem to be missing without this flag
+            $pkginst --force-confmiss $pkg
+        done
+        ovspresent=1
     fi
 
-    if [ "$DIST" = "Ubuntu" ]; then
-        sudo apt-get -y install $KERNEL_HEADERS
+    # Otherwise try distribution's OVS packages
+    if [ "$DIST" = "Ubuntu" ] && [ `expr $RELEASE '>=' 11.10` = 1 ]; then
+        if ! dpkg --get-selections | grep openvswitch-datapath; then
+            # If you've already installed a datapath, assume you
+            # know what you're doing and don't need dkms datapath.
+            # Otherwise, install it.
+            $install openvswitch-datapath-dkms
+        fi
+	if $install openvswitch-switch openvswitch-controller; then
+            echo "Ignoring error installing openvswitch-controller"
+        fi
+        ovspresent=1
     fi
 
+    # Switch can run on its own, but 
+    # Mininet should control the controller
+    if [ -e /etc/init.d/openvswitch-controller ]; then
+        if sudo service openvswitch-controller stop; then
+            echo "Stopped running controller"
+        fi
+        sudo update-rc.d openvswitch-controller disable
+    fi
+
+    if [ $ovspresent = 1 ]; then
+        echo "Done (hopefully) installing packages"
+        cd ~
+        return
+    fi
+
+    # Otherwise attempt to install from source
+
+    $install pkg-config gcc make python-dev libssl-dev libtool
+
+    if [ "$DIST" = "Debian" ]; then
+        if [ "$CODENAME" = "lenny" ]; then
+            $install git-core
+            # Install Autoconf 2.63+ backport from Debian Backports repo:
+            # Instructions from http://backports.org/dokuwiki/doku.php?id=instructions
+            sudo su -c "echo 'deb http://www.backports.org/debian lenny-backports main contrib non-free' >> /etc/apt/sources.list"
+            sudo apt-get update
+            sudo apt-get -y --force-yes install debian-backports-keyring
+            sudo apt-get -y --force-yes -t lenny-backports install autoconf
+        fi
+    else
+        $install git
+    fi    
+
     # Install OVS from release
     cd ~/
-    git clone git://openvswitch.org/openvswitch
+    git clone git://openvswitch.org/openvswitch $OVS_SRC
     cd $OVS_SRC
-    git checkout $OVS_RELEASE
+    git checkout $OVS_TAG
     ./boot.sh
     BUILDDIR=/lib/modules/${KERNEL_NAME}/build
     if [ ! -e $BUILDDIR ]; then
         echo "Creating build sdirectory $BUILDDIR"
         sudo mkdir -p $BUILDDIR
     fi
-	opts="--with-linux=$BUILDDIR"
-	mkdir -p $OVS_BUILD
-	cd $OVS_BUILD
+    opts="--with-linux=$BUILDDIR"
+    mkdir -p $OVS_BUILD
+    cd $OVS_BUILD
     ../configure $opts
     make
     sudo make install
-	# openflowd is deprecated, but for now copy it in
-	sudo cp tests/test-openflowd /usr/local/bin/ovs-openflowd
+
+    modprobe
+}
+
+function remove_ovs {
+    pkgs=`dpkg --get-selections | grep openvswitch | awk '{ print $1;}'`
+    echo "Removing existing Open vSwitch packages:"
+    echo $pkgs
+    if ! $remove $pkgs; then
+        echo "Not all packages removed correctly"
+    fi
+    # For some reason this doesn't happen
+    if scripts=`ls /etc/init.d/*openvswitch* 2>/dev/null`; then
+        echo $scripts
+        for s in $scripts; do
+            s=$(basename $s)
+            echo SCRIPT $s
+            sudo service $s stop
+            sudo rm -f /etc/init.d/$s
+            sudo update-rc.d -f $s remove
+        done
+    fi
+    echo "Done removing OVS"
 }
 
 # Install NOX with tutorial files
@@ -214,17 +334,17 @@ function nox {
     echo "Installing NOX w/tutorial files..."
 
     # Install NOX deps:
-    sudo apt-get -y install autoconf automake g++ libtool python python-twisted \
+    $install autoconf automake g++ libtool python python-twisted \
 		swig libssl-dev make
     if [ "$DIST" = "Debian" ]; then
-        sudo apt-get -y install libboost1.35-dev
+        $install libboost1.35-dev
     elif [ "$DIST" = "Ubuntu" ]; then
-        sudo apt-get -y install python-dev libboost-dev 
-        sudo apt-get -y install libboost-filesystem-dev
-        sudo apt-get -y install libboost-test-dev
+        $install python-dev libboost-dev 
+        $install libboost-filesystem-dev
+        $install libboost-test-dev
     fi
     # Install NOX optional deps:
-    sudo apt-get install -y libsqlite3-dev python-simplejson
+    $install libsqlite3-dev python-simplejson
 
     # Fetch NOX destiny
     cd ~/
@@ -234,7 +354,10 @@ function nox {
 
     # Apply patches
     git checkout -b tutorial-destiny
-    git am ~/mininet/util/nox-patches/*.patch
+    git am ~/mininet/util/nox-patches/*tutorial-port-nox-destiny*.patch
+    if [ "$DIST" = "Ubuntu" ] && [ `expr $RELEASE '>=' 12.04` = 1 ]; then
+        git am ~/mininet/util/nox-patches/*nox-ubuntu12-hacks.patch
+    fi
 
     # Build
     ./boot.sh
@@ -257,7 +380,7 @@ function oftest {
     echo "Installing oftest..."
 
     # Install deps:
-    sudo apt-get install -y tcpdump python-scapy
+    $install tcpdump python-scapy
 
     # Install oftest:
     cd ~/
@@ -271,7 +394,7 @@ function oftest {
 function cbench {
     echo "Installing cbench..."
     
-    sudo apt-get install -y libsnmp-dev libpcap-dev
+    $install libsnmp-dev libpcap-dev libconfig-dev
     cd ~/
     git clone git://openflow.org/oflops.git
     cd oflops
@@ -290,13 +413,13 @@ function other {
 
     # Install tcpdump and tshark, cmd-line packet dump tools.  Also install gitk,
     # a graphical git history viewer.
-    sudo apt-get install -y tcpdump tshark gitk
+    $install tcpdump tshark gitk
 
     # Install common text editors
-    sudo apt-get install -y vim nano emacs
+    $install vim nano emacs
 
     # Install NTP
-    sudo apt-get install -y ntp
+    $install ntp
 
     # Set git to colorize everything.
     git config --global color.diff auto
@@ -331,8 +454,8 @@ function all {
     kernel
     mn_deps
     of
+    wireshark
     ovs
-    modprobe
     nox
     oftest
     cbench
@@ -386,10 +509,12 @@ function usage {
     printf -- ' -f: install open(F)low\n' >&2
     printf -- ' -h: print this (H)elp message\n' >&2
     printf -- ' -k: install new (K)ernel\n' >&2
-    printf -- ' -m: install Open vSwitch kernel (M)odule\n' >&2
+    printf -- ' -m: install Open vSwitch kernel (M)odule from source dir\n' >&2
     printf -- ' -n: install mini(N)et dependencies + core files\n' >&2
+    printf -- ' -r: remove existing Open vSwitch packages\n' >&2
     printf -- ' -t: install o(T)her stuff\n' >&2
     printf -- ' -v: install open (V)switch\n' >&2
+    printf -- ' -w: install OpenFlow (w)ireshark dissector\n' >&2
     printf -- ' -x: install NO(X) OpenFlow controller\n' >&2
     printf -- ' -y: install (A)ll packages\n' >&2    
     
@@ -400,7 +525,7 @@ if [ $# -eq 0 ]
 then
     all
 else
-    while getopts 'abcdfhkmntvx' OPTION
+    while getopts 'abcdfhkmnrtvwx' OPTION
     do
       case $OPTION in
       a)    all;;
@@ -412,8 +537,10 @@ else
       k)    kernel;;
       m)    modprobe;;
       n)    mn_deps;;
+      r)    remove_ovs;;
       t)    other;;
       v)    ovs;;
+      w)    wireshark;;
       x)    nox;;
       ?)    usage;;
       esac

util/kbuild/cfs-nsfd-kernel/build-3.0.sh

@@ -0,0 +1,439 @@
+#!/bin/bash
+# Builds kernel with the new CFS Bandwidth patches
+# and nsfd/setns syscall patches.
+# Also builds Open vSwitch against the built kernel version.
+# Script only to be run on 64-bit systems; needs a few changes to run on
+# 32-bit ones.
+
+# If building for i386 (-t), make sure to install the following 32-bit libs:
+# sudo apt-get install ia32-libs lib32gcc1 libc6-i386 util-linux devscripts
+
+# Check for unitialized variables
+set -o nounset
+
+# Exit on any failure
+set -e
+
+# Location in which to download and build the kernel
+kdir=/usr/src
+
+# Kernel version to download
+kver=3.0.0
+
+# Save original directory for later.
+orig_dir=`pwd`
+
+# Default and custom kernel version string
+version_string=-with-cfs
+
+# Run menuconfig later?
+menuconfig=
+
+# Use localmodconfig?
+localmodconfig=
+
+# Build ubuntu kernel? must be 3.0.0 compatible
+ubuntu_release=
+ubuntu_default_release=ubuntu-oneiric
+ubuntu_base=3.0.0-14  # base tag and version for build
+ubuntu_tag=Ubuntu-$ubuntu_base.23
+ubuntu_flavor=generic
+#ubuntu_config=/boot/config-$ubuntu_base-$ubuntu_flavor
+ubuntu_config=${orig_dir}/config-3.0.9-with-cfs
+ubuntu_image=linux-image-$ubuntu_base-$ubuntu_flavor
+ubuntu_kver=3.0.9 # must match version that is actually built
+
+# OVS pkg string.  Not sure how to find this automatically.
+ovs_pkg_ver=1.2.0-1ubuntu3
+
+# Location of kernel config.  If not specified, use current .config.
+# was: ${orig_dir}/config-3.0.0-with-cfs
+kconfig=
+
+# Install only?
+install_only=
+
+# Use 32-bit?
+i386=
+
+function usage {
+    warn "Compiles kernel ${kver} with CBW, setns, and DCTCP patches in ${kdir}"
+    warn "Usage: build.sh [-huimlt] [-v 'versionstring']"
+    warn "-h help"
+    warn "-u build ubuntu kernel"
+    warn "-i install only (don't build)"
+    warn "-m use menuconfig"
+    warn "-l use localmodconfig"
+    warn "-v 'versionstring' use custom version string"
+    warn "-t build for i386 (32-bit)"
+}
+
+function parse_opts {
+    custom_version_string=
+    plus=
+    while getopts 'huimltv:' OPTION; do
+        case $OPTION in
+            h) usage; exit 0;;
+            u) ubuntu_release=$ubuntu_default_release;
+		kver=$ubuntu_kver; kconfig=$ubuntu_config;;
+	    i) install_only=true;;
+	    m) menuconfig=true;;
+	    l) localmodconfig=true;;
+	    v) custom_version_string=$OPTARG;;
+	    t) i386=true; plus=;;
+	    ?) usage; exit 1;;
+        esac
+    done
+    # Provide feedback which might be useful
+    if [[ "$custom_version_string" != "" ]]; then
+        warn "Using custom version_string: ${custom_version_string}"
+        version_string=$custom_version_string
+    else
+        warn "Using default version_string: ${version_string}"
+    fi
+    if [[ "$ubuntu_release" != "" ]]; then
+        warn "Building Ubuntu kernel for release ${ubuntu_release}"
+    fi
+}
+
+function warn {
+    # Echo the provided command in color text.
+    yellow='\e[0;33m' # Yellow
+    reset='\e[0m'
+    echo="echo -e"
+    if [ -n "${2+defined}" ]; then
+        echo="$echo $2"
+    fi
+    $echo "${yellow}$1${reset}"
+}
+
+function pre_check {
+    warn "Checking for git"
+    if [[ -z `which git` ]]; then
+        warn "You need git to download kernel.  Install? [Y/n] " -n
+        read answer
+        [[ -z $answer || $answer=="Y" || $answer == "y" ]] && \
+            sudo apt-get install git;
+    fi
+
+    warn "Checking for kernel-package build utilities"
+    if [[ -z `which make-kpkg` ]]; then
+        warn "You need kernel-package utilities to build the kernel.  Install? [Y/n] " -n
+        read answer
+        [[ -z $answer || $answer=="Y" || $answer == "y" ]] && \
+            sudo apt-get install kernel-package ncurses-dev;
+    fi
+}
+
+function fetch_kernel {
+    if [[ "$ubuntu_release" == "" ]]; then
+    srcdir=$kdir/linux-$kver
+    archive=git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-tip.git
+	tag=3.0.0
+    else
+	warn "Pre-installing $ubuntu_image"
+	#sudo apt-get install $ubuntu_image 
+	srcdir=$kdir/$ubuntu_release
+	archive=git://kernel.ubuntu.com/ubuntu/$ubuntu_release
+	tag=$ubuntu_tag
+    fi
+    if [[ -d $srcdir ]]; then
+        warn "Linux source exists in $srcdir, skipping.."
+        return
+    fi
+    warn "--> Fetching kernel $srcdir"
+    if git clone $archive $srcdir; then
+	return
+    fi
+    warn "Failed to fetch kernel from $archive"
+    if [[ "$ubuntu_release" == "" ]]; then
+        warn "Trying github"
+	archive=git://github.com/torvalds/linux.git
+	if git clone $archive $srcdir; then
+	    return
+	fi
+    fi
+    warn "Giving up."
+    exit 2
+}
+
+function work_around_kernel_package_bug {
+    warn "Applying workaround for kernel package bug..."
+    # Fix will likely break on any other kernel version, so watch out.
+    # From:
+    # https://bugs.launchpad.net/ubuntu/+source/kernel-package/+bug/58307/comments/16
+    sed -i -s 's/echo "+"/#echo "+"/' $srcdir/scripts/setlocalversion
+}
+
+function copy_patches {
+    warn "Copying patches..."
+    rm -rf $srcdir/patches
+    cp -r ${orig_dir}/../../linux-3.0.0-patches/ $srcdir/patches
+}
+
+function apply_patches {
+    cd $srcdir
+    if git checkout mininet ; then
+	# Assume mininet 
+        warn "Mininet branch already exists - not applying patches"
+        return
+    fi
+    if [[ "$tag" != "" ]] ; then
+	git checkout $tag
+    fi
+    git checkout -b mininet 
+    warn "Applying patches..."
+    git am -3 patches/*.patch
+    work_around_kernel_package_bug
+}
+
+# lxc/ns and cfs configuration flags
+
+config_y='
+CONFIG_GROUP_SCHED
+CONFIG_FAIR_GROUP_SCHED
+CONFIG_RT_GROUP_SCHED
+CONFIG_CGROUP_SCHED
+CONFIG_CGROUPS
+CONFIG_CGROUP_FREEZER
+CONFIG_CGROUP_DEVICE
+CONFIG_SCHED_AUTOGROUP
+CONFIG_BLK_CGROUP
+CONFIG_CFQ_GROUP_IOSCHED
+CONFIG_CGROUP_PERF
+CONFIG_CPUSETS
+CONFIG_PROC_PID_CPUSET
+CONFIG_CGROUP_CPUACCT
+CONFIG_RESOURCE_COUNTERS
+CONFIG_CGROUP_MEM_RES_CTLR
+CONFIG_CGROUP_MEM_RES_CTLR_SWAP
+CONFIG_MM_OWNER
+CONFIG_NAMESPACES
+CONFIG_UTS_NS
+CONFIG_IPC_NS
+CONFIG_USER_NS
+CONFIG_PID_NS
+CONFIG_NET_NS
+CONFIG_NET_CLS_CGROUP
+CONFIG_SECURITY_FILE_CAPABILITIES
+CONFIG_DEVPTS_MULTIPLE_INSTANCES
+CONFIG_VETH
+CONFIG_VLAN_8021Q
+CONFIG_MACVLAN
+CONFIG_CFS_BANDWIDTH
+CONFIG_NET_SCHED'
+
+config_m='
+CONFIG_BRIDGE
+CONFIG_NET_SCH_CBQ
+CONFIG_NET_SCH_HTB
+CONFIG_NET_SCH_HFSC
+CONFIG_NET_SCH_PRIO
+CONFIG_NET_SCH_MULTIQ
+CONFIG_NET_SCH_RED
+CONFIG_NET_SCH_SFB
+CONFIG_NET_SCH_SFQ
+CONFIG_NET_SCH_TEQL
+CONFIG_NET_SCH_TBF
+CONFIG_NET_SCH_GRED
+CONFIG_NET_SCH_DSMARK
+CONFIG_NET_SCH_NETEM
+CONFIG_NET_SCH_DRR
+CONFIG_NET_SCH_MQPRIO
+CONFIG_NET_SCH_CHOKE
+CONFIG_NET_SCH_QFQ
+CONFIG_NET_SCH_INGRESS
+'
+
+config_n='
+CONFIG_SECURITY_APPARMOR
+'
+
+function configure_kernel {
+    cd $srcdir
+    warn "Configuring kernel..."
+
+    if [[ "$menuconfig" == 'true' ]]; then
+        make menuconfig
+    else
+        if [[ "$kconfig" == "" ]]; then
+            warn "Using current kernel config..."
+        else
+            warn "Using specified kernel config: ${kconfig}..."
+            cp $kconfig .config
+        fi
+
+        warn "Making oldconfig..."
+        if [[ "$i386" == 'true' ]]; then
+            linux32=linux32
+        else
+            linux32=
+        fi
+        yes '' | $linux32 make oldconfig 1> /dev/null
+        if [[ "$localmodconfig" == 'true' ]]; then
+            warn "Making localmodconfig..."
+            yes '' | $linux32 make localmodconfig 1> /dev/null
+        fi
+        warn "Setting kernel flags for lxc and cbw..."
+        for flag in $config_y; do
+            if ! grep $flag .config 1> /dev/null; then
+                echo $flag=y >> .config
+            else
+                sed -i -s "s/# $flag is not set/$flag=y/" .config
+            fi
+        done
+        for flag in $config_m; do
+            if ! grep $flag .config 1> /dev/null; then 
+                echo $flag=m >> .config
+            else
+                sed -i -s "s/# $flag is not set/$flag=m/" .config
+                sed -i -s "s/$flag=y/$flag=m/" .config
+            fi
+        done
+        for flag in $config_n; do
+            if ! grep $flag .config 1> /dev/null; then
+                echo "# $flag is not set" >> .config
+            else
+                sed -i -s "s/$flag=y/# $flag is not set/" .config
+                sed -i -s "s/$flag=m/# $flag is not set/" .config
+            fi
+        done
+        for flag in $config_y $config_m; do 
+            grep $flag .config || echo "WARNING: $flag IS MISSING"
+        done
+        cp .config /tmp
+        warn "RAN CONFIG IN `pwd`"
+    fi
+}
+
+function build_kernel {
+    # Have your favourite build method here
+    # This is a standard Debian way of building the kernel
+    # The patches select cfs bandwidth automatically
+    warn "Building kernel-$version_string"
+    cd $srcdir
+    procs=`grep -c ^processor /proc/cpuinfo`
+    procs=`echo $procs + 2 | bc`
+    export CONCURRENCY_LEVEL=$procs
+    if [[ "$i386" == 'true' ]]; then
+        mkpkg_extra_args='--cross-compile - --arch i386'
+    else
+        mkpkg_extra_args=
+    fi
+    make-kpkg clean $mkpkg_extra_args
+    yes '' | fakeroot make-kpkg -j $procs $mkpkg_extra_args --initrd --append-to-version=${version_string} \
+        kernel_image kernel_headers
+}
+
+function mod_kernel_dpkg {
+    # Only needed for i386.
+    cd /usr/src
+    if [[ "$i386" == 'true' ]]; then
+        warn "Modifying deb-pkg names for i386"
+        # Based on instructions from http://dotcommie.net/?id=165
+        for pkg_type in linux-image linux-headers; do
+            pkg_name_orig=${pkg_type}-$kver${version_string}${plus}_$kver${version_string}${plus}-10.00.Custom_amd64.deb
+            #hook=`readlink -f set_debian_control_i386.sh`
+            hook=${orig_dir}/set_debian_control_i386.sh
+            warn "$pkg_name_orig"
+            fakeroot deb-reversion -s "" --hook $hook $pkg_name_orig
+            # Remove the 1 in the name that deb-reversion adds.
+            pkg_name_mod=${pkg_type}-$kver${version_string}${plus}_$kver${version_string}${plus}-10.00.Custom1_i386.deb
+            pkg_name_new=${pkg_type}-$kver${version_string}${plus}_$kver${version_string}${plus}-10.00.Custom_i386.deb
+            mv $pkg_name_mod $pkg_name_new
+            warn "Removing original package: $pkg_name_orig"
+            rm -f $pkg_name_orig
+       done
+    fi
+}
+
+function install_headers {
+    warn "Installing headers..."
+    sudo dpkg -i /usr/src/linux-headers-$kver${version_string}${plus}_$kver${version_string}${plus}-10.00.Custom_*.deb
+}
+
+function install_kernel {
+    warn "Installing kernel..."
+    sudo dpkg -i /usr/src/linux-image-$kver${version_string}${plus}_$kver${version_string}${plus}-10.00.Custom_*.deb
+}
+
+function build_initrd {
+    # Certain versions of Ubuntu install a make-kpkg"
+    # that does not build an initrd along with the rest of the kernel."
+    warn "Building initrd..."
+    #sudo mkdir -p /lib/modules/$kver${version_string}
+    sudo mkinitramfs -v -k -o /boot/initrd.img-$kver${version_string}${plus} $kver${version_string}${plus}
+}
+
+function build_ovs_datapath {
+    sudo apt-get install openvswitch-datapath-source
+    if [[ "$i386" == 'true' ]]; then
+        prepend='DEB_HOST_ARCH=i386 '
+    else
+        prepend=
+    fi
+    $prepend sudo module-assistant auto-build openvswitch-datapath -l $kver${version_string}${plus}
+}
+
+function mod_ovs_dpkg {
+    # Only needed for i386.
+    cd /usr/src
+	if [[ "$i386" == 'true' ]]; then
+        warn "Modifying deb-pkg names for i386"
+	    # Based on instructions from http://dotcommie.net/?id=165
+        for pkg_type in openvswitch-datapath-module; do
+            pkg_name_orig=${pkg_type}-$kver${version_string}${plus}_${ovs_pkg_ver}_amd64.deb
+            #hook=`readlink -f set_debian_control_i386.sh`
+            hook=${orig_dir}/set_debian_control_i386.sh
+            warn "$pkg_name_orig"
+            fakeroot deb-reversion -s "" --hook $hook $pkg_name_orig
+            # Remove the 1 in the name that deb-reversion adds.
+            pkg_name_mod=${pkg_type}-$kver${version_string}${plus}_${ovs_pkg_ver}1_i386.deb
+            pkg_name_new=${pkg_type}-$kver${version_string}${plus}_${ovs_pkg_ver}_i386.deb
+            mv $pkg_name_mod $pkg_name_new
+		    warn "Removing original package: $pkg_name_orig"
+		    rm -f $pkg_name_orig
+	   done
+	fi
+}
+
+function install_ovs_datapath {
+    warn "Installing ovs datapath"
+    sudo module-assistant install openvswitch-datapath -l $kver${version_string}${plus}
+}
+
+parse_opts $*
+
+if [[ "$install_only" != 'true' ]] ; then
+    pre_check
+
+    sudo chmod 777 $kdir
+    cd $kdir
+
+    fetch_kernel
+    copy_patches
+    apply_patches
+    configure_kernel
+    build_kernel
+    mod_kernel_dpkg
+    warn "******************************************"
+    warn "Check for kernel .deb installation file in /usr/src/ along with initrd."
+else
+    install_headers
+fi
+
+if [[ "$i386" != 'true' ]]; then
+    # Presumably we'll only want to install on a 64-bit machine.
+    install_kernel
+fi
+
+build_ovs_datapath
+mod_ovs_dpkg
+
+if [[ "$i386" != 'true' ]]; then
+    install_ovs_datapath
+    build_initrd
+fi
+
+cd $orig_dir
+warn "Done (hopefully)"

util/kbuild/cfs-nsfd-kernel/build-lxc-for-3.0.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+# Builds lxc for kernel patched with setns
+
+# Check for unitialized variables
+set -o nounset
+
+# Exit on any failure
+set -e
+
+# Kernel version to use
+kver=3.0
+
+# Location in which to download and build lxc
+lxcdir=$HOME
+kdir=/lib/modules/`uname -r`/build
+
+# lxc version to use
+lxcver=lxc-0.7.5
+
+# Save original directory for later.
+orig_dir=`pwd`
+
+function warn {
+    # Echo the provided command in color text.
+    yellow='\e[0;33m' # Yellow
+    reset='\e[0m'
+    echo="echo -e"
+    $echo "${yellow}$1${reset}"
+}
+
+function usage {
+    warn "Usage: $0 [lxc download location] [kernel location]"
+}
+
+
+if [[ "$#" > 2 ]]; then
+    warn "Invalid number of args passed."
+    usage
+    exit
+elif [[ "$#" == 0 ]]; then
+    warn "No args passed."
+    warn "Using default lxc location: ${lxcdir}." 
+    warn "Using default kernel location: ${kdir}."
+elif [[ "$#" == 1 ]]; then
+    lxcdir=$1
+    warn "Using custom lxc location: ${lxcdir}"
+    warn "Using default kernel location: ${kdir}"
+elif [[ "$#" == 2 ]]; then
+    lxcdir=$1
+    kdir=$2
+    warn "Using custom lxc location: ${lxcdir}"
+    warn "Using custom kernel location: ${kdir}"
+fi
+
+function pre_check {
+    warn "Checking for git"
+    if [[ -z `which git` ]]; then
+        read -p \
+            warn "You need git to download lxc.  Install? [Y/n] " \
+            answer;
+        [[ -z $answer || $answer=="Y" || $answer == "y" ]] && \
+            sudo apt-get install git;
+    fi
+
+    warn "Checking for linux source code"
+    if [[ ! -d ${kdir} ]]; then
+        warn "Error: Kernel doesn't exist in ${kdir}... exiting"
+        exit
+    fi
+}
+
+function fetch_lxc {
+    cd $lxcdir
+    warn "--> Fetching lxc"
+    if [[ -d lxc ]]; then
+        warn "lxc source exists, skipping.."
+        return
+    fi
+    git clone git://lxc.git.sourceforge.net/gitroot/lxc/lxc
+    cd lxc
+    git checkout $lxcver
+    cd ..
+}
+
+function copy_patches {
+    rm -rf lxc/patches
+    cp -r ${orig_dir}/../../lxc-$kver-patches lxc/patches
+}
+
+function apply_patches {
+    cd lxc
+    warn "Applying patches..."
+    git am -3 patches/*.patch
+}
+
+function build_lxc {
+    warn "Building lxc with kernel-${kver}..."
+    processors=`grep -c ^processor /proc/cpuinfo`
+    export CONCURRENCY_LEVEL=$processors
+    make distclean || true
+    ./autogen.sh
+    ./configure --with-linuxdir=${kdir}
+    make
+}
+
+function install_lxc {
+    warn "Installing lxc..."
+    sudo make install
+    # Seems to be missing
+    sudo mkdir -p /usr/local/var/lib/lxc
+}
+
+usage
+pre_check
+fetch_lxc
+copy_patches
+apply_patches
+build_lxc
+install_lxc
+cd $orig_dir
+warn "Done (hopefully)"

util/kbuild/cfs-nsfd-kernel/build.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+# Builds kernel with the new CFS Bandwidth patches
+# and nsfd/setns syscall patches.
+
+# Check for unitialized variables
+set -o nounset
+
+# Exit on any failure
+set -e
+
+# Location in which to download and build the kernel
+kdir=/usr/src
+
+# Kernel version to download
+kver=2.6.35
+
+# Save original directory for later.
+orig_dir=`pwd`
+
+# Kernel version string
+version_string=-with-cfs
+
+# Run menuconfig later?
+menuconfig=
+
+function warn {
+    # Echo the provided command in color text.
+    yellow='\e[0;33m' # Yellow
+    reset='\e[0m'
+    echo="echo -e"
+    $echo "${yellow}$1${reset}"
+}
+
+function usage {
+    warn "Usage: build.sh [version string] [menuconfig]"
+}
+
+
+if [[ "$#" > 2 ]]; then
+    warn "Invalid number of args passed."
+    usage
+    exit
+elif [[ "$#" == 0 ]]; then
+    warn "No args passed.  Using default version_string: ${version_string}"
+elif [[ "$#" == 1 ]]; then
+    warn "Using custom version_string: ${version_string}"
+    version_string=$1
+elif [[ "$#" == 2 && $2 != 'menuconfig' ]]; then
+    warn "Second arg is either menuconfig or missing."
+    usage
+else
+    version_string=$1
+    menuconfig=true
+fi
+
+function pre_check {
+    warn "Checking for kernel-package build utilities"
+    if [[ -z `which make-kpkg` ]]; then
+        read -p \
+            warn "You need kernel-package utilities to build the kernel.  Install? [Y/n] " \
+            answer;
+        [[ -z $answer || $answer=="Y" || $answer == "y" ]] && \
+            sudo apt-get install kernel-package ncurses-dev;
+    fi
+    
+    warn "Checking for quilt"
+    if [[ -z `which quilt` ]]; then
+        read -p \
+            warn "You need quilt to install patches.  Install? [Y/n] " \
+            answer;
+        [[ -z $answer || $answer=="Y" || $answer == "y" ]] && \
+            sudo apt-get install quilt;
+    fi
+}
+
+function fetch_kernel {
+    warn "--> Fetching kernel linux-$kver"
+    if [[ -f linux-$kver.tar.bz2 ]]; then
+        warn "File exists, skipping.."
+        return
+    fi
+    wget http://kernel.org/pub/linux/kernel/v2.6/linux-$kver.tar.bz2
+    warn "Unpacking kernel"
+    tar xjf linux-$kver.tar.bz2
+}
+
+function work_around_kernel_package_bug {
+    # Fix will likely break on any other kernel version, so watch out.
+    # From:
+    # https://bugs.launchpad.net/ubuntu/+source/kernel-package/+bug/58307/comments/16
+    sed -i -s 's/echo "+"/#echo "+"/' linux-${kver}/scripts/setlocalversion
+}
+
+function copy_patches {
+    rm -rf linux-$kver/patches
+    cp -r ${orig_dir}/../../linux-2.6.35-patches linux-$kver/patches
+}
+
+function apply_patches {
+    cd linux-$kver
+    # Apply patch series only if not applied previously.
+    # A better check would look at patches/series and make sure each entry
+    # in `quilt applied` was covered.
+    warn "Checking for applied patches"
+    quilt applied > quilt_applied_stdout 2> quilt_applied_stderr || true
+    if [[ `grep -c "No patches applied" quilt_applied_stderr` == 1 ]]; then
+        warn "Applying patches"
+        quilt push -a
+    else
+        warn "Skipped patches"
+    fi
+    rm quilt_applied
+}
+
+function build_kernel {
+    # Have your favourite build method here
+    # This is a standard Debian way of building the kernel
+    # The patches select cfs bandwidth automatically
+    warn "Building kernel..."
+
+    if [[ "$menuconfig" == 'true' ]]; then
+        make menuconfig
+    else
+        warn "Making oldconfig..."
+        yes "" | make oldconfig
+        warn "Making localmodconfig..."
+        make localmodconfig
+        warn "Enabling netns and cpubw..."
+        sed -i -s 's/# CONFIG_VETH is not set/CONFIG_VETH=y/' .config
+        sed -i -s 's/CONFIG_BRIDGE=y/CONFIG_BRIDGE=m/' .config
+        sed -i -s 's/# CONFIG_BRIDGE is not set/CONFIG_BRIDGE=m/' .config
+        sed -i -s 's/# CONFIG_CFS_BANDWIDTH is not set/CONFIG_CFS_BANDWIDTH=y/' .config
+        sed -i -s 's/# CONFIG_NET_NS is not set/CONFIG_NET_NS=y/' .config
+    fi
+
+    warn "Building kernel-$version_string"
+    processors=`grep -c ^processor /proc/cpuinfo`
+    export CONCURRENCY_LEVEL=$processors
+    yes "" | fakeroot make-kpkg --initrd --append-to-version=${version_string} kernel_image
+  
+    warn "******************************************"
+    warn "Check for kernel .deb installation file in ../ along with initrd."
+}
+
+function install_kernel {
+    warn "Installing kernel..."
+    sudo dpkg -i /usr/src/linux-image-$kver${version_string}_$kver${version_string}-10.00.Custom_amd64.deb
+}
+
+function build_initrd {
+    # Certain versions of Ubuntu install a make-kpkg"
+    # that does not build an initrd along with the rest of the kernel."
+    warn "Building initrd..."
+    #sudo mkdir -p /lib/modules/$kver${version_string}
+    sudo mkinitramfs -v -k -o /boot/initrd.img-$kver${version_string} $kver${version_string}
+}
+
+pre_check
+
+sudo chmod 777 $kdir
+cd $kdir
+
+fetch_kernel
+work_around_kernel_package_bug
+copy_patches
+apply_patches
+build_kernel
+install_kernel
+build_initrd
+warn "Done (hopefully)"

util/kbuild/cfs-nsfd-kernel/install-ubuntu.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Check for unitialized variables
+set -o nounset
+
+# Exit on any failure
+set -e
+
+debdir=/usr/src
+kver=3.0.9-with-cfs
+kbuild=/lib/modules/$kver/build
+
+if arch | grep 64 > /dev/null ; then arch=amd64; else arch=i386; fi
+
+headers=linux-headers-${kver}_${kver}-10.00.Custom_${arch}.deb
+image=linux-image-${kver}_${kver}-10.00.Custom_${arch}.deb
+ovs=openvswitch-datapath-module-${kver}_1.2.0-1ubuntu3_${arch}.deb
+
+echo "Mininet-hifi installer"
+
+echo "1. Checking for prereqs"
+  if [[ ! -e $debdir/$headers || ! -e $debdir/$image || 
+	! -e $debdir/$ovs ]]; then
+    echo "Can't find kernel packages"
+    echo "$debdir/$headers or $debdir/$image or $debdir/$ovs is missing"
+    exit 1
+  fi
+  if [[ "`ssh-add -l`" == "" ]]; then
+    echo "No SSH keys - nsdi repo checkout will fail."
+    exit 1
+  fi
+
+echo "2. Getting mainline Mininet from github"
+  cd ~
+  git clone git://github.com/mininet/mininet.git
+
+echo "3. Installing OpenFlow reference implementation"
+  mininet/util/install.sh -f
+
+echo "4. Installing Mininet core files"
+  mininet/util/install.sh -n
+
+echo "5. Adding nsdi repository"
+  cd ~/mininet
+  git remote add nsdi git@gitosis.stanford.edu:mininet-nsdi.git
+  git fetch nsdi
+  git checkout -b mininet-rt remotes/nsdi/mininet-rt
+  sudo make install
+
+echo "6. Installing kernel packages"
+  sudo dpkg -i $debdir/$headers
+  sudo dpkg -i $debdir/$image 
+  sudo dpkg -i $debdir/$ovs
+  
+echo "7. Fetching, building and installing Open vSwitch user code"
+  cd ~
+  git clone git://openvswitch.org/openvswitch
+  cd ~/openvswitch
+  git checkout v1.2.2
+  ./boot.sh
+  ./configure
+  make all
+  sudo make install
+  sudo cp tests/test-openflowd /usr/local/bin/ovs-openflowd
+
+echo "8. Building and installing custom lxc package"
+  sudo apt-get -y install libcap-dev
+  cd ~/mininet/util/kbuild/cfs-nsfd-kernel
+  ./build-lxc-for-3.0.sh $HOME $kbuild 
+
+echo "9. Setting up /cgroup"
+  sudo apt-get remove cgroup-lite
+  sudo mkdir /cgroup
+  sudo sh -c "echo 'cgroup /cgroup cgroup defaults 0 0' >> /etc/fstab"
+
+echo "10. Creating /etc/mn/host.conf"
+  sudo mkdir -p /etc/mn
+  sudo sh -c "echo 'lxc.utsname = mnhost' > /etc/mn/host.conf"
+  sudo sh -c "echo 'lxc.network.type = empty' >> /etc/mn/host.conf"
+
+echo "11. Getting rid of quiet boot"
+  sudo sed -i 's/quiet/text/' /etc/default/grub
+
+echo "Done! reboot to test"
+

util/kbuild/cfs-nsfd-kernel/install_lxc.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Install lxc from source, apply patch, install
+# (instructions tested with 2.6.35 only):
+sudo apt-get -y install libcap-dev quilt
+cd ~/
+git clone git://lxc.git.sourceforge.net/gitroot/lxc/lxc
+cd lxc
+git checkout lxc-0.7.2 -b lxc-0.7.2
+cp ~/mininet/util/kbuild/cfs-nsfd-kernel/lxc-patches.tar.gz .
+tar xzf lxc-patches.tar.gz
+# Modify patch.  Small change to the patch:  remove the 2nd argument to lxc_cgroup_path_get (it's set to NULL in the patch)
+sed -i -s 's/cgrouppath, NULL, my_args.name/cgrouppath, my_args.name/' patches/lxc-attach-bug-fix.patch
+quilt push -a
+./autogen.sh
+./configure
+make
+sudo make install

util/kbuild/cfs-nsfd-kernel/set_debian_control_i386.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+echo `pwd`
+sed -i -s "s/Architecture: amd64/Architecture: i386/" DEBIAN/control
+

util/kbuild/cfs-nsfd-kernel/update_ovs.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Re-build OVS for the kernel version defined below.
+
+OVS_DIR=~/openvswitch
+KERNEL_VER=`uname -r`
+#KERNEL_VER=2.6.35-with-cfs
+PROCESSORS=`grep -c ^processor /proc/cpuinfo`
+cd $OVS_DIR
+./configure --with-linux=/lib/modules/${KERNEL_VER}/build && \
+sudo make -j${PROCESSORS} && \
+sudo cp ./datapath/linux/openvswitch_mod.ko /lib/modules/${KERNEL_VER}/kernel/drivers/net && \
+echo "Running depmod..."
+sudo depmod -a ${KERNEL_VER}
+

util/linux-3.0.0-patches/0001-sched-Don-t-update-shares-twice-on-on_rq-parent.patch

@@ -0,0 +1,36 @@
+From 57cc69f4a6d27c0b3ef495589a1d4629a9f1fa3e Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Wed, 6 Jul 2011 22:30:37 -0700
+Subject: [PATCH 01/19] sched: Don't update shares twice on on_rq parent
+
+In dequeue_task_fair() we bail on dequeue when we encounter a parenting entity
+with additional weight.  However, we perform a double shares update on this
+entity as we continue the shares update traversal from this point, despite
+dequeue_entity() having already updated its queuing cfs_rq.
+Avoid this by starting from the parent when we resume.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110707053059.797714697@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched_fair.c |    3 +++
+ 1 files changed, 3 insertions(+), 0 deletions(-)
+
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index c768588..c80f030 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1370,6 +1370,9 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 			 */
+ 			if (task_sleep && parent_entity(se))
+ 				set_next_buddy(parent_entity(se));
++
++			/* avoid re-evaluating load for this entity */
++			se = parent_entity(se);
+ 			break;
+ 		}
+ 		flags |= DEQUEUE_SLEEP;
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0002-sched-Implement-hierarchical-task-accounting-for-SCH.patch

@@ -0,0 +1,168 @@
+From 4ec11a3e21874534f9ffa70a8878bb255618bb33 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:27 -0700
+Subject: [PATCH 02/19] sched: Implement hierarchical task accounting for SCHED_OTHER
+
+Introduce hierarchical task accounting for the group scheduling case in CFS, as
+well as promoting the responsibility for maintaining rq->nr_running to the
+scheduling classes.
+
+The primary motivation for this is that with scheduling classes supporting
+bandwidth throttling it is possible for entities participating in throttled
+sub-trees to not have root visible changes in rq->nr_running across activate
+and de-activate operations.  This in turn leads to incorrect idle and
+weight-per-task load balance decisions.
+
+This also allows us to make a small fixlet to the fastpath in pick_next_task()
+under group scheduling.
+
+Note: this issue also exists with the existing sched_rt throttling mechanism.
+This patch does not address that.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184756.878333391@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c          |    6 ++----
+ kernel/sched_fair.c     |   10 ++++++++--
+ kernel/sched_rt.c       |    5 ++++-
+ kernel/sched_stoptask.c |    2 ++
+ 4 files changed, 16 insertions(+), 7 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index fde6ff9..b015a0e 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -308,7 +308,7 @@ struct task_group root_task_group;
+ /* CFS-related fields in a runqueue */
+ struct cfs_rq {
+ 	struct load_weight load;
+-	unsigned long nr_running;
++	unsigned long nr_running, h_nr_running;
+ 
+ 	u64 exec_clock;
+ 	u64 min_vruntime;
+@@ -1830,7 +1830,6 @@ static void activate_task(struct rq *rq, struct task_struct *p, int flags)
+ 		rq->nr_uninterruptible--;
+ 
+ 	enqueue_task(rq, p, flags);
+-	inc_nr_running(rq);
+ }
+ 
+ /*
+@@ -1842,7 +1841,6 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
+ 		rq->nr_uninterruptible++;
+ 
+ 	dequeue_task(rq, p, flags);
+-	dec_nr_running(rq);
+ }
+ 
+ #ifdef CONFIG_IRQ_TIME_ACCOUNTING
+@@ -4226,7 +4224,7 @@ pick_next_task(struct rq *rq)
+ 	 * Optimization: we know that if all tasks are in
+ 	 * the fair class we can call that function directly:
+ 	 */
+-	if (likely(rq->nr_running == rq->cfs.nr_running)) {
++	if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+ 		p = fair_sched_class.pick_next_task(rq);
+ 		if (likely(p))
+ 			return p;
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index c80f030..f70bb4b 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1332,16 +1332,19 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 			break;
+ 		cfs_rq = cfs_rq_of(se);
+ 		enqueue_entity(cfs_rq, se, flags);
++		cfs_rq->h_nr_running++;
+ 		flags = ENQUEUE_WAKEUP;
+ 	}
+ 
+ 	for_each_sched_entity(se) {
+-		struct cfs_rq *cfs_rq = cfs_rq_of(se);
++		cfs_rq = cfs_rq_of(se);
++		cfs_rq->h_nr_running++;
+ 
+ 		update_cfs_load(cfs_rq, 0);
+ 		update_cfs_shares(cfs_rq);
+ 	}
+ 
++	inc_nr_running(rq);
+ 	hrtick_update(rq);
+ }
+ 
+@@ -1361,6 +1364,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 	for_each_sched_entity(se) {
+ 		cfs_rq = cfs_rq_of(se);
+ 		dequeue_entity(cfs_rq, se, flags);
++		cfs_rq->h_nr_running--;
+ 
+ 		/* Don't dequeue parent if it has other entities besides us */
+ 		if (cfs_rq->load.weight) {
+@@ -1379,12 +1383,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 	}
+ 
+ 	for_each_sched_entity(se) {
+-		struct cfs_rq *cfs_rq = cfs_rq_of(se);
++		cfs_rq = cfs_rq_of(se);
++		cfs_rq->h_nr_running--;
+ 
+ 		update_cfs_load(cfs_rq, 0);
+ 		update_cfs_shares(cfs_rq);
+ 	}
+ 
++	dec_nr_running(rq);
+ 	hrtick_update(rq);
+ }
+ 
+diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
+index 10d0182..1af971b 100644
+--- a/kernel/sched_rt.c
++++ b/kernel/sched_rt.c
+@@ -949,6 +949,8 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
+ 
+ 	if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)
+ 		enqueue_pushable_task(rq, p);
++
++	inc_nr_running(rq);
+ }
+ 
+ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
+@@ -959,6 +961,8 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
+ 	dequeue_rt_entity(rt_se);
+ 
+ 	dequeue_pushable_task(rq, p);
++
++	dec_nr_running(rq);
+ }
+ 
+ /*
+@@ -1851,4 +1855,3 @@ static void print_rt_stats(struct seq_file *m, int cpu)
+ 	rcu_read_unlock();
+ }
+ #endif /* CONFIG_SCHED_DEBUG */
+-
+diff --git a/kernel/sched_stoptask.c b/kernel/sched_stoptask.c
+index 6f43763..8b44e7f 100644
+--- a/kernel/sched_stoptask.c
++++ b/kernel/sched_stoptask.c
+@@ -34,11 +34,13 @@ static struct task_struct *pick_next_task_stop(struct rq *rq)
+ static void
+ enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
+ {
++	inc_nr_running(rq);
+ }
+ 
+ static void
+ dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
+ {
++	dec_nr_running(rq);
+ }
+ 
+ static void yield_task_stop(struct rq *rq)
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0003-sched-Introduce-primitives-to-account-for-CFS-bandwi.patch

@@ -0,0 +1,380 @@
+From 116f22667986ab86f1a00098a0daf9959b1f6df0 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:28 -0700
+Subject: [PATCH 03/19] sched: Introduce primitives to account for CFS bandwidth tracking
+
+In this patch we introduce the notion of CFS bandwidth, partitioned into
+globally unassigned bandwidth, and locally claimed bandwidth.
+
+ - The global bandwidth is per task_group, it represents a pool of unclaimed
+   bandwidth that cfs_rqs can allocate from.
+ - The local bandwidth is tracked per-cfs_rq, this represents allotments from
+   the global pool bandwidth assigned to a specific cpu.
+
+Bandwidth is managed via cgroupfs, adding two new interfaces to the cpu subsystem:
+ - cpu.cfs_period_us : the bandwidth period in usecs
+ - cpu.cfs_quota_us : the cpu bandwidth (in usecs) that this tg will be allowed
+   to consume over period above.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Nikhil Rao <ncrao@google.com>
+Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184756.972636699@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ init/Kconfig        |   12 +++
+ kernel/sched.c      |  201 ++++++++++++++++++++++++++++++++++++++++++++++++++-
+ kernel/sched_fair.c |   16 ++++
+ 3 files changed, 225 insertions(+), 4 deletions(-)
+
+diff --git a/init/Kconfig b/init/Kconfig
+index 412c21b..67579ed 100644
+--- a/init/Kconfig
++++ b/init/Kconfig
+@@ -715,6 +715,18 @@ config FAIR_GROUP_SCHED
+ 	depends on CGROUP_SCHED
+ 	default CGROUP_SCHED
+ 
++config CFS_BANDWIDTH
++	bool "CPU bandwidth provisioning for FAIR_GROUP_SCHED"
++	depends on EXPERIMENTAL
++	depends on FAIR_GROUP_SCHED
++	default n
++	help
++	  This option allows users to define CPU bandwidth rates (limits) for
++	  tasks running within the fair group scheduler.  Groups with no limit
++	  set are considered to be unconstrained and will run with no
++	  restriction.
++	  See tip/Documentation/scheduler/sched-bwc.txt for more information.
++
+ config RT_GROUP_SCHED
+ 	bool "Group scheduling for SCHED_RR/FIFO"
+ 	depends on EXPERIMENTAL
+diff --git a/kernel/sched.c b/kernel/sched.c
+index b015a0e..28d838b 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -244,6 +244,14 @@ struct cfs_rq;
+ 
+ static LIST_HEAD(task_groups);
+ 
++struct cfs_bandwidth {
++#ifdef CONFIG_CFS_BANDWIDTH
++	raw_spinlock_t lock;
++	ktime_t period;
++	u64 quota;
++#endif
++};
++
+ /* task group related information */
+ struct task_group {
+ 	struct cgroup_subsys_state css;
+@@ -275,6 +283,8 @@ struct task_group {
+ #ifdef CONFIG_SCHED_AUTOGROUP
+ 	struct autogroup *autogroup;
+ #endif
++
++	struct cfs_bandwidth cfs_bandwidth;
+ };
+ 
+ /* task_group_lock serializes the addition/removal of task groups */
+@@ -374,9 +384,48 @@ struct cfs_rq {
+ 
+ 	unsigned long load_contribution;
+ #endif
++#ifdef CONFIG_CFS_BANDWIDTH
++	int runtime_enabled;
++	s64 runtime_remaining;
++#endif
+ #endif
+ };
+ 
++#ifdef CONFIG_FAIR_GROUP_SCHED
++#ifdef CONFIG_CFS_BANDWIDTH
++static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
++{
++	return &tg->cfs_bandwidth;
++}
++
++static inline u64 default_cfs_period(void);
++
++static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
++{
++	raw_spin_lock_init(&cfs_b->lock);
++	cfs_b->quota = RUNTIME_INF;
++	cfs_b->period = ns_to_ktime(default_cfs_period());
++}
++
++static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++{
++	cfs_rq->runtime_enabled = 0;
++}
++
++static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
++{}
++#else
++static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
++static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
++static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
++
++static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
++{
++	return NULL;
++}
++#endif /* CONFIG_CFS_BANDWIDTH */
++#endif /* CONFIG_FAIR_GROUP_SCHED */
++
+ /* Real-Time classes' related field in a runqueue: */
+ struct rt_rq {
+ 	struct rt_prio_array active;
+@@ -7958,6 +8007,12 @@ static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
+ 	tg->cfs_rq[cpu] = cfs_rq;
+ 	init_cfs_rq(cfs_rq, rq);
+ 	cfs_rq->tg = tg;
++	cfs_rq->rq = rq;
++#ifdef CONFIG_SMP
++	/* allow initial update_cfs_load() to truncate */
++	cfs_rq->load_stamp = 1;
++#endif
++	init_cfs_rq_runtime(cfs_rq);
+ 
+ 	tg->se[cpu] = se;
+ 	/* se could be NULL for root_task_group */
+@@ -8093,6 +8148,7 @@ void __init sched_init(void)
+ 		 * We achieve this by letting root_task_group's tasks sit
+ 		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
+ 		 */
++		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
+ 		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ 
+@@ -8336,6 +8392,8 @@ static void free_fair_sched_group(struct task_group *tg)
+ {
+ 	int i;
+ 
++	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
++
+ 	for_each_possible_cpu(i) {
+ 		if (tg->cfs_rq)
+ 			kfree(tg->cfs_rq[i]);
+@@ -8363,6 +8421,8 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
+ 
+ 	tg->shares = NICE_0_LOAD;
+ 
++	init_cfs_bandwidth(tg_cfs_bandwidth(tg));
++
+ 	for_each_possible_cpu(i) {
+ 		cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
+ 				      GFP_KERNEL, cpu_to_node(i));
+@@ -8734,7 +8794,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
+ 	return walk_tg_tree(tg_schedulable, tg_nop, &data);
+ }
+ 
+-static int tg_set_bandwidth(struct task_group *tg,
++static int tg_set_rt_bandwidth(struct task_group *tg,
+ 		u64 rt_period, u64 rt_runtime)
+ {
+ 	int i, err = 0;
+@@ -8773,7 +8833,7 @@ int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
+ 	if (rt_runtime_us < 0)
+ 		rt_runtime = RUNTIME_INF;
+ 
+-	return tg_set_bandwidth(tg, rt_period, rt_runtime);
++	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
+ }
+ 
+ long sched_group_rt_runtime(struct task_group *tg)
+@@ -8798,7 +8858,7 @@ int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
+ 	if (rt_period == 0)
+ 		return -EINVAL;
+ 
+-	return tg_set_bandwidth(tg, rt_period, rt_runtime);
++	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
+ }
+ 
+ long sched_group_rt_period(struct task_group *tg)
+@@ -8988,6 +9048,128 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
+ 
+ 	return (u64) scale_load_down(tg->shares);
+ }
++
++#ifdef CONFIG_CFS_BANDWIDTH
++const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
++const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
++
++static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
++{
++	int i;
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
++	static DEFINE_MUTEX(mutex);
++
++	if (tg == &root_task_group)
++		return -EINVAL;
++
++	/*
++	 * Ensure we have at some amount of bandwidth every period.  This is
++	 * to prevent reaching a state of large arrears when throttled via
++	 * entity_tick() resulting in prolonged exit starvation.
++	 */
++	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
++		return -EINVAL;
++
++	/*
++	 * Likewise, bound things on the otherside by preventing insane quota
++	 * periods.  This also allows us to normalize in computing quota
++	 * feasibility.
++	 */
++	if (period > max_cfs_quota_period)
++		return -EINVAL;
++
++	mutex_lock(&mutex);
++	raw_spin_lock_irq(&cfs_b->lock);
++	cfs_b->period = ns_to_ktime(period);
++	cfs_b->quota = quota;
++	raw_spin_unlock_irq(&cfs_b->lock);
++
++	for_each_possible_cpu(i) {
++		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
++		struct rq *rq = rq_of(cfs_rq);
++
++		raw_spin_lock_irq(&rq->lock);
++		cfs_rq->runtime_enabled = quota != RUNTIME_INF;
++		cfs_rq->runtime_remaining = 0;
++		raw_spin_unlock_irq(&rq->lock);
++	}
++	mutex_unlock(&mutex);
++
++	return 0;
++}
++
++int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
++{
++	u64 quota, period;
++
++	period = ktime_to_ns(tg_cfs_bandwidth(tg)->period);
++	if (cfs_quota_us < 0)
++		quota = RUNTIME_INF;
++	else
++		quota = (u64)cfs_quota_us * NSEC_PER_USEC;
++
++	return tg_set_cfs_bandwidth(tg, period, quota);
++}
++
++long tg_get_cfs_quota(struct task_group *tg)
++{
++	u64 quota_us;
++
++	if (tg_cfs_bandwidth(tg)->quota == RUNTIME_INF)
++		return -1;
++
++	quota_us = tg_cfs_bandwidth(tg)->quota;
++	do_div(quota_us, NSEC_PER_USEC);
++
++	return quota_us;
++}
++
++int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
++{
++	u64 quota, period;
++
++	period = (u64)cfs_period_us * NSEC_PER_USEC;
++	quota = tg_cfs_bandwidth(tg)->quota;
++
++	if (period <= 0)
++		return -EINVAL;
++
++	return tg_set_cfs_bandwidth(tg, period, quota);
++}
++
++long tg_get_cfs_period(struct task_group *tg)
++{
++	u64 cfs_period_us;
++
++	cfs_period_us = ktime_to_ns(tg_cfs_bandwidth(tg)->period);
++	do_div(cfs_period_us, NSEC_PER_USEC);
++
++	return cfs_period_us;
++}
++
++static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft)
++{
++	return tg_get_cfs_quota(cgroup_tg(cgrp));
++}
++
++static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype,
++				s64 cfs_quota_us)
++{
++	return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us);
++}
++
++static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft)
++{
++	return tg_get_cfs_period(cgroup_tg(cgrp));
++}
++
++static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
++				u64 cfs_period_us)
++{
++	return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
++}
++
++#endif /* CONFIG_CFS_BANDWIDTH */
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ 
+ #ifdef CONFIG_RT_GROUP_SCHED
+@@ -9022,6 +9204,18 @@ static struct cftype cpu_files[] = {
+ 		.write_u64 = cpu_shares_write_u64,
+ 	},
+ #endif
++#ifdef CONFIG_CFS_BANDWIDTH
++	{
++		.name = "cfs_quota_us",
++		.read_s64 = cpu_cfs_quota_read_s64,
++		.write_s64 = cpu_cfs_quota_write_s64,
++	},
++	{
++		.name = "cfs_period_us",
++		.read_u64 = cpu_cfs_period_read_u64,
++		.write_u64 = cpu_cfs_period_write_u64,
++	},
++#endif
+ #ifdef CONFIG_RT_GROUP_SCHED
+ 	{
+ 		.name = "rt_runtime_us",
+@@ -9331,4 +9525,3 @@ struct cgroup_subsys cpuacct_subsys = {
+ 	.subsys_id = cpuacct_subsys_id,
+ };
+ #endif	/* CONFIG_CGROUP_CPUACCT */
+-
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index f70bb4b..91624cf 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1256,6 +1256,22 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
+ 		check_preempt_tick(cfs_rq, curr);
+ }
+ 
++
++/**************************************************
++ * CFS bandwidth control machinery
++ */
++
++#ifdef CONFIG_CFS_BANDWIDTH
++/*
++ * default period for cfs group bandwidth.
++ * default: 0.1s, units: nanoseconds
++ */
++static inline u64 default_cfs_period(void)
++{
++	return 100000000ULL;
++}
++#endif
++
+ /**************************************************
+  * CFS operations on tasks:
+  */
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0004-sched-Validate-CFS-quota-hierarchies.patch

@@ -0,0 +1,221 @@
+From e68a3cf7b0006f6d8c362833ebc96cbed01a263e Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:29 -0700
+Subject: [PATCH 04/19] sched: Validate CFS quota hierarchies
+
+Add constraints validation for CFS bandwidth hierarchies.
+
+Validate that:
+   max(child bandwidth) <= parent_bandwidth
+
+In a quota limited hierarchy, an unconstrained entity
+(e.g. bandwidth==RUNTIME_INF) inherits the bandwidth of its parent.
+
+This constraint is chosen over sum(child_bandwidth) as notion of over-commit is
+valuable within SCHED_OTHER.  Some basic code from the RT case is re-factored
+for reuse.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.083774572@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c |  112 +++++++++++++++++++++++++++++++++++++++++++++++++-------
+ 1 files changed, 98 insertions(+), 14 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 28d838b..75f2dd7 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -249,6 +249,7 @@ struct cfs_bandwidth {
+ 	raw_spinlock_t lock;
+ 	ktime_t period;
+ 	u64 quota;
++	s64 hierarchal_quota;
+ #endif
+ };
+ 
+@@ -1512,7 +1513,8 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load)
+ 	update_load_sub(&rq->load, load);
+ }
+ 
+-#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
++#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
++			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
+ typedef int (*tg_visitor)(struct task_group *, void *);
+ 
+ /*
+@@ -8694,12 +8696,7 @@ unsigned long sched_group_shares(struct task_group *tg)
+ }
+ #endif
+ 
+-#ifdef CONFIG_RT_GROUP_SCHED
+-/*
+- * Ensure that the real time constraints are schedulable.
+- */
+-static DEFINE_MUTEX(rt_constraints_mutex);
+-
++#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH)
+ static unsigned long to_ratio(u64 period, u64 runtime)
+ {
+ 	if (runtime == RUNTIME_INF)
+@@ -8707,6 +8704,13 @@ static unsigned long to_ratio(u64 period, u64 runtime)
+ 
+ 	return div64_u64(runtime << 20, period);
+ }
++#endif
++
++#ifdef CONFIG_RT_GROUP_SCHED
++/*
++ * Ensure that the real time constraints are schedulable.
++ */
++static DEFINE_MUTEX(rt_constraints_mutex);
+ 
+ /* Must be called with tasklist_lock held */
+ static inline int tg_has_rt_tasks(struct task_group *tg)
+@@ -8727,7 +8731,7 @@ struct rt_schedulable_data {
+ 	u64 rt_runtime;
+ };
+ 
+-static int tg_schedulable(struct task_group *tg, void *data)
++static int tg_rt_schedulable(struct task_group *tg, void *data)
+ {
+ 	struct rt_schedulable_data *d = data;
+ 	struct task_group *child;
+@@ -8791,7 +8795,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
+ 		.rt_runtime = runtime,
+ 	};
+ 
+-	return walk_tg_tree(tg_schedulable, tg_nop, &data);
++	return walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
+ }
+ 
+ static int tg_set_rt_bandwidth(struct task_group *tg,
+@@ -9050,14 +9054,17 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
+ }
+ 
+ #ifdef CONFIG_CFS_BANDWIDTH
++static DEFINE_MUTEX(cfs_constraints_mutex);
++
+ const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
+ const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
+ 
++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
++
+ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ {
+-	int i;
++	int i, ret = 0;
+ 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
+-	static DEFINE_MUTEX(mutex);
+ 
+ 	if (tg == &root_task_group)
+ 		return -EINVAL;
+@@ -9078,7 +9085,11 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 	if (period > max_cfs_quota_period)
+ 		return -EINVAL;
+ 
+-	mutex_lock(&mutex);
++	mutex_lock(&cfs_constraints_mutex);
++	ret = __cfs_schedulable(tg, period, quota);
++	if (ret)
++		goto out_unlock;
++
+ 	raw_spin_lock_irq(&cfs_b->lock);
+ 	cfs_b->period = ns_to_ktime(period);
+ 	cfs_b->quota = quota;
+@@ -9093,9 +9104,10 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 		cfs_rq->runtime_remaining = 0;
+ 		raw_spin_unlock_irq(&rq->lock);
+ 	}
+-	mutex_unlock(&mutex);
++out_unlock:
++	mutex_unlock(&cfs_constraints_mutex);
+ 
+-	return 0;
++	return ret;
+ }
+ 
+ int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
+@@ -9169,6 +9181,78 @@ static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
+ 	return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
+ }
+ 
++struct cfs_schedulable_data {
++	struct task_group *tg;
++	u64 period, quota;
++};
++
++/*
++ * normalize group quota/period to be quota/max_period
++ * note: units are usecs
++ */
++static u64 normalize_cfs_quota(struct task_group *tg,
++			       struct cfs_schedulable_data *d)
++{
++	u64 quota, period;
++
++	if (tg == d->tg) {
++		period = d->period;
++		quota = d->quota;
++	} else {
++		period = tg_get_cfs_period(tg);
++		quota = tg_get_cfs_quota(tg);
++	}
++
++	/* note: these should typically be equivalent */
++	if (quota == RUNTIME_INF || quota == -1)
++		return RUNTIME_INF;
++
++	return to_ratio(period, quota);
++}
++
++static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
++{
++	struct cfs_schedulable_data *d = data;
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
++	s64 quota = 0, parent_quota = -1;
++
++	if (!tg->parent) {
++		quota = RUNTIME_INF;
++	} else {
++		struct cfs_bandwidth *parent_b = tg_cfs_bandwidth(tg->parent);
++
++		quota = normalize_cfs_quota(tg, d);
++		parent_quota = parent_b->hierarchal_quota;
++
++		/*
++		 * ensure max(child_quota) <= parent_quota, inherit when no
++		 * limit is set
++		 */
++		if (quota == RUNTIME_INF)
++			quota = parent_quota;
++		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
++			return -EINVAL;
++	}
++	cfs_b->hierarchal_quota = quota;
++
++	return 0;
++}
++
++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
++{
++	struct cfs_schedulable_data data = {
++		.tg = tg,
++		.period = period,
++		.quota = quota,
++	};
++
++	if (quota != RUNTIME_INF) {
++		do_div(data.period, NSEC_PER_USEC);
++		do_div(data.quota, NSEC_PER_USEC);
++	}
++
++	return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
++}
+ #endif /* CONFIG_CFS_BANDWIDTH */
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ 
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0005-sched-Accumulate-per-cfs_rq-cpu-usage-and-charge-aga.patch

@@ -0,0 +1,217 @@
+From 50fe68ec9d454eced64cbfc29954ee64cc7225da Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:30 -0700
+Subject: [PATCH 05/19] sched: Accumulate per-cfs_rq cpu usage and charge against bandwidth
+
+Account bandwidth usage on the cfs_rq level versus the task_groups to which
+they belong.  Whether we are tracking bandwidth on a given cfs_rq is maintained
+under cfs_rq->runtime_enabled.
+
+cfs_rq's which belong to a bandwidth constrained task_group have their runtime
+accounted via the update_curr() path, which withdraws bandwidth from the global
+pool as desired.  Updates involving the global pool are currently protected
+under cfs_bandwidth->lock, local runtime is protected by rq->lock.
+
+This patch only assigns and tracks quota, no action is taken in the case that
+cfs_rq->runtime_used exceeds cfs_rq->runtime_assigned.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Nikhil Rao <ncrao@google.com>
+Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.179386821@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ include/linux/sched.h |    4 ++
+ kernel/sched.c        |    4 ++-
+ kernel/sched_fair.c   |   79 +++++++++++++++++++++++++++++++++++++++++++++++-
+ kernel/sysctl.c       |   10 ++++++
+ 4 files changed, 94 insertions(+), 3 deletions(-)
+
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index 14a6c7b..adfc8eb 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -2021,6 +2021,10 @@ static inline void sched_autogroup_fork(struct signal_struct *sig) { }
+ static inline void sched_autogroup_exit(struct signal_struct *sig) { }
+ #endif
+ 
++#ifdef CONFIG_CFS_BANDWIDTH
++extern unsigned int sysctl_sched_cfs_bandwidth_slice;
++#endif
++
+ #ifdef CONFIG_RT_MUTEXES
+ extern int rt_mutex_getprio(struct task_struct *p);
+ extern void rt_mutex_setprio(struct task_struct *p, int prio);
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 75f2dd7..cdbc7d3 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -248,7 +248,7 @@ struct cfs_bandwidth {
+ #ifdef CONFIG_CFS_BANDWIDTH
+ 	raw_spinlock_t lock;
+ 	ktime_t period;
+-	u64 quota;
++	u64 quota, runtime;
+ 	s64 hierarchal_quota;
+ #endif
+ };
+@@ -404,6 +404,7 @@ static inline u64 default_cfs_period(void);
+ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ {
+ 	raw_spin_lock_init(&cfs_b->lock);
++	cfs_b->runtime = 0;
+ 	cfs_b->quota = RUNTIME_INF;
+ 	cfs_b->period = ns_to_ktime(default_cfs_period());
+ }
+@@ -9093,6 +9094,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 	raw_spin_lock_irq(&cfs_b->lock);
+ 	cfs_b->period = ns_to_ktime(period);
+ 	cfs_b->quota = quota;
++	cfs_b->runtime = quota;
+ 	raw_spin_unlock_irq(&cfs_b->lock);
+ 
+ 	for_each_possible_cpu(i) {
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 91624cf..863c9ec 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -89,6 +89,20 @@ const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
+  */
+ unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL;
+ 
++#ifdef CONFIG_CFS_BANDWIDTH
++/*
++ * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
++ * each time a cfs_rq requests quota.
++ *
++ * Note: in the case that the slice exceeds the runtime remaining (either due
++ * to consumption or the quota being specified to be smaller than the slice)
++ * we will always only issue the remaining available time.
++ *
++ * default: 5 msec, units: microseconds
++  */
++unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
++#endif
++
+ static const struct sched_class fair_sched_class;
+ 
+ /**************************************************************
+@@ -305,6 +319,8 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
+ 
+ #endif	/* CONFIG_FAIR_GROUP_SCHED */
+ 
++static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
++				   unsigned long delta_exec);
+ 
+ /**************************************************************
+  * Scheduling class tree data structure manipulation methods:
+@@ -602,6 +618,8 @@ static void update_curr(struct cfs_rq *cfs_rq)
+ 		cpuacct_charge(curtask, delta_exec);
+ 		account_group_exec_runtime(curtask, delta_exec);
+ 	}
++
++	account_cfs_rq_runtime(cfs_rq, delta_exec);
+ }
+ 
+ static inline void
+@@ -1270,6 +1288,58 @@ static inline u64 default_cfs_period(void)
+ {
+ 	return 100000000ULL;
+ }
++
++static inline u64 sched_cfs_bandwidth_slice(void)
++{
++	return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
++}
++
++static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++{
++	struct task_group *tg = cfs_rq->tg;
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
++	u64 amount = 0, min_amount;
++
++	/* note: this is a positive sum as runtime_remaining <= 0 */
++	min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
++
++	raw_spin_lock(&cfs_b->lock);
++	if (cfs_b->quota == RUNTIME_INF)
++		amount = min_amount;
++	else if (cfs_b->runtime > 0) {
++		amount = min(cfs_b->runtime, min_amount);
++		cfs_b->runtime -= amount;
++	}
++	raw_spin_unlock(&cfs_b->lock);
++
++	cfs_rq->runtime_remaining += amount;
++}
++
++static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
++				     unsigned long delta_exec)
++{
++	if (!cfs_rq->runtime_enabled)
++		return;
++
++	cfs_rq->runtime_remaining -= delta_exec;
++	if (cfs_rq->runtime_remaining > 0)
++		return;
++
++	assign_cfs_rq_runtime(cfs_rq);
++}
++
++static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
++						   unsigned long delta_exec)
++{
++	if (!cfs_rq->runtime_enabled)
++		return;
++
++	__account_cfs_rq_runtime(cfs_rq, delta_exec);
++}
++
++#else
++static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
++				     unsigned long delta_exec) {}
+ #endif
+ 
+ /**************************************************
+@@ -4264,8 +4334,13 @@ static void set_curr_task_fair(struct rq *rq)
+ {
+ 	struct sched_entity *se = &rq->curr->se;
+ 
+-	for_each_sched_entity(se)
+-		set_next_entity(cfs_rq_of(se), se);
++	for_each_sched_entity(se) {
++		struct cfs_rq *cfs_rq = cfs_rq_of(se);
++
++		set_next_entity(cfs_rq, se);
++		/* ensure bandwidth has been allocated on our new cfs_rq */
++		account_cfs_rq_runtime(cfs_rq, 0);
++	}
+ }
+ 
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+diff --git a/kernel/sysctl.c b/kernel/sysctl.c
+index f175d98..b38ca7f 100644
+--- a/kernel/sysctl.c
++++ b/kernel/sysctl.c
+@@ -379,6 +379,16 @@ static struct ctl_table kern_table[] = {
+ 		.extra2		= &one,
+ 	},
+ #endif
++#ifdef CONFIG_CFS_BANDWIDTH
++	{
++		.procname	= "sched_cfs_bandwidth_slice_us",
++		.data		= &sysctl_sched_cfs_bandwidth_slice,
++		.maxlen		= sizeof(unsigned int),
++		.mode		= 0644,
++		.proc_handler	= proc_dointvec_minmax,
++		.extra1		= &one,
++	},
++#endif
+ #ifdef CONFIG_PROVE_LOCKING
+ 	{
+ 		.procname	= "prove_locking",
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0006-sched-Add-a-timer-to-handle-CFS-bandwidth-refresh.patch

@@ -0,0 +1,263 @@
+From c127107a0b9f7fe08dd11c84ecb6b307052b7688 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:31 -0700
+Subject: [PATCH 06/19] sched: Add a timer to handle CFS bandwidth refresh
+
+This patch adds a per-task_group timer which handles the refresh of the global
+CFS bandwidth pool.
+
+Since the RT pool is using a similar timer there's some small refactoring to
+share this support.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.277271273@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |  107 +++++++++++++++++++++++++++++++++++++++++----------
+ kernel/sched_fair.c |   40 +++++++++++++++++-
+ 2 files changed, 123 insertions(+), 24 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index cdbc7d3..4bb2d63 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -193,10 +193,28 @@ static inline int rt_bandwidth_enabled(void)
+ 	return sysctl_sched_rt_runtime >= 0;
+ }
+ 
+-static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
++static void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
+ {
+-	ktime_t now;
++	unsigned long delta;
++	ktime_t soft, hard, now;
+ 
++	for (;;) {
++		if (hrtimer_active(period_timer))
++			break;
++
++		now = hrtimer_cb_get_time(period_timer);
++		hrtimer_forward(period_timer, now, period);
++
++		soft = hrtimer_get_softexpires(period_timer);
++		hard = hrtimer_get_expires(period_timer);
++		delta = ktime_to_ns(ktime_sub(hard, soft));
++		__hrtimer_start_range_ns(period_timer, soft, delta,
++					 HRTIMER_MODE_ABS_PINNED, 0);
++	}
++}
++
++static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
++{
+ 	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
+ 		return;
+ 
+@@ -204,22 +222,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
+ 		return;
+ 
+ 	raw_spin_lock(&rt_b->rt_runtime_lock);
+-	for (;;) {
+-		unsigned long delta;
+-		ktime_t soft, hard;
+-
+-		if (hrtimer_active(&rt_b->rt_period_timer))
+-			break;
+-
+-		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
+-		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
+-
+-		soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
+-		hard = hrtimer_get_expires(&rt_b->rt_period_timer);
+-		delta = ktime_to_ns(ktime_sub(hard, soft));
+-		__hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
+-				HRTIMER_MODE_ABS_PINNED, 0);
+-	}
++	start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);
+ 	raw_spin_unlock(&rt_b->rt_runtime_lock);
+ }
+ 
+@@ -250,6 +253,9 @@ struct cfs_bandwidth {
+ 	ktime_t period;
+ 	u64 quota, runtime;
+ 	s64 hierarchal_quota;
++
++	int idle, timer_active;
++	struct hrtimer period_timer;
+ #endif
+ };
+ 
+@@ -400,6 +406,28 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
+ }
+ 
+ static inline u64 default_cfs_period(void);
++static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
++
++static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
++{
++	struct cfs_bandwidth *cfs_b =
++		container_of(timer, struct cfs_bandwidth, period_timer);
++	ktime_t now;
++	int overrun;
++	int idle = 0;
++
++	for (;;) {
++		now = hrtimer_cb_get_time(timer);
++		overrun = hrtimer_forward(timer, now, cfs_b->period);
++
++		if (!overrun)
++			break;
++
++		idle = do_sched_cfs_period_timer(cfs_b, overrun);
++	}
++
++	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
++}
+ 
+ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ {
+@@ -407,6 +435,9 @@ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ 	cfs_b->runtime = 0;
+ 	cfs_b->quota = RUNTIME_INF;
+ 	cfs_b->period = ns_to_ktime(default_cfs_period());
++
++	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
++	cfs_b->period_timer.function = sched_cfs_period_timer;
+ }
+ 
+ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+@@ -414,8 +445,34 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ 	cfs_rq->runtime_enabled = 0;
+ }
+ 
++/* requires cfs_b->lock, may release to reprogram timer */
++static void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
++{
++	/*
++	 * The timer may be active because we're trying to set a new bandwidth
++	 * period or because we're racing with the tear-down path
++	 * (timer_active==0 becomes visible before the hrtimer call-back
++	 * terminates).  In either case we ensure that it's re-programmed
++	 */
++	while (unlikely(hrtimer_active(&cfs_b->period_timer))) {
++		raw_spin_unlock(&cfs_b->lock);
++		/* ensure cfs_b->lock is available while we wait */
++		hrtimer_cancel(&cfs_b->period_timer);
++
++		raw_spin_lock(&cfs_b->lock);
++		/* if someone else restarted the timer then we're done */
++		if (cfs_b->timer_active)
++			return;
++	}
++
++	cfs_b->timer_active = 1;
++	start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
++}
++
+ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+-{}
++{
++	hrtimer_cancel(&cfs_b->period_timer);
++}
+ #else
+ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
+ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
+@@ -9064,7 +9121,7 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
+ 
+ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ {
+-	int i, ret = 0;
++	int i, ret = 0, runtime_enabled;
+ 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
+ 
+ 	if (tg == &root_task_group)
+@@ -9091,10 +9148,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 	if (ret)
+ 		goto out_unlock;
+ 
++	runtime_enabled = quota != RUNTIME_INF;
+ 	raw_spin_lock_irq(&cfs_b->lock);
+ 	cfs_b->period = ns_to_ktime(period);
+ 	cfs_b->quota = quota;
+ 	cfs_b->runtime = quota;
++
++	/* restart the period timer (if active) to handle new period expiry */
++	if (runtime_enabled && cfs_b->timer_active) {
++		/* force a reprogram */
++		cfs_b->timer_active = 0;
++		__start_cfs_bandwidth(cfs_b);
++	}
+ 	raw_spin_unlock_irq(&cfs_b->lock);
+ 
+ 	for_each_possible_cpu(i) {
+@@ -9102,7 +9167,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 		struct rq *rq = rq_of(cfs_rq);
+ 
+ 		raw_spin_lock_irq(&rq->lock);
+-		cfs_rq->runtime_enabled = quota != RUNTIME_INF;
++		cfs_rq->runtime_enabled = runtime_enabled;
+ 		cfs_rq->runtime_remaining = 0;
+ 		raw_spin_unlock_irq(&rq->lock);
+ 	}
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 863c9ec..e34c26c 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1306,9 +1306,16 @@ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ 	raw_spin_lock(&cfs_b->lock);
+ 	if (cfs_b->quota == RUNTIME_INF)
+ 		amount = min_amount;
+-	else if (cfs_b->runtime > 0) {
+-		amount = min(cfs_b->runtime, min_amount);
+-		cfs_b->runtime -= amount;
++	else {
++		/* ensure bandwidth timer remains active under consumption */
++		if (!cfs_b->timer_active)
++			__start_cfs_bandwidth(cfs_b);
++
++		if (cfs_b->runtime > 0) {
++			amount = min(cfs_b->runtime, min_amount);
++			cfs_b->runtime -= amount;
++			cfs_b->idle = 0;
++		}
+ 	}
+ 	raw_spin_unlock(&cfs_b->lock);
+ 
+@@ -1337,6 +1344,33 @@ static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 	__account_cfs_rq_runtime(cfs_rq, delta_exec);
+ }
+ 
++/*
++ * Responsible for refilling a task_group's bandwidth and unthrottling its
++ * cfs_rqs as appropriate. If there has been no activity within the last
++ * period the timer is deactivated until scheduling resumes; cfs_b->idle is
++ * used to track this state.
++ */
++static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
++{
++	int idle = 1;
++
++	raw_spin_lock(&cfs_b->lock);
++	/* no need to continue the timer with no bandwidth constraint */
++	if (cfs_b->quota == RUNTIME_INF)
++		goto out_unlock;
++
++	idle = cfs_b->idle;
++	cfs_b->runtime = cfs_b->quota;
++
++	/* mark as potentially idle for the upcoming period */
++	cfs_b->idle = 1;
++out_unlock:
++	if (idle)
++		cfs_b->timer_active = 0;
++	raw_spin_unlock(&cfs_b->lock);
++
++	return idle;
++}
+ #else
+ static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 				     unsigned long delta_exec) {}
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0007-sched-Expire-invalid-runtime.patch

@@ -0,0 +1,208 @@
+From bfd5537a5bca64bb37c64b3156bdbb85dbd46fae Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:32 -0700
+Subject: [PATCH 07/19] sched: Expire invalid runtime
+
+Since quota is managed using a global state but consumed on a per-cpu basis
+we need to ensure that our per-cpu state is appropriately synchronized.
+Most importantly, runtime that is state (from a previous period) should not be
+locally consumable.
+
+We take advantage of existing sched_clock synchronization about the jiffy to
+efficiently detect whether we have (globally) crossed a quota boundary above.
+
+One catch is that the direction of spread on sched_clock is undefined,
+specifically, we don't know whether our local clock is behind or ahead
+of the one responsible for the current expiration time.
+
+Fortunately we can differentiate these by considering whether the
+global deadline has advanced.  If it has not, then we assume our clock to be
+"fast" and advance our local expiration; otherwise, we know the deadline has
+truly passed and we expire our local runtime.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.379275352@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |    4 ++-
+ kernel/sched_fair.c |   90 +++++++++++++++++++++++++++++++++++++++++++++-----
+ 2 files changed, 84 insertions(+), 10 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 4bb2d63..6a0bcd5 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -253,6 +253,7 @@ struct cfs_bandwidth {
+ 	ktime_t period;
+ 	u64 quota, runtime;
+ 	s64 hierarchal_quota;
++	u64 runtime_expires;
+ 
+ 	int idle, timer_active;
+ 	struct hrtimer period_timer;
+@@ -393,6 +394,7 @@ struct cfs_rq {
+ #endif
+ #ifdef CONFIG_CFS_BANDWIDTH
+ 	int runtime_enabled;
++	u64 runtime_expires;
+ 	s64 runtime_remaining;
+ #endif
+ #endif
+@@ -9152,8 +9154,8 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 	raw_spin_lock_irq(&cfs_b->lock);
+ 	cfs_b->period = ns_to_ktime(period);
+ 	cfs_b->quota = quota;
+-	cfs_b->runtime = quota;
+ 
++	__refill_cfs_bandwidth_runtime(cfs_b);
+ 	/* restart the period timer (if active) to handle new period expiry */
+ 	if (runtime_enabled && cfs_b->timer_active) {
+ 		/* force a reprogram */
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index e34c26c..a97d19e 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1294,11 +1294,30 @@ static inline u64 sched_cfs_bandwidth_slice(void)
+ 	return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
+ }
+ 
++/*
++ * Replenish runtime according to assigned quota and update expiration time.
++ * We use sched_clock_cpu directly instead of rq->clock to avoid adding
++ * additional synchronization around rq->lock.
++ *
++ * requires cfs_b->lock
++ */
++static void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
++{
++	u64 now;
++
++	if (cfs_b->quota == RUNTIME_INF)
++		return;
++
++	now = sched_clock_cpu(smp_processor_id());
++	cfs_b->runtime = cfs_b->quota;
++	cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
++}
++
+ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ {
+ 	struct task_group *tg = cfs_rq->tg;
+ 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
+-	u64 amount = 0, min_amount;
++	u64 amount = 0, min_amount, expires;
+ 
+ 	/* note: this is a positive sum as runtime_remaining <= 0 */
+ 	min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
+@@ -1307,9 +1326,16 @@ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ 	if (cfs_b->quota == RUNTIME_INF)
+ 		amount = min_amount;
+ 	else {
+-		/* ensure bandwidth timer remains active under consumption */
+-		if (!cfs_b->timer_active)
++		/*
++		 * If the bandwidth pool has become inactive, then at least one
++		 * period must have elapsed since the last consumption.
++		 * Refresh the global state and ensure bandwidth timer becomes
++		 * active.
++		 */
++		if (!cfs_b->timer_active) {
++			__refill_cfs_bandwidth_runtime(cfs_b);
+ 			__start_cfs_bandwidth(cfs_b);
++		}
+ 
+ 		if (cfs_b->runtime > 0) {
+ 			amount = min(cfs_b->runtime, min_amount);
+@@ -1317,19 +1343,61 @@ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ 			cfs_b->idle = 0;
+ 		}
+ 	}
++	expires = cfs_b->runtime_expires;
+ 	raw_spin_unlock(&cfs_b->lock);
+ 
+ 	cfs_rq->runtime_remaining += amount;
++	/*
++	 * we may have advanced our local expiration to account for allowed
++	 * spread between our sched_clock and the one on which runtime was
++	 * issued.
++	 */
++	if ((s64)(expires - cfs_rq->runtime_expires) > 0)
++		cfs_rq->runtime_expires = expires;
+ }
+ 
+-static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+-				     unsigned long delta_exec)
++/*
++ * Note: This depends on the synchronization provided by sched_clock and the
++ * fact that rq->clock snapshots this value.
++ */
++static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ {
+-	if (!cfs_rq->runtime_enabled)
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
++	struct rq *rq = rq_of(cfs_rq);
++
++	/* if the deadline is ahead of our clock, nothing to do */
++	if (likely((s64)(rq->clock - cfs_rq->runtime_expires) < 0))
++		return;
++
++	if (cfs_rq->runtime_remaining < 0)
+ 		return;
+ 
++	/*
++	 * If the local deadline has passed we have to consider the
++	 * possibility that our sched_clock is 'fast' and the global deadline
++	 * has not truly expired.
++	 *
++	 * Fortunately we can check determine whether this the case by checking
++	 * whether the global deadline has advanced.
++	 */
++
++	if ((s64)(cfs_rq->runtime_expires - cfs_b->runtime_expires) >= 0) {
++		/* extend local deadline, drift is bounded above by 2 ticks */
++		cfs_rq->runtime_expires += TICK_NSEC;
++	} else {
++		/* global deadline is ahead, expiration has passed */
++		cfs_rq->runtime_remaining = 0;
++	}
++}
++
++static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
++				     unsigned long delta_exec)
++{
++	/* dock delta_exec before expiring quota (as it could span periods) */
+ 	cfs_rq->runtime_remaining -= delta_exec;
+-	if (cfs_rq->runtime_remaining > 0)
++	expire_cfs_rq_runtime(cfs_rq);
++
++	if (likely(cfs_rq->runtime_remaining > 0))
+ 		return;
+ 
+ 	assign_cfs_rq_runtime(cfs_rq);
+@@ -1360,7 +1428,12 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
+ 		goto out_unlock;
+ 
+ 	idle = cfs_b->idle;
+-	cfs_b->runtime = cfs_b->quota;
++	/* if we're going inactive then everything else can be deferred */
++	if (idle)
++		goto out_unlock;
++
++	__refill_cfs_bandwidth_runtime(cfs_b);
++
+ 
+ 	/* mark as potentially idle for the upcoming period */
+ 	cfs_b->idle = 1;
+@@ -1579,7 +1652,6 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
+ 
+ 	return wl;
+ }
+-
+ #else
+ 
+ static inline unsigned long effective_load(struct task_group *tg, int cpu,
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0008-sched-Add-support-for-throttling-group-entities.patch

@@ -0,0 +1,234 @@
+From 726bbbeef1579f5f981d2d98afda0304197b7e19 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:33 -0700
+Subject: [PATCH 08/19] sched: Add support for throttling group entities
+
+Now that consumption is tracked (via update_curr()) we add support to throttle
+group entities (and their corresponding cfs_rqs) in the case where this is no
+run-time remaining.
+
+Throttled entities are dequeued to prevent scheduling, additionally we mark
+them as throttled (using cfs_rq->throttled) to prevent them from becoming
+re-enqueued until they are unthrottled.  A list of a task_group's throttled
+entities are maintained on the cfs_bandwidth structure.
+
+Note: While the machinery for throttling is added in this patch the act of
+throttling an entity exceeding its bandwidth is deferred until later within
+the series.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.480608533@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |    7 ++++
+ kernel/sched_fair.c |   89 ++++++++++++++++++++++++++++++++++++++++++++++++--
+ 2 files changed, 92 insertions(+), 4 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 6a0bcd5..d631e42 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -257,6 +257,8 @@ struct cfs_bandwidth {
+ 
+ 	int idle, timer_active;
+ 	struct hrtimer period_timer;
++	struct list_head throttled_cfs_rq;
++
+ #endif
+ };
+ 
+@@ -396,6 +398,9 @@ struct cfs_rq {
+ 	int runtime_enabled;
+ 	u64 runtime_expires;
+ 	s64 runtime_remaining;
++
++	int throttled;
++	struct list_head throttled_list;
+ #endif
+ #endif
+ };
+@@ -438,6 +443,7 @@ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ 	cfs_b->quota = RUNTIME_INF;
+ 	cfs_b->period = ns_to_ktime(default_cfs_period());
+ 
++	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
+ 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ 	cfs_b->period_timer.function = sched_cfs_period_timer;
+ }
+@@ -445,6 +451,7 @@ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ {
+ 	cfs_rq->runtime_enabled = 0;
++	INIT_LIST_HEAD(&cfs_rq->throttled_list);
+ }
+ 
+ /* requires cfs_b->lock, may release to reprogram timer */
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index a97d19e..f6823e2 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1313,7 +1313,8 @@ static void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
+ 	cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
+ }
+ 
+-static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++/* returns 0 on failure to allocate runtime */
++static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ {
+ 	struct task_group *tg = cfs_rq->tg;
+ 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
+@@ -1354,6 +1355,8 @@ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+ 	 */
+ 	if ((s64)(expires - cfs_rq->runtime_expires) > 0)
+ 		cfs_rq->runtime_expires = expires;
++
++	return cfs_rq->runtime_remaining > 0;
+ }
+ 
+ /*
+@@ -1400,7 +1403,12 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 	if (likely(cfs_rq->runtime_remaining > 0))
+ 		return;
+ 
+-	assign_cfs_rq_runtime(cfs_rq);
++	/*
++	 * if we're unable to extend our runtime we resched so that the active
++	 * hierarchy can be throttled
++	 */
++	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
++		resched_task(rq_of(cfs_rq)->curr);
+ }
+ 
+ static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+@@ -1412,6 +1420,47 @@ static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 	__account_cfs_rq_runtime(cfs_rq, delta_exec);
+ }
+ 
++static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
++{
++	return cfs_rq->throttled;
++}
++
++static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
++{
++	struct rq *rq = rq_of(cfs_rq);
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
++	struct sched_entity *se;
++	long task_delta, dequeue = 1;
++
++	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
++
++	/* account load preceding throttle */
++	update_cfs_load(cfs_rq, 0);
++
++	task_delta = cfs_rq->h_nr_running;
++	for_each_sched_entity(se) {
++		struct cfs_rq *qcfs_rq = cfs_rq_of(se);
++		/* throttled entity or throttle-on-deactivate */
++		if (!se->on_rq)
++			break;
++
++		if (dequeue)
++			dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
++		qcfs_rq->h_nr_running -= task_delta;
++
++		if (qcfs_rq->load.weight)
++			dequeue = 0;
++	}
++
++	if (!se)
++		rq->nr_running -= task_delta;
++
++	cfs_rq->throttled = 1;
++	raw_spin_lock(&cfs_b->lock);
++	list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
++	raw_spin_unlock(&cfs_b->lock);
++}
++
+ /*
+  * Responsible for refilling a task_group's bandwidth and unthrottling its
+  * cfs_rqs as appropriate. If there has been no activity within the last
+@@ -1447,6 +1496,11 @@ out_unlock:
+ #else
+ static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 				     unsigned long delta_exec) {}
++
++static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
++{
++	return 0;
++}
+ #endif
+ 
+ /**************************************************
+@@ -1525,7 +1579,17 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 			break;
+ 		cfs_rq = cfs_rq_of(se);
+ 		enqueue_entity(cfs_rq, se, flags);
++
++		/*
++		 * end evaluation on encountering a throttled cfs_rq
++		 *
++		 * note: in the case of encountering a throttled cfs_rq we will
++		 * post the final h_nr_running increment below.
++		*/
++		if (cfs_rq_throttled(cfs_rq))
++			break;
+ 		cfs_rq->h_nr_running++;
++
+ 		flags = ENQUEUE_WAKEUP;
+ 	}
+ 
+@@ -1533,11 +1597,15 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 		cfs_rq = cfs_rq_of(se);
+ 		cfs_rq->h_nr_running++;
+ 
++		if (cfs_rq_throttled(cfs_rq))
++			break;
++
+ 		update_cfs_load(cfs_rq, 0);
+ 		update_cfs_shares(cfs_rq);
+ 	}
+ 
+-	inc_nr_running(rq);
++	if (!se)
++		inc_nr_running(rq);
+ 	hrtick_update(rq);
+ }
+ 
+@@ -1557,6 +1625,15 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 	for_each_sched_entity(se) {
+ 		cfs_rq = cfs_rq_of(se);
+ 		dequeue_entity(cfs_rq, se, flags);
++
++		/*
++		 * end evaluation on encountering a throttled cfs_rq
++		 *
++		 * note: in the case of encountering a throttled cfs_rq we will
++		 * post the final h_nr_running decrement below.
++		*/
++		if (cfs_rq_throttled(cfs_rq))
++			break;
+ 		cfs_rq->h_nr_running--;
+ 
+ 		/* Don't dequeue parent if it has other entities besides us */
+@@ -1579,11 +1656,15 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+ 		cfs_rq = cfs_rq_of(se);
+ 		cfs_rq->h_nr_running--;
+ 
++		if (cfs_rq_throttled(cfs_rq))
++			break;
++
+ 		update_cfs_load(cfs_rq, 0);
+ 		update_cfs_shares(cfs_rq);
+ 	}
+ 
+-	dec_nr_running(rq);
++	if (!se)
++		dec_nr_running(rq);
+ 	hrtick_update(rq);
+ }
+ 
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0009-sched-Add-support-for-unthrottling-group-entities.patch

@@ -0,0 +1,197 @@
+From b5898b8474a236451416cc68b2bea413c533f095 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:34 -0700
+Subject: [PATCH 09/19] sched: Add support for unthrottling group entities
+
+At the start of each period we refresh the global bandwidth pool.  At this time
+we must also unthrottle any cfs_rq entities who are now within bandwidth once
+more (as quota permits).
+
+Unthrottled entities have their corresponding cfs_rq->throttled flag cleared
+and their entities re-enqueued.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.574628950@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |    3 +
+ kernel/sched_fair.c |  127 +++++++++++++++++++++++++++++++++++++++++++++++++--
+ 2 files changed, 126 insertions(+), 4 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index d631e42..4b54a73 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -9178,6 +9178,9 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
+ 		raw_spin_lock_irq(&rq->lock);
+ 		cfs_rq->runtime_enabled = runtime_enabled;
+ 		cfs_rq->runtime_remaining = 0;
++
++		if (cfs_rq_throttled(cfs_rq))
++			unthrottle_cfs_rq(cfs_rq);
+ 		raw_spin_unlock_irq(&rq->lock);
+ 	}
+ out_unlock:
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index f6823e2..21e1c02 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1461,6 +1461,84 @@ static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+ 	raw_spin_unlock(&cfs_b->lock);
+ }
+ 
++static void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
++{
++	struct rq *rq = rq_of(cfs_rq);
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
++	struct sched_entity *se;
++	int enqueue = 1;
++	long task_delta;
++
++	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
++
++	cfs_rq->throttled = 0;
++	raw_spin_lock(&cfs_b->lock);
++	list_del_rcu(&cfs_rq->throttled_list);
++	raw_spin_unlock(&cfs_b->lock);
++
++	if (!cfs_rq->load.weight)
++		return;
++
++	task_delta = cfs_rq->h_nr_running;
++	for_each_sched_entity(se) {
++		if (se->on_rq)
++			enqueue = 0;
++
++		cfs_rq = cfs_rq_of(se);
++		if (enqueue)
++			enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
++		cfs_rq->h_nr_running += task_delta;
++
++		if (cfs_rq_throttled(cfs_rq))
++			break;
++	}
++
++	if (!se)
++		rq->nr_running += task_delta;
++
++	/* determine whether we need to wake up potentially idle cpu */
++	if (rq->curr == rq->idle && rq->cfs.nr_running)
++		resched_task(rq->curr);
++}
++
++static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
++		u64 remaining, u64 expires)
++{
++	struct cfs_rq *cfs_rq;
++	u64 runtime = remaining;
++
++	rcu_read_lock();
++	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
++				throttled_list) {
++		struct rq *rq = rq_of(cfs_rq);
++
++		raw_spin_lock(&rq->lock);
++		if (!cfs_rq_throttled(cfs_rq))
++			goto next;
++
++		runtime = -cfs_rq->runtime_remaining + 1;
++		if (runtime > remaining)
++			runtime = remaining;
++		remaining -= runtime;
++
++		cfs_rq->runtime_remaining += runtime;
++		cfs_rq->runtime_expires = expires;
++
++		/* we check whether we're throttled above */
++		if (cfs_rq->runtime_remaining > 0)
++			unthrottle_cfs_rq(cfs_rq);
++
++next:
++		raw_spin_unlock(&rq->lock);
++
++		if (!remaining)
++			break;
++	}
++	rcu_read_unlock();
++
++	return remaining;
++}
++
+ /*
+  * Responsible for refilling a task_group's bandwidth and unthrottling its
+  * cfs_rqs as appropriate. If there has been no activity within the last
+@@ -1469,23 +1547,64 @@ static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+  */
+ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
+ {
+-	int idle = 1;
++	u64 runtime, runtime_expires;
++	int idle = 1, throttled;
+ 
+ 	raw_spin_lock(&cfs_b->lock);
+ 	/* no need to continue the timer with no bandwidth constraint */
+ 	if (cfs_b->quota == RUNTIME_INF)
+ 		goto out_unlock;
+ 
+-	idle = cfs_b->idle;
++	throttled = !list_empty(&cfs_b->throttled_cfs_rq);
++	/* idle depends on !throttled (for the case of a large deficit) */
++	idle = cfs_b->idle && !throttled;
++
+ 	/* if we're going inactive then everything else can be deferred */
+ 	if (idle)
+ 		goto out_unlock;
+ 
+ 	__refill_cfs_bandwidth_runtime(cfs_b);
+ 
++	if (!throttled) {
++		/* mark as potentially idle for the upcoming period */
++		cfs_b->idle = 1;
++		goto out_unlock;
++	}
++
++	/*
++	 * There are throttled entities so we must first use the new bandwidth
++	 * to unthrottle them before making it generally available.  This
++	 * ensures that all existing debts will be paid before a new cfs_rq is
++	 * allowed to run.
++	 */
++	runtime = cfs_b->runtime;
++	runtime_expires = cfs_b->runtime_expires;
++	cfs_b->runtime = 0;
++
++	/*
++	 * This check is repeated as we are holding onto the new bandwidth
++	 * while we unthrottle.  This can potentially race with an unthrottled
++	 * group trying to acquire new bandwidth from the global pool.
++	 */
++	while (throttled && runtime > 0) {
++		raw_spin_unlock(&cfs_b->lock);
++		/* we can't nest cfs_b->lock while distributing bandwidth */
++		runtime = distribute_cfs_runtime(cfs_b, runtime,
++						 runtime_expires);
++		raw_spin_lock(&cfs_b->lock);
++
++		throttled = !list_empty(&cfs_b->throttled_cfs_rq);
++	}
+ 
+-	/* mark as potentially idle for the upcoming period */
+-	cfs_b->idle = 1;
++	/* return (any) remaining runtime */
++	cfs_b->runtime = runtime;
++	/*
++	 * While we are ensured activity in the period following an
++	 * unthrottle, this also covers the case in which the new bandwidth is
++	 * insufficient to cover the existing bandwidth deficit.  (Forcing the
++	 * timer to remain active while there are any throttled entities.)
++	 */
++	cfs_b->idle = 0;
+ out_unlock:
+ 	if (idle)
+ 		cfs_b->timer_active = 0;
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0010-sched-Allow-for-positional-tg_tree-walks.patch

@@ -0,0 +1,136 @@
+From b152339efae7eb1bdd9ec4e626121e9205299e9d Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:35 -0700
+Subject: [PATCH 10/19] sched: Allow for positional tg_tree walks
+
+Extend walk_tg_tree to accept a positional argument
+
+static int walk_tg_tree_from(struct task_group *from,
+			     tg_visitor down, tg_visitor up, void *data)
+
+Existing semantics are preserved, caller must hold rcu_lock() or sufficient
+analogue.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.677889157@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c |   50 +++++++++++++++++++++++++++++++++++++-------------
+ 1 files changed, 37 insertions(+), 13 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 4b54a73..813a4ce 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -1585,20 +1585,23 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load)
+ typedef int (*tg_visitor)(struct task_group *, void *);
+ 
+ /*
+- * Iterate the full tree, calling @down when first entering a node and @up when
+- * leaving it for the final time.
++ * Iterate task_group tree rooted at *from, calling @down when first entering a
++ * node and @up when leaving it for the final time.
++ *
++ * Caller must hold rcu_lock or sufficient equivalent.
+  */
+-static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
++static int walk_tg_tree_from(struct task_group *from,
++			     tg_visitor down, tg_visitor up, void *data)
+ {
+ 	struct task_group *parent, *child;
+ 	int ret;
+ 
+-	rcu_read_lock();
+-	parent = &root_task_group;
++	parent = from;
++
+ down:
+ 	ret = (*down)(parent, data);
+ 	if (ret)
+-		goto out_unlock;
++		goto out;
+ 	list_for_each_entry_rcu(child, &parent->children, siblings) {
+ 		parent = child;
+ 		goto down;
+@@ -1607,19 +1610,29 @@ up:
+ 		continue;
+ 	}
+ 	ret = (*up)(parent, data);
+-	if (ret)
+-		goto out_unlock;
++	if (ret || parent == from)
++		goto out;
+ 
+ 	child = parent;
+ 	parent = parent->parent;
+ 	if (parent)
+ 		goto up;
+-out_unlock:
+-	rcu_read_unlock();
+-
++out:
+ 	return ret;
+ }
+ 
++/*
++ * Iterate the full tree, calling @down when first entering a node and @up when
++ * leaving it for the final time.
++ *
++ * Caller must hold rcu_lock or sufficient equivalent.
++ */
++
++static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
++{
++	return walk_tg_tree_from(&root_task_group, down, up, data);
++}
++
+ static int tg_nop(struct task_group *tg, void *data)
+ {
+ 	return 0;
+@@ -8856,13 +8869,19 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)
+ 
+ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
+ {
++	int ret;
++
+ 	struct rt_schedulable_data data = {
+ 		.tg = tg,
+ 		.rt_period = period,
+ 		.rt_runtime = runtime,
+ 	};
+ 
+-	return walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
++	rcu_read_lock();
++	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
++	rcu_read_unlock();
++
++	return ret;
+ }
+ 
+ static int tg_set_rt_bandwidth(struct task_group *tg,
+@@ -9319,6 +9338,7 @@ static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
+ 
+ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
+ {
++	int ret;
+ 	struct cfs_schedulable_data data = {
+ 		.tg = tg,
+ 		.period = period,
+@@ -9330,7 +9350,11 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
+ 		do_div(data.quota, NSEC_PER_USEC);
+ 	}
+ 
+-	return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
++	rcu_read_lock();
++	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
++	rcu_read_unlock();
++
++	return ret;
+ }
+ #endif /* CONFIG_CFS_BANDWIDTH */
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0011-sched-Prevent-interactions-with-throttled-entities.patch

@@ -0,0 +1,230 @@
+From b7c5f316287ea56ecbc755110eaa032c588f2374 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:36 -0700
+Subject: [PATCH 11/19] sched: Prevent interactions with throttled entities
+
+From the perspective of load-balance and shares distribution, throttled
+entities should be invisible.
+
+However, both of these operations work on 'active' lists and are not
+inherently aware of what group hierarchies may be present.  In some cases this
+may be side-stepped (e.g. we could sideload via tg_load_down in load balance)
+while in others (e.g. update_shares()) it is more difficult to compute without
+incurring some O(n^2) costs.
+
+Instead, track hierarchicaal throttled state at time of transition.  This
+allows us to easily identify whether an entity belongs to a throttled hierarchy
+and avoid incorrect interactions with it.
+
+Also, when an entity leaves a throttled hierarchy we need to advance its
+time averaging for shares averaging so that the elapsed throttled time is not
+considered as part of the cfs_rq's operation.
+
+We also use this information to prevent buddy interactions in the wakeup and
+yield_to() paths.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.777916795@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |    2 +-
+ kernel/sched_fair.c |  103 ++++++++++++++++++++++++++++++++++++++++++++++++---
+ 2 files changed, 98 insertions(+), 7 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 813a4ce..523464e 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -399,7 +399,7 @@ struct cfs_rq {
+ 	u64 runtime_expires;
+ 	s64 runtime_remaining;
+ 
+-	int throttled;
++	int throttled, throttle_count;
+ 	struct list_head throttled_list;
+ #endif
+ #endif
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 21e1c02..3d7430b 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -725,6 +725,8 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
+ }
+ 
+ #ifdef CONFIG_FAIR_GROUP_SCHED
++/* we need this in update_cfs_load and load-balance functions below */
++static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
+ # ifdef CONFIG_SMP
+ static void update_cfs_rq_load_contribution(struct cfs_rq *cfs_rq,
+ 					    int global_update)
+@@ -747,7 +749,7 @@ static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
+ 	u64 now, delta;
+ 	unsigned long load = cfs_rq->load.weight;
+ 
+-	if (cfs_rq->tg == &root_task_group)
++	if (cfs_rq->tg == &root_task_group || throttled_hierarchy(cfs_rq))
+ 		return;
+ 
+ 	now = rq_of(cfs_rq)->clock_task;
+@@ -856,7 +858,7 @@ static void update_cfs_shares(struct cfs_rq *cfs_rq)
+ 
+ 	tg = cfs_rq->tg;
+ 	se = tg->se[cpu_of(rq_of(cfs_rq))];
+-	if (!se)
++	if (!se || throttled_hierarchy(cfs_rq))
+ 		return;
+ #ifndef CONFIG_SMP
+ 	if (likely(se->load.weight == tg->shares))
+@@ -1425,6 +1427,65 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
+ 	return cfs_rq->throttled;
+ }
+ 
++/* check whether cfs_rq, or any parent, is throttled */
++static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
++{
++	return cfs_rq->throttle_count;
++}
++
++/*
++ * Ensure that neither of the group entities corresponding to src_cpu or
++ * dest_cpu are members of a throttled hierarchy when performing group
++ * load-balance operations.
++ */
++static inline int throttled_lb_pair(struct task_group *tg,
++				    int src_cpu, int dest_cpu)
++{
++	struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
++
++	src_cfs_rq = tg->cfs_rq[src_cpu];
++	dest_cfs_rq = tg->cfs_rq[dest_cpu];
++
++	return throttled_hierarchy(src_cfs_rq) ||
++	       throttled_hierarchy(dest_cfs_rq);
++}
++
++/* updated child weight may affect parent so we have to do this bottom up */
++static int tg_unthrottle_up(struct task_group *tg, void *data)
++{
++	struct rq *rq = data;
++	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
++
++	cfs_rq->throttle_count--;
++#ifdef CONFIG_SMP
++	if (!cfs_rq->throttle_count) {
++		u64 delta = rq->clock_task - cfs_rq->load_stamp;
++
++		/* leaving throttled state, advance shares averaging windows */
++		cfs_rq->load_stamp += delta;
++		cfs_rq->load_last += delta;
++
++		/* update entity weight now that we are on_rq again */
++		update_cfs_shares(cfs_rq);
++	}
++#endif
++
++	return 0;
++}
++
++static int tg_throttle_down(struct task_group *tg, void *data)
++{
++	struct rq *rq = data;
++	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
++
++	/* group is entering throttled state, record last load */
++	if (!cfs_rq->throttle_count)
++		update_cfs_load(cfs_rq, 0);
++	cfs_rq->throttle_count++;
++
++	return 0;
++}
++
+ static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+ {
+ 	struct rq *rq = rq_of(cfs_rq);
+@@ -1435,7 +1496,9 @@ static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+ 	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
+ 
+ 	/* account load preceding throttle */
+-	update_cfs_load(cfs_rq, 0);
++	rcu_read_lock();
++	walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
++	rcu_read_unlock();
+ 
+ 	task_delta = cfs_rq->h_nr_running;
+ 	for_each_sched_entity(se) {
+@@ -1476,6 +1539,10 @@ static void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
+ 	list_del_rcu(&cfs_rq->throttled_list);
+ 	raw_spin_unlock(&cfs_b->lock);
+ 
++	update_rq_clock(rq);
++	/* update hierarchical throttle state */
++	walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
++
+ 	if (!cfs_rq->load.weight)
+ 		return;
+ 
+@@ -1620,6 +1687,17 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
+ {
+ 	return 0;
+ }
++
++static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
++{
++	return 0;
++}
++
++static inline int throttled_lb_pair(struct task_group *tg,
++				    int src_cpu, int dest_cpu)
++{
++	return 0;
++}
+ #endif
+ 
+ /**************************************************
+@@ -2521,6 +2599,9 @@ move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
+ 
+ 	for_each_leaf_cfs_rq(busiest, cfs_rq) {
+ 		list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
++			if (throttled_lb_pair(task_group(p),
++					      busiest->cpu, this_cpu))
++				break;
+ 
+ 			if (!can_migrate_task(p, busiest, this_cpu,
+ 						sd, idle, &pinned))
+@@ -2632,8 +2713,17 @@ static void update_shares(int cpu)
+ 	struct rq *rq = cpu_rq(cpu);
+ 
+ 	rcu_read_lock();
+-	for_each_leaf_cfs_rq(rq, cfs_rq)
++	/*
++	 * Iterates the task_group tree in a bottom up fashion, see
++	 * list_add_leaf_cfs_rq() for details.
++	 */
++	for_each_leaf_cfs_rq(rq, cfs_rq) {
++		/* throttled entities do not contribute to load */
++		if (throttled_hierarchy(cfs_rq))
++			continue;
++
+ 		update_shares_cpu(cfs_rq->tg, cpu);
++	}
+ 	rcu_read_unlock();
+ }
+ 
+@@ -2657,9 +2747,10 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
+ 		u64 rem_load, moved_load;
+ 
+ 		/*
+-		 * empty group
++		 * empty group or part of a throttled hierarchy
+ 		 */
+-		if (!busiest_cfs_rq->task_weight)
++		if (!busiest_cfs_rq->task_weight ||
++		    throttled_lb_pair(busiest_cfs_rq->tg, cpu_of(busiest), this_cpu))
+ 			continue;
+ 
+ 		rem_load = (u64)rem_load_move * busiest_weight;
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0012-sched-Prevent-buddy-interactions-with-throttled-enti.patch

@@ -0,0 +1,65 @@
+From 41f8be245b607e16567c13f6be065084b73c4977 Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:37 -0700
+Subject: [PATCH 12/19] sched: Prevent buddy interactions with throttled entities
+
+Buddies allow us to select "on-rq" entities without actually selecting them
+from a cfs_rq's rb_tree.  As a result we must ensure that throttled entities
+are not falsely nominated as buddies.  The fact that entities are dequeued
+within throttle_entity is not sufficient for clearing buddy status as the
+nomination may occur after throttling.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.886850167@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched_fair.c |   18 +++++++++++++++++-
+ 1 files changed, 17 insertions(+), 1 deletions(-)
+
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 3d7430b..3c0120e 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -2372,6 +2372,15 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
+ 	if (unlikely(se == pse))
+ 		return;
+ 
++	/*
++	 * This is possible from callers such as pull_task(), in which we
++	 * unconditionally check_prempt_curr() after an enqueue (which may have
++	 * lead to a throttle).  This both saves work and prevents false
++	 * next-buddy nomination below.
++	 */
++	if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
++		return;
++
+ 	if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
+ 		set_next_buddy(pse);
+ 		next_buddy_marked = 1;
+@@ -2380,6 +2389,12 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
+ 	/*
+ 	 * We can come here with TIF_NEED_RESCHED already set from new task
+ 	 * wake up path.
++	 *
++	 * Note: this also catches the edge-case of curr being in a throttled
++	 * group (e.g. via set_curr_task), since update_curr() (in the
++	 * enqueue of curr) will have resulted in resched being set.  This
++	 * prevents us from potentially nominating it as a false LAST_BUDDY
++	 * below.
+ 	 */
+ 	if (test_tsk_need_resched(curr))
+ 		return;
+@@ -2502,7 +2517,8 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp
+ {
+ 	struct sched_entity *se = &p->se;
+ 
+-	if (!se->on_rq)
++	/* throttled hierarchies are not runnable */
++	if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
+ 		return false;
+ 
+ 	/* Tell the scheduler that we'd really like pse to run next. */
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0013-sched-Migrate-throttled-tasks-on-HOTPLUG.patch

@@ -0,0 +1,69 @@
+From 7af3c5930e241d0bddc028e3d05ad396c32689be Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:38 -0700
+Subject: [PATCH 13/19] sched: Migrate throttled tasks on HOTPLUG
+
+Throttled tasks are invisisble to cpu-offline since they are not eligible for
+selection by pick_next_task().  The regular 'escape' path for a thread that is
+blocked at offline is via ttwu->select_task_rq, however this will not handle a
+throttled group since there are no individual thread wakeups on an unthrottle.
+
+Resolve this by unthrottling offline cpus so that threads can be migrated.
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184757.989000590@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c |   27 +++++++++++++++++++++++++++
+ 1 files changed, 27 insertions(+), 0 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 523464e..7b99d63 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -6310,6 +6310,30 @@ static void calc_global_load_remove(struct rq *rq)
+ 	rq->calc_load_active = 0;
+ }
+ 
++#ifdef CONFIG_CFS_BANDWIDTH
++static void unthrottle_offline_cfs_rqs(struct rq *rq)
++{
++	struct cfs_rq *cfs_rq;
++
++	for_each_leaf_cfs_rq(rq, cfs_rq) {
++		struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
++
++		if (!cfs_rq->runtime_enabled)
++			continue;
++
++		/*
++		 * clock_task is not advancing so we just need to make sure
++		 * there's some valid quota amount
++		 */
++		cfs_rq->runtime_remaining = cfs_b->quota;
++		if (cfs_rq_throttled(cfs_rq))
++			unthrottle_cfs_rq(cfs_rq);
++	}
++}
++#else
++static void unthrottle_offline_cfs_rqs(struct rq *rq) {}
++#endif
++
+ /*
+  * Migrate all tasks from the rq, sleeping tasks will be migrated by
+  * try_to_wake_up()->select_task_rq().
+@@ -6335,6 +6359,9 @@ static void migrate_tasks(unsigned int dead_cpu)
+ 	 */
+ 	rq->stop = NULL;
+ 
++	/* Ensure any throttled groups are reachable by pick_next_task */
++	unthrottle_offline_cfs_rqs(rq);
++
+ 	for ( ; ; ) {
+ 		/*
+ 		 * There's this thread running, bail when that's the only
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0014-sched-Throttle-entities-exceeding-their-allowed-band.patch

@@ -0,0 +1,133 @@
+From d71d241613f903255fec91ba9f959a633c724b4e Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:39 -0700
+Subject: [PATCH 14/19] sched: Throttle entities exceeding their allowed bandwidth
+
+With the machinery in place to throttle and unthrottle entities, as well as
+handle their participation (or lack there of) we can now enable throttling.
+
+There are 2 points that we must check whether it's time to set throttled state:
+ put_prev_entity() and enqueue_entity().
+
+- put_prev_entity() is the typical throttle path, we reach it by exceeding our
+  allocated run-time within update_curr()->account_cfs_rq_runtime() and going
+  through a reschedule.
+
+- enqueue_entity() covers the case of a wake-up into an already throttled
+  group.  In this case we know the group cannot be on_rq and can throttle
+  immediately.  Checks are added at time of put_prev_entity() and
+  enqueue_entity()
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184758.091415417@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched_fair.c |   52 +++++++++++++++++++++++++++++++++++++++++++++++++-
+ 1 files changed, 50 insertions(+), 2 deletions(-)
+
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 3c0120e..831a300 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -989,6 +989,8 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
+ 	se->vruntime = vruntime;
+ }
+ 
++static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
++
+ static void
+ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+ {
+@@ -1018,8 +1020,10 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+ 		__enqueue_entity(cfs_rq, se);
+ 	se->on_rq = 1;
+ 
+-	if (cfs_rq->nr_running == 1)
++	if (cfs_rq->nr_running == 1) {
+ 		list_add_leaf_cfs_rq(cfs_rq);
++		check_enqueue_throttle(cfs_rq);
++	}
+ }
+ 
+ static void __clear_buddies_last(struct sched_entity *se)
+@@ -1224,6 +1228,8 @@ static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
+ 	return se;
+ }
+ 
++static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
++
+ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
+ {
+ 	/*
+@@ -1233,6 +1239,9 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
+ 	if (prev->on_rq)
+ 		update_curr(cfs_rq);
+ 
++	/* throttle cfs_rqs exceeding runtime */
++	check_cfs_rq_runtime(cfs_rq);
++
+ 	check_spread(cfs_rq, prev);
+ 	if (prev->on_rq) {
+ 		update_stats_wait_start(cfs_rq, prev);
+@@ -1486,7 +1495,7 @@ static int tg_throttle_down(struct task_group *tg, void *data)
+ 	return 0;
+ }
+ 
+-static __used void throttle_cfs_rq(struct cfs_rq *cfs_rq)
++static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+ {
+ 	struct rq *rq = rq_of(cfs_rq);
+ 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
+@@ -1679,9 +1688,48 @@ out_unlock:
+ 
+ 	return idle;
+ }
++
++/*
++ * When a group wakes up we want to make sure that its quota is not already
++ * expired/exceeded, otherwise it may be allowed to steal additional ticks of
++ * runtime as update_curr() throttling can not not trigger until it's on-rq.
++ */
++static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
++{
++	/* an active group must be handled by the update_curr()->put() path */
++	if (!cfs_rq->runtime_enabled || cfs_rq->curr)
++		return;
++
++	/* ensure the group is not already throttled */
++	if (cfs_rq_throttled(cfs_rq))
++		return;
++
++	/* update runtime allocation */
++	account_cfs_rq_runtime(cfs_rq, 0);
++	if (cfs_rq->runtime_remaining <= 0)
++		throttle_cfs_rq(cfs_rq);
++}
++
++/* conditionally throttle active cfs_rq's from put_prev_entity() */
++static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++{
++	if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
++		return;
++
++	/*
++	 * it's possible for a throttled entity to be forced into a running
++	 * state (e.g. set_curr_task), in this case we're finished.
++	 */
++	if (cfs_rq_throttled(cfs_rq))
++		return;
++
++	throttle_cfs_rq(cfs_rq);
++}
+ #else
+ static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 				     unsigned long delta_exec) {}
++static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
++static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
+ 
+ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
+ {
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0015-sched-Add-exports-tracking-cfs-bandwidth-control-sta.patch

@@ -0,0 +1,125 @@
+From 11db1560b4dec193a20e2c78fb8238d9f14a1782 Mon Sep 17 00:00:00 2001
+From: Nikhil Rao <ncrao@google.com>
+Date: Thu, 21 Jul 2011 09:43:40 -0700
+Subject: [PATCH 15/19] sched: Add exports tracking cfs bandwidth control statistics
+
+This change introduces statistics exports for the cpu sub-system, these are
+added through the use of a stat file similar to that exported by other
+subsystems.
+
+The following exports are included:
+
+nr_periods:	number of periods in which execution occurred
+nr_throttled:	the number of periods above in which execution was throttle
+throttled_time:	cumulative wall-time that any cpus have been throttled for
+this group
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Nikhil Rao <ncrao@google.com>
+Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
+Reviewed-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184758.198901931@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |   21 +++++++++++++++++++++
+ kernel/sched_fair.c |    7 +++++++
+ 2 files changed, 28 insertions(+), 0 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 7b99d63..08d3aa0 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -259,6 +259,9 @@ struct cfs_bandwidth {
+ 	struct hrtimer period_timer;
+ 	struct list_head throttled_cfs_rq;
+ 
++	/* statistics */
++	int nr_periods, nr_throttled;
++	u64 throttled_time;
+ #endif
+ };
+ 
+@@ -399,6 +402,7 @@ struct cfs_rq {
+ 	u64 runtime_expires;
+ 	s64 runtime_remaining;
+ 
++	u64 throttled_timestamp;
+ 	int throttled, throttle_count;
+ 	struct list_head throttled_list;
+ #endif
+@@ -9383,6 +9387,19 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
+ 
+ 	return ret;
+ }
++
++static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
++		struct cgroup_map_cb *cb)
++{
++	struct task_group *tg = cgroup_tg(cgrp);
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
++
++	cb->fill(cb, "nr_periods", cfs_b->nr_periods);
++	cb->fill(cb, "nr_throttled", cfs_b->nr_throttled);
++	cb->fill(cb, "throttled_time", cfs_b->throttled_time);
++
++	return 0;
++}
+ #endif /* CONFIG_CFS_BANDWIDTH */
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ 
+@@ -9429,6 +9446,10 @@ static struct cftype cpu_files[] = {
+ 		.read_u64 = cpu_cfs_period_read_u64,
+ 		.write_u64 = cpu_cfs_period_write_u64,
+ 	},
++	{
++		.name = "stat",
++		.read_map = cpu_stats_show,
++	},
+ #endif
+ #ifdef CONFIG_RT_GROUP_SCHED
+ 	{
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 831a300..2060fc9 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1528,6 +1528,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
+ 		rq->nr_running -= task_delta;
+ 
+ 	cfs_rq->throttled = 1;
++	cfs_rq->throttled_timestamp = rq->clock;
+ 	raw_spin_lock(&cfs_b->lock);
+ 	list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
+ 	raw_spin_unlock(&cfs_b->lock);
+@@ -1545,8 +1546,10 @@ static void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
+ 
+ 	cfs_rq->throttled = 0;
+ 	raw_spin_lock(&cfs_b->lock);
++	cfs_b->throttled_time += rq->clock - cfs_rq->throttled_timestamp;
+ 	list_del_rcu(&cfs_rq->throttled_list);
+ 	raw_spin_unlock(&cfs_b->lock);
++	cfs_rq->throttled_timestamp = 0;
+ 
+ 	update_rq_clock(rq);
+ 	/* update hierarchical throttle state */
+@@ -1634,6 +1637,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
+ 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);
+ 	/* idle depends on !throttled (for the case of a large deficit) */
+ 	idle = cfs_b->idle && !throttled;
++	cfs_b->nr_periods += overrun;
+ 
+ 	/* if we're going inactive then everything else can be deferred */
+ 	if (idle)
+@@ -1647,6 +1651,9 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
+ 		goto out_unlock;
+ 	}
+ 
++	/* account preceding periods in which throttling occurred */
++	cfs_b->nr_throttled += overrun;
++
+ 	/*
+ 	 * There are throttled entities so we must first use the new bandwidth
+ 	 * to unthrottle them before making it generally available.  This
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0016-sched-Return-unused-runtime-on-group-dequeue.patch

@@ -0,0 +1,252 @@
+From 9baa7b654e1527bfec8f413f7372de6c4aeebb6a Mon Sep 17 00:00:00 2001
+From: Paul Turner <pjt@google.com>
+Date: Thu, 21 Jul 2011 09:43:41 -0700
+Subject: [PATCH 16/19] sched: Return unused runtime on group dequeue
+
+When a local cfs_rq blocks we return the majority of its remaining quota to the
+global bandwidth pool for use by other runqueues.
+
+We do this only when the quota is current and there is more than
+min_cfs_rq_quota [1ms by default] of runtime remaining on the rq.
+
+In the case where there are throttled runqueues and we have sufficient
+bandwidth to meter out a slice, a second timer is kicked off to handle this
+delivery, unthrottling where appropriate.
+
+Using a 'worst case' antagonist which executes on each cpu
+for 1ms before moving onto the next on a fairly large machine:
+
+no quota generations:
+
+ 197.47 ms       /cgroup/a/cpuacct.usage
+ 199.46 ms       /cgroup/a/cpuacct.usage
+ 205.46 ms       /cgroup/a/cpuacct.usage
+ 198.46 ms       /cgroup/a/cpuacct.usage
+ 208.39 ms       /cgroup/a/cpuacct.usage
+
+Since we are allowed to use "stale" quota our usage is effectively bounded by
+the rate of input into the global pool and performance is relatively stable.
+
+with quota generations [1s increments]:
+
+ 119.58 ms       /cgroup/a/cpuacct.usage
+ 119.65 ms       /cgroup/a/cpuacct.usage
+ 119.64 ms       /cgroup/a/cpuacct.usage
+ 119.63 ms       /cgroup/a/cpuacct.usage
+ 119.60 ms       /cgroup/a/cpuacct.usage
+
+The large deficit here is due to quota generations (/intentionally/) preventing
+us from now using previously stranded slack quota.  The cost is that this quota
+becomes unavailable.
+
+with quota generations and quota return:
+
+ 200.09 ms       /cgroup/a/cpuacct.usage
+ 200.09 ms       /cgroup/a/cpuacct.usage
+ 198.09 ms       /cgroup/a/cpuacct.usage
+ 200.09 ms       /cgroup/a/cpuacct.usage
+ 200.06 ms       /cgroup/a/cpuacct.usage
+
+By returning unused quota we're able to both stably consume our desired quota
+and prevent unintentional overages due to the abuse of slack quota from
+previous quota periods (especially on a large machine).
+
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184758.306848658@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ kernel/sched.c      |   15 +++++++-
+ kernel/sched_fair.c |  108 +++++++++++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 122 insertions(+), 1 deletions(-)
+
+diff --git a/kernel/sched.c b/kernel/sched.c
+index 08d3aa0..8be4ca2 100644
+--- a/kernel/sched.c
++++ b/kernel/sched.c
+@@ -256,7 +256,7 @@ struct cfs_bandwidth {
+ 	u64 runtime_expires;
+ 
+ 	int idle, timer_active;
+-	struct hrtimer period_timer;
++	struct hrtimer period_timer, slack_timer;
+ 	struct list_head throttled_cfs_rq;
+ 
+ 	/* statistics */
+@@ -418,6 +418,16 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
+ 
+ static inline u64 default_cfs_period(void);
+ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
++static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b);
++
++static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
++{
++	struct cfs_bandwidth *cfs_b =
++		container_of(timer, struct cfs_bandwidth, slack_timer);
++	do_sched_cfs_slack_timer(cfs_b);
++
++	return HRTIMER_NORESTART;
++}
+ 
+ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
+ {
+@@ -450,6 +460,8 @@ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
+ 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ 	cfs_b->period_timer.function = sched_cfs_period_timer;
++	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
++	cfs_b->slack_timer.function = sched_cfs_slack_timer;
+ }
+ 
+ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+@@ -485,6 +497,7 @@ static void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
+ {
+ 	hrtimer_cancel(&cfs_b->period_timer);
++	hrtimer_cancel(&cfs_b->slack_timer);
+ }
+ #else
+ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
+diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
+index 2060fc9..edf3b3e 100644
+--- a/kernel/sched_fair.c
++++ b/kernel/sched_fair.c
+@@ -1071,6 +1071,8 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
+ 		__clear_buddies_skip(se);
+ }
+ 
++static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
++
+ static void
+ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+ {
+@@ -1109,6 +1111,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+ 	if (!(flags & DEQUEUE_SLEEP))
+ 		se->vruntime -= cfs_rq->min_vruntime;
+ 
++	/* return excess runtime on last dequeue */
++	return_cfs_rq_runtime(cfs_rq);
++
+ 	update_min_vruntime(cfs_rq);
+ 	update_cfs_shares(cfs_rq);
+ }
+@@ -1696,6 +1701,108 @@ out_unlock:
+ 	return idle;
+ }
+ 
++/* a cfs_rq won't donate quota below this amount */
++static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
++/* minimum remaining period time to redistribute slack quota */
++static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
++/* how long we wait to gather additional slack before distributing */
++static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
++
++/* are we near the end of the current quota period? */
++static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
++{
++	struct hrtimer *refresh_timer = &cfs_b->period_timer;
++	u64 remaining;
++
++	/* if the call-back is running a quota refresh is already occurring */
++	if (hrtimer_callback_running(refresh_timer))
++		return 1;
++
++	/* is a quota refresh about to occur? */
++	remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
++	if (remaining < min_expire)
++		return 1;
++
++	return 0;
++}
++
++static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
++{
++	u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
++
++	/* if there's a quota refresh soon don't bother with slack */
++	if (runtime_refresh_within(cfs_b, min_left))
++		return;
++
++	start_bandwidth_timer(&cfs_b->slack_timer,
++				ns_to_ktime(cfs_bandwidth_slack_period));
++}
++
++/* we know any runtime found here is valid as update_curr() precedes return */
++static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++{
++	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
++	s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
++
++	if (slack_runtime <= 0)
++		return;
++
++	raw_spin_lock(&cfs_b->lock);
++	if (cfs_b->quota != RUNTIME_INF &&
++	    cfs_rq->runtime_expires == cfs_b->runtime_expires) {
++		cfs_b->runtime += slack_runtime;
++
++		/* we are under rq->lock, defer unthrottling using a timer */
++		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
++		    !list_empty(&cfs_b->throttled_cfs_rq))
++			start_cfs_slack_bandwidth(cfs_b);
++	}
++	raw_spin_unlock(&cfs_b->lock);
++
++	/* even if it's not valid for return we don't want to try again */
++	cfs_rq->runtime_remaining -= slack_runtime;
++}
++
++static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
++{
++	if (!cfs_rq->runtime_enabled || !cfs_rq->nr_running)
++		return;
++
++	__return_cfs_rq_runtime(cfs_rq);
++}
++
++/*
++ * This is done with a timer (instead of inline with bandwidth return) since
++ * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
++ */
++static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
++{
++	u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
++	u64 expires;
++
++	/* confirm we're still not at a refresh boundary */
++	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration))
++		return;
++
++	raw_spin_lock(&cfs_b->lock);
++	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) {
++		runtime = cfs_b->runtime;
++		cfs_b->runtime = 0;
++	}
++	expires = cfs_b->runtime_expires;
++	raw_spin_unlock(&cfs_b->lock);
++
++	if (!runtime)
++		return;
++
++	runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
++
++	raw_spin_lock(&cfs_b->lock);
++	if (expires == cfs_b->runtime_expires)
++		cfs_b->runtime = runtime;
++	raw_spin_unlock(&cfs_b->lock);
++}
++
+ /*
+  * When a group wakes up we want to make sure that its quota is not already
+  * expired/exceeded, otherwise it may be allowed to steal additional ticks of
+@@ -1737,6 +1844,7 @@ static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ 				     unsigned long delta_exec) {}
+ static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
+ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
++static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
+ 
+ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
+ {
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0017-sched-Add-documentation-for-bandwidth-control.patch

@@ -0,0 +1,148 @@
+From d5edecf4b5298b11f6d39e3029b7620ee06640e7 Mon Sep 17 00:00:00 2001
+From: Bharata B Rao <bharata@linux.vnet.ibm.com>
+Date: Thu, 21 Jul 2011 09:43:43 -0700
+Subject: [PATCH 17/19] sched: Add documentation for bandwidth control
+
+Basic description of usage and effect for CFS Bandwidth Control.
+
+Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
+Signed-off-by: Paul Turner <pjt@google.com>
+Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
+Link: http://lkml.kernel.org/r/20110721184758.498036116@google.com
+Signed-off-by: Ingo Molnar <mingo@elte.hu>
+---
+ Documentation/scheduler/sched-bwc.txt |  122 +++++++++++++++++++++++++++++++++
+ 1 files changed, 122 insertions(+), 0 deletions(-)
+ create mode 100644 Documentation/scheduler/sched-bwc.txt
+
+diff --git a/Documentation/scheduler/sched-bwc.txt b/Documentation/scheduler/sched-bwc.txt
+new file mode 100644
+index 0000000..f6b1873
+--- /dev/null
++++ b/Documentation/scheduler/sched-bwc.txt
+@@ -0,0 +1,122 @@
++CFS Bandwidth Control
++=====================
++
++[ This document only discusses CPU bandwidth control for SCHED_NORMAL.
++  The SCHED_RT case is covered in Documentation/scheduler/sched-rt-group.txt ]
++
++CFS bandwidth control is a CONFIG_FAIR_GROUP_SCHED extension which allows the
++specification of the maximum CPU bandwidth available to a group or hierarchy.
++
++The bandwidth allowed for a group is specified using a quota and period. Within
++each given "period" (microseconds), a group is allowed to consume only up to
++"quota" microseconds of CPU time.  When the CPU bandwidth consumption of a
++group exceeds this limit (for that period), the tasks belonging to its
++hierarchy will be throttled and are not allowed to run again until the next
++period.
++
++A group's unused runtime is globally tracked, being refreshed with quota units
++above at each period boundary.  As threads consume this bandwidth it is
++transferred to cpu-local "silos" on a demand basis.  The amount transferred
++within each of these updates is tunable and described as the "slice".
++
++Management
++----------
++Quota and period are managed within the cpu subsystem via cgroupfs.
++
++cpu.cfs_quota_us: the total available run-time within a period (in microseconds)
++cpu.cfs_period_us: the length of a period (in microseconds)
++cpu.stat: exports throttling statistics [explained further below]
++
++The default values are:
++	cpu.cfs_period_us=100ms
++	cpu.cfs_quota=-1
++
++A value of -1 for cpu.cfs_quota_us indicates that the group does not have any
++bandwidth restriction in place, such a group is described as an unconstrained
++bandwidth group.  This represents the traditional work-conserving behavior for
++CFS.
++
++Writing any (valid) positive value(s) will enact the specified bandwidth limit.
++The minimum quota allowed for the quota or period is 1ms.  There is also an
++upper bound on the period length of 1s.  Additional restrictions exist when
++bandwidth limits are used in a hierarchical fashion, these are explained in
++more detail below.
++
++Writing any negative value to cpu.cfs_quota_us will remove the bandwidth limit
++and return the group to an unconstrained state once more.
++
++Any updates to a group's bandwidth specification will result in it becoming
++unthrottled if it is in a constrained state.
++
++System wide settings
++--------------------
++For efficiency run-time is transferred between the global pool and CPU local
++"silos" in a batch fashion.  This greatly reduces global accounting pressure
++on large systems.  The amount transferred each time such an update is required
++is described as the "slice".
++
++This is tunable via procfs:
++	/proc/sys/kernel/sched_cfs_bandwidth_slice_us (default=5ms)
++
++Larger slice values will reduce transfer overheads, while smaller values allow
++for more fine-grained consumption.
++
++Statistics
++----------
++A group's bandwidth statistics are exported via 3 fields in cpu.stat.
++
++cpu.stat:
++- nr_periods: Number of enforcement intervals that have elapsed.
++- nr_throttled: Number of times the group has been throttled/limited.
++- throttled_time: The total time duration (in nanoseconds) for which entities
++  of the group have been throttled.
++
++This interface is read-only.
++
++Hierarchical considerations
++---------------------------
++The interface enforces that an individual entity's bandwidth is always
++attainable, that is: max(c_i) <= C. However, over-subscription in the
++aggregate case is explicitly allowed to enable work-conserving semantics
++within a hierarchy.
++  e.g. \Sum (c_i) may exceed C
++[ Where C is the parent's bandwidth, and c_i its children ]
++
++
++There are two ways in which a group may become throttled:
++	a. it fully consumes its own quota within a period
++	b. a parent's quota is fully consumed within its period
++
++In case b) above, even though the child may have runtime remaining it will not
++be allowed to until the parent's runtime is refreshed.
++
++Examples
++--------
++1. Limit a group to 1 CPU worth of runtime.
++
++	If period is 250ms and quota is also 250ms, the group will get
++	1 CPU worth of runtime every 250ms.
++
++	# echo 250000 > cpu.cfs_quota_us /* quota = 250ms */
++	# echo 250000 > cpu.cfs_period_us /* period = 250ms */
++
++2. Limit a group to 2 CPUs worth of runtime on a multi-CPU machine.
++
++	With 500ms period and 1000ms quota, the group can get 2 CPUs worth of
++	runtime every 500ms.
++
++	# echo 1000000 > cpu.cfs_quota_us /* quota = 1000ms */
++	# echo 500000 > cpu.cfs_period_us /* period = 500ms */
++
++	The larger period here allows for increased burst capacity.
++
++3. Limit a group to 20% of 1 CPU.
++
++	With 50ms period, 10ms quota will be equivalent to 20% of 1 CPU.
++
++	# echo 10000 > cpu.cfs_quota_us /* quota = 10ms */
++	# echo 50000 > cpu.cfs_period_us /* period = 50ms */
++
++	By using a small period here we are ensuring a consistent latency
++	response at the expense of burst capacity.
++
+-- 
+1.7.0.4
+

util/linux-3.0.0-patches/0019-dctcp-patch.patch

@@ -0,0 +1,437 @@
+From 46394a392b85376e0c17a4f84e2468a0b62ec5b6 Mon Sep 17 00:00:00 2001
+From: Vimalkumar <j.vimal+nf@gmail.com>
+Date: Wed, 7 Sep 2011 14:17:32 -0700
+Subject: [PATCH 19/19] dctcp patch
+
+---
+ include/linux/sysctl.h     |    3 +
+ include/linux/tcp.h        |   10 +++
+ include/net/tcp.h          |    3 +
+ kernel/sysctl_binary.c     |    3 +
+ net/ipv4/sysctl_net_ipv4.c |   21 +++++
+ net/ipv4/tcp_input.c       |  182 ++++++++++++++++++++++++++++++++++++++++----
+ net/ipv4/tcp_output.c      |   19 +++++-
+ 7 files changed, 225 insertions(+), 16 deletions(-)
+
+diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h
+index 11684d9..fd8c73a 100644
+--- a/include/linux/sysctl.h
++++ b/include/linux/sysctl.h
+@@ -425,6 +425,9 @@ enum
+ 	NET_TCP_ALLOWED_CONG_CONTROL=123,
+ 	NET_TCP_MAX_SSTHRESH=124,
+ 	NET_TCP_FRTO_RESPONSE=125,
++	NET_TCP_DELAYED_ACK=126,
++	NET_TCP_DCTCP_ENABLE=127,
++	NET_TCP_DCTCP_SHIFT_G=128,
+ };
+ 
+ enum {
+diff --git a/include/linux/tcp.h b/include/linux/tcp.h
+index e64f4c6..9d2ec1c 100644
+--- a/include/linux/tcp.h
++++ b/include/linux/tcp.h
+@@ -455,6 +455,16 @@ struct tcp_sock {
+ 	struct tcp_md5sig_info	*md5sig_info;
+ #endif
+ 
++/* DCTCP Specific Parameters */
++ 	u32	acked_bytes_ecn;
++ 	u32	acked_bytes_total;
++ 	u32	prior_ack;
++ 	u32	prior_rcv_nxt;
++ 	u32	dctcp_alpha;
++ 	u32	next_seq;
++ 	u32	ce_state;	/* 0: last pkt was non-ce , 1: last pkt was ce */
++ 	u32	delayed_ack_reserved;
++
+ 	/* When the cookie options are generated and exchanged, then this
+ 	 * object holds a reference to them (cookie_values->kref).  Also
+ 	 * contains related tcp_cookie_transactions fields.
+diff --git a/include/net/tcp.h b/include/net/tcp.h
+index cda30ea..b6b1480 100644
+--- a/include/net/tcp.h
++++ b/include/net/tcp.h
+@@ -223,6 +223,9 @@ extern int sysctl_tcp_max_orphans;
+ extern int sysctl_tcp_fack;
+ extern int sysctl_tcp_reordering;
+ extern int sysctl_tcp_ecn;
++extern int sysctl_tcp_delayed_ack;
++extern int sysctl_tcp_dctcp_enable;
++extern int sysctl_tcp_dctcp_shift_g;
+ extern int sysctl_tcp_dsack;
+ extern long sysctl_tcp_mem[3];
+ extern int sysctl_tcp_wmem[3];
+diff --git a/kernel/sysctl_binary.c b/kernel/sysctl_binary.c
+index 20dfc21..f232b5a 100644
+--- a/kernel/sysctl_binary.c
++++ b/kernel/sysctl_binary.c
+@@ -373,6 +373,9 @@ static const struct bin_table bin_net_ipv4_table[] = {
+ 	{ CTL_INT,	NET_TCP_FACK,				"tcp_fack" },
+ 	{ CTL_INT,	NET_TCP_REORDERING,			"tcp_reordering" },
+ 	{ CTL_INT,	NET_TCP_ECN,				"tcp_ecn" },
++	{ CTL_INT,	NET_TCP_DELAYED_ACK,			"tcp_delayed_ack" },
++	{ CTL_INT,	NET_TCP_DCTCP_ENABLE,			"tcp_dctcp_enable" },
++	{ CTL_INT,	NET_TCP_DCTCP_SHIFT_G,			"tcp_dctcp_shift_g" },
+ 	{ CTL_INT,	NET_TCP_DSACK,				"tcp_dsack" },
+ 	{ CTL_INT,	NET_TCP_MEM,				"tcp_mem" },
+ 	{ CTL_INT,	NET_TCP_WMEM,				"tcp_wmem" },
+diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
+index 57d0752..c896edf 100644
+--- a/net/ipv4/sysctl_net_ipv4.c
++++ b/net/ipv4/sysctl_net_ipv4.c
+@@ -440,6 +440,27 @@ static struct ctl_table ipv4_table[] = {
+ 		.proc_handler	= proc_dointvec
+ 	},
+ 	{
++		.procname	= "tcp_delayed_ack",
++		.data		= &sysctl_tcp_delayed_ack,
++		.maxlen		= sizeof(int),
++		.mode		= 0644,
++		.proc_handler	= proc_dointvec
++	},
++	{
++		.procname	= "tcp_dctcp_enable",
++		.data		= &sysctl_tcp_dctcp_enable,
++		.maxlen		= sizeof(int),
++		.mode		= 0644,
++		.proc_handler	= proc_dointvec
++	},
++	{
++		.procname	= "tcp_dctcp_shift_g",
++		.data		= &sysctl_tcp_dctcp_shift_g,
++		.maxlen		= sizeof(int),
++		.mode		= 0644,
++		.proc_handler	= proc_dointvec
++	},
++	{
+ 		.procname	= "tcp_dsack",
+ 		.data		= &sysctl_tcp_dsack,
+ 		.maxlen		= sizeof(int),
+diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
+index bef9f04..7b9829b 100644
+--- a/net/ipv4/tcp_input.c
++++ b/net/ipv4/tcp_input.c
+@@ -98,6 +98,13 @@ int sysctl_tcp_thin_dupack __read_mostly;
+ int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
+ int sysctl_tcp_abc __read_mostly;
+ 
++int sysctl_tcp_delayed_ack __read_mostly = 1;
++EXPORT_SYMBOL(sysctl_tcp_delayed_ack);
++int sysctl_tcp_dctcp_enable __read_mostly;
++EXPORT_SYMBOL(sysctl_tcp_dctcp_enable);
++int sysctl_tcp_dctcp_shift_g  __read_mostly = 4; /* g=1/2^4 */
++EXPORT_SYMBOL(sysctl_tcp_dctcp_shift_g);
++
+ #define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
+ #define FLAG_WIN_UPDATE		0x02 /* Incoming ACK was a window update.	*/
+ #define FLAG_DATA_ACKED		0x04 /* This ACK acknowledged new data.		*/
+@@ -217,16 +224,70 @@ static inline void TCP_ECN_withdraw_cwr(struct tcp_sock *tp)
+ 	tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
+ }
+ 
+-static inline void TCP_ECN_check_ce(struct tcp_sock *tp, struct sk_buff *skb)
++static inline void TCP_ECN_dctcp_check_ce(struct sock *sk, struct tcp_sock *tp, struct sk_buff *skb)
+ {
+ 	if (tp->ecn_flags & TCP_ECN_OK) {
+-		if (INET_ECN_is_ce(TCP_SKB_CB(skb)->flags))
+-			tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
+-		/* Funny extension: if ECT is not set on a segment,
+-		 * it is surely retransmit. It is not in ECN RFC,
+-		 * but Linux follows this rule. */
+-		else if (INET_ECN_is_not_ect((TCP_SKB_CB(skb)->flags)))
+-			tcp_enter_quickack_mode((struct sock *)tp);
++	  u32 temp_rcv_nxt;
++
++	  if (INET_ECN_is_ce(TCP_SKB_CB(skb)->flags)) {
++
++	    /* rcv_nxt is already update in previous process (tcp_rcv_established) */
++
++	    if(sysctl_tcp_dctcp_enable) {
++
++	      /* state has changed from CE=0 to CE=1 && delayed ack has not sent yet */
++	      if(tp->ce_state == 0 && tp->delayed_ack_reserved) {
++
++		/* save current rcv_nxt */
++		temp_rcv_nxt = tp->rcv_nxt;
++		/* generate previous ack with CE=0 */
++		tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
++		tp->rcv_nxt = tp->prior_rcv_nxt;
++		/* printk("CE=0 rcv_nxt= %u nxt= %u\n",tp->rcv_nxt, temp_rcv_nxt);  */
++		tcp_send_ack(sk);
++		/* recover current rcv_nxt */
++		tp->rcv_nxt = temp_rcv_nxt;
++	      }
++	      
++	      tp->ce_state = 1;
++	    }
++
++	    tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
++
++
++	    /* Funny extension: if ECT is not set on a segment,
++	     * it is surely retransmit. It is not in ECN RFC,
++	     * but Linux follows this rule. */
++	  } else if (INET_ECN_is_not_ect((TCP_SKB_CB(skb)->flags))) {
++	    tcp_enter_quickack_mode((struct sock *)tp);
++	  }else {
++	    /* It has ECT but it doesn't have CE */
++	    
++	    if(sysctl_tcp_dctcp_enable) {
++	      
++	      if(tp->ce_state != 0 && tp->delayed_ack_reserved) {
++		
++		/* save current rcv_nxt */
++		temp_rcv_nxt = tp->rcv_nxt;
++		/* generate previous ack with CE=1 */
++		tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
++		tp->rcv_nxt = tp->prior_rcv_nxt;
++		/* printk("CE=1 rcv_nxt= %u nxt= %u\n",tp->rcv_nxt, temp_rcv_nxt);  */
++		tcp_send_ack(sk);
++		/* recover current rcv_nxt */
++		tp->rcv_nxt = temp_rcv_nxt;
++	      }
++
++	      tp->ce_state = 0;
++
++	      /* deassert only when DCTCP is enabled */
++	      tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
++	    }
++
++	  }
++	    
++	  /* set current rcv_nxt to prior_rcv_nxt */
++	  tp->prior_rcv_nxt = tp->rcv_nxt;
+ 	}
+ }
+ 
+@@ -581,6 +642,8 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
+ 		 */
+ 		tcp_incr_quickack(sk);
+ 		icsk->icsk_ack.ato = TCP_ATO_MIN;
++
++		tp->ce_state = 0;
+ 	} else {
+ 		int m = now - icsk->icsk_ack.lrcvtime;
+ 
+@@ -601,7 +664,7 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
+ 	}
+ 	icsk->icsk_ack.lrcvtime = now;
+ 
+-	TCP_ECN_check_ce(tp, skb);
++	TCP_ECN_dctcp_check_ce(sk, tp, skb);
+ 
+ 	if (skb->len >= 128)
+ 		tcp_grow_window(sk, skb);
+@@ -827,19 +890,54 @@ void tcp_enter_cwr(struct sock *sk, const int set_ssthresh)
+ 	struct tcp_sock *tp = tcp_sk(sk);
+ 	const struct inet_connection_sock *icsk = inet_csk(sk);
+ 
++	__u32 ssthresh_old; 
++	__u32 cwnd_old;
++	__u32 cwnd_new;
++
+ 	tp->prior_ssthresh = 0;
+ 	tp->bytes_acked = 0;
+ 	if (icsk->icsk_ca_state < TCP_CA_CWR) {
+ 		tp->undo_marker = 0;
+-		if (set_ssthresh)
+-			tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
+-		tp->snd_cwnd = min(tp->snd_cwnd,
+-				   tcp_packets_in_flight(tp) + 1U);
++
++		if(!sysctl_tcp_dctcp_enable) {
++
++		  if (set_ssthresh)
++		    tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
++
++		  tp->snd_cwnd = min(tp->snd_cwnd,
++				     tcp_packets_in_flight(tp) + 1U);
++		  
++		}else {
++
++		  cwnd_new = max (tp->snd_cwnd - ((tp->snd_cwnd * tp->dctcp_alpha)>>11) , 2U);
++
++		  if(set_ssthresh) {
++		    
++		    ssthresh_old = tp->snd_ssthresh;
++		    tp->snd_ssthresh =  cwnd_new;
++		    
++		    /* printk("%llu alpha= %d ssth old= %d new= %d\n", */
++		    /* 		    			   ktime_to_us(ktime_get_real()), */
++		    /* 		    			   tp->dctcp_alpha, */
++		    /* 		    			   ssthresh_old, */
++		    /* 		    			   tp->snd_ssthresh); */
++		  }
++		  
++		  cwnd_old = tp->snd_cwnd;
++		  tp->snd_cwnd = cwnd_new;
++		  
++		  /* printk("%llu alpha= %d cwnd old= %d new= %d\n", */
++		  /* 		  			 ktime_to_us(ktime_get_real()), */
++		  /* 		  			 tp->dctcp_alpha, */
++		  /* 		  			 cwnd_old, */
++		  /* 		  			 tp->snd_cwnd); */
++		}
++		
+ 		tp->snd_cwnd_cnt = 0;
+ 		tp->high_seq = tp->snd_nxt;
+ 		tp->snd_cwnd_stamp = tcp_time_stamp;
+ 		TCP_ECN_queue_cwr(tp);
+-
++		
+ 		tcp_set_ca_state(sk, TCP_CA_CWR);
+ 	}
+ }
+@@ -2859,6 +2957,7 @@ static void tcp_try_to_open(struct sock *sk, int flag)
+ 		tcp_try_keep_open(sk);
+ 		tcp_moderate_cwnd(tp);
+ 	} else {
++	  if(!sysctl_tcp_dctcp_enable)
+ 		tcp_cwnd_down(sk, flag);
+ 	}
+ }
+@@ -3624,6 +3723,9 @@ static int tcp_ack(struct sock *sk, struct sk_buff *skb, int flag)
+ 	int prior_packets;
+ 	int frto_cwnd = 0;
+ 
++	__u32 alpha_old;
++	__u32 acked_bytes;
++
+ 	/* If the ack is older than previous acks
+ 	 * then we can probably ignore it.
+ 	 */
+@@ -3680,6 +3782,54 @@ static int tcp_ack(struct sock *sk, struct sk_buff *skb, int flag)
+ 		tcp_ca_event(sk, CA_EVENT_SLOW_ACK);
+ 	}
+ 
++	/* START: DCTCP Processing */
++
++	/* calc acked bytes */
++	if(after(ack,tp->prior_ack)) {
++	  acked_bytes = ack - tp->prior_ack;
++	} else {
++	  
++	  if(flag & FLAG_WIN_UPDATE) {
++	    /* Don't count when it is Window Updated ACK */
++	    acked_bytes = 0; 
++	    /* printk("acked_byte=0\n"); */
++	  }else {
++	    /* Count duplicate ACKs for Retransmission packets and so on as MSS size */
++	    acked_bytes = inet_csk(sk)->icsk_ack.rcv_mss;
++	  }
++	}
++
++	if(flag & FLAG_ECE) 
++	  tp->acked_bytes_ecn += acked_bytes;
++
++	tp->acked_bytes_total += acked_bytes;
++
++	tp->prior_ack = ack;
++
++	/* Expired RTT */
++        if (!before(tp->snd_una,tp->next_seq)) {
++
++	  /* For avoiding denominator == 1 */
++	  if(tp->acked_bytes_total == 0) tp->acked_bytes_total = 1;
++
++          alpha_old = tp->dctcp_alpha; 
++
++	  /* alpha = (1-g) * alpha + g * F */
++	  tp->dctcp_alpha = alpha_old - (alpha_old >> sysctl_tcp_dctcp_shift_g)
++	    + (tp->acked_bytes_ecn << (10 - sysctl_tcp_dctcp_shift_g)) / tp->acked_bytes_total;  
++	  
++	  if(tp->dctcp_alpha > 1024) tp->dctcp_alpha = 1024; /* round to 0-1024 */
++
++          /* printk("bytes_ecn= %d total= %d alpha: old= %d new= %d\n", */
++	  /* 	  		 tp->acked_bytes_ecn, tp->acked_bytes_total, alpha_old, tp->dctcp_alpha); */
++	  
++	  tp->acked_bytes_ecn = 0;
++	  tp->acked_bytes_total = 0;
++	  tp->next_seq = tp->snd_nxt;
++        }
++
++	/* END: DCTCP Processing */
++
+ 	/* We passed data and got it acked, remove any soft error
+ 	 * log. Something worked...
+ 	 */
+@@ -4480,7 +4630,7 @@ drop:
+ 		goto queue_and_out;
+ 	}
+ 
+-	TCP_ECN_check_ce(tp, skb);
++	TCP_ECN_dctcp_check_ce(sk, tp, skb);
+ 
+ 	if (tcp_try_rmem_schedule(sk, skb->truesize))
+ 		goto drop;
+@@ -4931,6 +5081,8 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
+ 	     __tcp_select_window(sk) >= tp->rcv_wnd) ||
+ 	    /* We ACK each frame or... */
+ 	    tcp_in_quickack_mode(sk) ||
++	    /* Delayed ACK is disabled or ... */
++	    sysctl_tcp_delayed_ack == 0 ||
+ 	    /* We have out of order data. */
+ 	    (ofo_possible && skb_peek(&tp->out_of_order_queue))) {
+ 		/* Then ack it now */
+diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
+index 882e0b0..2a4d1dc 100644
+--- a/net/ipv4/tcp_output.c
++++ b/net/ipv4/tcp_output.c
+@@ -308,7 +308,7 @@ static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb)
+ 	struct tcp_sock *tp = tcp_sk(sk);
+ 
+ 	tp->ecn_flags = 0;
+-	if (sysctl_tcp_ecn == 1) {
++	if (sysctl_tcp_ecn == 1 || sysctl_tcp_dctcp_enable) {
+ 		TCP_SKB_CB(skb)->flags |= TCPHDR_ECE | TCPHDR_CWR;
+ 		tp->ecn_flags = TCP_ECN_OK;
+ 	}
+@@ -878,6 +878,10 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
+ 	if (likely((tcb->flags & TCPHDR_SYN) == 0))
+ 		TCP_ECN_send(sk, skb, tcp_header_size);
+ 
++	/* In DCTCP, Assert ECT bit to all packets*/
++	if(sysctl_tcp_dctcp_enable)
++		INET_ECN_xmit(sk);
++
+ #ifdef CONFIG_TCP_MD5SIG
+ 	/* Calculate the MD5 hash, as we have all we need now */
+ 	if (md5) {
+@@ -2624,6 +2628,11 @@ int tcp_connect(struct sock *sk)
+ 	tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
+ 	TCP_ECN_send_syn(sk, buff);
+ 
++	/* Initialize DCTCP internal parameters */
++	tp->next_seq = tp->snd_nxt;
++	tp->acked_bytes_ecn = 0;
++	tp->acked_bytes_total = 0;
++
+ 	/* Send it off. */
+ 	TCP_SKB_CB(buff)->when = tcp_time_stamp;
+ 	tp->retrans_stamp = TCP_SKB_CB(buff)->when;
+@@ -2660,6 +2669,10 @@ void tcp_send_delayed_ack(struct sock *sk)
+ 	int ato = icsk->icsk_ack.ato;
+ 	unsigned long timeout;
+ 
++	/* Delayed ACK reserved flag for DCTCP */
++	struct tcp_sock *tp = tcp_sk(sk);
++	tp->delayed_ack_reserved = 1;
++
+ 	if (ato > TCP_DELACK_MIN) {
+ 		const struct tcp_sock *tp = tcp_sk(sk);
+ 		int max_ato = HZ / 2;
+@@ -2711,6 +2724,10 @@ void tcp_send_ack(struct sock *sk)
+ {
+ 	struct sk_buff *buff;
+ 
++	/* Delayed ACK reserved flag for DCTCP */
++	struct tcp_sock *tp = tcp_sk(sk);
++	tp->delayed_ack_reserved = 0;
++
+ 	/* If we have been reset, we may not send again. */
+ 	if (sk->sk_state == TCP_CLOSE)
+ 		return;
+-- 
+1.7.0.4
+

util/nox-patches/0002-nox-ubuntu12-hacks.patch

@@ -0,0 +1,175 @@
+From 166693d7cb640d4a41251b87e92c52d9c688196b Mon Sep 17 00:00:00 2001
+From: Bob Lantz <rlantz@cs.stanford.edu>
+Date: Mon, 14 May 2012 15:30:44 -0700
+Subject: [PATCH] Hacks to get NOX classic/destiny to compile under Ubuntu
+ 12.04
+
+Thanks to Srinivasu R. Kanduru for the initial patch.
+
+Apologies for the hacks - it is my hope that this will be fixed
+upstream eventually.
+
+---
+ config/ac_pkg_swig.m4                              |    7 ++++---
+ src/Make.vars                                      |    2 +-
+ src/nox/coreapps/pyrt/deferredcallback.cc          |    2 +-
+ src/nox/coreapps/pyrt/pyglue.cc                    |    2 +-
+ src/nox/coreapps/pyrt/pyrt.cc                      |    2 +-
+ src/nox/netapps/authenticator/auth.i               |    2 ++
+ src/nox/netapps/authenticator/flow_util.i          |    1 +
+ src/nox/netapps/routing/routing.i                  |    2 ++
+ .../switch_management/pyswitch_management.i        |    2 ++
+ src/nox/netapps/tests/tests.cc                     |    2 +-
+ src/nox/netapps/topology/pytopology.i              |    2 ++
+ 11 files changed, 18 insertions(+), 8 deletions(-)
+
+diff --git a/config/ac_pkg_swig.m4 b/config/ac_pkg_swig.m4
+index d12556e..9b608f2 100644
+--- a/config/ac_pkg_swig.m4
++++ b/config/ac_pkg_swig.m4
+@@ -78,9 +78,10 @@ AC_DEFUN([AC_PROG_SWIG],[
+                         if test -z "$available_patch" ; then
+                                 [available_patch=0]
+                         fi
+-                        if test $available_major -ne $required_major \
+-                                -o $available_minor -ne $required_minor \
+-                                -o $available_patch -lt $required_patch ; then
++			major_done=`test $available_major -gt $required_major`
++			minor_done=`test $available_minor -gt $required_minor`
++                        if test !$major_done -a !$minor_done \
++                                -a $available_patch -lt $required_patch ; then
+                                 AC_MSG_WARN([SWIG version >= $1 is required.  You have $swig_version.  You should look at http://www.swig.org])
+                                 SWIG=''
+                         else
+diff --git a/src/Make.vars b/src/Make.vars
+index d70d6aa..93b2879 100644
+--- a/src/Make.vars
++++ b/src/Make.vars
+@@ -53,7 +53,7 @@ AM_LDFLAGS += -export-dynamic
+ endif
+ 
+ # set python runtimefiles to be installed in the same directory as pkg
+-pkglib_SCRIPTS = $(NOX_RUNTIMEFILES) $(NOX_PYBUILDFILES)
++pkgdata_SCRIPTS = $(NOX_RUNTIMEFILES) $(NOX_PYBUILDFILES)
+ BUILT_SOURCES  = $(NOX_PYBUILDFILES)
+ 
+ # Runtime-files build and clean rules
+diff --git a/src/nox/coreapps/pyrt/deferredcallback.cc b/src/nox/coreapps/pyrt/deferredcallback.cc
+index 3a40fa7..111a586 100644
+--- a/src/nox/coreapps/pyrt/deferredcallback.cc
++++ b/src/nox/coreapps/pyrt/deferredcallback.cc
+@@ -69,7 +69,7 @@ DeferredCallback::get_instance(const Callback& c)
+     DeferredCallback* cb = new DeferredCallback(c);
+ 
+     // flag as used in *_wrap.cc....correct?
+-    return SWIG_Python_NewPointerObj(cb, s, SWIG_POINTER_OWN | 0);
++    return SWIG_Python_NewPointerObj(m, cb, s, SWIG_POINTER_OWN | 0);
+ }
+ 
+ bool
+diff --git a/src/nox/coreapps/pyrt/pyglue.cc b/src/nox/coreapps/pyrt/pyglue.cc
+index 48b9716..317fd04 100644
+--- a/src/nox/coreapps/pyrt/pyglue.cc
++++ b/src/nox/coreapps/pyrt/pyglue.cc
+@@ -874,7 +874,7 @@ to_python(const Flow& flow)
+     if (!s) {
+         throw std::runtime_error("Could not find Flow SWIG type_info");
+     }
+-    return SWIG_Python_NewPointerObj(f, s, SWIG_POINTER_OWN | 0);
++    return SWIG_Python_NewPointerObj(m, f, s, SWIG_POINTER_OWN | 0);
+ 
+ //     PyObject* dict = PyDict_New();
+ //     if (!dict) {
+diff --git a/src/nox/coreapps/pyrt/pyrt.cc b/src/nox/coreapps/pyrt/pyrt.cc
+index fbda461..8ec05d6 100644
+--- a/src/nox/coreapps/pyrt/pyrt.cc
++++ b/src/nox/coreapps/pyrt/pyrt.cc
+@@ -776,7 +776,7 @@ Python_event_manager::create_python_context(const Context* ctxt,
+                             pretty_print_python_exception());
+     }
+ 
+-    PyObject* pyctxt = SWIG_Python_NewPointerObj(p, s, 0);
++    PyObject* pyctxt = SWIG_Python_NewPointerObj(m, p, s, 0);
+     Py_INCREF(pyctxt); // XXX needed?
+ 
+     //Py_DECREF(m);
+diff --git a/src/nox/netapps/authenticator/auth.i b/src/nox/netapps/authenticator/auth.i
+index 1de1a17..bfa04e2 100644
+--- a/src/nox/netapps/authenticator/auth.i
++++ b/src/nox/netapps/authenticator/auth.i
+@@ -18,6 +18,8 @@
+ 
+ %module "nox.netapps.authenticator.pyauth"
+ 
++// Hack to get it to compile -BL
++%include "std_list.i"
+ %{
+ #include "core_events.hh"
+ #include "pyrt/pycontext.hh"
+diff --git a/src/nox/netapps/authenticator/flow_util.i b/src/nox/netapps/authenticator/flow_util.i
+index f67c3ef..2a314e2 100644
+--- a/src/nox/netapps/authenticator/flow_util.i
++++ b/src/nox/netapps/authenticator/flow_util.i
+@@ -32,6 +32,7 @@ using namespace vigil::applications;
+ %}
+ 
+ %include "common-defs.i"
++%include "std_list.i"
+ 
+ %import "netinet/netinet.i"
+ %import "pyrt/event.i"
+diff --git a/src/nox/netapps/routing/routing.i b/src/nox/netapps/routing/routing.i
+index 44ccb3d..f9221a2 100644
+--- a/src/nox/netapps/routing/routing.i
++++ b/src/nox/netapps/routing/routing.i
+@@ -17,6 +17,8 @@
+  */
+ %module "nox.netapps.routing.pyrouting"
+ 
++// Hack to get it to compile -BL
++%include "std_list.i"
+ %{
+ #include "pyrouting.hh"
+ #include "routing.hh"
+diff --git a/src/nox/netapps/switch_management/pyswitch_management.i b/src/nox/netapps/switch_management/pyswitch_management.i
+index 72bfed4..ad2c90d 100644
+--- a/src/nox/netapps/switch_management/pyswitch_management.i
++++ b/src/nox/netapps/switch_management/pyswitch_management.i
+@@ -18,6 +18,8 @@
+ 
+ %module "nox.netapps.pyswitch_management"
+ 
++// Hack to get it to compile -BL
++%include "std_list.i"
+ %{
+ #include "switch_management_proxy.hh"
+ #include "pyrt/pycontext.hh"
+diff --git a/src/nox/netapps/tests/tests.cc b/src/nox/netapps/tests/tests.cc
+index 20e900d..f027028 100644
+--- a/src/nox/netapps/tests/tests.cc
++++ b/src/nox/netapps/tests/tests.cc
+@@ -306,7 +306,7 @@ private:
+             throw runtime_error("Could not find PyContext SWIG type_info.");
+         }
+         
+-        PyObject* pyctxt = SWIG_Python_NewPointerObj(p, s, 0);
++        PyObject* pyctxt = SWIG_Python_NewPointerObj(m, p, s, 0);
+         assert(pyctxt);
+ 
+         Py_DECREF(m);
+diff --git a/src/nox/netapps/topology/pytopology.i b/src/nox/netapps/topology/pytopology.i
+index 94a9f4b..7a8cd94 100644
+--- a/src/nox/netapps/topology/pytopology.i
++++ b/src/nox/netapps/topology/pytopology.i
+@@ -18,6 +18,8 @@
+ 
+ %module "nox.netapps.topology"
+ 
++// Hack to get it to compile -BL
++%include "std_list.i"
+ %{
+ #include "pytopology.hh"
+ #include "pyrt/pycontext.hh"
+-- 
+1.7.5.4
+

util/nox-patches/README

@@ -1 +1,2 @@
-This patch adds the OpenFlow tutorial module source code to nox-destiny.
+0001: This patch adds the OpenFlow tutorial module source code to nox-destiny.
+0002: This patch hacks nox-destiny to compile on Ubuntu 12.04.

util/sch_htb-ofbuf/Makefile

@@ -0,0 +1,11 @@
+obj-m = sch_htb.o
+KVERSION = $(shell uname -r)
+all:
+	        make -C /lib/modules/$(KVERSION)/build M=$(PWD) modules
+install:
+		test -e /lib/modules/$(KVERSION)/kernel/net/sched/sch_htb.ko.bak || mv /lib/modules/$(KVERSION)/kernel/net/sched/sch_htb.ko /lib/modules/$(KVERSION)/kernel/net/sched/sch_htb.ko.bak
+		cp sch_htb.ko /lib/modules/$(KVERSION)/kernel/net/sched/sch_htb.ko
+		rmmod sch_htb
+		modprobe sch_htb
+clean:
+	        make -C /lib/modules/$(KVERSION)/build M=$(PWD) clean

util/sch_htb-ofbuf/README

@@ -0,0 +1,10 @@
+Modified sch_htb implementation with ofbuf support.
+
+To compile, just type make.  To use this module instead
+of regular sch_htb, do:
+
+0. make
+1. rmmod sch_htb
+2. insmod ./sch_htb.ko
+
+To revert, just rmmod sch_htb.

util/vm/install-mininet-vm.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This script is intended to install Mininet into
+# a brand-new Ubuntu (10.04 or 11.10) virtual machine,
+# to create a fully usable "tutorial" VM.
+
+set -e
+sudo sh -c 'cat >> /etc/sudoers' <<EOF
+openflow ALL=NOPASSWD: ALL
+EOF
+sudo sed -i -e 's/Default/#Default/' /etc/sudoers
+sudo sed -i -e 's/ubuntu/mininet-vm/' /etc/hostname
+sudo sed -i -e 's/ubuntu/mininet-vm/g' /etc/hosts
+sudo hostname `cat /etc/hostname`
+sudo sed -i -e 's/quiet splash/text/' /etc/default/grub
+sudo update-grub
+sudo sed -i -e 's/us.archive.ubuntu.com/mirrors.kernel.org/' \
+	/etc/apt/sources.list
+sudo apt-get update
+sudo apt-get -y install git-core openssh-server
+git clone git://github.com/mininet/mininet
+cd mininet
+# Check out branch for cs244
+git checkout -b cs244 origin/class/cs244
+cd
+time mininet/util/install.sh
+if ! grep NOX_CORE_DIR .bashrc; then
+  echo "export NOX_CORE_DIR=~/noxcore/build/src/" >> .bashrc
+fi
+echo <<EOF
+You may need to reboot and then:
+sudo dpkg-reconfigure openvswitch-datapath-dkms
+sudo service openvswitch-switch start
+EOF
+
+