selftests/tc-testing: implement tdc parallel test run

Use a Python process pool to run the tests in parallel.
Not all tests can run in parallel, for instance tests that are not
namespaced and tests that use netdevsim, as they can conflict with one
another.

The code logic will split the tests into serial and parallel.
For the parallel tests, we build batches of 32 tests and queue each
batch on the process pool. For the serial tests, they are queued as a
whole into the process pool, which in turn executes them concurrently
with the parallel tests.

Even though the tests serialize on rtnl_lock in the kernel, this feature
showed results with a ~3x speedup on the wall time for the entire test suite
running in a VM:
   Before - 4m32.502s
   After - 1m19.202s

Examples:
   In order to run tdc using 4 processes:
      ./tdc.py -J4 <...>
   In order to run tdc using 1 process:
      ./tdc.py -J1 <...> || ./tdc.py <...>

Note that the kernel configuration will affect the speed of the tests,
especially if such configuration slows down process creation and/or
fork().

Tested-by: Davide Caratti <dcaratti@redhat.com>
Signed-off-by: Pedro Tammela <pctammela@mojatatu.com>
Acked-by: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

tools/testing/selftests/tc-testing/plugin-lib/nsPlugin.py

@@ -3,46 +3,65 @@ import signal
 from string import Template
 import subprocess
 import time
+from multiprocessing import Pool
 from functools import cached_property
 from TdcPlugin import TdcPlugin
 
 from tdc_config import *
 
+def prepare_suite(obj, test):
+    original = obj.args.NAMES
+
+    if 'skip' in test and test['skip'] == 'yes':
+        return
+
+    if 'nsPlugin' not in test['plugins']:
+        return
+
+    shadow = {}
+    shadow['IP'] = original['IP']
+    shadow['TC'] = original['TC']
+    shadow['NS'] = '{}-{}'.format(original['NS'], test['random'])
+    shadow['DEV0'] = '{}id{}'.format(original['DEV0'], test['id'])
+    shadow['DEV1'] = '{}id{}'.format(original['DEV1'], test['id'])
+    shadow['DUMMY'] = '{}id{}'.format(original['DUMMY'], test['id'])
+    shadow['DEV2'] = original['DEV2']
+    obj.args.NAMES = shadow
+
+    if obj.args.namespace:
+        obj._ns_create()
+    else:
+        obj._ports_create()
+
+    # Make sure the netns is visible in the fs
+    while True:
+        obj._proc_check()
+        try:
+            ns = obj.args.NAMES['NS']
+            f = open('/run/netns/{}'.format(ns))
+            f.close()
+            break
+        except:
+            time.sleep(0.1)
+            continue
+
+    obj.args.NAMES = original
+
 class SubPlugin(TdcPlugin):
     def __init__(self):
         self.sub_class = 'ns/SubPlugin'
         super().__init__()
 
     def pre_suite(self, testcount, testlist):
+        from itertools import cycle
+
         super().pre_suite(testcount, testlist)
 
         print("Setting up namespaces and devices...")
 
-        original = self.args.NAMES
-
-        for t in testlist:
-            if 'skip' in t and t['skip'] == 'yes':
-                continue
-
-            if 'nsPlugin' not in t['plugins']:
-                continue
-
-            shadow = {}
-            shadow['IP'] = original['IP']
-            shadow['TC'] = original['TC']
-            shadow['NS'] = '{}-{}'.format(original['NS'], t['random'])
-            shadow['DEV0'] = '{}id{}'.format(original['DEV0'], t['id'])
-            shadow['DEV1'] = '{}id{}'.format(original['DEV1'], t['id'])
-            shadow['DUMMY'] = '{}id{}'.format(original['DUMMY'], t['id'])
-            shadow['DEV2'] = original['DEV2']
-            self.args.NAMES = shadow
-
-            if self.args.namespace:
-                self._ns_create()
-            else:
-                self._ports_create()
-
-        self.args.NAMES = original
+        with Pool(self.args.mp) as p:
+            it = zip(cycle([self]), testlist)
+            p.starmap(prepare_suite, it)
 
     def pre_case(self, caseinfo, test_skip):
         if self.args.verbose:
@@ -51,16 +70,6 @@ class SubPlugin(TdcPlugin):
         if test_skip:
             return
 
-        # Make sure the netns is visible in the fs
-        while True:
-            self._proc_check()
-            try:
-                ns = self.args.NAMES['NS']
-                f = open('/run/netns/{}'.format(ns))
-                f.close()
-                break
-            except:
-                continue
 
     def post_case(self):
         if self.args.verbose: