traffic: new version of model with packet_rate, version number

Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

8 files changed, 202 insertions, 94 deletions

diff --git a/python/samba/emulate/traffic.py b/python/samba/emulate/traffic.py
index b6097cdc120..807fa8244e2 100644
--- a/python/samba/emulate/traffic.py
+++ b/python/samba/emulate/traffic.py
@@ -54,6 +54,8 @@ from samba.compat import get_string
 from samba.logger import get_samba_logger
 import bisect
 
+CURRENT_MODEL_VERSION = 2   # save as this
+REQUIRED_MODEL_VERSION = 2  # load accepts this or greater
 SLEEP_OVERHEAD = 3e-4
 
 # we don't use None, because it complicates [de]serialisation
@@ -1136,7 +1138,7 @@ class TrafficModel(object):
         self.n = n
         self.dns_opcounts = defaultdict(int)
         self.cumulative_duration = 0.0
-        self.conversation_rate = [0, 1]
+        self.packet_rate = [0, 1]
 
     def learn(self, conversations, dns_opcounts={}):
         prev = 0.0
@@ -1149,10 +1151,15 @@ class TrafficModel(object):
             self.dns_opcounts[k] += v
 
         if len(conversations) > 1:
-            elapsed =\
-                conversations[-1].start_time - conversations[0].start_time
-            self.conversation_rate[0] = len(conversations)
-            self.conversation_rate[1] = elapsed
+            first = conversations[0].start_time
+            total = 0
+            last = first + 0.1
+            for c in conversations:
+                total += len(c)
+                last = max(last, c.packets[-1].timestamp)
+
+            self.packet_rate[0] = total
+            self.packet_rate[1] = last - first
 
         for c in conversations:
             client, server = c.guess_client_server(server)
@@ -1196,7 +1203,8 @@ class TrafficModel(object):
             'ngrams': ngrams,
             'query_details': query_details,
             'cumulative_duration': self.cumulative_duration,
-            'conversation_rate': self.conversation_rate,
+            'packet_rate': self.packet_rate,
+            'version': CURRENT_MODEL_VERSION
         }
         d['dns'] = self.dns_opcounts
 
@@ -1211,6 +1219,17 @@ class TrafficModel(object):
 
         d = json.load(f)
 
+        try:
+            version = d["version"]
+            if version < REQUIRED_MODEL_VERSION:
+                raise ValueError("the model file is version %d; "
+                                 "version %d is required" %
+                                 (version, REQUIRED_MODEL_VERSION))
+        except KeyError:
+                raise ValueError("the model file lacks a version number; "
+                                 "version %d is required" %
+                                 (REQUIRED_MODEL_VERSION))
+
         for k, v in d['ngrams'].items():
             k = tuple(str(k).split('\t'))
             values = self.ngrams.setdefault(k, [])
@@ -1232,18 +1251,22 @@ class TrafficModel(object):
                 self.dns_opcounts[k] += v
 
         self.cumulative_duration = d['cumulative_duration']
-        self.conversation_rate = d['conversation_rate']
-
-    def construct_conversation(self, timestamp=0.0, client=2, server=1,
-                               hard_stop=None, replay_speed=1):
-        """Construct a individual converation from the model."""
-
-        c = Conversation(timestamp, (server, client), conversation_id=client)
-
+        self.packet_rate = d['packet_rate']
+
+    def construct_conversation_sequence(self, timestamp=0.0,
+                                        hard_stop=None,
+                                        replay_speed=1,
+                                        ignore_before=0):
+        """Construct an individual conversation packet sequence from the
+        model.
+        """
+        c = []
         key = (NON_PACKET,) * (self.n - 1)
+        if ignore_before is None:
+            ignore_before = timestamp - 1
 
-        while key in self.ngrams:
-            p = random.choice(self.ngrams.get(key, NON_PACKET))
+        while True:
+            p = random.choice(self.ngrams.get(key, (NON_PACKET,)))
             if p == NON_PACKET:
                 break
 
@@ -1263,47 +1286,50 @@ class TrafficModel(object):
                 timestamp += wait
                 if hard_stop is not None and timestamp > hard_stop:
                     break
-                c.add_short_packet(timestamp, protocol, opcode, extra)
+                if timestamp >= ignore_before:
+                    c.append((timestamp, protocol, opcode, extra))
 
             key = key[1:] + (p,)
 
         return c
 
-    def generate_conversations(self, rate, duration, replay_speed=1):
+    def generate_conversations(self, scale, duration, replay_speed=1,
+                               server=1, client=2):
         """Generate a list of conversations from the model."""
 
-        # We run the simulation for at least ten times as long as our
-        # desired duration, and take a section near the start.
-        rate_n, rate_t  = self.conversation_rate
+        # We run the simulation for ten times as long as our desired
+        # duration, and take the section at the end.
+        lead_in = 9 * duration
+        rate_n, rate_t  = self.packet_rate
+        target_packets = int(duration * scale * rate_n / rate_t)
 
-        duration2 = max(rate_t, duration * 2)
-        n = rate * duration2 * rate_n / rate_t
+        conversations = []
+        n_packets = 0
+
+        while n_packets < target_packets:
+            start = random.uniform(-lead_in, duration)
+            c = self.construct_conversation_sequence(start,
+                                                     hard_stop=duration,
+                                                     replay_speed=replay_speed,
+                                                     ignore_before=0)
+            conversations.append(c)
+            n_packets += len(c)
+
+        print(("we have %d packets (target %d) in %d conversations at scale %f"
+               % (n_packets, target_packets, len(conversations), scale)),
+              file=sys.stderr)
+        conversations.sort()  # sorts by first element == start time
+        return seq_to_conversations(conversations)
 
-        server = 1
-        client = 2
 
-        conversations = []
-        end = duration2
-        start = end - duration
-
-        while client < n + 2:
-            start = random.uniform(0, duration2)
-            c = self.construct_conversation(start,
-                                            client,
-                                            server,
-                                            hard_stop=(duration2 * 5),
-                                            replay_speed=replay_speed)
-
-            c.forget_packets_outside_window(start, end)
-            c.renormalise_times(start)
-            if len(c) != 0:
-                conversations.append(c)
+def seq_to_conversations(seq, server=1, client=2):
+    conversations = []
+    for s in seq:
+        if s:
+            c = Conversation(s[0][0], (server, client), s)
             client += 1
-
-        print(("we have %d conversations at rate %f" %
-               (len(conversations), rate)), file=sys.stderr)
-        conversations.sort()
-        return conversations
+            conversations.append(c)
+    return conversations
 
 
 IP_PROTOCOLS = {
diff --git a/python/samba/tests/blackbox/testdata/traffic-sample-very-short.model b/python/samba/tests/blackbox/testdata/traffic-sample-very-short.model
index 8e299eb19e1..0de93ed1c8d 100644
--- a/python/samba/tests/blackbox/testdata/traffic-sample-very-short.model
+++ b/python/samba/tests/blackbox/testdata/traffic-sample-very-short.model
@@ -30,14 +30,14 @@
       "-": 1
     }
   }, 
-  "conversation_rate": [
-    2, 
-    0.12712717056274414
-  ], 
   "dns": {
     "1": 9, 
     "0": 9
   }, 
+  "packet_rate": [
+    50,
+    0.32482
+  ], 
   "query_details": {
     "rpc_netlogon:29": {
       "-": 1
@@ -56,5 +56,6 @@
       "": 1
     }
   }, 
-  "cumulative_duration": 0.39243292808532715
+  "cumulative_duration": 0.39243292808532715,
+  "version": 2
 }
\ No newline at end of file
diff --git a/python/samba/tests/blackbox/testdata/traffic_learner.expected b/python/samba/tests/blackbox/testdata/traffic_learner.expected
index 8e299eb19e1..3ae80899caf 100644
--- a/python/samba/tests/blackbox/testdata/traffic_learner.expected
+++ b/python/samba/tests/blackbox/testdata/traffic_learner.expected
@@ -1,4 +1,26 @@
 {
+  "packet_rate": [
+    10, 
+    0.22707200050354004
+  ], 
+  "query_details": {
+    "rpc_netlogon:29": {
+      "-": 1
+    }, 
+    "cldap:3": {
+      "\t\t\tNetlogon\t\t\t": 3
+    }, 
+    "ldap:3": {
+      "\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1, 
+      "2\tDC,DC\t\tcn\t\t\t": 1
+    }, 
+    "ldap:2": {
+      "\t\t\t\t\t\t": 1
+    }, 
+    "kerberos:": {
+      "": 1
+    }
+  }, 
   "ngrams": {
     "-\t-": {
       "cldap:3": 1, 
@@ -30,31 +52,10 @@
       "-": 1
     }
   }, 
-  "conversation_rate": [
-    2, 
-    0.12712717056274414
-  ], 
+  "version": 2, 
   "dns": {
     "1": 9, 
     "0": 9
   }, 
-  "query_details": {
-    "rpc_netlogon:29": {
-      "-": 1
-    }, 
-    "cldap:3": {
-      "\t\t\tNetlogon\t\t\t": 3
-    }, 
-    "ldap:3": {
-      "\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1, 
-      "2\tDC,DC\t\tcn\t\t\t": 1
-    }, 
-    "ldap:2": {
-      "\t\t\t\t\t\t": 1
-    }, 
-    "kerberos:": {
-      "": 1
-    }
-  }, 
   "cumulative_duration": 0.39243292808532715
 }
\ No newline at end of file
diff --git a/python/samba/tests/blackbox/testdata/traffic_replay-0.expected b/python/samba/tests/blackbox/testdata/traffic_replay-0.expected
new file mode 100644
index 00000000000..8f44438ef97
--- /dev/null
+++ b/python/samba/tests/blackbox/testdata/traffic_replay-0.expected
@@ -0,0 +1,18 @@
+0.011388	06		2	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.221447	06		2	1	ldap	2	unbindRequest							
+0.460878	06		3	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.581933	11		4	1	cldap	3	searchRequest				Netlogon			
+0.596977	11		4	1	cldap	3	searchRequest				Netlogon			
+0.611184	11		4	1	cldap	3	searchRequest				Netlogon			
+0.666808	06		3	1	ldap	2	unbindRequest							
+0.744297	06		4	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.768994	06		4	1	kerberos			
+0.772476	06		4	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.805442	11		5	1	cldap	3	searchRequest				Netlogon			
+0.805536	11		5	1	cldap	3	searchRequest				Netlogon			
+0.807659	11		5	1	cldap	3	searchRequest				Netlogon			
+0.808614	11		5	1	cldap	3	searchRequest				Netlogon			
+0.808819	11		5	1	cldap	3	searchRequest				Netlogon			
+0.865384	06		6	1	ldap	3	searchRequest				subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities			
+0.973595	06		5	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.974012	06		5	1	kerberos			
diff --git a/python/samba/tests/blackbox/testdata/traffic_replay-1.expected b/python/samba/tests/blackbox/testdata/traffic_replay-1.expected
new file mode 100644
index 00000000000..1ac696819bd
--- /dev/null
+++ b/python/samba/tests/blackbox/testdata/traffic_replay-1.expected
@@ -0,0 +1,19 @@
+0.011519	11		2	1	cldap	3	searchRequest				Netlogon			
+0.012916	11		2	1	cldap	3	searchRequest				Netlogon			
+0.158388	06		3	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.164506	06		2	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.166151	06		2	1	kerberos			
+0.166301	06		2	1	ldap	3	searchRequest				subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities			
+0.258932	06		4	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.259908	06		4	1	kerberos			
+0.260073	06		4	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.286044	06		5	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.295757	06		3	1	ldap	2	unbindRequest							
+0.459791	06		5	1	ldap	2	unbindRequest							
+0.553887	06		6	1	ldap	3	searchRequest				subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities			
+0.641127	11		7	1	cldap	3	searchRequest				Netlogon			
+0.641297	11		7	1	cldap	3	searchRequest				Netlogon			
+0.783989	06		6	1	ldap	2	unbindRequest							
+0.901096	06		7	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.915260	06		7	1	kerberos			
+0.915711	06		7	1	ldap	3	searchRequest	2	DC,DC		cn			
diff --git a/python/samba/tests/blackbox/testdata/traffic_replay.expected b/python/samba/tests/blackbox/testdata/traffic_replay.expected
index b29c8b04121..0c6b2a2fe52 100644
--- a/python/samba/tests/blackbox/testdata/traffic_replay.expected
+++ b/python/samba/tests/blackbox/testdata/traffic_replay.expected
@@ -1,10 +1,18 @@
-0.000000	06	3	5	1	ldap	3	searchRequest	2	DC,DC		cn			
-0.127127	11		6	1	cldap	3	searchRequest				Netlogon			
-0.127391	11		6	1	cldap	3	searchRequest				Netlogon			
-0.127689	11		6	1	cldap	3	searchRequest				Netlogon			
-0.227072	06	3	5	1	ldap	2	unbindRequest							
-0.282995	06	16	6	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
-0.287553	06	18	6	1	kerberos			
-0.287759	06	16	1	6	rpc_netlogon	29	NetrLogonGetDomainInfo	
-0.291161	06	19	6	1	ldap	3	searchRequest				subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities			
-0.292488	06	18	1	6	kerberos			
+0.040433	06		2	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.059203	06		2	1	kerberos			
+0.061641	06		2	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.535074	11		3	1	cldap	3	searchRequest				Netlogon			
+0.535369	11		3	1	cldap	3	searchRequest				Netlogon			
+0.536671	11		3	1	cldap	3	searchRequest				Netlogon			
+0.537238	11		3	1	cldap	3	searchRequest				Netlogon			
+0.537362	11		3	1	cldap	3	searchRequest				Netlogon			
+0.602824	11		4	1	cldap	3	searchRequest				Netlogon			
+0.640115	11		4	1	cldap	3	searchRequest				Netlogon			
+0.714546	06		3	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.715865	06		3	1	kerberos			
+0.716613	06		3	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.767674	06		4	1	rpc_netlogon	29	NetrLogonGetDomainInfo	
+0.778022	06		5	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.792356	06		4	1	kerberos			
+0.792763	06		4	1	ldap	3	searchRequest	2	DC,DC		cn			
+0.960412	06		5	1	ldap	2	unbindRequest							
diff --git a/python/samba/tests/blackbox/traffic_learner.py b/python/samba/tests/blackbox/traffic_learner.py
index dd7c7c15678..e6adce7a568 100644
--- a/python/samba/tests/blackbox/traffic_learner.py
+++ b/python/samba/tests/blackbox/traffic_learner.py
@@ -75,4 +75,4 @@ class TrafficLearnerTests(BlackboxTestCase):
             expected_details = {k: sorted(v) for k, v in expected.query_details.items()}
             self.assertEquals(expected_details, actual_details)
             self.assertEquals(expected.cumulative_duration, actual.cumulative_duration)
-            self.assertEquals(expected.conversation_rate, actual.conversation_rate)
+            self.assertEquals(expected.packet_rate, actual.packet_rate)
diff --git a/python/samba/tests/blackbox/traffic_replay.py b/python/samba/tests/blackbox/traffic_replay.py
index 015db2ed39b..a84d1a423e4 100644
--- a/python/samba/tests/blackbox/traffic_replay.py
+++ b/python/samba/tests/blackbox/traffic_replay.py
@@ -25,12 +25,13 @@ from samba.tests import BlackboxTestCase
 
 DATA_DIR      = "python/samba/tests/blackbox/testdata"
 SCRIPT        = "script/traffic_replay"
-FIXED         = "--fixed-password trafficreplay01%"
+FIXED         = "--fixed-password=trafficreplay01%"
 SERVER        = os.environ["SERVER"]
 PASSWORD      = os.environ["PASSWORD"]
 USER          = os.environ["USERNAME"]
-STD_OPTIONS   = "-U%s%%%s %s" % (USER, PASSWORD, SERVER)
-EXPECTED_NAME = os.path.join(DATA_DIR, "traffic_replay.expected")
+CREDS         = "-U%s%%%s" % (USER, PASSWORD)
+MODEL         = os.path.join(DATA_DIR, "traffic-sample-very-short.model")
+EXPECTED_OUTPUT = os.path.join(DATA_DIR, "traffic_replay-%s.expected")
 
 
 @contextmanager
@@ -49,7 +50,7 @@ class TrafficLearnerTests(BlackboxTestCase):
 
     def tearDown(self):
         options = "--clean-up"
-        command  = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
+        command = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
         self.check_run(command)
 
     def test_generate_users_only(self):
@@ -57,16 +58,48 @@ class TrafficLearnerTests(BlackboxTestCase):
            """
         options = ("--generate-users-only --number-of-users 20 "
                    "--number-of-groups 5 --average-groups-per-user 2")
-        command  = "%s %s %s %s" % (
-            SCRIPT, options, FIXED, STD_OPTIONS)
+        command = "%s %s %s %s %s" % (
+            SCRIPT, options, FIXED, CREDS, SERVER)
         self.check_run(command)
+        command = "%s %s %s %s %s %s" % (
+            SCRIPT, MODEL, options, FIXED, CREDS, SERVER)
+        self.check_run(command)
+
+    def test_summary_generation(self):
+        """Ensure a summary file is generated and the contents are correct"""
+
+        for i, opts in enumerate((["--random-seed=3"],
+                                  ["--random-seed=4"],
+                                  ["--random-seed=3",
+                                   "--conversation-persistence=0.5"],
+                                  ["--random-seed=3",
+                                   "--old-scale",
+                                   "--conversation-persistence=0.95"],
+                                  )):
+            with temp_file(self.tempdir) as output:
+                command = ([SCRIPT, MODEL,
+                            "--traffic-summary", output,
+                            "-D1", "-S0.1"] +
+                           opts +
+                           [FIXED, CREDS, SERVER])
+                self.check_run(command)
+                expected = open(EXPECTED_OUTPUT % i).read()
+                actual = open(output).read()
+                self.assertStringsEqual(expected, actual)
+
+    def test_summary_replay_no_fixed(self):
+        """Ensure a summary file with no fixed password fails
+           """
+        command = [SCRIPT, MODEL, CREDS, SERVER]
+        self.check_exit_code(command, 1)
 
     def test_model_replay(self):
         """Ensure a model can be replayed against a DC
            """
-
-        model   = "testdata/traffic-sample-very-short.model"
-        command = "%s %s-D 5 %s %s" % (SCRIPT, model, FIXED, STD_OPTIONS)
+        command = [SCRIPT, MODEL,
+                   FIXED,
+                   '-D2', '-S0.1',
+                   CREDS, SERVER]
         self.check_run(command)
 
     def test_generate_users_only_no_password(self):
@@ -74,5 +107,7 @@ class TrafficLearnerTests(BlackboxTestCase):
            """
         options = ("--generate-users-only --number-of-users 20 "
                    "--number-of-groups 5 --average-groups-per-user 2")
-        command  = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
+        command  = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
+        self.check_exit_code(command, 1)
+        command = "%s %s %s %s %s" % (SCRIPT, MODEL, options, CREDS, SERVER)
         self.check_exit_code(command, 1)