update

Author: dongtsi

eval.py

@@ -9,32 +9,19 @@
 sys.path.append('./KitNET')
 from KitNET.model import test_mut
 
-np.set_printoptions(precision=2)
-np.set_printoptions(threshold=10000000)
-np.set_printoptions(linewidth=140)
-np.set_printoptions(suppress=True)
-
-
-def logarithmic_compress(X, e=16.):
-    for i in range(X.shape[0]):
-        for j in range(X.shape[1]):
-            if X[i][j] <= 1. - e:
-                X[i][j] = -math.log(-X[i][j] - 1., e)
-            elif 1. - e < X[i][j] <= e - 1.:
-                X[i][j] = 1. / (e - 1.) * X[i][j]
-            else:
-                X[i][j] = math.log(X[i][j] + 1., e)
-    return X
-
 
 def Euclidean_Distance(v1, v2):
     dis = np.linalg.norm(v1 - v2)
     return dis
 
 
 class Analyzer:
-
-    def __init__(self, org_rmse_file, org_pcap_file, sta_data_file, model_save_path, limit=None):
+    def __init__(self,
+                 org_rmse_file,
+                 org_pcap_file,
+                 sta_data_file,
+                 model_save_path,
+                 limit=None):
 
         self.del_num = 0
 
@@ -53,7 +40,6 @@ def __init__(self, org_rmse_file, org_pcap_file, sta_data_file, model_save_path,
         self.grp_size = self.X_list[0].mal.shape[0]
 
         feature_list = []
-        # print("feature_list",np.array(self.feature_list).shape)
         for i in self.feature_list:
             for j in i:
                 feature_list.append(j)
@@ -66,7 +52,7 @@ def __init__(self, org_rmse_file, org_pcap_file, sta_data_file, model_save_path,
         self.feature_list[:, 82:99:4] = 0.
 
         # compiling mutated features
-        self.rmse_list = test_mut(self.feature_list,model_save_path)
+        self.rmse_list = test_mut(self.feature_list, model_save_path)
         self.rmse_list = np.array(self.rmse_list)
 
         # Analyzing partial data
@@ -79,7 +65,8 @@ def __init__(self, org_rmse_file, org_pcap_file, sta_data_file, model_save_path,
             self.glb_dis_list = self.glb_dis_list[:self.len]
 
         if len(self.rmse_list) < len(self.org_rmse_list) or limit is not None:
-            print("Warning: Statistics are incomplete !", len(self.rmse_list), "<", len(self.org_rmse_list))
+            print("Warning: Statistics are incomplete !", len(self.rmse_list),
+                  "<", len(self.org_rmse_list))
             self.org_pcap = self.org_pcap[:self.len]
             self.org_rmse_list = self.org_rmse_list[:self.len]
 
@@ -88,7 +75,8 @@ def del_outlier(self, extend=2):
         # print(self.feature_list.shape)
         org_rmse_mean = np.mean(self.org_rmse_list)
         for i in range(self.len):
-            if self.rmse_list[i] > self.org_rmse_list[i] * extend or self.rmse_list[i] > org_rmse_mean * extend:
+            if self.rmse_list[i] > self.org_rmse_list[
+                    i] * extend or self.rmse_list[i] > org_rmse_mean * extend:
                 self.rmse_list[i] = 0.
                 del_list.append(i)
                 for j in range(self.feature_list.shape[1]):
@@ -113,7 +101,12 @@ def save_mutated_traffic(self, mut_pcap_path):
         print("Total #pkts in mutated traffic:", len(pkt_List))
         wrpcap(mut_pcap_path, pkt_List)
 
-    def eval(self, AD_threshold, mimic_set_file, test_feat_file, knormer_file, need_mmr=False):
+    def eval(self,
+             AD_threshold,
+             mimic_set_file,
+             test_feat_file,
+             knormer_file,
+             need_mmr=False):
 
         print("1.Time elapse:")
         b = self.org_pcap[-1].time - self.org_pcap[0].time
@@ -142,7 +135,7 @@ def eval(self, AD_threshold, mimic_set_file, test_feat_file, knormer_file, need_
         a = np.mean(self.rmse_list)
         print("  original:", b, "mutated:", a)
         print("  PDR:", (b - a) / b)
-        print("-"*64)
+        print("-" * 64)
 
         print("# Detected:")
         b = self.org_rmse_list[self.org_rmse_list > AD_threshold].shape[0]
@@ -166,14 +159,16 @@ def eval(self, AD_threshold, mimic_set_file, test_feat_file, knormer_file, need_
             mut_dis = 0.
 
             for i in range(mal_feat.shape[0]):
-                org_dis += max(np.linalg.norm(mal_feat[i] - mimic_feat, axis=1))
-                mut_dis += min(np.linalg.norm(mut_feat[i] - mimic_feat, axis=1))
+                org_dis += max(np.linalg.norm(mal_feat[i] - mimic_feat,
+                                              axis=1))
+                mut_dis += min(np.linalg.norm(mut_feat[i] - mimic_feat,
+                                              axis=1))
             MMR = 1. - mut_dis / org_dis
             print("Feature Changed:")
             print("  Before:", org_dis, "After:", mut_dis)
             print("  MMR:", MMR)
 
-    def plt_rmse(self,AD_threshold):  # 对比原来的rmse和现在的rmse
+    def plt_rmse(self, AD_threshold):
 
         x = np.arange(0, self.len, 1)
         plt.figure()
@@ -182,11 +177,30 @@ def plt_rmse(self,AD_threshold):  # 对比原来的rmse和现在的rmse
         # plt.plot(x,[np.mean(np.log(self.org_rmse_list))]*self.len,c='#8A977B',alpha=0.5)
         # plt.plot(x,[np.mean(np.log(self.rmse_list))]*self.len, c='#FE4365',alpha=0.5)
 
-        plt.scatter(x, self.org_rmse_list, s=12, c='#8A977B', alpha=0.5, label="Before")
-        plt.scatter(x, self.rmse_list, s=12, c='#FE4365', alpha=0.5, label="After")
-        plt.plot(x, [np.mean(self.org_rmse_list)] * self.len, c='#8A977B', alpha=0.3, linewidth=4)
-        plt.plot(x, [np.mean(self.rmse_list)] * self.len, c='#FE4365', alpha=0.3, linewidth=4)
-        plt.plot(x, [AD_threshold] * self.len, c='black', linewidth=2,label="AD_threshold")
+        plt.scatter(x,
+                    self.org_rmse_list,
+                    s=12,
+                    c='#8A977B',
+                    alpha=0.5,
+                    label="Before")
+        plt.scatter(x,
+                    self.rmse_list,
+                    s=12,
+                    c='#FE4365',
+                    alpha=0.5,
+                    label="After")
+        plt.plot(x, [np.mean(self.org_rmse_list)] * self.len,
+                 c='#8A977B',
+                 alpha=0.3,
+                 linewidth=4)
+        plt.plot(x, [np.mean(self.rmse_list)] * self.len,
+                 c='#FE4365',
+                 alpha=0.3,
+                 linewidth=4)
+        plt.plot(x, [AD_threshold] * self.len,
+                 c='black',
+                 linewidth=2,
+                 label="AD_threshold")
         plt.title("RMSE change and mean")
         plt.xlabel('pkt no.')
         plt.ylabel('RMSE in Kitsune')
@@ -196,28 +210,49 @@ def plt_rmse(self,AD_threshold):  # 对比原来的rmse和现在的rmse
         plt.show()
 
 
-
-
 if __name__ == "__main__":
 
     parse = argparse.ArgumentParser()
-    parse.add_argument('-op', '--org_pcap_file', type=str, required=True,
+    parse.add_argument('-op',
+                       '--org_pcap_file',
+                       type=str,
+                       required=True,
                        help="original malicious (test) traffic (.pcap)")
 
-    parse.add_argument('-or', '--org_rmse_file', type=str, required=True,
-                       help="original RMSE file of test malicious traffic (.pkl)")
+    parse.add_argument(
+        '-or',
+        '--org_rmse_file',
+        type=str,
+        required=True,
+        help="original RMSE file of test malicious traffic (.pkl)")
 
-    parse.add_argument('-of', '--org_feat_file', type=str,
+    parse.add_argument('-of',
+                       '--org_feat_file',
+                       type=str,
                        help="original (test) feature (.npy)")
 
-    parse.add_argument('-b', '--mimic_set', type=str, required=True, help="benign features to mimic (.npy)")
-
-    parse.add_argument('-n', '--normalizer', type=str, required=True, help="compiled feature normalizer (.pkl)")
-
-    parse.add_argument('-sf', '--sta_file', type=str, default='./example/statistics.pkl',
+    parse.add_argument('-b',
+                       '--mimic_set',
+                       type=str,
+                       required=True,
+                       help="benign features to mimic (.npy)")
+
+    parse.add_argument('-n',
+                       '--normalizer',
+                       type=str,
+                       required=True,
+                       help="compiled feature normalizer (.pkl)")
+
+    parse.add_argument('-sf',
+                       '--sta_file',
+                       type=str,
+                       default='./example/statistics.pkl',
                        help="statistics to read(.pkl)")
 
-    parse.add_argument('-mf', '--model_file_path', type=str, default='./example/model.pkl',
+    parse.add_argument('-mf',
+                       '--model_file_path',
+                       type=str,
+                       default='./example/model.pkl',
                        help="model_file after training")
 
     arg = parse.parse_args()
@@ -237,15 +272,9 @@ def plt_rmse(self,AD_threshold):  # 对比原来的rmse和现在的rmse
         AD_threshold = pkl.load(f)
     print("AD_threshold:", AD_threshold)
 
-    a.eval(AD_threshold,arg.mimic_set,arg.org_feat_file,arg.normalizer,need_mmr=True)
+    a.eval(AD_threshold,
+           arg.mimic_set,
+           arg.org_feat_file,
+           arg.normalizer,
+           need_mmr=True)
     a.plt_rmse(AD_threshold)
-
-
-
-
-
-
-
-
-
-

initializer.py

@@ -20,43 +20,45 @@ def decide_has_pkt(crafted_pkt_prob):
 
 
 def initialize(
-                grp_size,                   # Number of pkts in each group
-                last_end_time,
-                groupList,                  # Pcap info in current group
-                max_time_extend,            # maximum time overhead (l_t)
-                max_cft_pkt,                # maximum crafted traffic overhead (l_c)
-                min_time_extend,
-                max_crafted_pkt_prob,
-               ):
+    grp_size,  # Number of pkts in each group
+    last_end_time,
+    groupList,  # Pcap info in current group
+    max_time_extend,  # maximum time overhead (l_t)
+    max_cft_pkt,  # maximum crafted traffic overhead (l_c)
+    min_time_extend,
+    max_crafted_pkt_prob,
+):
 
-    X = Unit(grp_size,max_cft_pkt)  # position vector
+    X = Unit(grp_size, max_cft_pkt)  # position vector
 
-    ics_time = 0                # accumulated increased ITA
+    ics_time = 0  # accumulated increased ITA
 
     for i in range(grp_size):
         if i == 0:
             itv = groupList[i].time - last_end_time
         else:
-            itv = groupList[i].time - groupList[i-1].time
+            itv = groupList[i].time - groupList[i - 1].time
         # ics_time += random.uniform(0,max_time_extend)*itv
-        ics_time += random.uniform(min_time_extend,max_time_extend)*itv
+        ics_time += random.uniform(min_time_extend, max_time_extend) * itv
         X.mal[i][0] = groupList[i].time + ics_time
 
     max_mal_itv = (groupList[-1].time - last_end_time) * (max_time_extend + 1)
 
     # building slot map
     slot_num = grp_size * max_cft_pkt
-    slot_itv = max_mal_itv/slot_num
+    slot_itv = max_mal_itv / slot_num
 
     # initializing crafted pkts
     crafted_pkt_prob = random.uniform(0, max_crafted_pkt_prob)
     nxt_mal_no = 0
 
     proto_max_lmt = []  # maximum protocol layer number
     for i in range(grp_size):
-        if groupList[i].haslayer(TCP) or groupList[i].haslayer(UDP) or groupList[i].haslayer(ICMP):
+        if groupList[i].haslayer(TCP) or groupList[i].haslayer(
+                UDP) or groupList[i].haslayer(ICMP):
             proto_max_lmt.append(3.)
-        elif groupList[i].haslayer(IP) or groupList[i].haslayer(IPv6) or groupList[i].haslayer(ARP):
+        elif groupList[i].haslayer(IP) or groupList[i].haslayer(
+                IPv6) or groupList[i].haslayer(ARP):
             proto_max_lmt.append(2.)
         elif groupList[i].haslayer(Ether):
             proto_max_lmt.append(1.)
@@ -69,12 +71,13 @@ def initialize(
             nxt_mal_no += 1
             if nxt_mal_no == grp_size:
                 break
-        if (not decide_has_pkt(crafted_pkt_prob)) or X.mal[nxt_mal_no][1] == max_cft_pkt:
+        if (not decide_has_pkt(crafted_pkt_prob)
+            ) or X.mal[nxt_mal_no][1] == max_cft_pkt:
             continue
         cft_no = int(round(X.mal[nxt_mal_no][1]))
 
         if proto_max_lmt[nxt_mal_no] == 3.:
-            X.craft[nxt_mal_no][cft_no][1] = random.choice([1.,2.,3.])
+            X.craft[nxt_mal_no][cft_no][1] = random.choice([1., 2., 3.])
             mtu = 1460
         elif proto_max_lmt[nxt_mal_no] == 2.:
             X.craft[nxt_mal_no][cft_no][1] = random.choice([1., 2.])
@@ -90,5 +93,4 @@ def initialize(
 
         X.mal[nxt_mal_no][1] += 1.
 
-    return X,proto_max_lmt
-
+    return X, proto_max_lmt

main.py

@@ -5,38 +5,55 @@
 from manipulator import Manipulator
 
 parse = argparse.ArgumentParser()
-parse.add_argument('-m', '--mal_pcap', type=str, required=True, help="input malicious traffic (.pcap)")
-
-parse.add_argument('-b', '--mimic_set', type=str, required=True, help="benign features to mimic (.npy)")
-
-parse.add_argument('-n', '--normalizer', type=str, required=True, help="compiled feature normalizer (.pkl)")
-
-parse.add_argument('-i', '--init_pcap', type=str, default='./_empty.pcap',
+parse.add_argument('-m',
+                   '--mal_pcap',
+                   type=str,
+                   required=True,
+                   help="input malicious traffic (.pcap)")
+
+parse.add_argument('-b',
+                   '--mimic_set',
+                   type=str,
+                   required=True,
+                   help="benign features to mimic (.npy)")
+
+parse.add_argument('-n',
+                   '--normalizer',
+                   type=str,
+                   required=True,
+                   help="compiled feature normalizer (.pkl)")
+
+parse.add_argument('-i',
+                   '--init_pcap',
+                   type=str,
+                   default='./_empty.pcap',
                    help="preparatory traffic (ignore this if you don't need)")
 
-parse.add_argument('-o', '--sta_file', type=str, default='./example/statistics.pkl',
+parse.add_argument('-o',
+                   '--sta_file',
+                   type=str,
+                   default='./example/statistics.pkl',
                    help="file saving the final statistics (.pkl)")
 
 arg = parse.parse_args()
 
-# 创建Manipulator
 m = Manipulator(arg.mal_pcap, arg.mimic_set, arg.normalizer, arg.init_pcap)
 
-# 选择配置参数
 max_iter, particle_num, local_grp_size = 3, 6, 3
 # max_iter,particle_num,local_grp_size = 4,8,4
 # max_iter,particle_num,local_grp_size = 5,10,5
 # max_iter,particle_num,local_grp_size = 3,10,5
 
 m.change_particle_params(w=0.7298, c1=1.49618, c2=1.49618)
-m.change_pso_params(max_iter=max_iter, particle_num=particle_num, grp_size=local_grp_size)
+m.change_pso_params(max_iter=max_iter,
+                    particle_num=particle_num,
+                    grp_size=local_grp_size)
 m.change_manipulator_params(grp_size=100,
                             min_time_extend=3.,
                             max_time_extend=6.,
                             max_cft_pkt=1,
                             max_crafted_pkt_prob=0.01)
 
-# 保存配置参数
 # m.save_configurations('./configurations.txt')
 
 # tmp_pcap_file = "_crafted.pcap"