faster pooling

optimization_algo/approaches/cma_approach_square_size.py

@@ -20,7 +20,8 @@ class cma_approach(object):
         res_step = 1,
         penalty_offest = 10000000000,
         seeding_type="optimised",
-        hybird = True
+        hybird = True,
+        print_to_csv = True
         ):
         self.target_col = target_col
         self.start = time.time()
@@ -47,6 +48,7 @@ class cma_approach(object):
         self.seeding_type = seeding_type
 
         self.max_res_available = max_res_unit
+        self.print_to_csv = print_to_csv
 
     def parse_topology_file(self):
         layers = []
@@ -219,21 +221,21 @@ class cma_approach(object):
 
             res_combintaions = [0] * self.es.popsize
 
-            pool = Pool(processes = cpu_count() - 4)
-            for result in pool.imap_unordered(self.evaluation_top_level, id_list):
-                scores[result[0]] = result[1]
-                if result[1] >= self.penalty_offest:
-                    invalid_sampling += 1
-                else:
-                    if not self.is_hybird:
-                        res_combintaions[result[0]] = result[2]
-            pool.close()
-            pool.join()
-
-            # for tup in samples:
-            #     _, scores[tup[0]] = evaluation_top_level(tup)
-            #     if scores[tup[0]] >= self.penalty_offest:
+            # pool = Pool(processes = cpu_count() - 4)
+            # for result in pool.imap_unordered(self.evaluation_top_level, id_list):
+            #     scores[result[0]] = result[1]
+            #     if result[1] >= self.penalty_offest:
             #         invalid_sampling += 1
+            #     else:
+            #         if not self.is_hybird:
+            #             res_combintaions[result[0]] = result[2]
+            # pool.close()
+            # pool.join()
+
+            for tup in id_list:
+                _, scores[tup[0]] = self.evaluation_top_level(tup)
+                if scores[tup[0]] >= self.penalty_offest:
+                    invalid_sampling += 1
 
             if not self.is_hybird:
                 best_in_iteration = min(scores)
@@ -273,13 +275,15 @@ class cma_approach(object):
                 ##print("RESULT NOT VALID")
                 ##print("Layer:", self.best_layer, "sum: ", sum(self.best_layer))
                 #print(self.penalty_layer(self.best_layer))
+                if self.print_to_csv:
+                    with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
+                        writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
+                        writer.writerow([self.target_col,self.i,self.k, self.topology_file, 0, 0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
+                    csvFile.close
 
-                with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
-                    writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
-                    writer.writerow([self.target_col,self.i,self.k, self.topology_file, 0,0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
-                csvFile.close
+                result = [self.target_col,self.i,self.k, self.topology_file, 0, 0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type]
 
-                return False
+                return False, result
 
             layer = self.regroup_layers(self.best_layer)
             max_latency, latencies, res, layers = self.eva_hybird_sq(layer)
@@ -314,11 +318,14 @@ class cma_approach(object):
         # print("Throughtput Ratio:", (1/max_latency)/(1/full_latency[full_max_idx]))
         # print("Latency increase:", (max_latency*self.k)/full_latency[full_max_idx])
 
-        with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
-            writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
-            writer.writerow([self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
-        csvFile.close
-        return True
+        if self.print_to_csv:
+            with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
+                writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
+                writer.writerow([self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
+            csvFile.close
+
+        result = [self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type]
+        return True, result
 
 if __name__ == "__main__":
     import csv
@@ -343,7 +350,8 @@ if __name__ == "__main__":
         res_step = 3,
         penalty_offest = 100000000000,
         seeding_type = seeding_type,
-        hybird = True
+        hybird = True,
+        print_to_csv = True
     )
 
     trials = 1
@@ -352,5 +360,5 @@ if __name__ == "__main__":
         es_hybird.run()
         trials += 1
 
-    k += 1
+    # k += 1
     #print("convergence takes", trials, "trials")

optimization_algo/approaches/requirements.txt

 cma
 hyperopt
 tqdm
-csv
-itertools
-multiprocessing
\ No newline at end of file
+# csv
+# itertools
+# multiprocessing
\ No newline at end of file

optimization_algo/approaches/sq_approach_faster.py

+from multiprocessing import Pool
+from os import cpu_count
+
+import path_constant as pc
+import cma_approach_square_size as ap
+
+nets = [
+    'FasterRCNN', 
+    'mobilenet',
+    'yolo_tiny',
+    'googlenet',
+    'alexnet',
+    'AlphaGoZero',
+    'ncf_rec',
+    'resnet_50_v1']
+
+layers = [
+    46, 27, 10, 58, 8, 8, 8, 53
+]
+
+max_partitions = [
+    23, 14, 9, 29, 8, 8, 8, 27, 13
+]
+
+max_res_sizes = [
+    1920*9*1, 1920*9*2, 1920*9*3
+]
+
+pop_size = [
+    100, 200, 500
+]
+
+targets = [
+    'DRAM_cycle', 'Cycles'
+]
+
+seedings = [
+    'allzeros', 'allzeros'
+]
+
+MAX_TRIAL = 20
+
+def trial(prop):
+    trials = 1
+    n, part, pop, tar, res, seed = prop
+
+    instance = ap.cma_approach(
+        path_to_datasrc = n+"_square_mem_bound.csv",
+        path_to_topology = n+".csv",
+        target_col = tar,
+
+        number_of_partition = part, max_iteration = 10000,
+        sigma = 0.5, population_size = pop,
+
+        max_res_unit = res, initial_res = 0,
+        res_step = 3,
+        penalty_offest = 100000000000,
+        seeding_type = seed,
+        hybird = True,
+        print_to_csv = False
+    )
+
+    instance.run()
+    success, out = instance.report()
+    while not success and trials < MAX_TRIAL:
+        instance.run()
+        success, out = instance.report()
+        trials += 1
+    return out
+
+    topology = sys.argv[1]
+    k = int(sys.argv[2])
+    population_size = int(sys.argv[3])
+    max_res_unit = int(sys.argv[4])
+    seeding_type = sys.argv[5]
+    target_col = sys.argv[6]
+
+if __name__ == "__main__":
+    import sys
+    import csv
+
+    MAX_TRIAL = sys.argv[1]
+    approaches = []
+
+    for idx, n in enumerate(nets):
+        for part in range(3, max_partitions[idx] + 1):
+            for seed in seedings:
+                for pop in pop_size:
+                    for tar in targets:
+                        for res in max_res_sizes:
+                            # print(n, part, pop, tar, res, seed)
+                            approaches.append((n, part, pop, tar, res, seed))
+
+    pool = Pool(processes = cpu_count() - 4)
+    for result in pool.imap_unordered(trial, approaches):
+        with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
+            writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
+            writer.writerow(result)
+    pool.close()
+    pool.join()
\ No newline at end of file