Commit e9d5294d authored by harry1080ti's avatar harry1080ti
Browse files

faster pooling

parent a33ae306
......@@ -20,7 +20,8 @@ class cma_approach(object):
res_step = 1,
penalty_offest = 10000000000,
seeding_type="optimised",
hybird = True
hybird = True,
print_to_csv = True
):
self.target_col = target_col
self.start = time.time()
......@@ -47,6 +48,7 @@ class cma_approach(object):
self.seeding_type = seeding_type
self.max_res_available = max_res_unit
self.print_to_csv = print_to_csv
def parse_topology_file(self):
layers = []
......@@ -219,21 +221,21 @@ class cma_approach(object):
res_combintaions = [0] * self.es.popsize
pool = Pool(processes = cpu_count() - 4)
for result in pool.imap_unordered(self.evaluation_top_level, id_list):
scores[result[0]] = result[1]
if result[1] >= self.penalty_offest:
invalid_sampling += 1
else:
if not self.is_hybird:
res_combintaions[result[0]] = result[2]
pool.close()
pool.join()
# for tup in samples:
# _, scores[tup[0]] = evaluation_top_level(tup)
# if scores[tup[0]] >= self.penalty_offest:
# pool = Pool(processes = cpu_count() - 4)
# for result in pool.imap_unordered(self.evaluation_top_level, id_list):
# scores[result[0]] = result[1]
# if result[1] >= self.penalty_offest:
# invalid_sampling += 1
# else:
# if not self.is_hybird:
# res_combintaions[result[0]] = result[2]
# pool.close()
# pool.join()
for tup in id_list:
_, scores[tup[0]] = self.evaluation_top_level(tup)
if scores[tup[0]] >= self.penalty_offest:
invalid_sampling += 1
if not self.is_hybird:
best_in_iteration = min(scores)
......@@ -273,13 +275,15 @@ class cma_approach(object):
##print("RESULT NOT VALID")
##print("Layer:", self.best_layer, "sum: ", sum(self.best_layer))
#print(self.penalty_layer(self.best_layer))
if self.print_to_csv:
with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
writer.writerow([self.target_col,self.i,self.k, self.topology_file, 0, 0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
csvFile.close
with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
writer.writerow([self.target_col,self.i,self.k, self.topology_file, 0,0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
csvFile.close
result = [self.target_col,self.i,self.k, self.topology_file, 0, 0, 0, 0, 0, 0, 0, layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type]
return False
return False, result
layer = self.regroup_layers(self.best_layer)
max_latency, latencies, res, layers = self.eva_hybird_sq(layer)
......@@ -314,11 +318,14 @@ class cma_approach(object):
# print("Throughtput Ratio:", (1/max_latency)/(1/full_latency[full_max_idx]))
# print("Latency increase:", (max_latency*self.k)/full_latency[full_max_idx])
with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
writer.writerow([self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
csvFile.close
return True
if self.print_to_csv:
with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
writer.writerow([self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type])
csvFile.close
result = [self.target_col,self.i,self.k, self.topology_file, 1,(1/max_latency), max_latency*self.k, 1/full_latency[full_max_idx], full_latency[full_max_idx], (1/max_latency)/(1/full_latency[full_max_idx]), (max_latency*self.k)/full_latency[full_max_idx], layer, res, self.end-self.start, self.es.sigma, self.seed_od,self.valid_sampling_percentage, self.trial, self.population_size, self.max_res_unit, self.seeding_type]
return True, result
if __name__ == "__main__":
import csv
......@@ -343,7 +350,8 @@ if __name__ == "__main__":
res_step = 3,
penalty_offest = 100000000000,
seeding_type = seeding_type,
hybird = True
hybird = True,
print_to_csv = True
)
trials = 1
......@@ -352,5 +360,5 @@ if __name__ == "__main__":
es_hybird.run()
trials += 1
k += 1
# k += 1
#print("convergence takes", trials, "trials")
cma
hyperopt
tqdm
csv
itertools
multiprocessing
\ No newline at end of file
# csv
# itertools
# multiprocessing
\ No newline at end of file
from multiprocessing import Pool
from os import cpu_count
import path_constant as pc
import cma_approach_square_size as ap
nets = [
'FasterRCNN',
'mobilenet',
'yolo_tiny',
'googlenet',
'alexnet',
'AlphaGoZero',
'ncf_rec',
'resnet_50_v1']
layers = [
46, 27, 10, 58, 8, 8, 8, 53
]
max_partitions = [
23, 14, 9, 29, 8, 8, 8, 27, 13
]
max_res_sizes = [
1920*9*1, 1920*9*2, 1920*9*3
]
pop_size = [
100, 200, 500
]
targets = [
'DRAM_cycle', 'Cycles'
]
seedings = [
'allzeros', 'allzeros'
]
MAX_TRIAL = 20
def trial(prop):
trials = 1
n, part, pop, tar, res, seed = prop
instance = ap.cma_approach(
path_to_datasrc = n+"_square_mem_bound.csv",
path_to_topology = n+".csv",
target_col = tar,
number_of_partition = part, max_iteration = 10000,
sigma = 0.5, population_size = pop,
max_res_unit = res, initial_res = 0,
res_step = 3,
penalty_offest = 100000000000,
seeding_type = seed,
hybird = True,
print_to_csv = False
)
instance.run()
success, out = instance.report()
while not success and trials < MAX_TRIAL:
instance.run()
success, out = instance.report()
trials += 1
return out
topology = sys.argv[1]
k = int(sys.argv[2])
population_size = int(sys.argv[3])
max_res_unit = int(sys.argv[4])
seeding_type = sys.argv[5]
target_col = sys.argv[6]
if __name__ == "__main__":
import sys
import csv
MAX_TRIAL = sys.argv[1]
approaches = []
for idx, n in enumerate(nets):
for part in range(3, max_partitions[idx] + 1):
for seed in seedings:
for pop in pop_size:
for tar in targets:
for res in max_res_sizes:
# print(n, part, pop, tar, res, seed)
approaches.append((n, part, pop, tar, res, seed))
pool = Pool(processes = cpu_count() - 4)
for result in pool.imap_unordered(trial, approaches):
with open(pc.RESULT_CSV_PATH+'cma_logmore_sq.csv', 'a') as csvFile:
writer = csv.writer(csvFile, delimiter=',', lineterminator="\n")
writer.writerow(result)
pool.close()
pool.join()
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment