Commit 21875cab authored by Jonathan Shahen's avatar Jonathan Shahen
Browse files

ReducToNomad complete; better timing for the ReduceTo and the RunSolver...

ReducToNomad complete; better timing for the ReduceTo and the RunSolver (incorperating a timing prefix so as to not over runeach EPOCH)
parent c445c6d2
......@@ -8,5 +8,6 @@
<classpathentry kind="lib" path="lib/hamcrest-core-1.3.jar"/>
<classpathentry kind="lib" path="lib/oplall.jar"/>
<classpathentry kind="lib" path="lib/commons-csv-1.3/commons-csv-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-math3-3.6.1/commons-math3-3.6.1.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
......@@ -4,7 +4,7 @@
<javadoc
access="private"
author="true"
classpath="lib/hamcrest-core-1.3.jar;lib/junit-4.12.jar;lib/commons-cli-1.2.jar;lib/commons-lang3-3.3.2.jar;lib/oplall.jar;lib/commons-csv-1.3/commons-csv-1.3.jar"
classpath="lib/hamcrest-core-1.3.jar;lib/junit-4.12.jar;lib/commons-cli-1.2.jar;lib/commons-lang3-3.3.2.jar;lib/oplall.jar;lib/commons-csv-1.3/commons-csv-1.3.jar;lib/commons-math3-3.6.1/commons-math3-3.6.1.jar"
destdir="doc"
windowtitle="Vagabond API"
doctitle="Vagabond API"
......
This diff is collapsed.
Apache Commons Math
Copyright 2001-2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
This product includes software developed for Orekit by
CS Systèmes d'Information (http://www.c-s.fr/)
Copyright 2010-2012 CS Systèmes d'Information
Apache Commons Math 3.6.1 RELEASE NOTES
The Apache Commons Math team is pleased to announce the release of
commons-math3-3.6.1
The Apache Commons Math project is a library of lightweight, self-contained
mathematics and statistics components addressing the most common practical
problems not immediately available in the Java programming language or
commons-lang.
This is a minor bug fix release. It incorporates three bug fixes since version
3.6. The minimum version of the Java platform required to compile and use
Commons Math 3.6.1 is Java 5.
Changes in this version include:
Fixed Bugs:
o MATH-1316: Fix code example in User Guide > Statistics > Multiple linear regression.
o MATH-1342: Fix ODE integrators when multiple events are nearly simultaneous.
Changes:
o MATH-1317: Add uniformity unit tests to "RandomGeneratorAbstractTest".
For complete information on Apache Commons Math, including instructions on how
to submit bug reports, patches, or suggestions for improvement, see the Apache
Commons Math website:
http://commons.apache.org/proper/commons-math/
......@@ -101,6 +101,9 @@ public class VagabondCUI {
System.out.println("-noplacementmaps -loglevel verbose "
+ "-settings tests/simpletestILP-NomadSubOptimal.properties !exit");
System.out.println("");
System.out.println("-loglevel verbose -settings tests/simpletestCNF.properties !exit");
System.out.println("-loglevel verbose -settings tests/simpletestNomad.properties !exit");
System.out.println("");
try {
BufferedReader bfr = new BufferedReader(new FileReader(previousCommandFilename));
previousCmd = bfr.readLine();
......
......@@ -21,7 +21,8 @@ import vagabond.reduction.ReduceTo;
import vagabond.reduction.RunSolver;
import vagabond.reduction.cnfsat.ReduceToCNFSAT;
import vagabond.reduction.cnfsat.RunSolverCNFSAT;
import vagabond.reduction.ilp.*;
import vagabond.reduction.ilp.ReduceToILP;
import vagabond.reduction.ilp.RunSolverILP;
import vagabond.reduction.nomad.ReduceToNomad;
import vagabond.reduction.nomad.RunSolverNomad;
import vagabond.results.ResultsManager;
......@@ -132,9 +133,14 @@ public class VagabondInstance {
* EPOCH LOOP - START
* ******************************************************************************
*/
for (int e = 0; e < settings.numberOfEpochs; e++) {
/* Timing */timing.startTimer("VagabondInstance::main()::epochLoop::" + e);
logger.info("Starting EPOCH " + e + " LOOP");
String tp;
for (int epochNum = 0; epochNum < settings.numberOfEpochs; epochNum++) {
tp = "Epoch_"
+ StringUtils.leftPad(epochNum + "", (int) Math.floor(Math.log10(settings.numberOfEpochs)) + 1,
"0")
+ "::";
/* Timing */timing.startTimer(tp + "VagabondInstance::main()::epochLoop::totalTime");
logger.info("Starting EPOCH " + epochNum + " LOOP");
switch (settings.reductionAlgorithm) {
case REDUCTION_SAT:
......@@ -146,13 +152,15 @@ public class VagabondInstance {
runSolver = new RunSolverILP();
// TODO Remove me!
/*
if (false) {
FileWriter fw = new FileWriter(settings.logFolder + "/ilpData.dat");
fw.write(WriteILPFile.toILPString(epochHistory));
fw.close();
throw new Exception("KILLING PROCESS! ILP wrote file out");
}
*/
break;
case REDUCTION_NOMAD:
reduction = new ReduceToNomad();
......@@ -173,11 +181,15 @@ public class VagabondInstance {
if (runSolver == null) { throw new Exception(
"Unable to create a RunSolver with the settings: " + settings.toString()); }
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::reduce");
// Setup the timer prefix
reduction.setTimerKeyPrefix(tp);
runSolver.setTimerKeyPrefix(tp);
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::reduce");
/**
* Reduce the current EpochHistory
*/
logger.info("Reducing for EPOCH " + e);
logger.info("Reducing for EPOCH " + epochNum);
// ERROR CHECKING if the reduction occurred any problems
if (reduction.reduce(epochHistory) == false) {
String placementMapFilePath = settings.logFolder + File.separator + settings.placementMapFile;
......@@ -190,21 +202,27 @@ public class VagabondInstance {
throw new Exception("Unable to reduce with the current PlacementMap");
}
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::reduce");
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::reduce");
// ###
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::load");
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::load");
// Load the Solver + Error checking
logger.info("Loading the Solver for EPOCH " + e);
logger.info("Loading the Solver for EPOCH " + epochNum);
if (runSolver.load(reduction) == false) { throw new Exception(
"Unable to load the reduction into the runSolver with the reduction: "
+ reduction.toString()); }
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::load");
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::load");
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::run");
// ###
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::run");
// Run The Solver
logger.info("Running the Solver for EPOCH " + e);
logger.info("Running the Solver for EPOCH " + epochNum);
EpochHistory newEpochHistory = runSolver.run();
/* Timing */timing.toggleTimer("VagabondInstance::main()::epochLoop::" + e + "::run");
/* Timing */timing.toggleTimer(tp + "VagabondInstance::main()::epochLoop::run");
// ###
// ERROR CHECKING for getting the new result
if (newEpochHistory == null) { throw new Exception(
......@@ -218,7 +236,8 @@ public class VagabondInstance {
}
if (settings.displayPlacementMapEveryEpoch) {
logger.info("Placement Map at the END of Epoch " + e + "\n" + epochHistory._latestPlacementMap);
logger.info(
"Placement Map at the END of Epoch " + epochNum + "\n" + epochHistory._latestPlacementMap);
}
if (settings.reductionAlgorithm == VagabondOptionString.REDUCTION_NOMAD) {
......@@ -226,7 +245,7 @@ public class VagabondInstance {
break;
}
/* Timing */stats.addEpochTime(timing.stopTimer("VagabondInstance::main()::epochLoop::" + e));
/* Timing */stats.addEpochTime(timing.stopTimer(tp + "VagabondInstance::main()::epochLoop::totalTime"));
}
/** *****************************************************************************
* ******************************************************************************
......
......@@ -8,6 +8,11 @@ import vagabond.pieces.EpochHistory;
*
*/
public interface ReduceTo {
/**
* A timer key prefix to distinguish between multiple runs (i.e. the current Epoch number)
* @param keyPrefix the text prefix to place at the start of every timing key
*/
public void setTimerKeyPrefix(String keyPrefix);
/**
* Reduces the EpochHistory into the problem space; stores a reduced form internally
......
......@@ -3,6 +3,11 @@ package vagabond.reduction;
import vagabond.pieces.EpochHistory;
public interface RunSolver {
/**
* A timer key prefix to distinguish between multiple runs (i.e. the current Epoch number)
* @param keyPrefix the text prefix to place at the start of every timing key
*/
public void setTimerKeyPrefix(String keyPrefix);
/**
* Loads up the reduction and performs any necessary operations to be ready to run the operation through the solver.
......
......@@ -4,6 +4,12 @@ import vagabond.pieces.EpochHistory;
import vagabond.reduction.ReduceTo;
public class ReduceToCNFSAT implements ReduceTo {
public String tp = "Epoch_Unknown";
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
public String toString() {
......
......@@ -6,6 +6,13 @@ import vagabond.reduction.RunSolver;
public class RunSolverCNFSAT implements RunSolver {
public String tp = "Epoch_Unknown";
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
public boolean load(ReduceTo r) {
// TODO Auto-generated method stub
......
......@@ -16,6 +16,8 @@ public class ReduceToILP implements ReduceTo {
public static Logger logger;
public static TimingManager timing;
public String tp = "Epoch_Unknown";
public EpochHistory _history = null;
public PlacementMap _currentPlacement = null;
public SumOfClientVMsPerMachine _sumPlacement = null;
......@@ -30,6 +32,11 @@ public class ReduceToILP implements ReduceTo {
timing = settings.timing;
}
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
......@@ -49,7 +56,7 @@ public class ReduceToILP implements ReduceTo {
@Override
public boolean reduce(EpochHistory eh) throws IloException {
/*TIMING*/timing.startTimer("ReduceToILP::reduce");
/*TIMING*/timing.startTimer(tp + "ReduceToILP::reduce");
_history = eh;
_currentPlacement = _history._latestPlacementMap;
......@@ -65,9 +72,9 @@ public class ReduceToILP implements ReduceTo {
oplF = new IloOplFactory();
IloOplErrorHandler errHandler = oplF.createOplErrorHandler(null);
/*TIMING*/timing.startTimer("ReduceToILP::reduce::readOPL");
/*TIMING*/timing.startTimer(tp + "ReduceToILP::reduce::readOPL");
IloOplModelSource modelSource = oplF.createOplModelSource(settings.dataFolder + "/ilp.mod");
/*TIMING*/timing.stopTimer("ReduceToILP::reduce::readOPL");
/*TIMING*/timing.stopTimer(tp + "ReduceToILP::reduce::readOPL");
IloOplSettings oplSettings = oplF.createOplSettings(errHandler);
IloOplModelDefinition def = oplF.createOplModelDefinition(modelSource, oplSettings);
......@@ -77,24 +84,24 @@ public class ReduceToILP implements ReduceTo {
cplex.setOut(null);
}
/*TIMING*/timing.startTimer("ReduceToILP::reduce::createOPLModel");
/*TIMING*/timing.startTimer(tp + "ReduceToILP::reduce::createOPLModel");
opl = oplF.createOplModel(def, cplex);
/*TIMING*/timing.stopTimer("ReduceToILP::reduce::createOPLModel");
/*TIMING*/timing.stopTimer(tp + "ReduceToILP::reduce::createOPLModel");
/*TIMING*/timing.startTimer("ReduceToILP::reduce::ILPCustomDataSource");
/*TIMING*/timing.startTimer(tp + "ReduceToILP::reduce::ILPCustomDataSource");
IloOplDataSource dataSource = new ILPCustomDataSource(oplF, _history, _sumPlacement);
/*TIMING*/timing.stopTimer("ReduceToILP::reduce::ILPCustomDataSource");
/*TIMING*/timing.stopTimer(tp + "ReduceToILP::reduce::ILPCustomDataSource");
opl.addDataSource(dataSource);
/*TIMING*/timing.startTimer("ReduceToILP::reduce::generate");
/*TIMING*/timing.startTimer(tp + "ReduceToILP::reduce::generate");
opl.generate();
/*TIMING*/timing.stopTimer("ReduceToILP::reduce::generate");
/*TIMING*/timing.stopTimer(tp + "ReduceToILP::reduce::generate");
// Set the optimization parameters
opl.getCplex().setParam(IloCplex.Param.MIP.Tolerances.MIPGap, settings.relativeMIPGapTolerance);
/*TIMING*/timing.stopTimer("ReduceToILP::reduce");
/*TIMING*/timing.stopTimer(tp + "ReduceToILP::reduce");
return true;
}
}
......@@ -8,13 +8,23 @@ import vagabond.pieces.PlacementMap;
import vagabond.reduction.ReduceTo;
import vagabond.reduction.RunSolver;
import vagabond.singleton.VagabondSettings;
import vagabond.timing.TimingManager;
public class RunSolverILP implements RunSolver {
public ReduceToILP rt;
private static Logger logger = null;
public static TimingManager timing;
public String tp = "Epoch_Unknown";
public RunSolverILP() {
logger = VagabondSettings.getInstance().getLogger();
timing = VagabondSettings.getInstance().timing;
}
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
......@@ -31,6 +41,7 @@ public class RunSolverILP implements RunSolver {
@Override
public EpochHistory run() throws IloException {
/*TIMING*/timing.toggleTimer(tp + "RunSolverILP::run");
if (rt.cplex.solve()) {
logger.fine("CPLEX OBJECTIVE: " + rt.cplex.getObjValue());
logger.fine("CPLEX Time: " + rt.cplex.getCplexTime());
......@@ -45,10 +56,13 @@ public class RunSolverILP implements RunSolver {
// End the model (clean up the resources)
rt.oplF.end();
/*TIMING*/timing.toggleTimer(tp + "RunSolverILP::run");
return rtn;
} else {
rt.oplF.end();
logger.severe("CPLEX could find no solution!");
/*TIMING*/timing.cancelTimer(tp + "RunSolverILP::run");
return null;
}
}
......
package vagabond.reduction.nomad;
import java.io.*;
import java.util.Date;
import java.util.logging.Logger;
import org.apache.commons.math3.util.Precision;
import vagabond.enums.PlacementMapDisplayStyle;
import vagabond.pieces.EpochHistory;
import vagabond.pieces.PlacementMap;
import vagabond.reduction.ReduceTo;
import vagabond.singleton.VagabondSettings;
import vagabond.timing.TimingManager;
public class ReduceToNomad implements ReduceTo {
public static VagabondSettings settings;
public static Logger logger;
public static TimingManager timing;
public String tp = "Epoch_Unknown";
public File nomadProgram;
public File nomadSettings;
public File nomadPlacement;
public ReduceToNomad() {
settings = VagabondSettings.getInstance();
logger = settings.getLogger();
timing = settings.timing;
nomadProgram = settings.getNomadExceutable();
nomadSettings = settings.getNomadSettingsFile();
nomadPlacement = settings.getNomadPlacementFile();
}
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("ReduceToNomad {");
// TODO Auto-generated method stub
sb.append("settingsFile=" + nomadSettings.getAbsolutePath() + ", ");
sb.append("placementFile=" + nomadPlacement.getAbsolutePath() + ", ");
sb.append("nomadProgram=" + nomadProgram.getAbsolutePath() + ", ");
sb.append("}");
return sb.toString();
}
@Override
public boolean reduce(EpochHistory eh) {
// TODO Auto-generated method stub
return false;
public boolean reduce(EpochHistory eh) throws IOException {
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::totalTime");
PlacementMap place = eh._latestPlacementMap;
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::writeSettingsFile");
FileWriter fws = new FileWriter(nomadSettings);
fws.write("Generated by Vagabond: " + settings.df.format(new Date()) + "\n");
fws.write("BUDGET=" + Precision.round(calculateMigrationPercentage(place, settings.migrationBudget), 3) + "\n");
fws.write("slidingT=" + settings.slidingWindow + "\n");
fws.write("numEpoch=" + settings.numberOfEpochs + "\n");
fws.write("numMachine=" + place.getNumberOfMachines() + "\n");
fws.write("numSlot=" + place.getMaxNumberOfSlotsPerMachine() + "\n");
fws.write("numClient=" + place.getNumberOfClients() + "\n");
fws.write("numVMRequest=" + place.getMaxNumberOfVMsPerClient() + "\n");
fws.write("PlacementMap=" + nomadPlacement.getName() + "\n");
fws.write("\n# You can ignore the lines below\n");
fws.write("percentInsert=0\n");
fws.write("<R,C>_Mode=0\n");
fws.write("Greedy=1\n");
fws.write("numRun=1\n");
fws.write("Opensys=0\n");
fws.write("SEED=10\n");
fws.close();
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::writeSettingsFile");
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::writePlacementFile");
FileWriter fwp = new FileWriter(nomadPlacement);
fwp.write(place.getFormatString(PlacementMapDisplayStyle.NOMAD));
fwp.close();
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::writePlacementFile");
/*TIMING*/timing.toggleTimer(tp + "ReduceToNomad::reduce::totalTime");
return true;
}
public static double calculateMigrationPercentage(PlacementMap place, int migrationBudget) {
return migrationBudget / place.getNumberOfVMs();
}
}
......@@ -5,6 +5,12 @@ import vagabond.reduction.ReduceTo;
import vagabond.reduction.RunSolver;
public class RunSolverNomad implements RunSolver {
public String tp = "Epoch_Unknown";
@Override
public void setTimerKeyPrefix(String keyPrefix) {
tp = keyPrefix;
}
@Override
public boolean load(ReduceTo r) {
......
package vagabond.singleton;
import java.io.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.logging.*;
import vagabond.VagabondInstance;
......@@ -34,8 +36,8 @@ public class VagabondSettings {
public static final String VERSION = "v0.0.2";
public static final String AUTHORS = "Jonathan Shahen <jmshahen@uwaterloo.ca>";
private static Logger logger = null;
public String logFile = "vagabond.log.csv";
// ################################################################
// FOLDERS
/**
* Stores the folder that contains all of the logs and results produced by this program
*/
......@@ -44,7 +46,74 @@ public class VagabondSettings {
* Stores the folder that contains data required by this program
*/
public String dataFolder = "data";
/**
* Stores the folder that contains the solver programs
*/
public String programFolder = "programs";
/**
* The sub-folder that stored the Nomad executable and files
* <p>
* This folder is located as a folder within {@link #programFolder}
*/
public String nomadFolder = "nomad-jonathan-edit";
/**
* The sub-folder that stores the CNF Solver and relevant files
* <p>
* This folder is located as a folder within {@link #programFolder}
*/
public String cnfSolverFolder = "lingeling-b85-3ee2a39-151109";
// ################################################################
// FILES
/**
* Stores the log file of what was recorded during the last run; does not store output to System.out!
* <p>
* In Excel you can make the data into a table by selecting all the data (CTRL+A) and then clicking
* <i>Insert-&gt;Table</i>;
* this will allow you to sort and limit columns by values you specify (Very handy for searching the log)
*/
public String logFile = "vagabond.log.csv";
/**
* Stores the output filename of the PlacementMap if the PlacementMap needs to be printed out
* @see vagabond.pieces.PlacementMap
*/
public String placementMapFile = "vagabond.error.placementMap.txt";
/**
* The file name of the timing results file to write out; SHOULD be stored under the folder {@link #logFolder}.
*/
public String timingFile = "vagabond.timing.csv";
/**
* The file name of the results results file to write out; SHOULD be stored under the folder {@link #logFolder}.
*/
public String resultsFile = "vagabond.results.txt";
// ################################################################
// PROGRAMS
/**
* Filename of the Nomad executable;
* can change this if you are on Linux/Mac to the binary compiled for that system.
*/
public String nomadProgram = "nomad.exe";
/**
* Filename of the CNF Solver executable;
* can change this if you are on Linux/Mac to the binary compiled for that system.
*/
public String cnfSolverProgram = "lingeling.exe";
/**
* Holds the specific filename of the settings.ini file that Nomad is expecting
* @see #getNomadSettingsFile()
*/
private String nomadSettingsFile = "settings.txt";
/**
* Holds the specific filename of the input placement map file that Nomad is expecting; must be referenced in the
* {@link #getNomadSettingsFile()} file.
* @see VagabondSettings#getNomadPlacementFile()
*/
public String nomadPlacementFile = "inputPlacement.txt";
// ################################################################
// LOGGING
private static Logger logger = null;
public ConsoleHandler consoleHandler = new ConsoleHandler();
/**
* The logging level of the program
......@@ -52,6 +121,7 @@ public class VagabondSettings {
public Level logLevel = Level.FINE;
public FileHandler fileHandler;
public Boolean WriteCSVFileHeader = true;
public DateFormat df = new SimpleDateFormat("yyyy/MM/dd hh:mm:ss.SSSZ");
/**
* A flag that when TRUE will print out to the logger on INFO the PlacementMap at the END of every epoch as well
......@@ -62,39 +132,28 @@ public class VagabondSettings {
*/
public Boolean displayPlacementMapEveryEpoch = true;
/**
* Stores the output filename of the PlacementMap if the PlacementMap needs to be printed out
* @see vagabond.pieces.PlacementMap
*/
public String placementMapFile = "vagabond.latest.placementMap.txt";
/**
* Stores the default display setting that should be used for placement maps
*/
public PlacementMapDisplayStyle displayStyle = PlacementMapDisplayStyle.CSV;
// ################################################################
// MANAGERS
/**
* The timing manager that holds all of the raw timing information for the instance.
*/
public TimingManager timing = new TimingManager();
/**
* The file name of the timing results file to write out; SHOULD be stored under the folder {@link #logFolder}.
*/
public String timingFile = "vagabond.timing.csv";