Skip to content
Snippets Groups Projects
Commit 854f13b5 authored by Bryant Curto's avatar Bryant Curto
Browse files

Merge branch 'master' into enable-ssl-tls

parents 4daf34e1 27f32fd8
No related branches found
No related tags found
No related merge requests found
Pipeline #70928 failed with stage
......@@ -18,7 +18,8 @@ typedef FredMutex shim_mutex_t;
#endif
static inline shim_thread_t* shim_thread_create(void (*start_routine)(void *), void* arg, bool bg = false) {
Fibre* f = new Fibre(CurrCluster(), bg);
Fibre* f = new Fibre(CurrCluster());
if (bg) f->setAffinity(Fibre::NoAffinity);
f->run(start_routine, arg);
return f;
}
......
......@@ -406,7 +406,7 @@ static void* scopemain(void* arg) {
Cluster** cluster = new Cluster*[clusterCount];
cluster[0] = &CurrCluster();
for (unsigned int c = 1; c < clusterCount; c += 1) {
cluster[c] = new Cluster;
cluster[c] = new Cluster(pollerCount);
}
#if defined __LIBFIBRE__
......
......@@ -16,6 +16,19 @@
******************************************************************************/
#include "runtime/Scheduler.h"
inline Fred* BaseProcessor::tryAll() {
Fred* nextFred;
if ((nextFred = tryLocal())) return nextFred;
#if TESTING_LOADBALANCING
if ((nextFred = tryStage())) return nextFred;
if (RuntimeWorkerPoll(*this)) {
if ((nextFred = tryLocal())) return nextFred;
}
if ((nextFred = trySteal())) return nextFred;
#endif
return nullptr;
}
inline Fred* BaseProcessor::tryLocal() {
Fred* f = readyQueue.dequeue();
if (f) {
......@@ -27,7 +40,7 @@ inline Fred* BaseProcessor::tryLocal() {
#if TESTING_LOADBALANCING
inline Fred* BaseProcessor::tryStage() {
Fred* f = scheduler.stage();
Fred* f = scheduler.getStaging(_friend<BaseProcessor>()).readyQueue.tryDequeue();
if (f) {
DBG::outl(DBG::Level::Scheduling, "tryStage: ", FmtHex(this), ' ', FmtHex(f));
if (f->checkAffinity(*this, _friend<BaseProcessor>())) {
......@@ -71,43 +84,31 @@ inline Fred* BaseProcessor::trySteal() {
}
#endif
inline Fred* BaseProcessor::scheduleInternal() {
Fred* nextFred;
if ((nextFred = tryLocal())) return nextFred;
#if TESTING_LOADBALANCING
if ((nextFred = tryStage())) return nextFred;
if (RuntimeWorkerPoll(*this)) {
if ((nextFred = tryLocal())) return nextFred;
}
if ((nextFred = trySteal())) return nextFred;
#endif
return nullptr;
}
inline Fred* BaseProcessor::scheduleNonblocking() {
#if TESTING_LOADBALANCING
#if TESTING_GO_IDLEMANAGER
return scheduleInternal();
return tryAll();
#else /* TESTING_GO_IDLEMANAGER */
if (scheduler.idleManager.tryGetReadyFred()) {
Fred* nextFred;
do { nextFred = scheduleInternal(); } while (!nextFred);
do { nextFred = tryAll(); } while (!nextFred);
return nextFred;
}
#endif
#else /* TESTING_LOADBALANCING */
if (readyCount.tryP()) {
Fred* nextFred;
do { nextFred = scheduleInternal(); } while (!nextFred);
do { nextFred = tryAll(); } while (!nextFred);
return nextFred;
}
#endif
return nullptr;
}
inline Fred& BaseProcessor::idleLoopSchedule() {
inline Fred& BaseProcessor::scheduleFromIdle() {
Fred* nextFred;
#if TESTING_LOADBALANCING && TESTING_GO_IDLEMANAGER
#if TESTING_LOADBALANCING
#if TESTING_GO_IDLEMANAGER
for (;;) {
scheduler.idleManager.incSpinning();
for (size_t i = 1; i < IdleSpinMax; i += 1) {
......@@ -126,12 +127,11 @@ inline Fred& BaseProcessor::idleLoopSchedule() {
return *nextFred;
}
}
#else
#else /* TESTING_GO_IDLEMANAGER */
for (size_t i = 1; i < IdleSpinMax; i += 1) {
nextFred = scheduleNonblocking();
if (nextFred) return *nextFred;
}
#if TESTING_LOADBALANCING
nextFred = scheduler.idleManager.getReadyFred(*this);
if (nextFred) {
DBG::outl(DBG::Level::Scheduling, "handover: ", FmtHex(this), ' ', FmtHex(nextFred));
......@@ -139,46 +139,50 @@ inline Fred& BaseProcessor::idleLoopSchedule() {
stats->handover.count();
return *nextFred;
}
#else /* TESTING_LOADBALANCING */
#endif /* TESTING_GO_IDLEMANAGER */
#else /* TESTING_LOADBALANCING */
for (size_t i = 1; i < IdleSpinMax; i += 1) {
nextFred = scheduleNonblocking();
if (nextFred) return *nextFred;
}
if (!readyCount.P()) haltSem.P(*this);
#endif
do { nextFred = scheduleInternal(); } while (!nextFred);
#endif /* TESTING_LOADBALANCING */
do { nextFred = tryAll(); } while (!nextFred);
return *nextFred;
#endif
}
void BaseProcessor::idleLoop(Fred* initFred) {
if (initFred) Fred::idleYieldTo(*initFred, _friend<BaseProcessor>());
for (;;) {
Fred& nextFred = idleLoopSchedule();
Fred& nextFred = scheduleFromIdle();
Fred::idleYieldTo(nextFred, _friend<BaseProcessor>());
}
}
void BaseProcessor::enqueueResume(Fred& f, _friend<Fred>) {
#if TESTING_LOADBALANCING
if (!scheduler.idleManager.addReadyFred(f)) enqueueFred(f);
#else
enqueueFred(f);
if (!readyCount.V()) haltSem.V(*this);
#endif
Fred* BaseProcessor::tryScheduleYield(_friend<Fred>) {
return tryLocal();
}
Fred& BaseProcessor::scheduleFull(_friend<Fred>) {
Fred* nextFred = scheduleNonblocking();
if (nextFred) return *nextFred;
return *idleFred;
Fred* BaseProcessor::tryScheduleYieldGlobal(_friend<Fred>) {
return tryAll();
}
Fred* BaseProcessor::scheduleYield(_friend<Fred>) {
return tryLocal();
Fred* BaseProcessor::trySchedulePreempt(Fred* currFred, _friend<Fred> fsc) {
if (currFred == idleFred) return nullptr;
return tryScheduleYieldGlobal(fsc);
}
Fred* BaseProcessor::scheduleYieldGlobal(_friend<Fred>) {
return scheduleInternal();
Fred& BaseProcessor::scheduleFull(_friend<Fred>) {
Fred* nextFred = scheduleNonblocking();
if (nextFred) return *nextFred;
return *idleFred;
}
Fred* BaseProcessor::schedulePreempt(Fred* currFred, _friend<Fred> fsc) {
if (currFred == idleFred) return nullptr;
return scheduleYieldGlobal(fsc);
void BaseProcessor::enqueueResume(Fred& f, _friend<Fred>) {
#if TESTING_LOADBALANCING
if (!scheduler.idleManager.addReadyFred(f)) enqueueFred(f);
#else
enqueueFred(f);
if (!readyCount.V()) haltSem.V(*this);
#endif
}
......@@ -92,19 +92,20 @@ class BaseProcessor : public DoubleLink<BaseProcessor,3> {
static const size_t HaltSpinMax = 64;
static const size_t IdleSpinMax = 1024;
inline Fred* tryLocal();
inline Fred* tryAll();
inline Fred* tryLocal();
#if TESTING_LOADBALANCING
inline Fred* tryStage();
inline Fred* trySteal();
BaseProcessor* localVictim;
BaseProcessor* globalVictim;
#else
Benaphore<> readyCount;
Benaphore<> readyCount;
#endif
HaltSemaphore haltSem;
Fred* handoverFred;
HaltSemaphore haltSem;
Fred* handoverFred;
#if TESTING_WAKE_FRED_WORKER
bool halting;
bool halting;
#endif
void enqueueFred(Fred& f) {
......@@ -112,9 +113,8 @@ class BaseProcessor : public DoubleLink<BaseProcessor,3> {
readyQueue.enqueue(f);
}
inline Fred* scheduleInternal();
inline Fred* scheduleNonblocking();
inline Fred& idleLoopSchedule();
inline Fred& scheduleFromIdle();
protected:
Scheduler& scheduler;
......@@ -148,21 +148,7 @@ public:
void setHalting(bool, _friend<IdleManager>) {}
#endif
#if TESTING_LOADBALANCING
Fred* tryDequeue(_friend<Scheduler>) {
return readyQueue.tryDequeue();
}
#endif
void enqueueYield(Fred& f, _friend<Fred>) { enqueueFred(f); }
void enqueueResume(Fred& f, _friend<Fred>);
Fred& scheduleFull(_friend<Fred>);
Fred* scheduleYield(_friend<Fred>);
Fred* scheduleYieldGlobal(_friend<Fred>);
Fred* schedulePreempt(Fred* currFred, _friend<Fred>);
Fred* suspend(_friend<IdleManager>) {
Fred* halt(_friend<IdleManager>) {
for (size_t i = 0; i < HaltSpinMax; i += 1) {
if fastpath(haltSem.tryP(*this)) return handoverFred;
Pause();
......@@ -172,12 +158,21 @@ public:
return handoverFred;
}
void resume(Fred* f, _friend<IdleManager>) {
void wake(Fred* f, _friend<IdleManager>) {
stats->wake.count();
handoverFred = f;
haltSem.V(*this);
}
Fred* tryScheduleYield(_friend<Fred>);
Fred* tryScheduleYieldGlobal(_friend<Fred>);
Fred* trySchedulePreempt(Fred* currFred, _friend<Fred>);
Fred& scheduleFull(_friend<Fred>);
void enqueueYield(Fred& f, _friend<Fred>) { enqueueFred(f); }
void enqueueResume(Fred& f, _friend<Fred>);
void reset(Scheduler& c, _friend<EventScope> token, const char* n = "Processor ") {
new (stats) FredStats::ProcessorStats(this, &c, n);
readyQueue.reset(*this, token);
......
......@@ -114,13 +114,13 @@ inline void Fred::yieldResume(Fred& nextFred) {
}
bool Fred::yield() {
Fred* nextFred = Context::CurrProcessor().scheduleYield(_friend<Fred>());
Fred* nextFred = Context::CurrProcessor().tryScheduleYield(_friend<Fred>());
if (nextFred) Context::CurrFred()->yieldTo(*nextFred);
return nextFred;
}
bool Fred::yieldGlobal() {
Fred* nextFred = Context::CurrProcessor().scheduleYieldGlobal(_friend<Fred>());
Fred* nextFred = Context::CurrProcessor().tryScheduleYieldGlobal(_friend<Fred>());
if (nextFred) Context::CurrFred()->yieldTo(*nextFred);
return nextFred;
}
......@@ -135,7 +135,7 @@ void Fred::idleYieldTo(Fred& nextFred, _friend<BaseProcessor>) {
void Fred::preempt() {
CHECK_PREEMPTION(0);
Fred* currFred = Context::CurrFred();
Fred* nextFred = Context::CurrProcessor().schedulePreempt(currFred, _friend<Fred>());
Fred* nextFred = Context::CurrProcessor().trySchedulePreempt(currFred, _friend<Fred>());
if (nextFred) currFred->switchFred<Yield>(*nextFred);
}
......@@ -155,9 +155,10 @@ void Fred::rebalance() {
BaseProcessor& Fred::migrate(BaseProcessor& proc) {
Fred* f = Context::CurrFred();
BaseProcessor* cproc = f->processor;
if (&cproc->getScheduler() != &proc.getScheduler() || !f->yieldGlobal()) {
f->yieldResume(Context::CurrProcessor().scheduleFull(_friend<Fred>()));
}
f->processor = &proc;
if (&cproc->getScheduler() == &proc.getScheduler() && f->yieldGlobal()) return *cproc;
Fred& nextFred = Context::CurrProcessor().scheduleFull(_friend<Fred>());
f->yieldResume(nextFred);
return *cproc;
}
......
......@@ -186,7 +186,7 @@ public:
// check affinity and potentially update processor during work-stealing
bool checkAffinity(BaseProcessor& newProcessor, _friend<BaseProcessor>) {
if (affinity == FixedAffinity) return true;
if (affinity > FixedAffinity) affinity -= 1;
if (affinity == TempAffinity) affinity = FixedAffinity;
processor = &newProcessor;
return false;
}
......
......@@ -49,7 +49,7 @@ class IdleManager {
nextProc->setHalting(false, _friend<IdleManager>());
__atomic_sub_fetch(&waitCounter, 1, __ATOMIC_SEQ_CST);
procLock.release();
nextProc->resume(fred, _friend<IdleManager>());
nextProc->wake(fred, _friend<IdleManager>());
return true;
}
public:
......@@ -68,7 +68,7 @@ public:
proc.setHalting(true, _friend<IdleManager>());
waitingProcs.push_front(proc);
procLock.release();
return proc.suspend(_friend<IdleManager>());
return proc.halt(_friend<IdleManager>());
}
IdleManager(cptr_t) : spinCounter(0), waitCounter(0) {}
void reset(cptr_t, _friend<EventScope>) {}
......@@ -90,7 +90,7 @@ class IdleManager {
proc.setHalting(true, _friend<IdleManager>());
waitingProcs.push_front(proc);
procLock.release();
return proc.suspend(_friend<IdleManager>());
return proc.halt(_friend<IdleManager>());
} else {
Fred* nextFred = waitingFreds.pop();
procLock.release();
......@@ -112,7 +112,7 @@ class IdleManager {
}
nextProc->setHalting(false, _friend<IdleManager>());
procLock.release();
nextProc->resume(&fred, _friend<IdleManager>());
nextProc->wake(&fred, _friend<IdleManager>());
}
}
......@@ -211,9 +211,7 @@ public:
}
#if TESTING_LOADBALANCING
Fred* stage() {
return stagingProc.tryDequeue(_friend<Scheduler>());
}
BaseProcessor& getStaging(_friend<BaseProcessor>) { return stagingProc; }
#endif
};
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment