mirror of
https://github.com/safing/portbase
synced 2025-09-13 08:39:49 +00:00
Finish modules/tasks revamp
This commit is contained in:
Daniel
parent
4e99dd2153
commit
71dabc1f23
11 changed files with 542 additions and 296 deletions
|
|
@ -1,9 +1,11 @@
|
|||
package modules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/safing/portbase/log"
|
||||
"github.com/tevino/abool"
|
||||
)
|
||||
|
||||
|
|
@ -12,162 +14,140 @@ import (
|
|||
// (2) sometimes there seems to some kind of race condition stuff, the test hangs and does not complete
|
||||
|
||||
var (
|
||||
closedChannel chan bool
|
||||
microTasks *int32
|
||||
microTasksThreshhold *int32
|
||||
microTaskFinished = make(chan struct{}, 1)
|
||||
|
||||
tasks *int32
|
||||
|
||||
mediumPriorityClearance chan bool
|
||||
lowPriorityClearance chan bool
|
||||
veryLowPriorityClearance chan bool
|
||||
|
||||
tasksDone chan bool
|
||||
tasksDoneFlag *abool.AtomicBool
|
||||
tasksWaiting chan bool
|
||||
tasksWaitingFlag *abool.AtomicBool
|
||||
mediumPriorityClearance = make(chan struct{})
|
||||
lowPriorityClearance = make(chan struct{})
|
||||
)
|
||||
|
||||
// StartMicroTask starts a new MicroTask. It will start immediately.
|
||||
func StartMicroTask() {
|
||||
atomic.AddInt32(tasks, 1)
|
||||
tasksDoneFlag.UnSet()
|
||||
}
|
||||
|
||||
// EndMicroTask MUST be always called when a MicroTask was previously started.
|
||||
func EndMicroTask() {
|
||||
c := atomic.AddInt32(tasks, -1)
|
||||
if c < 1 {
|
||||
if tasksDoneFlag.SetToIf(false, true) {
|
||||
tasksDone <- true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newTaskIsWaiting() {
|
||||
tasksWaiting <- true
|
||||
}
|
||||
|
||||
// StartMediumPriorityMicroTask starts a new MicroTask (waiting its turn) if channel receives.
|
||||
func StartMediumPriorityMicroTask() chan bool {
|
||||
if shutdownSignalClosed.IsSet() {
|
||||
return closedChannel
|
||||
}
|
||||
if tasksWaitingFlag.SetToIf(false, true) {
|
||||
defer newTaskIsWaiting()
|
||||
}
|
||||
return mediumPriorityClearance
|
||||
}
|
||||
|
||||
// StartLowPriorityMicroTask starts a new MicroTask (waiting its turn) if channel receives.
|
||||
func StartLowPriorityMicroTask() chan bool {
|
||||
if shutdownSignalClosed.IsSet() {
|
||||
return closedChannel
|
||||
}
|
||||
if tasksWaitingFlag.SetToIf(false, true) {
|
||||
defer newTaskIsWaiting()
|
||||
}
|
||||
return lowPriorityClearance
|
||||
}
|
||||
|
||||
// StartVeryLowPriorityMicroTask starts a new MicroTask (waiting its turn) if channel receives.
|
||||
func StartVeryLowPriorityMicroTask() chan bool {
|
||||
if shutdownSignalClosed.IsSet() {
|
||||
return closedChannel
|
||||
}
|
||||
if tasksWaitingFlag.SetToIf(false, true) {
|
||||
defer newTaskIsWaiting()
|
||||
}
|
||||
return veryLowPriorityClearance
|
||||
}
|
||||
|
||||
func start() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func stop() error {
|
||||
close(shutdownSignal)
|
||||
return nil
|
||||
}
|
||||
const (
|
||||
mediumPriorityMaxDelay = 1 * time.Second
|
||||
lowPriorityMaxDelay = 3 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
var microTasksVal int32
|
||||
microTasks = µTasksVal
|
||||
var microTasksThreshholdVal int32
|
||||
microTasksThreshhold = µTasksThreshholdVal
|
||||
}
|
||||
|
||||
closedChannel = make(chan bool, 0)
|
||||
close(closedChannel)
|
||||
// SetMaxConcurrentMicroTasks sets the maximum number of microtasks that should be run concurrently.
|
||||
func SetMaxConcurrentMicroTasks(n int) {
|
||||
if n < 4 {
|
||||
atomic.StoreInt32(microTasksThreshhold, 4)
|
||||
} else {
|
||||
atomic.StoreInt32(microTasksThreshhold, int32(n))
|
||||
}
|
||||
}
|
||||
|
||||
var t int32
|
||||
tasks = &t
|
||||
// StartMicroTask starts a new MicroTask with high priority. It will start immediately. The given function will be executed and panics caught. The supplied name should be a constant - the variable should never change as it won't be copied.
|
||||
func (m *Module) StartMicroTask(name *string, fn func(context.Context) error) error {
|
||||
atomic.AddInt32(microTasks, 1)
|
||||
return m.runMicroTask(name, fn)
|
||||
}
|
||||
|
||||
mediumPriorityClearance = make(chan bool, 0)
|
||||
lowPriorityClearance = make(chan bool, 0)
|
||||
veryLowPriorityClearance = make(chan bool, 0)
|
||||
// StartMediumPriorityMicroTask starts a new MicroTask with medium priority. It will wait until given a go (max 3 seconds). The given function will be executed and panics caught. The supplied name should be a constant - the variable should never change as it won't be copied.
|
||||
func (m *Module) StartMediumPriorityMicroTask(name *string, fn func(context.Context) error) error {
|
||||
// check if we can go immediately
|
||||
select {
|
||||
case <-mediumPriorityClearance:
|
||||
default:
|
||||
// wait for go or max delay
|
||||
select {
|
||||
case <-mediumPriorityClearance:
|
||||
case <-time.After(mediumPriorityMaxDelay):
|
||||
}
|
||||
}
|
||||
return m.runMicroTask(name, fn)
|
||||
}
|
||||
|
||||
tasksDone = make(chan bool, 1)
|
||||
tasksDoneFlag = abool.NewBool(true)
|
||||
tasksWaiting = make(chan bool, 1)
|
||||
tasksWaitingFlag = abool.NewBool(false)
|
||||
// StartLowPriorityMicroTask starts a new MicroTask with low priority. It will wait until given a go (max 15 seconds). The given function will be executed and panics caught. The supplied name should be a constant - the variable should never change as it won't be copied.
|
||||
func (m *Module) StartLowPriorityMicroTask(name *string, fn func(context.Context) error) error {
|
||||
// check if we can go immediately
|
||||
select {
|
||||
case <-lowPriorityClearance:
|
||||
default:
|
||||
// wait for go or max delay
|
||||
select {
|
||||
case <-lowPriorityClearance:
|
||||
case <-time.After(lowPriorityMaxDelay):
|
||||
}
|
||||
}
|
||||
return m.runMicroTask(name, fn)
|
||||
}
|
||||
|
||||
timoutTimerDuration := 1 * time.Second
|
||||
// timoutTimer := time.NewTimer(timoutTimerDuration)
|
||||
func (m *Module) runMicroTask(name *string, fn func(context.Context) error) (err error) {
|
||||
// start for module
|
||||
// hint: only microTasks global var is important for scheduling, others can be set here
|
||||
atomic.AddInt32(m.microTaskCnt, 1)
|
||||
m.waitGroup.Add(1)
|
||||
|
||||
go func() {
|
||||
microTaskManageLoop:
|
||||
for {
|
||||
// set up recovery
|
||||
defer func() {
|
||||
// recover from panic
|
||||
panicVal := recover()
|
||||
if panicVal != nil {
|
||||
me := m.NewPanicError(*name, "microtask", panicVal)
|
||||
me.Report()
|
||||
log.Errorf("%s: microtask %s panicked: %s\n%s", m.Name, *name, panicVal, me.StackTrace)
|
||||
err = me
|
||||
}
|
||||
|
||||
// wait for an event to start new tasks
|
||||
if !shutdownSignalClosed.IsSet() {
|
||||
|
||||
// reset timer
|
||||
// https://golang.org/pkg/time/#Timer.Reset
|
||||
// if !timoutTimer.Stop() {
|
||||
// <-timoutTimer.C
|
||||
// }
|
||||
// timoutTimer.Reset(timoutTimerDuration)
|
||||
|
||||
// wait for event to start a new task
|
||||
select {
|
||||
case <-tasksWaiting:
|
||||
if !tasksDoneFlag.IsSet() {
|
||||
continue microTaskManageLoop
|
||||
}
|
||||
case <-time.After(timoutTimerDuration):
|
||||
case <-tasksDone:
|
||||
case <-shutdownSignal:
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
// execute tasks until no tasks are waiting anymore
|
||||
if !tasksWaitingFlag.IsSet() {
|
||||
// wait until tasks are finished
|
||||
if !tasksDoneFlag.IsSet() {
|
||||
<-tasksDone
|
||||
}
|
||||
// signal module completion
|
||||
// microTasksModule.StopComplete()
|
||||
// exit
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// start new task, if none is started, check if we are shutting down
|
||||
select {
|
||||
case mediumPriorityClearance <- true:
|
||||
StartMicroTask()
|
||||
default:
|
||||
select {
|
||||
case lowPriorityClearance <- true:
|
||||
StartMicroTask()
|
||||
default:
|
||||
select {
|
||||
case veryLowPriorityClearance <- true:
|
||||
StartMicroTask()
|
||||
default:
|
||||
tasksWaitingFlag.UnSet()
|
||||
}
|
||||
}
|
||||
}
|
||||
// finish for module
|
||||
atomic.AddInt32(m.microTaskCnt, -1)
|
||||
m.waitGroup.Done()
|
||||
|
||||
// finish and possibly trigger next task
|
||||
atomic.AddInt32(microTasks, -1)
|
||||
select {
|
||||
case microTaskFinished <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}()
|
||||
|
||||
// run
|
||||
err = fn(m.Ctx)
|
||||
return //nolint:nakedret // need to use named return val in order to change in defer
|
||||
}
|
||||
|
||||
var (
|
||||
microTaskSchedulerStarted = abool.NewBool(false)
|
||||
)
|
||||
|
||||
func microTaskScheduler() {
|
||||
// only ever start once
|
||||
if !microTaskSchedulerStarted.SetToIf(false, true) {
|
||||
return
|
||||
}
|
||||
|
||||
microTaskManageLoop:
|
||||
for {
|
||||
if shutdownSignalClosed.IsSet() {
|
||||
close(mediumPriorityClearance)
|
||||
close(lowPriorityClearance)
|
||||
return
|
||||
}
|
||||
|
||||
if atomic.LoadInt32(microTasks) < atomic.LoadInt32(microTasksThreshhold) { // space left for firing task
|
||||
select {
|
||||
case mediumPriorityClearance <- struct{}{}:
|
||||
default:
|
||||
select {
|
||||
case taskTimeslot <- struct{}{}:
|
||||
continue microTaskManageLoop
|
||||
case mediumPriorityClearance <- struct{}{}:
|
||||
case lowPriorityClearance <- struct{}{}:
|
||||
}
|
||||
}
|
||||
// increase task counter
|
||||
atomic.AddInt32(microTasks, 1)
|
||||
} else {
|
||||
// wait for signal that a task was completed
|
||||
<-microTaskFinished
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue