liulei/jx-callback
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- big refactor tasksch

Browse Source
This commit is contained in:
gazebo
2018-10-20 15:05:02 +08:00
parent 2bfa5fe17f
commit 0d5a5189ed
15 changed files with 210 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
business/jxutils/tasksch/parallel_task.go
View File

@@ -14,7 +14,7 @@ const (
MaxParallelCount = 10
)
type WorkFunc func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error)
type WorkFunc func(task *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error)
type ResultHandlerFunc func(taskName string, result []interface{}, err error)
type ParallelConfig struct {
@@ -68,7 +68,7 @@ func (c *ParallelConfig) SetResultHandler(resultHandler ResultHandlerFunc) *Para
return c
}
func NewParallelTask(taskName string, userName string, config *ParallelConfig, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
func NewParallelTask(taskName string, config *ParallelConfig, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
if config == nil {
config = NewParallelConfig()
}
@@ -95,19 +95,8 @@ func NewParallelTask(taskName string, userName string, config *ParallelConfig, w
return task
}
func RunParallelTask(taskName string, userName string, config *ParallelConfig, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
task := NewParallelTask(taskName, userName, config, worker, itemList, params...)
task.Run()
return task
}
func RunTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
config := NewParallelConfig()
config.BatchSize = batchSize
config.IsContinueWhenError = isContinueWhenError
config.ParallelCount = parallelCount
config.ResultHandler = resultHandler
task := NewParallelTask(taskName, userName, config, worker, itemList, params...)
func RunParallelTask(taskName string, config *ParallelConfig, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
task := NewParallelTask(taskName, config, userName, worker, itemList, params...)
task.Run()
return task
}
@@ -128,7 +117,7 @@ func (task *ParallelTask) Run() {
chanRetVal = retVal
goto end
} else {
result, err := task.worker(job, task.params...)
result, err := task.worker(task, job, task.params...)
// globals.SugarLogger.Debugf("ParallelTask.Run %s, after call worker result:%v, err:%v", task.Name, result, err)
task.finishedOneJob(len(job), err)
if err == nil {
@@ -204,3 +193,8 @@ func (task *ParallelTask) Run() {
}
})
}
func (t *ParallelTask) AddChild(task ITask) {
t.BaseTask.AddChild(task)
task.SetParent(t)
}

14
business/jxutils/tasksch/parallel_task_test.go
View File

@@ -8,15 +8,12 @@ import (
"git.rosy.net.cn/baseapi/utils"
)
func TestRunTask(t *testing.T) {
func TestRunParallelTask(t *testing.T) {
itemList := make([]int, 100)
for k := range itemList {
itemList[k] = k
}
task := RunTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task := RunParallelTask("test", NewParallelConfig().SetParallelCount(100).SetBatchSize(7), "autotest", func(task *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -41,15 +38,12 @@ func TestRunTask(t *testing.T) {
t.Log(task.GetStatus())
}
func TestCancelTask(t *testing.T) {
func TestCancelParallelTask(t *testing.T) {
itemList := make([]int, 100)
for k := range itemList {
itemList[k] = k
}
task := RunTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task := RunParallelTask("test", NewParallelConfig().SetParallelCount(100).SetBatchSize(7), "autotest", func(task *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)

15
business/jxutils/tasksch/sequence_task.go
View File

@@ -12,7 +12,7 @@ const (
StepEnd = "End"
)
type SeqWorkFunc func(step int, params ...interface{}) (result interface{}, err error) // 只有最后一次返回结果保留
type SeqWorkFunc func(task *SeqTask, step int, params ...interface{}) (result interface{}, err error)
type SeqTask struct {
BaseTask
@@ -27,6 +27,12 @@ func NewSeqTask(taskName string, userName string, worker SeqWorkFunc, stepCount
return task
}
func RunSeqTask(taskName string, userName string, worker SeqWorkFunc, stepCount int, params ...interface{}) *SeqTask {
task := NewSeqTask(taskName, userName, worker, stepCount, params...)
task.Run()
return task
}
func (task *SeqTask) Run() {
task.run(func() {
globals.SugarLogger.Debugf("SeqTask.Run %s", task.Name)
@@ -38,7 +44,7 @@ func (task *SeqTask) Run() {
goto EndFor
default:
}
result, err := task.worker(i, task.params...)
result, err := task.worker(task, i, task.params...)
task.finishedOneJob(1, err)
if taskErr = err; taskErr != nil {
globals.SugarLogger.Infof("SeqTask.Run %s step:%d failed with error:%v", task.Name, i, err)
@@ -72,3 +78,8 @@ func (task *SeqTask) Run() {
close(task.quitChan)
})
}
func (t *SeqTask) AddChild(task ITask) {
t.BaseTask.AddChild(task)
task.SetParent(t)
}

4
business/jxutils/tasksch/sequence_task_test.go
View File

@@ -11,11 +11,11 @@ import (
func TestRunSeqTask(t *testing.T) {
var seqTask ITask
seqTask = NewSeqTask("TestSeqTask", "autotest", func(step int, params ...interface{}) (result interface{}, err error) {
seqTask = NewSeqTask("TestSeqTask", "autotest", func(task *SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
fmt.Println("ONE")
task2 := NewParallelTask("hello", "xjh", nil, func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task2 := NewParallelTask("hello", nil, "xjh", func(parallelTask *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
i := batchItemList[0].(int)
time.Sleep(2 * time.Second)
fmt.Println(i * 2)

13
business/jxutils/tasksch/task.go
View File

@@ -2,6 +2,7 @@ package tasksch
import (
"encoding/json"
"fmt"
"sync"
"time"
@@ -37,6 +38,7 @@ type ITask interface {
AddChild(task ITask)
GetChildren() TaskList
SetParent(parentTask ITask)
json.Marshaler
}
@@ -69,6 +71,7 @@ type BaseTask struct {
quitChan chan int
locker sync.RWMutex
parent ITask
}
func (s TaskList) Len() int {
@@ -199,6 +202,16 @@ func (t *BaseTask) GetChildren() (children TaskList) {
return children
}
func (t *BaseTask) SetParent(parentTask ITask) {
t.locker.Lock()
defer t.locker.Unlock()
if t.parent != nil {
panic(fmt.Sprintf("task:%s already have parent!", utils.Format4Output(t, false)))
}
t.parent = parentTask
}
/////////
func (t *BaseTask) MarshalJSON() ([]byte, error) {

18
business/jxutils/tasksch/task_man.go
View File

@@ -18,8 +18,14 @@ func init() {
defTaskMan.taskList = make(map[string]ITask)
}
func (m *TaskMan) RunTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
task := RunTask(taskName, isContinueWhenError, resultHandler, parallelCount, batchSize, userName, worker, itemList, params...)
func (m *TaskMan) RunParallelTask(taskName string, config *ParallelConfig, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
task := RunParallelTask(taskName, config, userName, worker, itemList, params...)
m.ManageTask(task)
return task
}
func (m *TaskMan) RunSeqTask(taskName string, userName string, worker SeqWorkFunc, stepCount int, params ...interface{}) *SeqTask {
task := RunSeqTask(taskName, userName, worker, stepCount, params...)
m.ManageTask(task)
return task
}
@@ -44,8 +50,12 @@ func (m *TaskMan) ManageTask(task ITask) ITask {
return task
}
func RunManagedTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
return defTaskMan.RunTask(taskName, isContinueWhenError, resultHandler, parallelCount, batchSize, userName, worker, itemList, params...)
func RunManagedParallelTask(taskName string, config *ParallelConfig, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *ParallelTask {
return defTaskMan.RunParallelTask(taskName, config, userName, worker, itemList, params...)
}
func RunManagedSeqTask(taskName string, userName string, worker SeqWorkFunc, stepCount int, params ...interface{}) *SeqTask {
return defTaskMan.RunSeqTask(taskName, userName, worker, stepCount, params...)
}
func GetTasks(taskID string, fromStatus, toStatus int, lastHours int) (taskList TaskList) {

10
business/jxutils/tasksch/task_man_test.go
View File

@@ -13,10 +13,11 @@ func TestTaskMan(t *testing.T) {
for k := range itemList {
itemList[k] = k
}
task1 := RunManagedTask("test", false, func(taskName string, result []interface{}, err error) {
config1 := NewParallelConfig().SetResultHandler(func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
}).SetParallelCount(100).SetBatchSize(7)
task1 := RunManagedParallelTask("test", config1, "autotest", func(task *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -27,10 +28,11 @@ func TestTaskMan(t *testing.T) {
return retSlice, nil
}, itemList, "a", "b", 1, 2)
task2 := RunManagedTask("test", false, func(taskName string, result []interface{}, err error) {
config2 := NewParallelConfig().SetResultHandler(func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
}).SetParallelCount(100).SetBatchSize(7)
task2 := RunManagedParallelTask("test", config2, "autotest", func(task *ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)