liulei/jx-callback
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- category sync almost ok.

Browse Source 
- refactor tasksch (isContinueWhenError)
This commit is contained in:
gazebo
2018-09-22 10:26:31 +08:00
parent 0623d31c71
commit 81089fb85c
13 changed files with 283 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
business/jxutils/tasksch/task.go
View File

@@ -22,22 +22,29 @@ const (
TaskStatusEnd = 4
)
const (
MaxParallelCount = 10
)
type WorkFunc func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error)
type ResultHandlerFunc func(taskName string, result []interface{}, err error)
type Task struct {
ID string `json:"id"`
Name string `json:"name"`
CreatedBy string `json:"createdBy"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
TerminatedAt time.Time `json:"terminatedAt"`
ParallelCount int `json:"parallelCount"`
TotalItemCount int `json:"totalItemCount"`
TotalJobCount int `json:"totalJobCount"`
FinishedItemCount int `json:"finishedItemCount"`
FinishedJobCount int `json:"finishedJobCount"`
Status int `json:"status"`
ID string `json:"id"`
Name string `json:"name"`
CreatedBy string `json:"createdBy"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
TerminatedAt time.Time `json:"terminatedAt"`
ParallelCount int `json:"parallelCount"`
TotalItemCount int `json:"totalItemCount"`
TotalJobCount int `json:"totalJobCount"`
FinishedItemCount int `json:"finishedItemCount"`
FinishedJobCount int `json:"finishedJobCount"`
FailedItemCount int `json:"failedItemCount"`
FailedJobCount int `json:"failedJobCount"`
IsContinueWhenError bool `json:"isContinueWhenError"`
Status int `json:"status"`
C <-chan int `json:"-"`
@@ -72,23 +79,31 @@ var (
ErrTaskIsCanceled = errors.New("任务被取消了")
)
func RunTask(taskName string, worker WorkFunc, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, itemList interface{}, params ...interface{}) *Task {
func RunTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *Task {
if parallelCount > MaxParallelCount {
parallelCount = MaxParallelCount
}
listLen := jxutils.GetSliceLen(itemList)
if parallelCount > listLen {
parallelCount = listLen
}
realItemList := utils.Interface2Slice(itemList)
jobList := jxutils.SplitSlice(realItemList, batchSize)
task := &Task{
ID: utils.GetUUID(),
Name: taskName,
CreatedAt: time.Now(),
CreatedBy: userName,
UpdatedAt: time.Now(),
TotalJobCount: len(jobList),
TotalItemCount: len(realItemList),
ParallelCount: parallelCount,
taskChan: make(chan []interface{}, parallelCount*100),
quitChan: make(chan int, parallelCount),
subFinishChan: make(chan interface{}, parallelCount),
finishChan: make(chan int, 2),
Status: TaskStatusWorking,
ID: utils.GetUUID(),
Name: taskName,
CreatedAt: time.Now(),
CreatedBy: userName,
UpdatedAt: time.Now(),
TotalJobCount: len(jobList),
TotalItemCount: len(realItemList),
ParallelCount: parallelCount,
taskChan: make(chan []interface{}, len(realItemList)+parallelCount), // 确保能装下所有taskitem，加结束标记
quitChan: make(chan int, parallelCount),
subFinishChan: make(chan interface{}, parallelCount),
finishChan: make(chan int, 2),
Status: TaskStatusWorking,
IsContinueWhenError: isContinueWhenError,
}
task.C = task.finishChan
go func() {
@@ -99,35 +114,34 @@ func RunTask(taskName string, worker WorkFunc, resultHandler ResultHandlerFunc,
retVal := make([]interface{}, 0)
for {
select {
case <-task.quitChan:
case <-task.quitChan: // 取消
goto end
case job := <-task.taskChan:
if job == nil {
if job == nil { // 任务完成
chanRetVal = retVal
goto end
} else {
result, err := worker(job, params...)
globals.SugarLogger.Debugf("RunTask %s, after call worker result:%v, err:%v", taskName, result, err)
task.finishedOneJob(len(job), err)
if err == nil {
task.finishedOneJob(len(job))
if result != nil {
retVal = append(retVal, utils.Interface2Slice(result)...)
}
} else {
} else if !isContinueWhenError { // 出错
chanRetVal = err
go func() {
task.Cancel()
}()
goto end
}
}
}
}
end:
// globals.SugarLogger.Debugf("RunTask %s, put to chann chanRetVal:%v", taskName, chanRetVal)
if task.GetStatus() < TaskStatusEndBegin {
globals.SugarLogger.Debugf("RunTask %s, put to chann chanRetVal:%v", taskName, chanRetVal)
task.locker.RLock()
if task.Status < TaskStatusEndBegin {
task.subFinishChan <- chanRetVal
}
task.locker.RUnlock()
}()
}
for _, job := range jobList {
@@ -141,10 +155,12 @@ func RunTask(taskName string, worker WorkFunc, resultHandler ResultHandlerFunc,
var taskErr error
for i := 0; i < parallelCount; i++ {
result := <-task.subFinishChan
// globals.SugarLogger.Debugf("RunTask %s, received from chann result:%v", taskName, result)
if err2, ok := result.(error); ok {
task.Cancel()
taskResult = nil
taskErr = err2
break
break // 出错情况下是否需要直接跳出？
} else if result != nil {
resultList := result.([]interface{})
taskResult = append(taskResult, resultList...)
@@ -201,11 +217,13 @@ func (t *Task) GetResult(duration time.Duration) (retVal []interface{}, err erro
}
func (t *Task) Cancel() {
if t.GetStatus() < TaskStatusEndBegin {
t.locker.Lock()
defer t.locker.Unlock()
if t.Status < TaskStatusEndBegin && t.Status != TaskStatusCanceling {
t.Status = TaskStatusCanceling
for i := 0; i < t.ParallelCount; i++ {
t.quitChan <- 0
}
t.setStatus(TaskStatusCanceling)
}
}
@@ -240,13 +258,18 @@ func (t *Task) GetStatus() int {
/////////
func (t *Task) finishedOneJob(itemCount int) {
func (t *Task) finishedOneJob(itemCount int, err error) {
t.locker.Lock()
defer t.locker.Unlock()
t.UpdatedAt = time.Now()
t.FinishedItemCount += itemCount
t.FinishedJobCount++
if err == nil {
t.FinishedItemCount += itemCount
t.FinishedJobCount++
} else {
t.FailedItemCount += itemCount
t.FailedJobCount++
}
}
func (t *Task) setStatus(status int) {

8
business/jxutils/tasksch/task_man.go
View File

@@ -17,8 +17,8 @@ func init() {
defTaskMan.taskList = make(map[string]*Task)
}
func (m *TaskMan) RunTask(taskName string, worker WorkFunc, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, itemList interface{}, params ...interface{}) *Task {
task := RunTask(taskName, worker, resultHandler, parallelCount, batchSize, userName, itemList, params...)
func (m *TaskMan) RunTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *Task {
task := RunTask(taskName, isContinueWhenError, resultHandler, parallelCount, batchSize, userName, worker, itemList, params...)
m.taskList[task.ID] = task
return task
}
@@ -34,8 +34,8 @@ func (m *TaskMan) GetTasks(taskID string, fromStatus, toStatus int, lastHours in
return taskList
}
func RunManagedTask(taskName string, worker WorkFunc, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, itemList interface{}, params ...interface{}) *Task {
return defTaskMan.RunTask(taskName, worker, resultHandler, parallelCount, batchSize, userName, itemList, params...)
func RunManagedTask(taskName string, isContinueWhenError bool, resultHandler ResultHandlerFunc, parallelCount, batchSize int, userName string, worker WorkFunc, itemList interface{}, params ...interface{}) *Task {
return defTaskMan.RunTask(taskName, isContinueWhenError, resultHandler, parallelCount, batchSize, userName, worker, itemList, params...)
}
func GetTasks(taskID string, fromStatus, toStatus int, lastHours int) (taskList []*Task) {

20
business/jxutils/tasksch/task_man_test.go
View File

@@ -13,7 +13,10 @@ func TestTaskMan(t *testing.T) {
for k := range itemList {
itemList[k] = k
}
task1 := RunManagedTask("test", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task1 := RunManagedTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -22,12 +25,12 @@ func TestTaskMan(t *testing.T) {
retSlice[k] = "hello:" + utils.Int2Str(batchItemList[k].(int)*2)
}
return retSlice, nil
}, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", itemList, "a", "b", 1, 2)
}, itemList, "a", "b", 1, 2)
task2 := RunManagedTask("test", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task2 := RunManagedTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -36,10 +39,7 @@ func TestTaskMan(t *testing.T) {
retSlice[k] = "hello:" + utils.Int2Str(batchItemList[k].(int)*2)
}
return retSlice, nil
}, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", itemList, "a", "b", 1, 2)
}, itemList, "a", "b", 1, 2)
time.Sleep(2 * time.Second)
task2.Cancel()
if task1.GetStatus() == task2.GetStatus() {

20
business/jxutils/tasksch/task_test.go
View File

@@ -13,7 +13,10 @@ func TestRunTask(t *testing.T) {
for k := range itemList {
itemList[k] = k
}
task := RunTask("test", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task := RunTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -22,10 +25,7 @@ func TestRunTask(t *testing.T) {
retSlice[k] = "hello:" + utils.Int2Str(batchItemList[k].(int)*2)
}
return retSlice, nil
}, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", itemList, "a", "b", 1, 2)
}, itemList, "a", "b", 1, 2)
result, err := task.GetResult(1 * time.Microsecond)
if err == nil || task.GetStatus() != TaskStatusWorking {
t.Fatal("task can not be done in 1 microsecond")
@@ -46,7 +46,10 @@ func TestCancelTask(t *testing.T) {
for k := range itemList {
itemList[k] = k
}
task := RunTask("test", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
task := RunTask("test", false, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", func(batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
sleepSecond := rand.Intn(5)
t.Logf("sleep %d seconds", sleepSecond)
time.Sleep(time.Duration(sleepSecond) * time.Second)
@@ -55,10 +58,7 @@ func TestCancelTask(t *testing.T) {
retSlice[k] = "hello:" + utils.Int2Str(batchItemList[k].(int)*2)
}
return retSlice, nil
}, func(taskName string, result []interface{}, err error) {
// t.Log("finished here")
// t.Log(utils.Format4Output(result, false))
}, 100, 7, "autotest", itemList, "a", "b", 1, 2)
}, itemList, "a", "b", 1, 2)
// time.Sleep(time.Second * 6)
t.Logf("finishedItemCount:%d, finishedJobCount:%d", task.GetFinishedItemCount(), task.GetFinishedJobCount())
task.Cancel()