- refactor jxcontent and tasksch (remove dependency from jxcontent to tasksch)

- send dingding msg to user when async task finished
This commit is contained in:
gazebo
2019-03-22 15:04:28 +08:00
parent 0f5445020a
commit 25ac631c5c
31 changed files with 1527 additions and 1488 deletions

View File

@@ -302,44 +302,45 @@ func (p *PurchaseHandler) RefreshAllSkusID(ctx *jxcontext.Context, parentTask ta
var skuPairs []*jdapi.SkuIDPair
const stepCount = 2
rootTask := tasksch.NewSeqTask("jd RefreshAllSkusID", ctx.GetUserName(), func(rootTask *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
err = dao.GetRows(db, &skuPairs, `
rootTask := tasksch.NewSeqTask("jd RefreshAllSkusID", ctx,
func(rootTask *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
err = dao.GetRows(db, &skuPairs, `
SELECT t1.id out_sku_id, t1.jd_id sku_id
FROM sku t1
WHERE t1.deleted_at = ?
`, utils.DefaultTimeValue)
default:
taskName := "RefreshAllSkusID update id"
if step != stepCount-1 {
taskName = "RefreshAllSkusID update uuid"
default:
taskName := "RefreshAllSkusID update id"
if step != stepCount-1 {
taskName = "RefreshAllSkusID update uuid"
}
task1 := tasksch.NewParallelTask(taskName, tasksch.NewParallelConfig().SetIsContinueWhenError(true).SetBatchSize(jdapi.MaxBatchSize4BatchUpdateOutSkuId), ctx,
func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuPairs := make([]*jdapi.SkuIDPair, len(batchItemList))
for k, v := range batchItemList {
pair := v.(*jdapi.SkuIDPair)
skuPairs[k] = &jdapi.SkuIDPair{
SkuId: pair.SkuId,
OutSkuId: pair.OutSkuId,
}
if step != stepCount-1 {
skuPairs[k].OutSkuId = utils.GetUUID()
}
}
globals.SugarLogger.Debug(utils.Format4Output(skuPairs, false))
if globals.EnableStoreWrite {
_, err = api.JdAPI.BatchUpdateOutSkuId(skuPairs)
}
return nil, err
}, skuPairs)
rootTask.AddChild(task1).Run()
_, err = task1.GetResult(0)
}
task1 := tasksch.NewParallelTask(taskName, tasksch.NewParallelConfig().SetIsContinueWhenError(true).SetBatchSize(jdapi.MaxBatchSize4BatchUpdateOutSkuId), ctx.GetUserName(), func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuPairs := make([]*jdapi.SkuIDPair, len(batchItemList))
for k, v := range batchItemList {
pair := v.(*jdapi.SkuIDPair)
skuPairs[k] = &jdapi.SkuIDPair{
SkuId: pair.SkuId,
OutSkuId: pair.OutSkuId,
}
if step != stepCount-1 {
skuPairs[k].OutSkuId = utils.GetUUID()
}
}
globals.SugarLogger.Debug(utils.Format4Output(skuPairs, false))
if globals.EnableStoreWrite {
_, err = api.JdAPI.BatchUpdateOutSkuId(skuPairs)
}
return nil, err
}, skuPairs)
rootTask.AddChild(task1).Run()
_, err = task1.GetResult(0)
}
return nil, err
}, stepCount)
ctx.SetTaskOrAddChild(rootTask, parentTask)
rootTask.Run()
return nil, err
}, stepCount)
tasksch.HandleTask(rootTask, parentTask, false).Run()
if !isAsync {
_, err = rootTask.GetResult(0)
}

View File

@@ -209,41 +209,42 @@ func (p *PurchaseHandler) RefreshAllStoresID(ctx *jxcontext.Context, parentTask
const stepCount = 3
var stores []*tJdStoreInfo
db := dao.GetDB()
rootTask := tasksch.NewSeqTask("jd RefreshAllStoresID", ctx.GetUserName(), func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
err = dao.GetRows(db, &stores, `
rootTask := tasksch.NewSeqTask("jd RefreshAllStoresID", ctx,
func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
err = dao.GetRows(db, &stores, `
SELECT t1.*, t2.vendor_store_id
FROM store t1
JOIN store_map t2 ON t1.id = t2.store_id AND t2.deleted_at = ? AND t2.vendor_id = ?
WHERE t1.deleted_at = ?
`, utils.DefaultTimeValue, model.VendorIDJD, utils.DefaultTimeValue)
default:
taskName := "jd RefreshAllStoresID update outSystemId"
if step != stepCount-1 {
taskName = "jd RefreshAllStoresID update to uuid"
}
task1 := tasksch.NewParallelTask(taskName, nil, ctx.GetUserName(), func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
store := batchItemList[0].(*tJdStoreInfo)
storeParams := map[string]interface{}{
"outSystemId": utils.Int2Str(int(store.ID)),
}
default:
taskName := "jd RefreshAllStoresID update outSystemId"
if step != stepCount-1 {
storeParams["outSystemId"] = utils.GetUUID()
taskName = "jd RefreshAllStoresID update to uuid"
}
if true { //globals.EnableStoreWrite {
err = api.JdAPI.UpdateStoreInfo4Open(store.VendorStoreID, ctx.GetUserName(), storeParams)
}
return nil, err
}, stores)
task.AddChild(task1).Run()
_, err = task1.GetResult(0)
}
return nil, err
}, stepCount)
task1 := tasksch.NewParallelTask(taskName, nil, ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
store := batchItemList[0].(*tJdStoreInfo)
storeParams := map[string]interface{}{
"outSystemId": utils.Int2Str(int(store.ID)),
}
if step != stepCount-1 {
storeParams["outSystemId"] = utils.GetUUID()
}
if true { //globals.EnableStoreWrite {
err = api.JdAPI.UpdateStoreInfo4Open(store.VendorStoreID, ctx.GetUserName(), storeParams)
}
return nil, err
}, stores)
task.AddChild(task1).Run()
_, err = task1.GetResult(0)
}
return nil, err
}, stepCount)
ctx.SetTaskOrAddChild(rootTask, parentTask)
rootTask.Run()
tasksch.HandleTask(rootTask, parentTask, false).Run()
if !isAsync {
_, err = rootTask.GetResult(0)
}

View File

@@ -54,106 +54,106 @@ func (p *PurchaseHandler) SyncStoreSkus(ctx *jxcontext.Context, parentTask tasks
if err = dao.GetRows(db, &storeSkus, sql, append(sqlParams, sqlWhereParams...)...); err != nil {
return "", err
}
task := tasksch.NewParallelTask("SyncStoresSkus京东", tasksch.NewParallelConfig().SetBatchSize(jdapi.MaxStoreSkuBatchSize).SetIsContinueWhenError(isContinueWhenError), ctx.GetUserName(), func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
var skuPriceInfoList []*jdapi.SkuPriceInfo
var skuVendibilityList []*jdapi.StockVendibility
var skuStockList []*jdapi.SkuStock
stationNo := batchItemList[0].(*tStoreSkuBindExt).VendorStoreID
var batchSkuIDs []int
for _, v := range batchItemList {
storeSku := v.(*tStoreSkuBindExt)
alreadyAddStock := false
if storeSku.JdSyncStatus&model.SyncFlagChangedMask != 0 {
batchSkuIDs = append(batchSkuIDs, storeSku.SkuID)
if storeSku.JdSyncStatus&(model.SyncFlagDeletedMask|model.SyncFlagNewMask) != 0 { // 关注或取消关注
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.SkuID),
StockQty: model.MaxStoreSkuStockQty,
}
if storeSku.DeletedAt != utils.DefaultTimeValue {
stock.StockQty = 0
} else {
alreadyAddStock = true
}
if stock.StockQty != 0 || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuStockList = append(skuStockList, stock)
}
}
if storeSku.JdSyncStatus&(model.SyncFlagPriceMask|model.SyncFlagNewMask) != 0 {
skuPriceInfoList = append(skuPriceInfoList, &jdapi.SkuPriceInfo{
OutSkuId: utils.Int2Str(storeSku.SkuID),
Price: jxutils.CaculateSkuVendorPrice(storeSku.Price, storeSku.PricePercentage),
})
}
if storeSku.JdSyncStatus&(model.SyncFlagSaleMask|model.SyncFlagNewMask) != 0 {
vendibility := &jdapi.StockVendibility{
OutSkuId: utils.Int2Str(storeSku.SkuID),
DoSale: true,
}
if storeSku.Status != model.StoreSkuBindStatusNormal {
vendibility.DoSale = false
} else if !alreadyAddStock { // 如果是设置可售则自动将库存加满
task := tasksch.NewParallelTask("SyncStoresSkus京东", tasksch.NewParallelConfig().SetBatchSize(jdapi.MaxStoreSkuBatchSize).SetIsContinueWhenError(isContinueWhenError), ctx,
func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
var skuPriceInfoList []*jdapi.SkuPriceInfo
var skuVendibilityList []*jdapi.StockVendibility
var skuStockList []*jdapi.SkuStock
stationNo := batchItemList[0].(*tStoreSkuBindExt).VendorStoreID
var batchSkuIDs []int
for _, v := range batchItemList {
storeSku := v.(*tStoreSkuBindExt)
alreadyAddStock := false
if storeSku.JdSyncStatus&model.SyncFlagChangedMask != 0 {
batchSkuIDs = append(batchSkuIDs, storeSku.SkuID)
if storeSku.JdSyncStatus&(model.SyncFlagDeletedMask|model.SyncFlagNewMask) != 0 { // 关注或取消关注
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.SkuID),
StockQty: model.MaxStoreSkuStockQty,
}
skuStockList = append(skuStockList, stock)
if storeSku.DeletedAt != utils.DefaultTimeValue {
stock.StockQty = 0
} else {
alreadyAddStock = true
}
if stock.StockQty != 0 || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuStockList = append(skuStockList, stock)
}
}
if vendibility.DoSale || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuVendibilityList = append(skuVendibilityList, vendibility)
if storeSku.JdSyncStatus&(model.SyncFlagPriceMask|model.SyncFlagNewMask) != 0 {
skuPriceInfoList = append(skuPriceInfoList, &jdapi.SkuPriceInfo{
OutSkuId: utils.Int2Str(storeSku.SkuID),
Price: jxutils.CaculateSkuVendorPrice(storeSku.Price, storeSku.PricePercentage),
})
}
if storeSku.JdSyncStatus&(model.SyncFlagSaleMask|model.SyncFlagNewMask) != 0 {
vendibility := &jdapi.StockVendibility{
OutSkuId: utils.Int2Str(storeSku.SkuID),
DoSale: true,
}
if storeSku.Status != model.StoreSkuBindStatusNormal {
vendibility.DoSale = false
} else if !alreadyAddStock { // 如果是设置可售则自动将库存加满
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.SkuID),
StockQty: model.MaxStoreSkuStockQty,
}
skuStockList = append(skuStockList, stock)
}
if vendibility.DoSale || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuVendibilityList = append(skuVendibilityList, vendibility)
}
}
}
}
}
syncMask := 0
errList := []error{}
if globals.EnableStoreWrite {
// todo 以下可以优化为并行操作
globals.SugarLogger.Debug(utils.Format4Output(skuVendibilityList, false), utils.Format4Output(skuPriceInfoList, false), utils.Format4Output(skuStockList, false))
if len(skuVendibilityList) > 0 {
if _, err = api.JdAPI.BatchUpdateVendibility("", stationNo, skuVendibilityList, ctx.GetUserName()); err == nil {
syncMask |= model.SyncFlagSaleMask
} else {
errList = append(errList, err)
syncMask := 0
errList := []error{}
if globals.EnableStoreWrite {
// todo 以下可以优化为并行操作
globals.SugarLogger.Debug(utils.Format4Output(skuVendibilityList, false), utils.Format4Output(skuPriceInfoList, false), utils.Format4Output(skuStockList, false))
if len(skuVendibilityList) > 0 {
if _, err = api.JdAPI.BatchUpdateVendibility("", stationNo, skuVendibilityList, ctx.GetUserName()); err == nil {
syncMask |= model.SyncFlagSaleMask
} else {
errList = append(errList, err)
}
}
if (err == nil || isContinueWhenError) && len(skuStockList) > 0 {
if _, err = api.JdAPI.BatchUpdateCurrentQtys("", stationNo, skuStockList, ctx.GetUserName()); err == nil {
syncMask |= model.SyncFlagNewMask | model.SyncFlagDeletedMask
} else {
errList = append(errList, err)
}
}
if (err == nil || isContinueWhenError) && len(skuPriceInfoList) > 0 {
if _, err = api.JdAPI.UpdateVendorStationPrice("", stationNo, skuPriceInfoList); err == nil {
syncMask |= model.SyncFlagPriceMask
} else {
errList = append(errList, err)
}
}
}
if (err == nil || isContinueWhenError) && len(skuStockList) > 0 {
if _, err = api.JdAPI.BatchUpdateCurrentQtys("", stationNo, skuStockList, ctx.GetUserName()); err == nil {
syncMask |= model.SyncFlagNewMask | model.SyncFlagDeletedMask
} else {
errList = append(errList, err)
}
if len(errList) == 0 {
syncMask = -1
}
if (err == nil || isContinueWhenError) && len(skuPriceInfoList) > 0 {
if _, err = api.JdAPI.UpdateVendorStationPrice("", stationNo, skuPriceInfoList); err == nil {
syncMask |= model.SyncFlagPriceMask
} else {
errList = append(errList, err)
}
}
}
if len(errList) == 0 {
syncMask = -1
}
if syncMask != 0 && len(batchSkuIDs) > 0 {
db := dao.GetDB() // 多线程问题
sql := `
if syncMask != 0 && len(batchSkuIDs) > 0 {
db := dao.GetDB() // 多线程问题
sql := `
UPDATE store_sku_bind t1
SET t1.jd_sync_status = t1.jd_sync_status & ?
` + sqlWhere0 + " AND t1.sku_id IN (" + dao.GenQuestionMarks(len(batchSkuIDs)) + ")"
if _, err = dao.ExecuteSQL(db, sql, ^syncMask, storeID, batchSkuIDs); err != nil {
errList = append(errList, err)
if _, err = dao.ExecuteSQL(db, sql, ^syncMask, storeID, batchSkuIDs); err != nil {
errList = append(errList, err)
}
}
}
if len(errList) == 1 {
err = errList[0]
} else if len(errList) > 1 {
err = fmt.Errorf("%v", errList)
}
return nil, err
}, storeSkus)
ctx.SetTaskOrAddChild(task, parentTask)
task.Run()
if len(errList) == 1 {
err = errList[0]
} else if len(errList) > 1 {
err = fmt.Errorf("%v", errList)
}
return nil, err
}, storeSkus)
tasksch.HandleTask(task, parentTask, false).Run()
if !isAsync {
_, err = task.GetResult(0)
}
@@ -185,83 +185,83 @@ func (p *PurchaseHandler) syncStoreSkus(ctx *jxcontext.Context, parentTask tasks
return "", err
}
stationNo := storeDetail.VendorStoreID
task := tasksch.NewParallelTask("SyncStoresSkus京东", tasksch.NewParallelConfig().SetBatchSize(jdapi.MaxStoreSkuBatchSize).SetIsContinueWhenError(isContinueWhenError), ctx.GetUserName(), func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (interface{}, error) {
var skuPriceInfoList []*jdapi.SkuPriceInfo
var skuVendibilityList []*jdapi.StockVendibility
var skuStockList []*jdapi.SkuStock
var batchSkuIDs []int
for _, v := range batchItemList {
storeSku := v.(*dao.StoreSkuSyncInfo)
alreadyAddStock := false
if storeSku.SkuSyncStatus&model.SyncFlagChangedMask != 0 || storeSku.BindID == 0 {
if storeSku.BindID != 0 {
batchSkuIDs = append(batchSkuIDs, storeSku.BindID)
}
if storeSku.SkuSyncStatus&(model.SyncFlagDeletedMask|model.SyncFlagNewMask) != 0 || storeSku.BindID == 0 { // 关注或取消关注
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.ID),
StockQty: model.MaxStoreSkuStockQty,
task := tasksch.NewParallelTask("SyncStoresSkus京东", tasksch.NewParallelConfig().SetBatchSize(jdapi.MaxStoreSkuBatchSize).SetIsContinueWhenError(isContinueWhenError), ctx,
func(t *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (interface{}, error) {
var skuPriceInfoList []*jdapi.SkuPriceInfo
var skuVendibilityList []*jdapi.StockVendibility
var skuStockList []*jdapi.SkuStock
var batchSkuIDs []int
for _, v := range batchItemList {
storeSku := v.(*dao.StoreSkuSyncInfo)
alreadyAddStock := false
if storeSku.SkuSyncStatus&model.SyncFlagChangedMask != 0 || storeSku.BindID == 0 {
if storeSku.BindID != 0 {
batchSkuIDs = append(batchSkuIDs, storeSku.BindID)
}
if storeSku.DeletedAt != utils.DefaultTimeValue || storeSku.BindID == 0 {
stock.StockQty = 0
} else {
alreadyAddStock = true
}
if stock.StockQty != 0 || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuStockList = append(skuStockList, stock)
}
}
if storeSku.SkuSyncStatus&(model.SyncFlagPriceMask|model.SyncFlagNewMask) != 0 {
skuPriceInfoList = append(skuPriceInfoList, &jdapi.SkuPriceInfo{
OutSkuId: utils.Int2Str(storeSku.ID),
Price: jxutils.CaculateSkuVendorPrice(int(storeSku.Price), int(storeDetail.PricePercentage)),
})
}
if storeSku.SkuSyncStatus&(model.SyncFlagSaleMask|model.SyncFlagNewMask) != 0 {
vendibility := &jdapi.StockVendibility{
OutSkuId: utils.Int2Str(storeSku.ID),
DoSale: true,
}
if storeSku.StoreSkuStatus != model.StoreSkuBindStatusNormal {
vendibility.DoSale = false
} else if !alreadyAddStock { // 如果是设置可售则自动将库存加满
if storeSku.SkuSyncStatus&(model.SyncFlagDeletedMask|model.SyncFlagNewMask) != 0 || storeSku.BindID == 0 { // 关注或取消关注
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.ID),
StockQty: model.MaxStoreSkuStockQty,
}
skuStockList = append(skuStockList, stock)
if storeSku.DeletedAt != utils.DefaultTimeValue || storeSku.BindID == 0 {
stock.StockQty = 0
} else {
alreadyAddStock = true
}
if stock.StockQty != 0 || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuStockList = append(skuStockList, stock)
}
}
if vendibility.DoSale || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuVendibilityList = append(skuVendibilityList, vendibility)
if storeSku.SkuSyncStatus&(model.SyncFlagPriceMask|model.SyncFlagNewMask) != 0 {
skuPriceInfoList = append(skuPriceInfoList, &jdapi.SkuPriceInfo{
OutSkuId: utils.Int2Str(storeSku.ID),
Price: jxutils.CaculateSkuVendorPrice(int(storeSku.Price), int(storeDetail.PricePercentage)),
})
}
if storeSku.SkuSyncStatus&(model.SyncFlagSaleMask|model.SyncFlagNewMask) != 0 {
vendibility := &jdapi.StockVendibility{
OutSkuId: utils.Int2Str(storeSku.ID),
DoSale: true,
}
if storeSku.StoreSkuStatus != model.StoreSkuBindStatusNormal {
vendibility.DoSale = false
} else if !alreadyAddStock { // 如果是设置可售则自动将库存加满
stock := &jdapi.SkuStock{
OutSkuId: utils.Int2Str(storeSku.ID),
StockQty: model.MaxStoreSkuStockQty,
}
skuStockList = append(skuStockList, stock)
}
if vendibility.DoSale || !storeskulock.IsJdStoreSkuLocked(stationNo, storeSku.JdID) {
skuVendibilityList = append(skuVendibilityList, vendibility)
}
}
}
}
}
globals.SugarLogger.Debugf("jd syncStoreSkus sync detail, storeID:%d, skuVendibilityList:%s, skuPriceInfoList:%s, skuStockList:%s", storeID, utils.Format4Output(skuVendibilityList, true), utils.Format4Output(skuPriceInfoList, true), utils.Format4Output(skuStockList, true))
if globals.EnableStoreWrite {
// todo 以下可以优化为并行操作
if len(skuVendibilityList) > 0 {
_, err = api.JdAPI.BatchUpdateVendibility("", stationNo, skuVendibilityList, ctx.GetUserName())
globals.SugarLogger.Debugf("jd syncStoreSkus sync detail, storeID:%d, skuVendibilityList:%s, skuPriceInfoList:%s, skuStockList:%s", storeID, utils.Format4Output(skuVendibilityList, true), utils.Format4Output(skuPriceInfoList, true), utils.Format4Output(skuStockList, true))
if globals.EnableStoreWrite {
// todo 以下可以优化为并行操作
if len(skuVendibilityList) > 0 {
_, err = api.JdAPI.BatchUpdateVendibility("", stationNo, skuVendibilityList, ctx.GetUserName())
}
if err == nil && len(skuStockList) > 0 {
_, err = api.JdAPI.BatchUpdateCurrentQtys("", stationNo, skuStockList, ctx.GetUserName())
}
if err == nil && len(skuPriceInfoList) > 0 {
_, err = api.JdAPI.UpdateVendorStationPrice("", stationNo, skuPriceInfoList)
}
}
if err == nil && len(skuStockList) > 0 {
_, err = api.JdAPI.BatchUpdateCurrentQtys("", stationNo, skuStockList, ctx.GetUserName())
}
if err == nil && len(skuPriceInfoList) > 0 {
_, err = api.JdAPI.UpdateVendorStationPrice("", stationNo, skuPriceInfoList)
}
}
if err == nil && len(batchSkuIDs) > 0 {
db := dao.GetDB() // 多线程问题
sql := `
if err == nil && len(batchSkuIDs) > 0 {
db := dao.GetDB() // 多线程问题
sql := `
UPDATE store_sku_bind t1
SET t1.jd_sync_status = 0
WHERE t1.id IN (` + dao.GenQuestionMarks(len(batchSkuIDs)) + ")"
_, err = dao.ExecuteSQL(db, sql, batchSkuIDs)
}
return nil, err
}, skus)
ctx.SetTaskOrAddChild(task, parentTask)
task.Run()
_, err = dao.ExecuteSQL(db, sql, batchSkuIDs)
}
return nil, err
}, skus)
tasksch.HandleTask(task, parentTask, false).Run()
if !isAsync {
_, err = task.GetResult(0)
}