liulei/jx-callback
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

上传图片至平台的并行数调整为2，经常出timeout，带宽不够

Browse Source
This commit is contained in:
gazebo
2019-09-26 17:01:11 +08:00
parent de5e69685e
commit 39051c2189
3 changed files with 1 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
business/jxstore/initdata/initdata.go
View File

@@ -286,61 +286,6 @@ func InitVendorCategory(ctx *jxcontext.Context, vendorID int) (num int64, err er
return num, err
}
func UploadWeimobImg4SkuName(ctx *jxcontext.Context, nameIDs []int, isAsync, isContinueWhenError bool) (hint string, err error) {
// db := dao.GetDB()
// skuNameList, err := dao.GetSkuNames(db, nameIDs)
// if err != nil {
// return "", err
// }
// rootTask := tasksch.NewParallelTask("UploadWeimobImg4SkuName", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
// func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
// skuName := batchItemList[0].(*model.SkuName)
// if skuName.Img != "" && skuName.ImgWeimob == "" {
// if globals.EnableWscStoreWrite {
// if skuName.ImgWeimob, err = api.WeimobAPI.UploadImgByURL(skuName.Img, ""); err == nil {
// _, err = dao.UpdateEntity(db, skuName, "ImgWeimob")
// }
// }
// }
// return nil, err
// }, skuNameList)
// tasksch.ManageTask(rootTask).Run()
// if !isAsync {
// _, err = rootTask.GetResult(0)
// } else {
// hint = rootTask.ID
// }
return hint, err
}
func UploadEbaiImg4SkuName(ctx *jxcontext.Context, nameIDs []int, isAsync, isContinueWhenError bool) (hint string, err error) {
db := dao.GetDB()
skuNameList, err := dao.GetSkuNames(db, nameIDs)
if err != nil {
return "", err
}
rootTask := tasksch.NewParallelTask("UploadEbaiImg4SkuName", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuName := batchItemList[0].(*model.SkuName)
if skuName.Img != "" && skuName.ImgEbai == "" {
if globals.EnableEbaiStoreWrite {
if skuName.ImgEbai, err = api.EbaiAPI.PictureUpload(skuName.Img, nil); err == nil {
_, err = dao.UpdateEntity(db, skuName, "ImgEbai")
}
}
}
return nil, err
}, skuNameList)
tasksch.ManageTask(rootTask).Run()
if !isAsync {
_, err = rootTask.GetResult(0)
} else {
hint = rootTask.ID
}
return hint, err
}
func UploadImg4Vendors(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hint string, err error) {
db := dao.GetDB()
rootTask := tasksch.NewSeqTask("合并SkuName图片至DataResource", ctx,
@@ -402,7 +347,7 @@ func UploadImg4Vendors(ctx *jxcontext.Context, isAsync, isContinueWhenError bool
dataResList, err := dao.GetNeedUploadDataResource(db)
if err == nil && len(dataResList) > 0 {
uploadTask := tasksch.NewParallelTask("批量上传图片至平台",
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(2), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
dataRes := batchItemList[0].(*model.DataResource)
_, err = datares.UploadImage2Vendors(ctx, task, dataRes, nil, false)

41
controllers/init_data.go
View File

@@ -2,7 +2,6 @@ package controllers
import (
"git.rosy.net.cn/jx-callback/business/jxstore/initdata"
"git.rosy.net.cn/jx-callback/business/jxutils"
"github.com/astaxie/beego"
)
@@ -53,46 +52,6 @@ func (c *InitDataController) InitVendorCategory() {
})
}
// @Title 将SkuName的图片上传到微盟
// @Description 将SkuName的图片上传到微盟
// @Param token header string true "认证token"
// @Param nameIDs formData string false "skuNameID"
// @Param isAsync formData bool false "是否异步操作"
// @Param isContinueWhenError formData bool false "单个同步失败是否继续，缺省false"
// @Success 200 {object} controllers.CallResult
// @Failure 200 {object} controllers.CallResult
// @router /UploadWeimobImg4SkuName [post]
func (c *InitDataController) UploadWeimobImg4SkuName() {
c.callUploadWeimobImg4SkuName(func(params *tInitdataUploadWeimobImg4SkuNameParams) (retVal interface{}, errCode string, err error) {
var nameIDs []int
if err = jxutils.Strings2Objs(params.NameIDs, &nameIDs); err != nil {
return retVal, "", err
}
retVal, err = initdata.UploadWeimobImg4SkuName(params.Ctx, nameIDs, params.IsAsync, params.IsContinueWhenError)
return retVal, "", err
})
}
// @Title 将SkuName的图片上传到微盟
// @Description 将SkuName的图片上传到微盟
// @Param token header string true "认证token"
// @Param nameIDs formData string false "skuNameID"
// @Param isAsync formData bool false "是否异步操作"
// @Param isContinueWhenError formData bool false "单个同步失败是否继续，缺省false"
// @Success 200 {object} controllers.CallResult
// @Failure 200 {object} controllers.CallResult
// @router /UploadEbaiImg4SkuName [post]
func (c *InitDataController) UploadEbaiImg4SkuName() {
c.callUploadEbaiImg4SkuName(func(params *tInitdataUploadEbaiImg4SkuNameParams) (retVal interface{}, errCode string, err error) {
var nameIDs []int
if err = jxutils.Strings2Objs(params.NameIDs, &nameIDs); err != nil {
return retVal, "", err
}
retVal, err = initdata.UploadEbaiImg4SkuName(params.Ctx, nameIDs, params.IsAsync, params.IsContinueWhenError)
return retVal, "", err
})
}
// @Title 将资源上传到平台（当前为饿百与美团）
// @Description 将资源上传到平台（当前为饿百与美团）
// @Param token header string true "认证token"

18
routers/commentsRouter_controllers.go
View File

@@ -556,15 +556,6 @@ func init() {
Filters: nil,
Params: nil})
beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"] = append(beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"],
beego.ControllerComments{
Method: "UploadEbaiImg4SkuName",
Router: `/UploadEbaiImg4SkuName`,
AllowHTTPMethods: []string{"post"},
MethodParams: param.Make(),
Filters: nil,
Params: nil})
beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"] = append(beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"],
beego.ControllerComments{
Method: "UploadImage4Vendors",
@@ -574,15 +565,6 @@ func init() {
Filters: nil,
Params: nil})
beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"] = append(beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:InitDataController"],
beego.ControllerComments{
Method: "UploadWeimobImg4SkuName",
Router: `/UploadWeimobImg4SkuName`,
AllowHTTPMethods: []string{"post"},
MethodParams: param.Make(),
Filters: nil,
Params: nil})
beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:JxShopController"] = append(beego.GlobalControllerRouter["git.rosy.net.cn/jx-callback/controllers:JxShopController"],
beego.ControllerComments{
Method: "JxMsg",