From 67f1c3bf770b39e56c198da2ceb39152c4f63ce0 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 09:12:49 +0800 Subject: [PATCH 01/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8DUploadImg4Vendors?= =?UTF-8?q?=E4=B8=ADImgHashCode=E5=8F=AF=E8=83=BD=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/initdata/initdata.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/business/jxstore/initdata/initdata.go b/business/jxstore/initdata/initdata.go index ae9977988..c31c387bc 100644 --- a/business/jxstore/initdata/initdata.go +++ b/business/jxstore/initdata/initdata.go @@ -324,6 +324,12 @@ func UploadImg4Vendors(ctx *jxcontext.Context, isAsync, isContinueWhenError bool CONCAT(t1.name, '_desc'), desc_img main_url, t1.desc_img_ebai ebai_url, IF(INSTR(t1.desc_img, "image.jxc4.com") > 0, t1.desc_img, '') qiniu_url, 2 FROM sku_name t1 + JOIN ( + SELECT img_hash_code, MAX(id) id, COUNT(*) ct + FROM sku_name + WHERE img_hash_code <> '' AND desc_img <> '' + GROUP BY 1 + ) t3 ON t3.id = t1.id LEFT JOIN data_resource t2 ON (t2.main_url <> '' AND t2.main_url = t1.desc_img) WHERE t1.desc_img <> '' AND t1.img_hash_code <> '' AND t2.id IS NULL; `) From 88fa351cbb2eaf8b4285625192dd655564b88456 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 09:23:11 +0800 Subject: [PATCH 02/12] =?UTF-8?q?=E5=B0=86DataResource.IsVendor=E6=94=B9?= =?UTF-8?q?=E4=B8=BAUseType?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/initdata/initdata.go | 2 +- business/jxutils/datares/datares.go | 4 ++-- business/model/common.go | 2 +- business/model/dao/common.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/business/jxstore/initdata/initdata.go b/business/jxstore/initdata/initdata.go index c31c387bc..0f4da6137 100644 --- a/business/jxstore/initdata/initdata.go +++ b/business/jxstore/initdata/initdata.go @@ -309,7 +309,7 @@ func UploadImg4Vendors(ctx *jxcontext.Context, isAsync, isContinueWhenError bool case 1: // 从SkuName添加缺失的图片至DataResource _, err = dao.ExecuteSQL(db, ` INSERT INTO data_resource(created_at, updated_at, last_operator, hash_code, - resource_type, name, main_url, ebai_url, qiniu_url, is_vendor) + resource_type, name, main_url, ebai_url, qiniu_url, use_type) SELECT t1.created_at, t1.created_at, t1.last_operator, t1.img_hash_code, CASE WHEN INSTR(t1.desc_img, ".jpg") > 0 OR INSTR(t1.desc_img, ".jpeg") > 0 THEN diff --git a/business/jxutils/datares/datares.go b/business/jxutils/datares/datares.go index 6f4534548..d0d87d7be 100644 --- a/business/jxutils/datares/datares.go +++ b/business/jxutils/datares/datares.go @@ -84,7 +84,7 @@ func RegisterDataResource(ctx *jxcontext.Context, name, resourceURL, mimeType, h HashCode: hashCode, ResourceType: mimeType, MainURL: resourceURL, - IsVendor: int8(imgType), + UseType: int8(imgType), } vendorID := jxutils.GuessDataResourceVendor(resourceURL) @@ -179,7 +179,7 @@ func UploadImage2Vendors(ctx *jxcontext.Context, parentTask tasksch.ITask, dataR func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { vendorID := batchItemList[0].(int) if handler := partner.GetPurchasePlatformFromVendorID(vendorID); handler != nil { - imgHint, err2 := handler.UploadImg(ctx, dataRes.MainURL, imgData, imgName, int(dataRes.IsVendor)) + imgHint, err2 := handler.UploadImg(ctx, dataRes.MainURL, imgData, imgName, int(dataRes.UseType)) if err = err2; err == nil { retVal = [][]interface{}{ []interface{}{ diff --git a/business/model/common.go b/business/model/common.go index 16ce08a1e..d9c36a28a 100644 --- a/business/model/common.go +++ b/business/model/common.go @@ -23,7 +23,7 @@ type DataResource struct { ResourceType string `orm:"size(48)" json:"resourceType"` // 资料的mime type Name string `orm:"size(48);index" json:"name"` - IsVendor int8 `json:"isVendor"` // 是否是平台需要的（平台需要才会上传到平台） + UseType int8 `json:"isVendor"` // 图资源使用方式（大于0才可能需要上传至平台） MainURL string `orm:"size(512);column(main_url);unique" json:"mainURL"` QiniuURL string `orm:"size(512);column(qiniu_url);index" json:"qiniuURL"` diff --git a/business/model/dao/common.go b/business/model/dao/common.go index fe36d8662..f3dbab71a 100644 --- a/business/model/dao/common.go +++ b/business/model/dao/common.go @@ -26,7 +26,7 @@ func GetNeedUploadDataResource(db *DaoDB) (dataResList []*model.DataResource, er sql := ` SELECT t1.* FROM data_resource t1 - WHERE t1.is_vendor <> 0 AND (t1.ebai_url = '' OR t1.mtwm_url = '')` + WHERE t1.use_type <> 0 AND (t1.ebai_url = '' OR t1.mtwm_url = '')` err = GetRows(db, &dataResList, sql) return dataResList, err } From 366a6afc63e96503d170e83fd787f26152d24c44 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 09:32:25 +0800 Subject: [PATCH 03/12] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E5=AF=B9=E4=BA=8ESkuNa?= =?UTF-8?q?me.DescImgEbai=E7=9A=84=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/model/dao/store_sku.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/business/model/dao/store_sku.go b/business/model/dao/store_sku.go index f88a6d6c2..bfcd21adc 100644 --- a/business/model/dao/store_sku.go +++ b/business/model/dao/store_sku.go @@ -17,9 +17,9 @@ var ( model.VendorIDMTWM: "mtwm_url", model.VendorIDEBAI: "ebai_url", } - descImgFieldMap = map[int]string{ - model.VendorIDEBAI: "desc_img_ebai", - } + // descImgFieldMap = map[int]string{ + // model.VendorIDEBAI: "desc_img_ebai", + // } ) type SkuStoreCatInfo struct { @@ -225,13 +225,13 @@ func GetStoreSkus2(db *DaoDB, vendorID, storeID int, skuIDs []int, isDirty bool) t3.id name_id, t3.prefix, t3.name, t3.unit, t3.upc, IF(t11.%s <> '', t11.%s, t3.img) img, IF(t12.%s <> '', t12.%s, t3.img2) img2, - IF(t13.%s <> '', t13.%s, t3.%s) desc_img, + t13.%s desc_img, t4.%s_category_id vendor_vendor_cat_id` fmtParams := []interface{}{ tableName, fieldPrefix, fieldPrefix, vendorSkuNameField, GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), - GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), GetDescImgFieldName(vendorID), + GetDataResFieldName(vendorID), fieldPrefix, } if isSingleStorePF { @@ -315,7 +315,7 @@ func GetFullStoreSkus(db *DaoDB, vendorID, storeID int) (skus []*StoreSkuSyncInf t3.id name_id, t3.prefix, t3.name, t3.unit, t3.upc, IF(t11.%s <> '', t11.%s, t3.img) img, IF(t12.%s <> '', t12.%s, t3.img2) img2, - IF(t13.%s <> '', t13.%s, t3.desc_img) desc_img, + t13.%s desc_img, t4.%s_category_id vendor_vendor_cat_id, t4.%s_sync_status store_cat_sync_status, t4.%s_id vendor_cat_id, t5sku.%s_sync_status sku_store_cat_sync_status, t5sku.%s_id sku_vendor_cat_id @@ -343,7 +343,7 @@ func GetFullStoreSkus(db *DaoDB, vendorID, storeID int) (skus []*StoreSkuSyncInf fieldPrefix, GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), - GetDataResFieldName(vendorID), GetDataResFieldName(vendorID), + GetDataResFieldName(vendorID), fieldPrefix, fieldPrefix, fieldPrefix, fieldPrefix, fieldPrefix, @@ -448,13 +448,13 @@ func GetDataResFieldName(vendorID int) (fieldName string) { return fieldName } -func GetDescImgFieldName(vendorID int) (fieldName string) { - fieldName = descImgFieldMap[vendorID] - if fieldName == "" { - fieldName = "desc_img" - } - return fieldName -} +// func GetDescImgFieldName(vendorID int) (fieldName string) { +// fieldName = descImgFieldMap[vendorID] +// if fieldName == "" { +// fieldName = "desc_img" +// } +// return fieldName +// } func GetStoresSkusInfo(db *DaoDB, storeIDs, skuIDs []int) (storeSkuList []*model.StoreSkuBind, err error) { sql := ` From 806202cba88128bb5277f45a32a84f8aeb1b63d5 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 09:39:56 +0800 Subject: [PATCH 04/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8Debai.UploadImg=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E8=AF=AF=E5=86=99ImgTypeMain=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/partner/purchase/ebai/ebai.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/business/partner/purchase/ebai/ebai.go b/business/partner/purchase/ebai/ebai.go index 5fe6874ca..c5cd5fe05 100644 --- a/business/partner/purchase/ebai/ebai.go +++ b/business/partner/purchase/ebai/ebai.go @@ -54,7 +54,7 @@ func (p *PurchaseHandler) UploadImg(ctx *jxcontext.Context, imgURL string, imgDa if globals.EnableEbaiStoreWrite { if imgType == model.ImgTypeMain { imgHint, err = api.EbaiAPI.PictureUpload(imgURL, imgData) - } else if imgType == model.ImgTypeMain { + } else if imgType == model.ImgTypeDesc { imgHint, err = api.EbaiAPI.SkuUploadRTF(p.getShopID4UploadRTF(), ebaiapi.BuildRFTFromImgs(imgURL)) } } else { From 918dda4851b3e6c44d6e816446b13a0172c28dde Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 09:59:31 +0800 Subject: [PATCH 05/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=97=A8=E5=BA=97?= =?UTF-8?q?=E9=85=8D=E9=80=81=E8=8C=83=E5=9B=B4=E6=97=B6=EF=BC=8C=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E6=95=B0=E6=8D=AE=E5=90=88=E6=B3=95=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/cms/store.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/business/jxstore/cms/store.go b/business/jxstore/cms/store.go index a48bdd0f0..0a63ad7ff 100644 --- a/business/jxstore/cms/store.go +++ b/business/jxstore/cms/store.go @@ -524,6 +524,20 @@ func checkCreateStore(store *model.Store) (err error) { return err } +func checkStoreDeliveryRange(deliveryRange string) (err error) { + if deliveryRange != "" { + points := jxutils.CoordinateStr2Points(deliveryRange) + for _, point := range points { + for _, coord := range point { + if coord == 0 || math.IsNaN(coord) { + return fmt.Errorf("坐标点数据非法:%s", deliveryRange) + } + } + } + } + return err +} + func UpdateStore(ctx *jxcontext.Context, storeID int, payload map[string]interface{}, userName string) (num int64, err error) { globals.SugarLogger.Debugf("UpdateStore storeID:%d, payload:%s", storeID, utils.Format4Output(payload, true)) if err = checkBankBranch(utils.Interface2String(payload["payeeBankBranchName"])); err != nil { @@ -537,6 +551,9 @@ func UpdateStore(ctx *jxcontext.Context, storeID int, payload map[string]interfa } valid := dao.StrictMakeMapByStructObject(payload, store, userName) + if err = checkStoreDeliveryRange(utils.Interface2String(valid["deliveryRange"])); err != nil { + return 0, err + } if globals.EnableWXAuth2 { if err = dao.ValidateRoles(db, utils.Interface2String(valid["marketManRole"]), utils.Interface2String(valid["operatorRole"]), utils.Interface2String(valid["operatorRole2"])); err != nil { return 0, err From c820a90bb7b5bc0d18922f92d9eebfa81fc325dc Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 10:21:00 +0800 Subject: [PATCH 06/12] up --- business/jxcallback/scheduler/basesch/basesch_ext.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/business/jxcallback/scheduler/basesch/basesch_ext.go b/business/jxcallback/scheduler/basesch/basesch_ext.go index 35d1e3fa8..ba83953db 100644 --- a/business/jxcallback/scheduler/basesch/basesch_ext.go +++ b/business/jxcallback/scheduler/basesch/basesch_ext.go @@ -55,7 +55,7 @@ func (c *BaseScheduler) CreateWaybillOnProviders(ctx *jxcontext.Context, order * } else if errList.GetErrListAsOne() == nil { err = fmt.Errorf("orderID:%s没有绑定有效的三方配送门店或没有剩下可用的三方配送", order.VendorOrderID) } else { - err = fmt.Errorf("orderID:%s所有运单失败：\n%s", order.VendorOrderID, errList.GetErrListAsOne().Error()) + err = fmt.Errorf("orderID:%s所有运单失败：%s", order.VendorOrderID, errList.GetErrListAsOne().Error()) } globals.SugarLogger.Infof("CreateWaybillOnProviders orderID:%s userName:%s error:%v", order.VendorOrderID, userName, err) return bills, err From 2ed3b3fa64796411c990a77c459dcefe3ddeb56a Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 12:08:27 +0800 Subject: [PATCH 07/12] tempop.CheckImages --- business/jxstore/tempop/tempop.go | 81 +++++++++++++++++++++++++++++++ business/model/common.go | 2 + controllers/temp_op.go | 31 ++---------- 3 files changed, 86 insertions(+), 28 deletions(-) diff --git a/business/jxstore/tempop/tempop.go b/business/jxstore/tempop/tempop.go index a8b3eb70a..cdb35ce78 100644 --- a/business/jxstore/tempop/tempop.go +++ b/business/jxstore/tempop/tempop.go @@ -1,7 +1,11 @@ package tempop import ( + "bytes" "fmt" + "image" + "image/jpeg" + "image/png" "regexp" "sync" "time" @@ -1103,3 +1107,80 @@ func CreateConsumerFromOrders(ctx *jxcontext.Context, vendorIDs []int, fromDate, } return hint, err } + +func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hint string, err error) { + db := dao.GetDB() + rootTask := tasksch.NewSeqTask("检查图片的有效性", ctx, + func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) { + switch step { + case 0: // 计算SkuName中缺失的hashCode（注意是DescImg不是Img） + var imgList []*model.DataResource + if err = dao.GetRows(db, &imgList, ` + SELECT t1.* + FROM data_resource t1 + WHERE t1.use_type = 1 + `); err == nil && len(imgList) > 0 { + calcTask := tasksch.NewParallelTask("检查京西图片的有效性", + tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, + func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { + dataRes := batchItemList[0].(*model.DataResource) + binaryData, hashCode, err2 := jxutils.DownloadFileByURL(dataRes.MainURL) + if err = err2; err == nil { + errMsg := "" + if hashCode != dataRes.HashCode { + errMsg += fmt.Sprintf("hashcode:%s", hashCode) + } + var img image.Image + if dataRes.ResourceType == "image/png" { + img, err = png.Decode(bytes.NewReader(binaryData)) + } else if dataRes.ResourceType == "image/jpeg" { + img, err = jpeg.Decode(bytes.NewReader(binaryData)) + } + if err == nil { + if img.Bounds().Dx() != 800 || img.Bounds().Dy() != 800 { + errMsg += fmt.Sprintf("size:%dx%d", img.Bounds().Dx(), img.Bounds().Dy()) + } + } + if errMsg != "" { + dataRes.Remark += errMsg + dao.UpdateEntity(db, dataRes, "Remark") + } + } + return retVal, err + }, imgList) + tasksch.HandleTask(calcTask, task, false).Run() + _, err = calcTask.GetResult(0) + } + case 1: // 从SkuName添加缺失的图片至DataResource + skuNameInfo, err2 := cms.GetSkuNames(ctx, "", false, map[string]interface{}{}, 0, -1) + if err = err2; err == nil && len(skuNameInfo.SkuNames) > 0 { + calcTask := tasksch.NewParallelTask("检查京西图片的有效性", + tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, + func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { + skuName := batchItemList[0].(*model.SkuNameExt) + if len(skuName.Skus) > 0 { + imgList, err2 := api.JdAPI.GetSkuPageImageInfo(skuName.Skus[0].JdID) + if err = err2; err == nil { + if len(imgList) == 0 { + err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, skuName.Skus[0].ID, skuName.Skus[0].JdID) + } + } + } + return retVal, err + }, skuNameInfo.SkuNames) + tasksch.HandleTask(calcTask, task, false).Run() + _, err = calcTask.GetResult(0) + } + } + return result, err + }, 2) + tasksch.HandleTask(rootTask, nil, true).Run() + if !isAsync { + if _, err = rootTask.GetResult(0); err == nil { + hint = "1" + } + } else { + hint = rootTask.ID + } + return hint, err +} diff --git a/business/model/common.go b/business/model/common.go index d9c36a28a..6fde833cc 100644 --- a/business/model/common.go +++ b/business/model/common.go @@ -29,4 +29,6 @@ type DataResource struct { QiniuURL string `orm:"size(512);column(qiniu_url);index" json:"qiniuURL"` EbaiURL string `orm:"size(512);column(ebai_url);index" json:"ebaiURL"` MtwmURL string `orm:"size(512);column(mtwm_url);index" json:"mtwmURL"` + + Remark string `orm:"size(1024)" json:"remark"` } diff --git a/controllers/temp_op.go b/controllers/temp_op.go index bc3ea66ff..8a86a2a3e 100644 --- a/controllers/temp_op.go +++ b/controllers/temp_op.go @@ -1,12 +1,10 @@ package controllers import ( - "git.rosy.net.cn/baseapi/platformapi/mtwmapi" "git.rosy.net.cn/jx-callback/business/jxstore/cms" "git.rosy.net.cn/jx-callback/business/jxstore/misc" "git.rosy.net.cn/jx-callback/business/jxstore/tempop" "git.rosy.net.cn/jx-callback/business/jxutils" - "git.rosy.net.cn/jx-callback/business/partner/purchase/mtwm" "github.com/astaxie/beego" ) @@ -238,37 +236,14 @@ func (c *TempOpController) RefreshMtpsWaybillFee() { // @Title 查询三方运单费用信息 // @Description 查询三方运单费用信息 // @Param token header string true "认证token" +// @Param isAsync query bool false "是否异步操作" +// @Param isContinueWhenError query bool false "单个同步失败是否继续，缺省false" // @Success 200 {object} controllers.CallResult // @Failure 200 {object} controllers.CallResult // @router /TestIt [get] func (c *TempOpController) TestIt() { c.callTestIt(func(params *tTempopTestItParams) (retVal interface{}, errCode string, err error) { - // // retVal, err = cms.OpenRemoteStoreByJxStatus(params.Ctx, nil, nil, false, true, true) - // // shopList, err := api.EbaiAPI.ShopList(ebaiapi.SysStatusAll) - // shopList, err := dao.GetStoresMapList(dao.GetDB(), []int{model.VendorIDEBAI}, nil, model.StoreStatusAll, model.StoreIsSyncYes, "") - // task := tasksch.NewParallelTask("TestIt", nil, params.Ctx, - // func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { - // // shopID := batchItemList[0].(*ebaiapi.ShopInfo).ShopID - // shopID := utils.Int2Str(batchItemList[0].(*model.StoreMap).StoreID) - // skuInfo, err := api.EbaiAPI.SkuList(shopID, &ebaiapi.SkuListParams{}) - // if err == nil && skuInfo.Total > 500 { - // errSkuCount := 0 - // for _, sku := range skuInfo.List { - // if sku.CustomCatIDs == "" { - // errSkuCount++ - // } - // } - // if errSkuCount > len(skuInfo.List)/3 { - // retVal = []string{shopID} - // } - // } - // return retVal, err - // }, shopList) - // task.Run() - // retVal, err = task.GetResult(0) - mtwm.OnCallbackMsg(&mtwmapi.CallbackMsg{ - Cmd: mtwmapi.MsgTypePrivateNumberDowngrade, - }) + retVal, err = tempop.CheckImages(params.Ctx, params.IsAsync, params.IsContinueWhenError) return retVal, "", err }) } From 355a1ce72fd950a6b0cf523b80144689133d3ff8 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 14:30:46 +0800 Subject: [PATCH 08/12] up --- business/jxstore/tempop/tempop.go | 42 +++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/business/jxstore/tempop/tempop.go b/business/jxstore/tempop/tempop.go index cdb35ce78..1f138ddb0 100644 --- a/business/jxstore/tempop/tempop.go +++ b/business/jxstore/tempop/tempop.go @@ -6,6 +6,7 @@ import ( "image" "image/jpeg" "image/png" + "net/http" "regexp" "sync" "time" @@ -1113,12 +1114,12 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin rootTask := tasksch.NewSeqTask("检查图片的有效性", ctx, func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) { switch step { - case 0: // 计算SkuName中缺失的hashCode（注意是DescImg不是Img） + case 0: // 检查图片的格式与大小 var imgList []*model.DataResource if err = dao.GetRows(db, &imgList, ` SELECT t1.* FROM data_resource t1 - WHERE t1.use_type = 1 + WHERE t1.use_type = 1 AND t1.remark <> '' `); err == nil && len(imgList) > 0 { calcTask := tasksch.NewParallelTask("检查京西图片的有效性", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, @@ -1128,7 +1129,13 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin if err = err2; err == nil { errMsg := "" if hashCode != dataRes.HashCode { - errMsg += fmt.Sprintf("hashcode:%s", hashCode) + errMsg = fmt.Sprintf("hashcode:%s", hashCode) + dataRes.HashCode = hashCode + } + resType := http.DetectContentType(binaryData) + if resType != dataRes.ResourceType { + errMsg += "," + resType + dataRes.ResourceType = resType } var img image.Image if dataRes.ResourceType == "image/png" { @@ -1137,13 +1144,17 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin img, err = jpeg.Decode(bytes.NewReader(binaryData)) } if err == nil { - if img.Bounds().Dx() != 800 || img.Bounds().Dy() != 800 { - errMsg += fmt.Sprintf("size:%dx%d", img.Bounds().Dx(), img.Bounds().Dy()) + if img != nil { + if img.Bounds().Dx() != 800 || img.Bounds().Dy() != 800 { + errMsg += fmt.Sprintf(",size:%dx%d", img.Bounds().Dx(), img.Bounds().Dy()) + } } + } else { + errMsg += "," + err.Error() } if errMsg != "" { - dataRes.Remark += errMsg - dao.UpdateEntity(db, dataRes, "Remark") + dataRes.Remark = errMsg + dao.UpdateEntity(dao.GetDB(), dataRes, "Remark", "ResourceType", "HashCode") } } return retVal, err @@ -1151,19 +1162,24 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin tasksch.HandleTask(calcTask, task, false).Run() _, err = calcTask.GetResult(0) } - case 1: // 从SkuName添加缺失的图片至DataResource + case 1: // 检查京东商品图片 skuNameInfo, err2 := cms.GetSkuNames(ctx, "", false, map[string]interface{}{}, 0, -1) if err = err2; err == nil && len(skuNameInfo.SkuNames) > 0 { calcTask := tasksch.NewParallelTask("检查京西图片的有效性", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { skuName := batchItemList[0].(*model.SkuNameExt) - if len(skuName.Skus) > 0 { - imgList, err2 := api.JdAPI.GetSkuPageImageInfo(skuName.Skus[0].JdID) - if err = err2; err == nil { - if len(imgList) == 0 { - err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, skuName.Skus[0].ID, skuName.Skus[0].JdID) + for _, sku := range skuName.Skus { + if sku.JdID != 0 { + imgList, err2 := api.JdAPI.GetSkuPageImageInfo(sku.JdID) + if err = err2; err == nil { + if len(imgList) == 0 { + err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, sku.ID, sku.JdID) + skuName.ImgHashCode = "np" + dao.UpdateEntity(dao.GetDB(), &skuName.SkuName, "ImgHashCode") + } } + break } } return retVal, err From e6cdf9fb86663ea943d4ec6f02072a199b4a0813 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 14:53:41 +0800 Subject: [PATCH 09/12] up --- business/jxstore/tempop/tempop.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/business/jxstore/tempop/tempop.go b/business/jxstore/tempop/tempop.go index 1f138ddb0..6d70cc584 100644 --- a/business/jxstore/tempop/tempop.go +++ b/business/jxstore/tempop/tempop.go @@ -1119,7 +1119,7 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin if err = dao.GetRows(db, &imgList, ` SELECT t1.* FROM data_resource t1 - WHERE t1.use_type = 1 AND t1.remark <> '' + WHERE t1.use_type = 1 `); err == nil && len(imgList) > 0 { calcTask := tasksch.NewParallelTask("检查京西图片的有效性", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, @@ -1165,7 +1165,7 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin case 1: // 检查京东商品图片 skuNameInfo, err2 := cms.GetSkuNames(ctx, "", false, map[string]interface{}{}, 0, -1) if err = err2; err == nil && len(skuNameInfo.SkuNames) > 0 { - calcTask := tasksch.NewParallelTask("检查京西图片的有效性", + calcTask := tasksch.NewParallelTask("检查京东图片的有效性", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { skuName := batchItemList[0].(*model.SkuNameExt) From 02e14a1b1d84e40ac19ee9a60e4b7b6e8187340a Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 15:37:27 +0800 Subject: [PATCH 10/12] =?UTF-8?q?CheckImages=E4=B8=AD=EF=BC=8C=E5=A6=82?= =?UTF-8?q?=E6=9E=9C=E5=95=86=E5=93=81=E4=B8=8B=E6=9E=B6=EF=BC=8C=E5=88=99?= =?UTF-8?q?=E4=B8=8D=E6=8A=A5=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/tempop/tempop.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/business/jxstore/tempop/tempop.go b/business/jxstore/tempop/tempop.go index 6d70cc584..07a97c4d0 100644 --- a/business/jxstore/tempop/tempop.go +++ b/business/jxstore/tempop/tempop.go @@ -1171,13 +1171,11 @@ func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hin skuName := batchItemList[0].(*model.SkuNameExt) for _, sku := range skuName.Skus { if sku.JdID != 0 { - imgList, err2 := api.JdAPI.GetSkuPageImageInfo(sku.JdID) - if err = err2; err == nil { - if len(imgList) == 0 { - err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, sku.ID, sku.JdID) - skuName.ImgHashCode = "np" - dao.UpdateEntity(dao.GetDB(), &skuName.SkuName, "ImgHashCode") - } + skuPageInfo, err2 := api.JdAPI.GetSkuPageInfo(sku.JdID) + if err = err2; err == nil && skuPageInfo.FixedStatus && len(skuPageInfo.Image) == 0 { + err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, sku.ID, sku.JdID) + skuName.ImgHashCode = "np" + dao.UpdateEntity(dao.GetDB(), &skuName.SkuName, "ImgHashCode") } break } From f4c13e21591737c962d0b881abcab2bb5fc2fdd0 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 16:24:56 +0800 Subject: [PATCH 11/12] =?UTF-8?q?GetStoresSkus=E5=92=8CGetStoreSkus?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8F=82=E6=95=B0isAct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/cms/store_sku.go | 60 +++++++++++++++++++----- business/jxstore/cms/store_sku_check.go | 2 +- business/jxstore/misc/store_score.go | 2 +- business/jxstore/misc/store_sku_sales.go | 4 +- controllers/cms_store_sku.go | 6 ++- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/business/jxstore/cms/store_sku.go b/business/jxstore/cms/store_sku.go index fd81163cb..40bf983fe 100644 --- a/business/jxstore/cms/store_sku.go +++ b/business/jxstore/cms/store_sku.go @@ -175,16 +175,55 @@ var ( } ) -func GetStoreSkus(ctx *jxcontext.Context, storeID int, skuIDs []int, isFocus bool, keyword string, isBySku bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { - return GetStoresSkus(ctx, []int{storeID}, skuIDs, isFocus, keyword, isBySku, params, offset, pageSize) +func GetStoreSkus(ctx *jxcontext.Context, storeID int, skuIDs []int, isFocus bool, keyword string, isBySku, isAct bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { + return GetStoresSkus(ctx, []int{storeID}, skuIDs, isFocus, keyword, isBySku, isAct, params, offset, pageSize) } -func getGetStoresSkusBaseSQL(db *dao.DaoDB, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku bool, params map[string]interface{}) (sql string, sqlParams []interface{}, err error) { +func getGetStoresSkusBaseSQL(db *dao.DaoDB, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku, isAct bool, params map[string]interface{}) (sql string, sqlParams []interface{}, err error) { sql = ` FROM sku_name t1 JOIN sku t2 FORCE INDEX(PRIMARY) ON t1.id = t2.name_id AND t2.deleted_at = ?/* AND t2.status = ?*/ JOIN store t3 ON t3.deleted_at = ? ` + sqlParams = []interface{}{ + utils.DefaultTimeValue, + // model.SkuStatusNormal, + utils.DefaultTimeValue, + } + if isAct { + sql += ` + JOIN ( + SELECT t2.store_id, t2.sku_id, + MIN(IF(t3.actual_act_price = 0, NULL, t3.actual_act_price)) actual_act_price, MIN(IF(t2.earning_price = 0, NULL, t2.earning_price)) earning_price /*non-zero min value*/ + FROM act t1 + JOIN act_store_sku t2 ON t2.act_id = t1.id AND t2.deleted_at = ? + JOIN act_store_sku_map t3 ON t3.bind_id = t2.id AND t3.act_id = t1.id AND (t3.sync_status & ? = 0 OR t1.type = ?) + JOIN act_map t4 ON t4.act_id = t1.id AND t4.vendor_id = t3.vendor_id AND t4.deleted_at = ? AND (t4.sync_status & ? = 0 OR t1.type = ?) + WHERE t1.deleted_at = ? AND t1.status = ? AND NOT (t1.begin_at > ? OR t1.end_at < ?)` + sqlParams = append(sqlParams, []interface{}{ + utils.DefaultTimeValue, + model.SyncFlagNewMask, + model.ActSkuFake, + utils.DefaultTimeValue, + model.SyncFlagNewMask, + model.ActSkuFake, + utils.DefaultTimeValue, + model.ActStatusCreated, + time.Now(), + time.Now(), + }) + if len(storeIDs) > 0 { + sql += " AND t2.store_id IN (" + dao.GenQuestionMarks(len(storeIDs)) + ")" + sqlParams = append(sqlParams, storeIDs) + } + if len(skuIDs) > 0 { + sql += " AND t2.sku_id IN (" + dao.GenQuestionMarks(len(skuIDs)) + ")" + sqlParams = append(sqlParams, skuIDs) + } + sql += ` + GROUP BY 1,2 + ) ta ON ta.store_id = t3.id AND ta.sku_id = t2.id` + } if !isFocus { sql += " LEFT" } @@ -193,15 +232,12 @@ func getGetStoresSkusBaseSQL(db *dao.DaoDB, storeIDs, skuIDs []int, isFocus bool LEFT JOIN sku_name_place_bind t5 ON t1.id = t5.name_id AND t3.city_code = t5.place_code WHERE t1.deleted_at = ? AND (t1.is_global = 1 OR t5.id IS NOT NULL OR 1 = ?)/* AND t1.status = ?*/ ` - sqlParams = []interface{}{ - utils.DefaultTimeValue, - // model.SkuStatusNormal, - utils.DefaultTimeValue, + sqlParams = append(sqlParams, []interface{}{ utils.DefaultTimeValue, utils.DefaultTimeValue, utils.Bool2Int(isFocus), // model.SkuStatusNormal, - } + }) if isFocus { sql += " AND ((t2.status = ? AND t1.status = ?) OR t4.status = ?)" sqlParams = append(sqlParams, model.SkuStatusNormal, model.SkuStatusNormal, model.SkuStatusNormal) @@ -318,11 +354,11 @@ func getGetStoresSkusBaseSQL(db *dao.DaoDB, storeIDs, skuIDs []int, isFocus bool return sql, sqlParams, err } -func GetStoresSkus(ctx *jxcontext.Context, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { - return GetStoresSkusNew(ctx, storeIDs, skuIDs, isFocus, keyword, isBySku, params, offset, pageSize) +func GetStoresSkus(ctx *jxcontext.Context, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku, isAct bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { + return GetStoresSkusNew(ctx, storeIDs, skuIDs, isFocus, keyword, isBySku, isAct, params, offset, pageSize) } -func GetStoresSkusNew(ctx *jxcontext.Context, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { +func GetStoresSkusNew(ctx *jxcontext.Context, storeIDs, skuIDs []int, isFocus bool, keyword string, isBySku, isAct bool, params map[string]interface{}, offset, pageSize int) (skuNamesInfo *StoreSkuNamesInfo, err error) { if !isFocus && !isBySku && (len(storeIDs) > 1 || len(storeIDs) == 0) { return nil, fmt.Errorf("未关注按SkuName只能查询单店") } @@ -330,7 +366,7 @@ func GetStoresSkusNew(ctx *jxcontext.Context, storeIDs, skuIDs []int, isFocus bo return nil, fmt.Errorf("GetStoresSkus必须指定storeIDs或skuIDs或分页") } db := dao.GetDB() - sql, sqlParams, err := getGetStoresSkusBaseSQL(db, storeIDs, skuIDs, isFocus, keyword, isBySku, params) + sql, sqlParams, err := getGetStoresSkusBaseSQL(db, storeIDs, skuIDs, isFocus, keyword, isBySku, isAct, params) if err != nil { return nil, err } diff --git a/business/jxstore/cms/store_sku_check.go b/business/jxstore/cms/store_sku_check.go index eed0d6c1f..7af18787a 100644 --- a/business/jxstore/cms/store_sku_check.go +++ b/business/jxstore/cms/store_sku_check.go @@ -330,7 +330,7 @@ func CheckSkuDiffBetweenJxAndVendor(ctx *jxcontext.Context, vendorIDList []int, if isGetJxSkuInfoData == false { //only get once jx sku info list every store id isGetJxSkuInfoData = true - jxSkuInfoData, _ := GetStoreSkus(ctx, storeID, []int{}, true, "", true, map[string]interface{}{}, 0, -1) + jxSkuInfoData, _ := GetStoreSkus(ctx, storeID, []int{}, true, "", true, false, map[string]interface{}{}, 0, -1) filterJxSkuInfoMap = GetFilterJxSkuInfoMap(jxSkuInfoData.SkuNames) } diff --git a/business/jxstore/misc/store_score.go b/business/jxstore/misc/store_score.go index 7079b5966..c938395b8 100644 --- a/business/jxstore/misc/store_score.go +++ b/business/jxstore/misc/store_score.go @@ -220,7 +220,7 @@ func GetAllStoreSkus(ctx *jxcontext.Context, parentTask tasksch.ITask, storeList taskFunc := func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { storeInfo := batchItemList[0].(*cms.StoreExt) storeID := storeInfo.ID - jxSkuInfoData, _ := cms.GetStoreSkus(ctx, storeID, []int{}, true, "", true, map[string]interface{}{}, 0, -1) + jxSkuInfoData, _ := cms.GetStoreSkus(ctx, storeID, []int{}, true, "", true, false,map[string]interface{}{}, 0, -1) jxSkuPriceMapData := make(map[int]int) for _, value := range jxSkuInfoData.SkuNames { for _, skuInfo := range value.Skus2 { diff --git a/business/jxstore/misc/store_sku_sales.go b/business/jxstore/misc/store_sku_sales.go index ec5173695..b001f5145 100644 --- a/business/jxstore/misc/store_sku_sales.go +++ b/business/jxstore/misc/store_sku_sales.go @@ -75,7 +75,7 @@ func GetStoreSkuSalesInfo(ctx *jxcontext.Context, storeID int) (outStoreSkuSales //得到当前门店商品数据 storeSkuMapData := make(map[int]*cms.StoreSkuNameExt) - storeSkuData, err := cms.GetStoreSkus(ctx, storeID, citySkuIDs, true, "", true, map[string]interface{}{}, 0, -1) + storeSkuData, err := cms.GetStoreSkus(ctx, storeID, citySkuIDs, true, "", true, false, map[string]interface{}{}, 0, -1) if err == nil { for _, value := range storeSkuData.SkuNames { for _, skuInfo := range value.Skus2 { @@ -95,7 +95,7 @@ func GetStoreSkuSalesInfo(ctx *jxcontext.Context, storeID int) (outStoreSkuSales taskFunc := func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { storeInfo := batchItemList[0].(*cms.StoreExt) storeID := storeInfo.ID - jxSkuInfoData, err := cms.GetStoreSkus(ctx, storeID, citySkuIDs, true, "", true, map[string]interface{}{}, 0, -1) + jxSkuInfoData, err := cms.GetStoreSkus(ctx, storeID, citySkuIDs, true, "", true, false, map[string]interface{}{}, 0, -1) jxSkuPriceMapData := make(map[int]int) for _, value := range jxSkuInfoData.SkuNames { for _, skuInfo := range value.Skus2 { diff --git a/controllers/cms_store_sku.go b/controllers/cms_store_sku.go index f58053f15..7f495a0ab 100644 --- a/controllers/cms_store_sku.go +++ b/controllers/cms_store_sku.go @@ -39,6 +39,7 @@ type StoreSkuController struct { // @Param offset query int false "门店列表起始序号（以0开始，缺省为0）" // @Param pageSize query int false "门店列表页大小（缺省为50，-1表示全部）" // @Param isBySku query bool false "是否按SKU分拆" +// @Param isAct query bool false "是否活动商品（包括正常活动与补贴）" // @Param jdSyncStatus query int false "京东同步标识" // @Param ebaiSyncStatus query int false "饿百同步标识" // @Param mtwmSyncStatus query int false "美团外卖同步标识" @@ -49,7 +50,7 @@ func (c *StoreSkuController) GetStoreSkus() { c.callGetStoreSkus(func(params *tStoreSkuGetStoreSkusParams) (retVal interface{}, errCode string, err error) { var skuIDs []int if err = jxutils.Strings2Objs(params.SkuIDs, &skuIDs); err == nil { - retVal, err = cms.GetStoreSkus(params.Ctx, params.StoreID, skuIDs, params.IsFocus, params.Keyword, params.IsBySku, params.MapData, params.Offset, params.PageSize) + retVal, err = cms.GetStoreSkus(params.Ctx, params.StoreID, skuIDs, params.IsFocus, params.Keyword, params.IsBySku, params.IsAct, params.MapData, params.Offset, params.PageSize) } return retVal, "", err }) @@ -78,6 +79,7 @@ func (c *StoreSkuController) GetStoreSkus() { // @Param offset query int false "门店列表起始序号（以0开始，缺省为0）" // @Param pageSize query int false "门店列表页大小（缺省为50，-1表示全部）" // @Param isBySku query bool false "是否按SKU分拆" +// @Param isAct query bool false "是否活动商品（包括正常活动与补贴）" // @Param jdSyncStatus query int false "京东同步标识" // @Param ebaiSyncStatus query int false "饿百同步标识" // @Param mtwmSyncStatus query int false "美团外卖同步标识" @@ -88,7 +90,7 @@ func (c *StoreSkuController) GetStoresSkus() { c.callGetStoresSkus(func(params *tStoreSkuGetStoresSkusParams) (retVal interface{}, errCode string, err error) { var storeIDs, skuIDs []int if err = jxutils.Strings2Objs(params.StoreIDs, &storeIDs, params.SkuIDs, &skuIDs); err == nil { - retVal, err = cms.GetStoresSkus(params.Ctx, storeIDs, skuIDs, params.IsFocus, params.Keyword, params.IsBySku, params.MapData, params.Offset, params.PageSize) + retVal, err = cms.GetStoresSkus(params.Ctx, storeIDs, skuIDs, params.IsFocus, params.Keyword, params.IsBySku, params.IsAct, params.MapData, params.Offset, params.PageSize) } return retVal, "", err }) From 4af8ebd8ef6dc7f3546fb99f0670f9cfe0b3b337 Mon Sep 17 00:00:00 2001 From: gazebo Date: Fri, 27 Sep 2019 16:56:51 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E4=B8=8D=E9=9C=80=E8=A6=81=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=E5=86=8D=E4=B8=BB=E5=8A=A8=E8=B0=83=E7=94=A8?= =?UTF-8?q?RegisterDataResource=E4=BA=86=EF=BC=8CRegisterDataResource?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E7=A9=BA=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- business/jxstore/cms/sku.go | 12 ++++---- business/jxutils/datares/datares.go | 39 ++++++++---------------- business/jxutils/datares/datares_test.go | 3 +- business/model/common.go | 7 +++-- business/partner/purchase/mtwm/mtwm.go | 2 +- business/userstore/food_recipe.go | 2 +- controllers/cms.go | 2 +- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/business/jxstore/cms/sku.go b/business/jxstore/cms/sku.go index de4f47e4d..16836bbdb 100644 --- a/business/jxstore/cms/sku.go +++ b/business/jxstore/cms/sku.go @@ -579,18 +579,18 @@ func AddSkuName(ctx *jxcontext.Context, skuNameExt *model.SkuNameExt, userName s // return nil, err // } // skuNameExt.ImgEbai = imgHintMap[model.VendorIDEBAI] - dataRes, err2 := datares.TryRegisterDataResource(ctx, skuNameExt.Name, skuNameExt.Img, true, model.ImgTypeMain, false) + dataRes, err2 := datares.TryRegisterDataResource(ctx, skuNameExt.Name, skuNameExt.Img, model.ImgTypeMain, false) if err = err2; err != nil { return nil, err } // skuNameExt.ImgHashCode = dataRes.HashCode skuNameExt.ImgEbai = dataRes.EbaiURL if skuNameExt.Img2 != "" { - datares.TryRegisterDataResource(ctx, skuNameExt.Name, skuNameExt.Img2, true, model.ImgTypeMain, true) + datares.TryRegisterDataResource(ctx, skuNameExt.Name, skuNameExt.Img2, model.ImgTypeMain, true) } if skuNameExt.DescImg != "" { - dataRes, err2 := datares.TryRegisterDataResource(ctx, skuNameExt.Name+"desc", skuNameExt.DescImg, true, model.ImgTypeDesc, false) + dataRes, err2 := datares.TryRegisterDataResource(ctx, skuNameExt.Name+"desc", skuNameExt.DescImg, model.ImgTypeDesc, false) if err = err2; err != nil { return nil, err } @@ -697,7 +697,7 @@ func UpdateSkuName(ctx *jxcontext.Context, nameID int, payload map[string]interf // } // // valid["ImgWeimob"] = imgHintMap[model.VendorIDWSC] // valid["ImgEbai"] = imgHintMap[model.VendorIDEBAI] - dataRes, err2 := datares.TryRegisterDataResource(ctx, skuName.Name, valid["img"].(string), true, model.ImgTypeMain, false) + dataRes, err2 := datares.TryRegisterDataResource(ctx, skuName.Name, valid["img"].(string), model.ImgTypeMain, false) if err = err2; err != nil { return 0, err } @@ -705,13 +705,13 @@ func UpdateSkuName(ctx *jxcontext.Context, nameID int, payload map[string]interf valid["ImgEbai"] = dataRes.EbaiURL } if valid["img2"] != nil { - datares.TryRegisterDataResource(ctx, skuName.Name, valid["img2"].(string), true, model.ImgTypeMain, true) + datares.TryRegisterDataResource(ctx, skuName.Name, valid["img2"].(string), model.ImgTypeMain, true) } if valid["descImg"] != nil { descImg := valid["descImg"].(string) if descImg != "" { - dataRes, err2 := datares.TryRegisterDataResource(ctx, skuName.Name+"_desc", descImg, true, model.ImgTypeDesc, false) + dataRes, err2 := datares.TryRegisterDataResource(ctx, skuName.Name+"_desc", descImg, model.ImgTypeDesc, false) if err = err2; err != nil { return 0, err } diff --git a/business/jxutils/datares/datares.go b/business/jxutils/datares/datares.go index d0d87d7be..e1c36bfd5 100644 --- a/business/jxutils/datares/datares.go +++ b/business/jxutils/datares/datares.go @@ -2,8 +2,8 @@ package datares import ( "fmt" + "net/http" "strings" - "time" "git.rosy.net.cn/jx-callback/business/jxutils" "git.rosy.net.cn/jx-callback/business/jxutils/jxcontext" @@ -17,12 +17,7 @@ import ( ) const ( - qiniuTokenExpires = 300 // 七牛TOKEN有效时间，5分钟 - dataResCacheTimeout = 24 * time.Hour -) - -var ( - dataResCache jxutils.SyncMapWithTimeout + qiniuTokenExpires = 300 // 七牛TOKEN有效时间，5分钟 ) type UploadResTokenInfo struct { @@ -50,7 +45,6 @@ func GetQiniuUploadToken(ctx *jxcontext.Context, suffix, hashCode string) (upTok Hit: imgURL != "", Img: imgURL, } - dataResCache.StoreWithTimeout(imgURL, hashCode, dataResCacheTimeout) return upTokenInfo, err } @@ -75,7 +69,9 @@ func getMimeTypeFromURL(resourceURL string) (mimeType string) { return mimeType } -func RegisterDataResource(ctx *jxcontext.Context, name, resourceURL, mimeType, hashCode string, resBinary []byte, imgType int, isAsync bool) (dataRes *model.DataResource, err error) { +// 此函数要求resBinary不能空，mimeType与hashCode必须是正确的 +func RegisterDataResource(ctx *jxcontext.Context, name, resourceURL, mimeType, hashCode string, resBinary []byte, imgType int, isAsyncUpload2Vendor bool) (dataRes *model.DataResource, err error) { + globals.SugarLogger.Debugf("RegisterDataResource, name:%s, resourceURL:%s, mimeType:%s, hashCode:%s, imgType:%d, isAsyncUpload2Vendor;%t", name, resourceURL, mimeType, hashCode, imgType, isAsyncUpload2Vendor) if model.ValidMimeTypes[mimeType] == nil { return nil, fmt.Errorf("MIME type:%s非法", mimeType) } @@ -106,17 +102,17 @@ func RegisterDataResource(ctx *jxcontext.Context, name, resourceURL, mimeType, h } if imgType > 0 { // 忽略上传错误 - UploadImage2Vendors(ctx, nil, dataRes, resBinary, isAsync) + UploadImage2Vendors(ctx, nil, dataRes, resBinary, isAsyncUpload2Vendor) } return dataRes, err } -func TryRegisterDataResource(ctx *jxcontext.Context, name, resourceURL string, isAllowDownLoad bool, imgType int, isAsync bool) (dataRes *model.DataResource, err error) { +func TryRegisterDataResource(ctx *jxcontext.Context, name, resourceURL string, imgType int, isAsyncUpload2Vendor bool) (dataRes *model.DataResource, err error) { + globals.SugarLogger.Debugf("TryRegisterDataResource, name:%s, resourceURL:%s, imgType:%d, isAsyncUpload2Vendor;%t", name, resourceURL, imgType, isAsyncUpload2Vendor) dataRes = &model.DataResource{ MainURL: resourceURL, } - db := dao.GetDB() - err = dao.GetEntity(db, dataRes, "MainURL") + err = dao.GetEntity(dao.GetDB(), dataRes, "MainURL") if err != nil { if !dao.IsNoRowsError(err) { return nil, err @@ -124,20 +120,11 @@ func TryRegisterDataResource(ctx *jxcontext.Context, name, resourceURL string, i } else { return dataRes, nil } - value, ok := dataResCache.Load(resourceURL) - var hashCode string - var resBinary []byte - if !ok { - if !isAllowDownLoad { - return nil, nil - } - if resBinary, hashCode, err = jxutils.DownloadFileByURL(resourceURL); err != nil { - return nil, err - } - } else { - hashCode = value.(string) + resBinary, hashCode, err := jxutils.DownloadFileByURL(resourceURL) + if err != nil { + return nil, err } - return RegisterDataResource(ctx, name, resourceURL, getMimeTypeFromURL(resourceURL), hashCode, resBinary, imgType, isAsync) + return RegisterDataResource(ctx, name, resourceURL, http.DetectContentType(resBinary), hashCode, resBinary, imgType, isAsyncUpload2Vendor) } func GetDataResource(ctx *jxcontext.Context, hashCode string) (resourceURL string, err error) { diff --git a/business/jxutils/datares/datares_test.go b/business/jxutils/datares/datares_test.go index 563c495c1..82080c758 100644 --- a/business/jxutils/datares/datares_test.go +++ b/business/jxutils/datares/datares_test.go @@ -6,6 +6,7 @@ import ( "git.rosy.net.cn/baseapi/utils" "git.rosy.net.cn/jx-callback/business/jxutils/jxcontext" + "git.rosy.net.cn/jx-callback/business/model" "git.rosy.net.cn/jx-callback/globals/api2" "git.rosy.net.cn/jx-callback/globals/testinit" @@ -36,7 +37,7 @@ func TestGetDataResource(t *testing.T) { } func TestTryRegisterDataResource(t *testing.T) { - dataRes, err := TryRegisterDataResource(jxcontext.AdminCtx, "test", "http://image.jxc4.com/image/8552261fb928be10f6406b13dc29570e.jpg", true, true, false) + dataRes, err := TryRegisterDataResource(jxcontext.AdminCtx, "test", "http://image.jxc4.com/image/37db6b94925e669e0a7b6844ba7ce160.jpg", model.ImgTypeMain, false) if err != nil { t.Fatal(err) } diff --git a/business/model/common.go b/business/model/common.go index 6fde833cc..661f24350 100644 --- a/business/model/common.go +++ b/business/model/common.go @@ -1,8 +1,9 @@ package model const ( - ImgTypeMain = 1 // 商品主图 - ImgTypeDesc = 2 // 商品描述详情 + ImgTypeLocal = 0 // 京西自己用的，不需要上传至平台 + ImgTypeMain = 1 // 商品主图 + ImgTypeDesc = 2 // 商品描述详情 ) var ( @@ -23,7 +24,7 @@ type DataResource struct { ResourceType string `orm:"size(48)" json:"resourceType"` // 资料的mime type Name string `orm:"size(48);index" json:"name"` - UseType int8 `json:"isVendor"` // 图资源使用方式（大于0才可能需要上传至平台） + UseType int8 `json:"useType"` // 图资源使用方式（大于0才可能需要上传至平台） MainURL string `orm:"size(512);column(main_url);unique" json:"mainURL"` QiniuURL string `orm:"size(512);column(qiniu_url);index" json:"qiniuURL"` diff --git a/business/partner/purchase/mtwm/mtwm.go b/business/partner/purchase/mtwm/mtwm.go index dfb172560..d291b9629 100644 --- a/business/partner/purchase/mtwm/mtwm.go +++ b/business/partner/purchase/mtwm/mtwm.go @@ -173,7 +173,7 @@ func (p *PurchaseHandler) UploadImg(ctx *jxcontext.Context, imgURL string, imgDa return "", fmt.Errorf("找不到一个美团门店来上传图片") } if globals.EnableMtwmStoreWrite { - if imgType > 0 { + if imgType > model.ImgTypeLocal { if imgData != nil { imgHint, err = api.MtwmAPI.ImageUpload(poiCode4UploadImg, imgName, imgData) } else { diff --git a/business/userstore/food_recipe.go b/business/userstore/food_recipe.go index a72e0e5c0..f3a1a8f4a 100644 --- a/business/userstore/food_recipe.go +++ b/business/userstore/food_recipe.go @@ -190,7 +190,7 @@ func tryRegisterDataRes4Recipe(ctx *jxcontext.Context, name, mainImg string, ste } errList := errlist.New() for _, v := range imgList { - _, err := datares.TryRegisterDataResource(ctx, name, v, true, 0, true) + _, err := datares.TryRegisterDataResource(ctx, name, v, model.ImgTypeLocal, true) errList.AddErr(err) } return errList.GetErrListAsOne() diff --git a/controllers/cms.go b/controllers/cms.go index 98e25fda3..3c42dbfa7 100644 --- a/controllers/cms.go +++ b/controllers/cms.go @@ -118,7 +118,7 @@ func (c *CmsController) GetQiniuUploadToken() { // @router /RegisterDataResource [post] func (c *CmsController) RegisterDataResource() { c.callRegisterDataResource(func(params *tCmsRegisterDataResourceParams) (retVal interface{}, errCode string, err error) { - retVal, err = datares.RegisterDataResource(params.Ctx, params.Name, params.ResourceURL, params.MimeType, params.HashCode, nil, 0, true) + // retVal, err = datares.RegisterDataResource(params.Ctx, params.Name, params.ResourceURL, params.MimeType, params.HashCode, nil, model.ImgTypeLocal, true) return retVal, "", err }) }