liulei/jx-callback
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整理DataResource相关的内容

Browse Source
This commit is contained in:
gazebo
2019-12-06 15:20:22 +08:00
parent b9421e7979
commit 7fb09f99d1
6 changed files with 201 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
business/jxstore/tempop/tempop.go
View File

@@ -4,10 +4,6 @@ import (
"bytes"
"context"
"fmt"
"image"
"image/jpeg"
"image/png"
"net/http"
"regexp"
"strings"
"sync"
@@ -1046,92 +1042,92 @@ func CreateConsumerFromOrders(ctx *jxcontext.Context, vendorIDs []int, fromDate,
}
func CheckImages(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hint string, err error) {
db := dao.GetDB()
rootTask := tasksch.NewSeqTask("检查图片的有效性", ctx,
func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0: // 检查图片的格式与大小
var imgList []*model.DataResource
if err = dao.GetRows(db, &imgList, `
SELECT t1.*
FROM data_resource t1
WHERE t1.use_type = 1
`); err == nil && len(imgList) > 0 {
calcTask := tasksch.NewParallelTask("检查京西图片的有效性",
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
dataRes := batchItemList[0].(*model.DataResource)
binaryData, hashCode, err2 := jxutils.DownloadFileByURL(dataRes.MainURL)
if err = err2; err == nil {
errMsg := ""
if hashCode != dataRes.HashCode {
errMsg = fmt.Sprintf("hashcode:%s", hashCode)
dataRes.HashCode = hashCode
}
resType := http.DetectContentType(binaryData)
if resType != dataRes.ResourceType {
errMsg += "," + resType
dataRes.ResourceType = resType
}
var img image.Image
if dataRes.ResourceType == "image/png" {
img, err = png.Decode(bytes.NewReader(binaryData))
} else if dataRes.ResourceType == "image/jpeg" {
img, err = jpeg.Decode(bytes.NewReader(binaryData))
}
if err == nil {
if img != nil {
if img.Bounds().Dx() != 800 || img.Bounds().Dy() != 800 {
errMsg += fmt.Sprintf(",size:%dx%d", img.Bounds().Dx(), img.Bounds().Dy())
}
}
} else {
errMsg += "," + err.Error()
}
if errMsg != "" {
dataRes.Remark = errMsg
dao.UpdateEntity(dao.GetDB(), dataRes, "Remark", "ResourceType", "HashCode")
}
}
return retVal, err
}, imgList)
tasksch.HandleTask(calcTask, task, false).Run()
_, err = calcTask.GetResult(0)
}
case 1: // 检查京东商品图片
skuNameInfo, err2 := cms.GetSkuNames(ctx, "", false, map[string]interface{}{}, 0, -1)
if err = err2; err == nil && len(skuNameInfo.SkuNames) > 0 {
calcTask := tasksch.NewParallelTask("检查京东图片的有效性",
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuName := batchItemList[0].(*model.SkuNameExt)
for _, sku := range skuName.Skus {
if sku.JdID != 0 {
skuPageInfo, err2 := api.JdAPI.GetSkuPageInfo(sku.JdID)
if err = err2; err == nil && skuPageInfo.FixedStatus && len(skuPageInfo.Image) == 0 {
err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, sku.ID, sku.JdID)
skuName.ImgHashCode = "np"
dao.UpdateEntity(dao.GetDB(), &skuName.SkuName, "ImgHashCode")
}
break
}
}
return retVal, err
}, skuNameInfo.SkuNames)
tasksch.HandleTask(calcTask, task, false).Run()
_, err = calcTask.GetResult(0)
}
}
return result, err
}, 2)
tasksch.HandleTask(rootTask, nil, true).Run()
if !isAsync {
if _, err = rootTask.GetResult(0); err == nil {
hint = "1"
}
} else {
hint = rootTask.ID
}
// db := dao.GetDB()
// rootTask := tasksch.NewSeqTask("检查图片的有效性", ctx,
// func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
// switch step {
// case 0: // 检查图片的格式与大小
// var imgList []*model.DataResource
// if err = dao.GetRows(db, &imgList, `
// SELECT t1.*
// FROM data_resource t1
// WHERE t1.use_type = 1
// `); err == nil && len(imgList) > 0 {
// calcTask := tasksch.NewParallelTask("检查京西图片的有效性",
// tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
// func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
// dataRes := batchItemList[0].(*model.DataResource)
// binaryData, hashCode, err2 := jxutils.DownloadFileByURL(dataRes.MainURL)
// if err = err2; err == nil {
// errMsg := ""
// if hashCode != dataRes.HashCode {
// errMsg = fmt.Sprintf("hashcode:%s", hashCode)
// dataRes.HashCode = hashCode
// }
// resType := http.DetectContentType(binaryData)
// if resType != dataRes.ResourceType {
// errMsg += "," + resType
// dataRes.ResourceType = resType
// }
// var img image.Image
// if dataRes.ResourceType == "image/png" {
// img, err = png.Decode(bytes.NewReader(binaryData))
// } else if dataRes.ResourceType == "image/jpeg" {
// img, err = jpeg.Decode(bytes.NewReader(binaryData))
// }
// if err == nil {
// if img != nil {
// if img.Bounds().Dx() != 800 || img.Bounds().Dy() != 800 {
// errMsg += fmt.Sprintf(",size:%dx%d", img.Bounds().Dx(), img.Bounds().Dy())
// }
// }
// } else {
// errMsg += "," + err.Error()
// }
// if errMsg != "" {
// dataRes.Remark = errMsg
// dao.UpdateEntity(dao.GetDB(), dataRes, "Remark", "ResourceType", "HashCode")
// }
// }
// return retVal, err
// }, imgList)
// tasksch.HandleTask(calcTask, task, false).Run()
// _, err = calcTask.GetResult(0)
// }
// case 1: // 检查京东商品图片
// skuNameInfo, err2 := cms.GetSkuNames(ctx, "", false, map[string]interface{}{}, 0, -1)
// if err = err2; err == nil && len(skuNameInfo.SkuNames) > 0 {
// calcTask := tasksch.NewParallelTask("检查京东图片的有效性",
// tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
// func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
// skuName := batchItemList[0].(*model.SkuNameExt)
// for _, sku := range skuName.Skus {
// if sku.JdID != 0 {
// skuPageInfo, err2 := api.JdAPI.GetSkuPageInfo(sku.JdID)
// if err = err2; err == nil && skuPageInfo.FixedStatus && len(skuPageInfo.Image) == 0 {
// err = fmt.Errorf("skuName:%d, sku:%d, jdid:%d, 没有图片", skuName.ID, sku.ID, sku.JdID)
// skuName.ImgHashCode = "np"
// dao.UpdateEntity(dao.GetDB(), &skuName.SkuName, "ImgHashCode")
// }
// break
// }
// }
// return retVal, err
// }, skuNameInfo.SkuNames)
// tasksch.HandleTask(calcTask, task, false).Run()
// _, err = calcTask.GetResult(0)
// }
// }
// return result, err
// }, 2)
// tasksch.HandleTask(rootTask, nil, true).Run()
// if !isAsync {
// if _, err = rootTask.GetResult(0); err == nil {
// hint = "1"
// }
// } else {
// hint = rootTask.ID
// }
return hint, err
}