Files
jx-callback/business/jxstore/initdata/initdata.go
2019-09-27 09:23:11 +08:00

583 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package initdata
import (
"bytes"
"context"
"fmt"
"strings"
"git.rosy.net.cn/baseapi/platformapi/autonavi"
"git.rosy.net.cn/baseapi/platformapi/jdapi"
"git.rosy.net.cn/baseapi/utils"
"git.rosy.net.cn/jx-callback/business/jxutils"
"git.rosy.net.cn/jx-callback/business/jxutils/datares"
"git.rosy.net.cn/jx-callback/business/jxutils/jxcontext"
"git.rosy.net.cn/jx-callback/business/jxutils/tasksch"
"git.rosy.net.cn/jx-callback/business/model"
"git.rosy.net.cn/jx-callback/business/model/dao"
"git.rosy.net.cn/jx-callback/business/partner"
"git.rosy.net.cn/jx-callback/business/partner/purchase/jd"
"git.rosy.net.cn/jx-callback/globals"
"git.rosy.net.cn/jx-callback/globals/api"
"github.com/qiniu/api.v7/storage"
)
func TruncateTable(db *dao.DaoDB, tableName string) (err error) {
_, err = dao.ExecuteSQL(db, "TRUNCATE TABLE "+tableName)
return err
}
func insertPlace(ctx *jxcontext.Context, db *dao.DaoDB, parent *autonavi.District, placeList []*autonavi.District) (err error) {
for _, v := range placeList {
if v.Level <= autonavi.DistrictLevelDistrict {
place := &model.Place{
Code: int(utils.Str2Int64(v.Adcode)),
Name: v.Name,
Level: int8(v.Level),
TelCode: v.CityCode,
Enabled: 1,
}
if parent != nil {
place.ParentCode = int(utils.Str2Int64(parent.Adcode))
}
dao.WrapAddIDCULEntity(place, ctx.GetUserName())
if err = dao.CreateEntity(db, place); err != nil {
return err
}
if err = insertPlace(ctx, db, v, v.Districts); err != nil {
return err
}
}
}
return nil
}
func InitPlace(ctx *jxcontext.Context) (err error) {
placeList, err2 := api.AutonaviAPI.GetDistricts(autonavi.DistrictLevelDistrict, "")
if err = err2; err != nil {
return err
}
placeList = placeList[0].Districts
db := dao.GetDB()
dao.Begin(db)
defer func() {
if r := recover(); r != nil || err != nil {
dao.Rollback(db)
if r != nil {
panic(r)
}
}
}()
if _, err = dao.ExecuteSQL(db, `
DELETE t1
FROM place t1
WHERE code < 9000000;
`); err != nil {
return err
}
if err = insertPlace(ctx, db, nil, placeList); err != nil {
return err
}
updateSqls := []string{
`
UPDATE place t1
JOIN jde_city t2 ON t1.code = t2.col_tencentAddressCode
SET t1.jd_code = t2.col_areaCode;
`,
`
UPDATE place t1
JOIN place t2 ON t1.parent_code = t2.code AND t2.jd_code != 0
JOIN jde_district t3 ON t1.name = t3.col_areaName AND t2.jd_code = t3.col_cityCode
SET t1.jd_code = t3.col_areaCode
WHERE t1.level = 3;
`,
`
UPDATE
place t1
JOIN ebde_places t2 ON t1.name = t2.col_city_name
SET t1.ebai_code = t2.col_city_id
WHERE t1.level = 1 OR t1.level = 2;
`,
`
UPDATE
place t1
JOIN place t1p ON t1.parent_code = t1p.code
JOIN ebde_places t2 ON t1.name = t2.col_city_name
JOIN ebde_places t2p ON t2.col_parent_id = t2p.col_city_id AND t1p.ebai_code = t2p.col_city_id
SET t1.ebai_code = t2.col_city_id
WHERE t1.level = 3;
`,
`
UPDATE
place t1
JOIN mtpsdeliveryprice t2 ON t1.code = t2.citycode
SET t1.mtps_price = t2.price;
`,
`
UPDATE
place t1
JOIN mtpsdeliveryprice t2 ON t1.name LIKE CONCAT(t2.cityname, '%')
SET t1.mtps_price = t2.price
WHERE t1.level = 2 AND t1.mtps_price = 0;
`,
`
UPDATE place t1
LEFT JOIN (
SELECT DISTINCT city_code
FROM store
UNION DISTINCT
SELECT DISTINCT place_code city_code
FROM sku_name_place_bind
) t2 ON t1.code = t2.city_code
SET t1.enabled = 0
WHERE t1.level = 2 AND t2.city_code IS NULL;
`,
}
for _, v := range updateSqls {
if _, err = dao.ExecuteSQL(db, v); err != nil {
return err
}
}
dao.Commit(db)
return err
}
func RefreshSkuNameImg(ctx *jxcontext.Context, parentTask tasksch.ITask, isForce, isAsync, isContinueWhenError bool) (hint string, err error) {
db := dao.GetDB()
var skuNameList []*model.SkuName
if err = dao.GetRows(db, &skuNameList, `
SELECT t1.id, t1.img, MAX(t2.jd_id) jd_id
FROM sku_name t1
JOIN sku t2 ON t2.name_id = t1.id AND t2.deleted_at = ?
WHERE t1.deleted_at = ?
GROUP BY 1,2
ORDER BY t1.id
`, utils.DefaultTimeValue, utils.DefaultTimeValue); err != nil {
return "", err
}
task := tasksch.NewParallelTask("RefreshSkuNameImg", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuName := batchItemList[0].(*model.SkuName)
if !jxutils.IsEmptyID(skuName.JdID) {
if skuName.Img == "" || isForce {
var imgList []*jdapi.SkuPageImg
if imgList, err = api.JdAPI.GetSkuPageImageInfo(skuName.JdID); err == nil {
if len(imgList) > 0 {
skuName.Img = imgList[0].Big
_, err = dao.UpdateEntity(db, skuName, "Img")
}
}
}
}
return nil, err
}, skuNameList)
tasksch.HandleTask(task, parentTask, true).Run()
if !isAsync {
_, err = task.GetResult(0)
} else {
hint = task.ID
}
return hint, err
}
// func RefreshImgMd5(ctx *jxcontext.Context, parentTask tasksch.ITask, isForce, isAsync, isContinueWhenError bool) (hint string, err error) {
// db := dao.GetDB()
// var skuNameList []*model.SkuName
// if err = dao.GetRows(db, &skuNameList, `
// SELECT *
// FROM sku_name
// WHERE deleted_at = ? AND img <> ''
// ORDER BY id
// `, utils.DefaultTimeValue); err != nil {
// return "", err
// }
// task := tasksch.NewParallelTask("InitSkuName calculate md5", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError), ctx,
// func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
// skuName := batchItemList[0].(*model.SkuName)
// if skuName.ImgHashCode == "" || isForce {
// response, err := http.Get(skuName.Img)
// if err == nil {
// defer response.Body.Close()
// if response.StatusCode == http.StatusOK {
// data, err2 := ioutil.ReadAll(response.Body)
// if err = err2; err == nil {
// skuName.ImgHashCode = fmt.Sprintf("%X", md5.Sum(data))
// db := dao.GetDB()
// _, err = dao.UpdateEntity(db, skuName, "ImgHashCode")
// }
// } else {
// err = platformapi.ErrHTTPCodeIsNot200
// }
// }
// }
// return nil, err
// }, skuNameList)
// tasksch.HandleTask(task, parentTask, true).Run()
// if !isAsync {
// _, err = task.GetResult(0)
// } else {
// hint = task.ID
// }
// return hint, err
// }
func InitSkuName(ctx *jxcontext.Context, isForce, isAsync, isContinueWhenError bool) (hint string, err error) {
rootTask := tasksch.NewSeqTask("InitSkuName", ctx,
func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0:
_, err = RefreshSkuNameImg(ctx, task, isForce, false, isContinueWhenError)
case 1:
// _, err = RefreshImgMd5(ctx, task, isForce, false, isContinueWhenError)
}
return nil, err
}, 2)
tasksch.ManageTask(rootTask).Run()
if !isAsync {
_, err = rootTask.GetResult(0)
} else {
hint = rootTask.ID
}
return hint, err
}
func InitVendorCategory(ctx *jxcontext.Context, vendorID int) (num int64, err error) {
if handler := partner.GetPurchasePlatformFromVendorID(vendorID); handler != nil {
cats, err2 := handler.GetVendorCategories(ctx)
if err2 != nil {
return num, err2
}
db := dao.GetDB()
dao.Begin(db)
defer func() {
if r := recover(); r != nil || err != nil {
dao.Rollback(db)
if r != nil {
panic(r)
}
}
}()
sql := `
DELETE
FROM sku_vendor_category
WHERE vendor_id = ?
`
if _, err = dao.ExecuteSQL(db, sql, vendorID); err != nil {
return num, err
}
for _, cat := range cats {
dao.WrapAddIDCULEntity(cat, ctx.GetUserName())
if err = dao.CreateEntity(db, cat); err != nil {
return num, err
}
}
dao.Commit(db)
num = int64(len(cats))
} else {
err = fmt.Errorf("找不到平台:%d", vendorID)
}
return num, err
}
func UploadImg4Vendors(ctx *jxcontext.Context, isAsync, isContinueWhenError bool) (hint string, err error) {
db := dao.GetDB()
rootTask := tasksch.NewSeqTask("合并SkuName图片至DataResource", ctx,
func(task *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
switch step {
case 0: // 计算SkuName中缺失的hashCode注意是DescImg不是Img
var skuNameList []*model.SkuName
if err = dao.GetRows(db, &skuNameList, `
SELECT t1.*
FROM sku_name t1
WHERE t1.desc_img <> '' AND t1.img_hash_code = ''
`); err == nil && len(skuNameList) > 0 {
calcTask := tasksch.NewParallelTask("UploadImg4Vendors calc hashCode",
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
skuName := batchItemList[0].(*model.SkuName)
_, skuName.ImgHashCode, err = jxutils.DownloadFileByURL(skuName.DescImg)
if err == nil {
dao.UpdateEntity(db, skuName, "ImgHashCode")
}
return retVal, err
}, skuNameList)
tasksch.HandleTask(calcTask, task, false).Run()
_, err = calcTask.GetResult(0)
}
case 1: // 从SkuName添加缺失的图片至DataResource
_, err = dao.ExecuteSQL(db, `
INSERT INTO data_resource(created_at, updated_at, last_operator, hash_code,
resource_type, name, main_url, ebai_url, qiniu_url, use_type)
SELECT t1.created_at, t1.created_at, t1.last_operator, t1.img_hash_code,
CASE
WHEN INSTR(t1.desc_img, ".jpg") > 0 OR INSTR(t1.desc_img, ".jpeg") > 0 THEN
'image/jpeg'
WHEN INSTR(t1.desc_img, ".png") > 0 OR INSTR(t1.desc_img, ".peg") > 0 THEN
'image/png'
WHEN INSTR(t1.desc_img, ".gif") THEN
'image/gif'
ELSE
''
END resource_type,
CONCAT(t1.name, '_desc'), desc_img main_url, t1.desc_img_ebai ebai_url,
IF(INSTR(t1.desc_img, "image.jxc4.com") > 0, t1.desc_img, '') qiniu_url, 2
FROM sku_name t1
JOIN (
SELECT img_hash_code, MAX(id) id, COUNT(*) ct
FROM sku_name
WHERE img_hash_code <> '' AND desc_img <> ''
GROUP BY 1
) t3 ON t3.id = t1.id
LEFT JOIN data_resource t2 ON (t2.main_url <> '' AND t2.main_url = t1.desc_img)
WHERE t1.desc_img <> '' AND t1.img_hash_code <> '' AND t2.id IS NULL;
`)
case 2: // 统一SkuName中同hashCode不同图片地址至同一地址
_, err = dao.ExecuteSQL(db, `
UPDATE sku_name t1
JOIN data_resource t2 ON t2.hash_code = t1.img_hash_code AND t2.main_url <> ''
SET t1.desc_img = t2.main_url
WHERE t1.img_hash_code <> '' AND t1.desc_img <> t2.main_url;
`)
case 3: // 上传DataResource中缺失的平台图片
dataResList, err2 := dao.GetNeedUploadDataResource(db)
if err = err2; err == nil && len(dataResList) > 0 {
uploadTask := tasksch.NewParallelTask("批量上传图片至平台",
tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(2), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
dataRes := batchItemList[0].(*model.DataResource)
_, err = datares.UploadImage2Vendors(ctx, task, dataRes, nil, false)
return nil, err
}, dataResList)
tasksch.HandleTask(uploadTask, task, true).Run()
_, err = uploadTask.GetResult(0)
}
}
return result, err
}, 4)
tasksch.HandleTask(rootTask, nil, true).Run()
if !isAsync {
if _, err = rootTask.GetResult(0); err == nil {
hint = "1"
}
} else {
hint = rootTask.ID
}
return hint, err
}
func getSkuNameKey(prefix, name, comment, specUnit, unit string, specQuality float32) string {
return fmt.Sprintf("%s-%s-%f-%s-%s", prefix, name, specQuality, specUnit, unit)
}
func BuildSkuFromEbaiStore(ctx *jxcontext.Context, baiduShopID int64, isAsync, isContinueWhenError bool) (hint string, err error) {
ebaiCatIDKey := "ebaiCatID"
catList, err := api.EbaiAPI.PageGetCustomCatList(baiduShopID)
if err != nil {
return hint, err
}
catMap := make(map[string]map[string]interface{})
allSkuInfo := make(map[string]map[string]map[string]map[string]interface{})
for _, cat := range catList {
catName := utils.Interface2String(cat["cat_name"])
if catName == "限时特惠" || catName == "应季热销" {
catName = "应季水果"
}
skuNameMap := allSkuInfo[catName]
if skuNameMap == nil {
skuNameMap = make(map[string]map[string]map[string]interface{})
allSkuInfo[catName] = skuNameMap
}
if catMap[catName] == nil {
catMap[catName] = cat
}
skuList, err2 := api.EbaiAPI.PageGetCustomSkuList(baiduShopID, utils.Str2Int64(utils.Interface2String(cat["cat_id"])))
if err = err2; err != nil {
return hint, err
}
for _, sku := range skuList {
_, _, _, specUnit, _, specQuality := jxutils.SplitSkuName(sku["upc_name"].(string))
if specQuality > 0 {
if catMap[catName][ebaiCatIDKey] == nil {
catMap[catName][ebaiCatIDKey] = utils.Interface2String(sku["cat3_id"])
}
skuNameKey := sku["photos"].([]interface{})[0].(map[string]interface{})["url"].(string)
if skuNameMap[skuNameKey] == nil {
skuNameMap[skuNameKey] = make(map[string]map[string]interface{})
}
skuNameMap[skuNameKey][fmt.Sprintf("%f.%s", specQuality, specUnit)] = sku
}
}
}
allSkuInfo2 := make(map[*model.SkuCategory][]*model.SkuNameExt)
catSeq := 0
for catName, catSkuNameMap := range allSkuInfo {
if len(catSkuNameMap) > 0 {
skuCat := &model.SkuCategory{
Name: catName,
ParentID: 0,
Level: 1,
Type: model.SkuCategoryNormal,
Seq: catSeq,
EbaiCategoryID: utils.Str2Int64(utils.Interface2String(catMap[catName][ebaiCatIDKey])),
}
catSeq++
for _, skuList := range catSkuNameMap {
var skuNameExt *model.SkuNameExt
for _, sku := range skuList {
prefix, name, comment, specUnit, unit, specQuality := jxutils.SplitSkuName(sku["upc_name"].(string))
if skuNameExt == nil {
skuNameExt = &model.SkuNameExt{
SkuName: model.SkuName{
Prefix: prefix,
Name: name,
// CategoryID:
IsGlobal: 1,
Unit: unit,
SpecQuality: specQuality,
SpecUnit: specUnit,
Price: int(jxutils.StandardPrice2Int(utils.MustInterface2Float64(sku["sale_price"]))),
Img: sku["photos"].([]interface{})[0].(map[string]interface{})["url"].(string),
Status: model.SkuStatusNormal,
IsSpu: 0,
},
}
if len(skuList) > 1 {
skuNameExt.Unit = "份"
skuNameExt.SpecQuality = 500
skuNameExt.SpecUnit = "g"
}
if skuNameExt.Unit == "份" {
tmpSpecQuality := specQuality
if specUnit == "kg" || specUnit == "l" {
tmpSpecQuality *= 1000
}
skuNameExt.Price = int(float32(skuNameExt.Price) * 500 / tmpSpecQuality)
}
}
mySku := &model.Sku{
// CategoryID:
// NameID:
Comment: comment,
SpecQuality: specQuality,
SpecUnit: specUnit,
Weight: int(utils.Str2Int64(utils.Interface2String(sku["weight"]))),
Status: model.SkuStatusNormal,
LinkID: int(jxutils.StandardPrice2Int(utils.MustInterface2Float64(sku["sale_price"]))), // 临时传递价格用
}
if sku["enabled"].(string) == "0" {
mySku.Status = model.SkuStatusDontSale
}
skuNameExt.Skus = append(skuNameExt.Skus, mySku)
}
allSkuInfo2[skuCat] = append(allSkuInfo2[skuCat], skuNameExt)
}
}
}
startOutSkuID := 1000
var skuCatList []*model.SkuCategory
for cat := range allSkuInfo2 {
skuCatList = append(skuCatList, cat)
}
putPolicy := storage.PutPolicy{
Scope: globals.QiniuBucket,
// Expires: 10 * 60,
}
cfg := &storage.Config{}
upToken := putPolicy.UploadToken(api.QiniuAPI)
rootTask := tasksch.NewSeqTask("BuildSkuFromEbaiStore", ctx,
func(rootTask *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) {
var catID string
cat := skuCatList[step]
skuNameExtList := allSkuInfo2[cat]
catID, err = addShopCategory(0, cat.Name, int(cat.Level), cat.Seq, ctx.GetUserName())
if err == nil {
shopCategories := []int64{utils.Str2Int64(catID)}
for _, skuNameExt := range skuNameExtList {
var imgContent []byte
skuNameExt.CategoryID = cat.ID
skuNameExt.Img = strings.Replace(skuNameExt.Img, "https://", "http://", 1)
if imgContent, _, err = jxutils.DownloadFileByURL(skuNameExt.Img); err != nil {
globals.SugarLogger.Infof("download pic %s failed with error:%v", skuNameExt.Img, err)
if !isContinueWhenError {
return nil, err
}
err = nil
continue
}
formUploader := storage.NewFormUploader(cfg)
ret := storage.PutRet{}
key := jxutils.GenPicFileName(".jpeg")
for i := 0; i < 3; i++ {
if err = formUploader.Put(context.Background(), &ret, upToken, key, bytes.NewReader(imgContent), int64(len(imgContent)), &storage.PutExtra{}); err == nil {
break
}
}
if err != nil {
globals.SugarLogger.Infof("upload pic %s failed with error:%v", skuNameExt.Img, err)
if !isContinueWhenError {
return nil, err
}
err = nil
continue
}
qiniuImgURL := jxutils.ComposeQiniuResURL(key) + "?imageMogr2/thumbnail/x800/gravity/Center/crop/800x800"
for _, sku := range skuNameExt.Skus {
jdCatID := 22410 // 其他国产水果
if cat.Name == "进口水果" {
jdCatID = 20342 // 其他进口水果
}
price := sku.LinkID
sku.LinkID = 0
skuName := jxutils.ComposeSkuName(skuNameExt.Prefix, skuNameExt.Name, sku.Comment, skuNameExt.Unit, sku.SpecQuality, sku.SpecUnit, jdapi.MaxSkuNameCharCount)
fixedStatus := 1
if sku.Status != model.SkuStatusNormal {
fixedStatus = 2
}
var vendorSkuID string
vendorSkuID, err = addSku(utils.Int2Str(startOutSkuID), jdCatID, shopCategories, jd.DefBrandID, skuName, price, jxutils.IntWeight2Float(sku.Weight), []string{qiniuImgURL}, fixedStatus, true, nil)
if err == nil {
globals.SugarLogger.Debugf("vendorSkuID=%s", vendorSkuID)
} else {
globals.SugarLogger.Infof("create %s failed with error:%v", skuName, err)
if !isContinueWhenError {
return nil, err
}
err = nil
}
// fmt.Printf("%s,[%s]%s-%f-%s, %s, %s\n", cat.Name, skuNameExt.Prefix, skuNameExt.Name, skuNameExt.SpecQuality, skuNameExt.SpecUnit, qiniuImgURL, skuName)
startOutSkuID++
// rootTask.Cancel()
// return nil, nil
}
}
} else {
return nil, err
}
return nil, err
}, len(skuCatList))
tasksch.ManageTask(rootTask).Run()
if !isAsync {
_, err = rootTask.GetResult(0)
} else {
hint = rootTask.ID
}
return hint, err
}
func addShopCategory(pid int64, shopCategoryName string, shopCategoryLevel, sort int, userName string) (catId string, err error) {
// globals.SugarLogger.Debug(pid, shopCategoryName, shopCategoryLevel, sort, userName)
// return "123", err
return api.JdAPI.AddShopCategory(pid, shopCategoryName, shopCategoryLevel, sort, userName)
}
func addSku(outSkuId string, cagtegoryId int, shopCategories []int64, brandId int, skuName string, skuPrice int, weight float32, images []string, fixedStatus int, isSale bool, addParams map[string]interface{}) (skuId string, err error) {
// globals.SugarLogger.Debug(outSkuId, cagtegoryId, shopCategories, brandId, skuName, skuPrice, weight, images, fixedStatus, isSale, addParams)
// return "456", err
return api.JdAPI.AddSku(outSkuId, cagtegoryId, shopCategories, brandId, skuName, skuPrice, weight, images, fixedStatus, isSale, addParams)
}