package initdata import ( "bytes" "context" "crypto/md5" "fmt" "io/ioutil" "net/http" "strings" "git.rosy.net.cn/baseapi/platformapi/autonavi" "git.rosy.net.cn/baseapi/platformapi/jdapi" "git.rosy.net.cn/baseapi/utils" "git.rosy.net.cn/jx-callback/business/jxutils" "git.rosy.net.cn/jx-callback/business/jxutils/jxcontext" "git.rosy.net.cn/jx-callback/business/jxutils/tasksch" "git.rosy.net.cn/jx-callback/business/model" "git.rosy.net.cn/jx-callback/business/model/dao" "git.rosy.net.cn/jx-callback/business/partner" "git.rosy.net.cn/jx-callback/business/partner/purchase/jd" "git.rosy.net.cn/jx-callback/globals" "git.rosy.net.cn/jx-callback/globals/api" "github.com/qiniu/api.v7/storage" ) func TruncateTable(db *dao.DaoDB, tableName string) (err error) { _, err = dao.ExecuteSQL(db, "TRUNCATE TABLE "+tableName) return err } func insertPlace(ctx *jxcontext.Context, db *dao.DaoDB, parent *autonavi.District, placeList []*autonavi.District) (err error) { for _, v := range placeList { if v.Level <= autonavi.DistrictLevelDistrict { place := &model.Place{ Code: int(utils.Str2Int64(v.Adcode)), Name: v.Name, Level: int8(v.Level), TelCode: v.CityCode, Enabled: 1, } if parent != nil { place.ParentCode = int(utils.Str2Int64(parent.Adcode)) } dao.WrapAddIDCULEntity(place, ctx.GetUserName()) if err = dao.CreateEntity(db, place); err != nil { return err } if err = insertPlace(ctx, db, v, v.Districts); err != nil { return err } } } return nil } func InitPlace(ctx *jxcontext.Context) (err error) { db := dao.GetDB() if err = TruncateTable(db, "place"); err == nil { placeList, err2 := api.AutonaviAPI.GetDistricts(autonavi.DistrictLevelDistrict, "") if err = err2; err != nil { return err } placeList = placeList[0].Districts dao.Begin(db) defer func() { dao.Rollback(db) }() if err = insertPlace(ctx, db, nil, placeList); err != nil { return err } updateSqls := []string{ ` UPDATE place t1 JOIN jde_city t2 ON t1.code = t2.col_tencentAddressCode SET t1.jd_code = t2.col_areaCode; `, ` UPDATE place t1 JOIN place t2 ON t1.parent_code = t2.code AND t2.jd_code != 0 JOIN jde_district t3 ON t1.name = t3.col_areaName AND t2.jd_code = t3.col_cityCode SET t1.jd_code = t3.col_areaCode WHERE t1.level = 3; `, ` UPDATE place t1 JOIN ebde_places t2 ON t1.name = t2.col_city_name SET t1.ebai_code = t2.col_city_id WHERE t1.level = 1 OR t1.level = 2; `, ` UPDATE place t1 JOIN place t1p ON t1.parent_code = t1p.code JOIN ebde_places t2 ON t1.name = t2.col_city_name JOIN ebde_places t2p ON t2.col_parent_id = t2p.col_city_id AND t1p.ebai_code = t2p.col_city_id SET t1.ebai_code = t2.col_city_id WHERE t1.level = 3; `, ` UPDATE place t1 JOIN mtpsdeliveryprice t2 ON t1.code = t2.citycode SET t1.mtps_price = t2.price; `, ` UPDATE place t1 JOIN mtpsdeliveryprice t2 ON t1.name LIKE CONCAT(t2.cityname, '%') SET t1.mtps_price = t2.price WHERE t1.level = 2 AND t1.mtps_price = 0; `, ` UPDATE place t1 LEFT JOIN ( SELECT DISTINCT city_code FROM store UNION DISTINCT SELECT DISTINCT place_code city_code FROM sku_name_place_bind ) t2 ON t1.code = t2.city_code SET t1.enabled = 0 WHERE t1.level = 2 AND t2.city_code IS NULL; `, } for _, v := range updateSqls { if _, err = dao.ExecuteSQL(db, v); err != nil { return err } } dao.Commit(db) } return err } func InitSkuName(ctx *jxcontext.Context, isForce, isAsync, isContinueWhenError bool) (hint string, err error) { db := dao.GetDB() var skuNameList []*model.SkuName if err = dao.GetRows(db, &skuNameList, ` SELECT * FROM sku_name WHERE deleted_at = ? AND img <> '' ORDER BY id `, utils.DefaultTimeValue); err != nil { return "", err } task := tasksch.NewParallelTask("InitSkuName calculate md5", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError), ctx, func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { skuName := batchItemList[0].(*model.SkuName) if skuName.ImgHashCode == "" || isForce { response, err := http.Get(skuName.Img) if err == nil { defer response.Body.Close() data, err2 := ioutil.ReadAll(response.Body) if err = err2; err == nil { skuName.ImgHashCode = fmt.Sprintf("%X", md5.Sum(data)) db := dao.GetDB() _, err = dao.UpdateEntity(db, skuName, "ImgHashCode") } } } return nil, err }, skuNameList) tasksch.ManageTask(task).Run() if !isAsync { _, err = task.GetResult(0) } else { hint = task.ID } return hint, err } func InitVendorCategory(ctx *jxcontext.Context, vendorID int) (num int64, err error) { if handler := partner.GetPurchasePlatformFromVendorID(vendorID); handler != nil { cats, err2 := handler.GetVendorCategories(ctx) if err2 != nil { return num, err2 } db := dao.GetDB() dao.Begin(db) defer func() { if r := recover(); r != nil || err != nil { dao.Rollback(db) if r != nil { panic(r) } } }() sql := ` DELETE FROM sku_vendor_category WHERE vendor_id = ? ` if _, err = dao.ExecuteSQL(db, sql, vendorID); err != nil { return num, err } for _, cat := range cats { dao.WrapAddIDCULEntity(cat, ctx.GetUserName()) if err = dao.CreateEntity(db, cat); err != nil { return num, err } } dao.Commit(db) num = int64(len(cats)) } else { err = fmt.Errorf("找不到平台:%d", vendorID) } return num, err } func UploadWeimobImg4SkuName(ctx *jxcontext.Context, nameIDs []int, isAsync, isContinueWhenError bool) (hint string, err error) { db := dao.GetDB() skuNameList, err := dao.GetSkuNames(db, nameIDs) if err != nil { return "", err } rootTask := tasksch.NewParallelTask("UploadWeimobImg4SkuName", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { skuName := batchItemList[0].(*model.SkuName) if skuName.Img != "" && skuName.ImgWeimob == "" { if globals.EnableWscStoreWrite { if skuName.ImgWeimob, err = api.WeimobAPI.UploadImgByURL(skuName.Img, ""); err == nil { _, err = dao.UpdateEntity(db, skuName, "ImgWeimob") } } } return nil, err }, skuNameList) tasksch.ManageTask(rootTask).Run() if !isAsync { _, err = rootTask.GetResult(0) } else { hint = rootTask.ID } return hint, err } func UploadEbaiImg4SkuName(ctx *jxcontext.Context, nameIDs []int, isAsync, isContinueWhenError bool) (hint string, err error) { db := dao.GetDB() skuNameList, err := dao.GetSkuNames(db, nameIDs) if err != nil { return "", err } rootTask := tasksch.NewParallelTask("UploadEbaiImg4SkuName", tasksch.NewParallelConfig().SetIsContinueWhenError(isContinueWhenError).SetParallelCount(5), ctx, func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { skuName := batchItemList[0].(*model.SkuName) if skuName.Img != "" && skuName.ImgEbai == "" { if globals.EnableEbaiStoreWrite { if skuName.ImgEbai, err = api.EbaiAPI.PictureUpload(skuName.Img, nil); err == nil { _, err = dao.UpdateEntity(db, skuName, "ImgEbai") } } } return nil, err }, skuNameList) tasksch.ManageTask(rootTask).Run() if !isAsync { _, err = rootTask.GetResult(0) } else { hint = rootTask.ID } return hint, err } func getSkuNameKey(prefix, name, comment, specUnit, unit string, specQuality float32) string { return fmt.Sprintf("%s-%s-%f-%s-%s", prefix, name, specQuality, specUnit, unit) } func BuildSkuFromEbaiStore(ctx *jxcontext.Context, baiduShopID int64, isAsync, isContinueWhenError bool) (hint string, err error) { ebaiCatIDKey := "ebaiCatID" catList, err := api.EbaiAPI.PageGetCustomCatList(baiduShopID) if err != nil { return hint, err } catMap := make(map[string]map[string]interface{}) allSkuInfo := make(map[string]map[string]map[string]map[string]interface{}) for _, cat := range catList { catName := utils.Interface2String(cat["cat_name"]) if catName == "限时特惠" || catName == "应季热销" { catName = "应季水果" } skuNameMap := allSkuInfo[catName] if skuNameMap == nil { skuNameMap = make(map[string]map[string]map[string]interface{}) allSkuInfo[catName] = skuNameMap } if catMap[catName] == nil { catMap[catName] = cat } skuList, err2 := api.EbaiAPI.PageGetCustomSkuList(baiduShopID, utils.Str2Int64(utils.Interface2String(cat["cat_id"]))) if err = err2; err != nil { return hint, err } for _, sku := range skuList { _, _, _, specUnit, _, specQuality := jxutils.SplitSkuName(sku["upc_name"].(string)) if specQuality > 0 { if catMap[catName][ebaiCatIDKey] == nil { catMap[catName][ebaiCatIDKey] = utils.Interface2String(sku["cat3_id"]) } skuNameKey := sku["photos"].([]interface{})[0].(map[string]interface{})["url"].(string) if skuNameMap[skuNameKey] == nil { skuNameMap[skuNameKey] = make(map[string]map[string]interface{}) } skuNameMap[skuNameKey][fmt.Sprintf("%f.%s", specQuality, specUnit)] = sku } } } allSkuInfo2 := make(map[*model.SkuCategory][]*model.SkuNameExt) catSeq := 0 for catName, catSkuNameMap := range allSkuInfo { if len(catSkuNameMap) > 0 { skuCat := &model.SkuCategory{ Name: catName, ParentID: 0, Level: 1, Type: model.SkuCategoryNormal, Seq: catSeq, EbaiCategoryID: utils.Str2Int64(utils.Interface2String(catMap[catName][ebaiCatIDKey])), } catSeq++ for _, skuList := range catSkuNameMap { var skuNameExt *model.SkuNameExt for _, sku := range skuList { prefix, name, comment, specUnit, unit, specQuality := jxutils.SplitSkuName(sku["upc_name"].(string)) if skuNameExt == nil { skuNameExt = &model.SkuNameExt{ SkuName: model.SkuName{ Prefix: prefix, Name: name, // CategoryID: IsGlobal: 1, Unit: unit, SpecQuality: specQuality, SpecUnit: specUnit, Price: int(jxutils.StandardPrice2Int(utils.MustInterface2Float64(sku["sale_price"]))), Img: sku["photos"].([]interface{})[0].(map[string]interface{})["url"].(string), Status: model.SkuStatusNormal, IsSpu: 0, }, } if len(skuList) > 1 { skuNameExt.Unit = "份" skuNameExt.SpecQuality = 500 skuNameExt.SpecUnit = "g" } if skuNameExt.Unit == "份" { tmpSpecQuality := specQuality if specUnit == "kg" || specUnit == "l" { tmpSpecQuality *= 1000 } skuNameExt.Price = int(float32(skuNameExt.Price) * 500 / tmpSpecQuality) } } mySku := &model.Sku{ // CategoryID: // NameID: Comment: comment, SpecQuality: specQuality, SpecUnit: specUnit, Weight: int(utils.Str2Int64(utils.Interface2String(sku["weight"]))), Status: model.SkuStatusNormal, LinkID: int(jxutils.StandardPrice2Int(utils.MustInterface2Float64(sku["sale_price"]))), // 临时传递价格用 } if sku["enabled"].(string) == "0" { mySku.Status = model.SkuStatusDontSale } skuNameExt.Skus = append(skuNameExt.Skus, mySku) } allSkuInfo2[skuCat] = append(allSkuInfo2[skuCat], skuNameExt) } } } startOutSkuID := 1000 var skuCatList []*model.SkuCategory for cat := range allSkuInfo2 { skuCatList = append(skuCatList, cat) } putPolicy := storage.PutPolicy{ Scope: globals.QiniuBucket, // Expires: 10 * 60, } cfg := &storage.Config{} upToken := putPolicy.UploadToken(api.QiniuAPI) rootTask := tasksch.NewSeqTask("BuildSkuFromEbaiStore", ctx, func(rootTask *tasksch.SeqTask, step int, params ...interface{}) (result interface{}, err error) { var catID string cat := skuCatList[step] skuNameExtList := allSkuInfo2[cat] catID, err = addShopCategory(0, cat.Name, int(cat.Level), cat.Seq, ctx.GetUserName()) if err == nil { shopCategories := []int64{utils.Str2Int64(catID)} for _, skuNameExt := range skuNameExtList { var imgContent []byte skuNameExt.CategoryID = cat.ID skuNameExt.Img = strings.Replace(skuNameExt.Img, "https://", "http://", 1) if imgContent, _, err = jxutils.DownloadFileByURL(skuNameExt.Img); err != nil { globals.SugarLogger.Infof("download pic %s failed with error:%v", skuNameExt.Img, err) if !isContinueWhenError { return nil, err } err = nil continue } formUploader := storage.NewFormUploader(cfg) ret := storage.PutRet{} key := jxutils.GenPicFileName(".jpeg") for i := 0; i < 3; i++ { if err = formUploader.Put(context.Background(), &ret, upToken, key, bytes.NewReader(imgContent), int64(len(imgContent)), &storage.PutExtra{}); err == nil { break } } if err != nil { globals.SugarLogger.Infof("upload pic %s failed with error:%v", skuNameExt.Img, err) if !isContinueWhenError { return nil, err } err = nil continue } qiniuImgURL := jxutils.ComposeQiniuResURL(key) + "?imageMogr2/thumbnail/x800/gravity/Center/crop/800x800" for _, sku := range skuNameExt.Skus { jdCatID := 22410 // 其他国产水果 if cat.Name == "进口水果" { jdCatID = 20342 // 其他进口水果 } price := sku.LinkID sku.LinkID = 0 skuName := jxutils.ComposeSkuName(skuNameExt.Prefix, skuNameExt.Name, sku.Comment, skuNameExt.Unit, sku.SpecQuality, sku.SpecUnit, jdapi.MaxSkuNameLen) fixedStatus := 1 if sku.Status != model.SkuStatusNormal { fixedStatus = 2 } var vendorSkuID string vendorSkuID, err = addSku(utils.Int2Str(startOutSkuID), jdCatID, shopCategories, jd.DefBrandID, skuName, price, jxutils.IntWeight2Float(sku.Weight), []string{qiniuImgURL}, fixedStatus, true, nil) if err == nil { globals.SugarLogger.Debugf("vendorSkuID=%s", vendorSkuID) } else { globals.SugarLogger.Infof("create %s failed with error:%v", skuName, err) if !isContinueWhenError { return nil, err } err = nil } // fmt.Printf("%s,[%s]%s-%f-%s, %s, %s\n", cat.Name, skuNameExt.Prefix, skuNameExt.Name, skuNameExt.SpecQuality, skuNameExt.SpecUnit, qiniuImgURL, skuName) startOutSkuID++ // rootTask.Cancel() // return nil, nil } } } else { return nil, err } return nil, err }, len(skuCatList)) tasksch.ManageTask(rootTask).Run() if !isAsync { _, err = rootTask.GetResult(0) } else { hint = rootTask.ID } return hint, err } func addShopCategory(pid int64, shopCategoryName string, shopCategoryLevel, sort int, userName string) (catId string, err error) { // globals.SugarLogger.Debug(pid, shopCategoryName, shopCategoryLevel, sort, userName) // return "123", err return api.JdAPI.AddShopCategory(pid, shopCategoryName, shopCategoryLevel, sort, userName) } func addSku(outSkuId string, cagtegoryId int, shopCategories []int64, brandId int, skuName string, skuPrice int, weight float32, images []string, fixedStatus int, isSale bool, addParams map[string]interface{}) (skuId string, err error) { // globals.SugarLogger.Debug(outSkuId, cagtegoryId, shopCategories, brandId, skuName, skuPrice, weight, images, fixedStatus, isSale, addParams) // return "456", err return api.JdAPI.AddSku(outSkuId, cagtegoryId, shopCategories, brandId, skuName, skuPrice, weight, images, fixedStatus, isSale, addParams) }