liulei/jx-callback
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- 优化net_spider

Browse Source
This commit is contained in:
gazebo
2019-09-11 11:54:54 +08:00
parent 85c8354873
commit f2a1cff346
4 changed files with 35 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
business/netspider/netspider.go
View File

@@ -26,7 +26,7 @@ func GetCityShops(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorIDs []
task := tasksch.NewParallelTask(fmt.Sprintf("GetCityShops:%d", cityCode), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx, task := tasksch.NewParallelTask(fmt.Sprintf("GetCityShops:%d", cityCode), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
vendorID := batchItemList[0].(int) vendorID := batchItemList[0].(int)
storeList, err2 := getStoreListByCoordinates(ctx, task, vendorID, utils.Int2Str(cityCode), coordList) storeList, err2 := getStoreListByCoordinates(ctx, task, vendorID, cityCode, coordList)
if err = err2; err == nil { if err = err2; err == nil {
retVal = storeList retVal = storeList
} }
@@ -40,8 +40,6 @@ func GetCityShops(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorIDs []
} }
err = nil err = nil
for _, v := range list { for _, v := range list {
v2 := v.(*model.PageShop)
v2.CityCode = cityCode
dao.WrapAddIDCULDEntity(v, ctx.GetUserName()) dao.WrapAddIDCULDEntity(v, ctx.GetUserName())
pageStoreList = append(pageStoreList, v.(*model.PageShop)) pageStoreList = append(pageStoreList, v.(*model.PageShop))
} }
@@ -49,7 +47,32 @@ func GetCityShops(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorIDs []
return pageStoreList, err return pageStoreList, err
} }
func getStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorID int, cityInfo string, coordList []*ditu.Coordinate) (storeList []*model.PageShop, err error) { func getStorePageInfo(ctx *jxcontext.Context, handler partner.IPurchasePlatformNetSpiderHandler, cityCode int, vendorStoreID string) (storePageInfo *model.PageShop, err error) {
storePageInfo, err = handler.GetStorePageInfo(ctx, vendorStoreID)
if err == nil && storePageInfo != nil {
if !(storePageInfo.Lng != 0 && storePageInfo.Lat != 0) {
storePageInfo.Lng, storePageInfo.Lat, storePageInfo.DistrictCode = api.AutonaviAPI.GetCoordinateFromAddress(storePageInfo.Address, utils.Int2Str(cityCode))
if storePageInfo.DistrictCode == 0 && cityCode != 0 {
storePageInfo.Lng, storePageInfo.Lat, storePageInfo.DistrictCode = api.AutonaviAPI.GetCoordinateFromAddress(storePageInfo.Address, "")
}
} else if storePageInfo.DistrictCode == 0 {
storePageInfo.DistrictCode = api.AutonaviAPI.GetCoordinateDistrictCode(storePageInfo.Lng, storePageInfo.Lat)
}
if storePageInfo.CityCode == 0 {
if storePageInfo.DistrictCode != 0 {
if place, err := dao.GetPlaceByCode(dao.GetDB(), storePageInfo.DistrictCode); err == nil {
storePageInfo.CityCode = place.ParentCode
}
}
if storePageInfo.CityCode == 0 {
storePageInfo.CityCode = cityCode
}
}
}
return storePageInfo, err
}
func getStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorID, cityCode int, coordList []*ditu.Coordinate) (storeList []*model.PageShop, err error) {
if len(coordList) > 0 { if len(coordList) > 0 {
if handler, _ := partner.GetPurchasePlatformFromVendorID(vendorID).(partner.IPurchasePlatformNetSpiderHandler); handler != nil { if handler, _ := partner.GetPurchasePlatformFromVendorID(vendorID).(partner.IPurchasePlatformNetSpiderHandler); handler != nil {
mainStoreIDList, _ := handler.GetStoreIDListByCoordinates(ctx, coordList[0]) mainStoreIDList, _ := handler.GetStoreIDListByCoordinates(ctx, coordList[0])
@@ -80,11 +103,8 @@ func getStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask,
task2 := tasksch.NewParallelTask(fmt.Sprintf("GetStoreListByCoordinate[%s] get detail", model.VendorChineseNames[vendorID]), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx, task2 := tasksch.NewParallelTask(fmt.Sprintf("GetStoreListByCoordinate[%s] get detail", model.VendorChineseNames[vendorID]), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx,
func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) { func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
storeID := batchItemList[0].(string) storeID := batchItemList[0].(string)
storePageInfo, err := handler.GetStorePageInfo(ctx, cityInfo, storeID) storePageInfo, err := getStorePageInfo(ctx, handler, cityCode, storeID)
if err == nil && storePageInfo != nil { if err == nil && storePageInfo != nil {
if storePageInfo.DistrictCode == 0 && (storePageInfo.Lng != 0 && storePageInfo.Lat != 0) {
storePageInfo.DistrictCode = api.AutonaviAPI.GetCoordinateDistrictCode(storePageInfo.Lng, storePageInfo.Lat)
}
return []interface{}{storePageInfo}, nil return []interface{}{storePageInfo}, nil
} }
return nil, err return nil, err
@@ -178,7 +198,7 @@ func RefreshPageStore(ctx *jxcontext.Context) (err error) {
} }
for _, v := range shopList { for _, v := range shopList {
if handler, _ := partner.GetPurchasePlatformFromVendorID(v.VendorID).(partner.IPurchasePlatformNetSpiderHandler); handler != nil { if handler, _ := partner.GetPurchasePlatformFromVendorID(v.VendorID).(partner.IPurchasePlatformNetSpiderHandler); handler != nil {
storePageInfo, err2 := handler.GetStorePageInfo(ctx, "", v.VendorStoreID) storePageInfo, err2 := getStorePageInfo(ctx, handler, v.CityCode, v.VendorStoreID)
if err2 == nil { if err2 == nil {
v.RecentOrderNum = storePageInfo.RecentOrderNum v.RecentOrderNum = storePageInfo.RecentOrderNum
v.SkuCount = storePageInfo.SkuCount v.SkuCount = storePageInfo.SkuCount

2
business/partner/partner_spider.go
View File

@@ -8,7 +8,7 @@ import (
type IPurchasePlatformNetSpiderHandler interface { type IPurchasePlatformNetSpiderHandler interface {
GetStoreIDListByCoordinates(ctx *jxcontext.Context, coord *ditu.Coordinate) (storeIDList []string, err error) GetStoreIDListByCoordinates(ctx *jxcontext.Context, coord *ditu.Coordinate) (storeIDList []string, err error)
GetStorePageInfo(ctx *jxcontext.Context, cityInfo, storeID string) (storePageInfo *model.PageShop, err error) GetStorePageInfo(ctx *jxcontext.Context, storeID string) (storePageInfo *model.PageShop, err error)
// GetStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask, cityInfo string, coordList []*ditu.Coordinate) (storeList []*model.PageShop, err error) // GetStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask, cityInfo string, coordList []*ditu.Coordinate) (storeList []*model.PageShop, err error)
} }

2
business/partner/purchase/ebai/net_spider.go
View File

@@ -22,7 +22,7 @@ func (c *PurchaseHandler) GetStoreIDListByCoordinates(ctx *jxcontext.Context, co
return storeIDList, nil return storeIDList, nil
} }
func (c *PurchaseHandler) GetStorePageInfo(ctx *jxcontext.Context, cityInfo, storeID string) (storePageInfo *model.PageShop, err error) { func (c *PurchaseHandler) GetStorePageInfo(ctx *jxcontext.Context, storeID string) (storePageInfo *model.PageShop, err error) {
shopInfo, err2 := api.EbaiAPI.GetStoreInfo2(storeID) shopInfo, err2 := api.EbaiAPI.GetStoreInfo2(storeID)
if err = err2; err == nil && shopInfo != nil { if err = err2; err == nil && shopInfo != nil {
lng, lat, _ := api.AutonaviAPI.CoordinateConvert(shopInfo.Longitude/100000, shopInfo.Latitude/100000, autonavi.CoordSysBaidu) lng, lat, _ := api.AutonaviAPI.CoordinateConvert(shopInfo.Longitude/100000, shopInfo.Latitude/100000, autonavi.CoordSysBaidu)

11
business/partner/purchase/jd/net_spider.go
View File

@@ -27,12 +27,9 @@ func (c *PurchaseHandler) GetStoreIDListByCoordinates(ctx *jxcontext.Context, co
return storeIDList, nil return storeIDList, nil
} }
func (c *PurchaseHandler) GetStorePageInfo(ctx *jxcontext.Context, cityInfo, storeID string) (storePageInfo *model.PageShop, err error) { func (c *PurchaseHandler) GetStorePageInfo(ctx *jxcontext.Context, storeID string) (storePageInfo *model.PageShop, err error) {
shopInfo, err2 := api.JdPageAPI.GetStoreInfo2(storeID) shopInfo, err2 := api.JdPageAPI.GetStoreInfo2(storeID)
if err = err2; err == nil && shopInfo != nil { if err = err2; err == nil && shopInfo != nil {
var lng, lat float64
districtCode := 0
lng, lat, districtCode = api.AutonaviAPI.GetCoordinateFromAddress(shopInfo.StoreInfo.StoreAddress, cityInfo)
return &model.PageShop{ return &model.PageShop{
Name: shopInfo.StoreInfo.StoreName, Name: shopInfo.StoreInfo.StoreName,
VendorID: model.VendorIDJD, VendorID: model.VendorIDJD,
@@ -41,9 +38,9 @@ func (c *PurchaseHandler) GetStorePageInfo(ctx *jxcontext.Context, cityInfo, sto
VendorStatus: utils.Int2Str(shopInfo.StoreInfo.StationStatus), VendorStatus: utils.Int2Str(shopInfo.StoreInfo.StationStatus),
Address: shopInfo.StoreInfo.StoreAddress, Address: shopInfo.StoreInfo.StoreAddress,
Lng: lng, Lng: 0,
Lat: lat, Lat: 0,
DistrictCode: districtCode, DistrictCode: 0,
Tel1: shopInfo.StoreInfo.StoreTel, Tel1: shopInfo.StoreInfo.StoreTel,