- 饿百与京东拉门店数据基本OK

business/netspider/netspider.go

@@ -0,0 +1,86 @@
+package netspider
+
+import (
+	"fmt"
+
+	"git.rosy.net.cn/baseapi/utils"
+	"git.rosy.net.cn/jx-callback/business/jxutils/ditu"
+	"git.rosy.net.cn/jx-callback/business/jxutils/jxcontext"
+	"git.rosy.net.cn/jx-callback/business/jxutils/tasksch"
+	"git.rosy.net.cn/jx-callback/business/model"
+	"git.rosy.net.cn/jx-callback/business/model/dao"
+	"git.rosy.net.cn/jx-callback/business/partner"
+)
+
+func GetCityShops(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorIDs []int, cityCode, radius, gridWith int) (pageStoreList []*model.PageShop, err error) {
+	coordList := ditu.GetCityCoordinateList(cityCode, radius, gridWith)
+	if len(coordList) > 0 {
+		task := tasksch.NewParallelTask("GetCityShops", nil, ctx,
+			func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
+				vendorID := batchItemList[0].(int)
+				storeList, err2 := getStoreListByCoordinates(ctx, task, vendorID, utils.Int2Str(cityCode), coordList)
+				if err = err2; err == nil {
+					retVal = storeList
+				}
+				return retVal, err
+			}, vendorIDs)
+		tasksch.AddChild(parentTask, task).Run()
+		list, err2 := task.GetResult(0)
+		if err = err2; err != nil && len(list) == 0 {
+			return nil, err
+		}
+		for _, v := range list {
+			v2 := v.(*model.PageShop)
+			v2.CityCode = cityCode
+			dao.WrapAddIDCULDEntity(v, ctx.GetUserName())
+			pageStoreList = append(pageStoreList, v.(*model.PageShop))
+		}
+	}
+	return pageStoreList, err
+}
+
+func getStoreListByCoordinates(ctx *jxcontext.Context, parentTask tasksch.ITask, vendorID int, cityInfo string, coordList []*ditu.Coordinate) (storeList []*model.PageShop, err error) {
+	if handler, _ := partner.GetPurchasePlatformFromVendorID(vendorID).(partner.IPurchasePlatformNetSpiderHandler); handler != nil {
+		task1 := tasksch.NewParallelTask(fmt.Sprintf("GetStoreListByCoordinate[%s] get list", model.VendorChineseNames[vendorID]), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx,
+			func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
+				pos := batchItemList[0].(*ditu.Coordinate)
+				storeIDList, err := handler.GetStoreIDListByCoordinates(ctx, pos)
+				if err != nil {
+					return nil, err
+				}
+				return storeIDList, nil
+			}, coordList)
+		tasksch.AddChild(parentTask, task1).Run()
+		fullStoreIDs, err2 := task1.GetResult(0)
+		if err = err2; err != nil && len(fullStoreIDs) == 0 {
+			return nil, err
+		}
+		storeIDMap := make(map[string]int)
+		for _, v := range fullStoreIDs {
+			storeIDMap[v.(string)] = 1
+		}
+		var storeIDs []string
+		for storeID := range storeIDMap {
+			storeIDs = append(storeIDs, storeID)
+		}
+
+		task2 := tasksch.NewParallelTask(fmt.Sprintf("GetStoreListByCoordinate[%s] get detail", model.VendorChineseNames[vendorID]), tasksch.NewParallelConfig().SetIsContinueWhenError(true), ctx,
+			func(task *tasksch.ParallelTask, batchItemList []interface{}, params ...interface{}) (retVal interface{}, err error) {
+				storeID := batchItemList[0].(string)
+				storePageInfo, err := handler.GetStorePageInfo(ctx, cityInfo, storeID)
+				if err == nil && storePageInfo != nil {
+					return []interface{}{storePageInfo}, nil
+				}
+				return nil, err
+			}, storeIDs)
+		tasksch.AddChild(parentTask, task2).Run()
+		shopList, err2 := task2.GetResult(0)
+		if err = err2; err != nil && len(shopList) == 0 {
+			return nil, err
+		}
+		for _, v := range shopList {
+			storeList = append(storeList, v.(*model.PageShop))
+		}
+	}
+	return storeList, err
+}

business/netspider/netspider_test.go

@@ -0,0 +1,41 @@
+package netspider
+
+import (
+	"testing"
+
+	"git.rosy.net.cn/jx-callback/business/jxutils/ditu"
+	"git.rosy.net.cn/jx-callback/business/jxutils/jxcontext"
+
+	"git.rosy.net.cn/baseapi/utils"
+	_ "git.rosy.net.cn/jx-callback/business/partner/purchase/ebai"
+	_ "git.rosy.net.cn/jx-callback/business/partner/purchase/jd"
+	"git.rosy.net.cn/jx-callback/globals/testinit"
+)
+
+func init() {
+	testinit.Init()
+}
+
+func TestGetStoreListByCoordinate(t *testing.T) {
+	storeList, err := getStoreListByCoordinates(jxcontext.AdminCtx, nil, 3, "成都",
+		[]*ditu.Coordinate{
+			&ditu.Coordinate{
+				Lng: 104.057218,
+				Lat: 30.6949,
+			},
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(utils.Format4Output(storeList, false))
+	t.Log(len(storeList))
+}
+
+func TestGetCityShops(t *testing.T) {
+	shopList, err := GetCityShops(jxcontext.AdminCtx, nil, []int{0, 3}, 510100, 5000, 3000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(utils.Format4Output(shopList, false))
+	t.Log(len(shopList))
+}