Accept Merge Request #35: (don -> mark)

Merge Request: 敏感词过滤
Created By: @Nathan drake
Accepted By: @XJH-Rosy
URL: https://coding.net/u/XJH-Rosy/p/jx-callback/git/merge/35

business/jxstore/cms/sku.go

@@ -508,6 +508,29 @@ func GetSkuNames(ctx *jxcontext.Context, keyword string, isBySku bool, params ma
 	return skuNamesInfo, err
 }
 
+func CheckHasSensitiveWord(word string) (bool, error) {
+	if hasSensitiveWord, sensitiveWord := IsSensitiveWordInList(word); hasSensitiveWord {
+		return true, errors.New(fmt.Sprintf("不能包含敏感词:[%s]", sensitiveWord))
+	}
+
+	return false, nil
+}
+
+func IsSensitiveWordInList(str string) (bool, string) {
+	wordList, err := dao.GetSensitiveWordList()
+	if err == nil {
+		for _, value := range wordList {
+			keyWord := value.Word
+			checkHas := strings.Contains(str, keyWord)
+			if checkHas {
+				return true, keyWord
+			}
+		}
+	}
+
+	return false, ""
+}
+
 func AddSkuName(ctx *jxcontext.Context, skuNameExt *model.SkuNameExt, userName string) (outSkuNameExt *model.SkuNameExt, err error) {
 	if skuNameExt.CategoryID == 0 {
 		return nil, errors.New("CategoryID不能为空")
@@ -520,6 +543,10 @@ func AddSkuName(ctx *jxcontext.Context, skuNameExt *model.SkuNameExt, userName s
 		}
 	}
 
+	if hasSensitiveWord, err := CheckHasSensitiveWord(skuNameExt.Name); hasSensitiveWord {
+		return nil, err
+	}
+
 	db := dao.GetDB()
 	dao.Begin(db)
 	defer func() {
@@ -622,6 +649,12 @@ func UpdateSkuName(ctx *jxcontext.Context, nameID int, payload map[string]interf
 	if err = dao.GetEntity(db, skuName); err != nil {
 		return 0, err
 	}
+	
+	newSkuName := utils.Interface2String(payload["name"])
+	if hasSensitiveWord, err := CheckHasSensitiveWord(newSkuName); hasSensitiveWord {
+		return 0, err
+	}
+
 	delete(payload, "isSpu")
 	delete(payload, "ImgHashCode")
 	delete(payload, "ImgWeimob")

business/jxstore/cms/store_sku_check.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
 	"git.rosy.net.cn/baseapi"
 	"git.rosy.net.cn/baseapi/utils"
 	"git.rosy.net.cn/jx-callback/business/jxutils"
@@ -20,7 +19,8 @@ import (
 
 const (
 	canWriteTolocal = false
-	isFilterToBeCreateAndNotSale = !false
+	needStatistic   = true
+	isFilterToBeCreateAndNotSale = true
 	parallelCount   = 5
 	fileExt         = ".xlsx"
 )
@@ -48,7 +48,26 @@ var (
 		"京西可售状态",
 		"平台可售状态",
 	}
+	statisticTitleList = []string{
+		"京西和平台商品状态",
+		"待创建",
+		"京西可售状态",
+		"数量",
+		"占比",
+	}
+	statisticDataList = [][]string{
+		{"京西有，平台无", "是", "下架"},
+		{"京西有，平台无", "是", "上架"},
+		{"京西有，平台无", "否", "下架"},
+		{"京西有，平台无", "否", "上架"},
 
+		{"京西无，平台有", "N/A", "N/A"},
+
+		{"京西有，平台有", "是", "下架"},
+		{"京西有，平台有", "是", "上架"},
+		{"京西有，平台有", "否", "下架"},
+		{"京西有，平台有", "否", "上架"},
+	}
 	diffData DiffDataLock
 
 	multiStoreAllSkuInfoMap  map[int]map[int]*partner.SkuNameInfo
@@ -74,6 +93,14 @@ type DiffData struct {
 	VendorStatus    string `json:"平台可售状态"`
 }
 
+type StatisticData struct {
+	JxVendorStatus      string `json:"京西和平台商品状态"`
+	ToBeCreate   		string `json:"待创建"`
+	JxSaleStatus 		string `json:"京西可售状态"`
+	Count           	string `json:"数量"`
+	Percent      		string `json:"占比"`
+}
+
 func (d *DiffDataLock) AppendData(vendorID int, diffData DiffData) {
 	d.locker.Lock()
 	defer d.locker.Unlock()
@@ -365,6 +392,63 @@ func IsJXCS() bool {
 	return globals.IsMainProductEnv()
 }
 
+func AddStatisticSheet(sheetName string, data []DiffData) (sheet *excel.Obj2ExcelSheetConfig) {
+	statisticData := make([]StatisticData, 9)
+	var count [9]int
+	var percent [9]float32
+	totalCount := len(data)
+	for _, rowData := range data {
+		if rowData.JxSkuName != "" && rowData.VendorSkuName == "" {
+			if rowData.ToBeCreate == "是" {
+				if rowData.JxStatus == "下架" {
+					count[0]++
+				} else if rowData.JxStatus == "上架" {
+					count[1]++
+				}
+			} else if rowData.ToBeCreate == "否" {
+				if rowData.JxStatus == "下架" {
+					count[2]++
+				} else if rowData.JxStatus == "上架" {
+					count[3]++
+				}
+			}
+		} else if rowData.JxSkuName == "" && rowData.VendorSkuName != "" {
+			if rowData.ToBeCreate == "" && rowData.JxStatus == "" {
+				count[4]++
+			}
+		} else if rowData.JxSkuName != "" && rowData.VendorSkuName != "" {
+			if rowData.ToBeCreate == "是" {
+				if rowData.JxStatus == "下架" {
+					count[5]++
+				} else if rowData.JxStatus == "上架" {
+					count[6]++
+				}
+			} else if rowData.ToBeCreate == "否" {
+				if rowData.JxStatus == "下架" {
+					count[7]++
+				} else if rowData.JxStatus == "上架" {
+					count[8]++
+				}
+			}
+		}
+	}
+	for index, value := range count {
+		percent[index] = float32(value * 100) / float32(totalCount)
+		countStr := utils.Int2Str(value)
+		percentStr := fmt.Sprintf("%.2f%%", percent[index])
+		subStatisticData := statisticDataList[index]
+		statisticData[index] = StatisticData{subStatisticData[0], subStatisticData[1], subStatisticData[2], countStr, percentStr}
+	}
+	sheetName = sheetName + "统计"
+	sheet = &excel.Obj2ExcelSheetConfig {
+		Title:       sheetName,
+		Data:        statisticData,
+		CaptionList: statisticTitleList,
+	}
+
+	return sheet
+}
+
 func WriteToExcel(task *tasksch.SeqTask, data map[int][]DiffData) {
 	var sheetList []*excel.Obj2ExcelSheetConfig
 	for key, value := range data {
@@ -375,8 +459,13 @@ func WriteToExcel(task *tasksch.SeqTask, data map[int][]DiffData) {
 			CaptionList: titleList,
 		}
 		sheetList = append(sheetList, excelConf)
+		if needStatistic {
+			statisticSheet := AddStatisticSheet(sheetName, value)
+			sheetList = append(sheetList, statisticSheet)
+		}
 		baseapi.SugarLogger.Debugf("WriteToExcel:append:%s count:%d", sheetName, len(value))
 	}
+
 	if len(sheetList) > 0 {
 		excelBin := excel.Obj2Excel(sheetList)
 		timeStr := utils.Int64ToStr(time.Now().Unix())

business/jxstore/cms/sync_store_sku.go

@@ -3,7 +3,8 @@ package cms
 import (
 	"fmt"
 	"time"
-
+	"regexp"
+	"strings"
 	"git.rosy.net.cn/baseapi/utils"
 	"git.rosy.net.cn/jx-callback/business/jxutils"
 
@@ -18,6 +19,10 @@ import (
 	"git.rosy.net.cn/jx-callback/business/partner"
 )
 
+var (
+	subSensitiveWordRegexp = regexp.MustCompile(`[^\[\]\"\}]`)
+)
+
 func CreateStoreCategoryByStoreSku(ctx *jxcontext.Context, vendorID, storeID int, vendorStoreID string, nameIDs, skuIDs []int) (err error) {
 	db := dao.GetDB()
 	dao.Begin(db)
@@ -413,6 +418,11 @@ func syncStoreSkuNew(ctx *jxcontext.Context, parentTask tasksch.ITask, isFull bo
 						}
 						if err == nil {
 							successList = batchedStoreSkuList
+						} else {
+							//handle error for sensitive words, if find, then insert to table sensitive_words
+							if sensitiveWord := GetSensitiveWord(singleStoreHandler, err.Error()); sensitiveWord != "" {
+								dao.InsertSensitiveWord(sensitiveWord, vendorID, ctx.GetUserName())
+							}
 						}
 						if len(successList) > 0 {
 							updateStoreSku(dao.GetDB(), vendorID, successList, model.SyncFlagNewMask)
@@ -579,3 +589,14 @@ func ClearRemoteStoreStuffAndSetNew(ctx *jxcontext.Context, parentTask tasksch.I
 	}
 	return rootTask.ID, err
 }
+
+func GetSensitiveWord(singleStoreHandler partner.ISingleStoreStoreSkuHandler, str string) string {
+	sensitiveWordRegexp := singleStoreHandler.GetSensitiveWordRegexp()
+	findResult := sensitiveWordRegexp.FindStringSubmatch(str)
+	if findResult != nil && len(findResult) > 1 {
+		findSubResult := subSensitiveWordRegexp.FindAllString(findResult[1], -1)
+		return strings.Join(findSubResult, "")
+	}
+
+	return ""
+}

business/model/dao/sensitive_words.go

@@ -0,0 +1,17 @@
+package dao
+
+import (
+	"git.rosy.net.cn/jx-callback/business/model"
+)
+
+func GetSensitiveWordList() (wordList []*model.SensitiveWord, err error) {
+	sql := `SELECT * FROM sensitive_word`
+	err = GetRows(nil, &wordList, sql)
+	return wordList, err
+}
+
+func InsertSensitiveWord(word string, vendorID int, userName string) error {
+	sensitiveWord := &model.SensitiveWord{Word: word, VendorID: vendorID}
+	WrapAddIDCULDEntity(sensitiveWord, userName)
+	return CreateEntity(nil, sensitiveWord)
+}

business/model/sensitive_words.go

@@ -0,0 +1,7 @@
+package model
+
+type SensitiveWord struct {
+	ModelIDCULD
+	VendorID int    `orm:"column(vendor_id)" json:"vendorID"`
+	Word     string `orm:"size(30);unique" json:"word"`
+}

business/partner/partner_store_sku.go

@@ -3,7 +3,7 @@ package partner
 import (
 	"math"
 	"time"
-
+	"regexp"
 	"git.rosy.net.cn/baseapi/utils"
 	"git.rosy.net.cn/jx-callback/business/jxutils/jxcontext"
 	"git.rosy.net.cn/jx-callback/business/jxutils/tasksch"
@@ -137,6 +137,8 @@ type ISingleStoreStoreSkuHandler interface {
 
 	IsErrCategoryExist(err error) (isExist bool)
 	IsErrCategoryNotExist(err error) (isNotExist bool)
+
+	GetSensitiveWordRegexp() *regexp.Regexp
 }
 
 func BuildSkuName(skuID int, vendorSkuID string) (skuName *SkuNameInfo) {

business/partner/purchase/ebai/store_sku2.go

@@ -2,7 +2,7 @@ package ebai
 
 import (
 	"time"
-
+	"regexp"
 	"git.rosy.net.cn/baseapi/platformapi/ebaiapi"
 	"git.rosy.net.cn/baseapi/utils"
 	"git.rosy.net.cn/jx-callback/business/jxutils"
@@ -22,6 +22,7 @@ var (
 		2: 15347484581335,  // 蔬菜类
 		3: 15347484581339,  // 其他蔬菜类休闲食品
 	}
+	sensitiveWordRegexp = regexp.MustCompile(`商品名称中含有敏感词(\[.*\])`)
 )
 
 func (p *PurchaseHandler) GetStoreSkusBatchSize(funcID int) (batchSize int) {
@@ -387,3 +388,7 @@ func vendorSkuList2Jx(vendorSkuList []*ebaiapi.SkuInfo) (skuNameList []*partner.
 	}
 	return skuNameList
 }
+
+func (p *PurchaseHandler) GetSensitiveWordRegexp() *regexp.Regexp {
+	return sensitiveWordRegexp
+}

business/partner/purchase/mtwm/store_sku2.go

@@ -1,6 +1,7 @@
 package mtwm
 
 import (
+	"regexp"
 	"git.rosy.net.cn/baseapi/platformapi/mtwmapi"
 	"git.rosy.net.cn/baseapi/utils"
 	"git.rosy.net.cn/jx-callback/business/jxutils"
@@ -24,6 +25,10 @@ const (
 	defVendorCatID = 200001903 // 生菜
 )
 
+var (
+	sensitiveWordRegexp = regexp.MustCompile(`包含敏感词：(\[.*\])`)
+)
+
 func (p *PurchaseHandler) GetStoreSkusBatchSize(funcID int) (batchSize int) {
 	switch funcID {
 	case partner.FuncUpdateStoreSkusStock, partner.FuncUpdateStoreSkusStatus, partner.FuncUpdateStoreSkusPrice:
@@ -365,3 +370,7 @@ func vendorSkuList2Jx(appFoodList []*mtwmapi.AppFood) (skuNameList []*partner.Sk
 	}
 	return skuNameList
 }
+
+func (p *PurchaseHandler) GetSensitiveWordRegexp() *regexp.Regexp {
+	return sensitiveWordRegexp
+}

globals/beegodb/beegodb.go

@@ -53,6 +53,7 @@ func Init() {
 	orm.RegisterModel(&model.NewConfig{})
 
 	orm.RegisterModel(&model.CasbinRule{})
+	orm.RegisterModel(&model.SensitiveWord{})
 	// create table
 	orm.RunSyncdb("default", false, true)
 }