- FilterMb4 added.

utils/utils.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"git.rosy.net.cn/baseapi"
 	"github.com/satori/go.uuid"
@@ -147,3 +148,15 @@ func SendFakeRequest(method, url, body, contentType string) (*http.Response, err
 	client := &http.Client{}
 	return client.Do(request)
 }
+
+// 过滤 utf8mb4（比如emoji表情）
+func FilterMb4(content string) string {
+	newContent := &strings.Builder{}
+	for _, value := range content {
+		size := utf8.RuneLen(value)
+		if size >= 1 && size <= 3 {
+			newContent.WriteRune(value)
+		}
+	}
+	return newContent.String()
+}

utils/utils_test.go

@@ -77,3 +77,28 @@ func TestJson(t *testing.T) {
 	t.Fatal(1)
 	t.Fail()
 }
+
+func BenchmarkFilterMb4(b *testing.B) {
+	b.ResetTimer()
+	str := `
+	{"code":"0","detail":null,"msg":"成功","result":{"maxPageSize":100,"page":1,"pageNo":1,"pageSize":20,"resultList":[{"acceptDeadline":"2018-07-18 09:00:00","adjustCount":0,"adjustId":0,"adjustIsExists":false,"appVersion":"H5","businessTag":"dj_new_cashier;dj_aging_nextday;dj_sx;","businessType":1,"buyerCity":"1315","buyerCityName":"厦门市","buyerCoordType":2,"buyerCountry":"3486","buyerCountryName":"湖里区","buyerFullAddress":"厦门市湖里区厦门湖里万达广场SOHO公寓D1栋308室","buyerFullName":"女神","buyerIp":1879427610,"buyerLat":24.50275,"buyerLng":118.17988,"buyerMobile":"13285650460","buyerPin":"JD_21e20cdd6f92b6d","buyerPinType":0,"buyerPoi":"厦门湖里万达广场SOHO公寓","buyerProvince":"0","buyerTelephone":"13285650460","cancelOrderIsReadable":true,"clientOrderPreStartDeliveryTime":"2018-07-18 09:30:00","deliveryBillNo":"817096693000041","deliveryCarrierName":"达达专送","deliveryCarrierNo":"9966","deliveryDelayAgreed":0,"deliveryPackageVolume":0.0,"deliveryPackageWeight":4.45,"deliveryStationName":"京西菜市-钟宅店","deliveryStationNo":"11736984","deliveryStationNoIsv":"100163","deliveryType":1,"discount":[{"discountCode":"201807171-101224909#317801315","discountDetailType":1,"discountName":"钟宅39减10","discountPrice":1000,"discountType":3,"orderShareRatioData":"promotionId=101224909\u0026venderShareProportion=100\u0026platShareProportion=0\u0026platPayMoney=0\u0026venderPayMoney=1000\u0026createPin=jialuluxm","skuIds":"2013749586,2013496693,2013267451,2013587798"}],"equipmentId":"864372039949845-6091f3baaa99","industryTag":3,"isDeleted":false,"isGroupon":false,"isJDGetCash":true,"lastFourDigitsOfBuyerMobile":"1007","localDeliveryMoney":0,"merchantPaymentDistanceFreightMoney":200,"middleNumBindingId":"12180717210601378227528","middleNumBindingTime":"2018-07-19 10:00:00","orderAdditionFreightMoney":0,"orderAgingType":9,"orderBalanceUsed":0,"orderBaseFreightMoney":400,"orderBaseReceivableFreight":0,"orderBuyerPayableMoney":4831,"orderBuyerRemark":"\n如遇缺货：其他商品继续配送（缺货商品退款）","orderDiscountMoney":1000,"orderDistanceStepFreight":0,"orderFinanceOrgCode":706,"orderFreightMoney":400,"orderGoodsMoney":5381,"orderId":817096693000041,"orderInvoiceOpenMark":2,"orderIsClosed":false,"orderJingdouMoney":0,"orderLadderFreightMoney":0,"orderNum":3,"orderPayType":4,"orderPreDeliveryTime":"2018-07-18 10:00:00","orderPreEndDeliveryTime":"2018-07-18 09:30:00","orderPreStartDeliveryTime":"2018-07-18 09:30:00","orderPurchaseTime":"2018-07-17 21:06:01","orderReceivableFreight":400,"orderSkuType":1,"orderStartTime":"2018-07-17 21:04:52","orderStatus":41000,"orderStatusTime":"2018-07-17 21:06:01","orderStockOwner":3,"orderTotalMoney":5381,"orderType":10000,"orderVenderChargeMoney":0,"orgCode":"320406","packagingMoney":50,"payChannel":9002,"pickDeadline":"2018-07-18 09:00:00","pickMark":1,"platformPointsDeductionMoney":0,"printMark":1,"produceStationName":"京西菜市-钟宅店","produceStationNo":"11736984","produceStationNoIsv":"100163","product":[{"adjustMode":0,"canteenMoney":0,"categoryId":"20307,20353,20362","firstCategoryName":"水果/蔬菜","isGift":false,"promotionType":1,"skuCount":1,"skuId":2013749586,"skuIdIsv":"4375","skuJdPrice":3800,"skuName":"土猪猪蹄约800g/份","skuSpuId":0,"skuStockOwner":0,"skuStorePrice":3800,"skuWeight":0.8},{"adjustMode":0,"canteenMoney":0,"categoryId":"20307,20353,20361","firstCategoryName":"水果/蔬菜","isGift":false,"promotionType":1,"skuCount":1,"skuId":2013496693,"skuIdIsv":"3569","skuJdPrice":85,"skuName":"小香葱约50g/份","skuSpuId":0,"skuStockOwner":0,"skuStorePrice":85,"skuWeight":0.05},{"adjustMode":0,"canteenMoney":0,"categoryId":"20307,20353,20362","firstCategoryName":"水果/蔬菜","isGift":false,"promotionType":1,"skuCount":1,"skuId":2013267451,"skuIdIsv":"3340","skuJdPrice":998,"skuName":"高山娃娃菜750g/包","skuSpuId":0,"skuStockOwner":0,"skuStorePrice":998,"skuWeight":3.0},{"adjustMode":0,"canteenMoney":0,"categoryId":"20307,20353,20362","firstCategoryName":"水果/蔬菜","isGift":false,"promotionType":1,"skuCount":1,"skuId":2013587798,"skuIdIsv":"4114","skuJdPrice":498,"skuName":"[鲜]甜白菜约600g/份","skuSpuId":0,"skuStockOwner":0,"skuStorePrice":498,"skuWeight":0.6}],"srcInnerOrderId":0,"srcInnerType":0,"srcOrderId":"817096693000041","srcOrderType":2,"srcPlatId":4,"storeLat":24.52385,"storeLng":118.16938,"tips":0,"ts":"2018-07-17 21:06:01","updatePin":"ocs-buffer","userTip":0,"venderBalanceDeductMoney":0,"venderIntegralDeductMoney":0,"yn":false}],"totalCount":1,"totalPage":1},"success":true}	
+	`
+	for i := 0; i < b.N; i++ {
+		_ = FilterMb4(str)
+	}
+}
+
+func TestFilterMb4(t *testing.T) {
+	str := `
+	{"realip":"120.52.149.107","@timestamp":"2018-07-17T21:30:19+08:00","host":"callback.jingxicaishi.com","request":"POST /djsw/pushDeliveryStatus HTTP/1.1","req_body":"timestamp=2018-07-17%2B21%253A30%253A19&sign=0027B8694AF7FF818A7E4FCFF13A8A5C&v=1.0&jd_param_json=%257B%2522createPin%2522%253A%2522JD_21028ea53427da6%2522%252C%2522deliveryCarrierName%2522%253A%2522%25E8%25BE%25BE%25E8%25BE%25BE%25E5%25B9%25B3%25E5%258F%25B0%2522%252C%2522deliveryCarrierNo%2522%253A%25229966%2522%252C%2522deliveryManName%2522%253A%2522%25E8%25AE%25B8%25E8%25BF%259E%25E8%25BE%2589%2522%252C%2522deliveryManNo%2522%253A%252213731713474%2522%252C%2522deliveryManPhone%2522%253A%252213731713474%2522%252C%2522deliveryStatus%2522%253A%252240%2522%252C%2522deliveryStatusTime%2522%253A%25222018-07-17%2B21%253A30%253A19%2522%252C%2522failType%2522%253A%2522%2522%252C%2522opTime%2522%253A%25221531834219636%2522%252C%2522orderId%2522%253A%2522817091517000222%2522%252C%2522remark%2522%253A%2522%25E5%25B7%25B2%25E5%25AE%258C%25E6%2588%2590%2522%257D&token=91633f2a-c5f5-4982-a925-a220d19095c3&app_key=1dba76d40cac446ca500c0391a0b6c9d&format=json","status":"200","size":38,"ua":"Jakarta Commons-HttpClient/3.1","cookie":"","req_time":"0.063","uri":"/djsw/pushDeliveryStatus","referer":"","xff":"","ups_status":"200","ups_addr":"172.27.0.10:8080","ups_time":"0.008"}
+	`
+	if FilterMb4(str) != str {
+		t.Fatal("FilterMb4 filtered normal characters")
+	}
+
+	str2 := "abcd中国人"
+	str3 := "😋" + str2 + "😋"
+	if FilterMb4(str3) != str2 {
+		t.Fatal("FilterMb4 can not filter mb4 characters")
+	}
+}