- access frequence limit

- output detail request when error

platformapi/ebaiapi/access_limit.go

@@ -0,0 +1,24 @@
+package ebaiapi
+
+import "git.rosy.net.cn/baseapi/platformapi"
+
+var (
+	apiLimitConfig = map[string]*platformapi.LimiterConfig{
+		"sku.create": &platformapi.LimiterConfig{
+			MaxAccessCount:  5 - 1,
+			TimeGapInSecond: 1,
+		},
+		"sku.delete": &platformapi.LimiterConfig{
+			MaxAccessCount:  1,
+			TimeGapInSecond: 1,
+		},
+		"sku.shop.category.delete": &platformapi.LimiterConfig{
+			MaxAccessCount:  5 - 2,
+			TimeGapInSecond: 1,
+		},
+		"sku.shop.category.create": &platformapi.LimiterConfig{
+			MaxAccessCount:  5 - 1,
+			TimeGapInSecond: 1,
+		},
+	}
+)

platformapi/ebaiapi/ebaiapi.go

@@ -33,11 +33,12 @@ type ResponseResult struct {
 }
 
 type API struct {
-	source  string
+	source       string
-	secret  string
+	secret       string
-	encrypt string
+	encrypt      string
-	client  *http.Client
+	client       *http.Client
-	config  *platformapi.APIConfig
+	config       *platformapi.APIConfig
+	speedLimiter *platformapi.Limiter
 }
 
 func New(source, secret string, config ...*platformapi.APIConfig) *API {
@@ -47,10 +48,11 @@ func New(source, secret string, config ...*platformapi.APIConfig) *API {
 		curConfig = *config[0]
 	}
 	api := &API{
-		source: source,
+		source:       source,
-		secret: secret,
+		secret:       secret,
-		client: &http.Client{Timeout: curConfig.ClientTimeout},
+		client:       &http.Client{Timeout: curConfig.ClientTimeout},
-		config: &curConfig,
+		config:       &curConfig,
+		speedLimiter: platformapi.New(apiLimitConfig),
 	}
 	return api
 }
@@ -70,6 +72,7 @@ func (a *API) signParams(params url.Values) string {
 }
 
 func (a *API) AccessAPI(cmd string, body map[string]interface{}) (retVal *ResponseResult, err error) {
+	a.speedLimiter.AccessAPI(cmd)
 	if body == nil {
 		body = make(map[string]interface{}, 0)
 	}
@@ -85,7 +88,6 @@ func (a *API) AccessAPI(cmd string, body map[string]interface{}) (retVal *Respon
 	}
 	params[signKey] = []string{a.signParams(params)}
 	encodedParams := params.Encode()
-
 	err = platformapi.AccessPlatformAPIWithRetry(a.client,
 		func() *http.Request {
 			request, _ := http.NewRequest(http.MethodPost, prodURL, strings.NewReader(encodedParams))

platformapi/limit_access_speed.go

@@ -0,0 +1,33 @@
+package platformapi
+
+import (
+	"time"
+
+	"github.com/juju/ratelimit"
+)
+
+type LimiterConfig struct {
+	MaxAccessCount  int
+	TimeGapInSecond int
+}
+
+type Limiter struct {
+	limitConfig map[string]*ratelimit.Bucket
+}
+
+func New(config map[string]*LimiterConfig) *Limiter {
+	limiter := &Limiter{
+		limitConfig: make(map[string]*ratelimit.Bucket),
+	}
+	for k, v := range config {
+		limiter.limitConfig[k] = ratelimit.NewBucketWithQuantum(time.Duration(v.TimeGapInSecond)*time.Second, int64(v.MaxAccessCount), int64(v.MaxAccessCount))
+	}
+	return limiter
+}
+
+func (l *Limiter) AccessAPI(apiName string) {
+	bucket := l.limitConfig[apiName]
+	if bucket != nil {
+		bucket.Wait(1)
+	}
+}

platformapi/limit_access_speed_test.go

@@ -0,0 +1,40 @@
+package platformapi
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"git.rosy.net.cn/baseapi/utils"
+)
+
+func TestLimitSpeed(t *testing.T) {
+	limiter := New(map[string]*LimiterConfig{
+		"limited1persecond": &LimiterConfig{
+			MaxAccessCount:  1,
+			TimeGapInSecond: 1,
+		},
+		"limited10per10second": &LimiterConfig{
+			MaxAccessCount:  10,
+			TimeGapInSecond: 10,
+		},
+	})
+
+	go func() {
+		count := 0
+		for {
+			limiter.AccessAPI("limited1persecond")
+			fmt.Printf("limited1persecond, time:%s, count:%d\n", utils.GetCurTimeStr(), count)
+			count++
+		}
+	}()
+	go func() {
+		count := 0
+		for {
+			limiter.AccessAPI("limited10per10second")
+			fmt.Printf("limited10per10second, time:%s, count:%d\n", utils.GetCurTimeStr(), count)
+			count++
+		}
+	}()
+	time.Sleep(30 * time.Second)
+}

platformapi/platformapi.go

@@ -1,7 +1,9 @@
 package platformapi
 
 import (
+	"bytes"
 	"errors"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -73,11 +75,17 @@ func init() {
 	structs.DefaultTagName = "json"
 }
 
+func getClonedData(r *bytes.Buffer) string {
+	return string(r.Bytes())
+}
+
 func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http.Request, config *APIConfig, handleResponse func(response *http.Response) (string, error)) error {
 	exceedLimitRetryCount := 0
 	recoverableErrorRetryCount := 0
 	for {
+		savedBuf := new(bytes.Buffer)
 		request := handleRequest()
+		request.Body = ioutil.NopCloser(io.TeeReader(request.Body, savedBuf))
 		response, err := client.Do(request)
 		if err != nil {
 			baseapi.SugarLogger.Debugf("AccessPlatformAPIWithRetry client.Get return err:%v", err)
@@ -86,16 +94,16 @@ func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http.
 				recoverableErrorRetryCount++
 				continue
 			} else {
-				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry access api request:%v, error:%v", request, err)
+				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry access api url:%v, request:%v, error:%v", request.URL, getClonedData(savedBuf), err)
 				return ErrAPIAccessFailed
 			}
 		}
 		defer response.Body.Close()
 		if response.StatusCode != 200 {
 			if bodyData, err := ioutil.ReadAll(response.Body); err == nil {
-				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, request:%v, response:%s", response.StatusCode, request, string(bodyData))
+				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, url:%v, request:%v, response:%s", response.StatusCode, request.URL, getClonedData(savedBuf), string(bodyData))
 			} else {
-				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, request:%v", response.StatusCode, request)
+				baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, url:%v, request:%v", response.StatusCode, request.URL, getClonedData(savedBuf))
 			}
 			return ErrHTTPCodeIsNot200
 		}
@@ -114,7 +122,7 @@ func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http.
 				continue
 			}
 		}
-		baseapi.SugarLogger.Infof("AccessPlatformAPIWithRetry failed, request:%v, response:%v, error:%v", request, response, err)
+		baseapi.SugarLogger.Infof("AccessPlatformAPIWithRetry failed, url:%v, request:%v, response:%v, error:%v", request.URL, getClonedData(savedBuf), response, err)
 		return err
 	}
 }