diff --git a/platformapi/ebaiapi/access_limit.go b/platformapi/ebaiapi/access_limit.go new file mode 100644 index 00000000..d307c55a --- /dev/null +++ b/platformapi/ebaiapi/access_limit.go @@ -0,0 +1,24 @@ +package ebaiapi + +import "git.rosy.net.cn/baseapi/platformapi" + +var ( + apiLimitConfig = map[string]*platformapi.LimiterConfig{ + "sku.create": &platformapi.LimiterConfig{ + MaxAccessCount: 5 - 1, + TimeGapInSecond: 1, + }, + "sku.delete": &platformapi.LimiterConfig{ + MaxAccessCount: 1, + TimeGapInSecond: 1, + }, + "sku.shop.category.delete": &platformapi.LimiterConfig{ + MaxAccessCount: 5 - 2, + TimeGapInSecond: 1, + }, + "sku.shop.category.create": &platformapi.LimiterConfig{ + MaxAccessCount: 5 - 1, + TimeGapInSecond: 1, + }, + } +) diff --git a/platformapi/ebaiapi/ebaiapi.go b/platformapi/ebaiapi/ebaiapi.go index 0888281f..d69f43f3 100644 --- a/platformapi/ebaiapi/ebaiapi.go +++ b/platformapi/ebaiapi/ebaiapi.go @@ -33,11 +33,12 @@ type ResponseResult struct { } type API struct { - source string - secret string - encrypt string - client *http.Client - config *platformapi.APIConfig + source string + secret string + encrypt string + client *http.Client + config *platformapi.APIConfig + speedLimiter *platformapi.Limiter } func New(source, secret string, config ...*platformapi.APIConfig) *API { @@ -47,10 +48,11 @@ func New(source, secret string, config ...*platformapi.APIConfig) *API { curConfig = *config[0] } api := &API{ - source: source, - secret: secret, - client: &http.Client{Timeout: curConfig.ClientTimeout}, - config: &curConfig, + source: source, + secret: secret, + client: &http.Client{Timeout: curConfig.ClientTimeout}, + config: &curConfig, + speedLimiter: platformapi.New(apiLimitConfig), } return api } @@ -70,6 +72,7 @@ func (a *API) signParams(params url.Values) string { } func (a *API) AccessAPI(cmd string, body map[string]interface{}) (retVal *ResponseResult, err error) { + a.speedLimiter.AccessAPI(cmd) if body == nil { body = make(map[string]interface{}, 0) } @@ -85,7 +88,6 @@ func (a *API) AccessAPI(cmd string, body map[string]interface{}) (retVal *Respon } params[signKey] = []string{a.signParams(params)} encodedParams := params.Encode() - err = platformapi.AccessPlatformAPIWithRetry(a.client, func() *http.Request { request, _ := http.NewRequest(http.MethodPost, prodURL, strings.NewReader(encodedParams)) diff --git a/platformapi/limit_access_speed.go b/platformapi/limit_access_speed.go new file mode 100644 index 00000000..74501c22 --- /dev/null +++ b/platformapi/limit_access_speed.go @@ -0,0 +1,33 @@ +package platformapi + +import ( + "time" + + "github.com/juju/ratelimit" +) + +type LimiterConfig struct { + MaxAccessCount int + TimeGapInSecond int +} + +type Limiter struct { + limitConfig map[string]*ratelimit.Bucket +} + +func New(config map[string]*LimiterConfig) *Limiter { + limiter := &Limiter{ + limitConfig: make(map[string]*ratelimit.Bucket), + } + for k, v := range config { + limiter.limitConfig[k] = ratelimit.NewBucketWithQuantum(time.Duration(v.TimeGapInSecond)*time.Second, int64(v.MaxAccessCount), int64(v.MaxAccessCount)) + } + return limiter +} + +func (l *Limiter) AccessAPI(apiName string) { + bucket := l.limitConfig[apiName] + if bucket != nil { + bucket.Wait(1) + } +} diff --git a/platformapi/limit_access_speed_test.go b/platformapi/limit_access_speed_test.go new file mode 100644 index 00000000..2909d66b --- /dev/null +++ b/platformapi/limit_access_speed_test.go @@ -0,0 +1,40 @@ +package platformapi + +import ( + "fmt" + "testing" + "time" + + "git.rosy.net.cn/baseapi/utils" +) + +func TestLimitSpeed(t *testing.T) { + limiter := New(map[string]*LimiterConfig{ + "limited1persecond": &LimiterConfig{ + MaxAccessCount: 1, + TimeGapInSecond: 1, + }, + "limited10per10second": &LimiterConfig{ + MaxAccessCount: 10, + TimeGapInSecond: 10, + }, + }) + + go func() { + count := 0 + for { + limiter.AccessAPI("limited1persecond") + fmt.Printf("limited1persecond, time:%s, count:%d\n", utils.GetCurTimeStr(), count) + count++ + } + }() + go func() { + count := 0 + for { + limiter.AccessAPI("limited10per10second") + fmt.Printf("limited10per10second, time:%s, count:%d\n", utils.GetCurTimeStr(), count) + count++ + } + }() + time.Sleep(30 * time.Second) +} diff --git a/platformapi/platformapi.go b/platformapi/platformapi.go index 2d19a954..86f8994e 100644 --- a/platformapi/platformapi.go +++ b/platformapi/platformapi.go @@ -1,7 +1,9 @@ package platformapi import ( + "bytes" "errors" + "io" "io/ioutil" "net" "net/http" @@ -73,11 +75,17 @@ func init() { structs.DefaultTagName = "json" } +func getClonedData(r *bytes.Buffer) string { + return string(r.Bytes()) +} + func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http.Request, config *APIConfig, handleResponse func(response *http.Response) (string, error)) error { exceedLimitRetryCount := 0 recoverableErrorRetryCount := 0 for { + savedBuf := new(bytes.Buffer) request := handleRequest() + request.Body = ioutil.NopCloser(io.TeeReader(request.Body, savedBuf)) response, err := client.Do(request) if err != nil { baseapi.SugarLogger.Debugf("AccessPlatformAPIWithRetry client.Get return err:%v", err) @@ -86,16 +94,16 @@ func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http. recoverableErrorRetryCount++ continue } else { - baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry access api request:%v, error:%v", request, err) + baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry access api url:%v, request:%v, error:%v", request.URL, getClonedData(savedBuf), err) return ErrAPIAccessFailed } } defer response.Body.Close() if response.StatusCode != 200 { if bodyData, err := ioutil.ReadAll(response.Body); err == nil { - baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, request:%v, response:%s", response.StatusCode, request, string(bodyData)) + baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, url:%v, request:%v, response:%s", response.StatusCode, request.URL, getClonedData(savedBuf), string(bodyData)) } else { - baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, request:%v", response.StatusCode, request) + baseapi.SugarLogger.Errorf("AccessPlatformAPIWithRetry HTTP code is:%d, url:%v, request:%v", response.StatusCode, request.URL, getClonedData(savedBuf)) } return ErrHTTPCodeIsNot200 } @@ -114,7 +122,7 @@ func AccessPlatformAPIWithRetry(client *http.Client, handleRequest func() *http. continue } } - baseapi.SugarLogger.Infof("AccessPlatformAPIWithRetry failed, request:%v, response:%v, error:%v", request, response, err) + baseapi.SugarLogger.Infof("AccessPlatformAPIWithRetry failed, url:%v, request:%v, response:%v, error:%v", request.URL, getClonedData(savedBuf), response, err) return err } }