retry only 5xx errors. Fixes #349 (#354)

Author: jay0lee

gcp/http.go

@@ -60,23 +60,26 @@ func newClient(oauthClientID, oauthClientSecret, oauthAuthURL, oauthTokenURL, re
 	return client, nil
 }
 
-// getWithRetry calls get() and retries on HTTP failure
-// (response code != 200).
+// getWithRetry calls get() and retries on HTTP temp failure
+// (response code 500-599).
 func getWithRetry(hc *http.Client, url string) (*http.Response, error) {
 	backoff := lib.Backoff{}
 	for {
 		response, err := get(hc, url)
 		if response != nil && response.StatusCode == http.StatusOK {
 			return response, err
-		}
-
-		p, retryAgain := backoff.Pause()
-		if !retryAgain {
-			log.Debugf("HTTP error %s, retry timeout hit", err, p)
+		} else if response != nil && response.StatusCode >= 500 && response.StatusCode <= 599 {
+			p, retryAgain := backoff.Pause()
+			if !retryAgain {
+				log.Debugf("HTTP error %s, retry timeout hit", err)
+				return response, err
+			}
+			log.Debugf("HTTP error %s, retrying after %s", err, p)
+			time.Sleep(p)
+		} else {
+			log.Debugf("Permanent HTTP error %s, will not retry", err)
 			return response, err
 		}
-		log.Debugf("HTTP error %s, retrying after %s", err, p)
-		time.Sleep(p)
 	}
 }
 
@@ -95,32 +98,35 @@ func get(hc *http.Client, url string) (*http.Response, error) {
 	response, err := hc.Do(request)
 	lock.Release()
 	if err != nil {
-		return nil, fmt.Errorf("GET failure: %s", err)
+		return response, fmt.Errorf("GET failure: %s", err)
 	}
 	if response.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("GET HTTP-level failure: %s %s", url, response.Status)
+		return response, fmt.Errorf("GET HTTP-level failure: %s %s", url, response.Status)
 	}
 
 	return response, nil
 }
 
-// postWithRetry calls post() and retries on HTTP failure
-// (response code != 200).
+// postWithRetry calls post() and retries on HTTP temp failure
+// (response code 500-599).
 func postWithRetry(hc *http.Client, url string, form url.Values) ([]byte, uint, int, error) {
 	backoff := lib.Backoff{}
 	for {
 		responseBody, gcpErrorCode, httpStatusCode, err := post(hc, url, form)
 		if responseBody != nil && httpStatusCode == http.StatusOK {
 			return responseBody, gcpErrorCode, httpStatusCode, err
-		}
-
-		p, retryAgain := backoff.Pause()
-		if !retryAgain {
-			log.Debugf("HTTP error %s, retry timeout hit", err, p)
+		} else if responseBody != nil && httpStatusCode >= 500 && httpStatusCode <= 599 {
+			p, retryAgain := backoff.Pause()
+			if !retryAgain {
+				log.Debugf("HTTP error %s, retry timeout hit", err)
+				return responseBody, gcpErrorCode, httpStatusCode, err
+			}
+			log.Debugf("HTTP error %s, retrying after %s", err, p)
+			time.Sleep(p)
+		} else {
+			log.Debugf("Permanent HTTP error %s, will not retry", err)
 			return responseBody, gcpErrorCode, httpStatusCode, err
 		}
-		log.Debugf("HTTP error %s, retrying after %s", err, p)
-		time.Sleep(p)
 	}
 }