perf: 请求GSE等底层系统接口响应非200时打印Header中的关键信息 #3179

src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkAIDevClient.java

@@ -73,7 +73,9 @@ public BkAIDevClient(MeterRegistry meterRegistry,
             meterRegistry,
             CommonMetricNames.BK_AI_DEV_API,
             getBkAIDevUrlSafely(bkApiGatewayProperties),
-            HttpHelperFactory.getDefaultHttpHelper()
+            HttpHelperFactory.getDefaultHttpHelper(
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
+            )
         );
         this.appProperties = appProperties;
         this.bkAIDevConfig = bkApiGatewayProperties.getBkAIDev();

src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java

@@ -79,7 +79,14 @@ public class EsbIamClient extends BkApiClient implements IIamClient {
     public EsbIamClient(MeterRegistry meterRegistry,
                         AppProperties appProperties,
                         EsbProperties esbProperties) {
-        super(meterRegistry, IAM_API, esbProperties.getService().getUrl(), HttpHelperFactory.getDefaultHttpHelper());
+        super(
+            meterRegistry,
+            IAM_API,
+            esbProperties.getService().getUrl(),
+            HttpHelperFactory.getDefaultHttpHelper(
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
+            )
+        );
         this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(),
             appProperties.getSecret(), "admin");
     }

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java

@@ -46,6 +46,7 @@
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.protocol.HttpCoreContext;
 import org.apache.http.util.EntityUtils;
+import org.slf4j.helpers.MessageFormatter;
 
 import java.io.IOException;
 
@@ -185,10 +186,12 @@ private HttpResponse execute(HttpRequestBase httpClientRequest,
                                  boolean throwExceptionWhenClientOrServerError) {
         int httpStatusCode = -1;
         String respStr = null;
+        Long contentLength = null;
         try (CloseableHttpResponse httpResponse = httpClient.execute(httpClientRequest, context)) {
             httpStatusCode = httpResponse.getStatusLine().getStatusCode();
             HttpEntity entity = httpResponse.getEntity();
             if (entity != null && entity.getContent() != null) {
+                contentLength = entity.getContentLength();
                 respStr = new String(EntityUtils.toByteArray(entity), CHARSET);
             }
             // 状态码>=400判定为失败
@@ -211,7 +214,12 @@ private HttpResponse execute(HttpRequestBase httpClientRequest,
                 return new HttpResponse(httpStatusCode, respStr, httpResponse.getAllHeaders());
             }
         } catch (IOException e) {
-            log.error("Request fail", e);
+            String message = MessageFormatter.format(
+                "Request fail, httpStatusCode={}, contentLength={}",
+                httpStatusCode,
+                contentLength
+            ).getMessage();
+            log.error(message, e);
             throw new InternalException(e, ErrorCode.API_ERROR);
         } finally {
             httpClientRequest.releaseConnection();

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java

@@ -62,7 +62,21 @@ private static WatchableHttpHelper getWatchableExtHelper(HttpHelper httpHelper)
     }
 
     public static WatchableHttpHelper getDefaultHttpHelper() {
-        HttpHelper baseHttpHelper = new BaseHttpHelper(DEFAULT_HTTP_CLIENT);
+        return getDefaultHttpHelper(null);
+    }
+
+    public static WatchableHttpHelper getDefaultHttpHelper(JobHttpClientFactory.HttpClientCustomizer customizer) {
+        HttpHelper baseHttpHelper = createHttpHelper(
+            15000,
+            15000,
+            15000,
+            500,
+            1000,
+            60,
+            false,
+            null,
+            customizer
+        );
         return getWatchableExtHelper(baseHttpHelper);
     }
 
@@ -84,7 +98,8 @@ public static HttpHelper createHttpHelper(int connRequestTimeout,
                                               int maxConnTotal,
                                               int timeToLive,
                                               boolean allowRetry,
-                                              HttpRequestRetryHandler retryHandler) {
+                                              HttpRequestRetryHandler retryHandler,
+                                              JobHttpClientFactory.HttpClientCustomizer customizer) {
         CloseableHttpClient httpClient = JobHttpClientFactory.createHttpClient(
             connRequestTimeout,
             connTimeout,
@@ -93,7 +108,8 @@ public static HttpHelper createHttpHelper(int connRequestTimeout,
             maxConnTotal,
             timeToLive,
             allowRetry,
-            retryHandler);
+            retryHandler,
+            customizer);
         return new BaseHttpHelper(httpClient);
     }
 }

src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/interceptor/LogBkApiRequestIdInterceptor.java

@@ -0,0 +1,71 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.esb.interceptor;
+
+import com.tencent.bk.job.common.constant.JobCommonHeaders;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.HttpStatus;
+import org.apache.http.StatusLine;
+import org.apache.http.protocol.HttpContext;
+
+/**
+ * 打印APIGW请求响应Header中的X-Bkapi-Request-Id，用于问题定位
+ */
+@Slf4j
+public class LogBkApiRequestIdInterceptor implements HttpResponseInterceptor {
+    @Override
+    public void process(HttpResponse response, HttpContext context) {
+        StatusLine statusLine = response.getStatusLine();
+        // 状态码为2xx的请求会在上层逻辑打印响应头X-Bkapi-Request-Id，此处不打印减少日志量
+        if (isStatusCodeSuccess(statusLine)) {
+            return;
+        }
+        // 获取并打印响应头X-Bkapi-Request-Id
+        String headerName = JobCommonHeaders.BK_GATEWAY_REQUEST_ID;
+        try {
+            if (response.containsHeader(headerName)) {
+                log.info(headerName + "=" + response.getFirstHeader(headerName).getValue());
+            } else {
+                log.info(headerName + " not found in the response");
+            }
+        } catch (Throwable t) {
+            log.warn("Failed to log header " + headerName, t);
+        }
+    }
+
+    /**
+     * 判断Http状态码是否为成功（2xx）系列状态码
+     *
+     * @param statusLine 状态行
+     * @return 是则返回true，否则返回false
+     */
+    private boolean isStatusCodeSuccess(StatusLine statusLine) {
+        return statusLine != null
+            && statusLine.getStatusCode() >= HttpStatus.SC_OK
+            && statusLine.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES;
+    }
+}

src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiClient.java

@@ -29,6 +29,7 @@
 import com.tencent.bk.job.common.constant.HttpMethodEnum;
 import com.tencent.bk.job.common.constant.JobCommonHeaders;
 import com.tencent.bk.job.common.esb.constants.EsbLang;
+import com.tencent.bk.job.common.esb.interceptor.LogBkApiRequestIdInterceptor;
 import com.tencent.bk.job.common.esb.metrics.EsbMetricTags;
 import com.tencent.bk.job.common.esb.model.BkApiAuthorization;
 import com.tencent.bk.job.common.esb.model.EsbResp;
@@ -46,6 +47,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.Header;
+import org.apache.http.HttpResponseInterceptor;
 import org.apache.http.message.BasicHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -391,4 +393,13 @@ protected Collection<Tag> getExtraMetricsTags() {
         return null;
     }
 
+    /**
+     * 获取打印APIGW RequestId的响应拦截器
+     *
+     * @return 响应拦截器
+     */
+    protected static HttpResponseInterceptor getLogBkApiRequestIdInterceptor() {
+        return new LogBkApiRequestIdInterceptor();
+    }
+
 }

src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java

@@ -89,7 +89,8 @@ public GseV2ApiClient(MeterRegistry meterRegistry,
                 2000,
                 60,
                 true,
-                new JobHttpRequestRetryHandler()
+                new JobHttpRequestRetryHandler(),
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
             )
         );
         gseBkApiAuthorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret());

src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java

@@ -66,7 +66,9 @@ public BkNoticeClient(MeterRegistry meterRegistry,
             meterRegistry,
             CommonMetricNames.BK_NOTICE_API,
             getBkNoticeUrlSafely(bkApiGatewayProperties),
-            HttpHelperFactory.getDefaultHttpHelper()
+            HttpHelperFactory.getDefaultHttpHelper(
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
+            )
         );
         this.appProperties = appProperties;
         authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret());

src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java

@@ -67,8 +67,14 @@ public class CmsiApiClient extends BkApiClient {
     public CmsiApiClient(EsbProperties esbProperties,
                          AppProperties appProperties,
                          MeterRegistry meterRegistry) {
-        super(meterRegistry, ESB_CMSI_API, esbProperties.getService().getUrl(),
-            HttpHelperFactory.getDefaultHttpHelper());
+        super(
+            meterRegistry,
+            ESB_CMSI_API,
+            esbProperties.getService().getUrl(),
+            HttpHelperFactory.getDefaultHttpHelper(
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
+            )
+        );
         this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(),
             appProperties.getSecret(), "admin");
     }

src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java

@@ -61,8 +61,14 @@ public class StandardLoginClient extends BkApiClient implements ILoginClient {
     private final AppProperties appProperties;
 
     public StandardLoginClient(EsbProperties esbProperties, AppProperties appProperties, MeterRegistry meterRegistry) {
-        super(meterRegistry, ESB_BK_LOGIN_API, esbProperties.getService().getUrl(),
-            HttpHelperFactory.getDefaultHttpHelper());
+        super(
+            meterRegistry,
+            ESB_BK_LOGIN_API,
+            esbProperties.getService().getUrl(),
+            HttpHelperFactory.getDefaultHttpHelper(
+                httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor())
+            )
+        );
         this.appProperties = appProperties;
     }