Merge pull request #31 from microsoft/sanitize_logs

Sanitize sas tokens in logs
microsoft · May 15, 2023 · 22ff453 · 22ff453
2 parents 5405a84 + b67aad5
commit 22ff453
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 11 deletions.
diff --git a/src/PackageUploader.ClientApi.Test/LogSanitizerTest.cs b/src/PackageUploader.ClientApi.Test/LogSanitizerTest.cs
@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using PackageUploader.ClientApi.Client.Ingestion.Sanitizers;
+
+namespace PackageUploader.ClientApi.Test;
+
+[TestClass]
+public class LogSanitizerTest
+{
+
+    [TestMethod]
+    public void SanitizeGamePackageResponse()
+    {
+        const string response = 
+            "{\"resourceType\":\"GamePackage\",\"packageType\":\"\",\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij\",\"uploadInfo\":{\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij\",\"xfusId\":\"12345678-1234-1234-1234-1234567890ab\",\"fileSasUri\":\"https://upload.xboxlive.com/upload/Asset/0123456789012345678?token=%3fsv%3d2021-01-01%26sr%3dc%26si%3d12345678-1234-1234-1234-1234567890ab2022-4-5-00-04-32%26sig%3d000000000000000000000000000000000000000000000000%26se%3d2022-04-10T00%253A37%253A32Z%26t%3dZZZ\",\"token\":\"?sv=2021-01-01%26sr%3dc%26si%3d12345678-1234-1234-1234-1234567890ab2022-4-5-00-04-32%26sig%3d000000000000000000000000000000000000000000000000%26se%3d2022-04-10T00%253A37%253A32Z%26t%3dZZZ\",\"uploadDomain\":\"https://upload.xboxlive.com\",\"xfusTenant\":\"ZZZ\"},\"state\":\"PendingUpload\",\"@odata.etag\":\"\"12345678-1234-1234-1234-1234567890ab\"\",\"id\":\"12345678-1234-1234-1234-1234567890ab\"}";
+        const string expectedSanitizedResponse = 
+            "{\"resourceType\":\"GamePackage\",\"packageType\":\"\",\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij\",\"uploadInfo\":{\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij\",\"xfusId\":\"12345678-1234-1234-1234-1234567890ab\",\"fileSasUri\":\"https://upload.xboxlive.com/upload/Asset/0123456789012345678?REDACTED\",\"token\":\"REDACTED\",\"uploadDomain\":\"https://upload.xboxlive.com\",\"xfusTenant\":\"ZZZ\"},\"state\":\"PendingUpload\",\"@odata.etag\":\"\"12345678-1234-1234-1234-1234567890ab\"\",\"id\":\"12345678-1234-1234-1234-1234567890ab\"}";
+
+        var sanitizedResponse = LogSanitizer.SanitizeJsonResponse(response);
+
+        Assert.AreEqual(expectedSanitizedResponse, sanitizedResponse);
+    }
+
+    [TestMethod]
+    public void SanitizePackageAssetResponse()
+    {
+        const string response =
+            "{\"resourceType\":\"PackageAsset\",\"type\":\"EraSymbolFile\",\"name\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip\",\"isCommitted\":true,\"packageId\":\"12345678-1234-1234-1234-1234567890ab\",\"packageType\":\"\",\"createdDate\":\"0001-01-01T00:00:00Z\",\"binarySizeInBytes\":229689131,\"uploadInfo\":{\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip\",\"xfusId\":\"12345678-1234-1234-1234-1234567890ab\",\"fileSasUri\":\"https://bloburl.blob.core.windows.net/12345678-1234-1234-1234-1234567890ab/Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip?sv=2021-01-01%26sr%3dc%26si%3d12345678-1234-1234-1234-1234567890ab2022-4-5-00-04-32%26sig%3d000000000000000000000000000000000000000000000000%26se%3d2022-04-10T00%253A37%253A32Z%26t%3dZZZ\",\"xfusTenant\":\"ZZZ\"},\"id\":\"EraSymbolFile\"}";
+        const string expectedSanitizedResponse =
+            "{\"resourceType\":\"PackageAsset\",\"type\":\"EraSymbolFile\",\"name\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip\",\"isCommitted\":true,\"packageId\":\"12345678-1234-1234-1234-1234567890ab\",\"packageType\":\"\",\"createdDate\":\"0001-01-01T00:00:00Z\",\"binarySizeInBytes\":229689131,\"uploadInfo\":{\"fileName\":\"Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip\",\"xfusId\":\"12345678-1234-1234-1234-1234567890ab\",\"fileSasUri\":\"https://bloburl.blob.core.windows.net/12345678-1234-1234-1234-1234567890ab/Microsoft.GamePackage_1.1.1.0_x64_abcdefghij_erasymbol.zip?REDACTED\",\"xfusTenant\":\"ZZZ\"},\"id\":\"EraSymbolFile\"}";
+
+        var sanitizedResponse = LogSanitizer.SanitizeJsonResponse(response);
+
+        Assert.AreEqual(expectedSanitizedResponse, sanitizedResponse);
+    }
+
+}
diff --git a/src/PackageUploader.ClientApi/Client/Ingestion/Client/HttpRestClient.cs b/src/PackageUploader.ClientApi/Client/Ingestion/Client/HttpRestClient.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using PackageUploader.ClientApi.Client.Ingestion.Models.Internal;
+using PackageUploader.ClientApi.Client.Ingestion.Sanitizers;
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
@@ -31,12 +32,13 @@ internal abstract class HttpRestClient : IHttpRestClient
 
     private static readonly MediaTypeHeaderValue JsonMediaTypeHeaderValue = new (MediaTypeNames.Application.Json);
     private const LogLevel VerboseLogLevel = LogLevel.Trace;
-    private const string SdkVersion = "SDK-V1.5.0";
+    private readonly string _sdkVersion;
 
-    protected HttpRestClient(ILogger logger, HttpClient httpClient)
+    protected HttpRestClient(ILogger logger, HttpClient httpClient, IngestionSdkVersion ingestionSdkVersion)
     {
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+        _sdkVersion = ingestionSdkVersion?.SdkVersion ?? "SDK-V1.0.0";
     }
 
     public async Task<T> GetAsync<T>(string subUrl, CancellationToken ct)
@@ -147,21 +149,21 @@ public async Task<TOut> PutAsync<TIn, TOut>(string subUrl, TIn body, IDictionary
         }
     }
 
-    private static HttpRequestMessage CreateJsonRequestMessage(HttpMethod method, string requestUri) =>
+    private HttpRequestMessage CreateJsonRequestMessage(HttpMethod method, string requestUri) =>
         CreateJsonRequestMessage(method, requestUri, (object) null, null);
 
-    private static HttpRequestMessage CreateJsonRequestMessage<T>(HttpMethod method, string requestUri, T inputValue) =>
+    private HttpRequestMessage CreateJsonRequestMessage<T>(HttpMethod method, string requestUri, T inputValue) =>
         CreateJsonRequestMessage(method, requestUri, inputValue, null);
 
-    private static HttpRequestMessage CreateJsonRequestMessage<T>(HttpMethod method, string requestUri, T inputValue, IDictionary<string, string> customHeaders)
+    private HttpRequestMessage CreateJsonRequestMessage<T>(HttpMethod method, string requestUri, T inputValue, IDictionary<string, string> customHeaders)
     {
         var request = new HttpRequestMessage(method, requestUri);
         if (inputValue is not null)
         {
             request.Content = JsonContent.Create(inputValue, JsonMediaTypeHeaderValue, DefaultJsonSerializerOptions);
         }
         request.Headers.Add("Request-ID", Guid.NewGuid().ToString());
-        request.Headers.Add("MethodOfAccess", SdkVersion);
+        request.Headers.Add("MethodOfAccess", _sdkVersion);
 
         if (customHeaders is not null && customHeaders.Any())
         {
@@ -189,8 +191,7 @@ private async Task LogRequestVerboseAsync(HttpRequestMessage request, Cancellati
         if (request.Content is not null)
         {
             var requestBody = await request.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
-            _logger.Log(VerboseLogLevel, "Request Body:");
-            _logger.Log(VerboseLogLevel, requestBody);
+            _logger.Log(VerboseLogLevel, "Request Body: {requestBody}", requestBody);
         }
     }
 
@@ -202,8 +203,9 @@ private async Task LogResponseVerboseAsync(HttpResponseMessage response, Cancell
         var serverRequestId = GetRequestIdFromHeaders(response.Headers);
         _logger.Log(VerboseLogLevel, "Response {statusCodeInt} {statusCode}: {reasonPhrase} [ServerRequestId: {serverRequestId}]", (int)response.StatusCode, response.StatusCode, response.ReasonPhrase ?? "", serverRequestId);
         var responseBody = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
-        _logger.Log(VerboseLogLevel, "Response Body:");
-        _logger.Log(VerboseLogLevel, responseBody);
+        responseBody = LogSanitizer.SanitizeJsonResponse(responseBody);
+
+        _logger.Log(VerboseLogLevel, "Response Body: {responseBody}", responseBody);
     }
 
     private void LogException(Exception ex)

diff --git a/src/PackageUploader.ClientApi/Client/Ingestion/IngestionExtensions.cs b/src/PackageUploader.ClientApi/Client/Ingestion/IngestionExtensions.cs
@@ -20,6 +20,7 @@ internal static class IngestionExtensions
     public static IServiceCollection AddIngestionService(this IServiceCollection services, IConfiguration config)
     {
         services.AddOptions<IngestionConfig>().Bind(config.GetSection(nameof(IngestionConfig))).ValidateDataAnnotations();
+        services.AddSingleton<IngestionSdkVersion>();
         services.AddScoped<IngestionAuthenticationDelegatingHandler>();
         services.AddHttpClient<IIngestionHttpClient, IngestionHttpClient>((serviceProvider, httpClient) =>
             {

diff --git a/src/PackageUploader.ClientApi/Client/Ingestion/IngestionHttpClient.cs b/src/PackageUploader.ClientApi/Client/Ingestion/IngestionHttpClient.cs
@@ -25,7 +25,7 @@ internal sealed class IngestionHttpClient : HttpRestClient, IIngestionHttpClient
 {
     private readonly ILogger<IngestionHttpClient> _logger;
 
-    public IngestionHttpClient(ILogger<IngestionHttpClient> logger, HttpClient httpClient) : base(logger, httpClient)
+    public IngestionHttpClient(ILogger<IngestionHttpClient> logger, HttpClient httpClient, IngestionSdkVersion ingestionSdkVersion) : base(logger, httpClient, ingestionSdkVersion)
     {
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
     }

diff --git a/src/PackageUploader.ClientApi/Client/Ingestion/IngestionSdkVersion.cs b/src/PackageUploader.ClientApi/Client/Ingestion/IngestionSdkVersion.cs
@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Reflection;
+
+namespace PackageUploader.ClientApi.Client.Ingestion;
+
+internal class IngestionSdkVersion
+{
+    public string SdkVersion { get; }
+
+    public IngestionSdkVersion()
+    {
+        SdkVersion = GetSdkVersion();
+    }
+
+    private static string GetSdkVersion()
+    {
+        var assembly = Assembly.GetExecutingAssembly();
+        var assemblyVersionAttribute = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
+
+        return assemblyVersionAttribute is null ?
+            $"SDK-V{assembly.GetName().Version?.ToString() ?? "1.0.0"}" :
+            $"SDK-V{assemblyVersionAttribute.InformationalVersion}";
+    }
+}
diff --git a/src/PackageUploader.ClientApi/Client/Ingestion/Sanitizers/LogSanitizer.cs b/src/PackageUploader.ClientApi/Client/Ingestion/Sanitizers/LogSanitizer.cs
@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Text.RegularExpressions;
+
+namespace PackageUploader.ClientApi.Client.Ingestion.Sanitizers;
+
+public static class LogSanitizer
+{
+    public static string SanitizeJsonResponse(string jsonResponse)
+    {
+        if (string.IsNullOrWhiteSpace(jsonResponse))
+            return jsonResponse;
+
+        // Sanitizing token from responses
+        var responseBody = Regex.Replace(jsonResponse, "\"token\":\\s*\"[^\"]+?([^\\/\"]+)\"", "\"token\":\"REDACTED\"");
+
+        // Sanitizing File Sas Uri from responses
+        var sasPropertyMatch = Regex.Match(responseBody, "\"fileSasUri\":\\s*\"[^\"]+?([^\\/\"]+)\"");
+        if (sasPropertyMatch.Success)
+            responseBody = responseBody.Replace(sasPropertyMatch.Groups[0].Value, sasPropertyMatch.Groups[0].Value.Split('?')[0] + "?REDACTED\"");
+
+        return responseBody;
+    }
+}