Merge branch 'master' into tester3

data/ignore_ids_safety_scan.json

@@ -1414,7 +1414,9 @@
                 "70612": "The maintainer and multiple third parties believe that this vulnerability isn't valid because users shouldn't use untrusted templates without sandboxing.",
                 "67599": "** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the highest version number, even if the user had intended to obtain a private package from a private index. This only affects use of the --extra-index-url option, and exploitation requires that the package does not already exist in the public index (and thus the attacker can put the package there with an arbitrary version number). NOTE: it has been reported that this is intended functionality and the user is responsible for using --extra-index-url securely.",
                 "70790": "Tqdm version 4.66.3 addresses CVE-2024-34062",
-                "67895": "CVE-2024-3651 impacts the idna.encode() function"
+                "67895": "CVE-2024-3651 impacts the idna.encode() function",
+                "71601": "Transformers version upgrade needs to be handled in a separate image",
+                "71670": "Pytorch version upgrade needs to be handled in a separate image"
             }
         },
         "inference-neuron": {

huggingface/pytorch/inference/docker/2.1/py3/Dockerfile.cpu

@@ -72,6 +72,7 @@ RUN apt-get update \
     libssl-dev \
     libxext6 \
     libxrender-dev \
+    linux-libc-dev \
     zlib1g-dev \
  && apt-get autoremove -y \
  && rm -rf /var/lib/apt/lists/* \

release_images_inference.yml

@@ -61,9 +61,11 @@ release_images:
     version: "0.28.0"
     arch_type: "x86"
     inference:
-      device_types: [ "cpu" ]
+      device_types: [ "gpu" ]
       python_versions: [ "py310" ]
       os_version: "ubuntu22.04"
+      lmi_version: "10.0.0"
+      cuda_version: "cu124"
       example: False
       disable_sm_tag: True
       force_release: False

tensorflow/inference/docker/2.14/py3/Dockerfile.ec2.cpu.core_packages.json

@@ -6,10 +6,13 @@
         "version_specifier": "<3.0"
     },
     "awscli": {
-        "version_specifier": "<2"
+        "version_specifier": ">=1.33.13,<2"
     },
     "tensorflow-serving-api": {
         "version_specifier": "==2.14.1",
         "skip": "True"
+    },
+    "urllib3": {
+        "version_specifier": ">=2.0.7"
     }
 }

tensorflow/inference/docker/2.14/py3/Dockerfile.sagemaker.cpu.core_packages.json

@@ -6,7 +6,7 @@
         "version_specifier":"<3.0"
     },
     "awscli":{
-        "version_specifier":"<2"
+        "version_specifier": ">=1.33.13,<2"
     },
     "tensorflow-serving-api":{
         "version_specifier":"==2.14.1",
@@ -17,5 +17,8 @@
     },
     "gunicorn":{
         "version_specifier":"==20.1.0"
+    },
+    "urllib3": {
+        "version_specifier": ">=2.0.7"
     }
 }

tensorflow/inference/docker/2.14/py3/cu118/Dockerfile.ec2.gpu.core_packages.json

@@ -6,10 +6,13 @@
         "version_specifier": "<3.0"
     },
     "awscli": {
-        "version_specifier": "<2"
+        "version_specifier": ">=1.33.13,<2"
     },
     "tensorflow-serving-api-gpu": {
         "version_specifier": "==2.14.1",
         "skip": "True"
+    },
+    "urllib3": {
+        "version_specifier": ">=2.0.7"
     }
 }

tensorflow/inference/docker/2.14/py3/cu118/Dockerfile.sagemaker.gpu.core_packages.json

@@ -6,7 +6,7 @@
         "version_specifier":"<3.0"
     },
     "awscli":{
-        "version_specifier":"<2"
+        "version_specifier": ">=1.33.13,<2"
     },
     "tensorflow-serving-api-gpu":{
         "version_specifier":"==2.14.1",
@@ -17,5 +17,8 @@
     },
     "gunicorn":{
         "version_specifier":"==20.1.0"
+    },
+    "urllib3": {
+        "version_specifier": ">=2.0.7"
     }
 }

tensorflow/training/docker/2.13/py3/Dockerfile.sagemaker.cpu.core_packages.json

@@ -1,6 +1,6 @@
 {
    "awscli":{
-      "version_specifier":"<2"
+      "version_specifier": ">=1.33.13,<2"
    },
    "pyyaml":{
       "version_specifier":">=6.0,<6.1"
@@ -51,7 +51,10 @@
    "sagemaker-tensorflow":{
       "version_specifier":"==2.13.0.1.19.1"
    },
-    "jinja2": {
+   "jinja2": {
       "version_specifier": ">=3.1.4"
-    }
+   },
+   "urllib3": {
+      "version_specifier": ">=2.0.7"
+   }
 }

tensorflow/training/docker/2.13/py3/cu118/Dockerfile.sagemaker.gpu.core_packages.json

@@ -1,6 +1,6 @@
 {
    "awscli":{
-      "version_specifier":"<2"
+      "version_specifier": ">=1.33.13,<2"
    },
    "pyyaml":{
       "version_specifier":">=6.0,<6.1"
@@ -54,7 +54,10 @@
    "sagemaker-tensorflow":{
       "version_specifier":"==2.13.0.1.19.1"
    },
-    "jinja2": {
+   "jinja2": {
       "version_specifier": ">=3.1.4"
-    }
+   },
+   "urllib3": {
+      "version_specifier": ">=2.0.7"
+   }
 }

tensorflow/training/docker/2.14/py3/Dockerfile.ec2.cpu.core_packages.json

@@ -1,6 +1,6 @@
 {
   "awscli": {
-    "version_specifier": "<2"
+    "version_specifier": ">=1.33.13,<2"
   },
   "protobuf": {
     "version_specifier": ">=3.20.3,<4"
@@ -31,5 +31,8 @@
   },
   "tensorflow-metadata": {
     "version_specifier": ">=1.14.0"
+  },
+  "urllib3": {
+    "version_specifier": ">=2.0.7"
   }
 }

tensorflow/training/docker/2.14/py3/Dockerfile.sagemaker.cpu.core_packages.json

@@ -1,6 +1,6 @@
 {
   "awscli": {
-    "version_specifier": "<2"
+    "version_specifier": ">=1.33.13,<2"
   },
   "cython": {
     "version_specifier": "<3.0"
@@ -63,5 +63,8 @@
   },
   "jinja2": {
     "version_specifier": ">=3.1.4"
+  },
+  "urllib3": {
+    "version_specifier": ">=2.0.7"
   }
 }

tensorflow/training/docker/2.14/py3/cu118/Dockerfile.ec2.gpu.core_packages.json

@@ -1,6 +1,6 @@
 {
   "awscli": {
-    "version_specifier": "<2"
+    "version_specifier": ">=1.33.13,<2"
   },
   "protobuf": {
     "version_specifier": ">=3.20.3,<4"
@@ -31,5 +31,8 @@
   },
   "tensorflow-metadata": {
     "version_specifier": ">=1.14.0"
+  },
+  "urllib3": {
+    "version_specifier": ">=2.0.7"
   }
 }

tensorflow/training/docker/2.14/py3/cu118/Dockerfile.sagemaker.gpu.core_packages.json

@@ -1,6 +1,6 @@
 {
   "awscli": {
-    "version_specifier": "<2"
+    "version_specifier": ">=1.33.13,<2"
   },
   "cython": {
     "version_specifier": "<3.0"
@@ -63,5 +63,8 @@
   },
   "jinja2": {
     "version_specifier": ">=3.1.4"
+  },
+  "urllib3": {
+    "version_specifier": ">=2.0.7"
   }
 }

test/dlc_tests/sanity/test_ecr_scan.py

@@ -30,6 +30,9 @@
     is_generic_image,
     get_allowlist_path_for_enhanced_scan_from_env_variable,
     get_ecr_scan_allowlist_path,
+    get_sha_of_an_image_from_ecr,
+    is_mainline_context,
+    is_test_phase,
 )
 from test.test_utils import ecr as ecr_utils
 from test.test_utils.security import (
@@ -81,6 +84,41 @@ def get_minimum_sev_threshold_level(image):
     return "HIGH"
 
 
+def upload_allowlist_to_image_data_storage_bucket(
+    ecr_client_for_enhanced_scanning_repo, ecr_enhanced_repo_uri, allowlist_for_daily_scans
+):
+    """
+    This method gets the unique identifier of the image from ecr and uses it as the path to
+    upload image's allowlisted vulnerabilities to the image-data-storage s3 bucket. This
+    information is used for the scanning dashboard to identify allowlisted vulnerabilities.
+
+    :param ecr_client_for_enhanced_scanning_repo: ecr boto3 client for image
+    :param ecr_enhanced_repo_uri: str Image URI from ecr
+    :param allowlist_for_daily_scans: list of allowlisted vulnerabilities for the image
+    """
+    image_sha = get_sha_of_an_image_from_ecr(
+        ecr_client_for_enhanced_scanning_repo, ecr_enhanced_repo_uri
+    )
+    s3_resource = boto3.resource("s3")
+    sts_client = boto3.client("sts")
+    account_id = sts_client.get_caller_identity().get("Account")
+    s3object = s3_resource.Object(
+        f"image-data-storage-{account_id}", image_sha + "/ecr_allowlist.json"
+    )
+    s3object.put(
+        Body=(
+            bytes(
+                json.dumps(
+                    allowlist_for_daily_scans.vulnerability_list,
+                    indent=4,
+                    cls=test_utils.EnhancedJSONEncoder,
+                ).encode("UTF-8")
+            )
+        )
+    )
+    LOGGER.info(f"ECR allowlist uploaded to image-data-storage s3 bucket")
+
+
 def conduct_preprocessing_of_images_before_running_ecr_scans(image, ecr_client, sts_client, region):
     """
     Conducts the following steps before starting any kind of ecr test:
@@ -220,6 +258,7 @@ def helper_function_for_leftover_vulnerabilities_from_enhanced_scanning(
 
     remaining_vulnerabilities = ecr_image_vulnerability_list - image_scan_allowlist
     LOGGER.info(f"ECR Enhanced Scanning test completed for image: {image}")
+    allowlist_for_daily_scans = image_scan_allowlist
 
     if remove_non_patchable_vulns:
         non_patchable_vulnerabilities = ECREnhancedScanVulnerabilityList(
@@ -238,6 +277,7 @@ def helper_function_for_leftover_vulnerabilities_from_enhanced_scanning(
             image_scan_allowlist=image_scan_allowlist,
             non_patchable_vulnerabilities=non_patchable_vulnerabilities,
         )
+        allowlist_for_daily_scans = future_allowlist
 
         # Note that ecr_enhanced_repo_uri will point to enhanced scan repo, thus we use image in the unique_s3 function below
         # as we want to upload the allowlist to the location that has repo of the actual image.
@@ -272,6 +312,12 @@ def helper_function_for_leftover_vulnerabilities_from_enhanced_scanning(
         LOGGER.info(
             f"[NonPatchableVulns] [image_uri:{ecr_enhanced_repo_uri}] {json.dumps(non_patchable_vulnerabilities.vulnerability_list, cls= test_utils.EnhancedJSONEncoder)}"
         )
+
+    if is_mainline_context() and is_test_phase() and not is_generic_image():
+        upload_allowlist_to_image_data_storage_bucket(
+            ecr_client_for_enhanced_scanning_repo, ecr_enhanced_repo_uri, allowlist_for_daily_scans
+        )
+
     return remaining_vulnerabilities, ecr_enhanced_repo_uri
 
 

test/test_utils/__init__.py

@@ -781,6 +781,10 @@ def is_time_for_invoking_ecr_scan_failure_routine_lambda():
     return current_utc_time.tm_hour == 16 and (0 < current_utc_time.tm_min < 20)
 
 
+def is_test_phase():
+    return "TEST_TYPE" in os.environ
+
+
 def _get_remote_override_flags():
     try:
         s3_client = boto3.client("s3")