[Hot-fix] Pinning DLAMIs to prevent failures due to nvidia-docker dep…

…recation (#3882)

* Pinning DLAMIs to prevent failures due to nvidia-docker deprecation

* buildspec override for traning and inf images

* Revert temp files

test/test_utils/__init__.py

@@ -58,7 +58,15 @@ def get_ami_id_boto3(region_name, ami_name_pattern, IncludeDeprecated=False):
         Owners=["amazon"],
         IncludeDeprecated=IncludeDeprecated,
     )
-    ami = max(ami_list["Images"], key=lambda x: x["CreationDate"])
+    #### TODO: remove/comment this when the 3881 is merged ####
+    # Date pinning AMI IDs due to nvidia-docker deprecation
+    filtered_images = [
+        element
+        for element in ami_list["Images"]
+        if datetime.strptime(element["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
+        < datetime.strptime("2024-05-02", "%Y-%m-%d")
+    ]
+    ami = max(filtered_images, key=lambda x: x["CreationDate"])
     return ami["ImageId"]