[Build][Test] Updating Torchserve for PT1.13 inference images (#3923)

* Updating Torchserve for PT1.13 inference images * Change TS version to 0.11.0 * Sanity test resolutions * Shift pyopenssl * Change the SM toolkit version * Run SM tests and change numpy in core packages * Comment AP tests * Add zstandard to core packages * Revert toml
aws · May 18, 2024 · 3e52562 · 3e52562
1 parent 6a4b37e
commit 3e52562
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 34 deletions.
diff --git a/pytorch/inference/buildspec-1-13.yml b/pytorch/inference/buildspec-1-13.yml
@@ -5,17 +5,17 @@ framework: &FRAMEWORK pytorch
 version: &VERSION 1.13.1
 short_version: &SHORT_VERSION "1.13"
 arch_type: x86
-autopatch_build: "True"
+# autopatch_build: "True"
 
 repository_info:
   inference_repository: &INFERENCE_REPOSITORY
     image_type: &INFERENCE_IMAGE_TYPE inference
     root: !join [ *FRAMEWORK, "/", *INFERENCE_IMAGE_TYPE ]
     repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE]
     repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/, *REPOSITORY_NAME ]
-    release_repository_name: &RELEASE_REPOSITORY_NAME !join [*FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE]
-    release_repository: &RELEASE_REPOSITORY !join [ *PROD_ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/,
-      *RELEASE_REPOSITORY_NAME ]
+    # release_repository_name: &RELEASE_REPOSITORY_NAME !join [*FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE]
+    # release_repository: &RELEASE_REPOSITORY !join [ *PROD_ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/,
+    #   *RELEASE_REPOSITORY_NAME ]
 
 context:
   inference_context: &INFERENCE_CONTEXT
@@ -41,9 +41,9 @@ images:
     python_version: &DOCKER_PYTHON_VERSION py3
     tag_python_version: &TAG_PYTHON_VERSION py39
     os_version: &OS_VERSION ubuntu20.04
-    torch_serve_version: &TORCHSERVE_VERSION 0.8.2
+    torch_serve_version: &TORCHSERVE_VERSION 0.11.0
     tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-ec2" ]
-    latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-ec2" ]
+    # latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-ec2" ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
     target: ec2
     context:
@@ -57,9 +57,9 @@ images:
     tag_python_version: &TAG_PYTHON_VERSION py39
     cuda_version: &CUDA_VERSION cu117
     os_version: &OS_VERSION ubuntu20.04
-    torch_serve_version: &TORCHSERVE_VERSION 0.8.2
+    torch_serve_version: &TORCHSERVE_VERSION 0.11.0
     tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-ec2" ]
-    latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-ec2" ]
+    # latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-ec2" ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile.,
                          *DEVICE_TYPE ]
     target: ec2
@@ -73,10 +73,10 @@ images:
     python_version: &DOCKER_PYTHON_VERSION py3
     tag_python_version: &TAG_PYTHON_VERSION py39
     os_version: &OS_VERSION ubuntu20.04
-    torch_serve_version: &TORCHSERVE_VERSION 0.8.2
-    tool_kit_version: &SM_TOOLKIT_VERSION 2.0.16
+    torch_serve_version: &TORCHSERVE_VERSION 0.11.0
+    tool_kit_version: &SM_TOOLKIT_VERSION 2.0.23
     tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-sagemaker" ]
-    latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-sagemaker" ]
+    # latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION, "-sagemaker" ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
     target: sagemaker
     context:
@@ -90,10 +90,10 @@ images:
     tag_python_version: &TAG_PYTHON_VERSION py39
     cuda_version: &CUDA_VERSION cu117
     os_version: &OS_VERSION ubuntu20.04
-    torch_serve_version: &TORCHSERVE_VERSION 0.8.2
-    tool_kit_version: &SM_TOOLKIT_VERSION 2.0.16
+    torch_serve_version: &TORCHSERVE_VERSION 0.11.0
+    tool_kit_version: &SM_TOOLKIT_VERSION 2.0.23
     tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-sagemaker" ]
-    latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-sagemaker" ]
+    # latest_release_tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, "-sagemaker" ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile.,
                          *DEVICE_TYPE ]
     target: sagemaker

diff --git a/pytorch/inference/docker/1.13/py3/Dockerfile.cpu b/pytorch/inference/docker/1.13/py3/Dockerfile.cpu
@@ -55,7 +55,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV LANG=C.UTF-8
 ENV LD_LIBRARY_PATH="/opt/conda/lib:${LD_LIBRARY_PATH}"
 ENV PATH=/opt/conda/bin:$PATH
-ENV TEMP=/home/model-server/tmp
+ENV TEMP=/tmp
 # Set MKL_THREADING_LAYER=GNU to prevent issues between torch and numpy/mkl
 ENV MKL_THREADING_LAYER=GNU
 ENV DLC_CONTAINER_TYPE=inference
@@ -127,6 +127,7 @@ RUN curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases
     typing \
     h5py \
     requests \
+    "zstandard>=0.21.0" \
     # Below 2 are included in miniconda base, but not mamba so need to install
     conda-content-trust \
     charset-normalizer \
@@ -151,6 +152,7 @@ RUN pip install --no-cache-dir --extra-index-url https://download.pytorch.org/wh
     "botocore==1.34.8" \
     "s3transfer==0.10.0" \
     "pillow>=10.0.1" \
+    "pyopenssl>=24.1.0" \
     ${TORCH_URL} \
     torchaudio==${TORCHAUDIO_VERSION} \
     torchvision==${TORCHVISION_VERSION} \
@@ -170,7 +172,7 @@ RUN pip install --no-cache-dir -U -r https://raw.githubusercontent.com/pytorch/s
 RUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
 
 RUN useradd -m model-server \
- && mkdir -p /home/model-server/tmp /opt/ml/model \
+ && mkdir -p /tmp /opt/ml/model \
  && chown -R model-server /home/model-server /opt/ml/model
 
 COPY torchserve-ec2-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
@@ -234,6 +236,8 @@ RUN conda install -y -c conda-forge \
     scikit-learn \
     pandas
 
+## Latest SM Toolkit needs numpy>=1.24.4 hence the latest version of numpy is installed in SM containers
+## Latest SM Toolkit does not install sagemaker-inference as its dependency
 RUN pip install --no-cache-dir sagemaker-pytorch-inference==${SM_TOOLKIT_VERSION}
 
 COPY torchserve-entrypoint.py /usr/local/bin/dockerd-entrypoint.py

diff --git a/pytorch/inference/docker/1.13/py3/Dockerfile.ec2.cpu.core_packages.json b/pytorch/inference/docker/1.13/py3/Dockerfile.ec2.cpu.core_packages.json
@@ -16,10 +16,10 @@
       "skip":"True"
    },
    "torchserve":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch-model-archiver":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch":{
       "version_specifier":"==1.13.1+cpu"
@@ -46,6 +46,9 @@
       "version_specifier": ">=10.0.1"
    },
    "pyopenssl": {
-      "version_specifier": ">=23.2.0"
+      "version_specifier": ">=24.1.0"
+   },
+   "zstandard": {
+      "version_specifier": ">=0.21.0"
    }
 }
diff --git a/pytorch/inference/docker/1.13/py3/Dockerfile.sagemaker.cpu.core_packages.json b/pytorch/inference/docker/1.13/py3/Dockerfile.sagemaker.cpu.core_packages.json
@@ -16,10 +16,10 @@
       "skip":"True"
    },
    "torchserve":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch-model-archiver":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch":{
       "version_specifier":"==1.13.1+cpu"
@@ -37,15 +37,18 @@
       "version_specifier":"<2"
    },
    "numpy":{
-      "version_specifier":">=1.22.2,<1.23"
+      "version_specifier":">=1.26.4"
    },
    "pyyaml":{
       "version_specifier":">=6.0"
    },
    "sagemaker-pytorch-inference":{
-      "version_specifier":"==2.0.16"
+      "version_specifier":"==2.0.23"
    },
    "pyopenssl": {
-      "version_specifier": ">=23.2.0"
+      "version_specifier": ">=24.1.0"
+   },
+   "zstandard": {
+      "version_specifier": ">=0.21.0"
    }
 }
diff --git a/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.ec2.gpu.core_packages.json b/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.ec2.gpu.core_packages.json
@@ -16,10 +16,10 @@
       "skip":"True"
    },
    "torchserve":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch-model-archiver":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch":{
       "version_specifier":"==1.13.1+cu117"
@@ -46,6 +46,9 @@
       "version_specifier": ">=10.0.1"
    },
    "pyopenssl": {
-      "version_specifier": ">=23.2.0"
+      "version_specifier": ">=24.1.0"
+   },
+   "zstandard": {
+      "version_specifier": ">=0.21.0"
    }
 }
diff --git a/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.gpu b/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.gpu
@@ -61,7 +61,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV LANG C.UTF-8
 ENV LD_LIBRARY_PATH="/opt/conda/lib:${LD_LIBRARY_PATH}"
 ENV PATH /opt/conda/bin:$PATH
-ENV TEMP=/home/model-server/tmp
+ENV TEMP=/tmp
 # Set MKL_THREADING_LAYER=GNU to prevent issues between torch and numpy/mkl
 ENV MKL_THREADING_LAYER=GNU
 ENV DLC_CONTAINER_TYPE=inference
@@ -188,6 +188,7 @@ RUN curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases
     h5py \
     requests \
     libgcc \
+    "zstandard>=0.21.0" \
     # Below 2 are included in miniconda base, but not mamba so need to install
     conda-content-trust \
     charset-normalizer \
@@ -212,6 +213,7 @@ RUN pip install --no-cache-dir --extra-index-url https://download.pytorch.org/wh
     "ipython>=8.10.0,<9.0" \
     "awscli<2" \
     "pillow>=10.0.1" \
+    "pyopenssl>=24.1.0" \
     ${TORCH_URL} \
     torchaudio==${TORCHAUDIO_VERSION} \
     torchvision==${TORCHVISION_VERSION} \
@@ -236,7 +238,7 @@ RUN cd tmp/ \
  && rm -rf tmp*
 
 RUN useradd -m model-server \
- && mkdir -p /home/model-server/tmp /opt/ml/model \
+ && mkdir -p /tmp /opt/ml/model \
  && chown -R model-server /home/model-server /opt/ml/model
 
 COPY torchserve-ec2-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
@@ -298,6 +300,8 @@ RUN conda install -y -c conda-forge \
     scikit-learn \
     pandas
 
+## Latest SM Toolkit needs numpy>=1.24.4 hence the latest version of numpy is installed in SM containers
+## Latest SM Toolkit does not install sagemaker-inference as its dependency
 RUN pip install --no-cache-dir sagemaker-pytorch-inference==${SM_TOOLKIT_VERSION}
 
 COPY torchserve-entrypoint.py /usr/local/bin/dockerd-entrypoint.py

diff --git a/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.sagemaker.gpu.core_packages.json b/pytorch/inference/docker/1.13/py3/cu117/Dockerfile.sagemaker.gpu.core_packages.json
@@ -16,10 +16,10 @@
       "skip":"True"
    },
    "torchserve":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch-model-archiver":{
-      "version_specifier":"==0.8.2"
+      "version_specifier":"==0.11.0"
    },
    "torch":{
       "version_specifier":"==1.13.1+cu117"
@@ -37,7 +37,7 @@
       "version_specifier":"<2"
    },
    "numpy":{
-      "version_specifier":">=1.22.2,<1.23"
+      "version_specifier":">=1.26.4"
    },
    "pyyaml":{
       "version_specifier":">=6.0"
@@ -46,9 +46,12 @@
       "version_specifier": ">=10.0.1"
    },
    "sagemaker-pytorch-inference":{
-      "version_specifier":"==2.0.16"
+      "version_specifier":"==2.0.23"
    },
    "pyopenssl": {
-      "version_specifier": ">=23.2.0"
+      "version_specifier": ">=24.1.0"
+   },
+   "zstandard": {
+      "version_specifier": ">=0.21.0"
    }
 }