From 899af22c25fdd5d896817e40514b5ea0e1537e90 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 21 Nov 2024 10:54:02 -0800 Subject: [PATCH] chore(CI): Build test vectors from source (#724) --- codebuild/py311/awses_local_mpl.yml | 2 ++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 2 ++ .../decrypt_golden_manifest_with_keyrings.yml | 2 ++ ...decrypt_golden_manifest_with_masterkey.yml | 2 +- .../py311/decrypt_keyrings_with_keyrings.yml | 2 ++ .../py311/decrypt_masterkey_with_keyrings.yml | 2 ++ .../decrypt_net_401_vectors_keyrings.yml | 2 ++ codebuild/py311/encrypt_keyrings.yml | 2 ++ .../generate_decrypt_vectors_keyrings.yml | 2 ++ codebuild/py312/awses_local_mpl.yml | 2 ++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 2 ++ .../decrypt_golden_manifest_with_keyrings.yml | 2 ++ ...decrypt_golden_manifest_with_masterkey.yml | 2 +- .../py312/decrypt_hkeyring_with_keyrings.yml | 2 ++ .../py312/decrypt_hkeyring_with_masterkey.yml | 2 +- .../py312/decrypt_keyrings_with_keyrings.yml | 2 ++ .../py312/decrypt_masterkey_with_keyrings.yml | 2 ++ .../decrypt_net_401_vectors_keyrings.yml | 2 ++ codebuild/py312/encrypt_keyrings.yml | 2 ++ .../generate_decrypt_vectors_keyrings.yml | 2 ++ .../generate_hkeyring_decrypt_vectors.yml | 2 ++ test_vector_handlers/requirements_mpl.txt | 2 +- .../scripts/install_mpl_test_vector_runner.sh | 24 +++++++++++++++++++ test_vector_handlers/tox.ini | 3 +++ 24 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 test_vector_handlers/scripts/install_mpl_test_vector_runner.sh diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index 859931aa3..c6a975df8 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -21,6 +21,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - tox diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml index d69ce9370..1eadf2958 100644 --- a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml @@ -22,6 +22,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | diff --git a/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml index 154863bcc..c65816d69 100644 --- a/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml +++ b/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download "golden manifest" - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip - unzip python-2.3.0.zip -d python-2.3.0 diff --git a/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml index 4ae4bb280..3b71759d3 100644 --- a/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml +++ b/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt-mpl" + TOXENV: "py311-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml index fec275d48..f202c9c69 100644 --- a/codebuild/py311/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip - unzip 311_keyrings.zip diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml index 8543077bd..1542b1acb 100644 --- a/codebuild/py311/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip - unzip 311_masterkey.zip diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 1a23f0917..b5e7d7233 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -19,6 +19,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py311/encrypt_keyrings.yml b/codebuild/py311/encrypt_keyrings.yml index 8b7cb94b4..8e315ca9f 100644 --- a/codebuild/py311/encrypt_keyrings.yml +++ b/codebuild/py311/encrypt_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 179ec0f12..0c9fc8333 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index c92265541..ea4d4f229 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -23,6 +23,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml index 810d16b74..2a8439253 100644 --- a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml @@ -22,6 +22,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | diff --git a/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml index 54bb10c33..c0442a10b 100644 --- a/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml +++ b/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download "golden manifest" - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip - unzip python-2.3.0.zip -d python-2.3.0 diff --git a/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml index df204d4c5..0e2ecb287 100644 --- a/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml +++ b/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-full_decrypt-mpl" + TOXENV: "py312-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml index 5bcd26738..844f4c464 100644 --- a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml +++ b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors # This manifest has coverage for both HKeyring and required encryption context CMM - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip diff --git a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml index be67235d7..04def1a98 100644 --- a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml +++ b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-full_decrypt-mpl" + TOXENV: "py312-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/decrypt_keyrings_with_keyrings.yml b/codebuild/py312/decrypt_keyrings_with_keyrings.yml index 3ab7058f9..27f9f1b5e 100644 --- a/codebuild/py312/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py312/decrypt_keyrings_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip - unzip 312_keyrings.zip diff --git a/codebuild/py312/decrypt_masterkey_with_keyrings.yml b/codebuild/py312/decrypt_masterkey_with_keyrings.yml index 21f646370..e22bd2ace 100644 --- a/codebuild/py312/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py312/decrypt_masterkey_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip - unzip 312_masterkey.zip diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml index aec3916e5..3156e06ab 100644 --- a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml @@ -19,6 +19,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py312/encrypt_keyrings.yml b/codebuild/py312/encrypt_keyrings.yml index 56a389e6f..db12d6c9d 100644 --- a/codebuild/py312/encrypt_keyrings.yml +++ b/codebuild/py312/encrypt_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml index ae79b86ce..1760333f0 100644 --- a/codebuild/py312/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml index b0a755360..15b4f485e 100644 --- a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml +++ b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers/test/aws-crypto-tools-test-vector-framework # Checkout WIP branch with manifest containing HKeyring and required EC CMM test cases diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt index 6995b3187..f1a309033 100644 --- a/test_vector_handlers/requirements_mpl.txt +++ b/test_vector_handlers/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptography-internal-mpl-testvectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@python-main-with-dafny-code#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file +aws-cryptographic-material-providers==1.7.4 diff --git a/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh new file mode 100644 index 000000000..b99226e66 --- /dev/null +++ b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh @@ -0,0 +1,24 @@ +# Builds the Python MPL TestVector runner from source. +# This package is used by the ESDK-Python test vectors for testing with the MPL. + +# This script is intended to be used by ESDK-Python's integration tests. +# You may need or want to make local changes to get this work on your machine. + +# Change to the directory of the script +cd "$(dirname "$0")" + +# Get MPL version specified in requirements.txt +export mplVersion=$(grep 'aws-cryptographic-material-providers==' ../requirements_mpl.txt | sed -E 's/.*==(.+)/\1/') + +# Clone MPL repo to get test vectors runner source code and the Dafny version to use +git clone --branch v$mplVersion --recurse-submodules https://github.com/aws/aws-cryptographic-material-providers-library.git + +# Download Dafny to build the test vector runner; get Dafny version from ESDK's project.properties file +export dafnyVersion=$(grep '^dafnyVersion=' aws-cryptographic-material-providers-library/project.properties | cut -d '=' -f 2) +curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip -L -o dafny.zip +unzip -qq dafny.zip && rm dafny.zip +export PATH="$PWD/dafny:$PATH" + +# Build MPL test vector runner from source +cd aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/ +make transpile_python diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index cdb1137fb..7fe77ff50 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,6 +51,9 @@ deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt + # This is the filepath used in ESDK-Python's integration tests. + # You may need to change this to run this project locally. + mpl: scripts/aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/runtimes/python .. commands = awses_local: {[testenv:base-command]commands}