limit pinning tests to CPM-downloaded projects (#599)

Modifies pinning tests from #530: * to only test projects that were downloaded by CPM (e.g. ignoring the `fmt` that might already exist in the build environment) * to echo out `pinned_versions.json` and `versions.json` in logs from failed tests, to make debugging faster ## Notes for Reviewers #592 proposes some testing changes that aren't specific to the goals of that PR. Since that PR might be stuck for a bit (rapidsai/build-planning#56 (comment)), this proposes pulling those out into a separate PR: * so that other changes in this project benefit from them * to shrink the diff of #592 and therefore the risk of merge conflicts ### How I tested this Pushed a commit with the new test error message content changes but keeping `fmt` in the failing tests, to confirm that the expected tests failed. <details><summary>got the expected outputs (click me)</summary> ```text The following tests FAILED: 698 - cpm_generate_pins-nested-makefile (Failed) 700 - cpm_generate_pins-nested-ninja (Failed) 702 - cpm_generate_pins-nested-ninja_multi-config (Failed) 722 - cpm_generate_pins-simple-makefile (Failed) 724 - cpm_generate_pins-simple-ninja (Failed) 726 - cpm_generate_pins-simple-ninja_multi-config (Failed) ``` And they failed in the expected way more informative logs! ```text CMake Error at CMakeLists.txt:51 (message): pinned fmt tag (10.2.1) should differ compared to baseline 10.2.1 pinned_versions.json: { "always_download" : true, "git_shallow" : false, "git_tag" : "${version}", "git_url" : "https://github.com/fmtlib/fmt.git", "version" : "10.2.1" } versions.json: { "git_tag" : "${version}", "git_url" : "https://github.com/fmtlib/fmt.git", "version" : "10.2.1" } ``` ([build link](https://github.com/rapidsai/rapids-cmake/actions/runs/8837613213/job/24267079292?pr=592)) </details> # Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: #599
rapidsai · May 1, 2024 · bc1c006 · bc1c006
1 parent 6301b3d
commit bc1c006
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 9 deletions.
diff --git a/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt b/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt
@@ -24,8 +24,15 @@ add_subdirectory(b)
 include(${rapids-cmake-dir}/cpm/init.cmake)
 rapids_cpm_init(GENERATE_PINNED_VERSIONS)
 
+# only check projects that were downloaded by CPM (ignore those already in the build environment)
+#
 # don't check cuco here as it uses hashes and that breaks our verify script
-set(projects-to-verify rmm fmt spdlog CCCL)
+foreach(proj IN ITEMS rmm fmt spdlog CCCL)
+  if(${proj}_SOURCE_DIR)
+    list(APPEND projects-to-verify ${proj})
+  endif()
+endforeach()
+
 add_custom_target(verify_generated_pins ALL
   COMMAND ${CMAKE_COMMAND} -S="${CMAKE_SOURCE_DIR}/verify/" -B"${CMAKE_BINARY_DIR}/verify_build"
   -D"rapids-cmake-dir=${rapids-cmake-dir}"

diff --git a/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt b/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt
@@ -32,22 +32,29 @@ foreach(proj IN LISTS projects-to-verify)
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository ${proj}_tag ${proj}_shallow ${proj}_exclude)
 endforeach()
 
-
-file(READ "${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json" json_data)
-
 # Verify that the pinned_versions is valid json by using it
 rapids_cpm_package_override("${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json")
 
+# helper macro for gathering versions.json and pinned_versions.json content to be printed in error messages
+macro(_get_json_data)
+  include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
+  include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
+  get_default_json(${proj} default_versions)
+  get_override_json(${proj} pinned_versions)
+endmacro()
+
 foreach(proj IN LISTS projects-to-verify)
   # Verify that each git_tag is now different.
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository pin_${proj}_tag pin_${proj}_shallow ${proj}_exclude)
   if(pin_${proj}_tag STREQUAL ${proj}_tag)
-    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}")
+    _get_json_data()
+    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 
   # Everything should have shallow marked as false
   # so that clones by SHA1 work
   if(pin_${proj}_shallow)
-    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}")
+    _get_json_data()
+    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 endforeach()
diff --git a/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt b/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt
@@ -22,7 +22,14 @@ include(${rapids-cmake-dir}/cpm/rmm.cmake)
 
 rapids_cpm_init(GENERATE_PINNED_VERSIONS)
 rapids_cpm_rmm()
-set(projects-to-verify rmm fmt spdlog CCCL)
+
+# only check projects that were downloaded by CPM (ignore those already in the build environment)
+foreach(proj IN ITEMS rmm fmt spdlog CCCL)
+  if(${proj}_SOURCE_DIR)
+    list(APPEND projects-to-verify ${proj})
+  endif()
+endforeach()
+
 add_custom_target(verify_generated_pins ALL
   COMMAND ${CMAKE_COMMAND} -S="${CMAKE_SOURCE_DIR}/verify/" -B"${CMAKE_BINARY_DIR}/verify_build"
   -D"rapids-cmake-dir=${rapids-cmake-dir}"

diff --git a/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt b/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt
@@ -35,16 +35,26 @@ endforeach()
 # Verify that the pinned_versions is valid json by using it
 rapids_cpm_package_override("${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json")
 
+# helper macro for gathering versions.json and pinned_versions.json content to be printed in error messages
+macro(_get_json_data)
+  include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
+  include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
+  get_default_json(${proj} default_versions)
+  get_override_json(${proj} pinned_versions)
+endmacro()
+
 foreach(proj IN LISTS projects-to-verify)
   # Verify that each git_tag is now different.
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository pin_${proj}_tag pin_${proj}_shallow ${proj}_exclude)
   if(pin_${proj}_tag STREQUAL ${proj}_tag)
-    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}")
+    _get_json_data()
+    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 
   # Everything should have shallow marked as false
   # so that clones by SHA1 work
   if(pin_${proj}_shallow)
-    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}")
+    _get_json_data()
+    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 endforeach()