Patch build files to also system libraries instead of bundled ones for the
libraries not supported or working only by specifying USE_SYSTEM_LIBS.  This
includes using the clog, cpuinfo, fbgemm, foxi, fp16, fxdiv, googletest,
httlib, ideep, miniz, nnpack, oneapi-dnnl, pocketfft, pthreadpool,
qnnpack-pytorch, tensorpipe, valgrind and xnnpack packages.

diff --git i/aten/src/ATen/CMakeLists.txt w/aten/src/ATen/CMakeLists.txt
index a3e841ca22a..632274abe5c 100644
--- i/aten/src/ATen/CMakeLists.txt
+++ w/aten/src/ATen/CMakeLists.txt
@@ -638,9 +638,9 @@ if(AT_NNPACK_ENABLED)
   list(APPEND ATen_CPU_DEPENDENCY_LIBS nnpack) # cpuinfo is added below
 endif()
 
-if(MKLDNN_FOUND)
-  list(APPEND ATen_CPU_DEPENDENCY_LIBS ${MKLDNN_LIBRARIES})
-endif(MKLDNN_FOUND)
+if(USE_MKLDNN)
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS DNNL::dnnl)
+endif(USE_MKLDNN)
 
 if(USE_MKLDNN_ACL)
     list(APPEND ATen_CPU_INCLUDE ${ACL_INCLUDE_DIRS})
diff --git i/caffe2/CMakeLists.txt w/caffe2/CMakeLists.txt
index 6cbaecc5d2e..339145fab63 100644
--- i/caffe2/CMakeLists.txt
+++ w/caffe2/CMakeLists.txt
@@ -91,9 +91,6 @@ if(NOT MSVC AND USE_XNNPACK)
   if(NOT TARGET fxdiv)
     set(FXDIV_BUILD_TESTS OFF CACHE BOOL "")
     set(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "")
-    add_subdirectory(
-      "${FXDIV_SOURCE_DIR}"
-      "${CMAKE_BINARY_DIR}/FXdiv")
   endif()
 endif()
 
@@ -1212,7 +1209,6 @@ if(USE_XPU)
 endif()
 
 if(NOT MSVC AND USE_XNNPACK)
-  TARGET_LINK_LIBRARIES(torch_cpu PRIVATE fxdiv)
 endif()
 
 # ==========================================================
@@ -1332,8 +1328,8 @@ endif()
 target_include_directories(torch_cpu PRIVATE
   ${TORCH_ROOT}/third_party/cpp-httplib)
 
-target_include_directories(torch_cpu PRIVATE
-  ${TORCH_ROOT}/third_party/nlohmann/include)
+find_package(httplib REQUIRED)
+target_link_libraries(torch_cpu PUBLIC httplib::httplib)
 
 install(DIRECTORY
   "${TORCH_SRC_DIR}/csrc"
@@ -1552,6 +1548,7 @@ target_link_libraries(torch_cpu PUBLIC c10)
 target_link_libraries(torch_cpu PUBLIC ${Caffe2_PUBLIC_DEPENDENCY_LIBS})
 target_link_libraries(torch_cpu PRIVATE ${Caffe2_DEPENDENCY_LIBS})
 target_link_libraries(torch_cpu PRIVATE ${Caffe2_DEPENDENCY_WHOLE_LINK_LIBS})
+target_link_libraries(torch_cpu PRIVATE miniz clog)
 if(USE_MPI)
   target_link_libraries(torch_cpu PRIVATE MPI::MPI_CXX)
 endif()
@@ -1819,7 +1816,7 @@ if(BUILD_MOBILE_TEST)
   foreach(test_src ${ATen_MOBILE_TEST_SRCS})
     get_filename_component(test_name ${test_src} NAME_WE)
     add_executable(${test_name} "${test_src}")
-    target_link_libraries(${test_name} torch_library gtest_main)
+    target_link_libraries(${test_name} torch_library gtest_main gtest)
     target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
     target_include_directories(${test_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>)
     target_include_directories(${test_name} PRIVATE ${ATen_CPU_INCLUDE})
@@ -1840,7 +1837,7 @@ if(BUILD_TEST)
         if(NOT MSVC)
           add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}" ../aten/src/ATen/native/quantized/AffineQuantizerBase.cpp)
           # TODO: Get rid of c10 dependency (which is only needed for the implementation of AT_ERROR)
-          target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main nlohmann)
+          target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main gtest nlohmann)
           if(USE_FBGEMM)
             target_link_libraries(${test_name}_${CPU_CAPABILITY} fbgemm)
           endif()
@@ -1860,7 +1857,7 @@ if(BUILD_TEST)
           endif()
         else()
           add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}")
-          target_link_libraries(${test_name}_${CPU_CAPABILITY} torch_library sleef gtest_main)
+          target_link_libraries(${test_name}_${CPU_CAPABILITY} torch_library sleef gtest_main gtest)
         endif()
         target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE $<INSTALL_INTERFACE:include>)
         target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>)
@@ -1901,7 +1898,7 @@ if(BUILD_TEST)
       add_executable(${test_name} "${test_src}")
       find_library(metal NAMES Metal)
       find_library(foundation NAMES Foundation)
-      target_link_libraries(${test_name} torch_library gtest_main ${metal} ${foundation})
+      target_link_libraries(${test_name} torch_library gtest_main gtest ${metal} ${foundation})
       target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
       target_include_directories(${test_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>)
       target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
@@ -1921,7 +1918,7 @@ if(BUILD_TEST)
     foreach(test_src ${Caffe2_GPU_TEST_SRCS})
       get_filename_component(test_name ${test_src} NAME_WE)
       add_executable(${test_name} "${test_src}")
-      target_link_libraries(${test_name} torch_library gtest_main)
+      target_link_libraries(${test_name} torch_library gtest_main gtest)
       if(USE_CUDNN AND ${test_name} MATCHES "cudnn")
         target_link_libraries(${test_name} torch::cudnn)
       endif()
@@ -1944,7 +1941,7 @@ if(BUILD_TEST)
       get_filename_component(test_name ${test_src} NAME_WE)
       add_executable(${test_name} "${test_src}")
       torch_compile_options(${test_name})
-      target_link_libraries(${test_name} torch_library gtest_main)
+      target_link_libraries(${test_name} torch_library gtest_main gtest)
       target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
       target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
       add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
@@ -1959,7 +1956,7 @@ if(BUILD_TEST)
     foreach(test_src ${Caffe2_VULKAN_TEST_SRCS})
       get_filename_component(test_name ${test_src} NAME_WE)
       add_executable(${test_name} "${test_src}")
-      target_link_libraries(${test_name} torch_library gtest_main)
+      target_link_libraries(${test_name} torch_library gtest_main gtest)
       target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
       target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
       add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
@@ -1991,7 +1988,7 @@ if(BUILD_TEST)
       if(test_src MATCHES "^.*\.hip$")
         set_target_properties(${test_name} PROPERTIES LINKER_LANGUAGE CXX)
       endif()
-      target_link_libraries(${test_name} torch_library gtest_main)
+      target_link_libraries(${test_name} torch_library gtest_main gtest)
       target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
       target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE} ${Caffe2_HIP_INCLUDE})
       target_compile_options(${test_name} PRIVATE ${HIP_CXX_FLAGS})
diff --git i/caffe2/serialize/CMakeLists.txt w/caffe2/serialize/CMakeLists.txt
index ebbff0f292a..dcded25908e 100644
--- i/caffe2/serialize/CMakeLists.txt
+++ w/caffe2/serialize/CMakeLists.txt
@@ -2,7 +2,6 @@ file(GLOB tmp *_test.cc)
 
 set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} ${tmp})
 list(APPEND Caffe2_CPU_SRCS
-  ${PROJECT_SOURCE_DIR}/third_party/miniz-3.0.2/miniz.c
   ${CMAKE_CURRENT_SOURCE_DIR}/inline_container.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/istream_adapter.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/file_adapter.cc
diff --git i/cmake/Dependencies.cmake w/cmake/Dependencies.cmake
index 903c212de81..a6972b40370 100644
--- i/cmake/Dependencies.cmake
+++ w/cmake/Dependencies.cmake
@@ -307,7 +307,7 @@ endif()
 # --- [ PocketFFT
 set(AT_POCKETFFT_ENABLED 0)
 if(NOT AT_MKL_ENABLED)
-  set(POCKETFFT_INCLUDE_DIR "${Torch_SOURCE_DIR}/third_party/pocketfft/")
+  set(POCKETFFT_INCLUDE_DIR "#POCKETFFT_INCLUDE_DIR")
   if(NOT EXISTS "${POCKETFFT_INCLUDE_DIR}")
     message(FATAL_ERROR "pocketfft directory not found, expected ${POCKETFFT_INCLUDE_DIR}")
   elseif(NOT EXISTS "${POCKETFFT_INCLUDE_DIR}/pocketfft_hdronly.h")
@@ -491,15 +491,6 @@ if(USE_PYTORCH_QNNPACK)
       set(PYTORCH_QNNPACK_BUILD_TESTS OFF CACHE BOOL "")
       set(PYTORCH_QNNPACK_BUILD_BENCHMARKS OFF CACHE BOOL "")
       set(PYTORCH_QNNPACK_LIBRARY_TYPE "static" CACHE STRING "")
-      add_subdirectory(
-        "${PYTORCH_QNNPACK_SOURCE_DIR}"
-        "${CONFU_DEPENDENCIES_BINARY_DIR}/pytorch_qnnpack")
-      # We build static versions of QNNPACK and pthreadpool but link
-      # them into a shared library for Caffe2, so they need PIC.
-      set_property(TARGET pytorch_qnnpack PROPERTY POSITION_INDEPENDENT_CODE ON)
-      set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON)
-      # QNNPACK depends on gemmlowp headers
-      target_include_directories(pytorch_qnnpack PRIVATE "${CAFFE2_THIRD_PARTY_ROOT}/gemmlowp")
     endif()
 
     list(APPEND Caffe2_DEPENDENCY_LIBS pytorch_qnnpack)
@@ -595,16 +586,15 @@ if(USE_XNNPACK AND NOT USE_SYSTEM_XNNPACK)
   list(APPEND Caffe2_DEPENDENCY_LIBS XNNPACK microkernels-prod)
 elseif(NOT TARGET XNNPACK AND USE_SYSTEM_XNNPACK)
   add_library(XNNPACK SHARED IMPORTED)
-  add_library(microkernels-prod SHARED IMPORTED)
+  add_library(microkernels-prod INTERFACE IMPORTED)
   find_library(XNNPACK_LIBRARY XNNPACK)
-  find_library(microkernels-prod_LIBRARY microkernels-prod)
   set_property(TARGET XNNPACK PROPERTY IMPORTED_LOCATION "${XNNPACK_LIBRARY}")
-  set_property(TARGET microkernels-prod PROPERTY IMPORTED_LOCATION "${microkernels-prod_LIBRARY}")
-  if(NOT XNNPACK_LIBRARY OR NOT microkernels-prod_LIBRARY)
+  set_property(TARGET microkernels-prod PROPERTY INTERFACE_LINK_LIBRARIES XNNPACK)
+  if(NOT XNNPACK_LIBRARY)
     message(FATAL_ERROR "Cannot find XNNPACK")
   endif()
   message("-- Found XNNPACK: ${XNNPACK_LIBRARY}")
-  list(APPEND Caffe2_DEPENDENCY_LIBS XNNPACK microkernels-prod)
+  list(APPEND Caffe2_DEPENDENCY_LIBS XNNPACK)
 endif()
 
 # ---[ Vulkan deps
@@ -658,9 +648,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR BUILD_MOBILE_TEST)
   set(INSTALL_GTEST OFF CACHE BOOL "Install gtest." FORCE)
   set(BUILD_GMOCK ON CACHE BOOL "Build gmock." FORCE)
 
-  add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest)
-  include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googletest/include)
-  include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googlemock/include)
 
   # We will not need to test benchmark lib itself.
   set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.")
@@ -692,16 +679,22 @@ if(USE_FBGEMM)
     set(FBGEMM_BUILD_TESTS OFF CACHE BOOL "")
     set(FBGEMM_BUILD_BENCHMARKS OFF CACHE BOOL "")
     set(FBGEMM_LIBRARY_TYPE "static" CACHE STRING "")
-    add_subdirectory("${FBGEMM_SOURCE_DIR}")
-
     if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-      target_compile_options_if_supported(asmjit -Wno-extra-semi)
-      target_compile_options_if_supported(fbgemm -Wno-extra-semi)
+      if(TARGET asmjit)
+        target_compile_options_if_supported(asmjit -Wno-extra-semi)
+      endif()
+      if(TARGET fbgemm)
+        target_compile_options_if_supported(fbgemm -Wno-extra-semi)
+      endif()
+    endif()
+    if(TARGET asmjit)
+      target_compile_options_if_supported(asmjit -Wno-unused-but-set-variable)
+      target_compile_options_if_supported(asmjit -Wno-unused-variable)
     endif()
-    target_compile_options_if_supported(asmjit -Wno-unused-but-set-variable)
-    target_compile_options_if_supported(asmjit -Wno-unused-variable)
   endif()
   if(USE_FBGEMM)
+    # Don't set compile definitions on system fbgemm
+    # target_compile_definitions(fbgemm PUBLIC DISABLE_FBGEMM_AUTOVEC)
     list(APPEND Caffe2_DEPENDENCY_LIBS fbgemm)
   endif()
 endif()
@@ -772,7 +765,7 @@ if(NOT TARGET fp16 AND NOT USE_SYSTEM_FP16)
       "${CONFU_DEPENDENCIES_BINARY_DIR}/FP16")
   endif()
 elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
-  add_library(fp16 STATIC "/usr/include/fp16.h")
+  add_library(fp16 STATIC "#FP16_INCLUDE_DIR")
   set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C)
 endif()
 list(APPEND Caffe2_DEPENDENCY_LIBS fp16)
@@ -1165,10 +1158,11 @@ if(USE_DISTRIBUTED AND USE_TENSORPIPE)
 
     # Tensorpipe uses cuda_add_library
     torch_update_find_cuda_flags()
-    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/tensorpipe)
     # Suppress warning to unblock libnop compilation by clang-17
     # See https://github.com/pytorch/pytorch/issues/151316
-    target_compile_options_if_supported(tensorpipe -Wno-missing-template-arg-list-after-template-kw)
+    if(TARGET tensorpipe)
+      target_compile_options_if_supported(tensorpipe -Wno-missing-template-arg-list-after-template-kw)
+    endif()
     # Workaround for relocation truncated to fit: R_AARCH64_CALL26 against symbol __aarch64_swp4_relax'
     # When compiling for ARMv8.0, build uv with embedded atomics, which are slightly slower
     # But are used only once during shutdown
@@ -1350,7 +1344,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX)
     endif()
     set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY})
     message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}")
-    list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx)
+    list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx onnx_optimizer)
   endif()
   # Recover the build shared libs option.
   set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS})
@@ -1513,9 +1507,8 @@ if(NOT INTERN_BUILD_MOBILE)
   endif()
   if(USE_MKLDNN)
     include(${CMAKE_CURRENT_LIST_DIR}/public/mkldnn.cmake)
-    if(MKLDNN_FOUND)
+    if(DNNL_FOUND)
       set(AT_MKLDNN_ENABLED 1)
-      include_directories(AFTER SYSTEM ${MKLDNN_INCLUDE_DIR})
     else()
       message(WARNING "MKLDNN could not be found.")
       caffe2_update_option(USE_MKLDNN OFF)
@@ -1590,7 +1583,7 @@ endif()
 set(FMT_INSTALL ON)
 set(TEMP_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
 set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE)
-add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/fmt)
+find_package(fmt)
 
 # Disable compiler feature checks for `fmt`.
 #
@@ -1599,7 +1592,6 @@ add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/fmt)
 # CMAKE_CXX_FLAGS in ways that break feature checks. Since we already know
 # `fmt` is compatible with a superset of the compilers that PyTorch is, it
 # shouldn't be too bad to just disable the checks.
-set_target_properties(fmt-header-only PROPERTIES INTERFACE_COMPILE_FEATURES "")
 
 list(APPEND Caffe2_DEPENDENCY_LIBS fmt::fmt-header-only)
 set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS} CACHE BOOL "Build shared libs" FORCE)
@@ -1695,4 +1687,5 @@ include_directories(nlohmann SYSTEM INTERFACE ${PROJECT_SOURCE_DIR}/third_party/
 
 # Include moodycamel
 add_library(moodycamel INTERFACE IMPORTED)
-include_directories(moodycamel SYSTEM INTERFACE ${PROJECT_SOURCE_DIR}/third_party/concurrentqueue)
+# Use system concurrentqueue - it's a header-only library
+include_directories(SYSTEM #CONCURRENTQUEUE_INCLUDE_DIR)
diff --git i/cmake/External/nnpack.cmake w/cmake/External/nnpack.cmake
index 8a4a310d629..f413d2e6181 100644
--- i/cmake/External/nnpack.cmake
+++ w/cmake/External/nnpack.cmake
@@ -40,7 +40,7 @@ endif()
 # (3) Android, iOS, Linux, macOS - supported
 ##############################################################################
 
-if(ANDROID OR IOS OR ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" OR ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
+if(FALSE)
   message(STATUS "Brace yourself, we are building NNPACK")
   set(CAFFE2_THIRD_PARTY_ROOT ${PROJECT_SOURCE_DIR}/third_party)
 
@@ -94,6 +94,5 @@ endif()
 # (4) Catch-all: not supported.
 ##############################################################################
 
-message(WARNING "Unknown platform - I don't know how to build NNPACK. "
-                "See cmake/External/nnpack.cmake for details.")
-set(USE_NNPACK OFF)
+set(NNPACK_FOUND TRUE)
+set(USE_NNPACK ON)
diff --git i/cmake/public/mkldnn.cmake w/cmake/public/mkldnn.cmake
index 87935625f9b..9f8fa3df81f 100644
--- i/cmake/public/mkldnn.cmake
+++ w/cmake/public/mkldnn.cmake
@@ -4,7 +4,7 @@ if(CPU_AARCH64)
   include(${CMAKE_CURRENT_LIST_DIR}/ComputeLibrary.cmake)
 endif()
 
-find_package(MKLDNN QUIET)
+find_package(DNNL REQUIRED)
 
 if(NOT TARGET caffe2::mkldnn)
   add_library(caffe2::mkldnn INTERFACE IMPORTED)
@@ -15,4 +15,4 @@ set_property(
   ${MKLDNN_INCLUDE_DIR})
 set_property(
   TARGET caffe2::mkldnn PROPERTY INTERFACE_LINK_LIBRARIES
-  ${MKLDNN_LIBRARIES})
+  DNNL::dnnl)
diff --git i/setup.py w/setup.py
index da38ba785f1..2d41bc13107 100644
--- i/setup.py
+++ w/setup.py
@@ -1060,13 +1060,9 @@ def build_deps() -> None:
     # Windows has very poor support for them.
     sym_files = [
         CWD / "tools/shared/_utils_internal.py",
-        CWD / "torch/utils/benchmark/utils/valgrind_wrapper/callgrind.h",
-        CWD / "torch/utils/benchmark/utils/valgrind_wrapper/valgrind.h",
     ]
     orig_files = [
         CWD / "torch/_utils_internal.py",
-        CWD / "third_party/valgrind-headers/callgrind.h",
-        CWD / "third_party/valgrind-headers/valgrind.h",
     ]
     for sym_file, orig_file in zip(sym_files, orig_files):
         same = False
diff --git i/test/cpp/aoti_abi_check/CMakeLists.txt w/test/cpp/aoti_abi_check/CMakeLists.txt
index 4146819e2f1..227413712cd 100644
--- i/test/cpp/aoti_abi_check/CMakeLists.txt
+++ w/test/cpp/aoti_abi_check/CMakeLists.txt
@@ -44,7 +44,7 @@ target_compile_definitions(test_aoti_abi_check PRIVATE USE_GTEST)
 
 # WARNING: DO NOT LINK torch!!!
 # The purpose is to check if the used aten/c10 headers are written in a header-only way
-target_link_libraries(test_aoti_abi_check PRIVATE gtest_main sleef)
+target_link_libraries(test_aoti_abi_check PRIVATE gtest_main gtest sleef)
 target_include_directories(test_aoti_abi_check PRIVATE ${ATen_CPU_INCLUDE})
 if(NOT USE_SYSTEM_SLEEF)
   target_include_directories(test_aoti_abi_check PRIVATE ${CMAKE_BINARY_DIR}/include)
@@ -66,7 +66,7 @@ foreach(test_src ${AOTI_ABI_CHECK_VEC_TEST_SRCS})
     separate_arguments(FLAGS UNIX_COMMAND "${FLAGS}")
     add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}")
 
-    target_link_libraries(${test_name}_${CPU_CAPABILITY} PRIVATE gtest_main sleef)
+    target_link_libraries(${test_name}_${CPU_CAPABILITY} PRIVATE gtest_main gtest sleef)
     target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE ${ATen_CPU_INCLUDE})
     if(NOT USE_SYSTEM_SLEEF)
       target_include_directories(${test_name}_${CPU_CAPABILITY} PRIVATE ${CMAKE_BINARY_DIR}/include)
diff --git i/test/cpp/aoti_inference/CMakeLists.txt w/test/cpp/aoti_inference/CMakeLists.txt
index 79e08471568..39002e7ac69 100644
--- i/test/cpp/aoti_inference/CMakeLists.txt
+++ w/test/cpp/aoti_inference/CMakeLists.txt
@@ -29,7 +29,7 @@ target_compile_definitions(test_aoti_inference PRIVATE USE_GTEST)
 
 target_link_libraries(test_aoti_inference PRIVATE
   torch
-  gtest_main
+  gtest_main gtest
   -Wl,--no-as-needed aoti_custom_class
 )
 
diff --git i/test/cpp/api/CMakeLists.txt w/test/cpp/api/CMakeLists.txt
index a92832a4d04..6f8ec13526c 100644
--- i/test/cpp/api/CMakeLists.txt
+++ w/test/cpp/api/CMakeLists.txt
@@ -50,7 +50,7 @@ endif()
 
 add_executable(test_api ${TORCH_API_TEST_SOURCES})
 target_include_directories(test_api PRIVATE ${ATen_CPU_INCLUDE})
-target_link_libraries(test_api PRIVATE torch gtest_main gmock)
+target_link_libraries(test_api PRIVATE torch gtest_main gtest gmock)
 
 if(USE_CUDA)
   target_compile_definitions(test_api PRIVATE "USE_CUDA")
diff --git i/test/cpp/c10d/CMakeLists.txt w/test/cpp/c10d/CMakeLists.txt
index 285a5dd2a73..6cb87fcbdd5 100644
--- i/test/cpp/c10d/CMakeLists.txt
+++ w/test/cpp/c10d/CMakeLists.txt
@@ -32,17 +32,17 @@ function(c10d_add_test test_src)
   endif()
 endfunction()
 
-c10d_add_test(BackoffTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST OFF)
-c10d_add_test(FileStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST})
-c10d_add_test(TCPStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST})
+c10d_add_test(BackoffTest.cpp LINK_LIBRARIES torch_cpu gtest_main gtest INSTALL_TEST OFF)
+c10d_add_test(FileStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main gtest INSTALL_TEST ${INSTALL_TEST})
+c10d_add_test(TCPStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main gtest INSTALL_TEST ${INSTALL_TEST})
 if(NOT WIN32)
-  c10d_add_test(HashStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST ${INSTALL_TEST})
+  c10d_add_test(HashStoreTest.cpp LINK_LIBRARIES torch_cpu gtest_main gtest INSTALL_TEST ${INSTALL_TEST})
 endif()
 
 if(USE_CUDA)
   if(USE_GLOO AND USE_C10D_GLOO)
-    c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main INSTALL_TEST ${INSTALL_TEST})
-    c10d_add_test(ProcessGroupGlooAsyncTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main INSTALL_TEST ${INSTALL_TEST})
+    c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main gtest gloo INSTALL_TEST ${INSTALL_TEST})
+    c10d_add_test(ProcessGroupGlooAsyncTest.cpp LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main gtest gloo INSTALL_TEST ${INSTALL_TEST})
   endif()
   if(USE_NCCL AND USE_C10D_NCCL)
     # NCCL is a private dependency of libtorch, but the tests include some
@@ -51,10 +51,10 @@ if(USE_CUDA)
     # a private dependency of the tests as well.
     c10d_add_test(
       ProcessGroupNCCLTest.cpp
-      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST})
+      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main gtest __caffe2_nccl INSTALL_TEST ${INSTALL_TEST})
     c10d_add_test(
       ProcessGroupNCCLErrorsTest.cpp
-      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_nccl INSTALL_TEST ${INSTALL_TEST})
+      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main gtest __caffe2_nccl INSTALL_TEST ${INSTALL_TEST})
     if(INSTALL_TEST)
       install(TARGETS c10d_cuda_test DESTINATION lib)
     endif()
@@ -66,14 +66,14 @@ if(USE_CUDA)
     # a private dependency of the tests as well.
     c10d_add_test(
       ProcessGroupUCCTest.cpp
-      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main __caffe2_ucc INSTALL_TEST ${INSTALL_TEST})
+      LINK_LIBRARIES torch_cpu c10d_cuda_test gtest_main gtest __caffe2_ucc INSTALL_TEST ${INSTALL_TEST})
     if(INSTALL_TEST)
       install(TARGETS c10d_cuda_test DESTINATION lib)
     endif()
   endif()
 else()
   if(USE_GLOO AND USE_C10D_GLOO)
-    c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu gtest_main INSTALL_TEST OFF)
+    c10d_add_test(ProcessGroupGlooTest.cpp LINK_LIBRARIES torch_cpu gtest_main gtest gloo INSTALL_TEST OFF)
   endif()
 endif()
 
diff --git i/test/cpp/dist_autograd/CMakeLists.txt w/test/cpp/dist_autograd/CMakeLists.txt
index 14fd7f7ae9a..9564d1270cb 100644
--- i/test/cpp/dist_autograd/CMakeLists.txt
+++ w/test/cpp/dist_autograd/CMakeLists.txt
@@ -7,7 +7,7 @@ if(USE_DISTRIBUTED AND NOT WIN32)
 
   add_executable(test_dist_autograd ${DIST_AUTOGRAD_TEST_SOURCES})
   target_include_directories(test_dist_autograd PRIVATE ${ATen_CPU_INCLUDE})
-  target_link_libraries(test_dist_autograd PRIVATE torch gtest_main)
+  target_link_libraries(test_dist_autograd PRIVATE torch gtest_main gtest)
 
   if(USE_CUDA)
     target_compile_definitions(test_dist_autograd PRIVATE "USE_CUDA")
diff --git i/test/cpp/jit/CMakeLists.txt w/test/cpp/jit/CMakeLists.txt
index 0b2a06b53c9..9c04aec99a8 100644
--- i/test/cpp/jit/CMakeLists.txt
+++ w/test/cpp/jit/CMakeLists.txt
@@ -126,7 +126,7 @@ if(USE_MKLDNN)
   target_link_libraries(test_jit PRIVATE caffe2::mkldnn)
 endif()
 
-set(JIT_TEST_DEPENDENCIES torch gtest_main jitbackend_test backend_with_compiler gmock)
+set(JIT_TEST_DEPENDENCIES torch gtest_main gtest jitbackend_test backend_with_compiler gmock)
 
 if(MSVC)
   list(APPEND JIT_TEST_DEPENDENCIES onnx_library)
diff --git i/test/cpp/lazy/CMakeLists.txt w/test/cpp/lazy/CMakeLists.txt
index 8bcd7ade611..131d0ae464c 100644
--- i/test/cpp/lazy/CMakeLists.txt
+++ w/test/cpp/lazy/CMakeLists.txt
@@ -28,7 +28,7 @@ add_executable(test_lazy
 # TODO temporary until we can delete the old gtest polyfills.
 target_compile_definitions(test_lazy PRIVATE USE_GTEST)
 
-set(LAZY_TEST_DEPENDENCIES torch gtest_main)
+set(LAZY_TEST_DEPENDENCIES torch gtest_main gtest)
 
 target_link_libraries(test_lazy PRIVATE ${LAZY_TEST_DEPENDENCIES})
 target_include_directories(test_lazy PRIVATE ${ATen_CPU_INCLUDE})
diff --git i/test/cpp/lite_interpreter_runtime/CMakeLists.txt w/test/cpp/lite_interpreter_runtime/CMakeLists.txt
index 5c27dd84f7c..27f7afdaeb4 100644
--- i/test/cpp/lite_interpreter_runtime/CMakeLists.txt
+++ w/test/cpp/lite_interpreter_runtime/CMakeLists.txt
@@ -21,7 +21,7 @@ target_include_directories(
   ${ATen_CPU_INCLUDE}
 )
 
-target_link_libraries(test_lite_interpreter_runtime PRIVATE torch gtest_main backend_with_compiler_runtime)
+target_link_libraries(test_lite_interpreter_runtime PRIVATE torch gtest_main gtest backend_with_compiler_runtime)
 
 if(LINUX)
   target_link_libraries(test_lite_interpreter_runtime PRIVATE "-Wl,--no-as-needed,$<TARGET_FILE:backend_with_compiler_runtime>,--as-needed")
diff --git i/test/cpp/nativert/CMakeLists.txt w/test/cpp/nativert/CMakeLists.txt
index 77636a2d109..04921418338 100644
--- i/test/cpp/nativert/CMakeLists.txt
+++ w/test/cpp/nativert/CMakeLists.txt
@@ -65,7 +65,7 @@ endif()
 # TODO temporary until we can delete the old gtest polyfills.
 target_compile_definitions(test_nativert PRIVATE USE_GTEST)
 
-set(NATIVERT_TEST_DEPENDENCIES torch gtest_main)
+set(NATIVERT_TEST_DEPENDENCIES torch gtest_main gtest)
 
 target_link_libraries(test_nativert PRIVATE ${CMAKE_DL_LIBS})
 target_link_libraries(test_nativert PRIVATE ${NATIVERT_TEST_DEPENDENCIES})
diff --git i/test/cpp/rpc/CMakeLists.txt w/test/cpp/rpc/CMakeLists.txt
index 454c54a0187..301319bbe71 100644
--- i/test/cpp/rpc/CMakeLists.txt
+++ w/test/cpp/rpc/CMakeLists.txt
@@ -5,7 +5,7 @@ set(TORCH_RPC_TEST_SOURCES
   ${TORCH_RPC_TEST_DIR}/test_wire_serialization.cpp
 )
 set(TORCH_RPC_TEST_DEPENDENCY_LIBS
-  torch gtest_main
+  torch gtest_main gtest
 )
 
 if(USE_GLOO)
diff --git i/test/inductor/cpp/CMakeLists.txt w/test/inductor/cpp/CMakeLists.txt
index 78a1ad8c2fd..e350270bf40 100644
--- i/test/inductor/cpp/CMakeLists.txt
+++ w/test/inductor/cpp/CMakeLists.txt
@@ -43,5 +43,5 @@ FetchContent_MakeAvailable(googletest)
 # TODO(voz): This is a little assumptive of just this one test, rewrite with real dir includes
 include_directories(${ATEN_INCLUDE})
 add_executable(test_cpp_prefix test_cpp_prefix.cpp ../../torchinductor/codegen/cpp_prefix.h)
-target_link_libraries(test_cpp_prefix gtest_main)
+target_link_libraries(test_cpp_prefix gtest_main gtest)
 add_test(NAME test_cpp_prefix COMMAND test_cpp_prefix)
diff --git i/test/mobile/lightweight_dispatch/CMakeLists.txt w/test/mobile/lightweight_dispatch/CMakeLists.txt
index 2064cea0d58..4090578fd6c 100644
--- i/test/mobile/lightweight_dispatch/CMakeLists.txt
+++ w/test/mobile/lightweight_dispatch/CMakeLists.txt
@@ -12,7 +12,7 @@ target_include_directories(test_codegen_unboxing PRIVATE ${ATen_CPU_INCLUDE})
 
 target_compile_definitions(test_codegen_unboxing PRIVATE USE_GTEST)
 
-set(TEST_UNBOXING_DEPENDENCIES torch gtest_main)
+set(TEST_UNBOXING_DEPENDENCIES torch gtest_main gtest)
 
 target_link_libraries(test_codegen_unboxing PRIVATE
   ${TEST_UNBOXING_DEPENDENCIES}
diff --git i/test/mobile/nnc/CMakeLists.txt w/test/mobile/nnc/CMakeLists.txt
index d38f0a41e22..136526b9dcd 100644
--- i/test/mobile/nnc/CMakeLists.txt
+++ w/test/mobile/nnc/CMakeLists.txt
@@ -11,7 +11,7 @@ add_executable(test_mobile_nnc
   ${MOBILE_NNC_TEST_SRCS}
 )
 
-target_link_libraries(test_mobile_nnc PRIVATE torch gtest_main)
+target_link_libraries(test_mobile_nnc PRIVATE torch gtest_main gtest)
 target_include_directories(test_mobile_nnc PRIVATE ${ATen_CPU_INCLUDE})
 target_compile_definitions(test_mobile_nnc PRIVATE USE_GTEST)
 
diff --git i/torch/CMakeLists.txt w/torch/CMakeLists.txt
index 3a3ca0f1236..9d4350a3b7e 100644
--- i/torch/CMakeLists.txt
+++ w/torch/CMakeLists.txt
@@ -82,9 +82,6 @@ set(TORCH_PYTHON_LINK_LIBRARIES
     Python::Module
     pybind::pybind11
     opentelemetry::api
-    httplib
-    nlohmann
-    moodycamel
     shm
     fmt::fmt-header-only
     ATEN_CPU_FILES_GEN_LIB)
