diff --git a/ports/dlib/portfile.cmake b/ports/dlib/portfile.cmake
index fa093539f161ab..183e533347fbc6 100644
--- a/ports/dlib/portfile.cmake
+++ b/ports/dlib/portfile.cmake
@@ -24,6 +24,11 @@ if (VCPKG_TARGET_IS_WINDOWS AND VCPKG_TARGET_ARCHITECTURE STREQUAL "arm64")
     set(COMMON_OPTIONS -DUSE_SSE2_INSTRUCTIONS=OFF)
 endif()
 
+set(dbg_opts "")
+if(VCPKG_TARGET_IS_WINDOWS)
+  set(dbg_opts -DDLIB_ENABLE_ASSERTS=ON)
+endif()
+
 vcpkg_cmake_configure(
     SOURCE_PATH "${SOURCE_PATH}"
     OPTIONS
@@ -39,12 +44,19 @@ vcpkg_cmake_configure(
         -DDLIB_WEBP_SUPPORT=OFF
         -DDLIB_USE_MKL_FFT=OFF
     OPTIONS_DEBUG
-        -DDLIB_ENABLE_ASSERTS=ON
+        ${dbg_opts}
         #-DDLIB_ENABLE_STACK_TRACE=ON
 )
 
 vcpkg_cmake_install()
 
+if(VCPKG_TARGET_IS_WINDOWS)
+  # Dlib encodes debug/release in its config.h. Patch it to respond to the NDEBUG macro instead. <- The below is using _DEBUG but there is no correct way to switch this on !windows
+  # Only windows defines _DEBUG in debug builds.
+  vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/dlib/config.h" "/* #undef ENABLE_ASSERTS */" "#if defined(_DEBUG)\n#define ENABLE_ASSERTS\n#endif")
+  vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/dlib/config.h" "#define DLIB_DISABLE_ASSERTS" "#if !defined(_DEBUG)\n#define DLIB_DISABLE_ASSERTS\n#endif")
+endif()
+
 vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/dlib)
 
 vcpkg_fixup_pkgconfig()
@@ -68,8 +80,4 @@ file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/dlib/cmake_utils/test_for_l
 file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/dlib/cmake_utils/test_for_libpng")
 file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/dlib/external/libpng/arm")
 
-# Dlib encodes debug/release in its config.h. Patch it to respond to the NDEBUG macro instead.
-vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/dlib/config.h" "/* #undef ENABLE_ASSERTS */" "#if defined(_DEBUG)\n#define ENABLE_ASSERTS\n#endif")
-vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/dlib/config.h" "#define DLIB_DISABLE_ASSERTS" "#if !defined(_DEBUG)\n#define DLIB_DISABLE_ASSERTS\n#endif")
-
 file(INSTALL "${SOURCE_PATH}/dlib/LICENSE.txt" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
diff --git a/ports/dlib/vcpkg.json b/ports/dlib/vcpkg.json
index 2c1008337a5879..c5f64563b0aa85 100644
--- a/ports/dlib/vcpkg.json
+++ b/ports/dlib/vcpkg.json
@@ -1,7 +1,7 @@
 {
   "name": "dlib",
   "version": "19.24",
-  "port-version": 3,
+  "port-version": 4,
   "description": "Modern C++ toolkit containing machine learning algorithms and tools for creating complex software in C++",
   "homepage": "https://github.com/davisking/dlib",
   "license": "BSL-1.0",
diff --git a/versions/baseline.json b/versions/baseline.json
index 4adf85fd4e729b..ecef33f21f2ad2 100644
--- a/versions/baseline.json
+++ b/versions/baseline.json
@@ -2270,7 +2270,7 @@
     },
     "dlib": {
       "baseline": "19.24",
-      "port-version": 3
+      "port-version": 4
     },
     "dlpack": {
       "baseline": "0.8",
diff --git a/versions/d-/dlib.json b/versions/d-/dlib.json
index 2d62df6293e255..8c4fb978506e2b 100644
--- a/versions/d-/dlib.json
+++ b/versions/d-/dlib.json
@@ -1,5 +1,10 @@
 {
   "versions": [
+    {
+      "git-tree": "f9c3d8205c42c8450e81864a313021ff76efec5f",
+      "version": "19.24",
+      "port-version": 4
+    },
     {
       "git-tree": "5a2fe0fe58b23b16ef1f94c5b68bbd560ac54649",
       "version": "19.24",