diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
index a5e53d06b68cc..3a049ec77fd1f 100644
--- a/cpp/cmake_modules/FindGFlags.cmake
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -69,6 +69,7 @@ if(GFLAGS_FOUND)
                        "${GFLAGS_SHARED_LIB}"
                        INCLUDE_DIRECTORIES
                        "${GFLAGS_INCLUDE_DIR}")
+    target_compile_definitions(gflags_shared INTERFACE "GFLAGS_IS_A_DLL=1")
     set(GFLAGS_STATIC_LIB FLASE)
     return()
   else()
@@ -111,7 +112,7 @@ if(GFLAGS_INCLUDE_DIR)
   else()
     set(GFLAGS_STATIC FALSE)
   endif()
-  if(GFLAGS_SHARED AND GFLAGS_STATIC)
+  if(GFLAGS_SHARED OR GFLAGS_STATIC)
     set(GFLAGS_FOUND TRUE)
     if(MSVC)
       set(GFLAGS_MSVC_DEPS "shlwapi.lib")
@@ -125,6 +126,12 @@ if(GFLAGS_INCLUDE_DIR)
                        "${GFLAGS_INCLUDE_DIR}"
                        DEPS
                        "${GFLAGS_MSVC_DEPS}")
+    if(GFLAGS_SHARED)
+      target_compile_definitions(gflags_shared INTERFACE "GFLAGS_IS_A_DLL=1")
+    endif()
+    if(GFLAGS_STATIC)
+      target_compile_definitions(gflags_static INTERFACE "GFLAGS_IS_A_DLL=0")
+    endif()
   else()
     set(GFLAGS_FOUND FALSE)
   endif()
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 56846c64e0e2a..7b99440ce3627 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -653,6 +653,7 @@ if(ARROW_NEED_GFLAGS)
     ADD_THIRDPARTY_LIB(gflags
       STATIC_LIB ${GFLAGS_STATIC_LIB})
     set(GFLAGS_LIBRARY gflags_static)
+    target_compile_definitions(${GFLAGS_LIBRARY} INTERFACE "GFLAGS_IS_A_DLL=0")
     if(MSVC)
       set_target_properties(${GFLAGS_LIBRARY}
 	PROPERTIES
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 81d94c9e8fbd1..e3c27fa5daea3 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -15,11 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// ARROW-3837: Depending on how gflags was built, this might be set to
-// 1 or 0 by default, making it so that we cannot statically link
-// gflags on Windows if it is set to 1
-#define GFLAGS_IS_A_DLL 0
-
 #include <cstdint>
 #include <cstdio>
 #include <cstring>