diff --git a/config/SundialsAddTest.cmake b/config/SundialsAddTest.cmake
index e965fed..a7fb1d2 100644
--- a/config/SundialsAddTest.cmake
+++ b/config/SundialsAddTest.cmake
@@ -70,7 +70,7 @@ MACRO(SUNDIALS_ADD_TEST NAME EXECUTABLE)
         "--verbose"
         "--testname=${NAME}"
         "--executablename=$<TARGET_FILE:${EXECUTABLE}>"
-        "--outputdir=${CMAKE_BINARY_DIR}/Testing/output"
+        "--outputdir=${TEST_OUTPUT_DIR}"
         )
 
       # do not diff the output and answer files
diff --git a/config/SundialsTesting.cmake b/config/SundialsTesting.cmake
index 815576a..d91801a 100644
--- a/config/SundialsTesting.cmake
+++ b/config/SundialsTesting.cmake
@@ -29,6 +29,13 @@ IF(SUNDIALS_DEVTESTS)
     ENDIF()
   ENDIF()
 
+  # Directory for test output
+  SET(TEST_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Testing/output)
+
+  IF(NOT EXISTS ${TEST_OUTPUT_DIR})
+    FILE(MAKE_DIRECTORY ${TEST_OUTPUT_DIR})
+  ENDIF()
+
   # look for the testRunner script in the test directory
   FIND_PROGRAM(TESTRUNNER testRunner PATHS test)
   HIDE_VARIABLE(TESTRUNNER)
diff --git a/test/testRunner b/test/testRunner
index f450ec2..f1c8deb 100755
--- a/test/testRunner
+++ b/test/testRunner
@@ -106,7 +106,8 @@ def main():
 
     # create output directory if necessary
     if not os.path.exists(outDir):
-        os.makedirs(outDir)
+        error("Output directory does not exist, it must be created.", outDir)
+        sys.exit(1)
     elif not os.path.isdir(outDir):
         error("Output directory exists but is not a directory, it must be deleted.", outDir)
         sys.exit(1)