- fix isolated console mode (i.e. launching from explorer.exe)

ZDoom · Oct 16, 2024 · 0a9eb47 · 0a9eb47
1 parent 81f2bef
commit 0a9eb47
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 18 deletions.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -1361,7 +1361,7 @@ endif()
 if( MSVC )
 	option ( CONSOLE_MODE "Compile as a console application" OFF )
 	if ( CONSOLE_MODE )
-		set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE /ENTRY:wWinMainCRTStartup" )
+		set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE /ENTRY:wmainCRTStartup" )
 	else()
 		set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:WINDOWS /ENTRY:wWinMainCRTStartup" )
 	endif()

diff --git a/src/common/platform/win32/i_main.cpp b/src/common/platform/win32/i_main.cpp
@@ -45,6 +45,11 @@
 #include <shellapi.h>
 #include <VersionHelpers.h>
 
+#include <iostream>
+#include <fstream>
+#include <io.h>
+#include <fcntl.h>
+
 #ifdef _MSC_VER
 #pragma warning(disable:4244)
 #endif
@@ -149,11 +154,22 @@ void I_SetIWADInfo()
 
 bool isConsoleApp()
 {
-	DWORD pids[2];
-	DWORD num_pids = GetConsoleProcessList(pids, 2);
-	bool win32con_is_exclusive = (num_pids <= 1);
+	static bool alreadychecked = false;
+	static bool returnvalue;
+
+	if (!alreadychecked)
+	{
+		DWORD pids[2];
+		DWORD num_pids = GetConsoleProcessList(pids, 2);
+		bool win32con_is_exclusive = (num_pids <= 1);
 
-	return GetConsoleWindow() != NULL && !win32con_is_exclusive;
+		returnvalue = ((GetConsoleWindow() != NULL && !win32con_is_exclusive) || (GetStdHandle(STD_OUTPUT_HANDLE) != NULL));
+		alreadychecked = true;
+	}
+
+	//printf("isConsoleApp is %i\n", returnvalue);
+
+	return returnvalue;
 }
 
 //==========================================================================
@@ -183,19 +199,7 @@ int DoMain (HINSTANCE hInstance)
 	if (isConsoleApp())
 	{
 		StdOut = GetStdHandle(STD_OUTPUT_HANDLE);
-		BY_HANDLE_FILE_INFORMATION info;
-
-		if (!GetFileInformationByHandle(StdOut, &info) && StdOut != nullptr)
-		{
-			SetConsoleCP(CP_UTF8);
-			SetConsoleOutputCP(CP_UTF8);
-			DWORD mode;
-			if (GetConsoleMode(StdOut, &mode))
-			{
-				if (SetConsoleMode(StdOut, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
-					FancyStdOut = IsWindows10OrGreater(); // Windows 8.1 and lower do not understand ANSI formatting.
-			}
-		}
+		FancyStdOut = IsWindows10OrGreater(); // Windows 8.1 and lower do not understand ANSI formatting.
 	}
 	else if (Args->CheckParm("-stdout") || Args->CheckParm("-norun"))
 	{
@@ -514,6 +518,11 @@ CUSTOM_CVAR(Bool, disablecrashlog, false, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
 //
 //==========================================================================
 
+int wmain()
+{
+    return wWinMain(GetModuleHandle(0), 0, GetCommandLineW(), SW_SHOW);
+}
+
 int WINAPI wWinMain (HINSTANCE hInstance, HINSTANCE nothing, LPWSTR cmdline, int nCmdShow)
 {
 	g_hInst = hInstance;

diff --git a/src/common/platform/win32/i_system.cpp b/src/common/platform/win32/i_system.cpp
@@ -102,6 +102,7 @@
 // PUBLIC FUNCTION PROTOTYPES ----------------------------------------------
 
 void DestroyCustomCursor();
+bool isConsoleApp();
 
 // PRIVATE FUNCTION PROTOTYPES ---------------------------------------------
 
@@ -306,6 +307,7 @@ static void PrintToStdOut(const char *cpt, HANDLE StdOut)
 			else break;
 		}
 	}
+
 	DWORD bytes_written;
 	WriteFile(StdOut, printData.GetChars(), (DWORD)printData.Len(), &bytes_written, NULL);
 	if (terminal)