From efdd9f41fbac987213f451695651b5aa2b8e600d Mon Sep 17 00:00:00 2001 From: Richard Lin Date: Mon, 29 Jul 2024 23:03:25 -0700 Subject: [PATCH] Update for re-using host Python process (#147) This doesn't affect the IDE compilation process, but the internal APIs changed. --- PolymorphicBlocks | 2 +- .../runner/CompileProcessHandler.scala | 43 ++++++++++--------- .../edg_ide/runner/DseProcessHandler.scala | 27 ++++-------- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/PolymorphicBlocks b/PolymorphicBlocks index 8ebe6731..f80beeba 160000 --- a/PolymorphicBlocks +++ b/PolymorphicBlocks @@ -1 +1 @@ -Subproject commit 8ebe673172883e512c76d1f3d354f3f157825e4b +Subproject commit f80beebac17bcd95f3f75ba76942bc45245ba669 diff --git a/src/main/scala/edg_ide/runner/CompileProcessHandler.scala b/src/main/scala/edg_ide/runner/CompileProcessHandler.scala index ed3a1be1..855c0df2 100644 --- a/src/main/scala/edg_ide/runner/CompileProcessHandler.scala +++ b/src/main/scala/edg_ide/runner/CompileProcessHandler.scala @@ -75,20 +75,20 @@ class DesignTopRunParams(workingDirectory: String, sdkHome: String, moduleName: } // a PythonInterface that uses the on-event hooks to log to the console -class LoggingPythonInterface( - console: ConsoleView, - interpreter: String, - pythonPaths: Seq[String] = Seq(), -) extends PythonInterface(interpreter = interpreter, pythonPaths = pythonPaths) { +class LoggingPythonInterface(interpreter: String, pythonPaths: Seq[String], console: ConsoleView) + extends ProtobufStdioSubprocess(interpreter = interpreter, pythonPaths = pythonPaths) { def forwardProcessOutput(): Unit = { - StreamUtils.forAvailable(processOutputStream) { data => + StreamUtils.forAvailable(outputStream) { data => console.print(new String(data), ConsoleViewContentType.NORMAL_OUTPUT) } - StreamUtils.forAvailable(processErrorStream) { data => + StreamUtils.forAvailable(errorStream) { data => console.print(new String(data), ConsoleViewContentType.ERROR_OUTPUT) } } +} +class LoggingCompilerInterface(interface: LoggingPythonInterface, console: ConsoleView) + extends PythonInterface(interface) { override def onLibraryRequest(element: ref.LibraryPath): Unit = { // this needs to be here to only print on requests that made it to Python (instead of just hit cache) console.print(s"Compile ${element.toSimpleString}\n", ConsoleViewContentType.LOG_DEBUG_OUTPUT) @@ -98,7 +98,7 @@ class LoggingPythonInterface( element: ref.LibraryPath, result: Errorable[(schema.Library.NS.Val, Option[edgrpc.Refinements])] ): Unit = { - forwardProcessOutput() + interface.forwardProcessOutput() result match { case Errorable.Error(msg) => console.print( @@ -127,7 +127,7 @@ class LoggingPythonInterface( values: Map[ref.LocalPath, ExprValue], result: Errorable[elem.HierarchyBlock] ): Unit = { - forwardProcessOutput() + interface.forwardProcessOutput() result match { case Errorable.Error(msg) => console.print( @@ -142,7 +142,7 @@ class LoggingPythonInterface( refinementPass: ref.LibraryPath, result: Errorable[Map[DesignPath, ExprValue]] ): Unit = { - forwardProcessOutput() + interface.forwardProcessOutput() result match { case Errorable.Error(msg) => console.print( @@ -157,7 +157,7 @@ class LoggingPythonInterface( backend: ref.LibraryPath, result: Errorable[Map[DesignPath, String]] ): Unit = { - forwardProcessOutput() + interface.forwardProcessOutput() result match { case Errorable.Error(msg) => console.print( @@ -275,14 +275,14 @@ class CompileProcessHandler( ) extends ProcessHandler with HasConsoleStages { var runThread: Option[Thread] = None - var pythonInterfaceOpt: Option[LoggingPythonInterface] = None + var pythonProcessOpt: Option[LoggingPythonInterface] = None override def destroyProcessImpl(): Unit = { - pythonInterfaceOpt.foreach { pythonInterface => + pythonProcessOpt.foreach { pythonInterface => pythonInterface.destroy() pythonInterface.forwardProcessOutput() console.print(f"Python subprocess terminated.\n", ConsoleViewContentType.ERROR_OUTPUT) - pythonInterfaceOpt = None + pythonProcessOpt = None } runThread.foreach(_.interrupt()) console.print(f"Compilation terminated.\n", ConsoleViewContentType.ERROR_OUTPUT) @@ -310,7 +310,7 @@ class CompileProcessHandler( console.print(s"Starting compilation of ${options.designName}\n", ConsoleViewContentType.LOG_INFO_OUTPUT) BlockVisualizerService(project).setDesignStale() - require(pythonInterfaceOpt.isEmpty) + require(pythonProcessOpt.isEmpty) var exitCode: Int = -1 try { @@ -323,10 +323,11 @@ class CompileProcessHandler( s"Using interpreter from configured SDK '$sdkName': $pythonCommand\n", ConsoleViewContentType.LOG_INFO_OUTPUT ) - val pythonInterface = new LoggingPythonInterface(console, pythonCommand, pythonPaths) - pythonInterfaceOpt = Some(pythonInterface) + val pythonProcess = new LoggingPythonInterface(pythonCommand, pythonPaths, console) + val pythonInterface = new LoggingCompilerInterface(pythonProcess, console) + pythonProcessOpt = Some(pythonProcess) - val packagePrefix = pythonInterface.packagePrefix + val packagePrefix = pythonProcess.packagePrefix if (packagePrefix.nonEmpty) { console.print(s"Using core prefix ${packagePrefix}\n", ConsoleViewContentType.LOG_INFO_OUTPUT) } @@ -479,11 +480,11 @@ class CompileProcessHandler( } } } - exitCode = pythonInterface.shutdown() - pythonInterface.forwardProcessOutput() // dump remaining process output (shouldn't happen) + exitCode = pythonProcess.shutdown() + pythonProcess.forwardProcessOutput() // dump remaining process output (shouldn't happen) } catch { // this generally shouldn't happen but is an overall catch-all and clean-up case e: Throwable => - pythonInterfaceOpt.foreach { pyIf => + pythonProcessOpt.foreach { pyIf => exitCode = pyIf.shutdown() pyIf.forwardProcessOutput() // dump remaining process output before the final error message } diff --git a/src/main/scala/edg_ide/runner/DseProcessHandler.scala b/src/main/scala/edg_ide/runner/DseProcessHandler.scala index 77d095ed..c11ee893 100644 --- a/src/main/scala/edg_ide/runner/DseProcessHandler.scala +++ b/src/main/scala/edg_ide/runner/DseProcessHandler.scala @@ -149,21 +149,9 @@ class DseProcessHandler(project: Project, options: DseRunConfigurationOptions, v // This structure is quite nasty, but is needed to give a stream handle in case something crashes, // in which case pythonInterface is not a valid reference - var pythonInterfaceOption: Option[PythonInterface] = None + var pythonProcessOpt: Option[LoggingPythonInterface] = None var exitCode: Int = -1 - def forwardProcessOutput(): Unit = { - pythonInterfaceOption.foreach { pyIf => - StreamUtils.forAvailable(pyIf.processOutputStream) { data => - console.print(new String(data), ConsoleViewContentType.NORMAL_OUTPUT) - } - } - pythonInterfaceOption.foreach { pyIf => - StreamUtils.forAvailable(pyIf.processErrorStream) { data => - console.print(new String(data), ConsoleViewContentType.ERROR_OUTPUT) - } - } - } try { val (pythonCommand, pythonPaths, sdkName) = @@ -176,8 +164,9 @@ class DseProcessHandler(project: Project, options: DseRunConfigurationOptions, v ConsoleViewContentType.LOG_INFO_OUTPUT ) - val pythonInterface = new LoggingPythonInterface(console, pythonCommand, pythonPaths) - pythonInterfaceOption = Some(pythonInterface) + val pythonProcess = new LoggingPythonInterface(pythonCommand, pythonPaths, console) + val pythonInterface = new LoggingCompilerInterface(pythonProcess, console) + pythonProcessOpt = Some(pythonProcess) (pythonInterface.getProtoVersion() match { case Errorable.Success(pyVersion) if pyVersion == Compiler.kExpectedProtoVersion => None @@ -345,12 +334,14 @@ class DseProcessHandler(project: Project, options: DseRunConfigurationOptions, v } } catch { case e: Throwable => - pythonInterfaceOption.foreach { pyIf => exitCode = pyIf.shutdown() } - forwardProcessOutput() // dump remaining process output first + pythonProcessOpt.foreach { pyIf => + exitCode = pyIf.shutdown() + pyIf.forwardProcessOutput() // dump remaining process output before the final error message + } - console.print(s"Compiler internal error: ${e.toString}\n", ConsoleViewContentType.ERROR_OUTPUT) val stackWriter = new StringWriter() e.printStackTrace(new PrintWriter(stackWriter)) + console.print(s"Compiler internal error: ${e.toString}\n", ConsoleViewContentType.ERROR_OUTPUT) console.print(stackWriter.toString, ConsoleViewContentType.ERROR_OUTPUT) }