Add basic distcc and ccache support to haikuporter

HaikuPorter/BuildPlatform.py

@@ -72,7 +72,15 @@ def resolveDependencies(self, dependencyInfoFiles, requiresTypes,
 			return
 		resolver = DependencyResolver(self, requiresTypes, repositories,
 									  **kwargs)
-		return resolver.determineRequiredPackagesFor(dependencyInfoFiles)
+		resolver.determineRequiredPackagesFor(dependencyInfoFiles)
+		if Configuration.shallUseCcache():
+		    resolver.injectDependency('Required by configuration', 'cmd:ccache')
+		if Configuration.shallUseDistcc():
+		    # pump needs its own switch and is out of my scope.
+		    # see https://preview.tinyurl.com/y32dmfrv -- possible breakage!
+		    resolver.injectDependency('Required by configuration', 'cmd:distcc')
+			# is it worth pulling in ssh for "secure pump"? Probably not.
+		return resolver.getResult()
 
 
 # -- BuildPlatformHaiku class -------------------------------------------------

HaikuPorter/Configuration.py

@@ -36,6 +36,22 @@ def isExecutable(path):
 
 # allowed types of the configuration file values
 haikuportsAttributes = {
+	'USE_DISTCC': {
+		'type': YesNo,
+		'required': False,
+		'default': False,
+		'extendable': Extendable.NO,
+		'indexable': False,
+		'setAttribute': 'useDistcc',
+	},
+	'USE_CCACHE': {
+		'type': YesNo,
+		'required': False,
+		'default': False,
+		'extendable': Extendable.NO,
+		'indexable': False,
+		'setAttribute': 'useCcache',
+	},
 	'ALLOW_UNTESTED': {
 		'type': YesNo,
 		'required': False,
@@ -253,6 +269,8 @@ def __init__(self):
 		self.vendor = None
 		self.packagesPath = None
 		self.sourceforgeMirror = None
+		self.useCcache = False
+		self.useDistcc = False
 
 		self._readConfigurationFile()
 
@@ -374,6 +392,22 @@ def getSourceforgeMirror():
 	def getVendor():
 		return Configuration.configuration.vendor
 
+	@staticmethod
+	def shallUseCcache():
+		return Configuration.configuration.useCcache
+
+	@staticmethod
+	def shallUseDistcc():
+		return Configuration.configuration.useDistcc
+
+	@staticmethod
+	def setUseDistcc(yes=True):
+		Configuration.configuration.useDistcc = yes
+
+	@staticmethod
+	def setUseCcache(yes=True):
+		Configuration.configuration.useCcache = yes
+
 	def _readConfigurationFile(self):
 		# Find the configuration file. It may be
 		# * specified on the command line,

HaikuPorter/DependencyResolver.py

@@ -117,6 +117,22 @@ def determineRequiredPackagesFor(self, dependencyInfoFiles):
 		self._satisfiedPackagesCache += result
 		return result
 
+	def injectDependency(self, reason, requirement):
+		self._addImmediate(reason, ResolvableExpression(requirement),
+			'build_prerequires', True)
+		self._buildDependencyGraph()
+		result = [
+			node.path for node in self._packageNodes
+		]
+		return result
+
+	def getResult(self):
+		result = [
+			node.path for node in self._packageNodes
+		]
+		return result
+
+
 	def _populateProvidesManager(self):
 		for repository in self._repositories:
 			for entry in os.listdir(repository):

HaikuPorter/Main.py

@@ -581,6 +581,10 @@ def _buildMainPort(self, port, testPort):
 					'resolving build dependencies failed: {}'.format(exception))
 				return
 		else:
+			if self.options.useCache:
+			    Configuration.setUseCcache(True)
+			if self.options.useDistcc:
+			    Configuration.setUseDistcc(True)
 			buildDependencies = port.resolveDependencies(
 				self.packageRepositories, testPort)
 
@@ -676,6 +680,12 @@ def _buildPort(self, port, parseRecipe, testPort):
 			if self.options.patch:
 				port.patchSource()
 
+		if self.options.useCcache:
+		    Configuration.setUseCcache(True)
+
+		if self.options.useDistcc:
+		    Configuration.setUseDistcc(True)
+
 		if self.options.build:
 			port.build(self.packagesPath, self.options.package, targetPath)
 

HaikuPorter/Options.py

@@ -326,6 +326,16 @@ def parseOptions():
 		dest='activeVersionsOnly', default=False,
 		help='only check in active versions of ports instead of all ports')
 
+	parser.add_option('--use-ccache', action='store_true',
+		dest='useCcache', default=False,
+		help='Use ccache to accelerate building by caching artefacts '
+		+ '(Uses ccache from ~)')
+
+	parser.add_option('--use-distcc', action='store_true',
+		dest='useDistcc', default=False,
+		help='Use Distcc to offload compilation to other machinesi '
+		+ '(uses configuration from ~/config)')
+
 	global __Options__
 
 	(__Options__, args) = parser.parse_args()

HaikuPorter/Port.py

@@ -44,6 +44,8 @@
 	getShellVariableSetters,
 	recipeActionScript,
 	setupChrootScript,
+	setupDistccScript,
+	setupCcacheScript,
 )
 from .Source import Source
 from .Utils import (
@@ -77,6 +79,12 @@ def __enter__(self):
 		shellEnv.update(self.envVars)
 		check_output(['bash', '-c', setupChrootScript], env=shellEnv,
 			cwd=self.path)
+		if Configuration.shallUseCcache():
+		    check_output(['bash', '-c', setupCcacheScript],
+			    env=shellEnv, cwd=self.path)
+		if Configuration.shallUseDistcc():
+		    check_output(['bash', '-c', setupDistccScript],
+			    env=shellEnv, cwd=self.path)
 		return self
 
 	def __exit__(self, ignoredType, value, traceback):
@@ -1564,6 +1572,11 @@ def _openShell(self, params, targetDir, env):
 		# define a terminal
 		shellEnv['TERM'] = 'xterm'
 
+		if Configuration.shallUseDistcc():
+		    shellEnv['HOME'] = '/boot/home'
+		    if Configuration.shallUseCcache():
+			shellEnv['CCACHE_PREFIX'] = 'distcc'
+
 		# execute the requested action via a shell ...
 		args = ['bash']
 		args += params

HaikuPorter/ShellScriptlets.py

@@ -762,6 +762,95 @@ def getShellVariableSetters(shellVariables):
 '''
 
 
+# -----------------------------------------------------------------------------
+
+# Shell scriptlet which sets up ccache in chroot
+# MUST be called BEFORE distcc setup!
+# we add distcc to the chain if req'd by passing an extra environment variable.
+# Something like CCACHE_PREFIX=distcc and we let computer find it for us
+# We don't remove this on success: cache might be reused after a patch
+setupCcacheScript = r'''
+
+if ! [ -e boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper ]; then
+
+    # this is the first thing in $PATH after . so we're jumping in front of gcc
+    mkdir -p boot/home/config/non-packaged/bin
+
+    # catch unprefixed invocations and redo them correctly
+    echo '#!/bin/bash
+	    exec '$targetArchitecture'-unknown-haiku-g${0:$[-2]} "$@"' > \
+	     boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper
+    chmod a+x boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper
+
+    # we add the prefix because otherwise both machines just use 'gcc'
+    # and i can't link linux objects to haiku objects so that fails
+    ln -s /bin/ccache boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-c++
+    ln -s /bin/ccache boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-g++
+    ln -s /bin/ccache boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-gcc
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/cc
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/gcc
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/g++
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/c++
+
+    # this stops the gxx from above from falling straight through to /bin/gxx
+    mkdir -p boot/home/config/bin
+    ln -s /bin/c++ boot/home/config/bin/$targetArchitecture-unknown-haiku-c++
+    ln -s /bin/g++ boot/home/config/bin/$targetArchitecture-unknown-haiku-g++
+    ln -s /bin/gcc boot/home/config/bin/$targetArchitecture-unknown-haiku-gcc
+    ln -s /bin/cc boot/home/config/bin/$targetArchitecture-unknown-haiku-cc
+fi
+
+# ?? We don't want to lose this but a soft link can't point to the real directory. it still helps...
+if ! [ -e boot/home/.ccache.ccache.conf ]; then
+    mkdir -p boot/home/.ccache
+    cp /boot/home/.ccache/ccache.conf boot/home/.ccache
+fi
+
+'''
+
+
+# -----------------------------------------------------------------------------
+
+# Shell scriptlet which sets up distcc in chroot
+setupDistccScript = r'''
+
+if ! [ -e boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper ]; then
+
+    # this is the first thing in $PATH after . so we're jumping in front of gcc
+    mkdir -p boot/home/config/non-packaged/bin
+
+    # catch unprefixed invocations and redo them correctly
+    echo '#!/bin/bash
+	    exec '$targetArchitecture'-unknown-haiku-g${0:$[-2]} "$@"' > \
+	     boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper
+    chmod a+x boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper
+    ln -s /bin/distcc boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-c++
+    ln -s /bin/distcc boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-g++
+    ln -s /bin/distcc boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-gcc
+
+    # we add the prefix because otherwise both machines just use 'gcc'
+    # and i can't link linux objects to haiku objects so that fails
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/cc
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/gcc
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/g++
+    ln -s /boot/home/config/non-packaged/bin/$targetArchitecture-unknown-haiku-wrapper boot/home/config/non-packaged/bin/c++
+
+    # this stops the gxx from above from falling straight through to /bin/gxx
+    mkdir -p boot/home/config/bin
+    ln -s /bin/c++ boot/home/config/bin/$targetArchitecture-unknown-haiku-c++
+    ln -s /bin/g++ boot/home/config/bin/$targetArchitecture-unknown-haiku-g++
+    ln -s /bin/gcc boot/home/config/bin/$targetArchitecture-unknown-haiku-gcc
+    ln -s /bin/cc boot/home/config/bin/$targetArchitecture-unknown-haiku-cc
+fi
+
+if ! [ -e boot/home/.distcc ]; then
+    # copy user's distcc setup into chroot so we know our volunteers etc
+    cp -R /boot/home/.distcc boot/home
+fi
+
+'''
+
+
 # -----------------------------------------------------------------------------
 
 # Shell scriptlet that prepares a chroot environment for entering.