-
Notifications
You must be signed in to change notification settings - Fork 28
How to build
Kazuaki Ishizaki edited this page Jan 22, 2016
·
20 revisions
This is a page to describe how to build Spark version of column-based RDD in array format.
$ mkdir -p ~/Spark
$ cd ~/Spark
$ git clone [email protected]:kiszk/spark-gpu.git
$ git clone [email protected]:MysterionRise/mavenized-jcuda.git
$ edit mavenized-jcuda/pom.xml
<jcuda.version>0.7.0a</jcuda.version> // set CUDA version (e.g. 0.7.0a, 0.7.5, ...)
$ cd mavenized-jcuda && mvn install && cd ..
$ cd spark-gpu && git checkout dev
$ export JAVA_HOME=<YOUR JAVA HOME> // e.g. JAVA_HOME=/usr/lib/jvm/java-8-openjdk-ppc64el
$ gpudev/util/compile.sh clean package install
$ mkdir -p ~/Spark
$ cd ~/Spark
$ git clone [email protected]:kiszk/spark-gpu.git
$ git clone [email protected]:MysterionRise/mavenized-jcuda.git
$ edit mavenized-jcuda/pom.xml
<jcuda.version>0.7.0a</jcuda.version> // set CUDA version (e.g. 0.7.0a, 0.7.5, ...)
$ cd mavenized-jcuda && mvn install && cd ..
$ cd spark-gpu && git checkout dev
$ export JAVA_HOME=<YOUR JAVA HOME> // e.g. JAVA_HOME=/usr/lib/jvm/java-8-openjdk-ppc64el
$ gpudev/util/compile.sh clean package install
// if you got an error for executing ng, you can enable ng for ppc64le
$ cd ~/Spark
$ git clone [email protected]:ibmsoe/zinc.git
$ cp zinc/src/universal/bin/nailgun spark-gpu/build/zinc-0.3.5.3/bin
$ mkdir -p spark-gpu/build/zinc-0.3.5.3/bin/ng/linuxppc64le
$ cp zinc/src/universal/bin/ng/linuxppc64le/ng spark-gpu/build/zinc-0.3.5.3/bin/ng/linuxppc64le
$ cd spark-gpu
$ gpudev/util/compile.sh clean package install
$ cd gpudev/util
$ ./testcore.sh -Dtest=none -DwildcardSuites=org.apache.spark.cuda.CUDAFunctionSuite // CUDA test set
$ ./testdev.sh // smaller test sets
$ ./testcore.sh // larger test sets
$ mkdir -p ~/Spark
$ cd ~/Spark
$ git clone [email protected]:kiszk/spark-gpu.git
$ git clone [email protected]:MysterionRise/mavenized-jcuda.git
$ export MAVEN_OPTS="-Xmx3g"
$ export JAVA_HOME=<YOUR JAVA HOME> // e.g. JAVA_HOME=/usr/lib/jvm/java-8-openjdk-ppc64el
$ ./make-distribution.sh --tgz -Dscala-2.11 -Phadoop-2.4 -Pyarn
// if you got an error for executing ng on ppc64le, please see instruction in "How to build for ppc64le"