start-worker

#!/bin/bash

if ! getent hosts spark-master; then
  echo "=== Cannot resolve the DNS entry for spark-master. Has the service been created yet, and is SkyDNS functional?"
  echo "=== See http://kubernetes.io/v1.1/docs/admin/dns.html for more details on DNS integration."
  echo "=== Sleeping 10s before pod exit."
  sleep 10
  exit 0
fi

# We don't this incoming service variables as it interferes directly with Spark.
#
# This is a bit subtle, if your service is named spark-master then k8 will
# create a variable SPARK_MASTER_PORT, which Spark will load, expecting a port number, and 
# get something like tcp://100.68.168.187:8080 .. and then it will crash with a
# error like so:
# java.lang.NumberFormatException: For input string: "tcp://10.108.205.234:7077"
#
# Good catch by Sai Varun Reddy Daram
# https://medium.com/@varunreddydaaram/kubernetes-did-not-work-with-apache-spark-de923ae7ab5c
unset SPARK_MASTER_PORT

# Writes any spark.* environmental variable to the spark-default.conf file
/write_configuration.py

# Run spark-class directly so that when it exits (or crashes), the pod restarts.
/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077 --webui-port 8081 --host $(hostname -i)