Merge tombstone updates (#3)

* Ensure tombstones created before kubexit started are read

This commit changes the `Watch` function inside the `tombstone` package to
also emit an initial event besides the `fsnotify` events. This initial event
is called immediatly when `Watch` is called and the watcher has been setup.

This change allows kubexit to detect tombstones written before kubexit was
started. This prevents possible race conditions as described by #8.

In order for this change to work, the `tombstone.EventHandler` type was
changed. It now requires a function with 3 arguments: The graveyard, the
tombstone and the operation instead of an `fsnotify.Event`. Reason being
that the initial event is not an `fsnotify.Event`. The functions implementing
an `tombstone.EventHandler` are changed accordingly.

This change on its own introduces a new bug where the tombstone is written
as part of an initial event, but the child process will still start because
`child.Start()` is being called after the watcher has been setup. To overcome
this issue, the shutdown state of the child is tracked in a new flag, which is
set if `ShutdownNow()` or `ShutdownWithTimeout()` is executed.

cmd/kubexit/main.go

@@ -96,7 +96,7 @@ func main() {
 	podName := os.Getenv("KUBEXIT_POD_NAME")
 	if podName == "" {
 		if len(birthDeps) > 0 {
-			log.Error(errors.New("missing env var: KUBEXIT_POD_NAME"), "missing env var", "var_name", "KUBEXIT_POD_NAME)
+			log.Error(errors.New("missing env var: KUBEXIT_POD_NAME"), "missing env var", "var_name", "KUBEXIT_POD_NAME")
 			os.Exit(2)
 		}
 		log.Info("Pod Name: N/A")
@@ -318,13 +318,11 @@ func onDeathOfAny(deathDeps []string, callback func()) tombstone.EventHandler {
 		deathDepSet[depName] = struct{}{}
 	}
 
-	return func(event fsnotify.Event) {
-		if event.Op&fsnotify.Create != fsnotify.Create && event.Op&fsnotify.Write != fsnotify.Write {
-			// ignore other events
+	return func(graveyard string, name string, op fsnotify.Op) {
+		if op != 0 && op&fsnotify.Create != fsnotify.Create && op&fsnotify.Write != fsnotify.Write {
+			// ignore events other than initial, create and write
 			return
 		}
-		graveyard := filepath.Dir(event.Name)
-		name := filepath.Base(event.Name)
 
 		log.Info("Tombstone modified:", "name", name)
 		if _, ok := deathDepSet[name]; !ok {

pkg/supervisor/supervisor.go

@@ -18,6 +18,7 @@ type Supervisor struct {
 	cmd           *exec.Cmd
 	sigCh         chan os.Signal
 	startStopLock sync.Mutex
+	shutdown      bool
 	shutdownTimer *time.Timer
 }
 
@@ -32,13 +33,18 @@ func New(name string, args ...string) *Supervisor {
 	cmd.Env = os.Environ()
 	return &Supervisor{
 		cmd: cmd,
+		shutdown: false,
 	}
 }
 
 func (s *Supervisor) Start() error {
 	s.startStopLock.Lock()
 	defer s.startStopLock.Unlock()
 
+	if s.shutdown {
+		return errors.New("not starting child process: shutdown already started")
+	}
+
 	log.Printf("Starting: %s\n", s)
 	if err := s.cmd.Start(); err != nil {
 		return fmt.Errorf("failed to start child process: %v", err)
@@ -90,6 +96,8 @@ func (s *Supervisor) ShutdownNow() error {
 	s.startStopLock.Lock()
 	defer s.startStopLock.Unlock()
 
+	s.shutdown = true
+
 	if !s.isRunning() {
 		log.Println("Skipping ShutdownNow: child process not running")
 		return nil
@@ -109,6 +117,8 @@ func (s *Supervisor) ShutdownWithTimeout(timeout time.Duration) error {
 	s.startStopLock.Lock()
 	defer s.startStopLock.Unlock()
 
+	s.shutdown = true
+
 	if !s.isRunning() {
 		log.Println("Skipping ShutdownWithTimeout: child process not running")
 		return nil

pkg/tombstone/tombstone.go

@@ -113,24 +113,24 @@ func Read(graveyard, name string) (*Tombstone, error) {
 	return &t, nil
 }
 
-type EventHandler func(fsnotify.Event)
+type EventHandler func(string, string, fsnotify.Op)
 
 // LoggingEventHandler is an example EventHandler that logs fsnotify events
-func LoggingEventHandler(event fsnotify.Event) {
-	if event.Op&fsnotify.Create == fsnotify.Create {
-		log.Info("Tombstone Watch: file created:", "name", event.Name)
+func LoggingEventHandler(graveyard string, tombstone string, op fsnotify.Op) {
+	if op&fsnotify.Create == fsnotify.Create {
+		log.Info("Tombstone Watch: file created:", "graveyard", graveyard, "tombstone", tombstone)
 	}
-	if event.Op&fsnotify.Remove == fsnotify.Remove {
-		log.Info("Tombstone Watch: file removed:", "name", event.Name)
+	if op&fsnotify.Remove == fsnotify.Remove {
+		log.Info("Tombstone Watch: file removed:", "graveyard", graveyard, "tombstone", tombstone)
 	}
-	if event.Op&fsnotify.Write == fsnotify.Write {
-		log.Info("Tombstone Watch: file modified:", "name", event.Name)
+	if op&fsnotify.Write == fsnotify.Write {
+		log.Info("Tombstone Watch: file modified:", "graveyard", graveyard, "tombstone", tombstone)
 	}
-	if event.Op&fsnotify.Rename == fsnotify.Rename {
-		log.Info("Tombstone Watch: file renamed:", "name", event.Name)
+	if op&fsnotify.Rename == fsnotify.Rename {
+		log.Info("Tombstone Watch: file renamed:", "graveyard", graveyard, "tombstone", tombstone)
 	}
-	if event.Op&fsnotify.Chmod == fsnotify.Chmod {
-		log.Info("Tombstone Watch: file chmoded:", "name", event.Name)
+	if op&fsnotify.Chmod == fsnotify.Chmod {
+		log.Info("Tombstone Watch: file chmoded:", "graveyard", graveyard, "tombstone", tombstone)
 	}
 }
 
@@ -153,7 +153,9 @@ func Watch(ctx context.Context, graveyard string, eventHandler EventHandler) err
 				if !ok {
 					return
 				}
-				eventHandler(event)
+				graveyard := filepath.Dir(event.Name)
+				tombstone := filepath.Base(event.Name)
+				eventHandler(graveyard, tombstone, event.Op)
 			case err, ok := <-watcher.Errors:
 				if !ok {
 					return
@@ -168,5 +170,21 @@ func Watch(ctx context.Context, graveyard string, eventHandler EventHandler) err
 	if err != nil {
 		return fmt.Errorf("failed to add watcher: %v", err)
 	}
+
+	// fire initial events after we started watching, this way no events are ever missed
+	f, err := os.Open(graveyard)
+	if err != nil {
+		return fmt.Errorf("failed to watch graveyard: %v", err)
+	}
+
+	files, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return fmt.Errorf("failed to watch for initial tombstones: %v", err)
+	}
+
+	for _, file := range files {
+		eventHandler(graveyard, file.Name(), 0)
+	}
 	return nil
 }