diff --git a/commands/commands.go b/commands/commands.go index b81b867f9a0..e50cf06a59b 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -208,9 +208,9 @@ type hugoBuilderCommon struct { baseURL string environment string - buildWatch bool - poll string - clock string + buildWatch bool + poll string + clock string gc bool diff --git a/commands/hugo.go b/commands/hugo.go index d2d99c7fabc..8e40b1c1cd0 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -835,10 +835,6 @@ func (c *commandeer) fullRebuild(changeType string) { // newWatcher creates a new watcher to watch filesystem events. func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*watcher.Batcher, error) { - if runtime.GOOS == "darwin" { - tweakLimit() - } - staticSyncer, err := newStaticSyncer(c) if err != nil { return nil, err diff --git a/go.mod b/go.mod index 6e7c1258e48..b75b3d4e3f3 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,7 @@ require ( github.com/aws/smithy-go v1.8.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/fsnotify/fsevents v0.1.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/golang-jwt/jwt/v4 v4.0.0 // indirect diff --git a/go.sum b/go.sum index b25dd9f2c02..783cbbd0749 100644 --- a/go.sum +++ b/go.sum @@ -249,6 +249,8 @@ github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/ github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsevents v0.1.1 h1:/125uxJvvoSDDBPen6yUZbil8J9ydKZnnl3TWWmvnkw= +github.com/fsnotify/fsevents v0.1.1/go.mod h1:+d+hS27T6k5J8CRaPLKFgwKYcpS7GwW3Ule9+SC2ZRc= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= diff --git a/watcher/filenotify/eventwatcher_darwin.go b/watcher/filenotify/eventwatcher_darwin.go new file mode 100644 index 00000000000..7cc92ae2320 --- /dev/null +++ b/watcher/filenotify/eventwatcher_darwin.go @@ -0,0 +1,306 @@ +// Copyright 2022 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +//go:build darwin && cgo + +package filenotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsevents" + "github.com/fsnotify/fsnotify" +) + +var ( + errFSEventsWatcherClosed = errors.New("fsEventsWatcher is closed") + errFSEventsWatcherStreamNotRegistered = errors.New("stream not registered") +) + +type eventStream struct { + *fsevents.EventStream + isDir bool + watcher *fsEventsWatcher + basePath string + removed chan bool +} + +type fsEventsWatcher struct { + streams map[string]*eventStream + events chan fsnotify.Event + errors chan error + mu sync.Mutex + done chan bool +} + +func (w *fsEventsWatcher) Events() <-chan fsnotify.Event { + select { + case <-w.done: + return nil + default: + return w.events + } +} + +func (w *fsEventsWatcher) Errors() <-chan error { + return w.errors +} + +func (w *fsEventsWatcher) Add(path string) error { + select { + case <-w.done: + return errFSEventsWatcherClosed + default: + } + + abs, err := filepath.Abs(path) + if err != nil { + return err + } + + w.mu.Lock() + _, found := w.streams[abs] + w.mu.Unlock() + + if found { + return fmt.Errorf("already registered: %s", abs) + } + + if !w.hasParentEventStreamPath(abs) { + if err := w.add(abs); err != nil { + return err + } + } + + if childPaths := w.getChildEventStreamPaths(abs); len(childPaths) > 0 { + if err := w.removePaths(childPaths); err != nil { + return err + } + } + // https://github.com/fsnotify/fsevents/issues/48 + if len(w.streams) > 4096 { + return fmt.Errorf("too many fsevent streams: %d\n", len(w.streams)) + } + + return nil +} + +func (w *fsEventsWatcher) add(path string) error { + dev, err := fsevents.DeviceForPath(path) + if err != nil { + return err + } + fi, err := os.Stat(path) + if err != nil { + return err + } + // Symlinked-path like "/temp" cannot be watched + evaled, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + + isDir := fi.IsDir() + es := &fsevents.EventStream{ + Paths: []string{evaled}, + Latency: 10 * time.Millisecond, + Device: dev, + Flags: fsevents.FileEvents | fsevents.WatchRoot, + } + stream := &eventStream{ + es, + isDir, + w, + path, + make(chan bool), + } + w.mu.Lock() + w.streams[path] = stream + w.mu.Unlock() + go func(stream *eventStream) { + stream.Start() + stream.Flush(true) + for { + select { + case <-stream.watcher.done: + case <-stream.removed: + stream.Flush(true) + stream.Stop() + return + case evs := <-stream.Events: + for _, evt := range evs { + err := stream.sendEvent(evt) + if err != nil { + return + } + } + } + } + }(stream) + return nil +} + +func matchEventFlag(t, m fsevents.EventFlags) bool { + return t&m == m +} + +func (s *eventStream) convertEventPath(path string) (string, error) { + // Symlinks-evaled path + path = "/" + path + + evaledBasePath, err := filepath.EvalSymlinks(s.basePath) + if err != nil { + return "", err + } + + rel := path[len(evaledBasePath):] + + return filepath.Join(s.basePath, rel), nil +} + +func (s *eventStream) convertEvent(e fsevents.Event) (fsnotify.Event, error) { + name, err := s.convertEventPath(e.Path) + + if err != nil { + return fsnotify.Event{}, err + } + + ne := fsnotify.Event{ + Name: name, + Op: 0, + } + if matchEventFlag(e.Flags, fsevents.ItemCreated) { + ne.Op = fsnotify.Create + return ne, nil + } + if matchEventFlag(e.Flags, fsevents.ItemRemoved) { + ne.Op = fsnotify.Remove + return ne, nil + } + if matchEventFlag(e.Flags, fsevents.ItemRenamed) { + ne.Op = fsnotify.Rename + return ne, nil + } + if matchEventFlag(e.Flags, fsevents.ItemModified) { + ne.Op = fsnotify.Write + return ne, nil + } + + return ne, nil +} + +func (s *eventStream) sendEvent(e fsevents.Event) error { + w := s.watcher + ne, err := s.convertEvent(e) + if err != nil { + return err + } + if ne.Op == 0 { + return nil + } + w.events <- ne + return nil +} + +func (w *fsEventsWatcher) sendErr(e error) { + w.errors <- e +} + +func (w *fsEventsWatcher) hasParentEventStreamPath(path string) bool { + for p, s := range w.streams { + if s.isDir && strings.HasPrefix(filepath.Dir(path), p) { + return true + } + } + return false +} + +func (w *fsEventsWatcher) getChildEventStreamPaths(path string) (children []string) { + for p := range w.streams { + if strings.HasPrefix(filepath.Dir(p), path) { + children = append(children, p) + } + } + + return +} + +func (w *fsEventsWatcher) Remove(path string) error { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + return w.remove(abs) +} + +func (w *fsEventsWatcher) removePaths(paths []string) error { + for _, p := range paths { + if err := w.remove(p); err != nil { + return err + } + } + return nil +} + +func (w *fsEventsWatcher) remove(path string) error { + w.mu.Lock() + defer w.mu.Unlock() + + stream, exists := w.streams[path] + if !exists { + return errFSEventsWatcherStreamNotRegistered + } + close(stream.removed) + delete(w.streams, path) + return nil +} + +func (w *fsEventsWatcher) Close() error { + select { + case <-w.done: + return nil + default: + } + + close(w.done) + for path := range w.streams { + err := w.remove(path) + if err != nil { + return err + } + } + return nil +} + +// NewFSEventsWatcher returns a fsevents file watcher +func NewFSEventsWatcher() (FileWatcher, error) { + w := &fsEventsWatcher{ + streams: make(map[string]*eventStream), + done: make(chan bool), + events: make(chan fsnotify.Event), + errors: make(chan error), + } + return w, nil +} + +// NewEventWatcher returns an FSEvents based file watcher on darwin +func NewEventWatcher() (FileWatcher, error) { + return NewFSEventsWatcher() +} diff --git a/watcher/filenotify/eventwatcher_other.go b/watcher/filenotify/eventwatcher_other.go new file mode 100644 index 00000000000..255d1132b6a --- /dev/null +++ b/watcher/filenotify/eventwatcher_other.go @@ -0,0 +1,22 @@ +// Copyright 2022 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +//go:build !darwin || !cgo + +package filenotify + +// NewEventWatcher returns an fsnotify based file watcher +func NewEventWatcher() (FileWatcher, error) { + return NewFsNotifyWatcher() +} diff --git a/watcher/filenotify/filenotify.go b/watcher/filenotify/filenotify.go index b9d0d2e14a1..7cd5a5fd03a 100644 --- a/watcher/filenotify/filenotify.go +++ b/watcher/filenotify/filenotify.go @@ -38,12 +38,3 @@ func NewPollingWatcher(interval time.Duration) FileWatcher { errors: make(chan error), } } - -// NewEventWatcher returns an fs-event based file watcher -func NewEventWatcher() (FileWatcher, error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &fsNotifyWatcher{watcher}, nil -} diff --git a/watcher/filenotify/fsevents_test.go b/watcher/filenotify/fsevents_test.go new file mode 100644 index 00000000000..2388b1edd4c --- /dev/null +++ b/watcher/filenotify/fsevents_test.go @@ -0,0 +1,211 @@ +//go:build darwin && cgo + +package filenotify + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + + qt "github.com/frankban/quicktest" +) + +func TestFSEventsAddRemove(t *testing.T) { + c := qt.New(t) + w, err := NewFSEventsWatcher() + + c.Assert(err, qt.IsNil) + c.Assert(w.Add("foo"), qt.Not(qt.IsNil)) + c.Assert(w.Remove("foo"), qt.Not(qt.IsNil)) + + f, err := ioutil.TempFile("", "asdf") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(f.Name()) + c.Assert(w.Add(f.Name()), qt.IsNil) + c.Assert(w.Remove(f.Name()), qt.IsNil) +} + +func TestFSEventsEvent(t *testing.T) { + c := qt.New(t) + + method := "fsevents" + + c.Run(fmt.Sprintf("%s, Watch dir", method), func(c *qt.C) { + dir, w := prepareFSEventsTest(c, method) + subdir := filepath.Join(dir, subdir1) + c.Assert(w.Add(subdir), qt.IsNil) + + filename := filepath.Join(subdir, "file1") + + // Write to one file. + c.Assert(ioutil.WriteFile(filename, []byte("changed"), 0600), qt.IsNil) + + var expected []fsnotify.Event + + expected = append(expected, fsnotify.Event{Name: filename, Op: fsnotify.Write}) + assertEvents(c, w, expected...) + + // Remove one file. + filename = filepath.Join(subdir, "file2") + c.Assert(os.Remove(filename), qt.IsNil) + assertEvents(c, w, fsnotify.Event{Name: filename, Op: fsnotify.Remove}) + + // Add one file. + filename = filepath.Join(subdir, "file3") + c.Assert(ioutil.WriteFile(filename, []byte("new"), 0600), qt.IsNil) + assertEvents(c, w, fsnotify.Event{Name: filename, Op: fsnotify.Create}) + + // Remove entire directory. + subdir = filepath.Join(dir, subdir2) + // Fsevent watcher fails if watched root is deleted + // so parent dir is added here + c.Assert(w.Add(dir), qt.IsNil) + + c.Assert(os.RemoveAll(subdir), qt.IsNil) + + expected = expected[:0] + + expected = append( + expected, + fsnotify.Event{Name: filepath.Join(subdir, "file2"), Op: fsnotify.Remove}, + fsnotify.Event{Name: filepath.Join(subdir, "file0"), Op: fsnotify.Remove}, + fsnotify.Event{Name: filepath.Join(subdir, "file1"), Op: fsnotify.Remove}, + fsnotify.Event{Name: subdir, Op: fsnotify.Remove}, + ) + assertEvents(c, w, expected...) + + }) + + c.Run(fmt.Sprintf("%s, Add should not trigger event", "fsevents"), func(c *qt.C) { + dir, w := prepareFSEventsTest(c, method) + subdir := filepath.Join(dir, subdir1) + + w.Add(subdir) + assertEvents(c, w) + + // Create a new sub directory and add it to the watcher. + subdir = filepath.Join(dir, subdir1, subdir2) + c.Assert(os.Mkdir(subdir, 0777), qt.IsNil) + w.Add(subdir) + // This should create only one event. + assertEvents(c, w, fsnotify.Event{Name: subdir, Op: fsnotify.Create}) + }) + +} + +func TestFSEventsClose(t *testing.T) { + c := qt.New(t) + w, err := NewFSEventsWatcher() + c.Assert(err, qt.IsNil) + f1, err := ioutil.TempFile("", "f1") + c.Assert(err, qt.IsNil) + f2, err := ioutil.TempFile("", "f2") + c.Assert(err, qt.IsNil) + filename1 := f1.Name() + filename2 := f2.Name() + f1.Close() + f2.Close() + + c.Assert(w.Add(filename1), qt.IsNil) + c.Assert(w.Add(filename2), qt.IsNil) + c.Assert(w.Close(), qt.IsNil) + c.Assert(w.Close(), qt.IsNil) + c.Assert(ioutil.WriteFile(filename1, []byte("new"), 0600), qt.IsNil) + c.Assert(ioutil.WriteFile(filename2, []byte("new"), 0600), qt.IsNil) + // No more event as the watchers are closed. + assertEvents(c, w) + + f2, err = ioutil.TempFile("", "f2") + c.Assert(err, qt.IsNil) + + defer os.Remove(f2.Name()) + + c.Assert(w.Add(f2.Name()), qt.Not(qt.IsNil)) + +} + +func prepareFSEventsTest(c *qt.C, id string) (string, FileWatcher) { + w, err := NewFSEventsWatcher() + c.Assert(err, qt.IsNil) + + dir := prepareTestDirWithSomeFiles(c, id) + + c.Cleanup(func() { + w.Close() + }) + err = waitForInit(dir, w) + c.Assert(err, qt.IsNil) + drainEvents(c, w) + return dir, w +} + +// FSEvents-wathcer notification can be unstable on starting. +// It contains past events or misses new events unexpectedly. +// Though, this may be no problem for actual uses (local server), it makes tests flaky. +// Thus, we have to wait until detecting a fresh write event. +func waitForInit(dir string, w FileWatcher) (err error) { + err = w.Add(dir) + if err != nil { + return err + } + defer func() { + e := w.Remove(dir) + if err == nil && e != nil { + err = e + } + }() + + f, err := ioutil.TempFile(dir, "testfile") + testfile := f.Name() + if err != nil { + return err + } + err = f.Close() + if err != nil { + return err + } + defer func() { + e := os.Remove(testfile) + if err == nil && e != nil { + err = e + } + }() + + wait := func() error { + createdOK := false + timeout := time.After(watchWaitTime * 2) + for i := 0; true; i++ { + select { + case evt := <-w.Events(): + if evt.Name != testfile { + continue + } + // first check: CREATE event of testfile + if evt.Op == fsnotify.Create { + createdOK = true + err := ioutil.WriteFile(testfile, []byte(fmt.Sprint(i)), 0600) + if err != nil { + return err + } + } + // second check: WRITE event of testfile + if createdOK && evt.Op == fsnotify.Write { + return nil + } + case e := <-w.Errors(): + return fmt.Errorf("got unexpected error waiting for FSEvents init %v", e) + case <-timeout: + return fmt.Errorf("timeout during waiting for FSEvents init") + } + } + return nil + } + return wait() +} diff --git a/watcher/filenotify/fsnotify.go b/watcher/filenotify/fsnotify.go index 19534128a31..4cca95ac53d 100644 --- a/watcher/filenotify/fsnotify.go +++ b/watcher/filenotify/fsnotify.go @@ -18,3 +18,11 @@ func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { func (w *fsNotifyWatcher) Errors() <-chan error { return w.Watcher.Errors } + +func NewFsNotifyWatcher() (FileWatcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &fsNotifyWatcher{watcher}, nil +} diff --git a/watcher/filenotify/poller_test.go b/watcher/filenotify/poller_test.go index b4723c7585d..8c37e13376b 100644 --- a/watcher/filenotify/poller_test.go +++ b/watcher/filenotify/poller_test.go @@ -221,7 +221,7 @@ func BenchmarkPoller(b *testing.B) { } func prepareTestDirWithSomeFiles(c *qt.C, id string) string { - dir, err := ioutil.TempDir("", fmt.Sprintf("test-poller-dir-%s", id)) + dir, err := ioutil.TempDir("", fmt.Sprintf("test-filenotify-dir-%s", id)) c.Assert(err, qt.IsNil) c.Assert(os.MkdirAll(filepath.Join(dir, subdir1), 0777), qt.IsNil) c.Assert(os.MkdirAll(filepath.Join(dir, subdir2), 0777), qt.IsNil) @@ -247,7 +247,7 @@ func preparePollTest(c *qt.C, poll bool) (string, FileWatcher) { w = NewPollingWatcher(watchWaitTime) } else { var err error - w, err = NewEventWatcher() + w, err = NewFsNotifyWatcher() c.Assert(err, qt.IsNil) }