From e4b1cfc241b285da2fad6d4ae9b4d98441c0b155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3nio=20Meireles?= Date: Sun, 6 Dec 2015 14:26:25 +0000 Subject: [PATCH] add an option to upload file to a VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit per @rimusz request Signed-off-by: António Meireles --- Godeps/Godeps.json | 22 +- .../_workspace/src/github.com/kr/fs/LICENSE | 27 + Godeps/_workspace/src/github.com/kr/fs/Readme | 3 + .../src/github.com/kr/fs/example_test.go | 19 + .../src/github.com/kr/fs/filesystem.go | 36 + .../_workspace/src/github.com/kr/fs/walk.go | 95 + .../src/github.com/kr/fs/walk_test.go | 209 +++ .../src/github.com/pkg/sftp/.gitignore | 8 + .../src/github.com/pkg/sftp/CONTRIBUTORS | 2 + .../src/github.com/pkg/sftp/LICENSE | 9 + .../src/github.com/pkg/sftp/README.md | 27 + .../src/github.com/pkg/sftp/attrs.go | 236 +++ .../src/github.com/pkg/sftp/attrs_stubs.go | 11 + .../src/github.com/pkg/sftp/attrs_test.go | 45 + .../src/github.com/pkg/sftp/attrs_unix.go | 17 + .../src/github.com/pkg/sftp/client.go | 1159 +++++++++++++ .../sftp/client_integration_darwin_test.go | 41 + .../pkg/sftp/client_integration_linux_test.go | 41 + .../pkg/sftp/client_integration_test.go | 1540 +++++++++++++++++ .../src/github.com/pkg/sftp/client_test.go | 86 + .../src/github.com/pkg/sftp/debug.go | 9 + .../src/github.com/pkg/sftp/example_test.go | 90 + .../examples/buffered-read-benchmark/main.go | 77 + .../examples/buffered-write-benchmark/main.go | 83 + .../pkg/sftp/examples/sftp-server/README.md | 12 + .../pkg/sftp/examples/sftp-server/main.go | 135 ++ .../examples/streaming-read-benchmark/main.go | 84 + .../streaming-write-benchmark/main.go | 84 + .../src/github.com/pkg/sftp/packet.go | 829 +++++++++ .../src/github.com/pkg/sftp/packet_test.go | 261 +++ .../src/github.com/pkg/sftp/release.go | 5 + .../src/github.com/pkg/sftp/server.go | 565 ++++++ .../pkg/sftp/server_integration_test.go | 673 +++++++ .../pkg/sftp/server_standalone/main.go | 39 + .../src/github.com/pkg/sftp/server_stubs.go | 12 + .../src/github.com/pkg/sftp/server_unix.go | 143 ++ .../src/github.com/pkg/sftp/sftp.go | 187 ++ .../src/github.com/pkg/sftp/wercker.yml | 34 + documentation/man/corectl-put.1 | 43 + documentation/man/corectl.1 | 2 +- documentation/markdown/corectl.md | 1 + documentation/markdown/corectl_put.md | 29 + pull.go | 2 +- run.go | 2 + ssh.go | 86 +- 45 files changed, 7114 insertions(+), 6 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/kr/fs/LICENSE create mode 100644 Godeps/_workspace/src/github.com/kr/fs/Readme create mode 100644 Godeps/_workspace/src/github.com/kr/fs/example_test.go create mode 100644 Godeps/_workspace/src/github.com/kr/fs/filesystem.go create mode 100644 Godeps/_workspace/src/github.com/kr/fs/walk.go create mode 100644 Godeps/_workspace/src/github.com/kr/fs/walk_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/.gitignore create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/CONTRIBUTORS create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/LICENSE create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/README.md create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_integration_darwin_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_integration_linux_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/debug.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/example_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-read-benchmark/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-write-benchmark/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-read-benchmark/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-write-benchmark/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/packet.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/release.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_integration_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/sftp.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml create mode 100644 documentation/man/corectl-put.1 create mode 100644 documentation/markdown/corectl_put.md diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index f9326c9..061c998 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { - "ImportPath": "github.com/TheNewNormal/corectl", - "GoVersion": "go1.5.1", + "ImportPath": "github.com/AntonioMeireles/corectl", + "GoVersion": "go1.5.2", "Packages": [ "./..." ], @@ -10,6 +10,16 @@ "Comment": "v0.1.0-21-g056c9bc", "Rev": "056c9bc7be7190eaa7715723883caffa5f8fa3e4" }, + { + "ImportPath": "github.com/TheNewNormal/corectl/image", + "Comment": "v0.0.10", + "Rev": "a117e5d7c645569cc52f58aad9fff7cf1cda89c6" + }, + { + "ImportPath": "github.com/TheNewNormal/corectl/uuid2ip", + "Comment": "v0.0.10", + "Rev": "a117e5d7c645569cc52f58aad9fff7cf1cda89c6" + }, { "ImportPath": "github.com/blang/semver", "Comment": "v3.0.1", @@ -33,6 +43,10 @@ "ImportPath": "github.com/inconshreveable/mousetrap", "Rev": "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" }, + { + "ImportPath": "github.com/kr/fs", + "Rev": "2788f0dbd16903de03cb8186e5c7d97b69ad387b" + }, { "ImportPath": "github.com/kr/pretty", "Comment": "go.weekly.2011-12-22-27-ge6ac2fc", @@ -55,6 +69,10 @@ "ImportPath": "github.com/mitchellh/mapstructure", "Rev": "281073eb9eb092240d33ef253c404f1cca550309" }, + { + "ImportPath": "github.com/pkg/sftp", + "Rev": "cbc2879daf7b8cff8faff8c509e4c727ba7291d5" + }, { "ImportPath": "github.com/rakyll/pb", "Rev": "dc507ad06b7462501281bb4691ee43f0b1d1ec37" diff --git a/Godeps/_workspace/src/github.com/kr/fs/LICENSE b/Godeps/_workspace/src/github.com/kr/fs/LICENSE new file mode 100644 index 0000000..7448756 --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/kr/fs/Readme b/Godeps/_workspace/src/github.com/kr/fs/Readme new file mode 100644 index 0000000..c95e13f --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/Readme @@ -0,0 +1,3 @@ +Filesystem Package + +http://godoc.org/github.com/kr/fs diff --git a/Godeps/_workspace/src/github.com/kr/fs/example_test.go b/Godeps/_workspace/src/github.com/kr/fs/example_test.go new file mode 100644 index 0000000..77e0db9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/example_test.go @@ -0,0 +1,19 @@ +package fs_test + +import ( + "fmt" + "os" + + "github.com/kr/fs" +) + +func ExampleWalker() { + walker := fs.Walk("/usr/lib") + for walker.Step() { + if err := walker.Err(); err != nil { + fmt.Fprintln(os.Stderr, err) + continue + } + fmt.Println(walker.Path()) + } +} diff --git a/Godeps/_workspace/src/github.com/kr/fs/filesystem.go b/Godeps/_workspace/src/github.com/kr/fs/filesystem.go new file mode 100644 index 0000000..f1c4805 --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/filesystem.go @@ -0,0 +1,36 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// FileSystem defines the methods of an abstract filesystem. +type FileSystem interface { + + // ReadDir reads the directory named by dirname and returns a + // list of directory entries. + ReadDir(dirname string) ([]os.FileInfo, error) + + // Lstat returns a FileInfo describing the named file. If the file is a + // symbolic link, the returned FileInfo describes the symbolic link. Lstat + // makes no attempt to follow the link. + Lstat(name string) (os.FileInfo, error) + + // Join joins any number of path elements into a single path, adding a + // separator if necessary. The result is Cleaned; in particular, all + // empty strings are ignored. + // + // The separator is FileSystem specific. + Join(elem ...string) string +} + +// fs represents a FileSystem provided by the os package. +type fs struct{} + +func (f *fs) ReadDir(dirname string) ([]os.FileInfo, error) { return ioutil.ReadDir(dirname) } + +func (f *fs) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) } + +func (f *fs) Join(elem ...string) string { return filepath.Join(elem...) } diff --git a/Godeps/_workspace/src/github.com/kr/fs/walk.go b/Godeps/_workspace/src/github.com/kr/fs/walk.go new file mode 100644 index 0000000..6ffa1e0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/walk.go @@ -0,0 +1,95 @@ +// Package fs provides filesystem-related functions. +package fs + +import ( + "os" +) + +// Walker provides a convenient interface for iterating over the +// descendants of a filesystem path. +// Successive calls to the Step method will step through each +// file or directory in the tree, including the root. The files +// are walked in lexical order, which makes the output deterministic +// but means that for very large directories Walker can be inefficient. +// Walker does not follow symbolic links. +type Walker struct { + fs FileSystem + cur item + stack []item + descend bool +} + +type item struct { + path string + info os.FileInfo + err error +} + +// Walk returns a new Walker rooted at root. +func Walk(root string) *Walker { + return WalkFS(root, new(fs)) +} + +// WalkFS returns a new Walker rooted at root on the FileSystem fs. +func WalkFS(root string, fs FileSystem) *Walker { + info, err := fs.Lstat(root) + return &Walker{ + fs: fs, + stack: []item{{root, info, err}}, + } +} + +// Step advances the Walker to the next file or directory, +// which will then be available through the Path, Stat, +// and Err methods. +// It returns false when the walk stops at the end of the tree. +func (w *Walker) Step() bool { + if w.descend && w.cur.err == nil && w.cur.info.IsDir() { + list, err := w.fs.ReadDir(w.cur.path) + if err != nil { + w.cur.err = err + w.stack = append(w.stack, w.cur) + } else { + for i := len(list) - 1; i >= 0; i-- { + path := w.fs.Join(w.cur.path, list[i].Name()) + w.stack = append(w.stack, item{path, list[i], nil}) + } + } + } + + if len(w.stack) == 0 { + return false + } + i := len(w.stack) - 1 + w.cur = w.stack[i] + w.stack = w.stack[:i] + w.descend = true + return true +} + +// Path returns the path to the most recent file or directory +// visited by a call to Step. It contains the argument to Walk +// as a prefix; that is, if Walk is called with "dir", which is +// a directory containing the file "a", Path will return "dir/a". +func (w *Walker) Path() string { + return w.cur.path +} + +// Stat returns info for the most recent file or directory +// visited by a call to Step. +func (w *Walker) Stat() os.FileInfo { + return w.cur.info +} + +// Err returns the error, if any, for the most recent attempt +// by Step to visit a file or directory. If a directory has +// an error, w will not descend into that directory. +func (w *Walker) Err() error { + return w.cur.err +} + +// SkipDir causes the currently visited directory to be skipped. +// If w is not on a directory, SkipDir has no effect. +func (w *Walker) SkipDir() { + w.descend = false +} diff --git a/Godeps/_workspace/src/github.com/kr/fs/walk_test.go b/Godeps/_workspace/src/github.com/kr/fs/walk_test.go new file mode 100644 index 0000000..6f5ad2a --- /dev/null +++ b/Godeps/_workspace/src/github.com/kr/fs/walk_test.go @@ -0,0 +1,209 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/kr/fs" +) + +type PathTest struct { + path, result string +} + +type Node struct { + name string + entries []*Node // nil if the entry is a file + mark int +} + +var tree = &Node{ + "testdata", + []*Node{ + {"a", nil, 0}, + {"b", []*Node{}, 0}, + {"c", nil, 0}, + { + "d", + []*Node{ + {"x", nil, 0}, + {"y", []*Node{}, 0}, + { + "z", + []*Node{ + {"u", nil, 0}, + {"v", nil, 0}, + }, + 0, + }, + }, + 0, + }, + }, + 0, +} + +func walkTree(n *Node, path string, f func(path string, n *Node)) { + f(path, n) + for _, e := range n.entries { + walkTree(e, filepath.Join(path, e.name), f) + } +} + +func makeTree(t *testing.T) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.entries == nil { + fd, err := os.Create(path) + if err != nil { + t.Errorf("makeTree: %v", err) + return + } + fd.Close() + } else { + os.Mkdir(path, 0770) + } + }) +} + +func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } + +func checkMarks(t *testing.T, report bool) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.mark != 1 && report { + t.Errorf("node %s mark = %d; expected 1", path, n.mark) + } + n.mark = 0 + }) +} + +// Assumes that each node name is unique. Good enough for a test. +// If clear is true, any incoming error is cleared before return. The errors +// are always accumulated, though. +func mark(path string, info os.FileInfo, err error, errors *[]error, clear bool) error { + if err != nil { + *errors = append(*errors, err) + if clear { + return nil + } + return err + } + name := info.Name() + walkTree(tree, tree.name, func(path string, n *Node) { + if n.name == name { + n.mark++ + } + }) + return nil +} + +func TestWalk(t *testing.T) { + makeTree(t) + errors := make([]error, 0, 10) + clear := true + markFn := func(walker *fs.Walker) (err error) { + for walker.Step() { + err = mark(walker.Path(), walker.Stat(), walker.Err(), &errors, clear) + if err != nil { + break + } + } + return err + } + // Expect no errors. + err := markFn(fs.Walk(tree.name)) + if err != nil { + t.Fatalf("no error expected, found: %s", err) + } + if len(errors) != 0 { + t.Fatalf("unexpected errors: %s", errors) + } + checkMarks(t, true) + errors = errors[0:0] + + // Test permission errors. Only possible if we're not root + // and only on some file systems (AFS, FAT). To avoid errors during + // all.bash on those file systems, skip during go test -short. + if os.Getuid() > 0 && !testing.Short() { + // introduce 2 errors: chmod top-level directories to 0 + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) + + // 3) capture errors, expect two. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + err := markFn(fs.Walk(tree.name)) + if err != nil { + t.Fatalf("expected no error return from Walk, got %s", err) + } + if len(errors) != 2 { + t.Errorf("expected 2 errors, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, true) + errors = errors[0:0] + + // 4) capture errors, stop after first error. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + clear = false // error will stop processing + err = markFn(fs.Walk(tree.name)) + if err == nil { + t.Fatalf("expected error return from Walk") + } + if len(errors) != 1 { + t.Errorf("expected 1 error, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, false) + errors = errors[0:0] + + // restore permissions + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) + } + + // cleanup + if err := os.RemoveAll(tree.name); err != nil { + t.Errorf("removeTree: %v", err) + } +} + +func TestBug3486(t *testing.T) { // http://code.google.com/p/go/issues/detail?id=3486 + root, err := filepath.EvalSymlinks(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + lib := filepath.Join(root, "lib") + src := filepath.Join(root, "src") + seenSrc := false + walker := fs.Walk(root) + for walker.Step() { + if walker.Err() != nil { + t.Fatal(walker.Err()) + } + + switch walker.Path() { + case lib: + walker.SkipDir() + case src: + seenSrc = true + } + } + if !seenSrc { + t.Fatalf("%q not seen", src) + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore b/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore new file mode 100644 index 0000000..9fc1e3d --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore @@ -0,0 +1,8 @@ +.*.swo +.*.swp + +server_standalone/server_standalone + +examples/sftp-server/id_rsa +examples/sftp-server/id_rsa.pub +examples/sftp-server/sftp-server diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/CONTRIBUTORS b/Godeps/_workspace/src/github.com/pkg/sftp/CONTRIBUTORS new file mode 100644 index 0000000..7eff823 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/CONTRIBUTORS @@ -0,0 +1,2 @@ +Dave Cheney +Saulius Gurklys diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/LICENSE b/Godeps/_workspace/src/github.com/pkg/sftp/LICENSE new file mode 100644 index 0000000..b7b5392 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) 2013, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/README.md b/Godeps/_workspace/src/github.com/pkg/sftp/README.md new file mode 100644 index 0000000..23301b3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/README.md @@ -0,0 +1,27 @@ +sftp +---- + +The `sftp` package provides support for file system operations on remote ssh servers using the SFTP subsystem. + +[![wercker status](https://app.wercker.com/status/7d3e9b916954ac3a7ed15a457938bac4/s/master "wercker status")](https://app.wercker.com/project/bykey/7d3e9b916954ac3a7ed15a457938bac4) + +usage and examples +------------------ + +See [godoc.org/github.com/pkg/sftp](http://godoc.org/github.com/pkg/sftp) for examples and usage. + +The basic operation of the package mirrors the facilities of the [os](http://golang.org/pkg/os) package. + +The Walker interface for directory traversal is heavily inspired by Keith Rarick's [fs](http://godoc.org/github.com/kr/fs) package. + +roadmap +------- + + * Currently all traffic with the server is serialized, this can be improved by allowing overlapping requests/responses. + * There is way too much duplication in the Client methods. If there was an unmarshal(interface{}) method this would reduce a heap of the duplication. + * Implement integration tests by talking directly to a real opensftp-server process. This shouldn't be too difficult to implement with a small refactoring to the sftp.NewClient method. These tests should be gated on an -sftp.integration test flag. _in progress_ + +contributing +------------ + +Features, Issues, and Pull Requests are always welcome. diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go new file mode 100644 index 0000000..7b359b5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go @@ -0,0 +1,236 @@ +package sftp + +// ssh_FXP_ATTRS support +// see http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-5 + +import ( + "os" + "syscall" + "time" +) + +const ( + ssh_FILEXFER_ATTR_SIZE = 0x00000001 + ssh_FILEXFER_ATTR_UIDGID = 0x00000002 + ssh_FILEXFER_ATTR_PERMISSIONS = 0x00000004 + ssh_FILEXFER_ATTR_ACMODTIME = 0x00000008 + ssh_FILEXFER_ATTR_EXTENDED = 0x80000000 +) + +// fileInfo is an artificial type designed to satisfy os.FileInfo. +type fileInfo struct { + name string + size int64 + mode os.FileMode + mtime time.Time + sys interface{} +} + +// Name returns the base name of the file. +func (fi *fileInfo) Name() string { return fi.name } + +// Size returns the length in bytes for regular files; system-dependent for others. +func (fi *fileInfo) Size() int64 { return fi.size } + +// Mode returns file mode bits. +func (fi *fileInfo) Mode() os.FileMode { return fi.mode } + +// ModTime returns the last modification time of the file. +func (fi *fileInfo) ModTime() time.Time { return fi.mtime } + +// IsDir returns true if the file is a directory. +func (fi *fileInfo) IsDir() bool { return fi.Mode().IsDir() } + +func (fi *fileInfo) Sys() interface{} { return fi.sys } + +// FileStat holds the original unmarshalled values from a call to READDIR or *STAT. +// It is exported for the purposes of accessing the raw values via os.FileInfo.Sys() +type FileStat struct { + Size uint64 + Mode uint32 + Mtime uint32 + Atime uint32 + Uid uint32 + Gid uint32 + Extended []StatExtended +} + +type StatExtended struct { + ExtType string + ExtData string +} + +func fileInfoFromStat(st *FileStat, name string) os.FileInfo { + fs := &fileInfo{ + name: name, + size: int64(st.Size), + mode: toFileMode(st.Mode), + mtime: time.Unix(int64(st.Mtime), 0), + sys: st, + } + return fs +} + +func fileStatFromInfo(fi os.FileInfo) (uint32, FileStat) { + mtime := fi.ModTime().Unix() + atime := mtime + var flags uint32 = ssh_FILEXFER_ATTR_SIZE | + ssh_FILEXFER_ATTR_PERMISSIONS | + ssh_FILEXFER_ATTR_ACMODTIME + + fileStat := FileStat{ + Size: uint64(fi.Size()), + Mode: fromFileMode(fi.Mode()), + Mtime: uint32(mtime), + Atime: uint32(atime), + } + + // os specific file stat decoding + fileStatFromInfoOs(fi, &flags, &fileStat) + + return flags, fileStat +} + +func unmarshalAttrs(b []byte) (*FileStat, []byte) { + flags, b := unmarshalUint32(b) + var fs FileStat + if flags&ssh_FILEXFER_ATTR_SIZE == ssh_FILEXFER_ATTR_SIZE { + fs.Size, b = unmarshalUint64(b) + } + if flags&ssh_FILEXFER_ATTR_UIDGID == ssh_FILEXFER_ATTR_UIDGID { + fs.Uid, b = unmarshalUint32(b) + } + if flags&ssh_FILEXFER_ATTR_UIDGID == ssh_FILEXFER_ATTR_UIDGID { + fs.Gid, b = unmarshalUint32(b) + } + if flags&ssh_FILEXFER_ATTR_PERMISSIONS == ssh_FILEXFER_ATTR_PERMISSIONS { + fs.Mode, b = unmarshalUint32(b) + } + if flags&ssh_FILEXFER_ATTR_ACMODTIME == ssh_FILEXFER_ATTR_ACMODTIME { + fs.Atime, b = unmarshalUint32(b) + fs.Mtime, b = unmarshalUint32(b) + } + if flags&ssh_FILEXFER_ATTR_EXTENDED == ssh_FILEXFER_ATTR_EXTENDED { + var count uint32 + count, b = unmarshalUint32(b) + ext := make([]StatExtended, count, count) + for i := uint32(0); i < count; i++ { + var typ string + var data string + typ, b = unmarshalString(b) + data, b = unmarshalString(b) + ext[i] = StatExtended{typ, data} + } + fs.Extended = ext + } + return &fs, b +} + +func marshalFileInfo(b []byte, fi os.FileInfo) []byte { + // attributes variable struct, and also variable per protocol version + // spec version 3 attributes: + // uint32 flags + // uint64 size present only if flag SSH_FILEXFER_ATTR_SIZE + // uint32 uid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 gid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 permissions present only if flag SSH_FILEXFER_ATTR_PERMISSIONS + // uint32 atime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 mtime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 extended_count present only if flag SSH_FILEXFER_ATTR_EXTENDED + // string extended_type + // string extended_data + // ... more extended data (extended_type - extended_data pairs), + // so that number of pairs equals extended_count + + flags, fileStat := fileStatFromInfo(fi) + + b = marshalUint32(b, flags) + if flags&ssh_FILEXFER_ATTR_SIZE != 0 { + b = marshalUint64(b, fileStat.Size) + } + if flags&ssh_FILEXFER_ATTR_UIDGID != 0 { + b = marshalUint32(b, fileStat.Uid) + b = marshalUint32(b, fileStat.Gid) + } + if flags&ssh_FILEXFER_ATTR_PERMISSIONS != 0 { + b = marshalUint32(b, fileStat.Mode) + } + if flags&ssh_FILEXFER_ATTR_ACMODTIME != 0 { + b = marshalUint32(b, fileStat.Atime) + b = marshalUint32(b, fileStat.Mtime) + } + + return b +} + +// toFileMode converts sftp filemode bits to the os.FileMode specification +func toFileMode(mode uint32) os.FileMode { + var fm = os.FileMode(mode & 0777) + switch mode & syscall.S_IFMT { + case syscall.S_IFBLK: + fm |= os.ModeDevice + case syscall.S_IFCHR: + fm |= os.ModeDevice | os.ModeCharDevice + case syscall.S_IFDIR: + fm |= os.ModeDir + case syscall.S_IFIFO: + fm |= os.ModeNamedPipe + case syscall.S_IFLNK: + fm |= os.ModeSymlink + case syscall.S_IFREG: + // nothing to do + case syscall.S_IFSOCK: + fm |= os.ModeSocket + } + if mode&syscall.S_ISGID != 0 { + fm |= os.ModeSetgid + } + if mode&syscall.S_ISUID != 0 { + fm |= os.ModeSetuid + } + if mode&syscall.S_ISVTX != 0 { + fm |= os.ModeSticky + } + return fm +} + +// fromFileMode converts from the os.FileMode specification to sftp filemode bits +func fromFileMode(mode os.FileMode) uint32 { + ret := uint32(0) + + if mode&os.ModeDevice != 0 { + if mode&os.ModeCharDevice != 0 { + ret |= syscall.S_IFCHR + } else { + ret |= syscall.S_IFBLK + } + } + if mode&os.ModeDir != 0 { + ret |= syscall.S_IFDIR + } + if mode&os.ModeSymlink != 0 { + ret |= syscall.S_IFLNK + } + if mode&os.ModeNamedPipe != 0 { + ret |= syscall.S_IFIFO + } + if mode&os.ModeSetgid != 0 { + ret |= syscall.S_ISGID + } + if mode&os.ModeSetuid != 0 { + ret |= syscall.S_ISUID + } + if mode&os.ModeSticky != 0 { + ret |= syscall.S_ISVTX + } + if mode&os.ModeSocket != 0 { + ret |= syscall.S_IFSOCK + } + + if mode&os.ModeType == 0 { + ret |= syscall.S_IFREG + } + ret |= uint32(mode & os.ModePerm) + + return ret +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go new file mode 100644 index 0000000..81cf3ea --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go @@ -0,0 +1,11 @@ +// +build !cgo,!plan9 windows android + +package sftp + +import ( + "os" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + // todo +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go new file mode 100644 index 0000000..d552905 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go @@ -0,0 +1,45 @@ +package sftp + +import ( + "bytes" + "os" + "reflect" + "testing" + "time" +) + +// ensure that attrs implemenst os.FileInfo +var _ os.FileInfo = new(fileInfo) + +var unmarshalAttrsTests = []struct { + b []byte + want *fileInfo + rest []byte +}{ + {marshal(nil, struct{ Flags uint32 }{}), &fileInfo{mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + }{ssh_FILEXFER_ATTR_SIZE, 20}), &fileInfo{size: 20, mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + Permissions uint32 + }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + Uid, Gid, Permissions uint32 + }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 1000, 1000, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, +} + +func TestUnmarshalAttrs(t *testing.T) { + for _, tt := range unmarshalAttrsTests { + stat, rest := unmarshalAttrs(tt.b) + got := fileInfoFromStat(stat, "") + tt.want.sys = got.Sys() + if !reflect.DeepEqual(got, tt.want) || !bytes.Equal(tt.rest, rest) { + t.Errorf("unmarshalAttrs(%#v): want %#v, %#v, got: %#v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go new file mode 100644 index 0000000..4165c2a --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go @@ -0,0 +1,17 @@ +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris +// +build cgo + +package sftp + +import ( + "os" + "syscall" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + if statt, ok := fi.Sys().(*syscall.Stat_t); ok { + *flags |= ssh_FILEXFER_ATTR_UIDGID + fileStat.Uid = statt.Uid + fileStat.Gid = statt.Gid + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client.go b/Godeps/_workspace/src/github.com/pkg/sftp/client.go new file mode 100644 index 0000000..b336dba --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client.go @@ -0,0 +1,1159 @@ +package sftp + +import ( + "bytes" + "encoding" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "path" + "sync" + "sync/atomic" + "time" + + "github.com/kr/fs" + + "golang.org/x/crypto/ssh" +) + +// MaxPacket sets the maximum size of the payload. +func MaxPacket(size int) func(*Client) error { + return func(c *Client) error { + if size < 1<<15 { + return fmt.Errorf("size must be greater or equal to 32k") + } + c.maxPacket = size + return nil + } +} + +// New creates a new SFTP client on conn. +func NewClient(conn *ssh.Client, opts ...func(*Client) error) (*Client, error) { + s, err := conn.NewSession() + if err != nil { + return nil, err + } + if err := s.RequestSubsystem("sftp"); err != nil { + return nil, err + } + pw, err := s.StdinPipe() + if err != nil { + return nil, err + } + pr, err := s.StdoutPipe() + if err != nil { + return nil, err + } + + return NewClientPipe(pr, pw, opts...) +} + +// NewClientPipe creates a new SFTP client given a Reader and a WriteCloser. +// This can be used for connecting to an SFTP server over TCP/TLS or by using +// the system's ssh client program (e.g. via exec.Command). +func NewClientPipe(rd io.Reader, wr io.WriteCloser, opts ...func(*Client) error) (*Client, error) { + sftp := &Client{ + w: wr, + r: rd, + maxPacket: 1 << 15, + inflight: make(map[uint32]chan<- result), + recvClosed: make(chan struct{}), + } + if err := sftp.applyOptions(opts...); err != nil { + wr.Close() + return nil, err + } + if err := sftp.sendInit(); err != nil { + wr.Close() + return nil, err + } + if err := sftp.recvVersion(); err != nil { + wr.Close() + return nil, err + } + go sftp.recv() + return sftp, nil +} + +// Client represents an SFTP session on a *ssh.ClientConn SSH connection. +// Multiple Clients can be active on a single SSH connection, and a Client +// may be called concurrently from multiple Goroutines. +// +// Client implements the github.com/kr/fs.FileSystem interface. +type Client struct { + w io.WriteCloser + r io.Reader + + maxPacket int // max packet size read or written. + nextid uint32 + + mu sync.Mutex // ensures only on request is in flight to the server at once + inflight map[uint32]chan<- result // outstanding requests + recvClosed chan struct{} // remote end has closed the connection +} + +// Close closes the SFTP session. +func (c *Client) Close() error { + err := c.w.Close() + <-c.recvClosed + return err +} + +// Create creates the named file mode 0666 (before umask), truncating it if +// it already exists. If successful, methods on the returned File can be +// used for I/O; the associated file descriptor has mode O_RDWR. +func (c *Client) Create(path string) (*File, error) { + return c.open(path, flags(os.O_RDWR|os.O_CREATE|os.O_TRUNC)) +} + +const sftpProtocolVersion = 3 // http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 + +func (c *Client) sendInit() error { + return sendPacket(c.w, sshFxInitPacket{ + Version: sftpProtocolVersion, // http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 + }) +} + +// returns the next value of c.nextid +func (c *Client) nextId() uint32 { + return atomic.AddUint32(&c.nextid, 1) +} + +func (c *Client) recvVersion() error { + typ, data, err := recvPacket(c.r) + if err != nil { + return err + } + if typ != ssh_FXP_VERSION { + return &unexpectedPacketErr{ssh_FXP_VERSION, typ} + } + + version, _ := unmarshalUint32(data) + if version != sftpProtocolVersion { + return &unexpectedVersionErr{sftpProtocolVersion, version} + } + + return nil +} + +// broadcastErr sends an error to all goroutines waiting for a response. +func (c *Client) broadcastErr(err error) { + c.mu.Lock() + listeners := make([]chan<- result, 0, len(c.inflight)) + for _, ch := range c.inflight { + listeners = append(listeners, ch) + } + c.mu.Unlock() + for _, ch := range listeners { + ch <- result{err: err} + } +} + +// recv continuously reads from the server and forwards responses to the +// appropriate channel. +func (c *Client) recv() { + defer close(c.recvClosed) + for { + typ, data, err := recvPacket(c.r) + if err != nil { + // Return the error to all listeners. + c.broadcastErr(err) + return + } + sid, _ := unmarshalUint32(data) + c.mu.Lock() + ch, ok := c.inflight[sid] + delete(c.inflight, sid) + c.mu.Unlock() + if !ok { + // This is an unexpected occurrence. Send the error + // back to all listeners so that they terminate + // gracefully. + c.broadcastErr(fmt.Errorf("sid: %v not fond", sid)) + return + } + ch <- result{typ: typ, data: data} + } +} + +// Walk returns a new Walker rooted at root. +func (c *Client) Walk(root string) *fs.Walker { + return fs.WalkFS(root, c) +} + +// ReadDir reads the directory named by dirname and returns a list of +// directory entries. +func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { + handle, err := c.opendir(p) + if err != nil { + return nil, err + } + defer c.close(handle) // this has to defer earlier than the lock below + var attrs []os.FileInfo + var done = false + for !done { + id := c.nextId() + typ, data, err1 := c.sendRequest(sshFxpReaddirPacket{ + Id: id, + Handle: handle, + }) + if err1 != nil { + err = err1 + done = true + break + } + switch typ { + case ssh_FXP_NAME: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIdErr{id, sid} + } + count, data := unmarshalUint32(data) + for i := uint32(0); i < count; i++ { + var filename string + filename, data = unmarshalString(data) + _, data = unmarshalString(data) // discard longname + var attr *FileStat + attr, data = unmarshalAttrs(data) + if filename == "." || filename == ".." { + continue + } + attrs = append(attrs, fileInfoFromStat(attr, path.Base(filename))) + } + case ssh_FXP_STATUS: + // TODO(dfc) scope warning! + err = eofOrErr(unmarshalStatus(id, data)) + done = true + default: + return nil, unimplementedPacketErr(typ) + } + } + if err == io.EOF { + err = nil + } + return attrs, err +} +func (c *Client) opendir(path string) (string, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpOpendirPacket{ + Id: id, + Path: path, + }) + if err != nil { + return "", err + } + switch typ { + case ssh_FXP_HANDLE: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIdErr{id, sid} + } + handle, _ := unmarshalString(data) + return handle, nil + case ssh_FXP_STATUS: + return "", unmarshalStatus(id, data) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Stat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the referent file. +func (c *Client) Stat(p string) (os.FileInfo, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpStatPacket{ + Id: id, + Path: p, + }) + if err != nil { + return nil, err + } + switch typ { + case ssh_FXP_ATTRS: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIdErr{id, sid} + } + attr, _ := unmarshalAttrs(data) + return fileInfoFromStat(attr, path.Base(p)), nil + case ssh_FXP_STATUS: + return nil, unmarshalStatus(id, data) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// Lstat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the symbolic link. +func (c *Client) Lstat(p string) (os.FileInfo, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpLstatPacket{ + Id: id, + Path: p, + }) + if err != nil { + return nil, err + } + switch typ { + case ssh_FXP_ATTRS: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIdErr{id, sid} + } + attr, _ := unmarshalAttrs(data) + return fileInfoFromStat(attr, path.Base(p)), nil + case ssh_FXP_STATUS: + return nil, unmarshalStatus(id, data) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// ReadLink reads the target of a symbolic link. +func (c *Client) ReadLink(p string) (string, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpReadlinkPacket{ + Id: id, + Path: p, + }) + if err != nil { + return "", err + } + switch typ { + case ssh_FXP_NAME: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIdErr{id, sid} + } + count, data := unmarshalUint32(data) + if count != 1 { + return "", unexpectedCount(1, count) + } + filename, _ := unmarshalString(data) // ignore dummy attributes + return filename, nil + case ssh_FXP_STATUS: + return "", unmarshalStatus(id, data) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Symlink creates a symbolic link at 'newname', pointing at target 'oldname' +func (c *Client) Symlink(oldname, newname string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpSymlinkPacket{ + Id: id, + Linkpath: newname, + Targetpath: oldname, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// setstat is a convience wrapper to allow for changing of various parts of the file descriptor. +func (c *Client) setstat(path string, flags uint32, attrs interface{}) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpSetstatPacket{ + Id: id, + Path: path, + Flags: flags, + Attrs: attrs, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// Chtimes changes the access and modification times of the named file. +func (c *Client) Chtimes(path string, atime time.Time, mtime time.Time) error { + type times struct { + Atime uint32 + Mtime uint32 + } + attrs := times{uint32(atime.Unix()), uint32(mtime.Unix())} + return c.setstat(path, ssh_FILEXFER_ATTR_ACMODTIME, attrs) +} + +// Chown changes the user and group owners of the named file. +func (c *Client) Chown(path string, uid, gid int) error { + type owner struct { + Uid uint32 + Gid uint32 + } + attrs := owner{uint32(uid), uint32(gid)} + return c.setstat(path, ssh_FILEXFER_ATTR_UIDGID, attrs) +} + +// Chmod changes the permissions of the named file. +func (c *Client) Chmod(path string, mode os.FileMode) error { + return c.setstat(path, ssh_FILEXFER_ATTR_PERMISSIONS, uint32(mode)) +} + +// Truncate sets the size of the named file. Although it may be safely assumed +// that if the size is less than its current size it will be truncated to fit, +// the SFTP protocol does not specify what behavior the server should do when setting +// size greater than the current size. +func (c *Client) Truncate(path string, size int64) error { + return c.setstat(path, ssh_FILEXFER_ATTR_SIZE, uint64(size)) +} + +// Open opens the named file for reading. If successful, methods on the +// returned file can be used for reading; the associated file descriptor +// has mode O_RDONLY. +func (c *Client) Open(path string) (*File, error) { + return c.open(path, flags(os.O_RDONLY)) +} + +// OpenFile is the generalized open call; most users will use Open or +// Create instead. It opens the named file with specified flag (O_RDONLY +// etc.). If successful, methods on the returned File can be used for I/O. +func (c *Client) OpenFile(path string, f int) (*File, error) { + return c.open(path, flags(f)) +} + +func (c *Client) open(path string, pflags uint32) (*File, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpOpenPacket{ + Id: id, + Path: path, + Pflags: pflags, + }) + if err != nil { + return nil, err + } + switch typ { + case ssh_FXP_HANDLE: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIdErr{id, sid} + } + handle, _ := unmarshalString(data) + return &File{c: c, path: path, handle: handle}, nil + case ssh_FXP_STATUS: + return nil, unmarshalStatus(id, data) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// close closes a handle handle previously returned in the response +// to SSH_FXP_OPEN or SSH_FXP_OPENDIR. The handle becomes invalid +// immediately after this request has been sent. +func (c *Client) close(handle string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpClosePacket{ + Id: id, + Handle: handle, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +func (c *Client) fstat(handle string) (*FileStat, error) { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpFstatPacket{ + Id: id, + Handle: handle, + }) + if err != nil { + return nil, err + } + switch typ { + case ssh_FXP_ATTRS: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIdErr{id, sid} + } + attr, _ := unmarshalAttrs(data) + return attr, nil + case ssh_FXP_STATUS: + return nil, unmarshalStatus(id, data) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// Get vfs stats from remote host. +// Implementing statvfs@openssh.com SSH_FXP_EXTENDED feature +// from http://www.opensource.apple.com/source/OpenSSH/OpenSSH-175/openssh/PROTOCOL?txt +func (c *Client) StatVFS(path string) (*StatVFS, error) { + // send the StatVFS packet to the server + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpStatvfsPacket{ + Id: id, + Path: path, + }) + if err != nil { + return nil, err + } + + switch typ { + // server responded with valid data + case ssh_FXP_EXTENDED_REPLY: + var response StatVFS + err = binary.Read(bytes.NewReader(data), binary.BigEndian, &response) + if err != nil { + return nil, errors.New("can not parse reply") + } + + return &response, nil + + // the resquest failed + case ssh_FXP_STATUS: + return nil, errors.New(fxp(ssh_FXP_STATUS).String()) + + default: + return nil, unimplementedPacketErr(typ) + } +} + +// Join joins any number of path elements into a single path, adding a +// separating slash if necessary. The result is Cleaned; in particular, all +// empty strings are ignored. +func (c *Client) Join(elem ...string) string { return path.Join(elem...) } + +// Remove removes the specified file or directory. An error will be returned if no +// file or directory with the specified path exists, or if the specified directory +// is not empty. +func (c *Client) Remove(path string) error { + err := c.removeFile(path) + if status, ok := err.(*StatusError); ok && status.Code == ssh_FX_FAILURE { + err = c.removeDirectory(path) + } + return err +} + +func (c *Client) removeFile(path string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpRemovePacket{ + Id: id, + Filename: path, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +func (c *Client) removeDirectory(path string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpRmdirPacket{ + Id: id, + Path: path, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// Rename renames a file. +func (c *Client) Rename(oldname, newname string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpRenamePacket{ + Id: id, + Oldpath: oldname, + Newpath: newname, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// result captures the result of receiving the a packet from the server +type result struct { + typ byte + data []byte + err error +} + +type idmarshaler interface { + id() uint32 + encoding.BinaryMarshaler +} + +func (c *Client) sendRequest(p idmarshaler) (byte, []byte, error) { + ch := make(chan result, 1) + c.dispatchRequest(ch, p) + s := <-ch + return s.typ, s.data, s.err +} + +func (c *Client) dispatchRequest(ch chan<- result, p idmarshaler) { + c.mu.Lock() + c.inflight[p.id()] = ch + if err := sendPacket(c.w, p); err != nil { + delete(c.inflight, p.id()) + c.mu.Unlock() + ch <- result{err: err} + return + } + c.mu.Unlock() +} + +// Creates the specified directory. An error will be returned if a file or +// directory with the specified path already exists, or if the directory's +// parent folder does not exist (the method cannot create complete paths). +func (c *Client) Mkdir(path string) error { + id := c.nextId() + typ, data, err := c.sendRequest(sshFxpMkdirPacket{ + Id: id, + Path: path, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return okOrErr(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// applyOptions applies options functions to the Client. +// If an error is encountered, option processing ceases. +func (c *Client) applyOptions(opts ...func(*Client) error) error { + for _, f := range opts { + if err := f(c); err != nil { + return err + } + } + return nil +} + +// File represents a remote file. +type File struct { + c *Client + path string + handle string + offset uint64 // current offset within remote file +} + +// Close closes the File, rendering it unusable for I/O. It returns an +// error, if any. +func (f *File) Close() error { + return f.c.close(f.handle) +} + +// Name returns the name of the file as presented to Open or Create. +func (f *File) Name() string { + return f.path +} + +const maxConcurrentRequests = 64 + +// Read reads up to len(b) bytes from the File. It returns the number of +// bytes read and an error, if any. EOF is signaled by a zero count with +// err set to io.EOF. +func (f *File) Read(b []byte) (int, error) { + // Split the read into multiple maxPacket sized concurrent reads + // bounded by maxConcurrentRequests. This allows reads with a suitably + // large buffer to transfer data at a much faster rate due to + // overlapping round trip times. + inFlight := 0 + desiredInFlight := 1 + offset := f.offset + ch := make(chan result) + type inflightRead struct { + b []byte + offset uint64 + } + reqs := map[uint32]inflightRead{} + type offsetErr struct { + offset uint64 + err error + } + var firstErr offsetErr + + sendReq := func(b []byte, offset uint64) { + reqId := f.c.nextId() + f.c.dispatchRequest(ch, sshFxpReadPacket{ + Id: reqId, + Handle: f.handle, + Offset: offset, + Len: uint32(len(b)), + }) + inFlight++ + reqs[reqId] = inflightRead{b: b, offset: offset} + } + + var read int + for len(b) > 0 || inFlight > 0 { + for inFlight < desiredInFlight && len(b) > 0 && firstErr.err == nil { + l := min(len(b), f.c.maxPacket) + rb := b[:l] + sendReq(rb, offset) + offset += uint64(l) + b = b[l:] + } + + if inFlight == 0 { + break + } + select { + case res := <-ch: + inFlight-- + if res.err != nil { + firstErr = offsetErr{offset: 0, err: res.err} + break + } + reqId, data := unmarshalUint32(res.data) + req, ok := reqs[reqId] + if !ok { + firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqId)} + break + } + delete(reqs, reqId) + switch res.typ { + case ssh_FXP_STATUS: + if firstErr.err == nil || req.offset < firstErr.offset { + firstErr = offsetErr{offset: req.offset, err: eofOrErr(unmarshalStatus(reqId, res.data))} + break + } + case ssh_FXP_DATA: + l, data := unmarshalUint32(data) + n := copy(req.b, data[:l]) + read += n + if n < len(req.b) { + sendReq(req.b[l:], req.offset+uint64(l)) + } + if desiredInFlight < maxConcurrentRequests { + desiredInFlight++ + } + default: + firstErr = offsetErr{offset: 0, err: unimplementedPacketErr(res.typ)} + break + } + } + } + // If the error is anything other than EOF, then there + // may be gaps in the data copied to the buffer so it's + // best to return 0 so the caller can't make any + // incorrect assumptions about the state of the buffer. + if firstErr.err != nil && firstErr.err != io.EOF { + read = 0 + } + f.offset += uint64(read) + return read, firstErr.err +} + +// WriteTo writes the file to w. The return value is the number of bytes +// written. Any error encountered during the write is also returned. +func (f *File) WriteTo(w io.Writer) (int64, error) { + fi, err := f.Stat() + if err != nil { + return 0, err + } + inFlight := 0 + desiredInFlight := 1 + offset := f.offset + writeOffset := offset + fileSize := uint64(fi.Size()) + ch := make(chan result) + type inflightRead struct { + b []byte + offset uint64 + } + reqs := map[uint32]inflightRead{} + pendingWrites := map[uint64][]byte{} + type offsetErr struct { + offset uint64 + err error + } + var firstErr offsetErr + + sendReq := func(b []byte, offset uint64) { + reqId := f.c.nextId() + f.c.dispatchRequest(ch, sshFxpReadPacket{ + Id: reqId, + Handle: f.handle, + Offset: offset, + Len: uint32(len(b)), + }) + inFlight++ + reqs[reqId] = inflightRead{b: b, offset: offset} + } + + var copied int64 + for firstErr.err == nil || inFlight > 0 { + for inFlight < desiredInFlight && firstErr.err == nil { + b := make([]byte, f.c.maxPacket) + sendReq(b, offset) + offset += uint64(f.c.maxPacket) + if offset > fileSize { + desiredInFlight = 1 + } + } + + if inFlight == 0 { + break + } + select { + case res := <-ch: + inFlight-- + if res.err != nil { + firstErr = offsetErr{offset: 0, err: res.err} + break + } + reqId, data := unmarshalUint32(res.data) + req, ok := reqs[reqId] + if !ok { + firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqId)} + break + } + delete(reqs, reqId) + switch res.typ { + case ssh_FXP_STATUS: + if firstErr.err == nil || req.offset < firstErr.offset { + firstErr = offsetErr{offset: req.offset, err: eofOrErr(unmarshalStatus(reqId, res.data))} + break + } + case ssh_FXP_DATA: + l, data := unmarshalUint32(data) + if req.offset == writeOffset { + nbytes, err := w.Write(data) + copied += int64(nbytes) + if err != nil { + firstErr = offsetErr{offset: req.offset + uint64(nbytes), err: err} + break + } + if nbytes < int(l) { + firstErr = offsetErr{offset: req.offset + uint64(nbytes), err: io.ErrShortWrite} + break + } + switch { + case offset > fileSize: + desiredInFlight = 1 + case desiredInFlight < maxConcurrentRequests: + desiredInFlight++ + } + writeOffset += uint64(nbytes) + for pendingData, ok := pendingWrites[writeOffset]; ok; pendingData, ok = pendingWrites[writeOffset] { + nbytes, err := w.Write(pendingData) + if err != nil { + firstErr = offsetErr{offset: writeOffset + uint64(nbytes), err: err} + break + } + if nbytes < len(pendingData) { + firstErr = offsetErr{offset: writeOffset + uint64(nbytes), err: io.ErrShortWrite} + break + } + writeOffset += uint64(nbytes) + inFlight-- + } + } else { + // Don't write the data yet because + // this response came in out of order + // and we need to wait for responses + // for earlier segments of the file. + inFlight++ // Pending writes should still be considered inFlight. + pendingWrites[req.offset] = data + } + default: + firstErr = offsetErr{offset: 0, err: unimplementedPacketErr(res.typ)} + break + } + } + } + if firstErr.err != io.EOF { + return copied, firstErr.err + } + return copied, nil + +} + +// Stat returns the FileInfo structure describing file. If there is an +// error. +func (f *File) Stat() (os.FileInfo, error) { + fs, err := f.c.fstat(f.handle) + if err != nil { + return nil, err + } + return fileInfoFromStat(fs, path.Base(f.path)), nil +} + +// Write writes len(b) bytes to the File. It returns the number of bytes +// written and an error, if any. Write returns a non-nil error when n != +// len(b). +func (f *File) Write(b []byte) (int, error) { + // Split the write into multiple maxPacket sized concurrent writes + // bounded by maxConcurrentRequests. This allows writes with a suitably + // large buffer to transfer data at a much faster rate due to + // overlapping round trip times. + inFlight := 0 + desiredInFlight := 1 + offset := f.offset + ch := make(chan result) + var firstErr error + written := len(b) + for len(b) > 0 || inFlight > 0 { + for inFlight < desiredInFlight && len(b) > 0 && firstErr == nil { + l := min(len(b), f.c.maxPacket) + rb := b[:l] + f.c.dispatchRequest(ch, sshFxpWritePacket{ + Id: f.c.nextId(), + Handle: f.handle, + Offset: offset, + Length: uint32(len(rb)), + Data: rb, + }) + inFlight++ + offset += uint64(l) + b = b[l:] + } + + if inFlight == 0 { + break + } + select { + case res := <-ch: + inFlight-- + if res.err != nil { + firstErr = res.err + break + } + switch res.typ { + case ssh_FXP_STATUS: + id, _ := unmarshalUint32(res.data) + err := okOrErr(unmarshalStatus(id, res.data)) + if err != nil && firstErr == nil { + firstErr = err + break + } + if desiredInFlight < maxConcurrentRequests { + desiredInFlight++ + } + default: + firstErr = unimplementedPacketErr(res.typ) + break + } + } + } + // If error is non-nil, then there may be gaps in the data written to + // the file so it's best to return 0 so the caller can't make any + // incorrect assumptions about the state of the file. + if firstErr != nil { + written = 0 + } + f.offset += uint64(written) + return written, firstErr +} + +// ReadFrom reads data from r until EOF and writes it to the file. The return +// value is the number of bytes read. Any error except io.EOF encountered +// during the read is also returned. +func (f *File) ReadFrom(r io.Reader) (int64, error) { + inFlight := 0 + desiredInFlight := 1 + offset := f.offset + ch := make(chan result) + var firstErr error + read := int64(0) + b := make([]byte, f.c.maxPacket) + for inFlight > 0 || firstErr == nil { + for inFlight < desiredInFlight && firstErr == nil { + n, err := r.Read(b) + if err != nil { + firstErr = err + } + f.c.dispatchRequest(ch, sshFxpWritePacket{ + Id: f.c.nextId(), + Handle: f.handle, + Offset: offset, + Length: uint32(n), + Data: b[:n], + }) + inFlight++ + offset += uint64(n) + read += int64(n) + } + + if inFlight == 0 { + break + } + select { + case res := <-ch: + inFlight-- + if res.err != nil { + firstErr = res.err + break + } + switch res.typ { + case ssh_FXP_STATUS: + id, _ := unmarshalUint32(res.data) + err := okOrErr(unmarshalStatus(id, res.data)) + if err != nil && firstErr == nil { + firstErr = err + break + } + if desiredInFlight < maxConcurrentRequests { + desiredInFlight++ + } + default: + firstErr = unimplementedPacketErr(res.typ) + break + } + } + } + if firstErr == io.EOF { + firstErr = nil + } + // If error is non-nil, then there may be gaps in the data written to + // the file so it's best to return 0 so the caller can't make any + // incorrect assumptions about the state of the file. + if firstErr != nil { + read = 0 + } + f.offset += uint64(read) + return read, firstErr +} + +// Seek implements io.Seeker by setting the client offset for the next Read or +// Write. It returns the next offset read. Seeking before or after the end of +// the file is undefined. Seeking relative to the end calls Stat. +func (f *File) Seek(offset int64, whence int) (int64, error) { + switch whence { + case os.SEEK_SET: + f.offset = uint64(offset) + case os.SEEK_CUR: + f.offset = uint64(int64(f.offset) + offset) + case os.SEEK_END: + fi, err := f.Stat() + if err != nil { + return int64(f.offset), err + } + f.offset = uint64(fi.Size() + offset) + default: + return int64(f.offset), unimplementedSeekWhence(whence) + } + return int64(f.offset), nil +} + +// Chown changes the uid/gid of the current file. +func (f *File) Chown(uid, gid int) error { + return f.c.Chown(f.path, uid, gid) +} + +// Chmod changes the permissions of the current file. +func (f *File) Chmod(mode os.FileMode) error { + return f.c.Chmod(f.path, mode) +} + +// Truncate sets the size of the current file. Although it may be safely assumed +// that if the size is less than its current size it will be truncated to fit, +// the SFTP protocol does not specify what behavior the server should do when setting +// size greater than the current size. +func (f *File) Truncate(size int64) error { + return f.c.Truncate(f.path, size) +} + +func min(a, b int) int { + if a > b { + return b + } + return a +} + +// okOrErr returns nil if Err.Code is SSH_FX_OK, otherwise it returns the error. +func okOrErr(err error) error { + if err, ok := err.(*StatusError); ok && err.Code == ssh_FX_OK { + return nil + } + return err +} + +func eofOrErr(err error) error { + if err, ok := err.(*StatusError); ok && err.Code == ssh_FX_EOF { + return io.EOF + } + return err +} + +func unmarshalStatus(id uint32, data []byte) error { + sid, data := unmarshalUint32(data) + if sid != id { + return &unexpectedIdErr{id, sid} + } + code, data := unmarshalUint32(data) + msg, data := unmarshalString(data) + lang, _, _ := unmarshalStringSafe(data) + return &StatusError{ + Code: code, + msg: msg, + lang: lang, + } +} + +func marshalStatus(b []byte, err StatusError) []byte { + b = marshalUint32(b, err.Code) + b = marshalString(b, err.msg) + b = marshalString(b, err.lang) + return b +} + +// flags converts the flags passed to OpenFile into ssh flags. +// Unsupported flags are ignored. +func flags(f int) uint32 { + var out uint32 + switch f & os.O_WRONLY { + case os.O_WRONLY: + out |= ssh_FXF_WRITE + case os.O_RDONLY: + out |= ssh_FXF_READ + } + if f&os.O_RDWR == os.O_RDWR { + out |= ssh_FXF_READ | ssh_FXF_WRITE + } + if f&os.O_APPEND == os.O_APPEND { + out |= ssh_FXF_APPEND + } + if f&os.O_CREATE == os.O_CREATE { + out |= ssh_FXF_CREAT + } + if f&os.O_TRUNC == os.O_TRUNC { + out |= ssh_FXF_TRUNC + } + if f&os.O_EXCL == os.O_EXCL { + out |= ssh_FXF_EXCL + } + return out +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_darwin_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_darwin_test.go new file mode 100644 index 0000000..949a042 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_darwin_test.go @@ -0,0 +1,41 @@ +package sftp + +import ( + "syscall" + "testing" +) + +func TestClientStatVFS(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + vfs, err := sftp.StatVFS("/") + if err != nil { + t.Fatal(err) + } + + // get system stats + s := syscall.Statfs_t{} + err = syscall.Statfs("/", &s) + if err != nil { + t.Fatal(err) + } + + // check some stats + if vfs.Files != uint64(s.Files) { + t.Fatal("fr_size does not match") + } + + if vfs.Bfree != uint64(s.Bfree) { + t.Fatal("f_bsize does not match") + } + + if vfs.Favail != uint64(s.Ffree) { + t.Fatal("f_namemax does not match") + } + + if vfs.Bavail != s.Bavail { + t.Fatal("f_bavail does not match") + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_linux_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_linux_test.go new file mode 100644 index 0000000..45aa137 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_linux_test.go @@ -0,0 +1,41 @@ +package sftp + +import ( + "syscall" + "testing" +) + +func TestClientStatVFS(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + vfs, err := sftp.StatVFS("/") + if err != nil { + t.Fatal(err) + } + + // get system stats + s := syscall.Statfs_t{} + err = syscall.Statfs("/", &s) + if err != nil { + t.Fatal(err) + } + + // check some stats + if vfs.Frsize != uint64(s.Frsize) { + t.Fatal("fr_size does not match") + } + + if vfs.Bsize != uint64(s.Bsize) { + t.Fatal("f_bsize does not match") + } + + if vfs.Namemax != uint64(s.Namelen) { + t.Fatal("f_namemax does not match") + } + + if vfs.Bavail != s.Bavail { + t.Fatal("f_bavail does not match") + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go new file mode 100644 index 0000000..94d4396 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go @@ -0,0 +1,1540 @@ +package sftp + +// sftp integration tests +// enable with -integration + +import ( + "crypto/sha1" + "flag" + "io" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "os/user" + "path" + "path/filepath" + "reflect" + "regexp" + "strconv" + "testing" + "testing/quick" + "time" + + "github.com/kr/fs" +) + +const ( + READONLY = true + READWRITE = false + NO_DELAY time.Duration = 0 + + debuglevel = "ERROR" // set to "DEBUG" for debugging +) + +var testServerImpl = flag.Bool("testserver", false, "perform integration tests against sftp package server instance") +var testIntegration = flag.Bool("integration", false, "perform integration tests against sftp server process") +var testSftp = flag.String("sftp", "/usr/lib/openssh/sftp-server", "location of the sftp server binary") + +type delayedWrite struct { + t time.Time + b []byte +} + +// delayedWriter wraps a writer and artificially delays the write. This is +// meant to mimic connections with various latencies. Error's returned from the +// underlying writer will panic so this should only be used over reliable +// connections. +type delayedWriter struct { + w io.WriteCloser + ch chan delayedWrite + closed chan struct{} +} + +func newDelayedWriter(w io.WriteCloser, delay time.Duration) io.WriteCloser { + ch := make(chan delayedWrite, 128) + closed := make(chan struct{}) + go func() { + for writeMsg := range ch { + time.Sleep(writeMsg.t.Add(delay).Sub(time.Now())) + n, err := w.Write(writeMsg.b) + if err != nil { + panic("write error") + } + if n < len(writeMsg.b) { + panic("showrt write") + } + } + w.Close() + close(closed) + }() + return delayedWriter{w: w, ch: ch, closed: closed} +} + +func (w delayedWriter) Write(b []byte) (int, error) { + bcopy := make([]byte, len(b)) + copy(bcopy, b) + w.ch <- delayedWrite{t: time.Now(), b: bcopy} + return len(b), nil +} + +func (w delayedWriter) Close() error { + close(w.ch) + <-w.closed + return nil +} + +func testClientGoSvr(t testing.TB, readonly bool, delay time.Duration) (*Client, *exec.Cmd) { + txPipeRd, txPipeWr := io.Pipe() + rxPipeRd, rxPipeWr := io.Pipe() + + server, err := NewServer(txPipeRd, rxPipeWr, os.Stderr, 0, readonly, ".") + if err != nil { + t.Fatal(err) + } + go server.Serve() + + var ctx io.WriteCloser = txPipeWr + if delay > NO_DELAY { + ctx = newDelayedWriter(ctx, delay) + } + + client, err := NewClientPipe(rxPipeRd, ctx) + if err != nil { + t.Fatal(err) + } + + // dummy command... + return client, exec.Command("true") +} + +// testClient returns a *Client connected to a localy running sftp-server +// the *exec.Cmd returned must be defer Wait'd. +func testClient(t testing.TB, readonly bool, delay time.Duration) (*Client, *exec.Cmd) { + if !*testIntegration { + t.Skip("skipping intergration test") + } + + if *testServerImpl { + return testClientGoSvr(t, readonly, delay) + } + + cmd := exec.Command(*testSftp, "-e", "-R", "-l", debuglevel) // log to stderr, read only + if !readonly { + cmd = exec.Command(*testSftp, "-e", "-l", debuglevel) // log to stderr + } + cmd.Stderr = os.Stdout + pw, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if delay > NO_DELAY { + pw = newDelayedWriter(pw, delay) + } + pr, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Skipf("could not start sftp-server process: %v", err) + } + + sftp, err := NewClientPipe(pr, pw) + if err != nil { + t.Fatal(err) + } + + return sftp, cmd +} + +func TestNewClient(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + + if err := sftp.Close(); err != nil { + t.Fatal(err) + } +} + +func TestClientLstat(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + want, err := os.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + got, err := sftp.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + if !sameFile(want, got) { + t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) + } +} + +func TestClientLstatMissing(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + os.Remove(f.Name()) + + _, err = sftp.Lstat(f.Name()) + if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_NO_SUCH_FILE { + t.Fatalf("Lstat: want: %v, got %#v", ssh_FX_NO_SUCH_FILE, err) + } +} + +func TestClientMkdir(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + dir, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + sub := path.Join(dir, "mkdir1") + if err := sftp.Mkdir(sub); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(sub); err != nil { + t.Fatal(err) + } +} + +func TestClientOpen(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + got, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + if err := got.Close(); err != nil { + t.Fatal(err) + } +} + +const seekBytes = 128 * 1024 + +type seek struct { + offset int64 +} + +func (s seek) Generate(r *rand.Rand, _ int) reflect.Value { + s.offset = int64(r.Int31n(seekBytes)) + return reflect.ValueOf(s) +} + +func (s seek) set(t *testing.T, r io.ReadSeeker) { + if _, err := r.Seek(s.offset, os.SEEK_SET); err != nil { + t.Fatalf("error while seeking with %+v: %v", s, err) + } +} + +func (s seek) current(t *testing.T, r io.ReadSeeker) { + const mid = seekBytes / 2 + + skip := s.offset / 2 + if s.offset > mid { + skip = -skip + } + + if _, err := r.Seek(mid, os.SEEK_SET); err != nil { + t.Fatalf("error seeking to midpoint with %+v: %v", s, err) + } + if _, err := r.Seek(skip, os.SEEK_CUR); err != nil { + t.Fatalf("error seeking from %d with %+v: %v", mid, s, err) + } +} + +func (s seek) end(t *testing.T, r io.ReadSeeker) { + if _, err := r.Seek(-s.offset, os.SEEK_END); err != nil { + t.Fatalf("error seeking from end with %+v: %v", s, err) + } +} + +func TestClientSeek(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + fOS, err := ioutil.TempFile("", "seek-test") + if err != nil { + t.Fatal(err) + } + defer fOS.Close() + + fSFTP, err := sftp.Open(fOS.Name()) + if err != nil { + t.Fatal(err) + } + defer fSFTP.Close() + + writeN(t, fOS, seekBytes) + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.set(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.set(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal absolute seeks: %v", err) + } + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.current(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.current(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal seeks from middle: %v", err) + } + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.end(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.end(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal seeks from end: %v", err) + } +} + +func TestClientCreate(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err != nil { + t.Fatal(err) + } + defer f2.Close() +} + +func TestClientAppend(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.OpenFile(f.Name(), os.O_RDWR|os.O_APPEND) + if err != nil { + t.Fatal(err) + } + defer f2.Close() +} + +func TestClientCreateFailed(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_PERMISSION_DENIED { + t.Fatalf("Create: want: %v, got %#v", ssh_FX_PERMISSION_DENIED, err) + } + if err == nil { + f2.Close() + } +} + +func TestClientFileName(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + + if got, want := f2.Name(), f.Name(); got != want { + t.Fatalf("Name: got %q want %q", want, got) + } +} + +func TestClientFileStat(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + want, err := os.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + + got, err := f2.Stat() + if err != nil { + t.Fatal(err) + } + + if !sameFile(want, got) { + t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) + } +} + +func TestClientRemove(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(f.Name()); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { + t.Fatal(err) + } +} + +func TestClientRemoveDir(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + dir, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(dir); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(dir); !os.IsNotExist(err) { + t.Fatal(err) + } +} + +func TestClientRemoveFailed(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(f.Name()); err == nil { + t.Fatalf("Remove(%v): want: permission denied, got %v", f.Name(), err) + } + if _, err := os.Lstat(f.Name()); err != nil { + t.Fatal(err) + } +} + +func TestClientRename(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".new" + if err := sftp.Rename(f.Name(), f2); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { + t.Fatal(err) + } + if _, err := os.Lstat(f2); err != nil { + t.Fatal(err) + } +} + +func TestClientReadLink(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".sym" + if err := os.Symlink(f.Name(), f2); err != nil { + t.Fatal(err) + } + if rl, err := sftp.ReadLink(f2); err != nil { + t.Fatal(err) + } else if rl != f.Name() { + t.Fatalf("unexpected link target: %v, not %v", rl, f.Name()) + } +} + +func TestClientSymlink(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".sym" + if err := sftp.Symlink(f.Name(), f2); err != nil { + t.Fatal(err) + } + if rl, err := sftp.ReadLink(f2); err != nil { + t.Fatal(err) + } else if rl != f.Name() { + t.Fatalf("unexpected link target: %v, not %v", rl, f.Name()) + } +} + +func TestClientChmod(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chmod(f.Name(), 0531); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(f.Name()); err != nil { + t.Fatal(err) + } else if stat.Mode()&os.ModePerm != 0531 { + t.Fatalf("invalid perm %o\n", stat.Mode()) + } +} + +func TestClientChmodReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chmod(f.Name(), 0531); err == nil { + t.Fatal("expected error") + } +} + +func TestClientChown(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + usr, err := user.Current() + if err != nil { + t.Fatal(err) + } + chownto, err := user.Lookup("daemon") // seems common-ish... + if err != nil { + t.Fatal(err) + } + + if usr.Uid != "0" { + t.Log("must be root to run chown tests") + t.Skip() + } + toUid, err := strconv.Atoi(chownto.Uid) + if err != nil { + t.Fatal(err) + } + toGid, err := strconv.Atoi(chownto.Gid) + if err != nil { + t.Fatal(err) + } + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + before, err := exec.Command("ls", "-nl", f.Name()).Output() + if err != nil { + t.Fatal(err) + } + if err := sftp.Chown(f.Name(), toUid, toGid); err != nil { + t.Fatal(err) + } + after, err := exec.Command("ls", "-nl", f.Name()).Output() + if err != nil { + t.Fatal(err) + } + + spaceRegex := regexp.MustCompile(`\s+`) + + beforeWords := spaceRegex.Split(string(before), -1) + if beforeWords[2] != "0" { + t.Fatalf("bad previous user? should be root") + } + afterWords := spaceRegex.Split(string(after), -1) + if afterWords[2] != chownto.Uid || afterWords[3] != chownto.Gid { + t.Fatalf("bad chown: %#v", afterWords) + } + t.Logf("before: %v", string(before)) + t.Logf(" after: %v", string(after)) +} + +func TestClientChownReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + usr, err := user.Current() + if err != nil { + t.Fatal(err) + } + chownto, err := user.Lookup("daemon") // seems common-ish... + if err != nil { + t.Fatal(err) + } + + if usr.Uid != "0" { + t.Log("must be root to run chown tests") + t.Skip() + } + toUid, err := strconv.Atoi(chownto.Uid) + if err != nil { + t.Fatal(err) + } + toGid, err := strconv.Atoi(chownto.Gid) + if err != nil { + t.Fatal(err) + } + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chown(f.Name(), toUid, toGid); err == nil { + t.Fatal("expected error") + } +} + +func TestClientChtimes(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + + atime := time.Date(2013, 2, 23, 13, 24, 35, 0, time.UTC) + mtime := time.Date(1985, 6, 12, 6, 6, 6, 0, time.UTC) + if err := sftp.Chtimes(f.Name(), atime, mtime); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(f.Name()); err != nil { + t.Fatal(err) + } else if stat.ModTime().Sub(mtime) != 0 { + t.Fatalf("incorrect mtime: %v vs %v", stat.ModTime(), mtime) + } +} + +func TestClientChtimesReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + + atime := time.Date(2013, 2, 23, 13, 24, 35, 0, time.UTC) + mtime := time.Date(1985, 6, 12, 6, 6, 6, 0, time.UTC) + if err := sftp.Chtimes(f.Name(), atime, mtime); err == nil { + t.Fatal("expected error") + } +} + +func TestClientTruncate(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + + if n, err := f.Write([]byte("hello world")); n != 11 || err != nil { + t.Fatal(err) + } + f.Close() + + if err := sftp.Truncate(fname, 5); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(fname); err != nil { + t.Fatal(err) + } else if stat.Size() != 5 { + t.Fatalf("unexpected size: %d", stat.Size()) + } +} + +func TestClientTruncateReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + + if n, err := f.Write([]byte("hello world")); n != 11 || err != nil { + t.Fatal(err) + } + f.Close() + + if err := sftp.Truncate(fname, 5); err == nil { + t.Fatal("expected error") + } + if stat, err := os.Stat(fname); err != nil { + t.Fatal(err) + } else if stat.Size() != 11 { + t.Fatalf("unexpected size: %d", stat.Size()) + } +} + +func sameFile(want, got os.FileInfo) bool { + return want.Name() == got.Name() && + want.Size() == got.Size() +} + +func TestClientReadSimple(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + f, err := ioutil.TempFile(d, "read-test") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + f.Write([]byte("hello")) + f.Close() + + f2, err := sftp.Open(fname) + if err != nil { + t.Fatal(err) + } + defer f2.Close() + stuff := make([]byte, 32) + n, err := f2.Read(stuff) + if err != nil && err != io.EOF { + t.Fatalf("err: %v", err) + } + if n != 5 { + t.Fatalf("n should be 5, is %v", n) + } + if string(stuff[0:5]) != "hello" { + t.Fatalf("invalid contents") + } +} + +func TestClientReadDir(t *testing.T) { + sftp1, cmd1 := testClient(t, READONLY, NO_DELAY) + sftp2, cmd2 := testClientGoSvr(t, READONLY, NO_DELAY) + defer cmd1.Wait() + defer cmd2.Wait() + defer sftp1.Close() + defer sftp2.Close() + + dir := "/dev/" + + d, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + defer d.Close() + osfiles, err := d.Readdir(4096) + if err != nil { + t.Fatal(err) + } + + sftp1Files, err := sftp1.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + sftp2Files, err := sftp2.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + osFilesByName := map[string]os.FileInfo{} + for _, f := range osfiles { + osFilesByName[f.Name()] = f + } + sftp1FilesByName := map[string]os.FileInfo{} + for _, f := range sftp1Files { + sftp1FilesByName[f.Name()] = f + } + sftp2FilesByName := map[string]os.FileInfo{} + for _, f := range sftp2Files { + sftp2FilesByName[f.Name()] = f + } + + if len(osFilesByName) != len(sftp1FilesByName) || len(sftp1FilesByName) != len(sftp2FilesByName) { + t.Fatalf("os gives %v, sftp1 gives %v, sftp2 gives %v", len(osFilesByName), len(sftp1FilesByName), len(sftp2FilesByName)) + } + + for name, osF := range osFilesByName { + sftp1F, ok := sftp1FilesByName[name] + if !ok { + t.Fatalf("%v present in os but not sftp1", name) + } + sftp2F, ok := sftp2FilesByName[name] + if !ok { + t.Fatalf("%v present in os but not sftp2", name) + } + + //t.Logf("%v: %v %v %v", name, osF, sftp1F, sftp2F) + if osF.Size() != sftp1F.Size() || sftp1F.Size() != sftp2F.Size() { + t.Fatalf("size %v %v %v", osF.Size(), sftp1F.Size(), sftp2F.Size()) + } + if osF.IsDir() != sftp1F.IsDir() || sftp1F.IsDir() != sftp2F.IsDir() { + t.Fatalf("isdir %v %v %v", osF.IsDir(), sftp1F.IsDir(), sftp2F.IsDir()) + } + if osF.ModTime().Sub(sftp1F.ModTime()) > time.Second || sftp1F.ModTime() != sftp2F.ModTime() { + t.Fatalf("modtime %v %v %v", osF.ModTime(), sftp1F.ModTime(), sftp2F.ModTime()) + } + if osF.Mode() != sftp1F.Mode() || sftp1F.Mode() != sftp2F.Mode() { + t.Fatalf("mode %x %x %x", osF.Mode(), sftp1F.Mode(), sftp2F.Mode()) + } + } +} + +var clientReadTests = []struct { + n int64 +}{ + {0}, + {1}, + {1000}, + {1024}, + {1025}, + {2048}, + {4096}, + {1 << 12}, + {1 << 13}, + {1 << 14}, + {1 << 15}, + {1 << 16}, + {1 << 17}, + {1 << 18}, + {1 << 19}, + {1 << 20}, +} + +func TestClientRead(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + for _, tt := range clientReadTests { + f, err := ioutil.TempFile(d, "read-test") + if err != nil { + t.Fatal(err) + } + defer f.Close() + hash := writeN(t, f, tt.n) + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + defer f2.Close() + hash2, n := readHash(t, f2) + if hash != hash2 || tt.n != n { + t.Errorf("Read: hash: want: %q, got %q, read: want: %v, got %v", hash, hash2, tt.n, n) + } + } +} + +// readHash reads r until EOF returning the number of bytes read +// and the hash of the contents. +func readHash(t *testing.T, r io.Reader) (string, int64) { + h := sha1.New() + tr := io.TeeReader(r, h) + read, err := io.Copy(ioutil.Discard, tr) + if err != nil { + t.Fatal(err) + } + return string(h.Sum(nil)), read +} + +// writeN writes n bytes of random data to w and returns the +// hash of that data. +func writeN(t *testing.T, w io.Writer, n int64) string { + rand, err := os.Open("/dev/urandom") + if err != nil { + t.Fatal(err) + } + defer rand.Close() + + h := sha1.New() + + mw := io.MultiWriter(w, h) + + written, err := io.CopyN(mw, rand, n) + if err != nil { + t.Fatal(err) + } + if written != n { + t.Fatalf("CopyN(%v): wrote: %v", n, written) + } + return string(h.Sum(nil)) +} + +var clientWriteTests = []struct { + n int + total int64 // cumulative file size +}{ + {0, 0}, + {1, 1}, + {0, 1}, + {999, 1000}, + {24, 1024}, + {1023, 2047}, + {2048, 4095}, + {1 << 12, 8191}, + {1 << 13, 16383}, + {1 << 14, 32767}, + {1 << 15, 65535}, + {1 << 16, 131071}, + {1 << 17, 262143}, + {1 << 18, 524287}, + {1 << 19, 1048575}, + {1 << 20, 2097151}, + {1 << 21, 4194303}, +} + +func TestClientWrite(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + f := path.Join(d, "writeTest") + w, err := sftp.Create(f) + if err != nil { + t.Fatal(err) + } + defer w.Close() + + for _, tt := range clientWriteTests { + got, err := w.Write(make([]byte, tt.n)) + if err != nil { + t.Fatal(err) + } + if got != tt.n { + t.Errorf("Write(%v): wrote: want: %v, got %v", tt.n, tt.n, got) + } + fi, err := os.Stat(f) + if err != nil { + t.Fatal(err) + } + if total := fi.Size(); total != tt.total { + t.Errorf("Write(%v): size: want: %v, got %v", tt.n, tt.total, total) + } + } +} + +// taken from github.com/kr/fs/walk_test.go + +type PathTest struct { + path, result string +} + +type Node struct { + name string + entries []*Node // nil if the entry is a file + mark int +} + +var tree = &Node{ + "testdata", + []*Node{ + {"a", nil, 0}, + {"b", []*Node{}, 0}, + {"c", nil, 0}, + { + "d", + []*Node{ + {"x", nil, 0}, + {"y", []*Node{}, 0}, + { + "z", + []*Node{ + {"u", nil, 0}, + {"v", nil, 0}, + }, + 0, + }, + }, + 0, + }, + }, + 0, +} + +func walkTree(n *Node, path string, f func(path string, n *Node)) { + f(path, n) + for _, e := range n.entries { + walkTree(e, filepath.Join(path, e.name), f) + } +} + +func makeTree(t *testing.T) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.entries == nil { + fd, err := os.Create(path) + if err != nil { + t.Errorf("makeTree: %v", err) + return + } + fd.Close() + } else { + os.Mkdir(path, 0770) + } + }) +} + +func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } + +func checkMarks(t *testing.T, report bool) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.mark != 1 && report { + t.Errorf("node %s mark = %d; expected 1", path, n.mark) + } + n.mark = 0 + }) +} + +// Assumes that each node name is unique. Good enough for a test. +// If clear is true, any incoming error is cleared before return. The errors +// are always accumulated, though. +func mark(path string, info os.FileInfo, err error, errors *[]error, clear bool) error { + if err != nil { + *errors = append(*errors, err) + if clear { + return nil + } + return err + } + name := info.Name() + walkTree(tree, tree.name, func(path string, n *Node) { + if n.name == name { + n.mark++ + } + }) + return nil +} + +func TestClientWalk(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + makeTree(t) + errors := make([]error, 0, 10) + clear := true + markFn := func(walker *fs.Walker) (err error) { + for walker.Step() { + err = mark(walker.Path(), walker.Stat(), walker.Err(), &errors, clear) + if err != nil { + break + } + } + return err + } + // Expect no errors. + err := markFn(sftp.Walk(tree.name)) + if err != nil { + t.Fatalf("no error expected, found: %s", err) + } + if len(errors) != 0 { + t.Fatalf("unexpected errors: %s", errors) + } + checkMarks(t, true) + errors = errors[0:0] + + // Test permission errors. Only possible if we're not root + // and only on some file systems (AFS, FAT). To avoid errors during + // all.bash on those file systems, skip during go test -short. + if os.Getuid() > 0 && !testing.Short() { + // introduce 2 errors: chmod top-level directories to 0 + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) + + // 3) capture errors, expect two. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + err := markFn(sftp.Walk(tree.name)) + if err != nil { + t.Fatalf("expected no error return from Walk, got %s", err) + } + if len(errors) != 2 { + t.Errorf("expected 2 errors, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, true) + errors = errors[0:0] + + // 4) capture errors, stop after first error. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + clear = false // error will stop processing + err = markFn(sftp.Walk(tree.name)) + if err == nil { + t.Fatalf("expected error return from Walk") + } + if len(errors) != 1 { + t.Errorf("expected 1 error, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, false) + errors = errors[0:0] + + // restore permissions + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) + } + + // cleanup + if err := os.RemoveAll(tree.name); err != nil { + t.Errorf("removeTree: %v", err) + } +} + +// sftp/issue/42, abrupt server hangup would result in client hangs. +func TestServerRoughDisconnect(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + + f, err := sftp.Open("/dev/zero") + if err != nil { + t.Fatal(err) + } + defer f.Close() + go func() { + time.Sleep(100 * time.Millisecond) + cmd.Process.Kill() + }() + + io.Copy(ioutil.Discard, f) + sftp.Close() +} + +func benchmarkRead(b *testing.B, bufsize int, delay time.Duration) { + size := 10*1024*1024 + 123 // ~10MiB + + // open sftp client + sftp, cmd := testClient(b, READONLY, delay) + defer cmd.Wait() + defer sftp.Close() + + buf := make([]byte, bufsize) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + offset := 0 + + f2, err := sftp.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + defer f2.Close() + + for offset < size { + n, err := io.ReadFull(f2, buf) + offset += n + if err == io.ErrUnexpectedEOF && offset != size { + b.Fatalf("read too few bytes! want: %d, got: %d", size, n) + } + + if err != nil { + b.Fatal(err) + } + + offset += n + } + } +} + +func BenchmarkRead1k(b *testing.B) { + benchmarkRead(b, 1*1024, NO_DELAY) +} + +func BenchmarkRead16k(b *testing.B) { + benchmarkRead(b, 16*1024, NO_DELAY) +} + +func BenchmarkRead32k(b *testing.B) { + benchmarkRead(b, 32*1024, NO_DELAY) +} + +func BenchmarkRead128k(b *testing.B) { + benchmarkRead(b, 128*1024, NO_DELAY) +} + +func BenchmarkRead512k(b *testing.B) { + benchmarkRead(b, 512*1024, NO_DELAY) +} + +func BenchmarkRead1MiB(b *testing.B) { + benchmarkRead(b, 1024*1024, NO_DELAY) +} + +func BenchmarkRead4MiB(b *testing.B) { + benchmarkRead(b, 4*1024*1024, NO_DELAY) +} + +func BenchmarkRead4MiBDelay10Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 10*time.Millisecond) +} + +func BenchmarkRead4MiBDelay50Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 50*time.Millisecond) +} + +func BenchmarkRead4MiBDelay150Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 150*time.Millisecond) +} + +func benchmarkWrite(b *testing.B, bufsize int, delay time.Duration) { + size := 10*1024*1024 + 123 // ~10MiB + + // open sftp client + sftp, cmd := testClient(b, false, delay) + defer cmd.Wait() + defer sftp.Close() + + data := make([]byte, size) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + offset := 0 + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err != nil { + b.Fatal(err) + } + defer f2.Close() + + for offset < size { + n, err := f2.Write(data[offset:min(len(data), offset+bufsize)]) + if err != nil { + b.Fatal(err) + } + + if offset+n < size && n != bufsize { + b.Fatalf("wrote too few bytes! want: %d, got: %d", size, n) + } + + offset += n + } + + f2.Close() + + fi, err := os.Stat(f.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != int64(size) { + b.Fatalf("wrong file size: want %d, got %d", size, fi.Size()) + } + + os.Remove(f.Name()) + } +} + +func BenchmarkWrite1k(b *testing.B) { + benchmarkWrite(b, 1*1024, NO_DELAY) +} + +func BenchmarkWrite16k(b *testing.B) { + benchmarkWrite(b, 16*1024, NO_DELAY) +} + +func BenchmarkWrite32k(b *testing.B) { + benchmarkWrite(b, 32*1024, NO_DELAY) +} + +func BenchmarkWrite128k(b *testing.B) { + benchmarkWrite(b, 128*1024, NO_DELAY) +} + +func BenchmarkWrite512k(b *testing.B) { + benchmarkWrite(b, 512*1024, NO_DELAY) +} + +func BenchmarkWrite1MiB(b *testing.B) { + benchmarkWrite(b, 1024*1024, NO_DELAY) +} + +func BenchmarkWrite4MiB(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, NO_DELAY) +} + +func BenchmarkWrite4MiBDelay10Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 10*time.Millisecond) +} + +func BenchmarkWrite4MiBDelay50Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 50*time.Millisecond) +} + +func BenchmarkWrite4MiBDelay150Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 150*time.Millisecond) +} + +func benchmarkCopyDown(b *testing.B, fileSize int64, delay time.Duration) { + // Create a temp file and fill it with zero's. + src, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer src.Close() + srcFilename := src.Name() + defer os.Remove(srcFilename) + zero, err := os.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + n, err := io.Copy(src, io.LimitReader(zero, fileSize)) + if err != nil { + b.Fatal(err) + } + if n < fileSize { + b.Fatal("short copy") + } + zero.Close() + src.Close() + + sftp, cmd := testClient(b, READONLY, delay) + defer cmd.Wait() + defer sftp.Close() + b.ResetTimer() + b.SetBytes(fileSize) + + for i := 0; i < b.N; i++ { + dst, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer os.Remove(dst.Name()) + + src, err := sftp.Open(srcFilename) + if err != nil { + b.Fatal(err) + } + defer src.Close() + n, err := io.Copy(dst, src) + if err != nil { + b.Fatalf("copy error: %v", err) + } + if n < fileSize { + b.Fatal("unable to copy all bytes") + } + dst.Close() + fi, err := os.Stat(dst.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != fileSize { + b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) + } + os.Remove(dst.Name()) + } +} + +func BenchmarkCopyDown10MiBDelay10Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 10*time.Millisecond) +} + +func BenchmarkCopyDown10MiBDelay50Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 50*time.Millisecond) +} + +func BenchmarkCopyDown10MiBDelay150Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 150*time.Millisecond) +} + +func benchmarkCopyUp(b *testing.B, fileSize int64, delay time.Duration) { + // Create a temp file and fill it with zero's. + src, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer src.Close() + srcFilename := src.Name() + defer os.Remove(srcFilename) + zero, err := os.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + n, err := io.Copy(src, io.LimitReader(zero, fileSize)) + if err != nil { + b.Fatal(err) + } + if n < fileSize { + b.Fatal("short copy") + } + zero.Close() + src.Close() + + sftp, cmd := testClient(b, false, delay) + defer cmd.Wait() + defer sftp.Close() + + b.ResetTimer() + b.SetBytes(fileSize) + + for i := 0; i < b.N; i++ { + tmp, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + tmp.Close() + defer os.Remove(tmp.Name()) + + dst, err := sftp.Create(tmp.Name()) + if err != nil { + b.Fatal(err) + } + defer dst.Close() + src, err := os.Open(srcFilename) + if err != nil { + b.Fatal(err) + } + defer src.Close() + n, err := io.Copy(dst, src) + if err != nil { + b.Fatalf("copy error: %v", err) + } + if n < fileSize { + b.Fatal("unable to copy all bytes") + } + + fi, err := os.Stat(tmp.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != fileSize { + b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) + } + os.Remove(tmp.Name()) + } +} + +func BenchmarkCopyUp10MiBDelay10Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 10*time.Millisecond) +} + +func BenchmarkCopyUp10MiBDelay50Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 50*time.Millisecond) +} + +func BenchmarkCopyUp10MiBDelay150Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 150*time.Millisecond) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go new file mode 100644 index 0000000..2da179c --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go @@ -0,0 +1,86 @@ +package sftp + +import ( + "io" + "os" + "testing" + + "github.com/kr/fs" +) + +// assert that *Client implements fs.FileSystem +var _ fs.FileSystem = new(Client) + +// assert that *File implements io.ReadWriteCloser +var _ io.ReadWriteCloser = new(File) + +var ok = &StatusError{Code: ssh_FX_OK} +var eof = &StatusError{Code: ssh_FX_EOF} +var fail = &StatusError{Code: ssh_FX_FAILURE} + +var eofOrErrTests = []struct { + err, want error +}{ + {nil, nil}, + {eof, io.EOF}, + {ok, ok}, + {io.EOF, io.EOF}, +} + +func TestEofOrErr(t *testing.T) { + for _, tt := range eofOrErrTests { + got := eofOrErr(tt.err) + if got != tt.want { + t.Errorf("eofOrErr(%#v): want: %#v, got: %#v", tt.err, tt.want, got) + } + } +} + +var okOrErrTests = []struct { + err, want error +}{ + {nil, nil}, + {eof, eof}, + {ok, nil}, + {io.EOF, io.EOF}, +} + +func TestOkOrErr(t *testing.T) { + for _, tt := range okOrErrTests { + got := okOrErr(tt.err) + if got != tt.want { + t.Errorf("okOrErr(%#v): want: %#v, got: %#v", tt.err, tt.want, got) + } + } +} + +var flagsTests = []struct { + flags int + want uint32 +}{ + {os.O_RDONLY, ssh_FXF_READ}, + {os.O_WRONLY, ssh_FXF_WRITE}, + {os.O_RDWR, ssh_FXF_READ | ssh_FXF_WRITE}, + {os.O_RDWR | os.O_CREATE | os.O_TRUNC, ssh_FXF_READ | ssh_FXF_WRITE | ssh_FXF_CREAT | ssh_FXF_TRUNC}, + {os.O_WRONLY | os.O_APPEND, ssh_FXF_WRITE | ssh_FXF_APPEND}, +} + +func TestFlags(t *testing.T) { + for i, tt := range flagsTests { + got := flags(tt.flags) + if got != tt.want { + t.Errorf("test %v: flags(%x): want: %x, got: %x", i, tt.flags, tt.want, got) + } + } +} + +func TestMissingLangTag(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fail() + } + }() + buf := marshalUint32([]byte{}, 0) + buf = marshalStatus(buf, StatusError{}) + _ = unmarshalStatus(0, buf[:len(buf)-4]) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/debug.go b/Godeps/_workspace/src/github.com/pkg/sftp/debug.go new file mode 100644 index 0000000..3e264ab --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/debug.go @@ -0,0 +1,9 @@ +// +build debug + +package sftp + +import "log" + +func debug(fmt string, args ...interface{}) { + log.Printf(fmt, args...) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go new file mode 100644 index 0000000..11dc1e1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go @@ -0,0 +1,90 @@ +package sftp_test + +import ( + "fmt" + "log" + "os" + "os/exec" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +func Example(conn *ssh.Client) { + // open an SFTP session over an existing ssh connection. + sftp, err := sftp.NewClient(conn) + if err != nil { + log.Fatal(err) + } + defer sftp.Close() + + // walk a directory + w := sftp.Walk("/home/user") + for w.Step() { + if w.Err() != nil { + continue + } + log.Println(w.Path()) + } + + // leave your mark + f, err := sftp.Create("hello.txt") + if err != nil { + log.Fatal(err) + } + if _, err := f.Write([]byte("Hello world!")); err != nil { + log.Fatal(err) + } + + // check it's there + fi, err := sftp.Lstat("hello.txt") + if err != nil { + log.Fatal(err) + } + log.Println(fi) +} + +func ExampleNewClientPipe() { + // Connect to a remote host and request the sftp subsystem via the 'ssh' + // command. This assumes that passwordless login is correctly configured. + cmd := exec.Command("ssh", "example.com", "-s", "sftp") + + // send errors from ssh to stderr + cmd.Stderr = os.Stderr + + // get stdin and stdout + wr, err := cmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + rd, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + + // start the process + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + defer cmd.Wait() + + // open the SFTP session + client, err := sftp.NewClientPipe(rd, wr) + if err != nil { + log.Fatal(err) + } + + // read a directory + list, err := client.ReadDir("/") + if err != nil { + log.Fatal(err) + } + + // print contents + for _, item := range list { + fmt.Println(item.Name()) + } + + // close the connection + client.Close() +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-read-benchmark/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-read-benchmark/main.go new file mode 100644 index 0000000..32c4d1f --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-read-benchmark/main.go @@ -0,0 +1,77 @@ +// buffered-read-benchmark benchmarks the peformance of reading +// from /dev/zero on the server to a []byte on the client via io.Copy. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + + "github.com/pkg/sftp" +) + +var ( + USER = flag.String("user", os.Getenv("USER"), "ssh username") + HOST = flag.String("host", "localhost", "ssh server hostname") + PORT = flag.Int("port", 22, "ssh server port") + PASS = flag.String("pass", os.Getenv("SOCKSIE_SSH_PASSWORD"), "ssh password") + SIZE = flag.Int("s", 1<<15, "set max packet size") +) + +func init() { + flag.Parse() +} + +func main() { + var auths []ssh.AuthMethod + if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil { + auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers)) + + } + if *PASS != "" { + auths = append(auths, ssh.Password(*PASS)) + } + + config := ssh.ClientConfig{ + User: *USER, + Auth: auths, + } + addr := fmt.Sprintf("%s:%d", *HOST, *PORT) + conn, err := ssh.Dial("tcp", addr, &config) + if err != nil { + log.Fatalf("unable to connect to [%s]: %v", addr, err) + } + defer conn.Close() + + c, err := sftp.NewClient(conn, sftp.MaxPacket(*SIZE)) + if err != nil { + log.Fatalf("unable to start sftp subsytem: %v", err) + } + defer c.Close() + + r, err := c.Open("/dev/zero") + if err != nil { + log.Fatal(err) + } + defer r.Close() + + const size = 1e9 + + log.Printf("reading %v bytes", size) + t1 := time.Now() + n, err := io.ReadFull(r, make([]byte, size)) + if err != nil { + log.Fatal(err) + } + if n != size { + log.Fatalf("copy: expected %v bytes, got %d", size, n) + } + log.Printf("read %v bytes in %s", size, time.Since(t1)) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-write-benchmark/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-write-benchmark/main.go new file mode 100644 index 0000000..0c38db6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/buffered-write-benchmark/main.go @@ -0,0 +1,83 @@ +// buffered-write-benchmark benchmarks the peformance of writing +// a single large []byte on the client to /dev/null on the server via io.Copy. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "syscall" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + + "github.com/pkg/sftp" +) + +var ( + USER = flag.String("user", os.Getenv("USER"), "ssh username") + HOST = flag.String("host", "localhost", "ssh server hostname") + PORT = flag.Int("port", 22, "ssh server port") + PASS = flag.String("pass", os.Getenv("SOCKSIE_SSH_PASSWORD"), "ssh password") + SIZE = flag.Int("s", 1<<15, "set max packet size") +) + +func init() { + flag.Parse() +} + +func main() { + var auths []ssh.AuthMethod + if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil { + auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers)) + + } + if *PASS != "" { + auths = append(auths, ssh.Password(*PASS)) + } + + config := ssh.ClientConfig{ + User: *USER, + Auth: auths, + } + addr := fmt.Sprintf("%s:%d", *HOST, *PORT) + conn, err := ssh.Dial("tcp", addr, &config) + if err != nil { + log.Fatalf("unable to connect to [%s]: %v", addr, err) + } + defer conn.Close() + + c, err := sftp.NewClient(conn, sftp.MaxPacket(*SIZE)) + if err != nil { + log.Fatalf("unable to start sftp subsytem: %v", err) + } + defer c.Close() + + w, err := c.OpenFile("/dev/null", syscall.O_WRONLY) + if err != nil { + log.Fatal(err) + } + defer w.Close() + + f, err := os.Open("/dev/zero") + if err != nil { + log.Fatal(err) + } + defer f.Close() + + const size = 1e9 + + log.Printf("writing %v bytes", size) + t1 := time.Now() + n, err := w.Write(make([]byte, size)) + if err != nil { + log.Fatal(err) + } + if n != size { + log.Fatalf("copy: expected %v bytes, got %d", size, n) + } + log.Printf("wrote %v bytes in %s", size, time.Since(t1)) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md new file mode 100644 index 0000000..bd96f2d --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md @@ -0,0 +1,12 @@ +Example SFTP server implementation +=== + +In order to use this example you will need an RSA key. + +On linux-like systems with openssh installed, you can use the command: + +``` +ssh-keygen -t rsa -f id_rsa +``` + +Then you will be able to run the sftp-server command in the current directory. diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go new file mode 100644 index 0000000..3dc3393 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go @@ -0,0 +1,135 @@ +// An example SFTP server implementation using the golang SSH package. +// Serves the whole filesystem visible to the user, and has a hard-coded username and password, +// so not for real use! +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "net" + "os" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// Based on example server code from golang.org/x/crypto/ssh and server_standalone +func main() { + + var ( + readOnly bool + debugLevelStr string + debugLevel int + debugStderr bool + rootDir string + ) + + flag.BoolVar(&readOnly, "R", false, "read-only server") + flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.StringVar(&debugLevelStr, "l", "none", "debug level") + flag.StringVar(&rootDir, "root", "", "root directory") + flag.Parse() + + debugStream := ioutil.Discard + if debugStderr { + debugStream = os.Stderr + debugLevel = 1 + } + + // An SSH server is represented by a ServerConfig, which holds + // certificate details and handles authentication of ServerConns. + config := &ssh.ServerConfig{ + PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + // Should use constant-time compare (or better, salt+hash) in + // a production setting. + fmt.Fprintf(debugStream, "Login: %s\n", c.User()) + if c.User() == "testuser" && string(pass) == "tiger" { + return nil, nil + } + return nil, fmt.Errorf("password rejected for %q", c.User()) + }, + } + + privateBytes, err := ioutil.ReadFile("id_rsa") + if err != nil { + log.Fatal("Failed to load private key", err) + } + + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + log.Fatal("Failed to parse private key", err) + } + + config.AddHostKey(private) + + // Once a ServerConfig has been configured, connections can be + // accepted. + listener, err := net.Listen("tcp", "0.0.0.0:2022") + if err != nil { + log.Fatal("failed to listen for connection", err) + } + fmt.Printf("Listening on %v\n", listener.Addr()) + + nConn, err := listener.Accept() + if err != nil { + log.Fatal("failed to accept incoming connection", err) + } + + // Before use, a handshake must be performed on the incoming + // net.Conn. + _, chans, reqs, err := ssh.NewServerConn(nConn, config) + if err != nil { + log.Fatal("failed to handshake", err) + } + fmt.Fprintf(debugStream, "SSH server established\n") + + // The incoming Request channel must be serviced. + go ssh.DiscardRequests(reqs) + + // Service the incoming Channel channel. + for newChannel := range chans { + // Channels have a type, depending on the application level + // protocol intended. In the case of an SFTP session, this is "subsystem" + // with a payload string of "sftp" + fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType()) + if newChannel.ChannelType() != "session" { + newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType()) + continue + } + channel, requests, err := newChannel.Accept() + if err != nil { + log.Fatal("could not accept channel.", err) + } + fmt.Fprintf(debugStream, "Channel accepted\n") + + // Sessions have out-of-band requests such as "shell", + // "pty-req" and "env". Here we handle only the + // "subsystem" request. + go func(in <-chan *ssh.Request) { + for req := range in { + fmt.Fprintf(debugStream, "Request: %v\n", req.Type) + ok := false + switch req.Type { + case "subsystem": + fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:]) + if string(req.Payload[4:]) == "sftp" { + ok = true + } + } + fmt.Fprintf(debugStream, " - accepted: %v\n", ok) + req.Reply(ok, nil) + } + }(requests) + + server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootDir) + if err != nil { + log.Fatal(err) + } + if err := server.Serve(); err != nil { + log.Fatal("sftp server completed with error:", err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-read-benchmark/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-read-benchmark/main.go new file mode 100644 index 0000000..dc901e9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-read-benchmark/main.go @@ -0,0 +1,84 @@ +// streaming-read-benchmark benchmarks the peformance of reading +// from /dev/zero on the server to /dev/null on the client via io.Copy. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + "syscall" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + + "github.com/pkg/sftp" +) + +var ( + USER = flag.String("user", os.Getenv("USER"), "ssh username") + HOST = flag.String("host", "localhost", "ssh server hostname") + PORT = flag.Int("port", 22, "ssh server port") + PASS = flag.String("pass", os.Getenv("SOCKSIE_SSH_PASSWORD"), "ssh password") + SIZE = flag.Int("s", 1<<15, "set max packet size") +) + +func init() { + flag.Parse() +} + +func main() { + var auths []ssh.AuthMethod + if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil { + auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers)) + + } + if *PASS != "" { + auths = append(auths, ssh.Password(*PASS)) + } + + config := ssh.ClientConfig{ + User: *USER, + Auth: auths, + } + addr := fmt.Sprintf("%s:%d", *HOST, *PORT) + conn, err := ssh.Dial("tcp", addr, &config) + if err != nil { + log.Fatalf("unable to connect to [%s]: %v", addr, err) + } + defer conn.Close() + + c, err := sftp.NewClient(conn, sftp.MaxPacket(*SIZE)) + if err != nil { + log.Fatalf("unable to start sftp subsytem: %v", err) + } + defer c.Close() + + r, err := c.Open("/dev/zero") + if err != nil { + log.Fatal(err) + } + defer r.Close() + + w, err := os.OpenFile("/dev/null", syscall.O_WRONLY, 0600) + if err != nil { + log.Fatal(err) + } + defer w.Close() + + const size int64 = 1e9 + + log.Printf("reading %v bytes", size) + t1 := time.Now() + n, err := io.Copy(w, io.LimitReader(r, size)) + if err != nil { + log.Fatal(err) + } + if n != size { + log.Fatalf("copy: expected %v bytes, got %d", size, n) + } + log.Printf("read %v bytes in %s", size, time.Since(t1)) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-write-benchmark/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-write-benchmark/main.go new file mode 100644 index 0000000..07a9ddb --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/streaming-write-benchmark/main.go @@ -0,0 +1,84 @@ +// streaming-write-benchmark benchmarks the peformance of writing +// from /dev/zero on the client to /dev/null on the server via io.Copy. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + "syscall" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + + "github.com/pkg/sftp" +) + +var ( + USER = flag.String("user", os.Getenv("USER"), "ssh username") + HOST = flag.String("host", "localhost", "ssh server hostname") + PORT = flag.Int("port", 22, "ssh server port") + PASS = flag.String("pass", os.Getenv("SOCKSIE_SSH_PASSWORD"), "ssh password") + SIZE = flag.Int("s", 1<<15, "set max packet size") +) + +func init() { + flag.Parse() +} + +func main() { + var auths []ssh.AuthMethod + if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil { + auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers)) + + } + if *PASS != "" { + auths = append(auths, ssh.Password(*PASS)) + } + + config := ssh.ClientConfig{ + User: *USER, + Auth: auths, + } + addr := fmt.Sprintf("%s:%d", *HOST, *PORT) + conn, err := ssh.Dial("tcp", addr, &config) + if err != nil { + log.Fatalf("unable to connect to [%s]: %v", addr, err) + } + defer conn.Close() + + c, err := sftp.NewClient(conn, sftp.MaxPacket(*SIZE)) + if err != nil { + log.Fatalf("unable to start sftp subsytem: %v", err) + } + defer c.Close() + + w, err := c.OpenFile("/dev/null", syscall.O_WRONLY) + if err != nil { + log.Fatal(err) + } + defer w.Close() + + f, err := os.Open("/dev/zero") + if err != nil { + log.Fatal(err) + } + defer f.Close() + + const size int64 = 1e9 + + log.Printf("writing %v bytes", size) + t1 := time.Now() + n, err := io.Copy(w, io.LimitReader(f, size)) + if err != nil { + log.Fatal(err) + } + if n != size { + log.Fatalf("copy: expected %v bytes, got %d", size, n) + } + log.Printf("wrote %v bytes in %s", size, time.Since(t1)) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/packet.go b/Godeps/_workspace/src/github.com/pkg/sftp/packet.go new file mode 100644 index 0000000..5954932 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/packet.go @@ -0,0 +1,829 @@ +package sftp + +import ( + "encoding" + "fmt" + "io" + "os" + "reflect" +) + +var ( + shortPacketError = fmt.Errorf("packet too short") +) + +const ( + debugDumpTxPacket = false + debugDumpRxPacket = false + debugDumpTxPacketBytes = false + debugDumpRxPacketBytes = false +) + +func marshalUint32(b []byte, v uint32) []byte { + return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func marshalUint64(b []byte, v uint64) []byte { + return marshalUint32(marshalUint32(b, uint32(v>>32)), uint32(v)) +} + +func marshalString(b []byte, v string) []byte { + return append(marshalUint32(b, uint32(len(v))), v...) +} + +func marshal(b []byte, v interface{}) []byte { + if v == nil { + return b + } + switch v := v.(type) { + case uint8: + return append(b, v) + case uint32: + return marshalUint32(b, v) + case uint64: + return marshalUint64(b, v) + case string: + return marshalString(b, v) + case os.FileInfo: + return marshalFileInfo(b, v) + default: + switch d := reflect.ValueOf(v); d.Kind() { + case reflect.Struct: + for i, n := 0, d.NumField(); i < n; i++ { + b = append(marshal(b, d.Field(i).Interface())) + } + return b + case reflect.Slice: + for i, n := 0, d.Len(); i < n; i++ { + b = append(marshal(b, d.Index(i).Interface())) + } + return b + default: + panic(fmt.Sprintf("marshal(%#v): cannot handle type %T", v, v)) + } + } +} + +func unmarshalUint32(b []byte) (uint32, []byte) { + v := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 + return v, b[4:] +} + +func unmarshalUint32Safe(b []byte) (uint32, []byte, error) { + var v uint32 = 0 + if len(b) < 4 { + return 0, nil, shortPacketError + } + v, b = unmarshalUint32(b) + return v, b, nil +} + +func unmarshalUint64(b []byte) (uint64, []byte) { + h, b := unmarshalUint32(b) + l, b := unmarshalUint32(b) + return uint64(h)<<32 | uint64(l), b +} + +func unmarshalUint64Safe(b []byte) (uint64, []byte, error) { + var v uint64 = 0 + if len(b) < 8 { + return 0, nil, shortPacketError + } + v, b = unmarshalUint64(b) + return v, b, nil +} + +func unmarshalString(b []byte) (string, []byte) { + n, b := unmarshalUint32(b) + return string(b[:n]), b[n:] +} + +func unmarshalStringSafe(b []byte) (string, []byte, error) { + n, b, err := unmarshalUint32Safe(b) + if err != nil { + return "", nil, err + } + if int64(n) > int64(len(b)) { + return "", nil, shortPacketError + } + return string(b[:n]), b[n:], nil +} + +// sendPacket marshals p according to RFC 4234. +func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error { + bb, err := m.MarshalBinary() + if err != nil { + return fmt.Errorf("marshal2(%#v): binary marshaller failed", err) + } + if debugDumpTxPacketBytes { + debug("send packet: %s %d bytes %x", fxp(bb[0]), len(bb), bb[1:]) + } else if debugDumpTxPacket { + debug("send packet: %s %d bytes", fxp(bb[0]), len(bb)) + } + l := uint32(len(bb)) + hdr := []byte{byte(l >> 24), byte(l >> 16), byte(l >> 8), byte(l)} + _, err = w.Write(hdr) + if err != nil { + return err + } + _, err = w.Write(bb) + return err +} + +func (svr *Server) sendPacket(m encoding.BinaryMarshaler) error { + // any responder can call sendPacket(); actual socket access must be serialized + svr.outMutex.Lock() + defer svr.outMutex.Unlock() + return sendPacket(svr.out, m) +} + +func recvPacket(r io.Reader) (uint8, []byte, error) { + var b = []byte{0, 0, 0, 0} + if _, err := io.ReadFull(r, b); err != nil { + return 0, nil, err + } + l, _ := unmarshalUint32(b) + b = make([]byte, l) + if _, err := io.ReadFull(r, b); err != nil { + debug("recv packet %d bytes: err %v", l, err) + return 0, nil, err + } + if debugDumpRxPacketBytes { + debug("recv packet: %s %d bytes %x", fxp(b[0]), l, b[1:]) + } else if debugDumpRxPacket { + debug("recv packet: %s %d bytes", fxp(b[0]), l) + } + return b[0], b[1:], nil +} + +type ExtensionPair struct { + Name string + Data string +} + +func unmarshalExtensionPair(b []byte) (ExtensionPair, []byte, error) { + ep := ExtensionPair{} + var err error = nil + ep.Name, b, err = unmarshalStringSafe(b) + if err != nil { + return ep, b, err + } + ep.Data, b, err = unmarshalStringSafe(b) + if err != nil { + return ep, b, err + } + return ep, b, err +} + +// Here starts the definition of packets along with their MarshalBinary +// implementations. +// Manually writing the marshalling logic wins us a lot of time and +// allocation. + +type sshFxInitPacket struct { + Version uint32 + Extensions []ExtensionPair +} + +func (p sshFxInitPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 // byte + uint32 + for _, e := range p.Extensions { + l += 4 + len(e.Name) + 4 + len(e.Data) + } + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_INIT) + b = marshalUint32(b, p.Version) + for _, e := range p.Extensions { + b = marshalString(b, e.Name) + b = marshalString(b, e.Data) + } + return b, nil +} + +func (p *sshFxInitPacket) UnmarshalBinary(b []byte) (err error) { + if p.Version, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + for len(b) > 0 { + ep := ExtensionPair{} + ep, b, err = unmarshalExtensionPair(b) + if err != nil { + return err + } + p.Extensions = append(p.Extensions, ep) + } + return nil +} + +type sshFxVersionPacket struct { + Version uint32 + Extensions []struct { + Name, Data string + } +} + +func (p sshFxVersionPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 // byte + uint32 + for _, e := range p.Extensions { + l += 4 + len(e.Name) + 4 + len(e.Data) + } + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_VERSION) + b = marshalUint32(b, p.Version) + for _, e := range p.Extensions { + b = marshalString(b, e.Name) + b = marshalString(b, e.Data) + } + return b, nil +} + +func marshalIdString(packetType byte, id uint32, str string) ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(str) + + b := make([]byte, 0, l) + b = append(b, packetType) + b = marshalUint32(b, id) + b = marshalString(b, str) + return b, nil +} + +func unmarshalIdString(b []byte, id *uint32, str *string) error { + var err error = nil + *id, b, err = unmarshalUint32Safe(b) + if err != nil { + return err + } + *str, b, err = unmarshalStringSafe(b) + if err != nil { + return err + } + return nil +} + +type sshFxpReaddirPacket struct { + Id uint32 + Handle string +} + +func (p sshFxpReaddirPacket) id() uint32 { return p.Id } + +func (p sshFxpReaddirPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_READDIR, p.Id, p.Handle) +} + +func (p *sshFxpReaddirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Handle) +} + +type sshFxpOpendirPacket struct { + Id uint32 + Path string +} + +func (p sshFxpOpendirPacket) id() uint32 { return p.Id } + +func (p sshFxpOpendirPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_OPENDIR, p.Id, p.Path) +} + +func (p *sshFxpOpendirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpLstatPacket struct { + Id uint32 + Path string +} + +func (p sshFxpLstatPacket) id() uint32 { return p.Id } + +func (p sshFxpLstatPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_LSTAT, p.Id, p.Path) +} + +func (p *sshFxpLstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpStatPacket struct { + Id uint32 + Path string +} + +func (p sshFxpStatPacket) id() uint32 { return p.Id } + +func (p sshFxpStatPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_LSTAT, p.Id, p.Path) +} + +func (p *sshFxpStatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpFstatPacket struct { + Id uint32 + Handle string +} + +func (p sshFxpFstatPacket) id() uint32 { return p.Id } + +func (p sshFxpFstatPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_FSTAT, p.Id, p.Handle) +} + +func (p *sshFxpFstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Handle) +} + +type sshFxpClosePacket struct { + Id uint32 + Handle string +} + +func (p sshFxpClosePacket) id() uint32 { return p.Id } + +func (p sshFxpClosePacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_CLOSE, p.Id, p.Handle) +} + +func (p *sshFxpClosePacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Handle) +} + +type sshFxpRemovePacket struct { + Id uint32 + Filename string +} + +func (p sshFxpRemovePacket) id() uint32 { return p.Id } + +func (p sshFxpRemovePacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_REMOVE, p.Id, p.Filename) +} + +func (p *sshFxpRemovePacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Filename) +} + +type sshFxpRmdirPacket struct { + Id uint32 + Path string +} + +func (p sshFxpRmdirPacket) id() uint32 { return p.Id } + +func (p sshFxpRmdirPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_RMDIR, p.Id, p.Path) +} + +func (p *sshFxpRmdirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpSymlinkPacket struct { + Id uint32 + Targetpath string + Linkpath string +} + +func (p sshFxpSymlinkPacket) id() uint32 { return p.Id } + +func (p sshFxpSymlinkPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Targetpath) + + 4 + len(p.Linkpath) + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_SYMLINK) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Targetpath) + b = marshalString(b, p.Linkpath) + return b, nil +} + +func (p *sshFxpSymlinkPacket) UnmarshalBinary(b []byte) error { + var err error = nil + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Targetpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Linkpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +type sshFxpReadlinkPacket struct { + Id uint32 + Path string +} + +func (p sshFxpReadlinkPacket) id() uint32 { return p.Id } + +func (p sshFxpReadlinkPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_READLINK, p.Id, p.Path) +} + +func (p *sshFxpReadlinkPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpRealpathPacket struct { + Id uint32 + Path string +} + +func (p sshFxpRealpathPacket) id() uint32 { return p.Id } + +func (p sshFxpRealpathPacket) MarshalBinary() ([]byte, error) { + return marshalIdString(ssh_FXP_READLINK, p.Id, p.Path) +} + +func (p *sshFxpRealpathPacket) UnmarshalBinary(b []byte) error { + return unmarshalIdString(b, &p.Id, &p.Path) +} + +type sshFxpNameAttr struct { + Name string + LongName string + Attrs []interface{} +} + +func (p sshFxpNameAttr) MarshalBinary() ([]byte, error) { + b := []byte{} + b = marshalString(b, p.Name) + b = marshalString(b, p.LongName) + for _, attr := range p.Attrs { + b = marshal(b, attr) + } + return b, nil +} + +type sshFxpNamePacket struct { + Id uint32 + NameAttrs []sshFxpNameAttr +} + +func (p sshFxpNamePacket) MarshalBinary() ([]byte, error) { + b := []byte{} + b = append(b, ssh_FXP_NAME) + b = marshalUint32(b, p.Id) + b = marshalUint32(b, uint32(len(p.NameAttrs))) + for _, na := range p.NameAttrs { + if ab, err := na.MarshalBinary(); err != nil { + return nil, err + } else { + b = append(b, ab...) + } + } + return b, nil +} + +type sshFxpOpenPacket struct { + Id uint32 + Path string + Pflags uint32 + Flags uint32 // ignored +} + +func (p sshFxpOpenPacket) id() uint32 { return p.Id } + +func (p sshFxpOpenPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + + 4 + len(p.Path) + + 4 + 4 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_OPEN) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Pflags) + b = marshalUint32(b, p.Flags) + return b, nil +} + +func (p *sshFxpOpenPacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return + } else if p.Pflags, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return + } + return +} + +type sshFxpReadPacket struct { + Id uint32 + Handle string + Offset uint64 + Len uint32 +} + +func (p sshFxpReadPacket) id() uint32 { return p.Id } + +func (p sshFxpReadPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Handle) + + 8 + 4 // uint64 + uint32 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_READ) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Handle) + b = marshalUint64(b, p.Offset) + b = marshalUint32(b, p.Len) + return b, nil +} + +func (p *sshFxpReadPacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return + } else if p.Len, b, err = unmarshalUint32Safe(b); err != nil { + return + } + return +} + +type sshFxpRenamePacket struct { + Id uint32 + Oldpath string + Newpath string +} + +func (p sshFxpRenamePacket) id() uint32 { return p.Id } + +func (p sshFxpRenamePacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Oldpath) + + 4 + len(p.Newpath) + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_RENAME) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Oldpath) + b = marshalString(b, p.Newpath) + return b, nil +} + +func (p *sshFxpRenamePacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if p.Oldpath, b, err = unmarshalStringSafe(b); err != nil { + return + } else if p.Newpath, b, err = unmarshalStringSafe(b); err != nil { + return + } + return +} + +type sshFxpWritePacket struct { + Id uint32 + Handle string + Offset uint64 + Length uint32 + Data []byte +} + +func (s sshFxpWritePacket) id() uint32 { return s.Id } + +func (s sshFxpWritePacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(s.Handle) + + 8 + 4 + // uint64 + uint32 + len(s.Data) + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_WRITE) + b = marshalUint32(b, s.Id) + b = marshalString(b, s.Handle) + b = marshalUint64(b, s.Offset) + b = marshalUint32(b, s.Length) + b = append(b, s.Data...) + return b, nil +} + +func (p *sshFxpWritePacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return + } else if uint32(len(b)) < p.Length { + err = shortPacketError + return + } else { + p.Data = append([]byte{}, b[:p.Length]...) + } + return +} + +type sshFxpMkdirPacket struct { + Id uint32 + Path string + Flags uint32 // ignored +} + +func (p sshFxpMkdirPacket) id() uint32 { return p.Id } + +func (p sshFxpMkdirPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Path) + + 4 // uint32 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_MKDIR) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Flags) + return b, nil +} + +func (p *sshFxpMkdirPacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + +type sshFxpSetstatPacket struct { + Id uint32 + Path string + Flags uint32 + Attrs interface{} +} + +type sshFxpFsetstatPacket struct { + Id uint32 + Handle string + Flags uint32 + Attrs interface{} +} + +func (p sshFxpSetstatPacket) id() uint32 { return p.Id } +func (p sshFxpFsetstatPacket) id() uint32 { return p.Id } + +func (p sshFxpSetstatPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Path) + + 4 // uint32 + uint64 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_SETSTAT) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Flags) + b = marshal(b, p.Attrs) + return b, nil +} + +func (p sshFxpFsetstatPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Handle) + + 4 // uint32 + uint64 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_FSETSTAT) + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Handle) + b = marshalUint32(b, p.Flags) + b = marshal(b, p.Attrs) + return b, nil +} + +func (p *sshFxpSetstatPacket) UnmarshalBinary(b []byte) error { + var err error = nil + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +func (p *sshFxpFsetstatPacket) UnmarshalBinary(b []byte) error { + var err error = nil + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +type sshFxpHandlePacket struct { + Id uint32 + Handle string +} + +func (p sshFxpHandlePacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_HANDLE} + b = marshalUint32(b, p.Id) + b = marshalString(b, p.Handle) + return b, nil +} + +type sshFxpStatusPacket struct { + Id uint32 + StatusError +} + +func (p sshFxpStatusPacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_STATUS} + b = marshalUint32(b, p.Id) + b = marshalStatus(b, p.StatusError) + return b, nil +} + +type sshFxpDataPacket struct { + Id uint32 + Length uint32 + Data []byte +} + +func (p sshFxpDataPacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_DATA} + b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.Length) + b = append(b, p.Data[:p.Length]...) + return b, nil +} + +func (p *sshFxpDataPacket) UnmarshalBinary(b []byte) (err error) { + if p.Id, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if uint32(len(b)) < p.Length { + return fmt.Errorf("truncated packet") + } else { + p.Data = make([]byte, p.Length) + copy(p.Data, b) + return nil + } +} + +type sshFxpStatvfsPacket struct { + Id uint32 + Path string +} + +func (p sshFxpStatvfsPacket) id() uint32 { return p.Id } + +func (p sshFxpStatvfsPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + len(p.Path) + + len("statvfs@openssh.com") + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_EXTENDED) + b = marshalUint32(b, p.Id) + b = marshalString(b, "statvfs@openssh.com") + b = marshalString(b, p.Path) + return b, nil +} + +type StatVFS struct { + Id uint32 + Bsize uint64 /* file system block size */ + Frsize uint64 /* fundamental fs block size */ + Blocks uint64 /* number of blocks (unit f_frsize) */ + Bfree uint64 /* free blocks in file system */ + Bavail uint64 /* free blocks for non-root */ + Files uint64 /* total file inodes */ + Ffree uint64 /* free file inodes */ + Favail uint64 /* free file inodes for to non-root */ + Fsid uint64 /* file system id */ + Flag uint64 /* bit mask of f_flag values */ + Namemax uint64 /* maximum filename length */ +} + +func (p *StatVFS) TotalSpace() uint64 { + return p.Frsize * p.Blocks +} + +func (p *StatVFS) FreeSpace() uint64 { + return p.Frsize * p.Bfree +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go new file mode 100644 index 0000000..88a1522 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go @@ -0,0 +1,261 @@ +package sftp + +import ( + "bytes" + "encoding" + "os" + "testing" +) + +var marshalUint32Tests = []struct { + v uint32 + want []byte +}{ + {1, []byte{0, 0, 0, 1}}, + {256, []byte{0, 0, 1, 0}}, + {^uint32(0), []byte{255, 255, 255, 255}}, +} + +func TestMarshalUint32(t *testing.T) { + for _, tt := range marshalUint32Tests { + got := marshalUint32(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalUint32(%d): want %v, got %v", tt.v, tt.want, got) + } + } +} + +var marshalUint64Tests = []struct { + v uint64 + want []byte +}{ + {1, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + {256, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0}}, + {^uint64(0), []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {1 << 32, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, +} + +func TestMarshalUint64(t *testing.T) { + for _, tt := range marshalUint64Tests { + got := marshalUint64(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalUint64(%d): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var marshalStringTests = []struct { + v string + want []byte +}{ + {"", []byte{0, 0, 0, 0}}, + {"/foo", []byte{0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f}}, +} + +func TestMarshalString(t *testing.T) { + for _, tt := range marshalStringTests { + got := marshalString(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalString(%q): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var marshalTests = []struct { + v interface{} + want []byte +}{ + {uint8(1), []byte{1}}, + {byte(1), []byte{1}}, + {uint32(1), []byte{0, 0, 0, 1}}, + {uint64(1), []byte{0, 0, 0, 0, 0, 0, 0, 1}}, + {"foo", []byte{0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f}}, + {[]uint32{1, 2, 3, 4}, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x4}}, +} + +func TestMarshal(t *testing.T) { + for _, tt := range marshalTests { + got := marshal(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshal(%v): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var unmarshalUint32Tests = []struct { + b []byte + want uint32 + rest []byte +}{ + {[]byte{0, 0, 0, 0}, 0, nil}, + {[]byte{0, 0, 1, 0}, 256, nil}, + {[]byte{255, 0, 0, 255}, 4278190335, nil}, +} + +func TestUnmarshalUint32(t *testing.T) { + for _, tt := range unmarshalUint32Tests { + got, rest := unmarshalUint32(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint32(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var unmarshalUint64Tests = []struct { + b []byte + want uint64 + rest []byte +}{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 0}, 0, nil}, + {[]byte{0, 0, 0, 0, 0, 0, 1, 0}, 256, nil}, + {[]byte{255, 0, 0, 0, 0, 0, 0, 255}, 18374686479671623935, nil}, +} + +func TestUnmarshalUint64(t *testing.T) { + for _, tt := range unmarshalUint64Tests { + got, rest := unmarshalUint64(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint64(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var unmarshalStringTests = []struct { + b []byte + want string + rest []byte +}{ + {marshalString(nil, ""), "", nil}, + {marshalString(nil, "blah"), "blah", nil}, +} + +func TestUnmarshalString(t *testing.T) { + for _, tt := range unmarshalStringTests { + got, rest := unmarshalString(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint64(%v): want %q, %#v, got %q, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var sendPacketTests = []struct { + p encoding.BinaryMarshaler + want []byte +}{ + {sshFxInitPacket{ + Version: 3, + Extensions: []ExtensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }, []byte{0x0, 0x0, 0x0, 0x26, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, + + {sshFxpOpenPacket{ + Id: 1, + Path: "/foo", + Pflags: flags(os.O_RDONLY), + }, []byte{0x0, 0x0, 0x0, 0x15, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, + + {sshFxpWritePacket{ + Id: 124, + Handle: "foo", + Offset: 13, + Length: uint32(len([]byte("bar"))), + Data: []byte("bar"), + }, []byte{0x0, 0x0, 0x0, 0x1b, 0x6, 0x0, 0x0, 0x0, 0x7c, 0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x0, 0x0, 0x3, 0x62, 0x61, 0x72}}, + + {sshFxpSetstatPacket{ + Id: 31, + Path: "/bar", + Flags: flags(os.O_WRONLY), + Attrs: struct { + Uid uint32 + Gid uint32 + }{1000, 100}, + }, []byte{0x0, 0x0, 0x0, 0x19, 0x9, 0x0, 0x0, 0x0, 0x1f, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x62, 0x61, 0x72, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x3, 0xe8, 0x0, 0x0, 0x0, 0x64}}, +} + +func TestSendPacket(t *testing.T) { + for _, tt := range sendPacketTests { + var w bytes.Buffer + sendPacket(&w, tt.p) + if got := w.Bytes(); !bytes.Equal(tt.want, got) { + t.Errorf("sendPacket(%v): want %#v, got %#v", tt.p, tt.want, got) + } + } +} + +func sp(p encoding.BinaryMarshaler) []byte { + var w bytes.Buffer + sendPacket(&w, p) + return w.Bytes() +} + +var recvPacketTests = []struct { + b []byte + want uint8 + rest []byte +}{ + {sp(sshFxInitPacket{ + Version: 3, + Extensions: []ExtensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }), ssh_FXP_INIT, []byte{0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, +} + +func TestRecvPacket(t *testing.T) { + for _, tt := range recvPacketTests { + r := bytes.NewReader(tt.b) + got, rest, _ := recvPacket(r) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("recvPacket(%#v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +func BenchmarkMarshalInit(b *testing.B) { + for i := 0; i < b.N; i++ { + sp(sshFxInitPacket{ + Version: 3, + Extensions: []ExtensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }) + } +} + +func BenchmarkMarshalOpen(b *testing.B) { + for i := 0; i < b.N; i++ { + sp(sshFxpOpenPacket{ + Id: 1, + Path: "/home/test/some/random/path", + Pflags: flags(os.O_RDONLY), + }) + } +} + +func BenchmarkMarshalWriteWorstCase(b *testing.B) { + data := make([]byte, 32*1024) + for i := 0; i < b.N; i++ { + sp(sshFxpWritePacket{ + Id: 1, + Handle: "someopaquehandle", + Offset: 0, + Length: uint32(len(data)), + Data: data, + }) + } +} + +func BenchmarkMarshalWrite1k(b *testing.B) { + data := make([]byte, 1024) + for i := 0; i < b.N; i++ { + sp(sshFxpWritePacket{ + Id: 1, + Handle: "someopaquehandle", + Offset: 0, + Length: uint32(len(data)), + Data: data, + }) + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/release.go b/Godeps/_workspace/src/github.com/pkg/sftp/release.go new file mode 100644 index 0000000..b695528 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/release.go @@ -0,0 +1,5 @@ +// +build !debug + +package sftp + +func debug(fmt string, args ...interface{}) {} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server.go b/Godeps/_workspace/src/github.com/pkg/sftp/server.go new file mode 100644 index 0000000..3b6c917 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server.go @@ -0,0 +1,565 @@ +package sftp + +// sftp server counterpart + +import ( + "encoding" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "syscall" + "time" +) + +const ( + sftpServerWorkerCount = 8 +) + +// Server is an SSH File Transfer Protocol (sftp) server. +// This is intended to provide the sftp subsystem to an ssh server daemon. +// This implementation currently supports most of sftp server protocol version 3, +// as specified at http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 +type Server struct { + in io.Reader + out io.WriteCloser + outMutex *sync.Mutex + debugStream io.Writer + debugLevel int + readOnly bool + rootDir string + lastId uint32 + pktChan chan rxPacket + openFiles map[string]*os.File + openFilesLock *sync.RWMutex + handleCount int + maxTxPacket uint32 + workerCount int +} + +func (svr *Server) nextHandle(f *os.File) string { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + svr.handleCount++ + handle := fmt.Sprintf("%d", svr.handleCount) + svr.openFiles[handle] = f + return handle +} + +func (svr *Server) closeHandle(handle string) error { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + if f, ok := svr.openFiles[handle]; ok { + delete(svr.openFiles, handle) + return f.Close() + } else { + return syscall.EBADF + } +} + +func (svr *Server) getHandle(handle string) (*os.File, bool) { + svr.openFilesLock.RLock() + defer svr.openFilesLock.RUnlock() + f, ok := svr.openFiles[handle] + return f, ok +} + +type serverRespondablePacket interface { + encoding.BinaryUnmarshaler + respond(svr *Server) error +} + +// Creates a new server instance around the provided streams. +// Various debug output will be written to debugStream, with verbosity set by debugLevel +// A subsequent call to Serve() is required. +func NewServer(in io.Reader, out io.WriteCloser, debugStream io.Writer, debugLevel int, readOnly bool, rootDir string) (*Server, error) { + if rootDir == "" { + if wd, err := os.Getwd(); err != nil { + return nil, err + } else { + rootDir = wd + } + } + return &Server{ + in: in, + out: out, + outMutex: &sync.Mutex{}, + debugStream: debugStream, + debugLevel: debugLevel, + readOnly: readOnly, + rootDir: rootDir, + pktChan: make(chan rxPacket, sftpServerWorkerCount), + openFiles: map[string]*os.File{}, + openFilesLock: &sync.RWMutex{}, + maxTxPacket: 1 << 15, + workerCount: sftpServerWorkerCount, + }, nil +} + +type rxPacket struct { + pktType fxp + pktBytes []byte +} + +// Unmarshal a single logical packet from the secure channel +func (svr *Server) rxPackets() error { + defer close(svr.pktChan) + + for { + pktType, pktBytes, err := recvPacket(svr.in) + if err == io.EOF { + fmt.Fprintf(svr.debugStream, "rxPackets loop done\n") + return nil + } else if err != nil { + fmt.Fprintf(svr.debugStream, "recvPacket error: %v\n", err) + return err + } + + svr.pktChan <- rxPacket{fxp(pktType), pktBytes} + } +} + +// Up to N parallel servers +func (svr *Server) sftpServerWorker(doneChan chan error) { + for pkt := range svr.pktChan { + if pkt, err := svr.decodePacket(pkt.pktType, pkt.pktBytes); err != nil { + fmt.Fprintf(svr.debugStream, "decodePacket error: %v\n", err) + doneChan <- err + return + } else { + //fmt.Fprintf(svr.debugStream, "pkt: %T %v\n", pkt, pkt) + pkt.respond(svr) + } + } + doneChan <- nil +} + +// Run this server until the streams stop or until the subsystem is stopped +func (svr *Server) Serve() error { + go svr.rxPackets() + doneChan := make(chan error) + for i := 0; i < svr.workerCount; i++ { + go svr.sftpServerWorker(doneChan) + } + for i := 0; i < svr.workerCount; i++ { + if err := <-doneChan; err != nil { + // abort early and shut down the session on un-decodable packets + break + } + } + fmt.Fprintf(svr.debugStream, "sftp server run finished\n") + // close any still-open files + for handle, file := range svr.openFiles { + fmt.Fprintf(svr.debugStream, "sftp server file with handle '%v' left open: %v\n", handle, file.Name()) + file.Close() + } + return svr.out.Close() +} + +func (svr *Server) decodePacket(pktType fxp, pktBytes []byte) (serverRespondablePacket, error) { + //pktId, restBytes := unmarshalUint32(pktBytes[1:]) + var pkt serverRespondablePacket = nil + switch pktType { + case ssh_FXP_INIT: + pkt = &sshFxInitPacket{} + case ssh_FXP_LSTAT: + pkt = &sshFxpLstatPacket{} + case ssh_FXP_OPEN: + pkt = &sshFxpOpenPacket{} + case ssh_FXP_CLOSE: + pkt = &sshFxpClosePacket{} + case ssh_FXP_READ: + pkt = &sshFxpReadPacket{} + case ssh_FXP_WRITE: + pkt = &sshFxpWritePacket{} + case ssh_FXP_FSTAT: + pkt = &sshFxpFstatPacket{} + case ssh_FXP_SETSTAT: + pkt = &sshFxpSetstatPacket{} + case ssh_FXP_FSETSTAT: + pkt = &sshFxpFsetstatPacket{} + case ssh_FXP_OPENDIR: + pkt = &sshFxpOpendirPacket{} + case ssh_FXP_READDIR: + pkt = &sshFxpReaddirPacket{} + case ssh_FXP_REMOVE: + pkt = &sshFxpRemovePacket{} + case ssh_FXP_MKDIR: + pkt = &sshFxpMkdirPacket{} + case ssh_FXP_RMDIR: + pkt = &sshFxpRmdirPacket{} + case ssh_FXP_REALPATH: + pkt = &sshFxpRealpathPacket{} + case ssh_FXP_STAT: + pkt = &sshFxpStatPacket{} + case ssh_FXP_RENAME: + pkt = &sshFxpRenamePacket{} + case ssh_FXP_READLINK: + pkt = &sshFxpReadlinkPacket{} + case ssh_FXP_SYMLINK: + pkt = &sshFxpSymlinkPacket{} + default: + return nil, fmt.Errorf("unhandled packet type: %s", pktType.String()) + } + if pkt == nil { + return nil, fmt.Errorf("unhandled packet type: %s", pktType.String()) + } + if err := pkt.UnmarshalBinary(pktBytes); err != nil { + return nil, err + } + return pkt, nil +} + +func (p sshFxInitPacket) respond(svr *Server) error { + return svr.sendPacket(sshFxVersionPacket{sftpProtocolVersion, nil}) +} + +type sshFxpStatResponse struct { + Id uint32 + info os.FileInfo +} + +func (p sshFxpStatResponse) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_ATTRS} + b = marshalUint32(b, p.Id) + b = marshalFileInfo(b, p.info) + return b, nil +} + +func (p sshFxpLstatPacket) respond(svr *Server) error { + // stat the requested file + if info, err := os.Lstat(p.Path); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + return svr.sendPacket(sshFxpStatResponse{p.Id, info}) + } +} + +func (p sshFxpStatPacket) respond(svr *Server) error { + // stat the requested file + if info, err := os.Stat(p.Path); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + return svr.sendPacket(sshFxpStatResponse{p.Id, info}) + } +} + +func (p sshFxpFstatPacket) respond(svr *Server) error { + if f, ok := svr.getHandle(p.Handle); !ok { + return svr.sendPacket(statusFromError(p.Id, syscall.EBADF)) + } else if info, err := f.Stat(); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + return svr.sendPacket(sshFxpStatResponse{p.Id, info}) + } +} + +func (p sshFxpMkdirPacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + // TODO FIXME: ignore flags field + err := os.Mkdir(p.Path, 0755) + return svr.sendPacket(statusFromError(p.Id, err)) +} + +func (p sshFxpRmdirPacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + err := os.Remove(p.Path) + return svr.sendPacket(statusFromError(p.Id, err)) +} + +func (p sshFxpRemovePacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + err := os.Remove(p.Filename) + return svr.sendPacket(statusFromError(p.Id, err)) +} + +func (p sshFxpRenamePacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + err := os.Rename(p.Oldpath, p.Newpath) + return svr.sendPacket(statusFromError(p.Id, err)) +} + +func (p sshFxpSymlinkPacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + err := os.Symlink(p.Targetpath, p.Linkpath) + return svr.sendPacket(statusFromError(p.Id, err)) +} + +var emptyFileStat = []interface{}{uint32(0)} + +func (p sshFxpReadlinkPacket) respond(svr *Server) error { + if f, err := os.Readlink(p.Path); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + return svr.sendPacket(sshFxpNamePacket{p.Id, []sshFxpNameAttr{sshFxpNameAttr{f, f, emptyFileStat}}}) + } +} + +func (p sshFxpRealpathPacket) respond(svr *Server) error { + if f, err := filepath.Abs(p.Path); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + f = filepath.Clean(f) + return svr.sendPacket(sshFxpNamePacket{p.Id, []sshFxpNameAttr{sshFxpNameAttr{f, f, emptyFileStat}}}) + } +} + +func (p sshFxpOpendirPacket) respond(svr *Server) error { + return sshFxpOpenPacket{p.Id, p.Path, ssh_FXF_READ, 0}.respond(svr) +} + +func (p sshFxpOpenPacket) respond(svr *Server) error { + osFlags := 0 + if p.Pflags&ssh_FXF_READ != 0 && p.Pflags&ssh_FXF_WRITE != 0 { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + osFlags |= os.O_RDWR + } else if p.Pflags&ssh_FXF_WRITE != 0 { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + osFlags |= os.O_WRONLY + } else if p.Pflags&ssh_FXF_READ != 0 { + osFlags |= os.O_RDONLY + } else { + // how are they opening? + return svr.sendPacket(statusFromError(p.Id, syscall.EINVAL)) + + } + + if p.Pflags&ssh_FXF_APPEND != 0 { + osFlags |= os.O_APPEND + } + if p.Pflags&ssh_FXF_CREAT != 0 { + osFlags |= os.O_CREATE + } + if p.Pflags&ssh_FXF_TRUNC != 0 { + osFlags |= os.O_TRUNC + } + if p.Pflags&ssh_FXF_EXCL != 0 { + osFlags |= os.O_EXCL + } + + if f, err := os.OpenFile(p.Path, osFlags, 0644); err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + handle := svr.nextHandle(f) + return svr.sendPacket(sshFxpHandlePacket{p.Id, handle}) + } +} + +func (p sshFxpClosePacket) respond(svr *Server) error { + return svr.sendPacket(statusFromError(p.Id, svr.closeHandle(p.Handle))) +} + +func (p sshFxpReadPacket) respond(svr *Server) error { + if f, ok := svr.getHandle(p.Handle); !ok { + return svr.sendPacket(statusFromError(p.Id, syscall.EBADF)) + } else { + if p.Len > svr.maxTxPacket { + p.Len = svr.maxTxPacket + } + ret := sshFxpDataPacket{Id: p.Id, Length: p.Len, Data: make([]byte, p.Len)} + if n, err := f.ReadAt(ret.Data, int64(p.Offset)); err != nil && (err != io.EOF || n == 0) { + return svr.sendPacket(statusFromError(p.Id, err)) + } else { + ret.Length = uint32(n) + return svr.sendPacket(ret) + } + } +} + +func (p sshFxpWritePacket) respond(svr *Server) error { + if svr.readOnly { + // shouldn't really get here, the open should have failed + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } + if f, ok := svr.getHandle(p.Handle); !ok { + return svr.sendPacket(statusFromError(p.Id, syscall.EBADF)) + } else { + _, err := f.WriteAt(p.Data, int64(p.Offset)) + return svr.sendPacket(statusFromError(p.Id, err)) + } +} + +func (p sshFxpReaddirPacket) respond(svr *Server) error { + if f, ok := svr.getHandle(p.Handle); !ok { + return svr.sendPacket(statusFromError(p.Id, syscall.EBADF)) + } else { + dirname := "" + dirents := []os.FileInfo{} + var err error = nil + + dirname = f.Name() + dirents, err = f.Readdir(128) + if err != nil { + return svr.sendPacket(statusFromError(p.Id, err)) + } + + ret := sshFxpNamePacket{p.Id, nil} + for _, dirent := range dirents { + ret.NameAttrs = append(ret.NameAttrs, sshFxpNameAttr{ + dirent.Name(), + runLs(dirname, dirent), + []interface{}{dirent}, + }) + } + return svr.sendPacket(ret) + } +} + +func (p sshFxpSetstatPacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } else { + // additional unmarshalling is required for each possibility here + b := p.Attrs.([]byte) + var err error = nil + + debug("setstat name \"%s\"", p.Path) + if (p.Flags & ssh_FILEXFER_ATTR_SIZE) != 0 { + var size uint64 = 0 + if size, b, err = unmarshalUint64Safe(b); err == nil { + err = os.Truncate(p.Path, int64(size)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_PERMISSIONS) != 0 { + var mode uint32 = 0 + if mode, b, err = unmarshalUint32Safe(b); err == nil { + err = os.Chmod(p.Path, os.FileMode(mode)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_ACMODTIME) != 0 { + var atime uint32 = 0 + var mtime uint32 = 0 + if atime, b, err = unmarshalUint32Safe(b); err != nil { + } else if mtime, b, err = unmarshalUint32Safe(b); err != nil { + } else { + atimeT := time.Unix(int64(atime), 0) + mtimeT := time.Unix(int64(mtime), 0) + err = os.Chtimes(p.Path, atimeT, mtimeT) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_UIDGID) != 0 { + var uid uint32 = 0 + var gid uint32 = 0 + if uid, b, err = unmarshalUint32Safe(b); err != nil { + } else if gid, b, err = unmarshalUint32Safe(b); err != nil { + } else { + err = os.Chown(p.Path, int(uid), int(gid)) + } + } + + return svr.sendPacket(statusFromError(p.Id, err)) + } +} + +func (p sshFxpFsetstatPacket) respond(svr *Server) error { + if svr.readOnly { + return svr.sendPacket(statusFromError(p.Id, syscall.EPERM)) + } else if f, ok := svr.getHandle(p.Handle); !ok { + return svr.sendPacket(statusFromError(p.Id, syscall.EBADF)) + } else { + // additional unmarshalling is required for each possibility here + b := p.Attrs.([]byte) + var err error = nil + + debug("fsetstat name \"%s\"", f.Name()) + if (p.Flags & ssh_FILEXFER_ATTR_SIZE) != 0 { + var size uint64 = 0 + if size, b, err = unmarshalUint64Safe(b); err == nil { + err = f.Truncate(int64(size)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_PERMISSIONS) != 0 { + var mode uint32 = 0 + if mode, b, err = unmarshalUint32Safe(b); err == nil { + err = f.Chmod(os.FileMode(mode)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_ACMODTIME) != 0 { + var atime uint32 = 0 + var mtime uint32 = 0 + if atime, b, err = unmarshalUint32Safe(b); err != nil { + } else if mtime, b, err = unmarshalUint32Safe(b); err != nil { + } else { + atimeT := time.Unix(int64(atime), 0) + mtimeT := time.Unix(int64(mtime), 0) + err = os.Chtimes(f.Name(), atimeT, mtimeT) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_UIDGID) != 0 { + var uid uint32 = 0 + var gid uint32 = 0 + if uid, b, err = unmarshalUint32Safe(b); err != nil { + } else if gid, b, err = unmarshalUint32Safe(b); err != nil { + } else { + err = f.Chown(int(uid), int(gid)) + } + } + + return svr.sendPacket(statusFromError(p.Id, err)) + } +} + +func errnoToSshErr(errno syscall.Errno) uint32 { + if errno == 0 { + return ssh_FX_OK + } else if errno == syscall.ENOENT { + return ssh_FX_NO_SUCH_FILE + } else if errno == syscall.EPERM { + return ssh_FX_PERMISSION_DENIED + } else { + return ssh_FX_FAILURE + } + + return uint32(errno) +} + +func statusFromError(id uint32, err error) sshFxpStatusPacket { + ret := sshFxpStatusPacket{ + Id: id, + StatusError: StatusError{ + // ssh_FX_OK = 0 + // ssh_FX_EOF = 1 + // ssh_FX_NO_SUCH_FILE = 2 ENOENT + // ssh_FX_PERMISSION_DENIED = 3 + // ssh_FX_FAILURE = 4 + // ssh_FX_BAD_MESSAGE = 5 + // ssh_FX_NO_CONNECTION = 6 + // ssh_FX_CONNECTION_LOST = 7 + // ssh_FX_OP_UNSUPPORTED = 8 + Code: ssh_FX_OK, + msg: "", + lang: "", + }, + } + if err != nil { + debug("statusFromError: error is %T %#v", err, err) + ret.StatusError.Code = ssh_FX_FAILURE + ret.StatusError.msg = err.Error() + if err == io.EOF { + ret.StatusError.Code = ssh_FX_EOF + } else if errno, ok := err.(syscall.Errno); ok { + ret.StatusError.Code = errnoToSshErr(errno) + } else if pathError, ok := err.(*os.PathError); ok { + debug("statusFromError: error is %T %#v", pathError.Err, pathError.Err) + if errno, ok := pathError.Err.(syscall.Errno); ok { + ret.StatusError.Code = errnoToSshErr(errno) + } + } + } + return ret +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_integration_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_integration_test.go new file mode 100644 index 0000000..2d887f0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_integration_test.go @@ -0,0 +1,673 @@ +package sftp + +// sftp server integration tests +// enable with -integration +// example invokation (darwin): gofmt -w `find . -name \*.go` && (cd server_standalone/ ; go build -tags debug) && go test -tags debug github.com/pkg/sftp -integration -v -sftp /usr/libexec/sftp-server -run ServerCompareSubsystems + +import ( + "bytes" + "encoding/hex" + "flag" + "fmt" + "io/ioutil" + "math/rand" + "net" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/kr/fs" + "golang.org/x/crypto/ssh" +) + +var testSftpClientBin = flag.String("sftp_client", "/usr/bin/sftp", "location of the sftp client binary") +var sshServerDebugStream = ioutil.Discard +var sftpServerDebugStream = ioutil.Discard +var sftpClientDebugStream = ioutil.Discard + +const ( + GOLANG_SFTP = true + OPENSSH_SFTP = false +) + +/*********************************************************************************************** + + +SSH server scaffolding; very simple, no strict auth. This is for unit testing, not real servers + + +***********************************************************************************************/ + +var ( + hostPrivateKeySigner ssh.Signer + privKey = []byte(` +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArhp7SqFnXVZAgWREL9Ogs+miy4IU/m0vmdkoK6M97G9NX/Pj +wf8I/3/ynxmcArbt8Rc4JgkjT2uxx/NqR0yN42N1PjO5Czu0dms1PSqcKIJdeUBV +7gdrKSm9Co4d2vwfQp5mg47eG4w63pz7Drk9+VIyi9YiYH4bve7WnGDswn4ycvYZ +slV5kKnjlfCdPig+g5P7yQYud0cDWVwyA0+kxvL6H3Ip+Fu8rLDZn4/P1WlFAIuc +PAf4uEKDGGmC2URowi5eesYR7f6GN/HnBs2776laNlAVXZUmYTUfOGagwLsEkx8x +XdNqntfbs2MOOoK+myJrNtcB9pCrM0H6um19uQIDAQABAoIBABkWr9WdVKvalgkP +TdQmhu3mKRNyd1wCl+1voZ5IM9Ayac/98UAvZDiNU4Uhx52MhtVLJ0gz4Oa8+i16 +IkKMAZZW6ro/8dZwkBzQbieWUFJ2Fso2PyvB3etcnGU8/Yhk9IxBDzy+BbuqhYE2 +1ebVQtz+v1HvVZzaD11bYYm/Xd7Y28QREVfFen30Q/v3dv7dOteDE/RgDS8Czz7w +jMW32Q8JL5grz7zPkMK39BLXsTcSYcaasT2ParROhGJZDmbgd3l33zKCVc1zcj9B +SA47QljGd09Tys958WWHgtj2o7bp9v1Ufs4LnyKgzrB80WX1ovaSQKvd5THTLchO +kLIhUAECgYEA2doGXy9wMBmTn/hjiVvggR1aKiBwUpnB87Hn5xCMgoECVhFZlT6l +WmZe7R2klbtG1aYlw+y+uzHhoVDAJW9AUSV8qoDUwbRXvBVlp+In5wIqJ+VjfivK +zgIfzomL5NvDz37cvPmzqIeySTowEfbQyq7CUQSoDtE9H97E2wWZhDkCgYEAzJdJ +k+NSFoTkHhfD3L0xCDHpRV3gvaOeew8524fVtVUq53X8m91ng4AX1r74dCUYwwiF +gqTtSSJfx2iH1xKnNq28M9uKg7wOrCKrRqNPnYUO3LehZEC7rwUr26z4iJDHjjoB +uBcS7nw0LJ+0Zeg1IF+aIdZGV3MrAKnrzWPixYECgYBsffX6ZWebrMEmQ89eUtFF +u9ZxcGI/4K8ErC7vlgBD5ffB4TYZ627xzFWuBLs4jmHCeNIJ9tct5rOVYN+wRO1k +/CRPzYUnSqb+1jEgILL6istvvv+DkE+ZtNkeRMXUndWwel94BWsBnUKe0UmrSJ3G +sq23J3iCmJW2T3z+DpXbkQKBgQCK+LUVDNPE0i42NsRnm+fDfkvLP7Kafpr3Umdl +tMY474o+QYn+wg0/aPJIf9463rwMNyyhirBX/k57IIktUdFdtfPicd2MEGETElWv +nN1GzYxD50Rs2f/jKisZhEwqT9YNyV9DkgDdGGdEbJNYqbv0qpwDIg8T9foe8E1p +bdErgQKBgAt290I3L316cdxIQTkJh1DlScN/unFffITwu127WMr28Jt3mq3cZpuM +Aecey/eEKCj+Rlas5NDYKsB18QIuAw+qqWyq0LAKLiAvP1965Rkc4PLScl3MgJtO +QYa37FK0p8NcDeUuF86zXBVutwS5nJLchHhKfd590ks57OROtm29 +-----END RSA PRIVATE KEY----- +`) +) + +func init() { + var err error + hostPrivateKeySigner, err = ssh.ParsePrivateKey(privKey) + if err != nil { + panic(err) + } +} + +func keyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + permissions := &ssh.Permissions{ + CriticalOptions: map[string]string{}, + Extensions: map[string]string{}, + } + return permissions, nil +} + +func pwAuth(conn ssh.ConnMetadata, pw []byte) (*ssh.Permissions, error) { + permissions := &ssh.Permissions{ + CriticalOptions: map[string]string{}, + Extensions: map[string]string{}, + } + return permissions, nil +} + +func basicServerConfig() *ssh.ServerConfig { + config := ssh.ServerConfig{ + Config: ssh.Config{ + MACs: []string{"hmac-sha1"}, + }, + PasswordCallback: pwAuth, + PublicKeyCallback: keyAuth, + } + config.AddHostKey(hostPrivateKeySigner) + return &config +} + +type sshServer struct { + useSubsystem bool + conn net.Conn + config *ssh.ServerConfig + sshConn *ssh.ServerConn + newChans <-chan ssh.NewChannel + newReqs <-chan *ssh.Request +} + +func sshServerFromConn(conn net.Conn, useSubsystem bool, config *ssh.ServerConfig) (*sshServer, error) { + // From a standard TCP connection to an encrypted SSH connection + sshConn, newChans, newReqs, err := ssh.NewServerConn(conn, config) + if err != nil { + return nil, err + } + + svr := &sshServer{useSubsystem, conn, config, sshConn, newChans, newReqs} + svr.listenChannels() + return svr, nil +} + +func (svr *sshServer) Wait() error { + return svr.sshConn.Wait() +} + +func (svr *sshServer) Close() error { + return svr.sshConn.Close() +} + +func (svr *sshServer) listenChannels() { + go func() { + for chanReq := range svr.newChans { + go svr.handleChanReq(chanReq) + } + }() + go func() { + for req := range svr.newReqs { + go svr.handleReq(req) + } + }() +} + +func (svr *sshServer) handleReq(req *ssh.Request) { + switch req.Type { + default: + rejectRequest(req) + } +} + +type sshChannelServer struct { + svr *sshServer + chanReq ssh.NewChannel + ch ssh.Channel + newReqs <-chan *ssh.Request +} + +type sshSessionChannelServer struct { + *sshChannelServer + env []string +} + +func (svr *sshServer) handleChanReq(chanReq ssh.NewChannel) { + fmt.Fprintf(sshServerDebugStream, "channel request: %v, extra: '%v'\n", chanReq.ChannelType(), hex.EncodeToString(chanReq.ExtraData())) + switch chanReq.ChannelType() { + case "session": + if ch, reqs, err := chanReq.Accept(); err != nil { + fmt.Fprintf(sshServerDebugStream, "fail to accept channel request: %v\n", err) + chanReq.Reject(ssh.ResourceShortage, "channel accept failure") + } else { + chsvr := &sshSessionChannelServer{ + sshChannelServer: &sshChannelServer{svr, chanReq, ch, reqs}, + env: append([]string{}, os.Environ()...), + } + chsvr.handle() + } + default: + chanReq.Reject(ssh.UnknownChannelType, "channel type is not a session") + } +} + +func (chsvr *sshSessionChannelServer) handle() { + // should maybe do something here... + go chsvr.handleReqs() +} + +func (chsvr *sshSessionChannelServer) handleReqs() { + for req := range chsvr.newReqs { + chsvr.handleReq(req) + } + fmt.Fprintf(sshServerDebugStream, "ssh server session channel complete\n") +} + +func (chsvr *sshSessionChannelServer) handleReq(req *ssh.Request) { + switch req.Type { + case "env": + chsvr.handleEnv(req) + case "subsystem": + chsvr.handleSubsystem(req) + default: + rejectRequest(req) + } +} + +func rejectRequest(req *ssh.Request) error { + fmt.Fprintf(sshServerDebugStream, "ssh rejecting request, type: %s\n", req.Type) + err := req.Reply(false, []byte{}) + if err != nil { + fmt.Fprintf(sshServerDebugStream, "ssh request reply had error: %v\n", err) + } + return err +} + +func rejectRequestUnmarshalError(req *ssh.Request, s interface{}, err error) error { + fmt.Fprintf(sshServerDebugStream, "ssh request unmarshaling error, type '%T': %v\n", s, err) + rejectRequest(req) + return err +} + +// env request form: +type sshEnvRequest struct { + Envvar string + Value string +} + +func (chsvr *sshSessionChannelServer) handleEnv(req *ssh.Request) error { + envReq := &sshEnvRequest{} + if err := ssh.Unmarshal(req.Payload, envReq); err != nil { + return rejectRequestUnmarshalError(req, envReq, err) + } + req.Reply(true, nil) + + found := false + for i, envstr := range chsvr.env { + if strings.HasPrefix(envstr, envReq.Envvar+"=") { + found = true + chsvr.env[i] = envReq.Envvar + "=" + envReq.Value + } + } + if !found { + chsvr.env = append(chsvr.env, envReq.Envvar+"="+envReq.Value) + } + + return nil +} + +// Payload: int: command size, string: command +type sshSubsystemRequest struct { + Name string +} + +type sshSubsystemExitStatus struct { + Status uint32 +} + +func (chsvr *sshSessionChannelServer) handleSubsystem(req *ssh.Request) error { + defer func() { + err1 := chsvr.ch.CloseWrite() + err2 := chsvr.ch.Close() + fmt.Fprintf(sshServerDebugStream, "ssh server subsystem request complete, err: %v %v\n", err1, err2) + }() + + subsystemReq := &sshSubsystemRequest{} + if err := ssh.Unmarshal(req.Payload, subsystemReq); err != nil { + return rejectRequestUnmarshalError(req, subsystemReq, err) + } + + // reply to the ssh client + + // no idea if this is actually correct spec-wise. + // just enough for an sftp server to start. + if subsystemReq.Name == "sftp" { + req.Reply(true, nil) + + if !chsvr.svr.useSubsystem { + // use the openssh sftp server backend; this is to test the ssh code, not the sftp code, + // or is used for comparison between our sftp subsystem and the openssh sftp subsystem + cmd := exec.Command(*testSftp, "-e", "-l", "DEBUG") // log to stderr + cmd.Stdin = chsvr.ch + cmd.Stdout = chsvr.ch + cmd.Stderr = sftpServerDebugStream + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() + } else { + sftpServer, err := NewServer(chsvr.ch, chsvr.ch, sftpServerDebugStream, 0, false, ".") + if err != nil { + return err + } + + // wait for the session to close + runErr := sftpServer.Serve() + exitStatus := uint32(1) + if runErr == nil { + exitStatus = uint32(0) + } + + _, exitStatusErr := chsvr.ch.SendRequest("exit-status", false, ssh.Marshal(sshSubsystemExitStatus{exitStatus})) + return exitStatusErr + } + } else { + return req.Reply(false, nil) + } +} + +/*********************************************************************************************** + + +Actual unit tests + + +***********************************************************************************************/ + +// starts an ssh server to test. returns: host string and port +func testServer(t *testing.T, useSubsystem bool, readonly bool) (net.Listener, string, int) { + if !*testIntegration { + t.Skip("skipping intergration test") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + host, portStr, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatal(err) + } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + fmt.Fprintf(sshServerDebugStream, "ssh server socket closed\n") + break + } + + go func() { + defer conn.Close() + sshSvr, err := sshServerFromConn(conn, useSubsystem, basicServerConfig()) + if err != nil { + t.Fatal(err) + } + err = sshSvr.Wait() + fmt.Fprintf(sshServerDebugStream, "ssh server finished, err: %v\n", err) + }() + } + }() + + return listener, host, port +} + +func runSftpClient(t *testing.T, script string, path string, host string, port int) (string, error) { + // if sftp client binary is unavailable, skip test + if _, err := os.Stat(*testSftpClientBin); err != nil { + t.Skip("sftp client binary unavailable") + } + cmd := exec.Command(*testSftpClientBin /*"-vvvv",*/, "-b", "-", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", "-o", "UserKnownHostsFile /dev/null", "-P", fmt.Sprintf("%d", port), fmt.Sprintf("%s:%s", host, path)) + stdout := &bytes.Buffer{} + cmd.Stdin = bytes.NewBufferString(script) + cmd.Stdout = stdout + cmd.Stderr = sftpClientDebugStream + if err := cmd.Start(); err != nil { + return "", err + } + err := cmd.Wait() + return string(stdout.Bytes()), err +} + +func TestServerCompareSubsystems(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + listenerOp, hostOp, portOp := testServer(t, OPENSSH_SFTP, READONLY) + defer listenerGo.Close() + defer listenerOp.Close() + + script := ` +ls / +ls -l / +ls /dev/ +ls -l /dev/ +ls -l /etc/ +ls -l /bin/ +ls -l /usr/bin/ +` + outputGo, err := runSftpClient(t, script, "/", hostGo, portGo) + if err != nil { + t.Fatal(err) + } + + outputOp, err := runSftpClient(t, script, "/", hostOp, portOp) + if err != nil { + t.Fatal(err) + } + + newlineRegex := regexp.MustCompile(`\r*\n`) + spaceRegex := regexp.MustCompile(`\s+`) + outputGoLines := newlineRegex.Split(outputGo, -1) + outputOpLines := newlineRegex.Split(outputOp, -1) + + for i, goLine := range outputGoLines { + if i > len(outputOpLines) { + t.Fatalf("output line count differs") + } + opLine := outputOpLines[i] + bad := false + if goLine != opLine { + goWords := spaceRegex.Split(goLine, -1) + opWords := spaceRegex.Split(opLine, -1) + // allow words[2] and [3] to be different as these are users & groups + for j, goWord := range goWords { + if j > len(opWords) { + bad = true + } + opWord := opWords[j] + if goWord != opWord && j != 2 && j != 3 { + bad = true + } + } + } + + if bad { + t.Errorf("outputs differ, go:\n%v\nopenssh:\n%v\n", goLine, opLine) + } + } +} + +var rng = rand.New(rand.NewSource(time.Now().Unix())) + +func randData(length int) []byte { + data := make([]byte, length) + for i := 0; i < length; i++ { + data[i] = byte(rng.Uint32()) + } + return data +} + +func randName() string { + return "sftp." + hex.EncodeToString(randData(16)) +} + +func TestServerMkdirRmdir(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpDir := "/tmp/" + randName() + defer os.RemoveAll(tmpDir) + + // mkdir remote + if _, err := runSftpClient(t, "mkdir "+tmpDir, "/", hostGo, portGo); err != nil { + t.Fatal(err) + } + + // directory should now exist + if _, err := os.Stat(tmpDir); err != nil { + t.Fatal(err) + } + + // now remove the directory + if _, err := runSftpClient(t, "rmdir "+tmpDir, "/", hostGo, portGo); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(tmpDir); err == nil { + t.Fatal("should have error after deleting the directory") + } +} + +func TestServerSymlink(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + link := "/tmp/" + randName() + defer os.RemoveAll(link) + + // now create a symbolic link within the new directory + if output, err := runSftpClient(t, "symlink /bin/sh "+link, "/", hostGo, portGo); err != nil { + t.Fatalf("failed: %v %v", err, string(output)) + } + + // symlink should now exist + if stat, err := os.Lstat(link); err != nil { + t.Fatal(err) + } else if (stat.Mode() & os.ModeSymlink) != os.ModeSymlink { + t.Fatalf("is not a symlink: %v", stat.Mode()) + } +} + +func TestServerPut(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpFileLocal := "/tmp/" + randName() + tmpFileRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpFileLocal) + defer os.RemoveAll(tmpFileRemote) + + t.Logf("put: local %v remote %v", tmpFileLocal, tmpFileRemote) + + // create a file with random contents. This will be the local file pushed to the server + tmpFileLocalData := randData(10 * 1024 * 1024) + if err := ioutil.WriteFile(tmpFileLocal, tmpFileLocalData, 0644); err != nil { + t.Fatal(err) + } + + // sftp the file to the server + if output, err := runSftpClient(t, "put "+tmpFileLocal+" "+tmpFileRemote, "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + // tmpFile2 should now exist, with the same contents + if tmpFileRemoteData, err := ioutil.ReadFile(tmpFileRemote); err != nil { + t.Fatal(err) + } else if string(tmpFileLocalData) != string(tmpFileRemoteData) { + t.Fatal("contents of file incorrect after put") + } +} + +func TestServerGet(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpFileLocal := "/tmp/" + randName() + tmpFileRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpFileLocal) + defer os.RemoveAll(tmpFileRemote) + + t.Logf("get: local %v remote %v", tmpFileLocal, tmpFileRemote) + + // create a file with random contents. This will be the remote file pulled from the server + tmpFileRemoteData := randData(10 * 1024 * 1024) + if err := ioutil.WriteFile(tmpFileRemote, tmpFileRemoteData, 0644); err != nil { + t.Fatal(err) + } + + // sftp the file to the server + if output, err := runSftpClient(t, "get "+tmpFileRemote+" "+tmpFileLocal, "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + // tmpFile2 should now exist, with the same contents + if tmpFileLocalData, err := ioutil.ReadFile(tmpFileLocal); err != nil { + t.Fatal(err) + } else if string(tmpFileLocalData) != string(tmpFileRemoteData) { + t.Fatal("contents of file incorrect after put") + } +} + +func compareDirectoriesRecursive(t *testing.T, aroot, broot string) { + walker := fs.Walk(aroot) + for walker.Step() { + if err := walker.Err(); err != nil { + t.Fatal(err) + } + // find paths + aPath := walker.Path() + aRel, err := filepath.Rel(aroot, aPath) + if err != nil { + t.Fatalf("could not find relative path for %v: %v", aPath, err) + } + bPath := path.Join(broot, aRel) + + if aRel == "." { + continue + } + + //t.Logf("comparing: %v a: %v b %v", aRel, aPath, bPath) + + // if a is a link, the sftp recursive copy won't have copied it. ignore + aLink, err := os.Lstat(aPath) + if err != nil { + t.Fatalf("could not lstat %v: %v", aPath, err) + } + if aLink.Mode()&os.ModeSymlink != 0 { + continue + } + + // stat the files + aFile, err := os.Stat(aPath) + if err != nil { + t.Fatalf("could not stat %v: %v", aPath, err) + } + bFile, err := os.Stat(bPath) + if err != nil { + t.Fatalf("could not stat %v: %v", bPath, err) + } + + // compare stats, with some leniency for the timestamp + if aFile.Mode() != bFile.Mode() { + t.Fatalf("modes different for %v: %v vs %v", aRel, aFile.Mode(), bFile.Mode()) + } + if !aFile.IsDir() { + if aFile.Size() != bFile.Size() { + t.Fatalf("sizes different for %v: %v vs %v", aRel, aFile.Size(), bFile.Size()) + } + } + timeDiff := aFile.ModTime().Sub(bFile.ModTime()) + if timeDiff > time.Second || timeDiff < -time.Second { + t.Fatalf("mtimes different for %v: %v vs %v", aRel, aFile.ModTime(), bFile.ModTime()) + } + + // compare contents + if !aFile.IsDir() { + if aContents, err := ioutil.ReadFile(aPath); err != nil { + t.Fatal(err) + } else if bContents, err := ioutil.ReadFile(bPath); err != nil { + t.Fatal(err) + } else if string(aContents) != string(bContents) { + t.Fatalf("contents different for %v", aRel) + } + } + } +} + +func TestServerPutRecursive(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + dirLocal, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + tmpDirRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpDirRemote) + + t.Logf("put recursive: local %v remote %v", dirLocal, tmpDirRemote) + + // push this directory (source code etc) recursively to the server + if output, err := runSftpClient(t, "mkdir "+tmpDirRemote+"\r\nput -r -P "+dirLocal+"/ "+tmpDirRemote+"/", "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + compareDirectoriesRecursive(t, dirLocal, path.Join(tmpDirRemote, path.Base(dirLocal))) +} + +func TestServerGetRecursive(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + dirRemote, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + tmpDirLocal := "/tmp/" + randName() + defer os.RemoveAll(tmpDirLocal) + + t.Logf("get recursive: local %v remote %v", tmpDirLocal, dirRemote) + + // pull this directory (source code etc) recursively from the server + if output, err := runSftpClient(t, "lmkdir "+tmpDirLocal+"\r\nget -r -P "+dirRemote+"/ "+tmpDirLocal+"/", "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + compareDirectoriesRecursive(t, dirRemote, path.Join(tmpDirLocal, path.Base(dirRemote))) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go new file mode 100644 index 0000000..77b8ec6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go @@ -0,0 +1,39 @@ +package main + +// small wrapper around sftp server that allows it to be used as a separate process subsystem call by the ssh server. +// in practice this will statically link; however this allows unit testing from the sftp client. + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + + "github.com/pkg/sftp" +) + +func main() { + var ( + readOnly bool + debugLevelStr string + debugLevel int + debugStderr bool + ) + + flag.BoolVar(&readOnly, "R", false, "read-only server") + flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.StringVar(&debugLevelStr, "l", "none", "debug level") + flag.Parse() + + debugStream := ioutil.Discard + if debugStderr { + debugStream = os.Stderr + debugLevel = 1 + } + + svr, _ := sftp.NewServer(os.Stdin, os.Stdout, debugStream, debugLevel, readOnly, "") + if err := svr.Serve(); err != nil { + fmt.Fprintf(debugStream, "sftp server completed with error: %v", err) + os.Exit(1) + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go new file mode 100644 index 0000000..3b1ddbd --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go @@ -0,0 +1,12 @@ +// +build !cgo,!plan9 windows android + +package sftp + +import ( + "os" + "path" +) + +func runLs(dirname string, dirent os.FileInfo) string { + return path.Join(dirname, dirent.Name()) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go new file mode 100644 index 0000000..8c3f0b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go @@ -0,0 +1,143 @@ +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris +// +build cgo + +package sftp + +import ( + "fmt" + "os" + "path" + "syscall" + "time" +) + +func runLsTypeWord(dirent os.FileInfo) string { + // find first character, the type char + // b Block special file. + // c Character special file. + // d Directory. + // l Symbolic link. + // s Socket link. + // p FIFO. + // - Regular file. + tc := '-' + mode := dirent.Mode() + if (mode & os.ModeDir) != 0 { + tc = 'd' + } else if (mode & os.ModeDevice) != 0 { + tc = 'b' + if (mode & os.ModeCharDevice) != 0 { + tc = 'c' + } + } else if (mode & os.ModeSymlink) != 0 { + tc = 'l' + } else if (mode & os.ModeSocket) != 0 { + tc = 's' + } else if (mode & os.ModeNamedPipe) != 0 { + tc = 'p' + } + + // owner + orc := '-' + if (mode & 0400) != 0 { + orc = 'r' + } + owc := '-' + if (mode & 0200) != 0 { + owc = 'w' + } + oxc := '-' + ox := (mode & 0100) != 0 + setuid := (mode & os.ModeSetuid) != 0 + if ox && setuid { + oxc = 's' + } else if setuid { + oxc = 'S' + } else if ox { + oxc = 'x' + } + + // group + grc := '-' + if (mode & 040) != 0 { + grc = 'r' + } + gwc := '-' + if (mode & 020) != 0 { + gwc = 'w' + } + gxc := '-' + gx := (mode & 010) != 0 + setgid := (mode & os.ModeSetgid) != 0 + if gx && setgid { + gxc = 's' + } else if setgid { + gxc = 'S' + } else if gx { + gxc = 'x' + } + + // all / others + arc := '-' + if (mode & 04) != 0 { + arc = 'r' + } + awc := '-' + if (mode & 02) != 0 { + awc = 'w' + } + axc := '-' + ax := (mode & 01) != 0 + sticky := (mode & os.ModeSticky) != 0 + if ax && sticky { + axc = 't' + } else if sticky { + axc = 'T' + } else if ax { + axc = 'x' + } + + return fmt.Sprintf("%c%c%c%c%c%c%c%c%c%c", tc, orc, owc, oxc, grc, gwc, gxc, arc, awc, axc) +} + +func runLsStatt(dirname string, dirent os.FileInfo, statt *syscall.Stat_t) string { + // example from openssh sftp server: + // crw-rw-rw- 1 root wheel 0 Jul 31 20:52 ttyvd + // format: + // {directory / char device / etc}{rwxrwxrwx} {number of links} owner group size month day [time (this year) | year (otherwise)] name + + typeword := runLsTypeWord(dirent) + numLinks := statt.Nlink + uid := statt.Uid + gid := statt.Gid + username := fmt.Sprintf("%d", uid) + groupname := fmt.Sprintf("%d", gid) + // TODO FIXME: uid -> username, gid -> groupname lookup for ls -l format output + + mtime := dirent.ModTime() + monthStr := mtime.Month().String()[0:3] + day := mtime.Day() + year := mtime.Year() + now := time.Now() + isOld := mtime.Before(now.Add(-time.Hour * 24 * 365 / 2)) + + yearOrTime := fmt.Sprintf("%02d:%02d", mtime.Hour(), mtime.Minute()) + if isOld { + yearOrTime = fmt.Sprintf("%d", year) + } + + return fmt.Sprintf("%s %4d %-8s %-8s %8d %s %2d %5s %s", typeword, numLinks, username, groupname, dirent.Size(), monthStr, day, yearOrTime, dirent.Name()) +} + +// ls -l style output for a file, which is in the 'long output' section of a readdir response packet +// this is a very simple (lazy) implementation, just enough to look almost like openssh in a few basic cases +func runLs(dirname string, dirent os.FileInfo) string { + dsys := dirent.Sys() + if dsys == nil { + } else if statt, ok := dsys.(*syscall.Stat_t); !ok { + } else { + return runLsStatt(dirname, dirent, statt) + } + + return path.Join(dirname, dirent.Name()) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go b/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go new file mode 100644 index 0000000..934684c --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go @@ -0,0 +1,187 @@ +// Package sftp implements the SSH File Transfer Protocol as described in +// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt +package sftp + +import ( + "fmt" +) + +const ( + ssh_FXP_INIT = 1 + ssh_FXP_VERSION = 2 + ssh_FXP_OPEN = 3 + ssh_FXP_CLOSE = 4 + ssh_FXP_READ = 5 + ssh_FXP_WRITE = 6 + ssh_FXP_LSTAT = 7 + ssh_FXP_FSTAT = 8 + ssh_FXP_SETSTAT = 9 + ssh_FXP_FSETSTAT = 10 + ssh_FXP_OPENDIR = 11 + ssh_FXP_READDIR = 12 + ssh_FXP_REMOVE = 13 + ssh_FXP_MKDIR = 14 + ssh_FXP_RMDIR = 15 + ssh_FXP_REALPATH = 16 + ssh_FXP_STAT = 17 + ssh_FXP_RENAME = 18 + ssh_FXP_READLINK = 19 + ssh_FXP_SYMLINK = 20 + ssh_FXP_STATUS = 101 + ssh_FXP_HANDLE = 102 + ssh_FXP_DATA = 103 + ssh_FXP_NAME = 104 + ssh_FXP_ATTRS = 105 + ssh_FXP_EXTENDED = 200 + ssh_FXP_EXTENDED_REPLY = 201 +) + +const ( + ssh_FX_OK = 0 + ssh_FX_EOF = 1 + ssh_FX_NO_SUCH_FILE = 2 + ssh_FX_PERMISSION_DENIED = 3 + ssh_FX_FAILURE = 4 + ssh_FX_BAD_MESSAGE = 5 + ssh_FX_NO_CONNECTION = 6 + ssh_FX_CONNECTION_LOST = 7 + ssh_FX_OP_UNSUPPORTED = 8 +) + +const ( + ssh_FXF_READ = 0x00000001 + ssh_FXF_WRITE = 0x00000002 + ssh_FXF_APPEND = 0x00000004 + ssh_FXF_CREAT = 0x00000008 + ssh_FXF_TRUNC = 0x00000010 + ssh_FXF_EXCL = 0x00000020 +) + +type fxp uint8 + +func (f fxp) String() string { + switch f { + case ssh_FXP_INIT: + return "SSH_FXP_INIT" + case ssh_FXP_VERSION: + return "SSH_FXP_VERSION" + case ssh_FXP_OPEN: + return "SSH_FXP_OPEN" + case ssh_FXP_CLOSE: + return "SSH_FXP_CLOSE" + case ssh_FXP_READ: + return "SSH_FXP_READ" + case ssh_FXP_WRITE: + return "SSH_FXP_WRITE" + case ssh_FXP_LSTAT: + return "SSH_FXP_LSTAT" + case ssh_FXP_FSTAT: + return "SSH_FXP_FSTAT" + case ssh_FXP_SETSTAT: + return "SSH_FXP_SETSTAT" + case ssh_FXP_FSETSTAT: + return "SSH_FXP_FSETSTAT" + case ssh_FXP_OPENDIR: + return "SSH_FXP_OPENDIR" + case ssh_FXP_READDIR: + return "SSH_FXP_READDIR" + case ssh_FXP_REMOVE: + return "SSH_FXP_REMOVE" + case ssh_FXP_MKDIR: + return "SSH_FXP_MKDIR" + case ssh_FXP_RMDIR: + return "SSH_FXP_RMDIR" + case ssh_FXP_REALPATH: + return "SSH_FXP_REALPATH" + case ssh_FXP_STAT: + return "SSH_FXP_STAT" + case ssh_FXP_RENAME: + return "SSH_FXP_RENAME" + case ssh_FXP_READLINK: + return "SSH_FXP_READLINK" + case ssh_FXP_SYMLINK: + return "SSH_FXP_SYMLINK" + case ssh_FXP_STATUS: + return "SSH_FXP_STATUS" + case ssh_FXP_HANDLE: + return "SSH_FXP_HANDLE" + case ssh_FXP_DATA: + return "SSH_FXP_DATA" + case ssh_FXP_NAME: + return "SSH_FXP_NAME" + case ssh_FXP_ATTRS: + return "SSH_FXP_ATTRS" + case ssh_FXP_EXTENDED: + return "SSH_FXP_EXTENDED" + case ssh_FXP_EXTENDED_REPLY: + return "SSH_FXP_EXTENDED_REPLY" + default: + return "unknown" + } +} + +type fx uint8 + +func (f fx) String() string { + switch f { + case ssh_FX_OK: + return "SSH_FX_OK" + case ssh_FX_EOF: + return "SSH_FX_EOF" + case ssh_FX_NO_SUCH_FILE: + return "SSH_FX_NO_SUCH_FILE" + case ssh_FX_PERMISSION_DENIED: + return "SSH_FX_PERMISSION_DENIED" + case ssh_FX_FAILURE: + return "SSH_FX_FAILURE" + case ssh_FX_BAD_MESSAGE: + return "SSH_FX_BAD_MESSAGE" + case ssh_FX_NO_CONNECTION: + return "SSH_FX_NO_CONNECTION" + case ssh_FX_CONNECTION_LOST: + return "SSH_FX_CONNECTION_LOST" + case ssh_FX_OP_UNSUPPORTED: + return "SSH_FX_OP_UNSUPPORTED" + default: + return "unknown" + } +} + +type unexpectedPacketErr struct { + want, got uint8 +} + +func (u *unexpectedPacketErr) Error() string { + return fmt.Sprintf("sftp: unexpected packet: want %v, got %v", fxp(u.want), fxp(u.got)) +} + +func unimplementedPacketErr(u uint8) error { + return fmt.Errorf("sftp: unimplemented packet type: got %v", fxp(u)) +} + +type unexpectedIdErr struct{ want, got uint32 } + +func (u *unexpectedIdErr) Error() string { + return fmt.Sprintf("sftp: unexpected id: want %v, got %v", u.want, u.got) +} + +func unimplementedSeekWhence(whence int) error { + return fmt.Errorf("sftp: unimplemented seek whence %v", whence) +} + +func unexpectedCount(want, got uint32) error { + return fmt.Errorf("sftp: unexpected count: want %v, got %v", want, got) +} + +type unexpectedVersionErr struct{ want, got uint32 } + +func (u *unexpectedVersionErr) Error() string { + return fmt.Sprintf("sftp: unexpected server version: want %v, got %v", u.want, u.got) +} + +type StatusError struct { + Code uint32 + msg, lang string +} + +func (s *StatusError) Error() string { return fmt.Sprintf("sftp: %q (%v)", s.msg, fx(s.Code)) } diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml b/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml new file mode 100644 index 0000000..1ca9d37 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml @@ -0,0 +1,34 @@ +box: wercker/golang +# Build definition +build: + # The steps that will be executed on build + steps: + # Sets the go workspace and places you package + # at the right place in the workspace tree + - setup-go-workspace + + # Gets the dependencies + - script: + name: go get + code: | + cd $WERCKER_SOURCE_DIR + go version + go get -t ./... + + # Build the project + - script: + name: go build + code: | + go build ./... + + # Test the project + - script: + name: go test + code: | + go test ./... + + - script: + name: go test -integration + cost: | + go test -intergration ./... + diff --git a/documentation/man/corectl-put.1 b/documentation/man/corectl-put.1 new file mode 100644 index 0000000..23c0cd8 --- /dev/null +++ b/documentation/man/corectl-put.1 @@ -0,0 +1,43 @@ +.TH "corectl" "1" "" " " "" "" + + +.SH NAME +.PP +corectl\-put \- copy file to inside VM + + +.SH SYNOPSIS +.PP +\fBcorectl put\fP [OPTIONS] + + +.SH DESCRIPTION +.PP +copy file to inside VM + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB\-\-debug\fP[=false] + adds extra verbosity, and options, for debugging purposes and/or power users + + +.SH EXAMPLE +.PP +.RS + +.nf + // copies 'filePath' into '/destinationPath' inside VMid + corectl put filePath VMid:/destinationPath + +.fi +.RE + + +.SH SEE ALSO +.PP +\fBcorectl(1)\fP + + +.SH HISTORY +.PP diff --git a/documentation/man/corectl.1 b/documentation/man/corectl.1 index 98e45f5..59d4f59 100644 --- a/documentation/man/corectl.1 +++ b/documentation/man/corectl.1 @@ -26,7 +26,7 @@ CoreOS over OSX made simple. .SH SEE ALSO .PP -, \fBcorectl\-kill(1)\fP, \fBcorectl\-load(1)\fP, \fBcorectl\-ls(1)\fP, \fBcorectl\-ps(1)\fP, \fBcorectl\-pull(1)\fP, \fBcorectl\-rm(1)\fP, \fBcorectl\-run(1)\fP, \fBcorectl\-ssh(1)\fP, \fBcorectl\-version(1)\fP +, \fBcorectl\-kill(1)\fP, \fBcorectl\-load(1)\fP, \fBcorectl\-ls(1)\fP, \fBcorectl\-ps(1)\fP, \fBcorectl\-pull(1)\fP, \fBcorectl\-put(1)\fP, \fBcorectl\-rm(1)\fP, \fBcorectl\-run(1)\fP, \fBcorectl\-ssh(1)\fP, \fBcorectl\-version(1)\fP .SH HISTORY diff --git a/documentation/markdown/corectl.md b/documentation/markdown/corectl.md index 888ebc9..6019179 100644 --- a/documentation/markdown/corectl.md +++ b/documentation/markdown/corectl.md @@ -24,6 +24,7 @@ corectl * [corectl ls](corectl_ls.md) - Lists locally available CoreOS images * [corectl ps](corectl_ps.md) - Lists running CoreOS instances * [corectl pull](corectl_pull.md) - Pulls a CoreOS image from upstream +* [corectl put](corectl_put.md) - copy file to inside VM * [corectl rm](corectl_rm.md) - Removes one or more CoreOS images from local fs * [corectl run](corectl_run.md) - Starts a new CoreOS instance * [corectl ssh](corectl_ssh.md) - Attach to or run commands inside a running CoreOS instance diff --git a/documentation/markdown/corectl_put.md b/documentation/markdown/corectl_put.md new file mode 100644 index 0000000..3a7807a --- /dev/null +++ b/documentation/markdown/corectl_put.md @@ -0,0 +1,29 @@ +## corectl put + +copy file to inside VM + +### Synopsis + + +copy file to inside VM + +``` +corectl put +``` + +### Examples + +``` + // copies 'filePath' into '/destinationPath' inside VMid + corectl put filePath VMid:/destinationPath +``` + +### Options inherited from parent commands + +``` + --debug[=false]: adds extra verbosity, and options, for debugging purposes and/or power users +``` + +### SEE ALSO +* [corectl](corectl.md) - CoreOS over OSX made simple. + diff --git a/pull.go b/pull.go index 40f9f3e..0fafec8 100644 --- a/pull.go +++ b/pull.go @@ -25,7 +25,7 @@ import ( "os" "path/filepath" "strings" - // XXX + "github.com/TheNewNormal/corectl/image" "github.com/blang/semver" diff --git a/run.go b/run.go index e803711..9e20e03 100644 --- a/run.go +++ b/run.go @@ -28,6 +28,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "time" "github.com/TheNewNormal/corectl/uuid2ip" @@ -203,6 +204,7 @@ func bootVM(vipre *viper.Viper) (err error) { } if err = c.Start(); err != nil { + c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} return fmt.Errorf("Aborting: unable to start in background. (%v)", err) } diff --git a/ssh.go b/ssh.go index 9a00fed..6f5835d 100644 --- a/ssh.go +++ b/ssh.go @@ -17,10 +17,14 @@ package main import ( "fmt" + "io" + "log" "os" "strings" "time" + "github.com/pkg/sftp" + "github.com/rakyll/pb" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/terminal" @@ -34,7 +38,7 @@ var ( PreRunE: func(cmd *cobra.Command, args []string) (err error) { vipre.BindPFlags(cmd.Flags()) if len(args) < 1 { - return fmt.Errorf("This command requires either at least " + + return fmt.Errorf("This command requires at least " + "one argument to work ") } return @@ -43,6 +47,22 @@ var ( Example: ` corectl ssh VMid // logins into VMid corectl ssh VMid "some commands" // runs 'some commands' inside VMid and exits`, } + scpCmd = &cobra.Command{ + Use: "put", + Aliases: []string{"copy", "cp", "scp"}, + Short: "copy file to inside VM", + PreRunE: func(cmd *cobra.Command, args []string) (err error) { + vipre.BindPFlags(cmd.Flags()) + if len(args) < 2 { + return fmt.Errorf("This command requires at least " + + "two argument to work ") + } + return + }, + RunE: scpCommand, + Example: ` // copies 'filePath' into '/destinationPath' inside VMid + corectl put filePath VMid:/destinationPath`, + } ) func sshCommand(cmd *cobra.Command, args []string) (err error) { @@ -133,7 +153,6 @@ func (vm VMInfo) startSSHsession() (c *sshClient, err error) { } func (c *sshClient) executeRemoteCommand(run string) (err error) { - if err = c.session.Run(run); err != nil && !strings.HasSuffix(err.Error(), "exited without exit status or exit signal") { return @@ -166,6 +185,69 @@ func vmInfo(id string) (vm VMInfo, err error) { return vm, fmt.Errorf("'%s' not found, or dead", id) } +func (c *sshClient) sCopy(source, destination, target string) (err error) { + var ( + ftp *sftp.Client + src *os.File + srcS, destS os.FileInfo + dest *sftp.File + bar *pb.ProgressBar + ) + + if ftp, err = sftp.NewClient(c.conn); err != nil { + return + } + defer ftp.Close() + log.Println("uploading '" + source + "' to '" + + target + ":" + destination + "'") + + if src, err = os.Open(source); err != nil { + return + } + defer src.Close() + if srcS, err = os.Stat(source); err != nil { + return + } + + if dest, err = ftp.Create(destination); err != nil { + // XXX + return + } + defer dest.Close() + bar = pb.New(int(srcS.Size())).SetUnits(pb.U_BYTES) + bar.Start() + defer bar.Finish() + if _, err = io.Copy(dest, src); err != nil { + return + } + + if destS, err = ftp.Stat(destination); err != nil { + return + } + if srcS.Size() != destS.Size() { + err = fmt.Errorf("something went wrong. " + + "destination file size != from sources'") + } + return +} + +func scpCommand(cmd *cobra.Command, args []string) (err error) { + var ( + session, vm = &sshClient{}, VMInfo{} + split = strings.Split(args[1], ":") + source, destination, target = args[0], split[1], split[0] + ) + if vm, err = vmInfo(target); err != nil { + return + } + if session, err = vm.startSSHsession(); err != nil { + return + } + defer session.close() + return session.sCopy(source, destination, target) +} + func init() { RootCmd.AddCommand(sshCmd) + RootCmd.AddCommand(scpCmd) }