From e38fe87bbf8c6b7a67232d2cba72b39465e0d626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 25 Apr 2023 16:52:13 +0200 Subject: [PATCH] Add apt credentials parser Add apt.auth.conf(5) machine credentials parser. See the parser code for documentation. This commit doesn't change the current behavior. It's only a preparation for future commits that will add support for different types of archives. --- internal/archive/credentials.go | 303 +++++++++++++++++++++++++++ internal/archive/credentials_test.go | 253 ++++++++++++++++++++++ internal/archive/export_test.go | 3 + 3 files changed, 559 insertions(+) create mode 100644 internal/archive/credentials.go create mode 100644 internal/archive/credentials_test.go diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go new file mode 100644 index 00000000..5cc9b429 --- /dev/null +++ b/internal/archive/credentials.go @@ -0,0 +1,303 @@ +package archive + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "sort" + "strings" +) + +// This file defines findCredentials() function for searching repository +// credentials in Apt configuration, see +// https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. + +// credentials contains matched non-empty Username and Password. +// Username is left empty if the search is unsuccessful. +type credentials struct { + Username string + Password string +} + +// Empty checks whether c represents unsuccessful search. +func (c credentials) Empty() bool { + return c.Username == "" +} + +// credentialsQuery contains parsed input URL data used for search. +type credentialsQuery struct { + scheme string + host string + port string + path string + needScheme bool +} + +// queryFor parses repoUrl into credentialsQuery and fills provided credentials with +// username and password if they are specified in repoUrl. +func queryFor(repoUrl string, creds *credentials) (*credentialsQuery, error) { + u, err := url.Parse(repoUrl) + if err != nil { + return nil, err + } + host := u.Host + port := u.Port() + if port != "" { + // u.Hostname() would remove brackets from IPv6 address but we + // need it verbatim for string search in netrc file. This is + // also faster because both u.Port() and u.Hostname() parse + // u.Host into port and hostname. + host = u.Host[0 : len(u.Host)-len(port)-1] + } + + query := credentialsQuery{ + scheme: u.Scheme, + host: host, + port: port, + path: u.Path, + // If the input URL specifies unencrypted scheme, the scheme in + // machine declarations in netrc file is not optional and must + // also match. + needScheme: u.Scheme != "https" && u.Scheme != "tor+https", + } + + if creds != nil { + creds.Username = u.User.Username() + creds.Password, _ = u.User.Password() + } + + return &query, nil +} + +// findCredentials searches credentials for repoUrl in configuration files in +// directory specified by CHISEL_AUTH_DIR environment variable if it's +// non-empty or /etc/apt/auth.conf.d. +func findCredentials(repoUrl string) (credentials, error) { + credentialsDir := "/etc/apt/auth.conf.d" + if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { + credentialsDir = v + } + return findCredentialsDir(repoUrl, credentialsDir) +} + +// findCredentialsDir searches for credentials for repoUrl in configuration +// files in credentialsDir directory. If the directory does not exist, empty +// credentials structure with nil err is returned. +// Only files that do not begin with dot and have either no or ".conf" +// extension are searched. The files are searched in ascending lexicographic +// order. The first file that contains machine declaration matching repoUrl +// ends the search. If no file contain matching machine declaration, empty +// credentials structure with nil err is returned. +func findCredentialsDir(repoUrl string, credentialsDir string) (creds credentials, err error) { + contents, err := os.ReadDir(credentialsDir) + if err != nil { + if os.IsNotExist(err) { + err = nil + } else { + err = fmt.Errorf("cannot open credentials directory: %w", err) + } + return + } + + confFiles := make([]string, 0, len(contents)) + for _, entry := range contents { + name := entry.Name() + if strings.HasPrefix(name, ".") { + continue + } + var info fs.FileInfo + info, err = entry.Info() + if err != nil { + return + } + if !info.Mode().IsRegular() { + continue + } + ext := filepath.Ext(name) + if ext == "" || ext == ".conf" { + confFiles = append(confFiles, name) + } + } + + // skip parsing repoUrl if there's nothing to match against + if len(confFiles) == 0 { + return + } + + sort.Strings(confFiles) + + query, err := queryFor(repoUrl, &creds) + if err != nil { + return + } + + errs := make([]error, 0, len(confFiles)) + + for _, file := range confFiles { + if !creds.Empty() { + break + } + + f, err := os.Open(filepath.Join(credentialsDir, file)) + if err != nil { + errs = append(errs, fmt.Errorf("cannot read credentials file: %w", err)) + continue + } + + if err = findCredentialsFile(query, f, &creds); err != nil { + errs = append(errs, fmt.Errorf("cannot find credentials: %w", err)) + } + } + + err = errors.Join(errs...) + return +} + +type netrcParser struct { + query *credentialsQuery + scanner *bufio.Scanner + creds *credentials +} + +// findCredentialsFile searches for credentials in netrc file matching query +// and fills creds with matched credentials if there's a match. The first match +// ends the search. +// +// The format of the netrc file is described in [1]. The parser is adapted from +// the Apt parser (see [2]). When the parser is looking for a matching machine +// declaration it disregards the current context and only considers the input +// token. For example when given the following netrc file +// +// machine http://acme.com/foo login u1 password machine +// machine http://acme.com/bar login u2 password p2 +// +// and http://acme.com/bar input URL, the second line won't match, because the +// second "machine" will be treated as start of machine declaration. This also +// means unknown tokens are ignored, so comments are not treated specially. +// +// When a matching machine declaration is found the search stops on next +// machine token or on end of file. This means that arbitrary number of login +// and password declarations (or in fact, any tokens) can follow a machine +// declaration. The last username and password declaration overrides the +// previous ones. For example when given the following netrc file +// +// machine http://acme.com login a foo login b password c bar login d password e +// +// and the input URL is http://acme.com, the matched username and password will +// be "d" and "e" respectively. Tokens foo and bar will be ignored. +// +// This parser diverges from the Apt parser in the following ways: +// 1. The port specification in machine declaration is optional whether or +// not a path is specified. While the Apt documentation[1] implies the +// same behavior, the code adheres to it only when the machine declaration +// does not specify a path, see line 96 in [2]. +// 2. When the input URL has unencrypted scheme and the machine declaration +// does not specify a scheme, it is skipped silently. The Apt parser warns +// the user about it, see line 113 in [2]. +// +// References: +// +// [1] https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html +// [2] https://salsa.debian.org/apt-team/apt/-/blob/d9039b24/apt-pkg/contrib/netrc.cc +// [3] https://salsa.debian.org/apt-team/apt/-/blob/4e04cbaf/methods/aptmethod.h#L560 +// [4] https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html +// [5] https://daniel.haxx.se/blog/2022/05/31/netrc-pains/ +func findCredentialsFile(query *credentialsQuery, netrc io.Reader, creds *credentials) error { + s := bufio.NewScanner(netrc) + s.Split(bufio.ScanWords) + p := netrcParser{ + query: query, + scanner: s, + creds: creds, + } + var err error + for state := netrcInvalid; state != nil; { + state, err = state(&p) + } + if err := p.scanner.Err(); err != nil { + return err + } + return err +} + +type netrcState func(*netrcParser) (netrcState, error) + +func netrcInvalid(p *netrcParser) (netrcState, error) { + for p.scanner.Scan() { + if p.scanner.Text() == "machine" { + return netrcMachine, nil + } + } + return nil, nil +} + +func netrcMachine(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting machine text") + } + token := p.scanner.Text() + if i := strings.Index(token, "://"); i != -1 { + if token[0:i] != p.query.scheme { + return netrcInvalid, nil + } + token = token[i+3:] + } else if p.query.needScheme { + return netrcInvalid, nil + } + if !strings.HasPrefix(token, p.query.host) { + return netrcInvalid, nil + } + token = token[len(p.query.host):] + if len(token) > 0 { + if token[0] == ':' { + if p.query.port == "" { + return netrcInvalid, nil + } + token = token[1:] + if !strings.HasPrefix(token, p.query.port) { + return netrcInvalid, nil + } + token = token[len(p.query.port):] + } + if !strings.HasPrefix(p.query.path, token) { + return netrcInvalid, nil + } + } + return netrcGoodMachine, nil +} + +func netrcGoodMachine(p *netrcParser) (netrcState, error) { +loop: + for p.scanner.Scan() { + switch p.scanner.Text() { + case "login": + return netrcUsername, nil + case "password": + return netrcPassword, nil + case "machine": + break loop + } + } + return nil, nil +} + +func netrcUsername(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting username text") + } + p.creds.Username = p.scanner.Text() + return netrcGoodMachine, nil +} + +func netrcPassword(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting password text") + } + p.creds.Password = p.scanner.Text() + return netrcGoodMachine, nil +} diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go new file mode 100644 index 00000000..87e657c9 --- /dev/null +++ b/internal/archive/credentials_test.go @@ -0,0 +1,253 @@ +package archive_test + +import ( + "io/ioutil" + "os" + "path/filepath" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/archive" +) + +type matchTest struct { + url string + err string + username string + password string +} + +type credentialsTest struct { + summary string + authDir map[string]string + matchTests []matchTest +} + +var credentialsTests = []credentialsTest{{ + summary: "Parsing", + authDir: map[string]string{ + "50test-logins": ` +machine example.netter login bar password foo +machine example.net login foo password bar + +machine example.org:90 login apt password apt +machine example.org:8080 +login +example password foobar + +machine example.org +login anonymous +password pass + +machine example.com/foo login user1 unknown token password pass1 +machine example.com/bar password pass2 login user2 + unknown token +machine example.com/user login user +machine example.netter login unused password firstentry +machine socks5h://example.last/debian login debian password rules +`, + }, + matchTests: []matchTest{ + {"https://example.net/foo", "", "foo", "bar"}, + {"https://user:pass@example.net/foo", "", "user", "pass"}, + {"https://example.org:90/foo", "", "apt", "apt"}, + {"https://example.org:8080/foo", "", "example", "foobar"}, + {"https://example.net:42/foo", "", "foo", "bar"}, + {"https://example.org/foo", "", "anonymous", "pass"}, + {"https://example.com/apt", "", "", ""}, + {"https://example.com/foo", "", "user1", "pass1"}, + {"https://example.com/fooo", "", "user1", "pass1"}, + {"https://example.com/fo", "", "", ""}, + {"https://example.com/bar", "", "user2", "pass2"}, + {"https://example.com/user", "", "user", ""}, + {"socks5h://example.last/debian", "", "debian", "rules"}, + {"socks5h://example.debian/", "", "", ""}, + {"socks5h://user:pass@example.debian/", "", "user", "pass"}, + }, +}, { + summary: "Bad file: No machine", + authDir: map[string]string{ + "50test-logins.conf": ` +foo example.org login foo1 password bar +machin example.org login foo2 password bar +machine2 example.org login foo3 password bar +`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "", ""}, + }, +}, { + summary: "Bad file: Ends machine", + authDir: map[string]string{ + "50test-logins.conf": ` +machine example.org login foo1 password bar +machine`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "foo1", "bar"}, + {"https://example.net/foo", ".*\\breached end of file while expecting machine text\\b.*", "", ""}, + {"https://foo:bar@example.net/foo", "", "foo", "bar"}, + }, +}, { + summary: "Bad file: Ends login", + authDir: map[string]string{ + "50test-logins.conf": ` +machine example.org login foo1 password bar +machine example.net login +`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "foo1", "bar"}, + {"https://example.net/foo", ".*\\breached end of file while expecting username text\\b.*", "", ""}, + {"https://foo:bar@example.net/foo", "", "foo", "bar"}, + }, +}, { + summary: "Matches only HTTPS", + authDir: map[string]string{ + "50test-logins.conf": ` +machine https.example login foo1 password bar +machine http://http.example login foo1 password bar +`, + }, + matchTests: []matchTest{ + {"https://https.example/foo", "", "foo1", "bar"}, + {"http://https.example/foo", "", "", ""}, + {"http://http.example/foo", "", "foo1", "bar"}, + {"https://http.example/foo", "", "", ""}, + }, +}, { + summary: "Password is machine", + authDir: map[string]string{ + "50test-logins.conf": ` +machine http://site1.com login u1 password machine +machine http://site2.com login u2 password p2 +`, + }, + matchTests: []matchTest{ + {"http://site1.com/foo", "", "u1", "machine"}, + {"http://site2.com/bar", "", "", ""}, + }, +}, { + summary: "Multiple login and password tokens", + authDir: map[string]string{ + "50test-logins.conf": ` +machine http://site1.com login a login b password c login d password e +machine http://site2.com login f password g +`, + }, + matchTests: []matchTest{ + {"http://site1.com/foo", "", "d", "e"}, + {"http://site2.com/bar", "", "f", "g"}, + }, +}, { + summary: "Empty auth dir", + authDir: map[string]string{}, + matchTests: []matchTest{ + {"https://example.com/foo", "", "", ""}, + {"http://zombo.com", "", "", ""}, + }, +}, { + summary: "Invalid input URL", + authDir: map[string]string{ + "logins": ` +machine login foo password bar login baz +`, + }, + matchTests: []matchTest{ + {":http:foo", "parse \":http:foo\": missing protocol scheme", "", ""}, + {"", "", "", ""}, // this is fine URL apparently, but won't ever match + {"https://login", "", "baz", "bar"}, + }, +}, { + summary: "First entry wins", + authDir: map[string]string{ + "logins": ` +machine http://example.com/foo login a password b +machine http://example.com/foo login c password d + +machine example.com/bar login e password f +machine http://example.com/bar login g password h + +machine http://example.com/baz login i password j +machine http://example.com/baz/qux login k password l +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "", "a", "b"}, + {"http://example.com/bar", "", "g", "h"}, + {"http://example.com/baz/qux", "", "i", "j"}, + }, +}, { + summary: "First file wins", + authDir: map[string]string{ + "10first": ` +machine http://example.com/foo login a password b +machine example.com/bar login e password f +machine http://example.com/baz login i password j +`, + "50second": ` +machine http://example.com/foo login b password c +machine http://example.com/bar login g password h +machine http://example.com/baz/qux login k password l +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "", "a", "b"}, + {"http://example.com/bar", "", "g", "h"}, + {"http://example.com/baz/qux", "", "i", "j"}, + }, +}} + +func (s *S) TestFindCredentialsDir(c *C) { + for _, credentialsTest := range credentialsTests { + authDir := c.MkDir() + for path, data := range credentialsTest.authDir { + fpath := filepath.Join(authDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = ioutil.WriteFile(fpath, []byte(data), 0644) + c.Assert(err, IsNil) + } + + for _, matchTest := range credentialsTest.matchTests { + c.Logf("Summary: %s for URL %s", credentialsTest.summary, matchTest.url) + creds, err := archive.FindCredentialsDir(matchTest.url, authDir) + if matchTest.err != "" { + c.Assert(err, ErrorMatches, matchTest.err) + } else { + c.Assert(err, IsNil) + } + c.Assert(creds.Username, Equals, matchTest.username) + c.Assert(creds.Password, Equals, matchTest.password) + } + } +} + +func (s *S) TestFindCredentialsDirMissingDir(c *C) { + var creds, emptyCreds archive.Credentials + var err error + + workDir := c.MkDir() + authDir := filepath.Join(workDir, "auth.conf.d") + + creds, err = archive.FindCredentialsDir("https://example.com/foo/bar", authDir) + c.Assert(err, IsNil) + c.Assert(creds, Equals, emptyCreds) + + err = os.Mkdir(authDir, 0755) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentialsDir("https://example.com/foo/bar", authDir) + c.Assert(err, IsNil) + c.Assert(creds, Equals, emptyCreds) + + confFile := filepath.Join(authDir, "example") + err = os.WriteFile(confFile, []byte("machine example.com login admin password swordfish"), 0600) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentialsDir("https://example.com/foo/bar", authDir) + c.Assert(err, IsNil) + c.Assert(creds, Not(Equals), emptyCreds) + c.Assert(creds.Username, Equals, "admin") + c.Assert(creds.Password, Equals, "swordfish") +} diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index c2bffea3..2dfbf594 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -14,3 +14,6 @@ func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) bulkDo = _bulkDo } } + +type Credentials = credentials +var FindCredentialsDir = findCredentialsDir