Skip to content

Commit

Permalink
feat: Add multiple URL support for externals
Browse files Browse the repository at this point in the history
  • Loading branch information
@twpayne
twpayne committed Dec 1, 2024
1 parent a5ea6ca commit 5f5e106
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 33 deletions.
12 changes: 7 additions & 5 deletions assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ listed in [`.chezmoiignore`](chezmoiignore.md)), all entries within the file are
also ignored.

Entries are indexed by target name relative to the directory of the
`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` field.
`type` can be either `file`, `archive`, `archive-file`, or `git-repo`. If the
entry's parent directories do not already exist in the source state then chezmoi
will create them as regular directories.
`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` and/or a
`urls` field. `type` can be either `file`, `archive`, `archive-file`, or
`git-repo`. If the entry's parent directories do not already exist in the source
state then chezmoi will create them as regular directories.

Entries may have the following fields:

Expand All @@ -39,6 +39,7 @@ Entries may have the following fields:
| `refreshPeriod` | duration | `0` | Refresh period |
| `stripComponents` | int | `0` | Number of leading directory components to strip from archives |
| `url` | string | *none* | URL |
| `urls` | []string | *none* | Extra URLs to try, in order |
| `checksum.sha256` | string | *none* | Expected SHA256 checksum of data |
| `checksum.sha384` | string | *none* | Expected SHA384 checksum of data |
| `checksum.sha512` | string | *none* | Expected SHA512 checksum of data |
Expand All @@ -49,7 +50,8 @@ Entries may have the following fields:
| `pull.args` | []string | *none* | Extra args to `git pull` |
| `archive.extractAppleDouble` | bool | `false` | If `true`, AppleDouble files are extracted |

`url` must be an `https://`, `http://`, or `file://` URL.
`url` must be an `https://`, `http://`, or `file://` URL. If `urls` is specified
then they are tried in order and the first URL that succeeds is used.

If any of the optional `checksum.sha256`, `checksum.sha384`, or
`checksum.sha512` fields are set, chezmoi will verify that the downloaded data
Expand Down
91 changes: 63 additions & 28 deletions internal/chezmoi/sourcestate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package chezmoi
import (
"bufio"
"bytes"
"cmp"
"context"
"crypto/sha256"
"encoding/hex"
Expand Down Expand Up @@ -108,6 +109,7 @@ type External struct {
RefreshPeriod Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"`
StripComponents int `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"`
URL string `json:"url" toml:"url" yaml:"url"`
URLs []string `json:"urls" toml:"urls" yaml:"urls"`
sourceAbsPath AbsPath
}

Expand Down Expand Up @@ -1558,11 +1560,12 @@ func (s *SourceState) executeTemplate(templateAbsPath AbsPath) ([]byte, error) {
func (s *SourceState) getExternalDataRaw(
ctx context.Context,
externalRelPath RelPath,
external *External,
urlStr string,
refreshPeriod Duration,
options *ReadOptions,
) ([]byte, error) {
// Handle file:// URLs by always reading from disk.
switch urlStruct, err := url.Parse(external.URL); {
switch urlStruct, err := url.Parse(urlStr); {
case err != nil:
return nil, err
case urlStruct.Scheme == "file":
Expand All @@ -1585,7 +1588,7 @@ func (s *SourceState) getExternalDataRaw(
if options != nil {
refreshExternals = options.RefreshExternals
}
urlSHA256 := sha256.Sum256([]byte(external.URL))
urlSHA256 := sha256.Sum256([]byte(urlStr))
cacheKey := hex.EncodeToString(urlSHA256[:])
cachedDataAbsPath := s.cacheDirAbsPath.JoinString("external", cacheKey)
switch refreshExternals {
Expand All @@ -1594,7 +1597,7 @@ func (s *SourceState) getExternalDataRaw(
case RefreshExternalsAuto:
// Use the cache, if available and within the refresh period.
if fileInfo, err := s.baseSystem.Stat(cachedDataAbsPath); err == nil {
if external.RefreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(external.RefreshPeriod)).After(now) {
if refreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(refreshPeriod)).After(now) {
if data, err := s.baseSystem.ReadFile(cachedDataAbsPath); err == nil {
return data, nil
}
Expand All @@ -1608,7 +1611,7 @@ func (s *SourceState) getExternalDataRaw(
}
}

req, err := http.NewRequestWithContext(ctx, http.MethodGet, external.URL, http.NoBody)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, http.NoBody)
if err != nil {
return nil, err
}
Expand All @@ -1627,7 +1630,7 @@ func (s *SourceState) getExternalDataRaw(
return nil, err
}
if resp.StatusCode < http.StatusOK || http.StatusMultipleChoices <= resp.StatusCode {
return nil, fmt.Errorf("%s: %s: %s", externalRelPath, external.URL, resp.Status)
return nil, fmt.Errorf("%s: %s: %s", externalRelPath, urlStr, resp.Status)
}

if err := MkdirAll(s.baseSystem, cachedDataAbsPath.Dir(), 0o700); err != nil {
Expand All @@ -1643,17 +1646,47 @@ func (s *SourceState) getExternalDataRaw(
return data, nil
}

// getExternalDataAndURL iterates over external.URL and external.URLs, returning
// the first data that is downloaded successfully and the URL it was downloaded
// from.
func (s *SourceState) getExternalDataAndURL(
ctx context.Context,
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, string, error) {
var firstURLStr string
var firstErr error
for _, urlStr := range append([]string{external.URL}, external.URLs...) {
if urlStr == "" {
continue
}
data, err := s.getExternalDataRaw(ctx, externalRelPath, urlStr, external.RefreshPeriod, options)
if err == nil {
return data, urlStr, nil
}
if firstURLStr == "" {
firstURLStr = urlStr
firstErr = err
}
}
if firstURLStr == "" {
return nil, "", fmt.Errorf("%s: no URL", externalRelPath)
}
return nil, firstURLStr, firstErr
}

// getExternalData reads the external data for externalRelPath from
// external.URL.
// external.URL or external.URLs, returning the data and URL.
func (s *SourceState) getExternalData(
ctx context.Context,
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, error) {
data, err := s.getExternalDataRaw(ctx, externalRelPath, external, options)
) ([]byte, string, error) {
data, urlStr, err := s.getExternalDataAndURL(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
return nil, "", err
}

var errs []error
Expand Down Expand Up @@ -1710,19 +1743,19 @@ func (s *SourceState) getExternalData(
}

if len(errs) != 0 {
return nil, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...))
return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...))
}

if external.Encrypted {
data, err = s.encryption.Decrypt(data)
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}
}

data, err = decompress(external.Decompress, data)
if err != nil {
return nil, fmt.Errorf("%s: %w", externalRelPath, err)
return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, err)
}

if external.Filter.Command != "" {
Expand All @@ -1731,11 +1764,11 @@ func (s *SourceState) getExternalData(
cmd.Stderr = os.Stderr
data, err = chezmoilog.LogCmdOutput(s.logger, cmd)
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}
}

return data, nil
return data, urlStr, nil
}

// newSourceStateDir returns a new SourceStateDir.
Expand Down Expand Up @@ -2312,7 +2345,7 @@ func (s *SourceState) readExternalArchive(
external *External,
options *ReadOptions,
) (map[RelPath][]SourceStateEntry, error) {
data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -2470,7 +2503,7 @@ func (s *SourceState) readExternalArchive(
sourceStateEntries[targetRelPath] = append(sourceStateEntries[targetRelPath], sourceStateEntry)
return nil
}); err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}

return s.populateImplicitParentDirs(externalRelPath, external, sourceStateEntries), nil
Expand All @@ -2483,16 +2516,16 @@ func (s *SourceState) readExternalArchiveData(
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, ArchiveFormat, error) {
data, err := s.getExternalData(ctx, externalRelPath, external, options)
) ([]byte, string, ArchiveFormat, error) {
data, urlStr, err := s.getExternalData(ctx, externalRelPath, external, options)
if err != nil {
return nil, ArchiveFormatUnknown, err
return nil, "", ArchiveFormatUnknown, err
}

externalURL, err := url.Parse(external.URL)
externalURL, err := url.Parse(urlStr)
if err != nil {
err := fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, ArchiveFormatUnknown, err
err := fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
return nil, urlStr, ArchiveFormatUnknown, err
}
urlPath := externalURL.Path
if external.Encrypted {
Expand All @@ -2504,7 +2537,7 @@ func (s *SourceState) readExternalArchiveData(
format = GuessArchiveFormat(urlPath, data)
}

return data, format, nil
return data, urlStr, format, nil
}

// readExternalArchiveFile reads a file from an external archive and returns its
Expand All @@ -2520,7 +2553,7 @@ func (s *SourceState) readExternalArchiveFile(
return nil, fmt.Errorf("%s: missing path", externalRelPath)
}

data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -2611,7 +2644,7 @@ func (s *SourceState) readExternalArchiveFile(
return nil, err
}
if sourceStateEntry == nil {
return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, external.URL)
return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, urlStr)
}

return s.populateImplicitParentDirs(externalRelPath, external, map[RelPath][]SourceStateEntry{
Expand Down Expand Up @@ -2728,7 +2761,8 @@ func (s *SourceState) readExternalFile(
options *ReadOptions,
) (map[RelPath][]SourceStateEntry, error) {
contentsFunc := sync.OnceValues(func() ([]byte, error) {
return s.getExternalData(ctx, externalRelPath, external, options)
data, _, err := s.getExternalData(ctx, externalRelPath, external, options)
return data, err
})
fileAttr := FileAttr{
Empty: true,
Expand Down Expand Up @@ -2880,7 +2914,8 @@ func (e *External) Path() AbsPath {
}

func (e *External) OriginString() string {
return e.URL + " defined in " + e.sourceAbsPath.String()
urlStr := cmp.Or(append([]string{e.URL}, e.URLs...)...)
return urlStr + " defined in " + e.sourceAbsPath.String()
}

// canonicalSourceStateEntry returns the canonical SourceStateEntry for the
Expand Down
27 changes: 27 additions & 0 deletions internal/cmd/testdata/scripts/externalfileurl.txtar
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,38 @@
# test that chezmoi apply reads from file:// URLs
exec chezmoi apply
cmp $HOME/.file golden/.file

chhome home2/user

# test that chezmoi apply reads from the first found URL
exec chezmoi apply
cmp $HOME/.file golden/.file2

chhome home3/user

# test that chezmoi apply returns an error from the first URL if all URLs return errors
! exec chezmoi apply
[!windows] stderr 'home3/user/\.local/share/file.txt: no such file or directory'
[windows] stderr 'home3/user/\.local/share/file.txt: The system cannot find the file specified\.'

-- golden/.file --
# contents of .file
-- golden/.file2 --
# contents of .file2
-- home/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt"
-- home/user/.local/share/file.txt --
# contents of .file
-- home2/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"]
-- home2/user/.local/share/file2.txt --
# contents of .file2
-- home3/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt"
urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"]

0 comments on commit 5f5e106

Please sign in to comment.