Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add multiple URL support for externals #4112

Merged
merged 1 commit into from
Dec 1, 2024
+97 −33
Merged

feat: Add multiple URL support for externals #4112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ listed in [`.chezmoiignore`](chezmoiignore.md)), all entries within the file are
also ignored.

Entries are indexed by target name relative to the directory of the
`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` field.
`type` can be either `file`, `archive`, `archive-file`, or `git-repo`. If the
entry's parent directories do not already exist in the source state then chezmoi
will create them as regular directories.
`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` and/or a
`urls` field. `type` can be either `file`, `archive`, `archive-file`, or
`git-repo`. If the entry's parent directories do not already exist in the source
state then chezmoi will create them as regular directories.

Entries may have the following fields:

Expand All @@ -39,6 +39,7 @@ Entries may have the following fields:
| `refreshPeriod` | duration | `0` | Refresh period |
| `stripComponents` | int | `0` | Number of leading directory components to strip from archives |
| `url` | string | *none* | URL |
| `urls` | []string | *none* | Extra URLs to try, in order |
| `checksum.sha256` | string | *none* | Expected SHA256 checksum of data |
| `checksum.sha384` | string | *none* | Expected SHA384 checksum of data |
| `checksum.sha512` | string | *none* | Expected SHA512 checksum of data |
Expand All @@ -49,7 +50,8 @@ Entries may have the following fields:
| `pull.args` | []string | *none* | Extra args to `git pull` |
| `archive.extractAppleDouble` | bool | `false` | If `true`, AppleDouble files are extracted |

`url` must be an `https://`, `http://`, or `file://` URL.
`url` must be an `https://`, `http://`, or `file://` URL. If `urls` is specified
then they are tried in order and the first URL that succeeds is used.

If any of the optional `checksum.sha256`, `checksum.sha384`, or
`checksum.sha512` fields are set, chezmoi will verify that the downloaded data
Expand Down
91 changes: 63 additions & 28 deletions internal/chezmoi/sourcestate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package chezmoi
import (
"bufio"
"bytes"
"cmp"
"context"
"crypto/sha256"
"encoding/hex"
Expand Down Expand Up @@ -108,6 +109,7 @@ type External struct {
RefreshPeriod Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"`
StripComponents int `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"`
URL string `json:"url" toml:"url" yaml:"url"`
URLs []string `json:"urls" toml:"urls" yaml:"urls"`
sourceAbsPath AbsPath
}

Expand Down Expand Up @@ -1558,11 +1560,12 @@ func (s *SourceState) executeTemplate(templateAbsPath AbsPath) ([]byte, error) {
func (s *SourceState) getExternalDataRaw(
ctx context.Context,
externalRelPath RelPath,
external *External,
urlStr string,
refreshPeriod Duration,
options *ReadOptions,
) ([]byte, error) {
// Handle file:// URLs by always reading from disk.
switch urlStruct, err := url.Parse(external.URL); {
switch urlStruct, err := url.Parse(urlStr); {
case err != nil:
return nil, err
case urlStruct.Scheme == "file":
Expand All @@ -1585,7 +1588,7 @@ func (s *SourceState) getExternalDataRaw(
if options != nil {
refreshExternals = options.RefreshExternals
}
urlSHA256 := sha256.Sum256([]byte(external.URL))
urlSHA256 := sha256.Sum256([]byte(urlStr))
cacheKey := hex.EncodeToString(urlSHA256[:])
cachedDataAbsPath := s.cacheDirAbsPath.JoinString("external", cacheKey)
switch refreshExternals {
Expand All @@ -1594,7 +1597,7 @@ func (s *SourceState) getExternalDataRaw(
case RefreshExternalsAuto:
// Use the cache, if available and within the refresh period.
if fileInfo, err := s.baseSystem.Stat(cachedDataAbsPath); err == nil {
if external.RefreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(external.RefreshPeriod)).After(now) {
if refreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(refreshPeriod)).After(now) {
if data, err := s.baseSystem.ReadFile(cachedDataAbsPath); err == nil {
return data, nil
}
Expand All @@ -1608,7 +1611,7 @@ func (s *SourceState) getExternalDataRaw(
}
}

req, err := http.NewRequestWithContext(ctx, http.MethodGet, external.URL, http.NoBody)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, http.NoBody)
if err != nil {
return nil, err
}
Expand All @@ -1627,7 +1630,7 @@ func (s *SourceState) getExternalDataRaw(
return nil, err
}
if resp.StatusCode < http.StatusOK || http.StatusMultipleChoices <= resp.StatusCode {
return nil, fmt.Errorf("%s: %s: %s", externalRelPath, external.URL, resp.Status)
return nil, fmt.Errorf("%s: %s: %s", externalRelPath, urlStr, resp.Status)
}

if err := MkdirAll(s.baseSystem, cachedDataAbsPath.Dir(), 0o700); err != nil {
Expand All @@ -1643,17 +1646,47 @@ func (s *SourceState) getExternalDataRaw(
return data, nil
}

// getExternalDataAndURL iterates over external.URL and external.URLs, returning
// the first data that is downloaded successfully and the URL it was downloaded
// from.
func (s *SourceState) getExternalDataAndURL(
ctx context.Context,
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, string, error) {
var firstURLStr string
var firstErr error
for _, urlStr := range append([]string{external.URL}, external.URLs...) {
if urlStr == "" {
continue
}
data, err := s.getExternalDataRaw(ctx, externalRelPath, urlStr, external.RefreshPeriod, options)
if err == nil {
return data, urlStr, nil
}
if firstURLStr == "" {
firstURLStr = urlStr
firstErr = err
}
}
if firstURLStr == "" {
return nil, "", fmt.Errorf("%s: no URL", externalRelPath)
}
return nil, firstURLStr, firstErr
}

// getExternalData reads the external data for externalRelPath from
// external.URL.
// external.URL or external.URLs, returning the data and URL.
func (s *SourceState) getExternalData(
ctx context.Context,
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, error) {
data, err := s.getExternalDataRaw(ctx, externalRelPath, external, options)
) ([]byte, string, error) {
data, urlStr, err := s.getExternalDataAndURL(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
return nil, "", err
}

var errs []error
Expand Down Expand Up @@ -1710,19 +1743,19 @@ func (s *SourceState) getExternalData(
}

if len(errs) != 0 {
return nil, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...))
return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...))
}

if external.Encrypted {
data, err = s.encryption.Decrypt(data)
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}
}

data, err = decompress(external.Decompress, data)
if err != nil {
return nil, fmt.Errorf("%s: %w", externalRelPath, err)
return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, err)
}

if external.Filter.Command != "" {
Expand All @@ -1731,11 +1764,11 @@ func (s *SourceState) getExternalData(
cmd.Stderr = os.Stderr
data, err = chezmoilog.LogCmdOutput(s.logger, cmd)
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}
}

return data, nil
return data, urlStr, nil
}

// newSourceStateDir returns a new SourceStateDir.
Expand Down Expand Up @@ -2312,7 +2345,7 @@ func (s *SourceState) readExternalArchive(
external *External,
options *ReadOptions,
) (map[RelPath][]SourceStateEntry, error) {
data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -2470,7 +2503,7 @@ func (s *SourceState) readExternalArchive(
sourceStateEntries[targetRelPath] = append(sourceStateEntries[targetRelPath], sourceStateEntry)
return nil
}); err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
}

return s.populateImplicitParentDirs(externalRelPath, external, sourceStateEntries), nil
Expand All @@ -2483,16 +2516,16 @@ func (s *SourceState) readExternalArchiveData(
externalRelPath RelPath,
external *External,
options *ReadOptions,
) ([]byte, ArchiveFormat, error) {
data, err := s.getExternalData(ctx, externalRelPath, external, options)
) ([]byte, string, ArchiveFormat, error) {
data, urlStr, err := s.getExternalData(ctx, externalRelPath, external, options)
if err != nil {
return nil, ArchiveFormatUnknown, err
return nil, "", ArchiveFormatUnknown, err
}

externalURL, err := url.Parse(external.URL)
externalURL, err := url.Parse(urlStr)
if err != nil {
err := fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
return nil, ArchiveFormatUnknown, err
err := fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err)
return nil, urlStr, ArchiveFormatUnknown, err
}
urlPath := externalURL.Path
if external.Encrypted {
Expand All @@ -2504,7 +2537,7 @@ func (s *SourceState) readExternalArchiveData(
format = GuessArchiveFormat(urlPath, data)
}

return data, format, nil
return data, urlStr, format, nil
}

// readExternalArchiveFile reads a file from an external archive and returns its
Expand All @@ -2520,7 +2553,7 @@ func (s *SourceState) readExternalArchiveFile(
return nil, fmt.Errorf("%s: missing path", externalRelPath)
}

data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -2611,7 +2644,7 @@ func (s *SourceState) readExternalArchiveFile(
return nil, err
}
if sourceStateEntry == nil {
return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, external.URL)
return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, urlStr)
}

return s.populateImplicitParentDirs(externalRelPath, external, map[RelPath][]SourceStateEntry{
Expand Down Expand Up @@ -2728,7 +2761,8 @@ func (s *SourceState) readExternalFile(
options *ReadOptions,
) (map[RelPath][]SourceStateEntry, error) {
contentsFunc := sync.OnceValues(func() ([]byte, error) {
return s.getExternalData(ctx, externalRelPath, external, options)
data, _, err := s.getExternalData(ctx, externalRelPath, external, options)
return data, err
})
fileAttr := FileAttr{
Empty: true,
Expand Down Expand Up @@ -2880,7 +2914,8 @@ func (e *External) Path() AbsPath {
}

func (e *External) OriginString() string {
return e.URL + " defined in " + e.sourceAbsPath.String()
urlStr := cmp.Or(append([]string{e.URL}, e.URLs...)...)
return urlStr + " defined in " + e.sourceAbsPath.String()
}

// canonicalSourceStateEntry returns the canonical SourceStateEntry for the
Expand Down
27 changes: 27 additions & 0 deletions internal/cmd/testdata/scripts/externalfileurl.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,38 @@
# test that chezmoi apply reads from file:// URLs
exec chezmoi apply
cmp $HOME/.file golden/.file

chhome home2/user

# test that chezmoi apply reads from the first found URL
exec chezmoi apply
cmp $HOME/.file golden/.file2

chhome home3/user

# test that chezmoi apply returns an error from the first URL if all URLs return errors
! exec chezmoi apply
[!windows] stderr 'home3/user/\.local/share/file.txt: no such file or directory'
[windows] stderr 'home3/user/\.local/share/file.txt: The system cannot find the file specified\.'

-- golden/.file --
# contents of .file
-- golden/.file2 --
# contents of .file2
-- home/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt"
-- home/user/.local/share/file.txt --
# contents of .file
-- home2/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"]
-- home2/user/.local/share/file2.txt --
# contents of .file2
-- home3/user/.local/share/chezmoi/.chezmoiexternal.toml --
[".file"]
type = "file"
url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt"
urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"]
Loading