diff --git a/assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md b/assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md index bd6f7d225e8..6e8e880a362 100644 --- a/assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md +++ b/assets/chezmoi.io/docs/reference/special-files/chezmoiexternal-format.md @@ -16,10 +16,10 @@ listed in [`.chezmoiignore`](chezmoiignore.md)), all entries within the file are also ignored. Entries are indexed by target name relative to the directory of the -`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` field. -`type` can be either `file`, `archive`, `archive-file`, or `git-repo`. If the -entry's parent directories do not already exist in the source state then chezmoi -will create them as regular directories. +`.chezmoiexternal.$FORMAT` file, and must have a `type` and a `url` and/or a +`urls` field. `type` can be either `file`, `archive`, `archive-file`, or +`git-repo`. If the entry's parent directories do not already exist in the source +state then chezmoi will create them as regular directories. Entries may have the following fields: @@ -39,6 +39,7 @@ Entries may have the following fields: | `refreshPeriod` | duration | `0` | Refresh period | | `stripComponents` | int | `0` | Number of leading directory components to strip from archives | | `url` | string | *none* | URL | +| `urls` | []string | *none* | Extra URLs to try, in order | | `checksum.sha256` | string | *none* | Expected SHA256 checksum of data | | `checksum.sha384` | string | *none* | Expected SHA384 checksum of data | | `checksum.sha512` | string | *none* | Expected SHA512 checksum of data | @@ -49,7 +50,8 @@ Entries may have the following fields: | `pull.args` | []string | *none* | Extra args to `git pull` | | `archive.extractAppleDouble` | bool | `false` | If `true`, AppleDouble files are extracted | -`url` must be an `https://`, `http://`, or `file://` URL. +`url` must be an `https://`, `http://`, or `file://` URL. If `urls` is specified +then they are tried in order and the first URL that succeeds is used. If any of the optional `checksum.sha256`, `checksum.sha384`, or `checksum.sha512` fields are set, chezmoi will verify that the downloaded data diff --git a/internal/chezmoi/sourcestate.go b/internal/chezmoi/sourcestate.go index 35b9055aaeb..2b79051ff0f 100644 --- a/internal/chezmoi/sourcestate.go +++ b/internal/chezmoi/sourcestate.go @@ -5,6 +5,7 @@ package chezmoi import ( "bufio" "bytes" + "cmp" "context" "crypto/sha256" "encoding/hex" @@ -108,6 +109,7 @@ type External struct { RefreshPeriod Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"` StripComponents int `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"` URL string `json:"url" toml:"url" yaml:"url"` + URLs []string `json:"urls" toml:"urls" yaml:"urls"` sourceAbsPath AbsPath } @@ -1558,11 +1560,12 @@ func (s *SourceState) executeTemplate(templateAbsPath AbsPath) ([]byte, error) { func (s *SourceState) getExternalDataRaw( ctx context.Context, externalRelPath RelPath, - external *External, + urlStr string, + refreshPeriod Duration, options *ReadOptions, ) ([]byte, error) { // Handle file:// URLs by always reading from disk. - switch urlStruct, err := url.Parse(external.URL); { + switch urlStruct, err := url.Parse(urlStr); { case err != nil: return nil, err case urlStruct.Scheme == "file": @@ -1585,7 +1588,7 @@ func (s *SourceState) getExternalDataRaw( if options != nil { refreshExternals = options.RefreshExternals } - urlSHA256 := sha256.Sum256([]byte(external.URL)) + urlSHA256 := sha256.Sum256([]byte(urlStr)) cacheKey := hex.EncodeToString(urlSHA256[:]) cachedDataAbsPath := s.cacheDirAbsPath.JoinString("external", cacheKey) switch refreshExternals { @@ -1594,7 +1597,7 @@ func (s *SourceState) getExternalDataRaw( case RefreshExternalsAuto: // Use the cache, if available and within the refresh period. if fileInfo, err := s.baseSystem.Stat(cachedDataAbsPath); err == nil { - if external.RefreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(external.RefreshPeriod)).After(now) { + if refreshPeriod == 0 || fileInfo.ModTime().Add(time.Duration(refreshPeriod)).After(now) { if data, err := s.baseSystem.ReadFile(cachedDataAbsPath); err == nil { return data, nil } @@ -1608,7 +1611,7 @@ func (s *SourceState) getExternalDataRaw( } } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, external.URL, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, http.NoBody) if err != nil { return nil, err } @@ -1627,7 +1630,7 @@ func (s *SourceState) getExternalDataRaw( return nil, err } if resp.StatusCode < http.StatusOK || http.StatusMultipleChoices <= resp.StatusCode { - return nil, fmt.Errorf("%s: %s: %s", externalRelPath, external.URL, resp.Status) + return nil, fmt.Errorf("%s: %s: %s", externalRelPath, urlStr, resp.Status) } if err := MkdirAll(s.baseSystem, cachedDataAbsPath.Dir(), 0o700); err != nil { @@ -1643,17 +1646,47 @@ func (s *SourceState) getExternalDataRaw( return data, nil } +// getExternalDataAndURL iterates over external.URL and external.URLs, returning +// the first data that is downloaded successfully and the URL it was downloaded +// from. +func (s *SourceState) getExternalDataAndURL( + ctx context.Context, + externalRelPath RelPath, + external *External, + options *ReadOptions, +) ([]byte, string, error) { + var firstURLStr string + var firstErr error + for _, urlStr := range append([]string{external.URL}, external.URLs...) { + if urlStr == "" { + continue + } + data, err := s.getExternalDataRaw(ctx, externalRelPath, urlStr, external.RefreshPeriod, options) + if err == nil { + return data, urlStr, nil + } + if firstURLStr == "" { + firstURLStr = urlStr + firstErr = err + } + } + if firstURLStr == "" { + return nil, "", fmt.Errorf("%s: no URL", externalRelPath) + } + return nil, firstURLStr, firstErr +} + // getExternalData reads the external data for externalRelPath from -// external.URL. +// external.URL or external.URLs, returning the data and URL. func (s *SourceState) getExternalData( ctx context.Context, externalRelPath RelPath, external *External, options *ReadOptions, -) ([]byte, error) { - data, err := s.getExternalDataRaw(ctx, externalRelPath, external, options) +) ([]byte, string, error) { + data, urlStr, err := s.getExternalDataAndURL(ctx, externalRelPath, external, options) if err != nil { - return nil, err + return nil, "", err } var errs []error @@ -1710,19 +1743,19 @@ func (s *SourceState) getExternalData( } if len(errs) != 0 { - return nil, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...)) + return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, errors.Join(errs...)) } if external.Encrypted { data, err = s.encryption.Decrypt(data) if err != nil { - return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err) + return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err) } } data, err = decompress(external.Decompress, data) if err != nil { - return nil, fmt.Errorf("%s: %w", externalRelPath, err) + return nil, urlStr, fmt.Errorf("%s: %w", externalRelPath, err) } if external.Filter.Command != "" { @@ -1731,11 +1764,11 @@ func (s *SourceState) getExternalData( cmd.Stderr = os.Stderr data, err = chezmoilog.LogCmdOutput(s.logger, cmd) if err != nil { - return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err) + return nil, urlStr, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err) } } - return data, nil + return data, urlStr, nil } // newSourceStateDir returns a new SourceStateDir. @@ -2312,7 +2345,7 @@ func (s *SourceState) readExternalArchive( external *External, options *ReadOptions, ) (map[RelPath][]SourceStateEntry, error) { - data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options) + data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options) if err != nil { return nil, err } @@ -2470,7 +2503,7 @@ func (s *SourceState) readExternalArchive( sourceStateEntries[targetRelPath] = append(sourceStateEntries[targetRelPath], sourceStateEntry) return nil }); err != nil { - return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err) + return nil, fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err) } return s.populateImplicitParentDirs(externalRelPath, external, sourceStateEntries), nil @@ -2483,16 +2516,16 @@ func (s *SourceState) readExternalArchiveData( externalRelPath RelPath, external *External, options *ReadOptions, -) ([]byte, ArchiveFormat, error) { - data, err := s.getExternalData(ctx, externalRelPath, external, options) +) ([]byte, string, ArchiveFormat, error) { + data, urlStr, err := s.getExternalData(ctx, externalRelPath, external, options) if err != nil { - return nil, ArchiveFormatUnknown, err + return nil, "", ArchiveFormatUnknown, err } - externalURL, err := url.Parse(external.URL) + externalURL, err := url.Parse(urlStr) if err != nil { - err := fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err) - return nil, ArchiveFormatUnknown, err + err := fmt.Errorf("%s: %s: %w", externalRelPath, urlStr, err) + return nil, urlStr, ArchiveFormatUnknown, err } urlPath := externalURL.Path if external.Encrypted { @@ -2504,7 +2537,7 @@ func (s *SourceState) readExternalArchiveData( format = GuessArchiveFormat(urlPath, data) } - return data, format, nil + return data, urlStr, format, nil } // readExternalArchiveFile reads a file from an external archive and returns its @@ -2520,7 +2553,7 @@ func (s *SourceState) readExternalArchiveFile( return nil, fmt.Errorf("%s: missing path", externalRelPath) } - data, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options) + data, urlStr, format, err := s.readExternalArchiveData(ctx, externalRelPath, external, options) if err != nil { return nil, err } @@ -2611,7 +2644,7 @@ func (s *SourceState) readExternalArchiveFile( return nil, err } if sourceStateEntry == nil { - return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, external.URL) + return nil, fmt.Errorf("%s: path not found in %s", external.ArchivePath, urlStr) } return s.populateImplicitParentDirs(externalRelPath, external, map[RelPath][]SourceStateEntry{ @@ -2728,7 +2761,8 @@ func (s *SourceState) readExternalFile( options *ReadOptions, ) (map[RelPath][]SourceStateEntry, error) { contentsFunc := sync.OnceValues(func() ([]byte, error) { - return s.getExternalData(ctx, externalRelPath, external, options) + data, _, err := s.getExternalData(ctx, externalRelPath, external, options) + return data, err }) fileAttr := FileAttr{ Empty: true, @@ -2880,7 +2914,8 @@ func (e *External) Path() AbsPath { } func (e *External) OriginString() string { - return e.URL + " defined in " + e.sourceAbsPath.String() + urlStr := cmp.Or(append([]string{e.URL}, e.URLs...)...) + return urlStr + " defined in " + e.sourceAbsPath.String() } // canonicalSourceStateEntry returns the canonical SourceStateEntry for the diff --git a/internal/cmd/testdata/scripts/externalfileurl.txtar b/internal/cmd/testdata/scripts/externalfileurl.txtar index 081d9ebaef0..72be4bea959 100644 --- a/internal/cmd/testdata/scripts/externalfileurl.txtar +++ b/internal/cmd/testdata/scripts/externalfileurl.txtar @@ -1,11 +1,38 @@ +# test that chezmoi apply reads from file:// URLs exec chezmoi apply cmp $HOME/.file golden/.file +chhome home2/user + +# test that chezmoi apply reads from the first found URL +exec chezmoi apply +cmp $HOME/.file golden/.file2 + +chhome home3/user + +# test that chezmoi apply returns an error from the first URL if all URLs return errors +! exec chezmoi apply +[!windows] stderr 'home3/user/\.local/share/file.txt: no such file or directory' +[windows] stderr 'home3/user/\.local/share/file.txt: The system cannot find the file specified\.' + -- golden/.file -- # contents of .file +-- golden/.file2 -- +# contents of .file2 -- home/user/.local/share/chezmoi/.chezmoiexternal.toml -- [".file"] type = "file" url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt" -- home/user/.local/share/file.txt -- # contents of .file +-- home2/user/.local/share/chezmoi/.chezmoiexternal.toml -- +[".file"] + type = "file" + urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"] +-- home2/user/.local/share/file2.txt -- +# contents of .file2 +-- home3/user/.local/share/chezmoi/.chezmoiexternal.toml -- +[".file"] + type = "file" + url = "file://{{ .chezmoi.homeDir }}/.local/share/file.txt" + urls = ["file://{{ .chezmoi.homeDir }}/.local/share/file1.txt", "file://{{ .chezmoi.homeDir }}/.local/share/file2.txt"]