From 4210c93ec7e49bb589c190b8146b1c0dcbcef95f Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 27 Aug 2024 15:52:49 -0300 Subject: [PATCH] fix: make number of concurrent downloads configurable --- docs/Configuration.md | 7 ++++--- internal/cmd/update.go | 2 +- internal/config/config.go | 7 +++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index fdaf4e7e9..a2868da9f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -47,9 +47,10 @@ Most configuration can be set directly using environment variables or flags. The The `StorageConfig` struct contains settings related to storage. -| Environment variable | Default | Required | Description | -| -------------------- | ------------- | -------- | --------------------------------------- | -| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. | +| Environment variable | Default | Required | Description | +| -------------------- | ----------------- | -------- | ---------------------------------------- | +| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. | +| `SHIORI_MAX_PAR_DL` | (num logicl CPU) | No | Number of parallel articles to download. | #### The data Directory diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 918ea2a1a..4635f4bde 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -132,7 +132,7 @@ func updateHandler(cmd *cobra.Command, args []string) { chDone := make(chan struct{}) chProblem := make(chan int, 10) chMessage := make(chan interface{}, 10) - semaphore := make(chan struct{}, 10) + semaphore := make(chan struct{}, cfg.Storage.MaxParDl) cInfo.Println("Downloading article(s)...") diff --git a/internal/config/config.go b/internal/config/config.go index 390b642d6..3506261f6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "time" @@ -86,6 +87,7 @@ type DatabaseConfig struct { type StorageConfig struct { DataDir string `env:"DIR"` // Using DIR to be backwards compatible with the old config + MaxParDl int `env:"MAX_PAR_DL,default=-1"` } type Config struct { @@ -109,6 +111,10 @@ func (c Config) SetDefaults(logger *logrus.Logger, portableMode bool) { } } + if c.Storage.MaxParDl == -1 { + c.Storage.MaxParDl = runtime.NumCPU() + } + // Set default database url if not set if c.Database.DBMS == "" && c.Database.URL == "" { c.Database.URL = fmt.Sprintf("sqlite:///%s", filepath.Join(c.Storage.DataDir, "shiori.db")) @@ -124,6 +130,7 @@ func (c *Config) DebugConfiguration(logger *logrus.Logger) { logger.Debugf(" SHIORI_DATABASE_URL: %s", c.Database.URL) logger.Debugf(" SHIORI_DBMS: %s", c.Database.DBMS) logger.Debugf(" SHIORI_DIR: %s", c.Storage.DataDir) + logger.Debugf(" SHIORI_MAX_PAR_DL: %d", c.Storage.MaxParDl) logger.Debugf(" SHIORI_HTTP_ENABLED: %t", c.Http.Enabled) logger.Debugf(" SHIORI_HTTP_PORT: %d", c.Http.Port) logger.Debugf(" SHIORI_HTTP_ADDRESS: %s", c.Http.Address)