Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: make number of concurrent downloads configurable #972

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix: make number of concurrent downloads configurable #972

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
555936b
fix: make number of concurrent downloads configurable
Oppen Aug 27, 2024
7f2b185
Merge branch 'master' into fix/configurable_concurrency_limits
fmartingr Sep 22, 2024
cba4f84
Typo and nonsensical default
Oppen Sep 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions docs/Configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ Most configuration can be set directly using environment variables or flags. The

The `StorageConfig` struct contains settings related to storage.

| Environment variable | Default | Required | Description |
| -------------------- | ------------- | -------- | --------------------------------------- |
| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. |
| Environment variable | Default | Required | Description |
| -------------------- | ----------------- | -------- | ---------------------------------------- |
| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. |
| `SHIORI_MAX_PAR_DL` | (num logicl CPU) | No | Number of parallel articles to download. |
Oppen marked this conversation as resolved.
Show resolved Hide resolved

#### The data Directory

Expand Down
8 changes: 4 additions & 4 deletions internal/cmd/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func checkCmd() *cobra.Command {
}

func checkHandler(cmd *cobra.Command, args []string) {
_, deps := initShiori(cmd.Context(), cmd)
cfg, deps := initShiori(cmd.Context(), cmd)

// Parse flags
skipConfirm, _ := cmd.Flags().GetBool("yes")
Expand Down Expand Up @@ -69,9 +69,9 @@ func checkHandler(cmd *cobra.Command, args []string) {

wg := sync.WaitGroup{}
chDone := make(chan struct{})
chProblem := make(chan int, 10)
chMessage := make(chan interface{}, 10)
semaphore := make(chan struct{}, 10)
chProblem := make(chan int, cfg.Storage.MaxParDl)
chMessage := make(chan interface{}, cfg.Storage.MaxParDl)
semaphore := make(chan struct{}, cfg.Storage.MaxParDl)

for i, book := range bookmarks {
wg.Add(1)
Expand Down
6 changes: 3 additions & 3 deletions internal/cmd/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,9 @@ func updateHandler(cmd *cobra.Command, args []string) {
mx := sync.RWMutex{}
wg := sync.WaitGroup{}
chDone := make(chan struct{})
chProblem := make(chan int, 10)
chMessage := make(chan interface{}, 10)
semaphore := make(chan struct{}, 10)
chProblem := make(chan int, cfg.Storage.MaxParDl)
chMessage := make(chan interface{}, cfg.Storage.MaxParDl)
semaphore := make(chan struct{}, cfg.Storage.MaxParDl)

cInfo.Println("Downloading article(s)...")

Expand Down
7 changes: 7 additions & 0 deletions internal/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"time"

Expand Down Expand Up @@ -86,6 +87,7 @@ type DatabaseConfig struct {

type StorageConfig struct {
DataDir string `env:"DIR"` // Using DIR to be backwards compatible with the old config
MaxParDl int `env:"MAX_PAR_DL,default=-1"`
fmartingr marked this conversation as resolved.
Show resolved Hide resolved
Oppen marked this conversation as resolved.
Show resolved Hide resolved
}

type Config struct {
Expand All @@ -109,6 +111,10 @@ func (c Config) SetDefaults(logger *logrus.Logger, portableMode bool) {
}
}

if c.Storage.MaxParDl == -1 {
Oppen marked this conversation as resolved.
Show resolved Hide resolved
c.Storage.MaxParDl = runtime.NumCPU()
}

// Set default database url if not set
if c.Database.DBMS == "" && c.Database.URL == "" {
c.Database.URL = fmt.Sprintf("sqlite:///%s", filepath.Join(c.Storage.DataDir, "shiori.db"))
Expand All @@ -124,6 +130,7 @@ func (c *Config) DebugConfiguration(logger *logrus.Logger) {
logger.Debugf(" SHIORI_DATABASE_URL: %s", c.Database.URL)
logger.Debugf(" SHIORI_DBMS: %s", c.Database.DBMS)
logger.Debugf(" SHIORI_DIR: %s", c.Storage.DataDir)
logger.Debugf(" SHIORI_MAX_PAR_DL: %d", c.Storage.MaxParDl)
logger.Debugf(" SHIORI_HTTP_ENABLED: %t", c.Http.Enabled)
logger.Debugf(" SHIORI_HTTP_PORT: %d", c.Http.Port)
logger.Debugf(" SHIORI_HTTP_ADDRESS: %s", c.Http.Address)
Expand Down
5 changes: 3 additions & 2 deletions internal/http/routes/api/v1/bookmarks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,13 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
}
// TODO: limit request to 20

cfg := r.deps.Config
// Fetch data from internet
mx := sync.RWMutex{}
wg := sync.WaitGroup{}
chDone := make(chan struct{})
chProblem := make(chan int, 10)
semaphore := make(chan struct{}, 10)
chProblem := make(chan int, cfg.Storage.MaxParDl)
semaphore := make(chan struct{}, cfg.Storage.MaxParDl)

for i, book := range bookmarks {
wg.Add(1)
Expand Down
Loading