diff --git a/config/config_test.go b/config/config_test.go index f74b8fcc..a3dd2c13 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2960,7 +2960,7 @@ func TestWaybackMeiliEndpoint(t *testing.T) { }{ { endpoint: "", - expected: defWaybackMeiliEndpoint, + expected: defMeiliEndpoint, }, { endpoint: "https://example.com", @@ -2979,7 +2979,7 @@ func TestWaybackMeiliEndpoint(t *testing.T) { t.Fatalf(`Parsing environment variables failed: %v`, err) } - got := opts.WaybackMeiliEndpoint() + got := opts.MeiliEndpoint() if got != test.expected { t.Fatalf(`Unexpected set meilisearch endpoint got %s instead of %s`, got, test.expected) } @@ -2996,7 +2996,7 @@ func TestWaybackMeiliIndexing(t *testing.T) { }{ { indexing: "", - expected: defWaybackMeiliIndexing, + expected: defMeiliIndexing, }, { indexing: "foo-bar", @@ -3015,7 +3015,7 @@ func TestWaybackMeiliIndexing(t *testing.T) { t.Fatalf(`Parsing environment variables failed: %v`, err) } - got := opts.WaybackMeiliIndexing() + got := opts.MeiliIndexing() if got != test.expected { t.Fatalf(`Unexpected set meilisearch indexing got %s instead of %s`, got, test.expected) } @@ -3032,7 +3032,7 @@ func TestWaybackMeiliApikey(t *testing.T) { }{ { apikey: "", - expected: defWaybackMeiliApikey, + expected: defMeiliApikey, }, { apikey: "foo.bar", @@ -3051,7 +3051,7 @@ func TestWaybackMeiliApikey(t *testing.T) { t.Fatalf(`Parsing environment variables failed: %v`, err) } - got := opts.WaybackMeiliApikey() + got := opts.MeiliApikey() if got != test.expected { t.Fatalf(`Unexpected set meilisearch api key got %s instead of %s`, got, test.expected) } @@ -3095,6 +3095,78 @@ func TestEnabledMeilisearch(t *testing.T) { } } +func TestOmnivoreApikey(t *testing.T) { + t.Parallel() + + var tests = []struct { + apikey string + expected string + }{ + { + apikey: "", + expected: defOmnivoreApikey, + }, + { + apikey: "foo.bar", + expected: "foo.bar", + }, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + os.Clearenv() + os.Setenv("WAYBACK_OMNIVORE_APIKEY", test.apikey) + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.OmnivoreApikey() + if got != test.expected { + t.Fatalf(`Unexpected set Omnivore api key got %s instead of %s`, got, test.expected) + } + }) + } +} + +func TestEnabledOmnivore(t *testing.T) { + t.Parallel() + + var tests = []struct { + apikey string + expected bool + }{ + { + apikey: "", + expected: false, + }, + { + apikey: "foo-bar", + expected: true, + }, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + os.Clearenv() + os.Setenv("WAYBACK_OMNIVORE_APIKEY", test.apikey) + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.EnabledOmnivore() + if got != test.expected { + t.Fatalf(`Unexpected enabled meilisearch got %t instead of %t`, got, test.expected) + } + }) + } +} + func TestEnableServices(t *testing.T) { tests := []struct { name string diff --git a/config/options.go b/config/options.go index e9a93b3e..22dd655e 100644 --- a/config/options.go +++ b/config/options.go @@ -98,9 +98,11 @@ const ( defWaybackFallback = false defProxy = "" - defWaybackMeiliEndpoint = "" - defWaybackMeiliIndexing = "capsules" - defWaybackMeiliApikey = "" + defMeiliEndpoint = "" + defMeiliIndexing = "capsules" + defMeiliApikey = "" + + defOmnivoreApikey = "" maxAttachSizeTelegram = 50000000 // 50MB maxAttachSizeDiscord = 8000000 // 8MB @@ -141,6 +143,8 @@ type Options struct { irc *irc onion *onion xmpp *xmpp + omnivore *omnivore + meili *meili listenAddr string chromeRemoteAddr string @@ -154,10 +158,6 @@ type Options struct { waybackMaxRetries int waybackUserAgent string waybackFallback bool - - waybackMeiliEndpoint string - waybackMeiliIndexing string - waybackMeiliApikey string } type ipfs struct { @@ -253,28 +253,35 @@ type xmpp struct { helptext string } +type meili struct { + endpoint string + indexing string + apikey string +} + +type omnivore struct { + apikey string +} + // NewOptions returns Options with default values. func NewOptions() *Options { opts := &Options{ - debug: defDebug, - logTime: defLogTime, - logLevel: defLogLevel, - overTor: defOverTor, - metrics: defMetrics, - listenAddr: defListenAddr, - chromeRemoteAddr: defChromeRemoteAddr, - enabledChromeRemote: defEnabledChromeRemote, - boltPathname: defBoltPathname, - poolingSize: defPoolingSize, - storageDir: defStorageDir, - maxMediaSize: defMaxMediaSize, - waybackTimeout: defWaybackTimeout, - waybackMaxRetries: defWaybackMaxRetries, - waybackUserAgent: defWaybackUserAgent, - waybackFallback: defWaybackFallback, - waybackMeiliEndpoint: defWaybackMeiliEndpoint, - waybackMeiliIndexing: defWaybackMeiliIndexing, - waybackMeiliApikey: defWaybackMeiliApikey, + debug: defDebug, + logTime: defLogTime, + logLevel: defLogLevel, + overTor: defOverTor, + metrics: defMetrics, + listenAddr: defListenAddr, + chromeRemoteAddr: defChromeRemoteAddr, + enabledChromeRemote: defEnabledChromeRemote, + boltPathname: defBoltPathname, + poolingSize: defPoolingSize, + storageDir: defStorageDir, + maxMediaSize: defMaxMediaSize, + waybackTimeout: defWaybackTimeout, + waybackMaxRetries: defWaybackMaxRetries, + waybackUserAgent: defWaybackUserAgent, + waybackFallback: defWaybackFallback, ipfs: &ipfs{ host: defIPFSHost, port: defIPFSPort, @@ -357,6 +364,14 @@ func NewOptions() *Options { noTLS: defXMPPNoTLS, helptext: defXMPPHelptext, }, + meili: &meili{ + endpoint: defMeiliEndpoint, + indexing: defMeiliIndexing, + apikey: defMeiliApikey, + }, + omnivore: &omnivore{ + apikey: defOmnivoreApikey, + }, } return opts @@ -926,24 +941,34 @@ func (o *Options) WaybackFallback() bool { return o.waybackFallback } -// WaybackMeiliEndpoint returns the Meilisearch API endpoint. -func (o *Options) WaybackMeiliEndpoint() string { - return o.waybackMeiliEndpoint +// MeiliEndpoint returns the Meilisearch API endpoint. +func (o *Options) MeiliEndpoint() string { + return o.meili.endpoint } -// WaybackMeiliIndexing returns the Meilisearch indexing name. -func (o *Options) WaybackMeiliIndexing() string { - return o.waybackMeiliIndexing +// MeiliIndexing returns the Meilisearch indexing name. +func (o *Options) MeiliIndexing() string { + return o.meili.indexing } -// WaybackMeiliApikey returns the Meilisearch admin apikey. -func (o *Options) WaybackMeiliApikey() string { - return o.waybackMeiliApikey +// MeiliApikey returns the Meilisearch admin apikey. +func (o *Options) MeiliApikey() string { + return o.meili.apikey } // EnabledMeilisearch returns whether enable meilisearch server. func (o *Options) EnabledMeilisearch() bool { - return o.WaybackMeiliEndpoint() != "" + return o.MeiliEndpoint() != "" +} + +// OmnivoreApikey returns the Omnivore apikey. +func (o *Options) OmnivoreApikey() string { + return o.omnivore.apikey +} + +// EnabledOmnivore returns whether enable Omnivore. +func (o *Options) EnabledOmnivore() bool { + return o.OmnivoreApikey() != "" } // HTTPdEnabled returns whether enable HTTP daemon service. diff --git a/config/parser.go b/config/parser.go index 73cb27cc..d3ede970 100644 --- a/config/parser.go +++ b/config/parser.go @@ -222,11 +222,13 @@ func (p *Parser) parseLines(lines []string) (err error) { case "WAYBACK_FALLBACK": p.opts.waybackFallback = parseBool(val, defWaybackFallback) case "WAYBACK_MEILI_ENDPOINT": - p.opts.waybackMeiliEndpoint = parseString(val, defWaybackMeiliEndpoint) + p.opts.meili.endpoint = parseString(val, defMeiliEndpoint) case "WAYBACK_MEILI_INDEXING": - p.opts.waybackMeiliIndexing = parseString(val, defWaybackMeiliIndexing) + p.opts.meili.indexing = parseString(val, defMeiliIndexing) case "WAYBACK_MEILI_APIKEY": - p.opts.waybackMeiliApikey = parseString(val, defWaybackMeiliApikey) + p.opts.meili.apikey = parseString(val, defMeiliApikey) + case "WAYBACK_OMNIVORE_APIKEY": + p.opts.omnivore.apikey = parseString(val, defOmnivoreApikey) default: if os.Getenv(key) == "" && val != "" { os.Setenv(key, val) diff --git a/docs/environment.md b/docs/environment.md index eb929a14..6c0116ab 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -40,6 +40,7 @@ Use the `-c` / `--config` option to specify the build definition file to use. | - | `WAYBACK_MEILI_ENDPOINT` | - | Meilisearch API endpoint | | - | `WAYBACK_MEILI_INDEXING` | `capsules` | Meilisearch indexing name | | - | `WAYBACK_MEILI_APIKEY` | - | Meilisearch admin API key | +| - | `WAYBACK_OMNIVORE_APIKEY` | - | Omnivore API key | | `-d`, `--daemon` | - | - | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` | | `--ia` | `WAYBACK_ENABLE_IA` | `true` | Wayback webpages to **Internet Archive** | | `--is` | `WAYBACK_ENABLE_IS` | `true` | Wayback webpages to **Archive Today** | diff --git a/docs/integrations/omnivore.md b/docs/integrations/omnivore.md new file mode 100644 index 00000000..044a76fd --- /dev/null +++ b/docs/integrations/omnivore.md @@ -0,0 +1,11 @@ +--- +title: Publish to Omnivore +--- + +Omnivore lets you save and organize articles, newsletters, and documents for later reading. It also syncs with popular knowledge management systems and offers text-to-speech and distraction free features. + +## Configuration + +Create API key, place key in environment or configuration file: + +- `WAYBACK_OMNIVORE_APIKEY`: Omnivore API key. diff --git a/docs/integrations/omnivore.zh.md b/docs/integrations/omnivore.zh.md new file mode 100644 index 00000000..ab03f1e0 --- /dev/null +++ b/docs/integrations/omnivore.zh.md @@ -0,0 +1,11 @@ +--- +title: 发布到Omnivore +--- + +Omnivore 可让您保存和整理文章、简报和文档以供日后阅读。它还可与流行的知识管理系统同步，并提供文本转语音和无干扰功能。 + +## 配置 + +创建 API 密钥，将密钥放置在环境或配置文件中： + +- `WAYBACK_OMNIVORE_APIKEY`: Omnivore API key. diff --git a/go.mod b/go.mod index df6a6bb3..942471c9 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,9 @@ require ( github.com/gabriel-vasile/mimetype v1.4.2 github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b github.com/go-shiori/obelisk v0.0.0-20230316095823-42f6a2f99d9d + github.com/goccy/go-json v0.10.3 github.com/google/go-github/v40 v40.0.0 + github.com/google/uuid v1.3.0 github.com/gookit/color v1.5.3 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.1 @@ -36,7 +38,6 @@ require ( github.com/wabarc/archive.is v1.4.0 github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896 github.com/wabarc/ghostarchive v0.1.1 - github.com/wabarc/go-anonfile v0.1.0 github.com/wabarc/go-catbox v0.1.0 github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb github.com/wabarc/imgbb v1.0.0 @@ -95,7 +96,6 @@ require ( github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect - github.com/google/uuid v1.3.0 // indirect github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/boxo v0.8.1 // indirect diff --git a/go.sum b/go.sum index 6476fbbc..6bc19ae4 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q= github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -479,13 +481,10 @@ github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896 h1:c3uD+IKXpN github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896/go.mod h1:yEmUMlNO2PPAxIvo/Hf/VxOrCS5SBwL2/vCW8pyTWjA= github.com/wabarc/ghostarchive v0.1.1 h1:iGnDwvUixynKGlUTtxGfWAD1p6QRtCj7pAYCqgZNZTQ= github.com/wabarc/ghostarchive v0.1.1/go.mod h1:+HB72/CrKK4at+QhWDKRZYB6WLLyE/xlXr6/LgfqFro= -github.com/wabarc/go-anonfile v0.1.0 h1:M4jKUAMROxxVaqLQt30ONmaHA/0YnvyJtX8qg/zI9+I= -github.com/wabarc/go-anonfile v0.1.0/go.mod h1:CH1LzSKQ0x/RlKpEG2gi+hXaOOZrfeRx6Rp717o+65A= github.com/wabarc/go-catbox v0.1.0 h1:/UhV9md3MJrjZtm+EToSyFjawXgPiHSExLNRqsWNisg= github.com/wabarc/go-catbox v0.1.0/go.mod h1:Zjs9Y55f2WOwGWwmKSCrUuMfwh+nDktkjub9jgHq4CQ= github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8= github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E= -github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E= github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E= github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ= github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb h1:psEAY4wXvhXp/Hp5CJWgAOKWqhvAom+/hOjK+Qscx7o= diff --git a/ingress/register/publish.go b/ingress/register/publish.go index d7b07d08..c1ebe788 100644 --- a/ingress/register/publish.go +++ b/ingress/register/publish.go @@ -12,6 +12,7 @@ import ( _ "github.com/wabarc/wayback/publish/meili" _ "github.com/wabarc/wayback/publish/nostr" _ "github.com/wabarc/wayback/publish/notion" + _ "github.com/wabarc/wayback/publish/omnivore" _ "github.com/wabarc/wayback/publish/relaychat" _ "github.com/wabarc/wayback/publish/slack" _ "github.com/wabarc/wayback/publish/telegram" diff --git a/metrics/metrics.go b/metrics/metrics.go index 49743a8e..3aee1ed4 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -26,17 +26,18 @@ const ( ServiceTwitter = "twitter" ServiceXMPP = "xmpp" - PublishIRC = "irc" // IRC channel - PublishGithub = "github" // GitHub issues - PublishNotion = "notion" // Notion page - PublishChannel = "telegram" // Telegram channel - PublishMstdn = "mastodon" // Mastodon toot - PublishDiscord = "discord" // Discord channel - PublishTwitter = "twitter" - PublishMatrix = "matrix" - PublishSlack = "slack" - PublishNostr = "nostr" - PublishMeili = "meili" + PublishIRC = "irc" // IRC channel + PublishGithub = "github" // GitHub issues + PublishNotion = "notion" // Notion page + PublishChannel = "telegram" // Telegram channel + PublishMstdn = "mastodon" // Mastodon toot + PublishDiscord = "discord" // Discord channel + PublishTwitter = "twitter" + PublishMatrix = "matrix" + PublishSlack = "slack" + PublishNostr = "nostr" + PublishMeili = "meili" + PublishOmnivore = "omnivore" StatusRequest = "request" StatusSuccess = "success" diff --git a/publish/meili/meili.go b/publish/meili/meili.go index 49bd16f6..8b19ae0d 100644 --- a/publish/meili/meili.go +++ b/publish/meili/meili.go @@ -55,7 +55,7 @@ type Meili struct { // New returns a Meilisearch client. func New(client *http.Client, opts *config.Options) *Meili { - endpoint, apikey, idxname := opts.WaybackMeiliEndpoint(), opts.WaybackMeiliApikey(), opts.WaybackMeiliIndexing() + endpoint, apikey, idxname := opts.MeiliEndpoint(), opts.MeiliApikey(), opts.MeiliIndexing() if client == nil { client = &http.Client{ diff --git a/publish/omnivore/doc.go b/publish/omnivore/doc.go new file mode 100644 index 00000000..32bd1923 --- /dev/null +++ b/publish/omnivore/doc.go @@ -0,0 +1,5 @@ +// Copyright 2024 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package omnivore // import "github.com/wabarc/wayback/publish/omnivore" diff --git a/publish/omnivore/omnivore.go b/publish/omnivore/omnivore.go new file mode 100644 index 00000000..a5c6fc28 --- /dev/null +++ b/publish/omnivore/omnivore.go @@ -0,0 +1,153 @@ +// Copyright 2024 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package omnivore // import "github.com/wabarc/wayback/publish/omnivore" + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/goccy/go-json" + "github.com/google/uuid" + "github.com/wabarc/logger" + "github.com/wabarc/wayback" + "github.com/wabarc/wayback/config" + "github.com/wabarc/wayback/errors" + "github.com/wabarc/wayback/ingress" + "github.com/wabarc/wayback/metrics" + "github.com/wabarc/wayback/publish" + "github.com/wabarc/wayback/reduxer" +) + +const ( + defaultClientTimeout = 10 * time.Second + defaultApiEndpoint = "https://api-prod.omnivore.app/api/graphql" +) + +var mutation = ` +mutation SaveUrl($input: SaveUrlInput!) { + saveUrl(input: $input) { + ... on SaveSuccess { + url + clientRequestId + } + ... on SaveError { + errorCodes + message + } + } +} +` + +// Interface guard +var _ publish.Publisher = (*Omnivore)(nil) + +type Omnivore struct { + bot *http.Client + opts *config.Options +} + +type errorResponse struct { + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +type successResponse struct { + Data struct { + SaveUrl struct { + Url string `json:"url"` + ClientRequestId string `json:"clientRequestId"` + } `json:"saveUrl"` + } `json:"data"` +} + +// New returns a omnivore client. +func New(client *http.Client, opts *config.Options) *Omnivore { + if opts.OmnivoreApikey() == "" { + logger.Debug("Onmnivore integration access token is required") + return nil + } + + bot := ingress.Client() + if client != nil { + bot = client + } + bot.Timeout = defaultClientTimeout + + return &Omnivore{bot: bot, opts: opts} +} + +// Publish save url to the Omnivore of the given cols and args. +func (o *Omnivore) Publish(_ context.Context, _ reduxer.Reduxer, cols []wayback.Collect, args ...string) error { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusRequest) + + if len(cols) == 0 { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusFailure) + return errors.New("publish to omnivore: collects empty") + } + + var payload = map[string]interface{}{ + "query": mutation, + "variables": map[string]interface{}{ + "input": map[string]interface{}{ + "clientRequestId": uuid.NewString(), + "source": "api", + "url": cols[0].Src, + }, + }, + } + b, err := json.Marshal(payload) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, defaultApiEndpoint, bytes.NewReader(b)) + if err != nil { + return err + } + + req.Header.Set("Authorization", o.opts.OmnivoreApikey()) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", o.opts.WaybackUserAgent()) + + resp, err := o.bot.Do(req) + if err != nil { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusFailure) + return err + } + + defer resp.Body.Close() + b, err = io.ReadAll(resp.Body) + if err != nil { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusFailure) + return fmt.Errorf("omnivore: failed to parse response: %s", err) + } + + if resp.StatusCode >= 400 { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusFailure) + var errResponse errorResponse + if err = json.Unmarshal(b, &errResponse); err != nil { + return fmt.Errorf("omnivore: failed to save URL: status=%d %s", resp.StatusCode, string(b)) + } + return fmt.Errorf("omnivore: failed to save URL: status=%d %s", resp.StatusCode, errResponse.Errors[0].Message) + } + + var successReponse successResponse + if err = json.Unmarshal(b, &successReponse); err != nil { + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusFailure) + return fmt.Errorf("omnivore: failed to parse response, however the request appears successful, is the url correct?: status=%d %s", resp.StatusCode, string(b)) + } + + metrics.IncrementPublish(metrics.PublishOmnivore, metrics.StatusSuccess) + return nil +} + +// Shutdown shuts down the Omnivore publish service, it always return a nil error. +func (o *Omnivore) Shutdown() error { + return nil +} diff --git a/publish/omnivore/omnivore_test.go b/publish/omnivore/omnivore_test.go new file mode 100644 index 00000000..3f83510a --- /dev/null +++ b/publish/omnivore/omnivore_test.go @@ -0,0 +1,60 @@ +// Copyright 2024 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package omnivore // import "github.com/wabarc/wayback/publish/omnivore" + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/wabarc/helper" + "github.com/wabarc/wayback/config" + "github.com/wabarc/wayback/publish" + "github.com/wabarc/wayback/reduxer" +) + +const saveURLResp = `{"data":{"saveUrl":{"url":"https://omnivore.app/repo/links/cff02ab5-c36e-4efe-a976-2de32dc1685d","clientRequestId":"cff02ab5-c36e-4efe-a976-2de32dc1685d"}}}` + +func TestPublish(t *testing.T) { + t.Setenv("WAYBACK_OMNIVORE_APIKEY", "foo") + opts, _ := config.NewParser().ParseEnvironmentVariables() + + httpClient, mux, server := helper.MockServer() + defer server.Close() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/graphql": + fmt.Fprintln(w, saveURLResp) + default: + fmt.Fprintln(w, `{}`) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + o := New(httpClient, opts) + got := o.Publish(ctx, reduxer.BundleExample(), publish.Collects) + if got != nil { + t.Errorf("unexpected save url got %v", got) + } +} + +func TestShutdown(t *testing.T) { + opts, _ := config.NewParser().ParseEnvironmentVariables() + + httpClient, _, server := helper.MockServer() + defer server.Close() + + no := New(httpClient, opts) + err := no.Shutdown() + if err != nil { + t.Errorf("Unexpected shutdown: %v", err) + } +} diff --git a/publish/omnivore/setup.go b/publish/omnivore/setup.go new file mode 100644 index 00000000..86302329 --- /dev/null +++ b/publish/omnivore/setup.go @@ -0,0 +1,27 @@ +// Copyright 2024 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package omnivore // import "github.com/wabarc/wayback/publish/omnivore" + +import ( + "github.com/wabarc/wayback/config" + "github.com/wabarc/wayback/publish" +) + +func init() { + publish.Register(publish.FlagOmnivore, setup) +} + +func setup(opts *config.Options) *publish.Module { + if opts.EnabledOmnivore() { + publisher := New(nil, opts) + + return &publish.Module{ + Publisher: publisher, + Opts: opts, + } + } + + return nil +} diff --git a/publish/publish.go b/publish/publish.go index 69c6a4e6..bd0e83e1 100644 --- a/publish/publish.go +++ b/publish/publish.go @@ -32,6 +32,7 @@ const ( FlagNotion // FlagNotion is a flag for notion publish service FlagGitHub // FlagGitHub is a flag for github publish service FlagMeili // FlagMeili is a flag for meilisearch publish service + FlagOmnivore // FlagOmnivore is a flag for Omnivore publish service ) // Publisher is the interface that wraps the basic Publish method. @@ -72,6 +73,8 @@ func (f Flag) String() string { return "github" case FlagMeili: return "meilisearch" + case FlagOmnivore: + return "omnivore" default: return "unknown" } diff --git a/reduxer/reduxer.go b/reduxer/reduxer.go index f768d774..771a7571 100644 --- a/reduxer/reduxer.go +++ b/reduxer/reduxer.go @@ -177,7 +177,7 @@ func Do(ctx context.Context, opts *config.Options, urls ...*url.URL) (Reduxer, e g.Go(func() error { basename := strings.TrimSuffix(helper.FileName(uri.String(), ""), ".html") basename = strings.TrimSuffix(basename, ".htm") - ctx = context.WithValue(ctx, ctxBasenameKey, basename) + ctx = context.WithValue(ctx, ctxBasenameKey, basename) // nolint:staticcheck shot, er := capture(ctx, opts, uri, dir) if er != nil { diff --git a/wayback.1 b/wayback.1 index d90f95f5..6c57f0e4 100644 --- a/wayback.1 +++ b/wayback.1 @@ -189,6 +189,9 @@ Meilisearch indexing name.\&. .B WAYBACK_MEILI_APIKEY Meilisearch admin API key.\&. .TP +.B WAYBACK_OMNIVORE_APIKEY +Omnivore API key.\&. +.TP .B WAYBACK_BOLT_PATH File path of bolt database. default ./wayback.db\&. .TP diff --git a/wayback.conf b/wayback.conf index cea43bb5..3dc5f333 100644 --- a/wayback.conf +++ b/wayback.conf @@ -55,6 +55,7 @@ WAYBACK_XMPP_HELPTEXT=Hi,\n\nI'm a 🤖 to help you backup webpages more easily. WAYBACK_MEILI_ENDPOINT= WAYBACK_MEILI_INDEXING=capsules WAYBACK_MEILI_APIKEY= +WAYBACK_OMNIVORE_APIKEY= WAYBACK_USE_TOR=false WAYBACK_ONION_PRIVKEY= WAYBACK_ONION_LOCAL_PORT=8964