Add support publish to Omnivore

wabarc · Aug 11, 2024 · b779341 · b779341
1 parent 33d2b59
commit b779341
Show file tree

Hide file tree

Showing 16 changed files with 416 additions and 58 deletions.
diff --git a/config/config_test.go b/config/config_test.go
@@ -2960,7 +2960,7 @@ func TestWaybackMeiliEndpoint(t *testing.T) {
 	}{
 		{
 			endpoint: "",
-			expected: defWaybackMeiliEndpoint,
+			expected: defMeiliEndpoint,
 		},
 		{
 			endpoint: "https://example.com",
@@ -2979,7 +2979,7 @@ func TestWaybackMeiliEndpoint(t *testing.T) {
 				t.Fatalf(`Parsing environment variables failed: %v`, err)
 			}
 
-			got := opts.WaybackMeiliEndpoint()
+			got := opts.MeiliEndpoint()
 			if got != test.expected {
 				t.Fatalf(`Unexpected set meilisearch endpoint got %s instead of %s`, got, test.expected)
 			}
@@ -2996,7 +2996,7 @@ func TestWaybackMeiliIndexing(t *testing.T) {
 	}{
 		{
 			indexing: "",
-			expected: defWaybackMeiliIndexing,
+			expected: defMeiliIndexing,
 		},
 		{
 			indexing: "foo-bar",
@@ -3015,7 +3015,7 @@ func TestWaybackMeiliIndexing(t *testing.T) {
 				t.Fatalf(`Parsing environment variables failed: %v`, err)
 			}
 
-			got := opts.WaybackMeiliIndexing()
+			got := opts.MeiliIndexing()
 			if got != test.expected {
 				t.Fatalf(`Unexpected set meilisearch indexing got %s instead of %s`, got, test.expected)
 			}
@@ -3032,7 +3032,7 @@ func TestWaybackMeiliApikey(t *testing.T) {
 	}{
 		{
 			apikey:   "",
-			expected: defWaybackMeiliApikey,
+			expected: defMeiliApikey,
 		},
 		{
 			apikey:   "foo.bar",
@@ -3051,7 +3051,7 @@ func TestWaybackMeiliApikey(t *testing.T) {
 				t.Fatalf(`Parsing environment variables failed: %v`, err)
 			}
 
-			got := opts.WaybackMeiliApikey()
+			got := opts.MeiliApikey()
 			if got != test.expected {
 				t.Fatalf(`Unexpected set meilisearch api key got %s instead of %s`, got, test.expected)
 			}
@@ -3095,6 +3095,78 @@ func TestEnabledMeilisearch(t *testing.T) {
 	}
 }
 
+func TestOmnivoreApikey(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		apikey   string
+		expected string
+	}{
+		{
+			apikey:   "",
+			expected: defOmnivoreApikey,
+		},
+		{
+			apikey:   "foo.bar",
+			expected: "foo.bar",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			os.Clearenv()
+			os.Setenv("WAYBACK_OMNIVORE_APIKEY", test.apikey)
+
+			parser := NewParser()
+			opts, err := parser.ParseEnvironmentVariables()
+			if err != nil {
+				t.Fatalf(`Parsing environment variables failed: %v`, err)
+			}
+
+			got := opts.OmnivoreApikey()
+			if got != test.expected {
+				t.Fatalf(`Unexpected set Omnivore api key got %s instead of %s`, got, test.expected)
+			}
+		})
+	}
+}
+
+func TestEnabledOmnivore(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		apikey   string
+		expected bool
+	}{
+		{
+			apikey:   "",
+			expected: false,
+		},
+		{
+			apikey:   "foo-bar",
+			expected: true,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			os.Clearenv()
+			os.Setenv("WAYBACK_OMNIVORE_APIKEY", test.apikey)
+
+			parser := NewParser()
+			opts, err := parser.ParseEnvironmentVariables()
+			if err != nil {
+				t.Fatalf(`Parsing environment variables failed: %v`, err)
+			}
+
+			got := opts.EnabledOmnivore()
+			if got != test.expected {
+				t.Fatalf(`Unexpected enabled meilisearch got %t instead of %t`, got, test.expected)
+			}
+		})
+	}
+}
+
 func TestEnableServices(t *testing.T) {
 	tests := []struct {
 		name     string

diff --git a/config/options.go b/config/options.go
@@ -98,9 +98,11 @@ const (
 	defWaybackFallback     = false
 	defProxy               = ""
 
-	defWaybackMeiliEndpoint = ""
-	defWaybackMeiliIndexing = "capsules"
-	defWaybackMeiliApikey   = ""
+	defMeiliEndpoint = ""
+	defMeiliIndexing = "capsules"
+	defMeiliApikey   = ""
+
+	defOmnivoreApikey = ""
 
 	maxAttachSizeTelegram = 50000000   // 50MB
 	maxAttachSizeDiscord  = 8000000    // 8MB
@@ -141,6 +143,8 @@ type Options struct {
 	irc      *irc
 	onion    *onion
 	xmpp     *xmpp
+	omnivore *omnivore
+	meili    *meili
 
 	listenAddr          string
 	chromeRemoteAddr    string
@@ -253,28 +257,35 @@ type xmpp struct {
 	helptext string
 }
 
+type meili struct {
+	endpoint string
+	indexing string
+	apikey   string
+}
+
+type omnivore struct {
+	apikey string
+}
+
 // NewOptions returns Options with default values.
 func NewOptions() *Options {
 	opts := &Options{
-		debug:                defDebug,
-		logTime:              defLogTime,
-		logLevel:             defLogLevel,
-		overTor:              defOverTor,
-		metrics:              defMetrics,
-		listenAddr:           defListenAddr,
-		chromeRemoteAddr:     defChromeRemoteAddr,
-		enabledChromeRemote:  defEnabledChromeRemote,
-		boltPathname:         defBoltPathname,
-		poolingSize:          defPoolingSize,
-		storageDir:           defStorageDir,
-		maxMediaSize:         defMaxMediaSize,
-		waybackTimeout:       defWaybackTimeout,
-		waybackMaxRetries:    defWaybackMaxRetries,
-		waybackUserAgent:     defWaybackUserAgent,
-		waybackFallback:      defWaybackFallback,
-		waybackMeiliEndpoint: defWaybackMeiliEndpoint,
-		waybackMeiliIndexing: defWaybackMeiliIndexing,
-		waybackMeiliApikey:   defWaybackMeiliApikey,
+		debug:               defDebug,
+		logTime:             defLogTime,
+		logLevel:            defLogLevel,
+		overTor:             defOverTor,
+		metrics:             defMetrics,
+		listenAddr:          defListenAddr,
+		chromeRemoteAddr:    defChromeRemoteAddr,
+		enabledChromeRemote: defEnabledChromeRemote,
+		boltPathname:        defBoltPathname,
+		poolingSize:         defPoolingSize,
+		storageDir:          defStorageDir,
+		maxMediaSize:        defMaxMediaSize,
+		waybackTimeout:      defWaybackTimeout,
+		waybackMaxRetries:   defWaybackMaxRetries,
+		waybackUserAgent:    defWaybackUserAgent,
+		waybackFallback:     defWaybackFallback,
 		ipfs: &ipfs{
 			host:   defIPFSHost,
 			port:   defIPFSPort,
@@ -357,6 +368,14 @@ func NewOptions() *Options {
 			noTLS:    defXMPPNoTLS,
 			helptext: defXMPPHelptext,
 		},
+		meili: &meili{
+			endpoint: defMeiliEndpoint,
+			indexing: defMeiliIndexing,
+			apikey:   defMeiliApikey,
+		},
+		omnivore: &omnivore{
+			apikey: defOmnivoreApikey,
+		},
 	}
 
 	return opts
@@ -926,24 +945,34 @@ func (o *Options) WaybackFallback() bool {
 	return o.waybackFallback
 }
 
-// WaybackMeiliEndpoint returns the Meilisearch API endpoint.
-func (o *Options) WaybackMeiliEndpoint() string {
-	return o.waybackMeiliEndpoint
+// MeiliEndpoint returns the Meilisearch API endpoint.
+func (o *Options) MeiliEndpoint() string {
+	return o.meili.endpoint
 }
 
-// WaybackMeiliIndexing returns the Meilisearch indexing name.
-func (o *Options) WaybackMeiliIndexing() string {
-	return o.waybackMeiliIndexing
+// MeiliIndexing returns the Meilisearch indexing name.
+func (o *Options) MeiliIndexing() string {
+	return o.meili.indexing
 }
 
-// WaybackMeiliApikey returns the Meilisearch admin apikey.
-func (o *Options) WaybackMeiliApikey() string {
-	return o.waybackMeiliApikey
+// MeiliApikey returns the Meilisearch admin apikey.
+func (o *Options) MeiliApikey() string {
+	return o.meili.apikey
 }
 
 // EnabledMeilisearch returns whether enable meilisearch server.
 func (o *Options) EnabledMeilisearch() bool {
-	return o.WaybackMeiliEndpoint() != ""
+	return o.MeiliEndpoint() != ""
+}
+
+// OmnivoreApikey returns the Omnivore apikey.
+func (o *Options) OmnivoreApikey() string {
+	return o.omnivore.apikey
+}
+
+// EnabledOmnivore returns whether enable Omnivore.
+func (o *Options) EnabledOmnivore() bool {
+	return o.OmnivoreApikey() != ""
 }
 
 // HTTPdEnabled returns whether enable HTTP daemon service.

diff --git a/config/parser.go b/config/parser.go
@@ -222,11 +222,13 @@ func (p *Parser) parseLines(lines []string) (err error) {
 		case "WAYBACK_FALLBACK":
 			p.opts.waybackFallback = parseBool(val, defWaybackFallback)
 		case "WAYBACK_MEILI_ENDPOINT":
-			p.opts.waybackMeiliEndpoint = parseString(val, defWaybackMeiliEndpoint)
+			p.opts.meili.endpoint = parseString(val, defMeiliEndpoint)
 		case "WAYBACK_MEILI_INDEXING":
-			p.opts.waybackMeiliIndexing = parseString(val, defWaybackMeiliIndexing)
+			p.opts.meili.indexing = parseString(val, defMeiliIndexing)
 		case "WAYBACK_MEILI_APIKEY":
-			p.opts.waybackMeiliApikey = parseString(val, defWaybackMeiliApikey)
+			p.opts.meili.apikey = parseString(val, defMeiliApikey)
+		case "WAYBACK_OMNIVORE_APIKEY":
+			p.opts.omnivore.apikey = parseString(val, defOmnivoreApikey)
 		default:
 			if os.Getenv(key) == "" && val != "" {
 				os.Setenv(key, val)

diff --git a/docs/environment.md b/docs/environment.md
@@ -40,6 +40,7 @@ Use the `-c` / `--config` option to specify the build definition file to use.
 | -                   | `WAYBACK_MEILI_ENDPOINT`          | -                          | Meilisearch API endpoint                                     |
 | -                   | `WAYBACK_MEILI_INDEXING`          | `capsules`                 | Meilisearch indexing name                                    |
 | -                   | `WAYBACK_MEILI_APIKEY`            | -                          | Meilisearch admin API key                                    |
+| -                   | `WAYBACK_OMNIVORE_APIKEY`         | -                          | Omnivore API key                                             |
 | `-d`, `--daemon`    | -                                 | -                          | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` |
 | `--ia`              | `WAYBACK_ENABLE_IA`               | `true`                     | Wayback webpages to **Internet Archive**                     |
 | `--is`              | `WAYBACK_ENABLE_IS`               | `true`                     | Wayback webpages to **Archive Today**                        |

diff --git a/go.mod b/go.mod
@@ -16,7 +16,9 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b
 	github.com/go-shiori/obelisk v0.0.0-20230316095823-42f6a2f99d9d
+	github.com/goccy/go-json v0.10.3
 	github.com/google/go-github/v40 v40.0.0
+	github.com/google/uuid v1.3.0
 	github.com/gookit/color v1.5.3
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.1
@@ -36,7 +38,6 @@ require (
 	github.com/wabarc/archive.is v1.4.0
 	github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
 	github.com/wabarc/ghostarchive v0.1.1
-	github.com/wabarc/go-anonfile v0.1.0
 	github.com/wabarc/go-catbox v0.1.0
 	github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb
 	github.com/wabarc/imgbb v1.0.0
@@ -95,7 +96,6 @@ require (
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/ipfs/boxo v0.8.1 // indirect

diff --git a/go.sum b/go.sum
@@ -169,6 +169,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q=
 github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -479,13 +481,10 @@ github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896 h1:c3uD+IKXpN
 github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896/go.mod h1:yEmUMlNO2PPAxIvo/Hf/VxOrCS5SBwL2/vCW8pyTWjA=
 github.com/wabarc/ghostarchive v0.1.1 h1:iGnDwvUixynKGlUTtxGfWAD1p6QRtCj7pAYCqgZNZTQ=
 github.com/wabarc/ghostarchive v0.1.1/go.mod h1:+HB72/CrKK4at+QhWDKRZYB6WLLyE/xlXr6/LgfqFro=
-github.com/wabarc/go-anonfile v0.1.0 h1:M4jKUAMROxxVaqLQt30ONmaHA/0YnvyJtX8qg/zI9+I=
-github.com/wabarc/go-anonfile v0.1.0/go.mod h1:CH1LzSKQ0x/RlKpEG2gi+hXaOOZrfeRx6Rp717o+65A=
 github.com/wabarc/go-catbox v0.1.0 h1:/UhV9md3MJrjZtm+EToSyFjawXgPiHSExLNRqsWNisg=
 github.com/wabarc/go-catbox v0.1.0/go.mod h1:Zjs9Y55f2WOwGWwmKSCrUuMfwh+nDktkjub9jgHq4CQ=
 github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8=
 github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
-github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ=
 github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb h1:psEAY4wXvhXp/Hp5CJWgAOKWqhvAom+/hOjK+Qscx7o=

diff --git a/ingress/register/publish.go b/ingress/register/publish.go
@@ -12,6 +12,7 @@ import (
 	_ "github.com/wabarc/wayback/publish/meili"
 	_ "github.com/wabarc/wayback/publish/nostr"
 	_ "github.com/wabarc/wayback/publish/notion"
+	_ "github.com/wabarc/wayback/publish/omnivore"
 	_ "github.com/wabarc/wayback/publish/relaychat"
 	_ "github.com/wabarc/wayback/publish/slack"
 	_ "github.com/wabarc/wayback/publish/telegram"

diff --git a/metrics/metrics.go b/metrics/metrics.go
@@ -26,17 +26,18 @@ const (
 	ServiceTwitter  = "twitter"
 	ServiceXMPP     = "xmpp"
 
-	PublishIRC     = "irc"      // IRC channel
-	PublishGithub  = "github"   // GitHub issues
-	PublishNotion  = "notion"   // Notion page
-	PublishChannel = "telegram" // Telegram channel
-	PublishMstdn   = "mastodon" // Mastodon toot
-	PublishDiscord = "discord"  // Discord channel
-	PublishTwitter = "twitter"
-	PublishMatrix  = "matrix"
-	PublishSlack   = "slack"
-	PublishNostr   = "nostr"
-	PublishMeili   = "meili"
+	PublishIRC      = "irc"      // IRC channel
+	PublishGithub   = "github"   // GitHub issues
+	PublishNotion   = "notion"   // Notion page
+	PublishChannel  = "telegram" // Telegram channel
+	PublishMstdn    = "mastodon" // Mastodon toot
+	PublishDiscord  = "discord"  // Discord channel
+	PublishTwitter  = "twitter"
+	PublishMatrix   = "matrix"
+	PublishSlack    = "slack"
+	PublishNostr    = "nostr"
+	PublishMeili    = "meili"
+	PublishOmnivore = "omnivore"
 
 	StatusRequest = "request"
 	StatusSuccess = "success"

diff --git a/publish/meili/meili.go b/publish/meili/meili.go
@@ -55,7 +55,7 @@ type Meili struct {
 
 // New returns a Meilisearch client.
 func New(client *http.Client, opts *config.Options) *Meili {
-	endpoint, apikey, idxname := opts.WaybackMeiliEndpoint(), opts.WaybackMeiliApikey(), opts.WaybackMeiliIndexing()
+	endpoint, apikey, idxname := opts.MeiliEndpoint(), opts.MeiliApikey(), opts.MeiliIndexing()
 
 	if client == nil {
 		client = &http.Client{

diff --git a/publish/omnivore/doc.go b/publish/omnivore/doc.go
@@ -0,0 +1,5 @@
+// Copyright 2024 Wayback Archiver. All rights reserved.
+// Use of this source code is governed by the GNU GPL v3
+// license that can be found in the LICENSE file.
+
+package omnivore // import "github.com/wabarc/wayback/publish/omnivore"