diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 8238a5be..9bbf6629 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -49,6 +49,7 @@ jobs: with: language: ${{ matrix.language }} config-file: './.github/codeql/codeql-config.yml' + egress-policy: audit nancy: name: Sonatype Nancy @@ -85,3 +86,4 @@ jobs: with: scan-type: 'fs' sarif: 'filesystem.sarif' + egress-policy: audit diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 40eed65f..5bc5e980 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -99,6 +99,7 @@ jobs: go-mips64: ${{ matrix.mips64 }} go-mipsle: ${{ matrix.mipsle }} artifact-path: ./build/binary/wayback* + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} @@ -122,6 +123,7 @@ jobs: go-arch: ${{ matrix.arch }} go-arm: ${{ matrix.arm }} artifact-path: build/package/wayback*.deb + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} @@ -132,6 +134,7 @@ jobs: product: wayback params: 'make rpm' artifact-path: build/package/wayback*.rpm + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} @@ -145,6 +148,7 @@ jobs: build/aur/.SRCINFO build/aur/PKGBUILD build/aur/wayback*.pkg.tar.zst + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} @@ -153,8 +157,9 @@ jobs: uses: wabarc/.github/.github/workflows/reusable-builder-snap.yml@main with: product: wayback - channel: edge + channel: stable publish: ${{ github.repository == 'wabarc/wayback' && github.event_name == 'push' }} + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} @@ -167,5 +172,6 @@ jobs: version: edge params: 'make build' artifact-path: org.wabarc.wayback-*.x86_64.flatpak + egress-policy: audit secrets: wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 05a06f70..1a996844 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,13 +54,13 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@2e205a28d0e1da00c5f53b161f4067b052c61f34 # v1.5.0 with: - egress-policy: block + egress-policy: audit disable-telemetry: true allowed-endpoints: > ghcr.io:443 github.com:443 api.github.com:443 - pkg-containers.githubusercontent.com:443 + *.githubusercontent.com docker.io:443 auth.docker.io:443 index.docker.io:443 @@ -78,9 +78,7 @@ jobs: - name: Install Cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # v2.8.1 - with: - cosign-release: 'v1.13.1' + uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0 - name: Prepare id: prep @@ -158,14 +156,14 @@ jobs: *.cache-from=type=local,src=/tmp/.image-cache/image *.cache-to=type=local,dest=/tmp/.image-cache-new/image - - name: Sign image with a key + - name: Siging image if: github.event_name != 'pull_request' env: TAGS: ${{ steps.meta.outputs.tags }} COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} COSIGN_PASSWORD: ${{secrets.COSIGN_PASSPHARSE}} run: | - cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} + cosign sign --yes --key env://COSIGN_PRIVATE_KEY ${TAGS} - name: Check manifest if: github.event_name != 'pull_request' @@ -201,14 +199,14 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@2e205a28d0e1da00c5f53b161f4067b052c61f34 # v1.5.0 with: - egress-policy: block + egress-policy: audit disable-telemetry: true allowed-endpoints: > ghcr.io:443 github.com:443 api.github.com:443 docker.io:443 - pkg-containers.githubusercontent.com:443 + *.githubusercontent.com auth.docker.io:443 registry-1.docker.io:443 production.cloudflare.docker.com:443 @@ -224,9 +222,7 @@ jobs: - name: Install Cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # v2.8.1 - with: - cosign-release: 'v1.13.1' + uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0 - name: Prepare id: prep @@ -304,7 +300,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} COSIGN_PASSWORD: ${{secrets.COSIGN_PASSPHARSE}} run: | - cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} + cosign sign --yes --key env://COSIGN_PRIVATE_KEY ${TAGS} - name: Check manifest if: github.event_name != 'pull_request' @@ -331,11 +327,12 @@ jobs: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - #actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status with: scan-type: 'image' image-ref: '${{ needs.publish.outputs.image }}:${{ needs.publish.outputs.version }}' sarif: 'container-standalone.sarif' + egress-policy: 'audit' trivy-bundle: name: Trivy for bundle @@ -344,8 +341,9 @@ jobs: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - #actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status with: scan-type: 'image' image-ref: '${{ needs.allinone.outputs.image }}:${{ needs.allinone.outputs.version }}' sarif: 'container-bundle.sarif' + egress-policy: 'audit' diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 484ca9b8..f26f3c2d 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -26,3 +26,5 @@ jobs: license: name: License Checker uses: wabarc/.github/.github/workflows/reusable-license.yml@main + with: + egress-policy: audit diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 63ac71c9..129af4e1 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -29,6 +29,8 @@ jobs: golangci: name: golangci-lint uses: wabarc/.github/.github/workflows/reusable-golangci.yml@main + with: + egress-policy: audit shellcheck: name: ShellCheck diff --git a/.licenserc.yaml b/.licenserc.yaml index 8525e8f8..453834ce 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -45,3 +45,14 @@ header: - 'mkdocs.yml' comment: on-failure + +dependency: + files: + - go.mod + licenses: + - name: github.com/multiformats/go-base36 + version: v0.2.0 + license: Apache-2.0 OR MIT + - name: github.com/multiformats/go-multicodec + version: v0.9.0 + license: Apache-2.0 OR MIT diff --git a/README.md b/README.md index 4a91bbea..c0f5ae86 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,7 @@ [![World Wide Web](https://img.shields.io/badge/Web-15aabf.svg)](https://wabarc.eu.org/) [![Nostr](https://img.shields.io/badge/Nostr-8e44ad.svg)](https://iris.to/#/profile/npub1gm4xeu8wlt6aa56zenutkwa0ppjng5axsscv424d0xvv5jalxxzs4hjukz) -Wayback is a tool that supports running as a command-line tool and docker container, purpose to snapshot webpage to time capsules. - -Supported Golang version: See [.github/workflows/testing.yml](./.github/workflows/testing.yml) +Wayback is a web archiving and playback tool that allows users to capture and preserve web content. It provides an IM-style interface for receiving and presenting archived web content, and a search and playback service for retrieving previously archived pages. Wayback is designed to be used by web archivists, researchers, and anyone who wants to preserve web content and access it in the future. ## Features @@ -32,7 +30,11 @@ Supported Golang version: See [.github/workflows/testing.yml](./.github/workflow - Supports storing archived files to disk for offline use - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving. -## Installation +## Getting Started + +For a comprehensive guide, please refer to the complete [documentation](https://docs.wabarc.eu.org/). + +### Installation The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/wabarc/wayback/releases) and place the executable file in your PATH. @@ -45,7 +47,7 @@ go install github.com/wabarc/wayback/cmd/wayback@latest From GitHub Releases: ```sh -curl -fsSL https://github.com/wabarc/wayback/raw/main/install.sh | sh +curl -fsSL https://get.wabarc.eu.org | sh ``` or via [Bina](https://bina.egoist.dev/): @@ -92,9 +94,9 @@ brew tap wabarc/wayback brew install wayback ``` -## Usage +### Usage -### Command line +#### Command line ```sh $ wayback -h @@ -208,100 +210,13 @@ cat url.txt | wayback #### Configuration Parameters -By default, `wayback` looks for configuration options from this files, the following are parsed: - -- `./wayback.conf` -- `~/wayback.conf` -- `/etc/wayback.conf` - -Use the `-c` / `--config` option to specify the build definition file to use. - -You can also specify configuration options either via command flags or via environment variables, an overview of all options below. - -| Flags | Environment Variable | Default | Description | -| ------------------- | --------------------------------- | -------------------------- | ------------------------------------------------------------ | -| `--debug` | `DEBUG` | `false` | Enable debug mode, override `LOG_LEVEL` | -| `-c`, `--config` | - | - | Configuration file path, defaults: `./wayback.conf`, `~/wayback.conf`, `/etc/wayback.conf` | -| - | `LOG_TIME` | `true` | Display the date and time in log messages | -| - | `LOG_LEVEL` | `info` | Log level, supported level are `debug`, `info`, `warn`, `error`, `fatal`, defaults to `info` | -| - | `ENABLE_METRICS` | `false` | Enable metrics collector | -| - | `WAYBACK_LISTEN_ADDR` | `0.0.0.0:8964` | The listen address for the HTTP server | -| - | `CHROME_BIN` | - | Preferred to sets the path to the Chrome executable | -| - | `CHROME_REMOTE_ADDR` | - | Chrome/Chromium remote debugging address, for screenshot, format: `host:port`, `wss://domain.tld` | -| - | `WAYBACK_PROXY` | - | Proxy address, e.g. `socks5://127.0.0.1:1080` | -| - | `WAYBACK_POOLING_SIZE` | `3` | Number of worker pool for wayback at once | -| - | `WAYBACK_BOLT_PATH` | `./wayback.db` | File path of bolt database | -| - | `WAYBACK_STORAGE_DIR` | - | Directory to store binary file, e.g. PDF, html file | -| - | `WAYBACK_MAX_MEDIA_SIZE` | `512MB` | Max size to limit download stream media | -| - | `WAYBACK_MEDIA_SITES` | - | Extra media websites wish to be supported, separate with comma | -| - | `WAYBACK_TIMEOUT` | `300` | Timeout for single wayback request, defaults to 300 second | -| - | `WAYBACK_MAX_RETRIES` | `2` | Max retries for single wayback request, defaults to 2 | -| - | `WAYBACK_USERAGENT` | `WaybackArchiver/1.0` | User-Agent for a wayback request | -| - | `WAYBACK_FALLBACK` | `off` | Use Google cache as a fallback if the original webpage is unavailable | -| - | `WAYBACK_MEILI_ENDPOINT` | - | Meilisearch API endpoint | -| - | `WAYBACK_MEILI_INDEXING` | `capsules` | Meilisearch indexing name | -| - | `WAYBACK_MEILI_APIKEY` | - | Meilisearch admin API key | -| `-d`, `--daemon` | - | - | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` | -| `--ia` | `WAYBACK_ENABLE_IA` | `true` | Wayback webpages to **Internet Archive** | -| `--is` | `WAYBACK_ENABLE_IS` | `true` | Wayback webpages to **Archive Today** | -| `--ip` | `WAYBACK_ENABLE_IP` | `false` | Wayback webpages to **IPFS** | -| `--ph` | `WAYBACK_ENABLE_PH` | `false` | Wayback webpages to **[Telegra.ph](https://telegra.ph)**, required Chrome/Chromium | -| `--ipfs-host` | `WAYBACK_IPFS_HOST` | `127.0.0.1` | IPFS daemon service host | -| `-p`, `--ipfs-port` | `WAYBACK_IPFS_PORT` | `5001` | IPFS daemon service port | -| `-m`, `--ipfs-mode` | `WAYBACK_IPFS_MODE` | `pinner` | IPFS mode for preserve webpage, e.g. `daemon`, `pinner` | -| - | `WAYBACK_IPFS_TARGET` | `web3storage` | The IPFS pinning service is used to store files, supported pinners: infura, pinata, nftstorage, web3storage. | -| - | `WAYBACK_IPFS_APIKEY` | - | Apikey of the IPFS pinning service | -| - | `WAYBACK_IPFS_SECRET` | - | Secret of the IPFS pinning service | -| - | `WAYBACK_GITHUB_TOKEN` | - | GitHub Personal Access Token, required the `repo` scope | -| - | `WAYBACK_GITHUB_OWNER` | - | GitHub account name | -| - | `WAYBACK_GITHUB_REPO` | - | GitHub repository to publish results | -| - | `WAYBACK_NOTION_TOKEN` | - | Notion integration token | -| - | `WAYBACK_NOTION_DATABASE_ID` | - | Notion database ID for archiving results | -| `-t`, `--token` | `WAYBACK_TELEGRAM_TOKEN` | - | Telegram Bot API Token | -| `--chatid` | `WAYBACK_TELEGRAM_CHANNEL` | - | The Telegram public/private channel id to publish archive result | -| - | `WAYBACK_TELEGRAM_HELPTEXT` | - | The help text for Telegram command | -| - | `WAYBACK_MASTODON_SERVER` | - | Domain of Mastodon instance | -| - | `WAYBACK_MASTODON_KEY` | - | The client key of your Mastodon application | -| - | `WAYBACK_MASTODON_SECRET` | - | The client secret of your Mastodon application | -| - | `WAYBACK_MASTODON_TOKEN` | - | The access token of your Mastodon application | -| - | `WAYBACK_TWITTER_CONSUMER_KEY` | - | The customer key of your Twitter application | -| - | `WAYBACK_TWITTER_CONSUMER_SECRET` | - | The customer secret of your Twitter application | -| - | `WAYBACK_TWITTER_ACCESS_TOKEN` | - | The access token of your Twitter application | -| - | `WAYBACK_TWITTER_ACCESS_SECRET` | - | The access secret of your Twitter application | -| - | `WAYBACK_IRC_NICK` | - | IRC nick | -| - | `WAYBACK_IRC_PASSWORD` | - | IRC password | -| - | `WAYBACK_IRC_CHANNEL` | - | IRC channel | -| - | `WAYBACK_IRC_SERVER` | `irc.libera.chat:6697` | IRC server, required TLS | -| - | `WAYBACK_MATRIX_HOMESERVER` | `https://matrix.org` | Matrix homeserver | -| - | `WAYBACK_MATRIX_USERID` | - | Matrix unique user ID, format: `@foo:example.com` | -| - | `WAYBACK_MATRIX_ROOMID` | - | Matrix internal room ID, format: `!bar:example.com` | -| - | `WAYBACK_MATRIX_PASSWORD` | - | Matrix password | -| - | `WAYBACK_DISCORD_BOT_TOKEN` | - | Discord bot authorization token | -| - | `WAYBACK_DISCORD_CHANNEL` | - | Discord channel ID, [find channel ID](https://support.discord.com/hc/en-us/articles/206346498-Where-can-I-find-my-server-ID-) | -| - | `WAYBACK_DISCORD_HELPTEXT` | - | The help text for Discord command | -| - | `WAYBACK_SLACK_APP_TOKEN` | - | App-Level Token of Slack app | -| - | `WAYBACK_SLACK_BOT_TOKEN` | - | `Bot User OAuth Token` for Slack workspace, use `User OAuth Token` if requires create external link | -| - | `WAYBACK_SLACK_CHANNEL` | - | Channel ID of Slack channel | -| - | `WAYBACK_SLACK_HELPTEXT` | - | The help text for Slack slash command | -| - | `WAYBACK_NOSTR_RELAY_URL` | `wss://nostr.developer.li` | Nostr relay server url, multiple separated by comma | -| - | `WAYBACK_NOSTR_PRIVATE_KEY` | - | The private key of a Nostr account | -| - | `WAYBACK_XMPP_JID` | - | The JID of a XMPP account | -| - | `WAYBACK_XMPP_PASSWORD` | - | The password of a XMPP account | -| - | `WAYBACK_XMPP_NOTLS` | - | Connect to XMPP server without TLS | -| - | `WAYBACK_XMPP_HELPTEXT` | - | The help text for XMPP command | -| `--tor` | `WAYBACK_USE_TOR` | `false` | Snapshot webpage via Tor anonymity network | -| `--tor-key` | `WAYBACK_ONION_PRIVKEY` | - | The private key for Tor Hidden Service | -| - | `WAYBACK_ONION_LOCAL_PORT` | `8964` | Local port for Tor Hidden Service, also support for a **reverse proxy**. This is ignored if `WAYBACK_LISTEN_ADDR` is set. | -| - | `WAYBACK_ONION_REMOTE_PORTS` | `80` | Remote ports for Tor Hidden Service, e.g. `WAYBACK_ONION_REMOTE_PORTS=80,81` | -| - | `WAYBACK_ONION_DISABLED` | `false` | Disable onion service | -| - | `WAYBACK_SLOT` | - | Pinning service for IPFS mode of pinner, see [ipfs-pinner](https://github.com/wabarc/ipfs-pinner#supported-pinning-services) | -| - | `WAYBACK_APIKEY` | - | API key for pinning service | -| - | `WAYBACK_SECRET` | - | API secret for pinning service | - -If both of the definition file and environment variables are specified, they are all will be read and apply, -and preferred from the environment variable for the same item. - -Prints the resulting options of the targets with `--print`, in a Go struct with type, without running the `wayback`. +Look at the [full list of configuration options](docs/environment.md). + +## Deployment + +- [wabarc/on-heroku](https://github.com/wabarc/on-heroku) +- [wabarc/on-github](https://github.com/wabarc/on-github) +- [wabarc/on-render](https://github.com/wabarc/on-render) ### Docker/Podman @@ -322,15 +237,36 @@ docker run -d wabarc/wayback wayback -d telegram -t YOUR-BOT-TOKEN -c YOUR-CHANN /> -## Deployment +## Screenshots -- [wabarc/on-heroku](https://github.com/wabarc/on-heroku) -- [wabarc/on-github](https://github.com/wabarc/on-github) -- [wabarc/on-render](https://github.com/wabarc/on-render) +
Click to see screenshots of the services. -## Documentation +### Discord +![Discord](./docs/assets/discord-server.png) -For a comprehensive guide, please refer to the complete [documentation](https://docs.wabarc.eu.org/). +### Web Service +![Web](./docs/assets/web.png) + +### Mastodon +![Mastodon](./docs/assets/mastodon.png) + +### Matrix +![Matrix Room](./docs/assets/matrix-room.png) + +### IRC +![IRC](./docs/assets/irc.png) + +### Slack +![Slack Channel](./docs/assets/slack-channel.png) + +### Telegram +![Telegram Bot](./docs/assets/telegram.png) +![Telegram Channel](./docs/assets/telegram-channel.png) + +### XMPP +![XMPP](./docs/assets/xmpp.png) + +
## Contributing diff --git a/docs/assets/discord-server.png b/docs/assets/discord-server.png new file mode 100644 index 00000000..21bff5b9 Binary files /dev/null and b/docs/assets/discord-server.png differ diff --git a/docs/assets/irc.png b/docs/assets/irc.png new file mode 100644 index 00000000..912e6a33 Binary files /dev/null and b/docs/assets/irc.png differ diff --git a/docs/assets/mastodon.png b/docs/assets/mastodon.png new file mode 100644 index 00000000..319b0d0c Binary files /dev/null and b/docs/assets/mastodon.png differ diff --git a/docs/assets/matrix-room.png b/docs/assets/matrix-room.png new file mode 100644 index 00000000..736e11f5 Binary files /dev/null and b/docs/assets/matrix-room.png differ diff --git a/docs/assets/slack-channel.png b/docs/assets/slack-channel.png new file mode 100644 index 00000000..f2df3bc4 Binary files /dev/null and b/docs/assets/slack-channel.png differ diff --git a/docs/assets/telegram-channel.png b/docs/assets/telegram-channel.png new file mode 100644 index 00000000..556eb98c Binary files /dev/null and b/docs/assets/telegram-channel.png differ diff --git a/docs/assets/telegram.png b/docs/assets/telegram.png new file mode 100644 index 00000000..a13bdaf1 Binary files /dev/null and b/docs/assets/telegram.png differ diff --git a/docs/assets/web.png b/docs/assets/web.png new file mode 100644 index 00000000..70c0aaba Binary files /dev/null and b/docs/assets/web.png differ diff --git a/docs/assets/xmpp.png b/docs/assets/xmpp.png new file mode 100644 index 00000000..35e1c94b Binary files /dev/null and b/docs/assets/xmpp.png differ diff --git a/docs/changelog.md b/docs/changelog.md index 6b2aca39..b580d095 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add docker-compose.yml ([#367](https://github.com/wabarc/wayback/pull/367)) - Add option to disable onion service ([#372](https://github.com/wabarc/wayback/pull/372)) - Add support for setting preferred path to Chrome executable ([#375](https://github.com/wabarc/wayback/pull/375)) +- Add support for XMPP ([#380](https://github.com/wabarc/wayback/pull/380)) ### Changed - Set NoWait to true for tor listen config ([#368](https://github.com/wabarc/wayback/pull/368)) @@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactor and reorganize service entry ([#389](https://github.com/wabarc/wayback/pull/389)) - Replace `fatih/color` with `gookit/color` ([#390](https://github.com/wabarc/wayback/pull/390)) - Using concurrency to manage new workflow runs ([#384](https://github.com/wabarc/wayback/pull/384)) +- Improve documentation ([#385](https://github.com/wabarc/wayback/pull/385)) ## [0.19.1] - 2023-03-21 diff --git a/docs/environment.md b/docs/environment.md new file mode 100644 index 00000000..c496b6c7 --- /dev/null +++ b/docs/environment.md @@ -0,0 +1,98 @@ +# Configuration Parameters + +Wayback can use a configuration file and environment variables. + +If both of the definition file and environment variables are specified, they are all will be read and apply, +and preferred from the environment variable for the same item. + +Prints the resulting options of the targets with `--print`, in a Go struct with type, without running the `wayback`. + +By default, `wayback` looks for configuration options from this files, the following are parsed: + +- `./wayback.conf` +- `~/wayback.conf` +- `/etc/wayback.conf` + +Use the `-c` / `--config` option to specify the build definition file to use. + +## Configuration Options + +| Flags | Environment Variable | Default | Description | +| ------------------- | --------------------------------- | -------------------------- | ------------------------------------------------------------ | +| `--debug` | `DEBUG` | `false` | Enable debug mode, override `LOG_LEVEL` | +| `-c`, `--config` | - | - | Configuration file path, defaults: `./wayback.conf`, `~/wayback.conf`, `/etc/wayback.conf` | +| - | `LOG_TIME` | `true` | Display the date and time in log messages | +| - | `LOG_LEVEL` | `info` | Log level, supported level are `debug`, `info`, `warn`, `error`, `fatal`, defaults to `info` | +| - | `ENABLE_METRICS` | `false` | Enable metrics collector | +| - | `WAYBACK_LISTEN_ADDR` | `0.0.0.0:8964` | The listen address for the HTTP server | +| - | `CHROME_BIN` | - | Preferred to sets the path to the Chrome executable | +| - | `CHROME_REMOTE_ADDR` | - | Chrome/Chromium remote debugging address, for screenshot, format: `host:port`, `wss://domain.tld` | +| - | `WAYBACK_PROXY` | - | Proxy address, e.g. `socks5://127.0.0.1:1080` | +| - | `WAYBACK_POOLING_SIZE` | `3` | Number of worker pool for wayback at once | +| - | `WAYBACK_BOLT_PATH` | `./wayback.db` | File path of bolt database | +| - | `WAYBACK_STORAGE_DIR` | - | Directory to store binary file, e.g. PDF, html file | +| - | `WAYBACK_MAX_MEDIA_SIZE` | `512MB` | Max size to limit download stream media | +| - | `WAYBACK_MEDIA_SITES` | - | Extra media websites wish to be supported, separate with comma | +| - | `WAYBACK_TIMEOUT` | `300` | Timeout for single wayback request, defaults to 300 second | +| - | `WAYBACK_MAX_RETRIES` | `2` | Max retries for single wayback request, defaults to 2 | +| - | `WAYBACK_USERAGENT` | `WaybackArchiver/1.0` | User-Agent for a wayback request | +| - | `WAYBACK_FALLBACK` | `off` | Use Google cache as a fallback if the original webpage is unavailable | +| - | `WAYBACK_MEILI_ENDPOINT` | - | Meilisearch API endpoint | +| - | `WAYBACK_MEILI_INDEXING` | `capsules` | Meilisearch indexing name | +| - | `WAYBACK_MEILI_APIKEY` | - | Meilisearch admin API key | +| `-d`, `--daemon` | - | - | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` | +| `--ia` | `WAYBACK_ENABLE_IA` | `true` | Wayback webpages to **Internet Archive** | +| `--is` | `WAYBACK_ENABLE_IS` | `true` | Wayback webpages to **Archive Today** | +| `--ip` | `WAYBACK_ENABLE_IP` | `false` | Wayback webpages to **IPFS** | +| `--ph` | `WAYBACK_ENABLE_PH` | `false` | Wayback webpages to **[Telegra.ph](https://telegra.ph)**, required Chrome/Chromium | +| `--ipfs-host` | `WAYBACK_IPFS_HOST` | `127.0.0.1` | IPFS daemon service host | +| `-p`, `--ipfs-port` | `WAYBACK_IPFS_PORT` | `5001` | IPFS daemon service port | +| `-m`, `--ipfs-mode` | `WAYBACK_IPFS_MODE` | `pinner` | IPFS mode for preserve webpage, e.g. `daemon`, `pinner` | +| - | `WAYBACK_IPFS_TARGET` | `web3storage` | The IPFS pinning service is used to store files, supported pinners: infura, pinata, nftstorage, web3storage. | +| - | `WAYBACK_IPFS_APIKEY` | - | Apikey of the IPFS pinning service | +| - | `WAYBACK_IPFS_SECRET` | - | Secret of the IPFS pinning service | +| - | `WAYBACK_GITHUB_TOKEN` | - | GitHub Personal Access Token, required the `repo` scope | +| - | `WAYBACK_GITHUB_OWNER` | - | GitHub account name | +| - | `WAYBACK_GITHUB_REPO` | - | GitHub repository to publish results | +| - | `WAYBACK_NOTION_TOKEN` | - | Notion integration token | +| - | `WAYBACK_NOTION_DATABASE_ID` | - | Notion database ID for archiving results | +| `-t`, `--token` | `WAYBACK_TELEGRAM_TOKEN` | - | Telegram Bot API Token | +| `--chatid` | `WAYBACK_TELEGRAM_CHANNEL` | - | The Telegram public/private channel id to publish archive result | +| - | `WAYBACK_TELEGRAM_HELPTEXT` | - | The help text for Telegram command | +| - | `WAYBACK_MASTODON_SERVER` | - | Domain of Mastodon instance | +| - | `WAYBACK_MASTODON_KEY` | - | The client key of your Mastodon application | +| - | `WAYBACK_MASTODON_SECRET` | - | The client secret of your Mastodon application | +| - | `WAYBACK_MASTODON_TOKEN` | - | The access token of your Mastodon application | +| - | `WAYBACK_TWITTER_CONSUMER_KEY` | - | The customer key of your Twitter application | +| - | `WAYBACK_TWITTER_CONSUMER_SECRET` | - | The customer secret of your Twitter application | +| - | `WAYBACK_TWITTER_ACCESS_TOKEN` | - | The access token of your Twitter application | +| - | `WAYBACK_TWITTER_ACCESS_SECRET` | - | The access secret of your Twitter application | +| - | `WAYBACK_IRC_NICK` | - | IRC nick | +| - | `WAYBACK_IRC_PASSWORD` | - | IRC password | +| - | `WAYBACK_IRC_CHANNEL` | - | IRC channel | +| - | `WAYBACK_IRC_SERVER` | `irc.libera.chat:6697` | IRC server, required TLS | +| - | `WAYBACK_MATRIX_HOMESERVER` | `https://matrix.org` | Matrix homeserver | +| - | `WAYBACK_MATRIX_USERID` | - | Matrix unique user ID, format: `@foo:example.com` | +| - | `WAYBACK_MATRIX_ROOMID` | - | Matrix internal room ID, format: `!bar:example.com` | +| - | `WAYBACK_MATRIX_PASSWORD` | - | Matrix password | +| - | `WAYBACK_DISCORD_BOT_TOKEN` | - | Discord bot authorization token | +| - | `WAYBACK_DISCORD_CHANNEL` | - | Discord channel ID, [find channel ID](https://support.discord.com/hc/en-us/articles/206346498-Where-can-I-find-my-server-ID-) | +| - | `WAYBACK_DISCORD_HELPTEXT` | - | The help text for Discord command | +| - | `WAYBACK_SLACK_APP_TOKEN` | - | App-Level Token of Slack app | +| - | `WAYBACK_SLACK_BOT_TOKEN` | - | `Bot User OAuth Token` for Slack workspace, use `User OAuth Token` if requires create external link | +| - | `WAYBACK_SLACK_CHANNEL` | - | Channel ID of Slack channel | +| - | `WAYBACK_SLACK_HELPTEXT` | - | The help text for Slack slash command | +| - | `WAYBACK_NOSTR_RELAY_URL` | `wss://nostr.developer.li` | Nostr relay server url, multiple separated by comma | +| - | `WAYBACK_NOSTR_PRIVATE_KEY` | - | The private key of a Nostr account | +| - | `WAYBACK_XMPP_JID` | - | The JID of a XMPP account | +| - | `WAYBACK_XMPP_PASSWORD` | - | The password of a XMPP account | +| - | `WAYBACK_XMPP_NOTLS` | - | Connect to XMPP server without TLS | +| - | `WAYBACK_XMPP_HELPTEXT` | - | The help text for XMPP command | +| `--tor` | `WAYBACK_USE_TOR` | `false` | Snapshot webpage via Tor anonymity network | +| `--tor-key` | `WAYBACK_ONION_PRIVKEY` | - | The private key for Tor Hidden Service | +| - | `WAYBACK_ONION_LOCAL_PORT` | `8964` | Local port for Tor Hidden Service, also support for a **reverse proxy**. This is ignored if `WAYBACK_LISTEN_ADDR` is set. | +| - | `WAYBACK_ONION_REMOTE_PORTS` | `80` | Remote ports for Tor Hidden Service, e.g. `WAYBACK_ONION_REMOTE_PORTS=80,81` | +| - | `WAYBACK_ONION_DISABLED` | `false` | Disable onion service | +| - | `WAYBACK_SLOT` | - | Pinning service for IPFS mode of pinner, see [ipfs-pinner](https://github.com/wabarc/ipfs-pinner#supported-pinning-services) | +| - | `WAYBACK_APIKEY` | - | API key for pinning service | +| - | `WAYBACK_SECRET` | - | API secret for pinning service | diff --git a/docs/index.md b/docs/index.md index 2836ddc3..50e23b8d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ -Wayback is a web archiving and replay tool that enables users to capturing and preserving web content. It provides an interface to crawl and archive the contents of the web, and a search and replay service to access previously archived pages. Wayback is designed to be used by web archivists, researchers, and anyone who wants to preserve web content and access it in the future. +Wayback is a web archiving and playback tool that allows users to capture and preserve web content. It provides an IM-style interface for receiving and presenting archived web content, and a search and playback service for retrieving previously archived pages. Wayback is designed to be used by web archivists, researchers, and anyone who wants to preserve web content and access it in the future. Wayback is an open-source web archiving application written in Go. With a modular and customizable architecture, it is designed to be flexible and adaptable to various use cases and environments. It provides support for multiple storage backends and integration with other services. -Whether you need to archive a single web page or a large collection of websites, Wayback can help you capture and store web content for posterity. With its user-friendly interface and powerful features, Wayback is a valuable tool for anyone interested in web archiving and preservation. +Whether you need to archive a single web page or a large collection of web sites, Wayback can help you capture and preserve web content for posterity. With an easy-to-use interface and powerful features, Wayback is a valuable tool for anyone interested in web archiving and preservation. ## Features diff --git a/docs/index.zh.md b/docs/index.zh.md index 194e2a46..16a31fa9 100644 --- a/docs/index.zh.md +++ b/docs/index.zh.md @@ -2,11 +2,11 @@ title: 简介 --- -Wayback是一个网络存档和回放工具，可以帮助用户捕获和保存网络内容。它提供了一个会话界面，用于爬取和存档网络内容，以及一个搜索和回放服务，用于访问先前存档的页面。Wayback旨在供网络存档员、研究人员以及任何想要保存网络内容并在未来访问它的人使用。 +Wayback是一个网络归档和回放工具，允许用户抓取和保存网络内容。它提供了一个IM风格的界面来接收和展示存档的网络内容，以及一个搜索和回放服务来检索以前存档的网页。Wayback是为网络档案员、研究人员和任何想保存网络内容并在未来访问这些内容的人设计的。 Wayback是用Go编写的开源网络存档应用程序。具有模块化和可定制化的架构，它被设计成灵活和适应各种用例和环境。它提供了对多个存储后端的支持，并与其他服务集成。 -无论您需要存档单个网页还是大量网站的集合，Wayback都可以帮助您为后人捕获和存储网络内容。凭借其用户友好的界面和强大的功能，Wayback是对网络存档和保护感兴趣的任何人都非常有价值的工具。 +无论您是需要归档一个网页，还是需要归档一大批网站，Wayback都可以帮助你抓取和保存网络内容，以备后用。凭借易于使用的界面和强大的功能，Wayback对于任何对网络归档和保存感兴趣的人来说都是一个宝贵的工具。 ## 特性 diff --git a/mkdocs.yml b/mkdocs.yml index a3948057..31a7fe03 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,7 +43,8 @@ nav: - Installation: 'installation.md' - Usage: - 'Command Line': 'command-line.md' - - 'Service': 'service.md' + - 'Daemon Service': 'service.md' + - 'Configurations': 'environment.md' - Deployment: 'deployment.md' - Integrations: - Discord: 'integrations/discord.md' @@ -120,7 +121,8 @@ plugins: Installation: 安装 Usage: 使用 'Command Line': 命令行 - 'Service': 守护服务 + 'Daemon Service': 守护服务 + Configurations: 配置选项 Deployment: 部署 Integrations: 集成 Resources: 资源 diff --git a/snapcraft.yaml b/snapcraft.yaml index 815f21f7..c3b7eeb4 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -12,23 +12,34 @@ description: | A toolkit for snapshot webpage to the Wayback Machine. Website https://github.com/wabarc/wayback -base: core18 +base: core20 grade: stable confinement: strict compression: lzo + architectures: - #- build-on: [i386, amd64, arm64, armhf, ppc64el, s390x] - - build-on: amd64 + - amd64 + - arm64 + - armhf + - i386 parts: wayback: plugin: go source: https://github.com/wabarc/wayback.git - go-importpath: github.com/wabarc/wayback/cmd/wayback + override-build: | + make build + install $SNAPCRAFT_PART_BUILD/build/binary/wayback -D $SNAPCRAFT_PART_INSTALL/bin/wayback + mkdir -p $SNAPCRAFT_PART_INSTALL/bin + if [ ! -e $SNAPCRAFT_PART_INSTALL/bin/wayback ]; then + ln -s $SNAPCRAFT_PART_INSTALL/bin/wayback $SNAPCRAFT_PART_INSTALL/bin/wayback + fi + build-packages: + - build-essential apps: wayback: - command: wayback + command: bin/wayback plugs: - home - network