Initial commit

.github/dependabot.yml

@@ -0,0 +1,18 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 10
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+

.github/workflows/linter.yml

@@ -0,0 +1,24 @@
+name: Linter
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+    types: [ opened, synchronize, reopened ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout default branch
+        uses: actions/checkout@v2
+
+      - name: Lint Code Base
+        uses: github/super-linter@v4
+        env:
+          DEFAULT_BRANCH: 'main'
+          VALIDATE_ALL_CODEBASE: false
+          VALIDATE_JSON: false
+          VALIDATE_ANSIBLE: false
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,82 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+env:
+  PRODUCT: warcraft
+
+jobs:
+  build:
+    name: Checkout, build, archive, upload
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.16
+
+    - name: Build fat binary
+      run: make all-arch
+
+    - name: Archive binary
+      run: make releases
+
+    - name: Upload archived binary
+      uses: actions/upload-artifact@v2
+      with:
+        name: ${{ env.PRODUCT }}
+        path: build/package/${{ env.PRODUCT }}*
+
+  checksum:
+    name: Get archived packages checksum
+    runs-on: ubuntu-latest
+    needs: build
+    outputs:
+      digest: ${{ steps.digest.outputs.result }}
+    steps:
+    - name: Download math result from build job
+      uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.PRODUCT }}
+        path: .
+
+    - name: Create all binary digest
+      id: digest
+      run: |
+        digest=$(find ${{ env.PRODUCT }}* -type f -exec sha256sum {} +)
+        digest="${digest//$'%'/%25}"
+        digest="${digest//$'\n'/%0A}"
+        echo "::set-output name=result::$digest"
+
+  release:
+    name: Create and upload release
+    runs-on: ubuntu-latest
+    needs: [build, checksum]
+    steps:
+    - name: Download math result from build and checksum jobs
+      uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.PRODUCT }}
+        path: ${{ env.PRODUCT }}
+
+    - name: Create Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+      with:
+        body: |
+          **Digests in this release:**
+
+          ```
+          ${{ needs.checksum.outputs.digest }}
+          ```
+        files: ${{ env.PRODUCT }}/*${{ env.PRODUCT }}*
+        draft: false
+        prerelease: true

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Stale
+
+on:
+  schedule:
+    - cron: "0 3 * * 6"
+
+jobs:
+  stale:
+    name: Stale
+    runs-on: ubuntu-latest
+    steps:
+      - name: Mark stale issues and pull requests
+        uses: actions/stale@v3
+        with:
+          repo-token: ${{ github.token }}
+          stale-issue-message: "This issue is stale because it has been open 120 days with no activity. Remove stale label or comment or this will be closed in 5 days"
+          stale-pr-message: 'It has been open 120 days with no activity. Remove stale label or comment or this will be closed in 5 days'
+          days-before-stale: 120
+          days-before-close: 5

.github/workflows/testing.yml

@@ -0,0 +1,70 @@
+name: Testing
+
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - "**/*.go"
+      - "go.mod"
+      - "go.sum"
+      - ".github/workflows/testing.yml"
+  pull_request:
+    branches: [ main ]
+    types: [ opened, synchronize, reopened ]
+    paths:
+      - "**/*.go"
+      - "go.mod"
+      - "go.sum"
+      - ".github/workflows/testing.yml"
+
+jobs:
+  test:
+    name: Testing
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        go: [ "1.12", "1.13", "1.14", "1.15", "1.16" ]
+
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go }}
+
+    - name: Set up Chocolatey
+      if: matrix.os == 'windows-latest'
+      uses: crazy-max/ghaction-chocolatey@v1
+      with:
+        args: -h
+
+    - name: Install Wget
+      if: matrix.os == 'windows-latest'
+      run: |
+        choco install wget
+        wget --help
+
+    - name: Check out code base
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Run test
+      run: |
+        make test
+        make test-cover
+
+    - name: Upload coverage
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage-${{ matrix.os }}-go${{ matrix.go }}
+        path: coverage.*
+
+    - name: Run integration test
+      run: make test-integration

.gitignore

@@ -0,0 +1,2 @@
+/build
+**.warc.gz

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Wayback Archiver
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,84 @@
+export GO111MODULE = on
+export GOPROXY = https://proxy.golang.org
+
+NAME = warcraft
+BINDIR ?= ./build/binary
+PACKDIR ?= ./build/package
+GOBUILD := CGO_ENABLED=0 go build --ldflags="-s -w" -v
+GOFILES := $(wildcard ./cmd/warcraft/*.go)
+VERSION := $(shell git describe --tags `git rev-list --tags --max-count=1`)
+VERSION := $(VERSION:v%=%)
+PROJECT := github.com/wabarc/warcraft
+PACKAGES := $(shell go list ./...)
+
+PLATFORM_LIST = \
+	darwin-amd64 \
+	darwin-arm64 \
+	linux-amd64
+
+WINDOWS_ARCH_LIST = \
+	windows-amd64
+
+.PHONY: all
+all: linux-amd64 darwin-amd64 windows-amd64
+
+darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ $(GOFILES)
+
+darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ $(GOFILES)
+
+linux-amd64:
+	GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ $(GOFILES)
+
+windows-amd64:
+	GOOS=windows GOARCH=amd64 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe $(GOFILES)
+
+fmt:
+	@echo "-> Running go fmt"
+	@go fmt $(PACKAGES)
+
+test:
+	@echo "-> Running go test"
+	@CGO_ENABLED=1 go test -v -race -cover -coverprofile=coverage.out -covermode=atomic ./...
+
+test-integration:
+	@echo 'mode: atomic' > coverage.out
+	@go list ./... | xargs -n1 -I{} sh -c 'CGO_ENABLED=1 go test -race -tags=integration -covermode=atomic -coverprofile=coverage.tmp -coverpkg $(go list ./... | tr "\n" ",") {} && tail -n +2 coverage.tmp >> coverage.out || exit 255'
+	@rm coverage.tmp
+
+test-cover:
+	@echo "-> Running go tool cover"
+	@go tool cover -func=coverage.out
+	@go tool cover -html=coverage.out -o coverage.html
+
+bench:
+	@echo "-> Running benchmark"
+	@go test -v -bench .
+
+profile:
+	@echo "-> Running profile"
+	@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
+
+tar_releases := $(addsuffix .gz, $(PLATFORM_LIST))
+zip_releases := $(addsuffix .zip, $(WINDOWS_ARCH_LIST))
+
+$(tar_releases): %.gz : %
+	@mkdir -p $(PACKDIR)
+	chmod +x $(BINDIR)/$(NAME)-$(basename $@)
+	tar -czf $(PACKDIR)/$(NAME)-$(basename $@)-$(VERSION).tar.gz --transform "s/$(notdir $(BINDIR))//g" $(BINDIR)/$(NAME)-$(basename $@)
+
+$(zip_releases): %.zip : %
+	@mkdir -p $(PACKDIR)
+	zip -m -j $(PACKDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe
+
+all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST)
+
+releases: $(tar_releases) $(zip_releases)
+
+clean:
+	rm -f $(PACKDIR)/*
+	rm -f *.warc.gz
+
+tag:
+	git tag v$(VERSION)

README.md

@@ -0,0 +1,53 @@
+# warcraft
+
+`warcraft` is a toolkit to help download webpage as `warc` file using wget.
+
+## Installation
+
+The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/wabarc/warcraft/releases) and place the executable file in your PATH.
+
+Via Golang package get command
+
+```sh
+go get -u github.com/wabarc/warcraft/cmd/warcraft
+```
+
+From [gobinaries.com](https://gobinaries.com):
+
+```sh
+$ curl -sf https://gobinaries.com/wabarc/warcraft | sh
+```
+
+## Usage
+
+Command-line:
+
+```sh
+$ warcraft
+A CLI tool help download webpage as warc file using wget.
+
+Usage:
+
+  warcraft [options] [url1] ... [urlN]
+```
+
+Go package:
+```go
+import (
+        "fmt"
+
+        "github.com/wabarc/warcraft"
+)
+
+func main() {
+        if b, err := warcraft.NewWarcraft(nil).Download(url); err != nil {
+            fmt.Fprintf(os.Stderr, "warcraft: %v\n", err)
+        } else {
+            fmt.Fprintf(os.Stdout, "%s  %s\n", url, string(b))
+        }
+}
+```
+
+## License
+
+This software is released under the terms of the MIT. See the [LICENSE](https://github.com/wabarc/warcraft/blob/main/LICENSE) file for details.