Merge pull request #221 from SiaFoundation/nate/siad-migrate

utility

.github/workflows/publish_migrate.yml

@@ -0,0 +1,179 @@
+name: Publish - Migration Utility
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on new SemVer tags
+  push:
+    branches:
+      - master
+    tags: 
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - name: Test
+        uses: ./.github/actions/test
+  build-linux:
+    runs-on: ubuntu-latest
+    needs: [ test ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - name: Setup
+        run: |
+          sudo apt update
+          sudo apt install -y gcc-aarch64-linux-gnu
+          go generate ./...
+      - name: Build amd64
+        env:
+          CGO_ENABLED: 1
+          GOOS: linux
+          GOARCH: amd64
+        run: |
+          mkdir -p release
+          ZIP_OUTPUT=release/migrate_${GOOS}_${GOARCH}.zip
+          go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/migrate
+          cp README.md LICENSE bin/
+          zip -qj $ZIP_OUTPUT bin/*
+      - name: Build arm64
+        env:
+          CGO_ENABLED: 1
+          GOOS: linux
+          GOARCH: arm64
+          CC: aarch64-linux-gnu-gcc
+        run: |
+          mkdir -p release
+          ZIP_OUTPUT=release/migrate_${GOOS}_${GOARCH}.zip
+          go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/migrate
+          cp README.md LICENSE bin/
+          zip -qj $ZIP_OUTPUT bin/*
+      - uses: actions/upload-artifact@v3
+        with:
+          name: migrate
+          path: release/
+  build-mac: 
+    runs-on: macos-latest
+    needs: [ test ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - name: Setup
+        env:
+          APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }}
+          APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }}
+          APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
+          APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }}
+          APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }}
+          APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }}
+          APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
+        run: |
+          # extract apple cert
+          APPLE_CERT_PATH=$RUNNER_TEMP/apple_cert.p12
+          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
+          echo -n "$APPLE_CERT_B64" | base64 --decode --output $APPLE_CERT_PATH
+
+          # extract apple key
+          mkdir -p ~/private_keys
+          APPLE_API_KEY_PATH=~/private_keys/AuthKey_$APPLE_API_KEY.p8
+          echo -n "$APPLE_KEY_B64" | base64 --decode --output $APPLE_API_KEY_PATH
+
+          # create temp keychain
+          security create-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+          security default-keychain -s $KEYCHAIN_PATH
+          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
+          security unlock-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+
+          # import keychain
+          security import $APPLE_CERT_PATH -P $APPLE_CERT_PASSWORD -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
+          security find-identity -v $KEYCHAIN_PATH -p codesigning
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $APPLE_KEYCHAIN_PASSWORD $KEYCHAIN_PATH
+
+          # generate
+          go generate ./...
+      - name: Build amd64
+        env:
+          APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }}
+          APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }}
+          APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
+          APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }}
+          APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }}
+          APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }}
+          APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
+          CGO_ENABLED: 1
+          GOOS: darwin
+          GOARCH: amd64
+        run: |
+          ZIP_OUTPUT=release/migrate_${GOOS}_${GOARCH}.zip
+          mkdir -p release
+          go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/migrate
+          cp README.md LICENSE bin/
+          /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/migrate
+          ditto -ck bin $ZIP_OUTPUT
+          xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT
+      - name: Build arm64
+        env:
+          APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }}
+          APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }}
+          APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
+          APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }}
+          APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }}
+          APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }}
+          APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
+          CGO_ENABLED: 1
+          GOOS: darwin
+          GOARCH: arm64
+        run: |
+          ZIP_OUTPUT=release/migrate_${GOOS}_${GOARCH}.zip
+          mkdir -p release
+          go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/migrate
+          cp README.md LICENSE bin/
+          /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/migrate
+          ditto -ck bin $ZIP_OUTPUT
+          xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT
+      - uses: actions/upload-artifact@v3
+        with:
+          name: migrate
+          path: release/
+  build-windows: 
+    runs-on: windows-latest
+    needs: [ test ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - name: Setup
+        shell: bash
+        run: |
+          dotnet tool install --global AzureSignTool
+          go generate ./...
+      - name: Build amd64
+        env:
+          CGO_ENABLED: 1
+          GOOS: windows
+          GOARCH: amd64
+        shell: bash
+        run: |
+          mkdir -p release
+          ZIP_OUTPUT=release/migrate_${GOOS}_${GOARCH}.zip
+          go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/migrate
+          azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/migrate.exe
+          cp README.md LICENSE bin/
+          7z a $ZIP_OUTPUT ./bin/*
+      - uses: actions/upload-artifact@v3
+        with:
+          name: migrate
+          path: release/

cmd/hostd/main.go

@@ -223,6 +223,9 @@ func main() {
 	flag.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on")
 	// log
 	flag.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "log level (debug, info, warn, error)")
+	// migrate
+	var migrateDestructive bool
+	flag.BoolVar(&migrateDestructive, "migrate.destructive", migrateDestructive, "destructively migrates a siad host to hostd")
 	flag.Parse()
 
 	switch flag.Arg(0) {

cmd/migrate/main.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"go.sia.tech/hostd/internal/migrate"
+	"go.sia.tech/hostd/persist/sqlite"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+// read a line of input from stdin
+func readInput(context string) (string, error) {
+	fmt.Printf("%s: ", context)
+	input, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	fmt.Println()
+	return string(input), err
+}
+
+func verifyDirectory(dataPath string) error {
+	// check if the directory exists
+	if stat, err := os.Stat(dataPath); err != nil {
+		return fmt.Errorf("failed to stat directory: %w", err)
+	} else if !stat.IsDir() {
+		return fmt.Errorf("not a directory: %s", dataPath)
+	}
+
+	requiredPaths := []string{
+		filepath.Join(dataPath, "consensus", "consensus.db"),
+		filepath.Join(dataPath, "host", "host.json"),
+		filepath.Join(dataPath, "host", "host.db"),
+		filepath.Join(dataPath, "host", "contractmanager", "contractmanager.json"),
+	}
+
+	for _, path := range requiredPaths {
+		if _, err := os.Stat(path); err != nil {
+			return fmt.Errorf("failed to stat required file %q: %w", path, err)
+		}
+	}
+	return nil
+}
+
+func main() {
+	var logLevel string
+	flag.StringVar(&logLevel, "log.level", "info", "set the log level")
+	flag.Parse()
+
+	// configure console logging note: this is configured before anything else
+	// to have consistent logging. File logging will be added after the cli
+	// flags and config is parsed
+	consoleCfg := zap.NewProductionEncoderConfig()
+	consoleCfg.TimeKey = "" // prevent duplicate timestamps
+	consoleCfg.EncodeTime = zapcore.RFC3339TimeEncoder
+	consoleCfg.EncodeDuration = zapcore.StringDurationEncoder
+	consoleCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder
+	consoleCfg.StacktraceKey = ""
+	consoleCfg.CallerKey = ""
+	consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg)
+
+	// configure logging
+	var level zap.AtomicLevel
+	switch logLevel {
+	case "debug":
+		level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	case "info":
+		level = zap.NewAtomicLevelAt(zap.InfoLevel)
+	case "warn":
+		level = zap.NewAtomicLevelAt(zap.WarnLevel)
+	case "error":
+		level = zap.NewAtomicLevelAt(zap.ErrorLevel)
+	default:
+		panic("invalid log level")
+	}
+
+	// only log info messages to console unless stdout logging is enabled
+	consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), level)
+	log := zap.New(consoleCore, zap.AddCaller())
+	defer log.Sync()
+	// redirect stdlib log to zap
+	zap.RedirectStdLog(log)
+
+	fmt.Println("siad Host Migrator")
+	fmt.Println("This tool will migrate your old siad host to hostd")
+	fmt.Println("It will migrate any existing contracts, storage folders, and settings")
+	fmt.Println("It will not migrate your wallet, but you can sweep it to a new hostd seed after migration")
+	fmt.Println("")
+	fmt.Println("Please make sure your computer will not go to turn off during the migration")
+	readInput("Press enter to continue")
+
+	// read the siad directory
+	fmt.Println("Where is your siad data directory? Enter the path and press enter to continue.")
+	dir, err := readInput("siad data directory")
+	if err != nil {
+		log.Fatal("failed to read input", zap.Error(err))
+	}
+
+	// check to make sure everything is in order
+	fmt.Println("Checking directory structure...")
+	if err := verifyDirectory(dir); err != nil {
+		log.Fatal("failed to verify siad directory structure", zap.Error(err))
+	}
+
+	// ask if the migration should be destructive
+	var destructive bool
+	fmt.Println("Do you want to delete the old data files during migration?")
+	fmt.Println("Unless you have double the available disk space, it is required to delete the old data files to free up space for the new storage volumes")
+	fmt.Println("This will free up disk space, but you will no longer be able to use siad. If the migration fails, you may be left in a broken state.")
+
+	for {
+		answer, err := readInput("delete old data files? (yes/no)")
+		if err != nil {
+			log.Fatal("failed to read input", zap.Error(err))
+		}
+		switch answer {
+		case "yes":
+			destructive = true
+		case "no":
+			destructive = false
+		default:
+			fmt.Println("Please enter yes or no")
+			fmt.Println("")
+			continue
+		}
+		break
+	}
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGINT)
+	defer cancel()
+
+	db, err := sqlite.OpenDatabase(filepath.Join(dir, "hostd.db"), log.Named("sqlite3"))
+	if err != nil {
+		log.Fatal("failed to open database", zap.Error(err))
+	}
+	defer db.Close()
+
+	log.Info("starting siad migration", zap.String("siadDir", dir), zap.Bool("destructive", destructive))
+
+	start := time.Now()
+	if err := migrate.Siad(ctx, db, dir, destructive, log.Named("migrate")); err != nil {
+		log.Fatal("siad migration failed", zap.Error(err))
+	}
+
+	log.Info("siad migration complete!!!", zap.Duration("elapsed", time.Since(start)))
+	log.Info("please start hostd with the same data directory as siad to continue using your host")
+	log.Info("you will also need to generate a new 12-word wallet seed and send your balance from the old wallet to the new wallet")
+}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/cloudflare/cloudflare-go v0.75.0
 	github.com/hashicorp/golang-lru/v2 v2.0.5
 	github.com/mattn/go-sqlite3 v1.14.17
+	gitlab.com/NebulousLabs/bolt v1.4.4
 	gitlab.com/NebulousLabs/encoding v0.0.0-20200604091946-456c3dc907fe
 	go.sia.tech/core v0.1.12-0.20231011160830-b58e9e8ec3ce
 	go.sia.tech/jape v0.10.0
@@ -38,7 +39,6 @@ require (
 	github.com/klauspost/reedsolomon v1.11.8 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	gitlab.com/NebulousLabs/bolt v1.4.4 // indirect
 	gitlab.com/NebulousLabs/demotemutex v0.0.0-20151003192217-235395f71c40 // indirect
 	gitlab.com/NebulousLabs/entropy-mnemonics v0.0.0-20181018051301-7532f67e3500 // indirect
 	gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 // indirect