Use jsoniter for all marshaling and unmarshaling

We run on phones so every bit of performance matters to the users and
clearly this library is much faster. This is easier than benchmarking
everything. At this point we've proven it is a good idea.

CHANGELOG.md

@@ -20,7 +20,8 @@
 
 ### Fixed 
 
-- ...
+- Improved performance by using "github.com/json-iterator/go" instead of
+  "encoding/json".
 
 ### Security 
 

cmd/log-debugger/debugger/event.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/cmd/log-debugger/debugger/log"
 	"github.com/planetary-social/scuttlego/logging"
 	"github.com/planetary-social/scuttlego/service/domain/messages"
@@ -143,7 +144,7 @@ func prettifyBody(body string) (string, error) {
 			annotatedNotes[note.Ref().String()] = fmt.Sprintf("%s (sequence=%d, receive=%t replicate=%t)", string(b), note.Sequence(), note.Receive(), note.Replicate())
 		}
 
-		j, err := json.Marshal(annotatedNotes)
+		j, err := jsoniter.Marshal(annotatedNotes)
 		if err != nil {
 			return "", errors.Wrap(err, "error marshaling annotated notes")
 		}

internal/fixtures/fixtures.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	cryptorand "crypto/rand"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"math"
 	"math/rand"
@@ -16,6 +15,7 @@ import (
 	"time"
 
 	"github.com/dgraph-io/badger/v3"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/internal"
 	"github.com/planetary-social/scuttlego/logging"
 	"github.com/planetary-social/scuttlego/service/app/common"
@@ -192,7 +192,7 @@ func SomeRawContent() message.RawContent {
 		"type":       SomeString(),
 		SomeString(): SomeString(),
 	}
-	j, err := json.Marshal(v)
+	j, err := jsoniter.Marshal(v)
 	if err != nil {
 		panic(err)
 	}

migrations/migrations.go

@@ -2,11 +2,11 @@ package migrations
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"github.com/boreq/errors"
 	"github.com/hashicorp/go-multierror"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/internal"
 	"github.com/planetary-social/scuttlego/logging"
 )
@@ -182,7 +182,7 @@ func (r Runner) runMigration(ctx context.Context, migration Migration, onRunning
 		return errors.Wrap(err, "error loading state")
 	}
 
-	humanReadableState, err := json.Marshal(state)
+	humanReadableState, err := jsoniter.Marshal(state)
 	if err != nil {
 		return errors.Wrap(err, "state json marshal error")
 	}

service/adapters/badger/ban_list_repository.go

@@ -1,11 +1,11 @@
 package badger
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/boreq/errors"
 	"github.com/dgraph-io/badger/v3"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/adapters/badger/utils"
 	"github.com/planetary-social/scuttlego/service/app/commands"
 	"github.com/planetary-social/scuttlego/service/domain/bans"
@@ -102,7 +102,7 @@ func (b BanListRepository) CreateFeedMapping(ref refs.Feed) error {
 
 	bucket := b.mappingBucket()
 
-	v, err := json.Marshal(storedBannableRef{
+	v, err := jsoniter.Marshal(storedBannableRef{
 		Typ: storedBannableRefTypeFeed,
 		Ref: ref.String(),
 	})
@@ -150,7 +150,7 @@ func (b BanListRepository) LookupMapping(hash bans.Hash) (commands.BannableRef,
 	}
 
 	var v storedBannableRef
-	if err := json.Unmarshal(value, &v); err != nil {
+	if err := jsoniter.Unmarshal(value, &v); err != nil {
 		return commands.BannableRef{}, errors.Wrap(err, "unmarshal failed")
 	}
 

service/adapters/migrations/badger_storage.go

@@ -1,10 +1,9 @@
 package migrations
 
 import (
-	"encoding/json"
-
 	"github.com/boreq/errors"
 	"github.com/dgraph-io/badger/v3"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/migrations"
 	"github.com/planetary-social/scuttlego/service/adapters/badger/utils"
 )
@@ -42,7 +41,7 @@ func (b *BadgerStorage) LoadState(name string) (migrations.State, error) {
 			return errors.Wrap(err, "error getting value")
 		}
 
-		if err := json.Unmarshal(value, &state); err != nil {
+		if err := jsoniter.Unmarshal(value, &state); err != nil {
 			return errors.Wrap(err, "error unmarshaling state")
 		}
 
@@ -58,7 +57,7 @@ func (b *BadgerStorage) LoadState(name string) (migrations.State, error) {
 }
 
 func (b *BadgerStorage) SaveState(name string, state migrations.State) error {
-	marshaledState, err := json.Marshal(state)
+	marshaledState, err := jsoniter.Marshal(state)
 	if err != nil {
 		return errors.Wrap(err, "error marshaling state")
 	}

service/domain/feeds/content/transport/mapping_contact.go

@@ -1,10 +1,10 @@
 package transport
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/internal"
 	"github.com/planetary-social/scuttlego/service/domain/feeds/content/known"
 	"github.com/planetary-social/scuttlego/service/domain/refs"
@@ -24,12 +24,12 @@ var ContactMapping = MessageContentMapping{
 			return nil, errors.Wrap(err, "could not marshal contact action")
 		}
 
-		return json.Marshal(t)
+		return jsoniter.Marshal(t)
 	},
 	Unmarshal: func(b []byte) (known.KnownMessageContent, error) {
 		var t transportContact
 
-		if err := json.Unmarshal(b, &t); err != nil {
+		if err := jsoniter.Unmarshal(b, &t); err != nil {
 			return nil, errors.Wrap(err, "json unmarshal failed")
 		}
 

service/domain/feeds/content/transport/mapping_pub.go

@@ -1,9 +1,8 @@
 package transport
 
 import (
-	"encoding/json"
-
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/domain/feeds/content/known"
 	"github.com/planetary-social/scuttlego/service/domain/refs"
 )
@@ -21,12 +20,12 @@ var PubMapping = MessageContentMapping{
 			},
 		}
 
-		return json.Marshal(t)
+		return jsoniter.Marshal(t)
 	},
 	Unmarshal: func(b []byte) (known.KnownMessageContent, error) {
 		var t transportPub
 
-		if err := json.Unmarshal(b, &t); err != nil {
+		if err := jsoniter.Unmarshal(b, &t); err != nil {
 			return nil, errors.Wrap(err, "json unmarshal failed")
 		}
 

service/domain/feeds/content/transport/marshaler.go

@@ -1,11 +1,11 @@
 package transport
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/logging"
 	"github.com/planetary-social/scuttlego/service/domain/feeds/content"
 	"github.com/planetary-social/scuttlego/service/domain/feeds/content/known"
@@ -74,7 +74,7 @@ func (m *Marshaler) Unmarshal(b message.RawContent) (known.KnownMessageContent,
 
 func (m *Marshaler) identifyContentType(b message.RawContent) (known.MessageContentType, error) {
 	var typ MessageContentType
-	if err := json.Unmarshal(b.Bytes(), &typ); err != nil {
+	if err := jsoniter.Unmarshal(b.Bytes(), &typ); err != nil {
 		return "", errors.Wrap(err, "json unmarshal of message content type failed")
 	}
 	if typ.MessageContentType == "" {

service/domain/feeds/formats/format_scuttlebutt.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/domain/feeds"
 	"github.com/planetary-social/scuttlego/service/domain/feeds/message"
 	"github.com/planetary-social/scuttlego/service/domain/identity"
@@ -91,7 +92,7 @@ func (s *Scuttlebutt) Load(verifiedRawMessage message.VerifiedRawMessage) (messa
 	}
 
 	var dmsg legacy.DeserializedMessage
-	if err := json.Unmarshal(verifiedRawMessage.Bytes(), &dmsg); err != nil {
+	if err := jsoniter.Unmarshal(verifiedRawMessage.Bytes(), &dmsg); err != nil {
 		return message.MessageWithoutId{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -100,7 +101,7 @@ func (s *Scuttlebutt) Load(verifiedRawMessage message.VerifiedRawMessage) (messa
 
 func (s *Scuttlebutt) Peek(raw message.RawMessage) (feeds.PeekedMessage, error) {
 	var dmsg peekedSsbMessage
-	if err := json.Unmarshal(raw.Bytes(), &dmsg); err != nil {
+	if err := jsoniter.Unmarshal(raw.Bytes(), &dmsg); err != nil {
 		return feeds.PeekedMessage{}, errors.Wrap(err, "json unmarshal failed")
 	}
 

service/domain/messages/blobs_createWants.go

@@ -1,11 +1,11 @@
 package messages
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/domain/blobs"
 	"github.com/planetary-social/scuttlego/service/domain/refs"
 	"github.com/planetary-social/scuttlego/service/domain/transport/rpc"
@@ -33,7 +33,7 @@ type BlobsCreateWantsResponse struct {
 func NewBlobsCreateWantsResponseFromBytes(b []byte) (BlobsCreateWantsResponse, error) {
 	m := make(blobsCreateWantsResponseTransport)
 
-	if err := json.Unmarshal(b, &m); err != nil {
+	if err := jsoniter.Unmarshal(b, &m); err != nil {
 		return BlobsCreateWantsResponse{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -99,7 +99,7 @@ func (resp BlobsCreateWantsResponse) MarshalJSON() ([]byte, error) {
 		panic("not all cases are covered")
 	}
 
-	return json.Marshal(m)
+	return jsoniter.Marshal(m)
 }
 
 type blobsCreateWantsResponseTransport map[string]int64

service/domain/messages/blobs_get.go

@@ -1,10 +1,9 @@
 package messages
 
 import (
-	"encoding/json"
-
 	"github.com/boreq/errors"
 	"github.com/hashicorp/go-multierror"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/domain/blobs"
 	"github.com/planetary-social/scuttlego/service/domain/refs"
 	"github.com/planetary-social/scuttlego/service/domain/transport/rpc"
@@ -76,7 +75,7 @@ func NewBlobsGetArgumentsFromBytes(b []byte) (BlobsGetArguments, error) {
 func newBlobsGetArgumentsFromBytesString(b []byte) (BlobsGetArguments, error) {
 	var args []string
 
-	if err := json.Unmarshal(b, &args); err != nil {
+	if err := jsoniter.Unmarshal(b, &args); err != nil {
 		return BlobsGetArguments{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -95,7 +94,7 @@ func newBlobsGetArgumentsFromBytesString(b []byte) (BlobsGetArguments, error) {
 func newBlobsGetArgumentsFromBytesObject(b []byte) (BlobsGetArguments, error) {
 	var args []blobsGetArgumentsTransport
 
-	if err := json.Unmarshal(b, &args); err != nil {
+	if err := jsoniter.Unmarshal(b, &args); err != nil {
 		return BlobsGetArguments{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -149,7 +148,7 @@ func (a BlobsGetArguments) MarshalJSON() ([]byte, error) {
 			a.hash.String(),
 		}
 
-		return json.Marshal(args)
+		return jsoniter.Marshal(args)
 	}
 
 	args := []blobsGetArgumentsTransport{
@@ -168,7 +167,7 @@ func (a BlobsGetArguments) MarshalJSON() ([]byte, error) {
 		args[0].Max = &v
 	}
 
-	return json.Marshal(args)
+	return jsoniter.Marshal(args)
 }
 
 func (a BlobsGetArguments) Hash() refs.Blob {

service/domain/messages/blobs_get_test.go

@@ -136,7 +136,7 @@ func TestNewBlobsGetArgumentsFromBytesObject(t *testing.T) {
 		{
 			Name:          "different_key_and_hash",
 			Payload:       `[{"hash":"&2b3zi3R00MZ6X+9jXgZMCS6/N1W1PGM2leOEKvpKQjA=.sha256","key":"&eb3zi3R00MZ6X+9jXgZMCS6/N1W1PGM2leOEKvpKQjA=.sha256","max":5242880}]`,
-			ExpectedError: errors.New("2 errors occurred:\n\t* error unmarshaling arguments as string: json unmarshal failed: json: cannot unmarshal object into Go value of type string\n\t* error unmarshaling arguments as object: could not create a blob ref: key and hash are set but have different values\n\n"),
+			ExpectedError: errors.New("2 errors occurred:\n\t* error unmarshaling arguments as string: json unmarshal failed: []string: ReadString: expects \" or n, but found {, error found in #2 byte of ...|[{\"hash\":\"&2|..., bigger context ...|[{\"hash\":\"&2b3zi3R00MZ6X+9jXgZMCS6/N1W1PGM2leOEKvpKQ|...\n\t* error unmarshaling arguments as object: could not create a blob ref: key and hash are set but have different values\n\n"),
 		},
 	}
 

service/domain/messages/create_history_stream.go

@@ -177,7 +177,7 @@ func (c CreateHistoryStreamResponse) MarshalJSON() ([]byte, error) {
 		Value:     c.value.Bytes(),
 		Timestamp: c.timestamp.UnixMilli(),
 	}
-	return json.Marshal(transport)
+	return jsoniter.Marshal(transport)
 }
 
 func sequencePointerToIntPointer(sequence *message.Sequence) *int {

service/domain/messages/ebt_replicate.go

@@ -1,7 +1,6 @@
 package messages
 
 import (
-	"encoding/json"
 	"strconv"
 
 	"github.com/boreq/errors"
@@ -52,7 +51,7 @@ func NewEbtReplicateArguments(
 func NewEbtReplicateArgumentsFromBytes(b []byte) (EbtReplicateArguments, error) {
 	var args []ebtReplicateArgumentsTransport
 
-	if err := json.Unmarshal(b, &args); err != nil {
+	if err := jsoniter.Unmarshal(b, &args); err != nil {
 		return EbtReplicateArguments{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -92,7 +91,7 @@ func (c EbtReplicateArguments) MarshalJSON() ([]byte, error) {
 			Format:  f,
 		},
 	}
-	return json.Marshal(transport)
+	return jsoniter.Marshal(transport)
 }
 
 const ebtReplicateFormatTransportClassic = "classic"

service/domain/messages/invite_use.go

@@ -1,9 +1,8 @@
 package messages
 
 import (
-	"encoding/json"
-
 	"github.com/boreq/errors"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/planetary-social/scuttlego/service/domain/refs"
 	"github.com/planetary-social/scuttlego/service/domain/transport/rpc"
 )
@@ -42,7 +41,7 @@ func NewInviteUseArguments(feed refs.Identity) (InviteUseArguments, error) {
 func NewInviteUseArgumentsFromBytes(b []byte) (InviteUseArguments, error) {
 	var args []inviteUseArgumentsTransport
 
-	if err := json.Unmarshal(b, &args); err != nil {
+	if err := jsoniter.Unmarshal(b, &args); err != nil {
 		return InviteUseArguments{}, errors.Wrap(err, "json unmarshal failed")
 	}
 
@@ -59,7 +58,7 @@ func NewInviteUseArgumentsFromBytes(b []byte) (InviteUseArguments, error) {
 }
 
 func (i InviteUseArguments) MarshalJSON() ([]byte, error) {
-	return json.Marshal([]inviteUseArgumentsTransport{
+	return jsoniter.Marshal([]inviteUseArgumentsTransport{
 		{
 			Feed: i.feed.String(),
 		},