feat: include meta tag in api

Taskfile.yaml

@@ -85,7 +85,7 @@ tasks:
   test-all:
     desc: Run all tests.
     cmds:
-      - go test -timeout 10s -tags test ./...
+      - go test -v -timeout 10s -tags test github.com/bangumi/server/internal/tag/...
     env:
       TEST_MYSQL: "1"
       TEST_REDIS: "1"

canal/canal.go

@@ -59,7 +59,7 @@ func Main() error {
 
 		// driver and connector
 		fx.Provide(
-			driver.NewMysqlConnectionPool,
+			driver.NewMysqlSqlDB,
 			driver.NewRedisClient, logger.Copy, cache.NewRedisCache,
 			subject.NewMysqlRepo, search.New, session.NewMysqlRepo, session.New,
 			driver.NewS3,

cmd/archive/main.go

@@ -80,7 +80,7 @@ func start(out string) {
 	err := fx.New(
 		fx.NopLogger,
 		fx.Provide(
-			driver.NewMysqlConnectionPool, dal.NewDB,
+			driver.NewMysqlSqlDB, dal.NewGormDB,
 
 			config.NewAppConfig, logger.Copy,
 
@@ -210,7 +210,7 @@ type Subject struct {
 
 type Tag struct {
 	Name  string `json:"name"`
-	Count int    `json:"count"`
+	Count uint   `json:"count"`
 }
 
 func exportSubjects(q *query.Query, w io.Writer) {

cmd/gen/gorm/main.go

@@ -92,12 +92,12 @@ func main() {
 		panic("failed to read config: " + err.Error())
 	}
 
-	conn, err := driver.NewMysqlConnectionPool(c)
+	conn, err := driver.NewMysqlSqlDB(c)
 	if err != nil {
 		panic(err)
 	}
 
-	db, err := dal.NewDB(conn, c)
+	db, err := dal.NewGormDB(conn, c)
 	if err != nil {
 		panic(err)
 	}

cmd/web/cmd.go

@@ -41,6 +41,7 @@ import (
 	"github.com/bangumi/server/internal/revision"
 	"github.com/bangumi/server/internal/search"
 	"github.com/bangumi/server/internal/subject"
+	"github.com/bangumi/server/internal/tag"
 	"github.com/bangumi/server/internal/timeline"
 	"github.com/bangumi/server/internal/user"
 	"github.com/bangumi/server/web"
@@ -65,7 +66,8 @@ func start() error {
 		fx.Provide(
 			config.AppConfigReader(config.AppTypeHTTP),
 			driver.NewRedisClientWithMetrics, // redis
-			driver.NewMysqlConnectionPool,    // mysql
+			driver.NewMysqlSqlDB,             // mysql
+			driver.NewRueidisClient,
 			func() *resty.Client {
 				httpClient := resty.New().SetJSONEscapeHTML(false)
 				httpClient.JSONUnmarshal = json.Unmarshal
@@ -74,6 +76,8 @@ func start() error {
 			},
 		),
 
+		fx.Invoke(dal.SetupMetrics),
+
 		dal.Module,
 
 		fx.Provide(
@@ -87,6 +91,8 @@ func start() error {
 
 			dam.New, subject.NewMysqlRepo, subject.NewCachedRepo, person.NewMysqlRepo,
 
+			tag.NewCachedRepo, tag.NewMysqlRepo,
+
 			auth.NewService, person.NewService, search.New,
 		),
 

dal/fx.go

@@ -27,7 +27,7 @@ import (
 
 var Module = fx.Module("dal",
 	fx.Provide(
-		NewDB,
+		NewGormDB,
 		query.Use,
 		NewMysqlTransaction,
 		func(db *sql.DB) *sqlx.DB {

dal/metrics.go

@@ -22,7 +22,7 @@ import (
 	"gorm.io/gorm"
 )
 
-func setupMetrics(db *gorm.DB, conn *sql.DB) error {
+func SetupMetrics(db *gorm.DB, conn *sql.DB) error {
 	var DatabaseQuery = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "chii_db_execute_total",

dal/new.go

@@ -27,7 +27,7 @@ import (
 	"github.com/bangumi/server/internal/pkg/logger"
 )
 
-func NewDB(conn *sql.DB, c config.AppConfig) (*gorm.DB, error) {
+func NewGormDB(conn *sql.DB, c config.AppConfig) (*gorm.DB, error) {
 	var gLog gormLogger.Interface
 	if c.Debug.Gorm {
 		logger.Info("enable gorm debug mode, will log all sql")
@@ -51,9 +51,5 @@ func NewDB(conn *sql.DB, c config.AppConfig) (*gorm.DB, error) {
 		return nil, errgo.Wrap(err, "create dal")
 	}
 
-	if err = setupMetrics(db, conn); err != nil {
-		return nil, errgo.Wrap(err, "setup metrics")
-	}
-
 	return db, nil
 }

dal/new_test.go

@@ -31,9 +31,9 @@ func TestNewDB(t *testing.T) {
 	cfg, err := config.NewAppConfig()
 	require.NoError(t, err)
 
-	conn, err := driver.NewMysqlConnectionPool(cfg)
+	conn, err := driver.NewMysqlSqlDB(cfg)
 	require.NoError(t, err)
-	db, err := dal.NewDB(conn, cfg)
+	db, err := dal.NewGormDB(conn, cfg)
 	require.NoError(t, err)
 
 	err = db.Exec("select 0;").Error

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/prometheus/client_golang v1.20.4
 	github.com/redis/go-redis/v9 v9.6.1
+	github.com/redis/rueidis v1.0.47
 	github.com/samber/lo v1.47.0
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -338,8 +338,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
-github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
+github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
+github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -400,6 +400,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
 github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
 github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
 github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/redis/rueidis v1.0.47 h1:41UdeXOo4eJuW+cfpUJuLtVGyO0QJY3A2rEYgJWlfHs=
+github.com/redis/rueidis v1.0.47/go.mod h1:by+34b0cFXndxtYmPAHpoTHO5NkosDlBvhexoTURIxM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -513,6 +515,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

internal/cachekey/cachekey.go

@@ -54,3 +54,7 @@ func Index(id model.IndexID) string {
 func User(id model.UserID) string {
 	return resPrefix + "user:" + strconv.FormatUint(uint64(id), 10)
 }
+
+func SubjectMetaTag(id model.SubjectID) string {
+	return resPrefix + "subject:meta-tags:" + strconv.FormatUint(uint64(id), 10)
+}

internal/collections/infra/mysql_repo.go

@@ -333,16 +333,16 @@ func (r mysqlRepo) reCountSubjectTags(ctx context.Context, tx *query.Query,
 		return errgo.Trace(err)
 	}
 
-	var count = make(map[string]int)
-	var countMap = make(map[string]uint32)
+	var count = make(map[string]uint)
+	var countMap = make(map[string]uint)
 
 	for _, tag := range tagList {
 		if !dam.ValidateTag(tag.Tag.Name) {
 			continue
 		}
 
 		count[tag.Tag.Name]++
-		countMap[tag.Tag.Name] = tag.Tag.Results
+		countMap[tag.Tag.Name] = uint(tag.Tag.Results)
 	}
 
 	var phpTags = make([]subject.Tag, 0, len(count))
@@ -351,7 +351,7 @@ func (r mysqlRepo) reCountSubjectTags(ctx context.Context, tx *query.Query,
 		phpTags = append(phpTags, subject.Tag{
 			Name:       lo.ToPtr(name),
 			Count:      c,
-			TotalCount: int(countMap[name]),
+			TotalCount: uint(countMap[name]),
 		})
 	}
 

internal/model/subject.go

@@ -17,8 +17,9 @@ package model
 const subjectLocked = 2
 
 type Tag struct {
-	Name  string
-	Count int
+	Name       string
+	Count      uint
+	TotalCount uint
 }
 
 type Subject struct {

internal/pkg/cache/noop.go

@@ -17,6 +17,8 @@ package cache
 import (
 	"context"
 	"time"
+
+	"github.com/redis/rueidis"
 )
 
 func NewNoop() RedisCache {
@@ -36,3 +38,7 @@ func (n noop) Set(context.Context, string, any, time.Duration) error {
 func (n noop) Del(context.Context, ...string) error {
 	return nil
 }
+
+func (n noop) mget(ctx context.Context, key []string) rueidis.RedisResult {
+	return rueidis.RedisResult{}
+}

internal/pkg/cache/redis.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/redis/go-redis/v9"
+	"github.com/redis/rueidis"
 	"github.com/trim21/errgo"
 	"go.uber.org/zap"
 
@@ -34,15 +35,18 @@ type RedisCache interface {
 	Get(ctx context.Context, key string, value any) (bool, error)
 	Set(ctx context.Context, key string, value any, ttl time.Duration) error
 	Del(ctx context.Context, keys ...string) error
+
+	mget(ctx context.Context, key []string) rueidis.RedisResult
 }
 
 // NewRedisCache create a redis backed cache.
-func NewRedisCache(cli *redis.Client) RedisCache {
-	return redisCache{r: cli}
+func NewRedisCache(cli *redis.Client, ru rueidis.Client) RedisCache {
+	return redisCache{r: cli, ru: ru}
 }
 
 type redisCache struct {
-	r *redis.Client
+	r  *redis.Client
+	ru rueidis.Client
 }
 
 func (c redisCache) Get(ctx context.Context, key string, value any) (bool, error) {
@@ -66,6 +70,14 @@ func (c redisCache) Get(ctx context.Context, key string, value any) (bool, error
 	return true, nil
 }
 
+func (c redisCache) mget(ctx context.Context, keys []string) rueidis.RedisResult {
+	return c.ru.Do(ctx, c.ru.B().Mget().Key(keys...).Build())
+}
+
+func MGet[T any](c RedisCache, ctx context.Context, keys []string, value *[]T) error {
+	return rueidis.DecodeSliceOfJSON(c.mget(ctx, keys), value)
+}
+
 func (c redisCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
 	b, err := marshalBytes(value)
 	if err != nil {

internal/pkg/driver/mysql.go

@@ -31,7 +31,8 @@ import (
 
 var setLoggerOnce = sync.Once{}
 
-func NewMysqlConnectionPool(c config.AppConfig) (*sql.DB, error) {
+//nolint:stylecheck
+func NewMysqlSqlDB(c config.AppConfig) (*sql.DB, error) {
 	setLoggerOnce.Do(func() {
 		_ = mysql.SetLogger(logger.StdAt(zap.ErrorLevel))
 	})

internal/pkg/driver/redis.go

@@ -16,9 +16,12 @@ package driver
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/redis/go-redis/v9"
+	"github.com/redis/rueidis"
 	"github.com/trim21/errgo"
 	"go.uber.org/zap"
 
@@ -63,3 +66,21 @@ func NewRedisClientWithMetrics(c config.AppConfig) (*redis.Client, error) {
 
 	return cli, nil
 }
+
+func NewRueidisClient(c config.AppConfig) (rueidis.Client, error) {
+	u, err := url.Parse(c.RedisURL)
+	if err != nil {
+		return nil, err
+	}
+
+	password, _ := u.User.Password()
+	cli, err := rueidis.NewClient(rueidis.ClientOption{
+		InitAddress: []string{fmt.Sprintf("%s:%s", u.Hostname(), u.Port())},
+		Password:    password,
+	})
+	if err != nil {
+		return cli, err
+	}
+
+	return cli, nil
+}

internal/pkg/test/fx.go

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, version 3.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+package test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/go-resty/resty/v2"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/fx"
+	"go.uber.org/zap"
+
+	"github.com/bangumi/server/config"
+	"github.com/bangumi/server/dal"
+	"github.com/bangumi/server/internal/pkg/cache"
+	"github.com/bangumi/server/internal/pkg/driver"
+)
+
+func Fx(t *testing.T, target ...fx.Option) {
+	t.Helper()
+	err := fx.New(
+		append(target, fx.NopLogger,
+
+			// driver and connector
+			fx.Provide(
+				config.AppConfigReader(config.AppTypeHTTP),
+				driver.NewRedisClient,   // redis
+				driver.NewRueidisClient, // redis
+				driver.NewMysqlSqlDB,    // mysql
+				func() *resty.Client {
+					httpClient := resty.New().SetJSONEscapeHTML(false)
+					httpClient.JSONUnmarshal = json.Unmarshal
+					httpClient.JSONMarshal = json.Marshal
+					return httpClient
+				},
+			),
+
+			dal.Module,
+
+			fx.Provide(cache.NewRedisCache, zap.NewNop),
+		)...,
+	).Err()
+
+	require.NoError(t, err)
+}

internal/pkg/test/gorm.go

@@ -56,7 +56,7 @@ func GetGorm(tb testing.TB) *gorm.DB {
 
 func newGorm(tb testing.TB, c config.AppConfig) (*gorm.DB, error) {
 	tb.Helper()
-	conn, err := driver.NewMysqlConnectionPool(c)
+	conn, err := driver.NewMysqlSqlDB(c)
 	if err != nil {
 		return nil, errgo.Wrap(err, "sql.Open")
 	}