refactor: 重构项目结构并优化路由组织

2025-10-28 12:20:16 +08:00
parent ea9ecb770d
commit 819a2eb8ec
7 changed files with 50 additions and 33 deletions
--- a/internal/controller/dataController.go
+++ b/internal/controller/dataController.go
@@ -0,0 +1,128 @@
+package controller
+
+import (
+	"dypid/global"
+	"dypid/internal/db"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
+)
+
+func ReadDataHandler(c *gin.Context) {
+	//解析输入数据
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+
+	retData := global.RDB.LPop(global.RCtx, fmt.Sprintf("list:%s", input.Token)).Val()
+	if retData == "null" {
+		c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
+		return
+	}
+
+	c.String(http.StatusOK, retData)
+}
+
+func WriteDataHandler(c *gin.Context) {
+	//解析输入数据
+	input := struct {
+		Token string `form:"token" binding:"required"`
+		Data  string `form:"data" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+
+	//数据获取
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	dataIndex, err := getDataIndex(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
+
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
+	if err != nil && err.Error() != "ERR item exists" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	luaScript := `
+local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
+local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
+local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
+local dedupValue = ARGV[1] -- ARGV[1]: 去重值
+local rawData = ARGV[2]    -- ARGV[2]: 原始数据
+
+-- 检查布隆过滤器中是否已存在该值
+local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
+
+-- 如果已存在，返回已去重标记
+if exists == 1 then
+    return "已去重"
+end
+
+-- 添加到布隆过滤器
+redis.call('CF.ADD', dedupKey, dedupValue)
+-- 添加到列表
+redis.call('LPUSH', listKey, rawData)
+
+redis.call('LPUSH', deleteListKey, dedupValue)
+
+-- 返回成功结果
+return "ok"
+`
+	result, err := redis.NewScript(luaScript).Run(
+		global.RCtx,
+		global.RDB,
+		[]string{
+			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
+			fmt.Sprintf("list:%s", input.Token),
+			fmt.Sprintf("delete-list:%s", input.Token),
+		},
+		dedupValue,
+		input.Data,
+	).Result()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()})
+		return
+	}
+
+	if resultMap, ok := result.(string); ok {
+		c.JSON(http.StatusOK, gin.H{"result": resultMap})
+		return
+	}
+
+	c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
+}
+
+func createCF(bloomFilter string, capacity int64) error {
+	_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
+	return err
+}
+
+func getDataIndex(token string) (index map[string]int, err error) {
+	dataFormat, err := db.GetDataFormat(token)
+	if err != nil {
+		return nil, err
+	}
+
+	f := strings.Split(dataFormat, "----")
+	index = make(map[string]int)
+	for i, s := range f {
+		index[s] = i
+	}
+	return index, nil
+}
--- a/internal/controller/tokenController.go
+++ b/internal/controller/tokenController.go
@@ -0,0 +1,235 @@
+package controller
+
+import (
+	"dypid/global"
+	"dypid/internal/db"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
+)
+
+func ListTokenHandler(c *gin.Context) {
+	c.JSON(http.StatusOK, gin.H{"result": db.ListToken()})
+}
+
+func CreateTokenHandler(c *gin.Context) {
+	//解析输入数据
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+		Notes       string `form:"notes"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+	//检查Token是否存在
+	if db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "创建Token失败，Token已经存在"})
+		return
+	}
+
+	//创建Token
+	err := db.CreateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	//返回
+	c.JSON(http.StatusOK, gin.H{"result": "ok"})
+}
+
+func UpdateTokenHandler(c *gin.Context) {
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+		Notes       string `form:"notes"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "更改失败，Token不存在"})
+		return
+	}
+
+	err := db.UpdateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"result": "ok"})
+}
+
+func DeleteTokenHandler(c *gin.Context) {
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
+		return
+	}
+
+	err := db.DeleteToken(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"result": "ok"})
+}
+
+func GetTokenInfoHandler(c *gin.Context) {
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
+		return
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "获取信息失败，Token不存在"})
+		return
+	}
+
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取去重对象失败 " + err.Error()})
+		return
+	}
+	dataFormat, err := db.GetDataFormat(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取数据格式失败 " + err.Error()})
+		return
+	}
+
+	output := struct {
+		Token            string `json:"token"`
+		DedupObject      string `json:"dedup_object"`
+		DataFormat       string `json:"data_format"`
+		DedupItemsNumber int64  `json:"dedup_items_number"`
+		CacheListNumber  int64  `json:"cache_list_number"`
+	}{}
+	output.Token = input.Token
+	output.DedupObject = dedupObject
+	output.DataFormat = dataFormat
+	output.DedupItemsNumber = global.RDB.CFInfo(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val().NumItemsInserted
+	output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val()
+
+	c.JSON(http.StatusOK, gin.H{"result": output})
+}
+
+func DeleteTokenInfoHandler(c *gin.Context) {
+	//解析输入数据
+	input := struct {
+		Token      string `form:"token" binding:"required"`
+		DedupBF    string `form:"dedup_bf"`
+		CacheList  string `form:"cache_list"`
+		BothNumber string `form:"both_number"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
+		return
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
+		return
+	}
+
+	//检查token是否存在
+	_, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Token不存在" + err.Error()})
+		return
+	}
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	//删除去重对象
+	switch input.DedupBF {
+	case "":
+	case "all":
+		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
+		global.RDB.Del(global.RCtx, keys...)
+		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.DedupBF)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
+			return
+		}
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除原始数据
+	switch input.CacheList {
+	case "":
+	case "all":
+		global.RDB.Del(global.RCtx, "list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.CacheList)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
+			return
+		}
+
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除去重参考值和原始数据
+	switch input.BothNumber {
+	case "":
+	default:
+		i, err := strconv.Atoi(input.BothNumber)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
+			return
+		}
+
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+			pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+	}
+
+	//输出信息
+	c.JSON(http.StatusOK, gin.H{"result": "ok"})
+}
--- a/internal/db/local.go
+++ b/internal/db/local.go
@@ -0,0 +1,132 @@
+package db
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+)
+
+var localDB []Token
+
+type Token struct {
+	Token       string `json:"token"`
+	DedupObject string `json:"dedup_object"`
+	DataFormat  string `json:"data_format"`
+	Notes       string `json:"notes"`
+}
+
+func InitLocalDB() {
+start:
+	data, err := os.ReadFile("./token.json")
+	if os.IsNotExist(err) {
+		os.WriteFile("./token.json", []byte("[]"), 0644)
+		goto start
+	} else if err != nil {
+		fmt.Println(err)
+	}
+
+	err = json.Unmarshal(data, &localDB)
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func CheckToken(token string) bool {
+	InitLocalDB()
+	for _, t := range localDB {
+		if t.Token == token {
+			return true
+		}
+	}
+	return false
+}
+
+func ListToken() []Token {
+	InitLocalDB()
+	return localDB
+}
+
+func CreateToken(token string, dedupObject string, dataFormat string, Notes string) error {
+	InitLocalDB()
+	for _, t := range localDB {
+		if t.Token == token {
+			return errors.New("token已经存在")
+		}
+	}
+
+	a := append(localDB, Token{token, dedupObject, dataFormat, Notes})
+	marshal, err := json.Marshal(a)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile("./token.json", marshal, 0644); err != nil {
+		return err
+	}
+	InitLocalDB()
+	return nil
+}
+
+func UpdateToken(token string, dedupObject string, dataFormat string, Notes string) error {
+	InitLocalDB()
+
+	for i, t := range localDB {
+		if t.Token == token {
+			localDB[i].DedupObject = dedupObject
+			localDB[i].DataFormat = dataFormat
+			localDB[i].Notes = Notes
+
+			marshal, err := json.Marshal(localDB)
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile("./token.json", marshal, 0644); err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+
+	InitLocalDB()
+	return errors.New("token not found")
+}
+
+func DeleteToken(token string) error {
+	InitLocalDB()
+
+	for i, t := range localDB {
+		if t.Token == token {
+			localDB = append(localDB[:i], localDB[i+1:]...)
+
+			marshal, err := json.Marshal(localDB)
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile("./token.json", marshal, 0644); err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+
+	InitLocalDB()
+	return errors.New("token not found")
+}
+
+func GetDedupObject(token string) (dedupObject string, err error) {
+	for i, t := range localDB {
+		if t.Token == token {
+			return localDB[i].DedupObject, nil
+		}
+	}
+	return "", errors.New("未找到Token")
+}
+
+func GetDataFormat(token string) (dataFormat string, err error) {
+	for i, t := range localDB {
+		if t.Token == token {
+			return localDB[i].DataFormat, nil
+		}
+	}
+	return "", errors.New("未找到Token")
+}
--- a/internal/db/redis.go
+++ b/internal/db/redis.go
@@ -0,0 +1,26 @@
+package db
+
+import (
+	"dypid/config"
+	"dypid/global"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// InitRedis 初始化Redis
+func InitRedis() {
+	global.RDB = redis.NewClient(&redis.Options{
+		Addr:     config.APPConfig.Redis.Host,
+		Password: config.APPConfig.Redis.Password,
+		DB:       config.APPConfig.Redis.DB,
+		//PoolSize: 1000,
+	})
+	ping := global.RDB.Ping(global.RCtx)
+	if ping.Err() != nil {
+		fmt.Println("Redis初始化失败：", ping.Err())
+		panic("Redis error")
+	} else {
+		fmt.Println("Redis初始化完成")
+	}
+}
--- a/internal/model/data.go
+++ b/internal/model/data.go
@@ -0,0 +1,16 @@
+package model
+
+type Data struct {
+	Token     string `form:"token" json:"token"`
+	Uid       string `form:"uid" json:"uid"`
+	Pid       string `form:"pid" json:"pid"`
+	SecId     string `form:"secid" json:"secid"`
+	Dyid      string `form:"dyid" json:"dyid"`
+	Key       string `form:"key" json:"key"`
+	CommentId string `form:"comment_id" json:"comment_id"`
+	Id1       string `form:"id1" json:"id1"`
+	Id2       string `form:"id2" json:"id2"`
+	Id3       string `form:"id3" json:"id3"`
+	Id4       string `form:"id4" json:"id4"`
+	Id5       string `form:"id5" json:"id5"`
+}