ygxbnet
bc15b19474
部署开发环境 / deploy-dev (push) Successful in 2m6s
- 在数据控制器中引入去重对象逻辑 - 根据去重对象配置决定是否执行去重操作 - 当去重对象为 false 时不进行去重直接写入数据库 - 在管理界面令牌管理页面添加不去重选项 - 表格中显示去重对象时将 false 显示为不去重中文描述
144 lines
3.7 KiB
Go
144 lines
3.7 KiB
Go
package controller
|
|
|
|
import (
|
|
"dypid/internal/db"
|
|
"dypid/internal/global"
|
|
"dypid/internal/model"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
func ReadDataHandler(c *gin.Context) {
|
|
//解析输入数据
|
|
input := struct {
|
|
Token string `form:"token" binding:"required"`
|
|
}{}
|
|
if err := c.ShouldBindQuery(&input); err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
retData := global.RDB.LPop(global.RCtx, fmt.Sprintf("list:%s", input.Token)).Val()
|
|
if retData == "null" {
|
|
c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
|
|
return
|
|
}
|
|
|
|
c.String(http.StatusOK, retData)
|
|
}
|
|
|
|
func WriteDataHandler(c *gin.Context) {
|
|
//解析输入数据
|
|
input := struct {
|
|
Token string `form:"token" binding:"required"`
|
|
Data string `form:"data" binding:"required"`
|
|
}{}
|
|
if err := c.ShouldBindQuery(&input); err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
//获取去重对象
|
|
dedupObject, err := db.GetDedupObject(input.Token)
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
|
|
//判断数据是否需要去重
|
|
//如果数据不需要去重则直接写入数据库
|
|
if dedupObject == "false" {
|
|
global.RDB.LPush(global.RCtx, fmt.Sprintf("list:%s", input.Token), input.Data)
|
|
c.JSON(http.StatusOK, model.APIResponse{
|
|
Code: 200,
|
|
Message: "数据写入成功",
|
|
})
|
|
return
|
|
}
|
|
|
|
//数据要去重，下面函数会执行
|
|
//获取数据索引
|
|
dataIndex, err := getDataIndex(input.Token)
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
|
|
|
|
err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
|
|
if err != nil && err.Error() != "ERR item exists" {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
luaScript := `
|
|
local dedupKey = KEYS[1] -- KEYS[1]: 去重键 (dedup:token:object)
|
|
local listKey = KEYS[2] -- KEYS[2]: 列表键 (list:token)
|
|
local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
|
|
local dedupValue = ARGV[1] -- ARGV[1]: 去重值
|
|
local rawData = ARGV[2] -- ARGV[2]: 原始数据
|
|
|
|
-- 检查布隆过滤器中是否已存在该值
|
|
local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
|
|
|
|
-- 如果已存在，返回已去重标记
|
|
if exists == 1 then
|
|
return "已去重"
|
|
end
|
|
|
|
-- 添加到布隆过滤器
|
|
redis.call('CF.ADD', dedupKey, dedupValue)
|
|
-- 添加到列表
|
|
redis.call('LPUSH', listKey, rawData)
|
|
|
|
redis.call('LPUSH', deleteListKey, dedupValue)
|
|
|
|
-- 返回成功结果
|
|
return "ok"
|
|
`
|
|
result, err := redis.NewScript(luaScript).Run(
|
|
global.RCtx,
|
|
global.RDB,
|
|
[]string{
|
|
fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
|
|
fmt.Sprintf("list:%s", input.Token),
|
|
fmt.Sprintf("delete-list:%s", input.Token),
|
|
},
|
|
dedupValue,
|
|
input.Data,
|
|
).Result()
|
|
if err != nil {
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
if resultMap, ok := result.(string); ok {
|
|
c.JSON(http.StatusOK, gin.H{"result": resultMap})
|
|
return
|
|
}
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
|
|
}
|
|
|
|
func createCF(bloomFilter string, capacity int64) error {
|
|
_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
|
|
return err
|
|
}
|
|
|
|
func getDataIndex(token string) (index map[string]int, err error) {
|
|
dataFormat, err := db.GetDataFormat(token)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f := strings.Split(dataFormat, "----")
|
|
index = make(map[string]int)
|
|
for i, s := range f {
|
|
index[s] = i
|
|
}
|
|
return index, nil
|
|
}
|