125 lines
3.1 KiB
Go
125 lines
3.1 KiB
Go
package controller
|
|
|
|
import (
|
|
"dypid/db"
|
|
"dypid/global"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
func ReadDataHandler(c *gin.Context) {
|
|
input := struct {
|
|
Token string `form:"token" binding:"required"`
|
|
}{}
|
|
if err := c.ShouldBindQuery(&input); err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", input.Token))
|
|
if lLen.Val() == 0 {
|
|
c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
|
|
return
|
|
}
|
|
retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", input.Token)).Val()[1]
|
|
|
|
c.String(http.StatusOK, retData)
|
|
}
|
|
|
|
func WriteDataHandler(c *gin.Context) {
|
|
//解析输入数据
|
|
input := struct {
|
|
Token string `form:"token" binding:"required"`
|
|
Data string `form:"data" binding:"required"`
|
|
}{}
|
|
if err := c.ShouldBindQuery(&input); err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
//数据获取
|
|
dedupObject, err := db.GetDedupObject(input.Token)
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
dataIndex, err := getDataIndex(input.Token)
|
|
if err != nil {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
|
|
|
|
err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100000000)
|
|
if err != nil && err.Error() != "ERR item exists" {
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
return
|
|
}
|
|
luaScript := `
|
|
local dedupKey = KEYS[1] -- KEYS[1]: 去重键 (dedup:token:object)
|
|
local listKey = KEYS[2] -- KEYS[2]: 列表键 (list:token)
|
|
local dedupValue = ARGV[1] -- ARGV[1]: 去重值
|
|
local rawData = ARGV[2] -- ARGV[2]: 原始数据
|
|
|
|
-- 检查布隆过滤器中是否已存在该值
|
|
local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
|
|
|
|
-- 如果已存在，返回已去重标记
|
|
if exists == 1 then
|
|
return "已去重"
|
|
end
|
|
|
|
-- 添加到布隆过滤器
|
|
redis.call('CF.ADD', dedupKey, dedupValue)
|
|
-- 添加到列表
|
|
redis.call('LPUSH', listKey, rawData)
|
|
|
|
-- 返回成功结果
|
|
return "ok"
|
|
`
|
|
result, err := redis.NewScript(luaScript).Run(
|
|
global.RCtx,
|
|
global.RDB,
|
|
[]string{
|
|
fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
|
|
fmt.Sprintf("list:%s", input.Token),
|
|
},
|
|
dedupValue,
|
|
input.Data,
|
|
).Result()
|
|
if err != nil {
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()})
|
|
return
|
|
}
|
|
|
|
if resultMap, ok := result.(string); ok {
|
|
c.JSON(http.StatusOK, gin.H{"result": resultMap})
|
|
return
|
|
}
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
|
|
}
|
|
|
|
func createCF(bloomFilter string, capacity int64) error {
|
|
_, err := global.RDB.CFReserve(global.RCtx, bloomFilter, capacity).Result()
|
|
return err
|
|
}
|
|
|
|
func getDataIndex(token string) (index map[string]int, err error) {
|
|
dataFormat, err := db.GetDataFormat(token)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f := strings.Split(dataFormat, "----")
|
|
index = make(map[string]int)
|
|
for i, s := range f {
|
|
index[s] = i
|
|
}
|
|
return index, nil
|
|
}
|