diff --git a/controller/dataController.go b/controller/dataController.go index 912b66c..4ddd9b4 100644 --- a/controller/dataController.go +++ b/controller/dataController.go @@ -3,52 +3,65 @@ package controller import ( "dypid/db" "dypid/global" - "dypid/model" - "encoding/json" "fmt" + "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/redis/go-redis/v9" ) func ReadDataHandler(c *gin.Context) { - lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", c.Query("token"))) + input := struct { + Token string `form:"token" binding:"required"` + }{} + if err := c.ShouldBindQuery(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()}) + return + } + + lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", input.Token)) if lLen.Val() == 0 { - c.JSON(200, gin.H{"result": "数据库没有数据"}) + c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"}) return } - retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", c.Query("token"))) - newData := model.Data{} - err := json.Unmarshal([]byte(retData.Val()[1]), &newData) - if err != nil { - c.JSON(400, gin.H{"error": err.Error()}) - return - } - c.JSON(200, gin.H{"result": newData}) + retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", input.Token)).Val()[1] + + c.String(http.StatusOK, retData) } + func WriteDataHandler(c *gin.Context) { - data := model.Data{} - - if err := c.BindQuery(&data); err != nil { - c.JSON(400, gin.H{"error": err.Error()}) + input := struct { + Token string `form:"token" binding:"required"` + Data string `form:"data" binding:"required"` + }{} + if err := c.ShouldBindQuery(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()}) return } - dedupObject, err := db.GetDedupObject(data.Token) + + dedupObject, err := db.GetDedupObject(input.Token) if err != nil { - c.JSON(400, gin.H{"error": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - err = createBF(fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject), 0.01, 100000000) - if err != nil && err.Error() != "ERR item exists" { - c.JSON(400, gin.H{"error": err.Error()}) + dataIndex, err := getDataIndex(input.Token) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } + dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]] + err = createBF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 0.01, 100000000) + if err != nil && err.Error() != "ERR item exists" { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } luaScript := ` local dedupKey = KEYS[1] -- KEYS[1]: 去重键 (dedup:token:object) local listKey = KEYS[2] -- KEYS[2]: 列表键 (list:token) local dedupValue = ARGV[1] -- ARGV[1]: 去重值 -local jsonData = ARGV[2] -- ARGV[2]: JSON序列化的数据 +local rawData = ARGV[2] -- ARGV[2]: 原始数据 -- 检查布隆过滤器中是否已存在该值 local exists = redis.call('BF.EXISTS', dedupKey, dedupValue) @@ -61,41 +74,49 @@ end -- 添加到布隆过滤器 redis.call('BF.ADD', dedupKey, dedupValue) -- 添加到列表 -redis.call('LPUSH', listKey, jsonData) +redis.call('LPUSH', listKey, rawData) -- 返回成功结果 return "ok" ` - k1 := fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject) - k2 := fmt.Sprintf("list:%s", data.Token) - v1 := c.Query(dedupObject) - v2, err := json.Marshal(data) - if err != nil { - c.JSON(500, gin.H{"error": "JSON序列化失败 " + err.Error()}) - return - } - result, err := redis.NewScript(luaScript).Run( global.RCtx, global.RDB, - []string{k1, k2}, - v1, - string(v2), + []string{ + fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), + fmt.Sprintf("list:%s", input.Token), + }, + dedupValue, + input.Data, ).Result() if err != nil { - c.JSON(500, gin.H{"error": "Redis操作失败 " + err.Error()}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()}) return } if resultMap, ok := result.(string); ok { - c.JSON(200, gin.H{"result": resultMap}) + c.JSON(http.StatusOK, gin.H{"result": resultMap}) return } - c.JSON(500, gin.H{"error": "WriteDataHandler 错误"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"}) } func createBF(bloomFilter string, errorRate float64, capacity int64) error { _, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result() return err } + +func getDataIndex(token string) (index map[string]int, err error) { + dataFormat, err := db.GetDataFormat(token) + if err != nil { + return nil, err + } + + f := strings.Split(dataFormat, "----") + index = make(map[string]int) + for i, s := range f { + index[s] = i + } + return index, nil +} diff --git a/controller/tokenController.go b/controller/tokenController.go index dace8d1..24e8a35 100644 --- a/controller/tokenController.go +++ b/controller/tokenController.go @@ -13,50 +13,63 @@ func ListTokenHandler(c *gin.Context) { } func CreateTokenHandler(c *gin.Context) { - if c.Query("token") == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"}) - return - } else if c.Query("dedup_object") == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"}) + //解析输入数据 + input := struct { + Token string `form:"token" binding:"required"` + DedupObject string `form:"dedup_object" binding:"required"` + DataFormat string `form:"data_format" binding:"required"` + }{} + if err := c.ShouldBindQuery(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()}) return } - err := db.CreateToken(c.Query("token"), c.Query("dedup_object")) + //创建Token + err := db.CreateToken(input.Token, input.DedupObject, input.DataFormat) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + + //返回 c.JSON(http.StatusOK, gin.H{"result": "ok"}) } func UpdateTokenHandler(c *gin.Context) { - if c.Query("token") == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"}) - return - } else if c.Query("dedup_object") == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"}) + input := struct { + Token string `form:"token" binding:"required"` + DedupObject string `form:"dedup_object" binding:"required"` + DataFormat string `form:"data_format" binding:"required"` + }{} + if err := c.ShouldBindQuery(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()}) return } - err := db.UpdateToken(c.Query("token"), c.Query("dedup_object")) + err := db.UpdateToken(input.Token, input.DedupObject, input.DataFormat) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + c.JSON(http.StatusOK, gin.H{"result": "ok"}) } func DeleteTokenHandler(c *gin.Context) { - if c.Query("token") == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"}) + input := struct { + Token string `form:"token" binding:"required"` + }{} + if err := c.ShouldBindQuery(&input); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()}) return } - err := db.DeleteToken(c.Query("token")) + err := db.DeleteToken(input.Token) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + c.JSON(http.StatusOK, gin.H{"result": "ok"}) } @@ -74,15 +87,22 @@ func GetTokenInfoHandler(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "获取去重对象失败 " + err.Error()}) return } + dataFormat, err := db.GetDataFormat(input.Token) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "获取数据格式失败 " + err.Error()}) + return + } output := struct { Token string `json:"token"` DedupObject string `json:"dedup_object"` + DataFormat string `json:"data_format"` DedupItemsNumber int64 `json:"dedup_items_number"` CacheListNumber int64 `json:"cache_list_number"` }{} output.Token = input.Token output.DedupObject = dedupObject + output.DataFormat = dataFormat output.DedupItemsNumber = global.RDB.BFCard(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val() output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val() diff --git a/db/local.go b/db/local.go index 10aa0eb..5db4bda 100644 --- a/db/local.go +++ b/db/local.go @@ -12,6 +12,7 @@ var localDB []Token type Token struct { Token string `json:"token"` DedupObject string `json:"dedup_object"` + DataFormat string `json:"data_format"` } func InitLocalDB() { @@ -35,7 +36,7 @@ func ListToken() []Token { return localDB } -func CreateToken(token string, dedupObject string) error { +func CreateToken(token string, dedupObject string, dataFormat string) error { InitLocalDB() for _, t := range localDB { if t.Token == token { @@ -43,7 +44,7 @@ func CreateToken(token string, dedupObject string) error { } } - a := append(localDB, Token{token, dedupObject}) + a := append(localDB, Token{token, dedupObject, dataFormat}) marshal, err := json.Marshal(a) if err != nil { return err @@ -55,12 +56,13 @@ func CreateToken(token string, dedupObject string) error { return nil } -func UpdateToken(token string, dedupObject string) error { +func UpdateToken(token string, dedupObject string, dataFormat string) error { InitLocalDB() for i, t := range localDB { if t.Token == token { localDB[i].DedupObject = dedupObject + localDB[i].DataFormat = dataFormat marshal, err := json.Marshal(localDB) if err != nil { @@ -72,6 +74,7 @@ func UpdateToken(token string, dedupObject string) error { return nil } } + InitLocalDB() return errors.New("token not found") } @@ -93,6 +96,7 @@ func DeleteToken(token string) error { return nil } } + InitLocalDB() return errors.New("token not found") } @@ -105,3 +109,12 @@ func GetDedupObject(token string) (dedupObject string, err error) { } return "", errors.New("未找到Token") } + +func GetDataFormat(token string) (dataFormat string, err error) { + for i, t := range localDB { + if t.Token == token { + return localDB[i].DataFormat, nil + } + } + return "", errors.New("未找到Token") +} diff --git a/tool/update-dypid.go b/tool/update-dypid.go index 1d76c94..a3fe7fe 100644 --- a/tool/update-dypid.go +++ b/tool/update-dypid.go @@ -16,19 +16,6 @@ import ( "github.com/spf13/viper" ) -type UploadData struct { - Dyid string // dyid - Uid string // uid - Secid string // secid - Pid string // pid - CommentId string // comment_id - Id1 string // id1 - Id2 string // id2 - Id3 string // id3 - Id4 string // id4 - Id5 string // id5 -} - var httpClient = &http.Client{ Transport: &http.Transport{ MaxIdleConns: 200, @@ -43,10 +30,6 @@ func main() { viper.SetDefault("url", "http://localhost:8080") viper.SetDefault("token", "") viper.SetDefault("thread-count", 10) - viper.SetDefault("index.uid", 0) - viper.SetDefault("index.secid", 1) - viper.SetDefault("index.pid", 2) - viper.SetDefault("index.comment-id", 3) //设置配置文件名和路径 ./config.toml viper.AddConfigPath(".") viper.SetConfigName("config") @@ -89,19 +72,10 @@ func main() { time.Sleep(2 * time.Second) } } -func uploadDataToServer(data UploadData) error { +func uploadDataToServer(data string) error { params := url.Values{} - params.Add("token", viper.GetString("token")) - params.Add("dyid", data.Dyid) - params.Add("uid", data.Uid) - params.Add("secid", data.Secid) - params.Add("pid", data.Pid) - params.Add("comment_id", data.CommentId) - params.Add("id1", data.Id1) - params.Add("id2", data.Id2) - params.Add("id3", data.Id3) - params.Add("id4", data.Id4) - params.Add("id5", data.Id5) + params.Set("token", viper.GetString("token")) + params.Set("data", data) resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader("")) if err != nil { @@ -188,35 +162,8 @@ func processLines(lines <-chan string, workerID int, filePath string) { if strings.TrimSpace(line) == "" { continue } - - // 使用----分割行数据 - parts := strings.Split(line, "----") - for i := 0; i < 10; i++ { - parts = append(parts, "") - } - - // 确保有足够的字段 - if len(parts) < 3 { - fmt.Printf("Worker %d (文件 %s): 行数据字段不足，跳过: %s\n", workerID, filePath, line) - continue - } - - // 创建上传数据结构 - uploadData := UploadData{ - Uid: parts[viper.GetInt("index.uid")], - Secid: parts[viper.GetInt("index.secid")], - Pid: parts[viper.GetInt("index.pid")], - CommentId: parts[viper.GetInt("index.comment-id")], - Dyid: parts[4], - Id1: parts[5], - Id2: parts[6], - Id3: parts[7], - Id4: parts[8], - Id5: parts[9], - } - // 上传数据 - if err := uploadDataToServer(uploadData); err != nil { + if err := uploadDataToServer(line); err != nil { fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err) } } diff --git a/web/index.html b/web/index.html index 9e5fc8f..659652b 100644 --- a/web/index.html +++ b/web/index.html @@ -1,13 +1,13 @@ - + - Vite App - - -
- - + dypid + + +
+ + diff --git a/web/src/views/TokenDetailView.vue b/web/src/views/TokenDetailView.vue index 70597c7..8b1fd13 100644 --- a/web/src/views/TokenDetailView.vue +++ b/web/src/views/TokenDetailView.vue @@ -2,6 +2,10 @@ import {useCounterStore} from "@/stores/counter.ts"; import {ref, watch} from 'vue' import axios from "@/axios.ts"; +import {useRoute} from "vue-router" + + +const route = useRoute() const result = ref() const value = ref('') @@ -67,7 +71,8 @@ axios.get('/api/token').then(res => {