feat(data): 添加数据格式支持更改数据写入读取

2025-09-03 20:10:19 +08:00
parent deb958b71b
commit 4c1e0d417d
7 changed files with 205 additions and 142 deletions
--- a/controller/dataController.go
+++ b/controller/dataController.go
@@ -3,52 +3,65 @@ package controller
 import (
 	"dypid/db"
 	"dypid/global"
-	"dypid/model"
-	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"

 	"github.com/gin-gonic/gin"
 	"github.com/redis/go-redis/v9"
 )

 func ReadDataHandler(c *gin.Context) {
-	lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", c.Query("token")))
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+
+	lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", input.Token))
 	if lLen.Val() == 0 {
-		c.JSON(200, gin.H{"result": "数据库没有数据"})
+		c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
 		return
 	}
-	retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", c.Query("token")))
-	newData := model.Data{}
-	err := json.Unmarshal([]byte(retData.Val()[1]), &newData)
-	if err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
-		return
-	}
-	c.JSON(200, gin.H{"result": newData})
+	retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", input.Token)).Val()[1]
+
+	c.String(http.StatusOK, retData)
 }
+
 func WriteDataHandler(c *gin.Context) {
-	data := model.Data{}
-
-	if err := c.BindQuery(&data); err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
+	input := struct {
+		Token string `form:"token" binding:"required"`
+		Data  string `form:"data" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
-	dedupObject, err := db.GetDedupObject(data.Token)
+
+	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
-	err = createBF(fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject), 0.01, 100000000)
-	if err != nil && err.Error() != "ERR item exists" {
-		c.JSON(400, gin.H{"error": err.Error()})
+	dataIndex, err := getDataIndex(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
+	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]

+	err = createBF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 0.01, 100000000)
+	if err != nil && err.Error() != "ERR item exists" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
 	luaScript := `
 local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
 local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
 local dedupValue = ARGV[1] -- ARGV[1]: 去重值
-local jsonData = ARGV[2]   -- ARGV[2]: JSON序列化的数据
+local rawData = ARGV[2]    -- ARGV[2]: 原始数据

 -- 检查布隆过滤器中是否已存在该值
 local exists = redis.call('BF.EXISTS', dedupKey, dedupValue)
@@ -61,41 +74,49 @@ end
 -- 添加到布隆过滤器
 redis.call('BF.ADD', dedupKey, dedupValue)
 -- 添加到列表
-redis.call('LPUSH', listKey, jsonData)
+redis.call('LPUSH', listKey, rawData)

 -- 返回成功结果
 return "ok"
 `
-	k1 := fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject)
-	k2 := fmt.Sprintf("list:%s", data.Token)
-	v1 := c.Query(dedupObject)
-	v2, err := json.Marshal(data)
-	if err != nil {
-		c.JSON(500, gin.H{"error": "JSON序列化失败 " + err.Error()})
-		return
-	}
-
 	result, err := redis.NewScript(luaScript).Run(
 		global.RCtx,
 		global.RDB,
-		[]string{k1, k2},
-		v1,
-		string(v2),
+		[]string{
+			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
+			fmt.Sprintf("list:%s", input.Token),
+		},
+		dedupValue,
+		input.Data,
 	).Result()
 	if err != nil {
-		c.JSON(500, gin.H{"error": "Redis操作失败 " + err.Error()})
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()})
 		return
 	}

 	if resultMap, ok := result.(string); ok {
-		c.JSON(200, gin.H{"result": resultMap})
+		c.JSON(http.StatusOK, gin.H{"result": resultMap})
 		return
 	}

-	c.JSON(500, gin.H{"error": "WriteDataHandler 错误"})
+	c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
 }

 func createBF(bloomFilter string, errorRate float64, capacity int64) error {
 	_, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result()
 	return err
 }
+
+func getDataIndex(token string) (index map[string]int, err error) {
+	dataFormat, err := db.GetDataFormat(token)
+	if err != nil {
+		return nil, err
+	}
+
+	f := strings.Split(dataFormat, "----")
+	index = make(map[string]int)
+	for i, s := range f {
+		index[s] = i
+	}
+	return index, nil
+}
--- a/controller/tokenController.go
+++ b/controller/tokenController.go
@@ -13,50 +13,63 @@ func ListTokenHandler(c *gin.Context) {
 }

 func CreateTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
-		return
-	} else if c.Query("dedup_object") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"})
+	//解析输入数据
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}

-	err := db.CreateToken(c.Query("token"), c.Query("dedup_object"))
+	//创建Token
+	err := db.CreateToken(input.Token, input.DedupObject, input.DataFormat)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
+	//返回
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }

 func UpdateTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
-		return
-	} else if c.Query("dedup_object") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"})
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}

-	err := db.UpdateToken(c.Query("token"), c.Query("dedup_object"))
+	err := db.UpdateToken(input.Token, input.DedupObject, input.DataFormat)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }

 func DeleteTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}

-	err := db.DeleteToken(c.Query("token"))
+	err := db.DeleteToken(input.Token)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }

@@ -74,15 +87,22 @@ func GetTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取去重对象失败 " + err.Error()})
 		return
 	}
+	dataFormat, err := db.GetDataFormat(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取数据格式失败 " + err.Error()})
+		return
+	}

 	output := struct {
 		Token            string `json:"token"`
 		DedupObject      string `json:"dedup_object"`
+		DataFormat       string `json:"data_format"`
 		DedupItemsNumber int64  `json:"dedup_items_number"`
 		CacheListNumber  int64  `json:"cache_list_number"`
 	}{}
 	output.Token = input.Token
 	output.DedupObject = dedupObject
+	output.DataFormat = dataFormat
 	output.DedupItemsNumber = global.RDB.BFCard(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val()
 	output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val()

--- a/db/local.go
+++ b/db/local.go
@@ -12,6 +12,7 @@ var localDB []Token
 type Token struct {
 	Token       string `json:"token"`
 	DedupObject string `json:"dedup_object"`
+	DataFormat  string `json:"data_format"`
 }

 func InitLocalDB() {
@@ -35,7 +36,7 @@ func ListToken() []Token {
 	return localDB
 }

-func CreateToken(token string, dedupObject string) error {
+func CreateToken(token string, dedupObject string, dataFormat string) error {
 	InitLocalDB()
 	for _, t := range localDB {
 		if t.Token == token {
@@ -43,7 +44,7 @@ func CreateToken(token string, dedupObject string) error {
 		}
 	}

-	a := append(localDB, Token{token, dedupObject})
+	a := append(localDB, Token{token, dedupObject, dataFormat})
 	marshal, err := json.Marshal(a)
 	if err != nil {
 		return err
@@ -55,12 +56,13 @@ func CreateToken(token string, dedupObject string) error {
 	return nil
 }

-func UpdateToken(token string, dedupObject string) error {
+func UpdateToken(token string, dedupObject string, dataFormat string) error {
 	InitLocalDB()

 	for i, t := range localDB {
 		if t.Token == token {
 			localDB[i].DedupObject = dedupObject
+			localDB[i].DataFormat = dataFormat

 			marshal, err := json.Marshal(localDB)
 			if err != nil {
@@ -72,6 +74,7 @@ func UpdateToken(token string, dedupObject string) error {
 			return nil
 		}
 	}
+
 	InitLocalDB()
 	return errors.New("token not found")
 }
@@ -93,6 +96,7 @@ func DeleteToken(token string) error {
 			return nil
 		}
 	}
+
 	InitLocalDB()
 	return errors.New("token not found")
 }
@@ -105,3 +109,12 @@ func GetDedupObject(token string) (dedupObject string, err error) {
 	}
 	return "", errors.New("未找到Token")
 }
+
+func GetDataFormat(token string) (dataFormat string, err error) {
+	for i, t := range localDB {
+		if t.Token == token {
+			return localDB[i].DataFormat, nil
+		}
+	}
+	return "", errors.New("未找到Token")
+}
--- a/tool/update-dypid.go
+++ b/tool/update-dypid.go
@@ -16,19 +16,6 @@ import (
 	"github.com/spf13/viper"
 )

-type UploadData struct {
-	Dyid      string // dyid
-	Uid       string // uid
-	Secid     string // secid
-	Pid       string // pid
-	CommentId string // comment_id
-	Id1       string // id1
-	Id2       string // id2
-	Id3       string // id3
-	Id4       string // id4
-	Id5       string // id5
-}
-
 var httpClient = &http.Client{
 	Transport: &http.Transport{
 		MaxIdleConns:        200,
@@ -43,10 +30,6 @@ func main() {
 	viper.SetDefault("url", "http://localhost:8080")
 	viper.SetDefault("token", "")
 	viper.SetDefault("thread-count", 10)
-	viper.SetDefault("index.uid", 0)
-	viper.SetDefault("index.secid", 1)
-	viper.SetDefault("index.pid", 2)
-	viper.SetDefault("index.comment-id", 3)
 	//设置配置文件名和路径 ./config.toml
 	viper.AddConfigPath(".")
 	viper.SetConfigName("config")
@@ -89,19 +72,10 @@ func main() {
 		time.Sleep(2 * time.Second)
 	}
 }
-func uploadDataToServer(data UploadData) error {
+func uploadDataToServer(data string) error {
 	params := url.Values{}
-	params.Add("token", viper.GetString("token"))
-	params.Add("dyid", data.Dyid)
-	params.Add("uid", data.Uid)
-	params.Add("secid", data.Secid)
-	params.Add("pid", data.Pid)
-	params.Add("comment_id", data.CommentId)
-	params.Add("id1", data.Id1)
-	params.Add("id2", data.Id2)
-	params.Add("id3", data.Id3)
-	params.Add("id4", data.Id4)
-	params.Add("id5", data.Id5)
+	params.Set("token", viper.GetString("token"))
+	params.Set("data", data)

 	resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
 	if err != nil {
@@ -188,35 +162,8 @@ func processLines(lines <-chan string, workerID int, filePath string) {
 		if strings.TrimSpace(line) == "" {
 			continue
 		}
-
-		// 使用----分割行数据
-		parts := strings.Split(line, "----")
-		for i := 0; i < 10; i++ {
-			parts = append(parts, "")
-		}
-
-		// 确保有足够的字段
-		if len(parts) < 3 {
-			fmt.Printf("Worker %d (文件 %s): 行数据字段不足，跳过: %s\n", workerID, filePath, line)
-			continue
-		}
-
-		// 创建上传数据结构
-		uploadData := UploadData{
-			Uid:       parts[viper.GetInt("index.uid")],
-			Secid:     parts[viper.GetInt("index.secid")],
-			Pid:       parts[viper.GetInt("index.pid")],
-			CommentId: parts[viper.GetInt("index.comment-id")],
-			Dyid:      parts[4],
-			Id1:       parts[5],
-			Id2:       parts[6],
-			Id3:       parts[7],
-			Id4:       parts[8],
-			Id5:       parts[9],
-		}
-
 		// 上传数据
-		if err := uploadDataToServer(uploadData); err != nil {
+		if err := uploadDataToServer(line); err != nil {
 			fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err)
 		}
 	}
--- a/web/index.html
+++ b/web/index.html
@@ -4,7 +4,7 @@
    <meta charset="UTF-8">
    <link rel="icon" href="/favicon.ico">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Vite App</title>
+    <title>dypid</title>
 </head>
 <body>
 <div id="app"></div>
--- a/web/src/views/TokenDetailView.vue
+++ b/web/src/views/TokenDetailView.vue
@@ -2,6 +2,10 @@
 import {useCounterStore} from "@/stores/counter.ts";
 import {ref, watch} from 'vue'
 import axios from "@/axios.ts";
+import {useRoute} from "vue-router"
+
+
+const route = useRoute()

 const result = ref()
 const value = ref('')
@@ -67,7 +71,8 @@ axios.get('/api/token').then(res => {

 <template>
  <b>当前Token：</b>
-  <el-select v-model="value" placeholder="选择Token" style="width: 240px">
+  <b v-if="!useCounterStore().isAdmin">{{ route.query.token }}</b>
+  <el-select v-if="useCounterStore().isAdmin" v-model="value" placeholder="选择Token" style="width: 240px">
    <el-option
        v-for="item in options"
        :key="item.value"
@@ -85,6 +90,7 @@ axios.get('/api/token').then(res => {
      border
  >
    <el-descriptions-item label="去重对象">{{ result?.dedup_object }}</el-descriptions-item>
+    <el-descriptions-item label="上传数据格式">{{ result?.data_format }}</el-descriptions-item>
    <el-descriptions-item label="去重记录值">{{ result?.dedup_items_number }}</el-descriptions-item>
    <el-descriptions-item label="Redis中数据条数">{{ result?.cache_list_number }}</el-descriptions-item>
  </el-descriptions>
--- a/web/src/views/TokenManageView.vue
+++ b/web/src/views/TokenManageView.vue
@@ -2,6 +2,8 @@
 import {ref} from "vue";
 import axios from "@/axios.ts";
 import {ElMessage, ElMessageBox} from 'element-plus'
+import {useCounterStore} from "@/stores/counter.ts";
+import {useRouter} from "vue-router";

 const tableData = ref([])

@@ -14,23 +16,27 @@ const value = ref('')
 const options = [
  {
    value: 'uid',
-    label: 'uid',
-  },
-  {
-    value: 'pid',
-    label: 'pid',
  },
  {
    value: 'secid',
-    label: 'secid',
  },
  {
-    value: 'dyid',
-    label: 'dyid',
+    value: 'pid',
  },
  {
    value: 'comment_id',
-    label: 'comment_id',
+  },
+  {
+    value: 'dyid',
+  }
+]
+
+const dataFormat = ref('')
+const dataFormatOptions = [
+  {
+    value: 'uid----secid----pid----comment_id',
+  }, {
+    value: 'uid----secid',
  }
 ]

@@ -38,7 +44,8 @@ const addToken = () => {
  axios.post('/api/token', {}, {
    params: {
      token: input.value,
-      dedup_object: value.value
+      dedup_object: value.value,
+      data_format: dataFormat.value
    }
  }).then(response => {
    if (response.data.result == "ok") {
@@ -55,18 +62,18 @@ const addToken = () => {
  })
 }

-import {useCounterStore} from "@/stores/counter.ts";
-import {useRouter} from "vue-router";

 const router = useRouter()
 const viewDetails = (row: any) => {
  useCounterStore().token = row.token
  router.push({
-    name: "TokenDetail"
+    name: "TokenDetail",
+    query: {token: row.token}
  })
 }

 var rowOut: any
+
 const dedupObjectVisible = ref(false)
 const dialogDedupObjectVisible = (row: any) => {
  rowOut = row
@@ -77,7 +84,8 @@ const updateDedupObject = () => {
  axios.put('/api/token', {}, {
    params: {
      token: rowOut.token,
-      dedup_object: value.value
+      dedup_object: value.value,
+      data_format: rowOut.data_format
    }
  }).then(res => {
    if (res.data.result == "ok") {
@@ -92,7 +100,34 @@ const updateDedupObject = () => {
  }).catch(error => {
    ElMessage.error(error.response?.data?.error)
  })
+}

+const dataFormatVisible = ref(false)
+const dialogDataFormatVisible = (row: any) => {
+  rowOut = row
+  dataFormatVisible.value = true
+}
+const updateDataFormat = () => {
+  dataFormatVisible.value = false
+  axios.put('/api/token', {}, {
+    params: {
+      token: rowOut.token,
+      dedup_object: rowOut.dedup_object,
+      data_format: dataFormat.value
+    }
+  }).then(res => {
+    if (res.data.result == "ok") {
+      ElMessage({
+        message: '更改成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
 }

 const deleteToken = (row: any) => {
@@ -127,7 +162,6 @@ const checkPassword = () => {
    ElMessage.error('密码错误')
  }
 }
-
 </script>

 <template>
@@ -143,12 +177,18 @@ const checkPassword = () => {
  <!--管理员-->
  <div v-if="useCounterStore().isAdmin">
    <!--添加Token-->
-    <el-input v-model="input" style="width: 200px" placeholder="请输入Token名称"/>
-    <el-select v-model="value" placeholder="选择去重对象" style="width: 200px">
+    <el-input v-model="input" style="width: 150px" placeholder="请输入Token名称"/>
+    <el-select v-model="value" placeholder="选择去重对象" style="width: 150px">
      <el-option
          v-for="item in options"
          :key="item.value"
-          :label="item.label"
+          :value="item.value"
+      />
+    </el-select>
+    <el-select v-model="dataFormat" placeholder="选择数据格式" style="width: 280px">
+      <el-option
+          v-for="item in dataFormatOptions"
+          :key="item.value"
          :value="item.value"
      />
    </el-select>
@@ -156,12 +196,14 @@ const checkPassword = () => {

    <!--Token列表-->
    <el-table :data="tableData" style="width: 100%">
-      <el-table-column prop="token" label="Token" width="180"/>
-      <el-table-column prop="dedup_object" label="去重对象" width="180"/>
+      <el-table-column prop="token" label="Token" width="150"/>
+      <el-table-column prop="dedup_object" label="去重对象" width="150"/>
+      <el-table-column prop="data_format" label="上传数据格式" width="280"/>
      <el-table-column label="操作">
        <template #default="scope">
          <el-button @click="viewDetails(scope.row)">查看详细</el-button>
          <el-button @click="dialogDedupObjectVisible(scope.row)" type="primary">更改去重对象</el-button>
+          <el-button @click="dialogDataFormatVisible(scope.row)" type="primary">更改数据格式</el-button>
          <el-popconfirm
              width="180"
              title="确认删除此Token吗"
@@ -183,7 +225,6 @@ const checkPassword = () => {
        <el-option
            v-for="item in options"
            :key="item.value"
-            :label="item.label"
            :value="item.value"
        />
      </el-select>
@@ -193,6 +234,21 @@ const checkPassword = () => {
        </el-button>
      </template>
    </el-dialog>
+
+    <el-dialog v-model="dataFormatVisible" title="更改数据格式" width="400">
+      <el-select v-model="dataFormat" placeholder="选择数据格式" style="width: 280px">
+        <el-option
+            v-for="item in dataFormatOptions"
+            :key="item.value"
+            :value="item.value"
+        />
+      </el-select>
+      <template #footer>
+        <el-button type="primary" @click="updateDataFormat">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
  </div>
 </template>