refactor: 修改SecId字段名，添加上传工具进度展示和映射更改

tool/update-dypid.go

@@ -19,7 +19,7 @@ import (
 type UploadData struct {
 	Dyid      string // dyid
 	Uid       string // uid
-	Secuid    string // secuid
+	Secid     string // secid
 	Pid       string // pid
 	CommentId string // comment_id
 	Id1       string // id1
@@ -31,8 +31,8 @@ type UploadData struct {
 
 var httpClient = &http.Client{
 	Transport: &http.Transport{
-		MaxIdleConns:        100,
-		MaxIdleConnsPerHost: 10,
+		MaxIdleConns:        200,
+		MaxIdleConnsPerHost: 100,
 		IdleConnTimeout:     30 * time.Second,
 	},
 	Timeout: 30 * time.Second,
@@ -40,10 +40,13 @@ var httpClient = &http.Client{
 
 func main() {
 	//程序配置
-	//设置默认值
 	viper.SetDefault("url", "http://localhost:8080")
 	viper.SetDefault("token", "")
 	viper.SetDefault("thread-count", 10)
+	viper.SetDefault("index.uid", 0)
+	viper.SetDefault("index.secid", 1)
+	viper.SetDefault("index.pid", 2)
+	viper.SetDefault("index.comment-id", 3)
 	//设置配置文件名和路径 ./config.toml
 	viper.AddConfigPath(".")
 	viper.SetConfigName("config")
@@ -61,10 +64,10 @@ func main() {
 	})
 
 	//检测./upload
-	fmt.Println("程序启动成功，正在检测upload")
-	os.Mkdir("./upload", os.ModePerm)
+	fmt.Println("程序启动成功，正在检测txt文件")
+	//os.Mkdir("./upload", os.ModePerm)
 	for {
-		files, err := getTxtFiles("./upload")
+		files, err := getTxtFiles("./")
 		if err != nil {
 			fmt.Println(err)
 			return
@@ -91,7 +94,7 @@ func uploadDataToServer(data UploadData) error {
 	params.Add("token", viper.GetString("token"))
 	params.Add("dyid", data.Dyid)
 	params.Add("uid", data.Uid)
-	params.Add("secuid", data.Secuid)
+	params.Add("secid", data.Secid)
 	params.Add("pid", data.Pid)
 	params.Add("comment_id", data.CommentId)
 	params.Add("id1", data.Id1)
@@ -101,7 +104,12 @@ func uploadDataToServer(data UploadData) error {
 	params.Add("id5", data.Id5)
 
 	resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
-	_, _ = io.Copy(io.Discard, resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp != nil {
+		_, _ = io.Copy(io.Discard, resp.Body)
+	}
 	return err
 }
 
@@ -159,6 +167,9 @@ func processFile(filePath string) {
 		line := scanner.Text()
 		lines <- line
 		lineCount++
+		if lineCount%10000 == 0 {
+			fmt.Printf("文件【%s】处理进度：%v%%\n", filePath, float64(lineCount)/40000*100)
+		}
 	}
 
 	close(lines)
@@ -168,7 +179,7 @@ func processFile(filePath string) {
 		fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
 	}
 
-	fmt.Printf("文件 %s 处理完成，共处理 %d 行数据\n", filePath, lineCount)
+	fmt.Printf("文件【%s】处理完成，共处理 %d 行数据\n", filePath, lineCount)
 }
 
 func processLines(lines <-chan string, workerID int, filePath string) {
@@ -192,11 +203,11 @@ func processLines(lines <-chan string, workerID int, filePath string) {
 
 		// 创建上传数据结构
 		uploadData := UploadData{
-			Dyid:      parts[0],
-			Uid:       parts[1],
-			Secuid:    parts[2],
-			Pid:       parts[3],
-			CommentId: parts[4],
+			Uid:       parts[viper.GetInt("index.uid")],
+			Secid:     parts[viper.GetInt("index.secid")],
+			Pid:       parts[viper.GetInt("index.pid")],
+			CommentId: parts[viper.GetInt("index.comment-id")],
+			Dyid:      parts[4],
 			Id1:       parts[5],
 			Id2:       parts[6],
 			Id3:       parts[7],
@@ -208,8 +219,5 @@ func processLines(lines <-chan string, workerID int, filePath string) {
 		if err := uploadDataToServer(uploadData); err != nil {
 			fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err)
 		}
-
-		// 添加短暂延迟避免服务器过载
-		//time.Sleep(10 * time.Millisecond)
 	}
 }