refactor(uploader): 优化文件上传并发处理逻辑

- 将循环方式从索引遍历改为range方式
- 添加处理进程创建注释说明
- 在进度更新后添加延时避免过度频繁更新
- 修复空行处理时计数器递增逻辑
- 优化panic恢复处理流程

frontend/src/App.vue

@@ -9,7 +9,8 @@ import {configModel} from "@/model.ts";
 import Config = config.Config;
 
 // const serverUrl = ref('')
-const token = ref('')
+const hasVideoToken = ref('')
+const noVideoToken = ref('')
 const checkDir = ref('')
 const concurrentFiles = ref(1)
 const uploadThreads = ref(1)
@@ -59,6 +60,7 @@ const selectDirectory = () => {
   SelectPath().then((path) => {
     if (path) {
       checkDir.value = path
+      writeCheckDir()
     } else {
       ElMessage.warning('未选择目录，不更改配置')
     }
@@ -70,8 +72,8 @@ const startRun = () => {
   //   ElMessage.warning('请输入服务器地址')
   //   return
   // }
-  if (!token.value) {
+	if (!hasVideoToken.value || !noVideoToken.value) {
-    ElMessage.error('请输入Token')
+		ElMessage.error('请输入两个上传 Token')
     return
   }
   if (!checkDir.value) {
@@ -104,8 +106,11 @@ const clearLog = () => {
 // const writeServerUrl =() => {
 //   WriteConfig("url", serverUrl.value)
 // }
-const writeToken = () => {
+const writeHasVideoToken = () => {
-  WriteConfig(configModel.Token, token.value)
+  WriteConfig(configModel.HasVideoToken, hasVideoToken.value)
+}
+const writeNoVideoToken = () => {
+  WriteConfig(configModel.NoVideoToken, noVideoToken.value)
 }
 const writeCheckDir = () => {
   WriteConfig(configModel.CheckDir, checkDir.value)
@@ -124,7 +129,8 @@ const writeUploadThreads = () => {
 try {
   GetConfig().then((config: Config) => {
     // serverUrl.value = config.url
-    token.value = config.token
+    hasVideoToken.value = config.has_video_token
+    noVideoToken.value = config.no_video_token
     checkDir.value = config.check_dir
     concurrentFiles.value = config.handle_file_count
     uploadThreads.value = config.thread_count
@@ -167,8 +173,15 @@ try {
       <!--      </div>-->
 
       <div class="form-item">
-        <label>Token</label>
+        <label>有视频 Token</label>
-        <el-input v-model="token" placeholder="请输入Token" :disabled="isRunning" @change="writeToken()"/>
+        <el-input v-model="hasVideoToken" placeholder="有视频账号上传使用的 Token" :disabled="isRunning"
+                  @change="writeHasVideoToken()"/>
+      </div>
+
+      <div class="form-item">
+        <label>无视频 Token</label>
+        <el-input v-model="noVideoToken" placeholder="无视频账号上传使用的 Token" :disabled="isRunning"
+                  @change="writeNoVideoToken()"/>
       </div>
 
       <div class="form-item">
@@ -187,7 +200,7 @@ try {
 
       <div class="form-item">
         <label>单文件上传线程</label>
-        <el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"
+        <el-input-number v-model="uploadThreads" :min="1" :max="500" :disabled="isRunning"
                          @change="writeUploadThreads()"/>
       </div>
 

frontend/src/components/ConfirmClearDialog.vue

@@ -40,9 +40,9 @@ const handleConfirm = () => {
       align-center
   >
     <template #header>
-      是否确认清空上传文件
+      是否确认清空需要上传文件
     </template>
-    <div class="hint-text">以下文件将会被清空并移动到tmp文件夹进行上传，您是否确认</div>
+    <div class="hint-text">以下文件将会被清空并移动到 tmp 文件夹进行上传，您是否确认</div>
     <div class="dialog-content">
       <div class="file-list">
         <div v-for="(file, index) in fileList" :key="index" class="file-item">

frontend/src/model.ts

@@ -1,6 +1,8 @@
 export enum configModel {
     Url = "url",
     Token = "token",
+    HasVideoToken = "has-video-token",
+    NoVideoToken = "no-video-token",
     ThreadCount = "thread-count",
     HandleFileCount = "handle-file-count",
     IsRunOnStart = "is-run-on-start",

frontend/wailsjs/go/models.ts

@@ -3,6 +3,8 @@ export namespace config {
 	export class Config {
 	    url: string;
 	    token: string;
+	    has_video_token: string;
+	    no_video_token: string;
 	    thread_count: number;
 	    handle_file_count: number;
 	    is_run_on_start: boolean;
@@ -17,6 +19,8 @@ export namespace config {
 	        if ('string' === typeof source) source = JSON.parse(source);
 	        this.url = source["url"];
 	        this.token = source["token"];
+	        this.has_video_token = source["has_video_token"];
+	        this.no_video_token = source["no_video_token"];
 	        this.thread_count = source["thread_count"];
 	        this.handle_file_count = source["handle_file_count"];
 	        this.is_run_on_start = source["is_run_on_start"];

go.mod

@@ -5,6 +5,7 @@ go 1.26
 require (
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/spf13/viper v1.21.0
+	github.com/tidwall/gjson v1.14.2
 	github.com/wailsapp/wails/v2 v2.12.0
 	golang.org/x/sync v0.20.0
 )
@@ -37,6 +38,8 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tkrajina/go-reflector v0.5.8 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect

go.sum

@@ -81,6 +81,12 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tkrajina/go-reflector v0.5.8 h1:yPADHrwmUbMq4RGEyaOUpz2H90sRsETNVpjzo3DLVQQ=
 github.com/tkrajina/go-reflector v0.5.8/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

internal/api/api.go

@@ -7,8 +7,11 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
+
+	"github.com/tidwall/gjson"
 )
 
 var httpClient = &http.Client{
@@ -31,9 +34,9 @@ func init() {
 func InitConn() {
 	wg := &sync.WaitGroup{}
 
-	for i := 0; i < 10; i++ {
+	for range 10 {
 		wg.Go(func() {
-			for i := 0; i < 50; i++ {
+			for range 50 {
 				resp, err := httpClient.Get(config.APPConfig.Url + "/api/test")
 				if err != nil {
 					fmt.Println(err)
@@ -56,8 +59,31 @@ func UploadDataToServer(ctx context.Context, data string) error {
 		<-limit
 	}()
 
+	//根据账号是否有视频判断要存储的token
 	params := url.Values{}
-	params.Set("token", config.APPConfig.Token)
+	secUserId := strings.Split(data, "----")[1]
+	hasVideo, err := CheckHasVideo(secUserId)
+	if err != nil {
+		for err != nil {
+			fmt.Println("抖音请求api重试")
+			//定时器定时100毫秒
+			ticker := time.NewTicker(100 * time.Millisecond)
+
+			select {
+			case <-ctx.Done():
+				return err
+			case <-ticker.C:
+				ticker.Stop()
+				hasVideo, err = CheckHasVideo(secUserId)
+			}
+		}
+	}
+	if hasVideo {
+		params.Set("token", config.APPConfig.HasVideoToken)
+	} else {
+		params.Set("token", config.APPConfig.NoVideoToken)
+	}
+
 	params.Set("data", data)
 
 	//http://127.0.0.1:8080/api/data?token=123456&data=123456
@@ -82,3 +108,41 @@ func UploadDataToServer(ctx context.Context, data string) error {
 
 	return err
 }
+
+var douYinClient = &http.Client{}
+
+func CheckHasVideo(secUserId string) (bool, error) {
+	req, err := http.NewRequest(
+		"GET",
+		"https://imdesktop.douyin.com/aweme/v1/web/user/profile/other/?sec_user_id="+secUserId,
+		nil,
+	)
+
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+	req.Header.Add("User-Agent", "Apifox/1.0.0 (https://apifox.com)")
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Host", "imdesktop.douyin.com")
+	req.Header.Add("Connection", "keep-alive")
+
+	res, err := douYinClient.Do(req)
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+
+	if gjson.Get(string(body), "user.aweme_count").Int() > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}

internal/config/config.go

@@ -11,6 +11,8 @@ import (
 type Config struct {
 	Url                string `json:"url" mapstructure:"url"`
 	Token              string `json:"token" mapstructure:"token"`
+	HasVideoToken      string `json:"has_video_token" mapstructure:"has-video-token"`
+	NoVideoToken       string `json:"no_video_token" mapstructure:"no-video-token"`
 	ThreadCount        int    `json:"thread_count" mapstructure:"thread-count"`
 	HandleFileCount    int    `json:"handle_file_count" mapstructure:"handle-file-count"`
 	IsRunOnStart       bool   `json:"is_run_on_start" mapstructure:"is-run-on-start"`
@@ -24,6 +26,8 @@ var configMu sync.Mutex
 const (
 	Url                = "url"
 	Token              = "token"
+	HasVideoToken      = "has-video-token"
+	NoVideoToken       = "no-video-token"
 	ThreadCount        = "thread-count"
 	HandleFileCount    = "handle-file-count"
 	IsRunOnStart       = "is-run-on-start"
@@ -34,8 +38,11 @@ const (
 func InitConfig() {
 	// 设置默认配置
 	defaultConfig := Config{
-		Url:                "http://127.0.0.1:8080",
+		//Url:                "http://127.0.0.1:8080",
+		Url:                "http://112.124.71.39:8080",
 		Token:              "",
+		HasVideoToken:      "1234",
+		NoVideoToken:       "5678",
 		ThreadCount:        10,
 		HandleFileCount:    25,
 		IsRunOnStart:       false,
@@ -44,6 +51,8 @@ func InitConfig() {
 	}
 	viper.SetDefault(Url, defaultConfig.Url)
 	viper.SetDefault(Token, defaultConfig.Token)
+	viper.SetDefault(HasVideoToken, defaultConfig.HasVideoToken)
+	viper.SetDefault(NoVideoToken, defaultConfig.NoVideoToken)
 	viper.SetDefault(ThreadCount, defaultConfig.ThreadCount)
 	viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
 	viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)

internal/uploader/uploader.go

@@ -44,10 +44,12 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 	AddLog(logChan, `单文件上传线程: `+strconv.Itoa(config.APPConfig.ThreadCount))
 	AddLog(logChan, "===============================================")
 
+	//创建连接池
 	AddLog(logChan, "正在创建连接池（连接池可避免首次大量上传时出现网络错误）")
 	api.InitConn()
 	AddLog(logChan, "创建连接池完成，开始运行程序")
 
+	//清除进度
 	progress.Clear()
 
 	//推送上传进度
@@ -57,6 +59,8 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 			case <-ctx.Done():
 				return
 			default:
+			}
+
 			var pg []Progress
 			progress.Range(func(_, value any) bool {
 				pg = append(pg, value.(Progress))
@@ -66,12 +70,13 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 
 			time.Sleep(250 * time.Millisecond)
 		}
-		}
 	}()
 
 	//开启上传程序
 	for {
 		uploadData(ctx, logChan)
+
+		//延时1分钟运行
 		select {
 		case <-ctx.Done():
 			return
@@ -116,22 +121,8 @@ func uploadData(ctx context.Context, logChan *chan string) {
 			return
 		}
 
-		//是否向用户提示清空文件，并复制文件到tmp
+		//向用户提示要清空文件，获取确认
-		if config.APPConfig.ClearFilesNoPrompt {
+		if config.APPConfig.ClearFilesNoPrompt == false {
-			//不用提示直接复制文件到tmp
-			for _, p := range f {
-				err := copyFile(p, "./tmp/"+filepath.Base(p))
-				if err != nil {
-					AddLog(logChan, "复制文件失败："+err.Error())
-				} else {
-					files = append(files, "./tmp/"+filepath.Base(p))
-					err := os.Truncate(p, 0)
-					if err != nil {
-						AddLog(logChan, "清空文件失败："+err.Error())
-					}
-				}
-			}
-		} else {
 			//提示用户
 			wailsruntime.EventsEmit(ctx, "clear-files", f)
 
@@ -141,8 +132,21 @@ func uploadData(ctx context.Context, logChan *chan string) {
 				confirm <- optionalData[0].(bool)
 			})
 
-			if <-confirm {
+			if <-confirm == false {
+				//取消上传
+				AddLog(logChan, "已取消上传，1分钟后再运行")
+				return
+			}
+		}
+
+		//复制文件到tmp目录
 		for _, p := range f {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
 			err := copyFile(p, "./tmp/"+filepath.Base(p))
 			if err != nil {
 				AddLog(logChan, "复制文件失败："+err.Error())
@@ -154,25 +158,22 @@ func uploadData(ctx context.Context, logChan *chan string) {
 				}
 			}
 		}
-			} else {
-				AddLog(logChan, "已取消上传，1分钟后再运行")
-				return
-			}
-		}
 	}
 
 	//检测到文件
 	//统计文件行数
-	var fInfo = make(map[string]fileInfo)
+	var filesInfo = make(map[string]fileInfo)
-
-	AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
 
 	isAllEmpty := true
+
+	AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
 	for _, filePath := range files {
 		select {
 		case <-ctx.Done():
 			return
 		default:
+		}
+
 		file, err := os.Open(filePath)
 		if err != nil {
 			AddLog(logChan, "打开文件失败："+err.Error())
@@ -189,7 +190,7 @@ func uploadData(ctx context.Context, logChan *chan string) {
 		if lineCount == 0 {
 			continue
 		}
-			fInfo[filepath.Base(filePath)] = fileInfo{
+		filesInfo[filepath.Base(filePath)] = fileInfo{
 			FilePath:  filePath,
 			FileLines: lineCount,
 		}
@@ -198,7 +199,6 @@ func uploadData(ctx context.Context, logChan *chan string) {
 
 		AddLog(logChan, fmt.Sprintf("%s 文件行数：%v", filepath.Base(filePath), lineCount))
 	}
-	}
 
 	if isAllEmpty {
 		AddLog(logChan, "所有文件都为空，不进行上传")
@@ -207,7 +207,7 @@ func uploadData(ctx context.Context, logChan *chan string) {
 
 	//刷新文件上传进度
 	progress.Clear()
-	for fileName, info := range fInfo {
+	for fileName, info := range filesInfo {
 		progress.Store(fileName,
 			Progress{
 				FileName:   fileName,
@@ -220,18 +220,23 @@ func uploadData(ctx context.Context, logChan *chan string) {
 
 	// 使用 errgroup 控制同时处理的文件数，并开始上传文件任务
 	g, egctx := errgroup.WithContext(ctx)
-	g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
+	// 设置同时处理文件数
+	g.SetLimit(config.APPConfig.HandleFileCount)
 	// 执行文件上传任务参数（文件路径，文件行数）
-	for fileName, info := range fInfo {
+	for fileName, info := range filesInfo {
 		select {
 		case <-egctx.Done():
 			return
 		default:
+		}
+
 		g.Go(func() error {
 			select {
 			case <-egctx.Done():
 				return egctx.Err()
 			default:
+			}
+
 			AddLog(logChan, "正在上传文件："+fileName)
 
 			processFile(egctx, logChan, info.FilePath, info.FileLines)
@@ -240,28 +245,28 @@ func uploadData(ctx context.Context, logChan *chan string) {
 			case <-egctx.Done():
 				return egctx.Err()
 			default:
+			}
+
 			//上传完成，删除缓存文件
 			err := os.Remove(info.FilePath)
 			if err != nil {
 				AddLog(logChan, "删除缓存文件失败："+err.Error())
 			}
 			return nil
-					}
-				}
 		})
 	}
-	}
 
 	select {
 	case <-ctx.Done():
 		return
 	default:
+	}
+
 	// 等待所有任务完成
 	g.Wait()
 
 	AddLog(logChan, "所有任务执行完成！")
 	AddLog(logChan, fmt.Sprintf("上传完成，耗时：%s", time.Since(start).String()))
-	}
 }
 
 // copyFile 快速拷贝文件 src -> dst
@@ -329,17 +334,19 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 	lines := make(chan string, 200)
 	var countLine int32 = 0
 	// 创建指定个worker同时处理文件上传
-	for i := 0; i < config.APPConfig.ThreadCount; i++ {
+	for i := range config.APPConfig.ThreadCount {
 		select {
 		case <-ctx.Done():
 			close(lines)
 			return
 		default:
+		}
+
+		//创建处理进程
 		go func() {
 			processLines(ctx, logChan, &lines, i, filePath, &countLine)
 		}()
 	}
-	}
 
 	// 读取文件并发送到通道
 	scanner := bufio.NewScanner(file)
@@ -350,13 +357,15 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 				fmt.Println("panic:", f+":"+strconv.Itoa(l), r)
 			}
 		}()
+
 		for scanner.Scan() {
 			select {
 			case <-ctx.Done():
 				return
 			default:
-				lines <- scanner.Text()
 			}
+
+			lines <- scanner.Text()
 		}
 	}()
 
@@ -367,6 +376,8 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 			close(lines) //关闭processLines中的上传线程
 			return
 		default:
+		}
+
 		progress.Store(filepath.Base(filePath),
 			Progress{
 				FileName:   filepath.Base(filePath),
@@ -375,7 +386,7 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 				Percentage: int(float64(countLine) / float64(fileLines) * 100),
 			},
 		)
-		}
+		time.Sleep(50 * time.Millisecond)
 	}
 	//上传完成，进度设为100
 	progress.Store(filepath.Base(filePath),
@@ -403,8 +414,11 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
 		case <-ctx.Done():
 			return
 		default:
+		}
+
 		// 跳过空行
 		if strings.TrimSpace(line) == "" {
+			atomic.AddInt32(countLine, 1)
 			continue
 		}
 		// 上传数据
@@ -413,7 +427,6 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
 		}
 		atomic.AddInt32(countLine, 1)
 	}
-	}
 }
 
 // AddLog 添加日志

main.go

@@ -30,7 +30,7 @@ func main() {
 		},
 		BackgroundColour: &options.RGBA{R: 27, G: 38, B: 54, A: 1},
 		OnStartup:        app.startup,
-		Bind: []interface{}{
+		Bind: []any{
 			app,
 		},
 	})