refactor(uploader): 优化文件上传并发处理逻辑

- 将循环方式从索引遍历改为range方式
- 添加处理进程创建注释说明
- 在进度更新后添加延时避免过度频繁更新
- 修复空行处理时计数器递增逻辑
- 优化panic恢复处理流程

frontend/package.json

@@ -11,6 +11,7 @@
     "type-check": "vue-tsc --build"
   },
   "dependencies": {
+    "@element-plus/icons-vue": "^2.3.2",
     "element-plus": "^2.13.7",
     "vue": "^3.5.32"
   },

frontend/package.json.md5

@@ -1 +1 @@
-aa914e6b4676ee4621ced7ad6d81c58c
+05225657934ff66d822c925754c951bf

frontend/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      '@element-plus/icons-vue':
+        specifier: ^2.3.2
+        version: 2.3.2(vue@3.5.33(typescript@6.0.3))
       element-plus:
         specifier: ^2.13.7
         version: 2.13.7(typescript@6.0.3)(vue@3.5.33(typescript@6.0.3))

frontend/src/App.vue

@@ -1,14 +1,16 @@
 <script lang="ts" setup>
-import {ref, nextTick, computed} from 'vue'
+import {computed, nextTick, ref} from 'vue'
-import {ElMessage} from 'element-plus'
+import {ElMessage, ElMessageBox} from 'element-plus'
-import {ElMessageBox} from 'element-plus'
+import ConfirmClearDialog from './components/ConfirmClearDialog.vue'
-import {SelectPath, GetConfig, WriteConfig, StartUpload, StopUpload} from '../wailsjs/go/main/App';
+import {GetConfig, SelectPath, StartUpload, StopUpload, WriteConfig} from '../wailsjs/go/main/App';
 import {config} from "../wailsjs/go/models.ts";
-import Config = config.Config;
 import {EventsOn, LogPrint} from "../wailsjs/runtime";
+import {configModel} from "@/model.ts";
+import Config = config.Config;
 
 // const serverUrl = ref('')
-const token = ref('')
+const hasVideoToken = ref('')
+const noVideoToken = ref('')
 const checkDir = ref('')
 const concurrentFiles = ref(1)
 const uploadThreads = ref(1)
@@ -19,6 +21,10 @@ const logOutput = ref<string[]>([])
 const logContentRef = ref<HTMLElement>()
 const logRoll = ref(true)
 
+const clearDialogVisible = ref(false)
+const filesToClear = ref<string[]>([])
+const noPromptClear = ref(false)
+
 interface FileProgress {
   name: string
   uploaded: number
@@ -54,6 +60,7 @@ const selectDirectory = () => {
   SelectPath().then((path) => {
     if (path) {
       checkDir.value = path
+      writeCheckDir()
     } else {
       ElMessage.warning('未选择目录，不更改配置')
     }
@@ -65,8 +72,8 @@ const startRun = () => {
   //   ElMessage.warning('请输入服务器地址')
   //   return
   // }
-  if (!token.value) {
+	if (!hasVideoToken.value || !noVideoToken.value) {
-    ElMessage.error('请输入Token')
+		ElMessage.error('请输入两个上传 Token')
     return
   }
   if (!checkDir.value) {
@@ -99,17 +106,20 @@ const clearLog = () => {
 // const writeServerUrl =() => {
 //   WriteConfig("url", serverUrl.value)
 // }
-const writeToken =() => {
+const writeHasVideoToken = () => {
-  WriteConfig("token", token.value)
+  WriteConfig(configModel.HasVideoToken, hasVideoToken.value)
+}
+const writeNoVideoToken = () => {
+  WriteConfig(configModel.NoVideoToken, noVideoToken.value)
 }
 const writeCheckDir = () => {
-  WriteConfig("check-dir", checkDir.value)
+  WriteConfig(configModel.CheckDir, checkDir.value)
 }
 const writeConcurrentFiles = () => {
-  WriteConfig("handle-file-count", concurrentFiles.value)
+  WriteConfig(configModel.HandleFileCount, concurrentFiles.value)
 }
 const writeUploadThreads = () => {
-  WriteConfig("thread-count", uploadThreads.value)
+  WriteConfig(configModel.ThreadCount, uploadThreads.value)
 }
 // const writeAutoStart = () => {
 //   WriteConfig("is-run-on-start", autoStart.value)
@@ -119,11 +129,13 @@ const writeUploadThreads =() => {
 try {
   GetConfig().then((config: Config) => {
     // serverUrl.value = config.url
-    token.value = config.token
+    hasVideoToken.value = config.has_video_token
+    noVideoToken.value = config.no_video_token
     checkDir.value = config.check_dir
     concurrentFiles.value = config.handle_file_count
     uploadThreads.value = config.thread_count
     // autoStart.value = config.is_run_on_start
+    noPromptClear.value = config.clear_files_no_prompt
 
     LogPrint(`[${new Date().toLocaleString()}] 配置已加载`)
   })
@@ -131,6 +143,7 @@ try {
   console.log(e)
 }
 
+try {
   EventsOn("is-run", (run) => {
     isRunning.value = run
   })
@@ -140,6 +153,15 @@ EventsOn("progress", (progress) => {
   EventsOn("log", (msg) => {
     addLog(msg)
   })
+  EventsOn("clear-files", (files) => {
+    filesToClear.value = files
+    if (!noPromptClear.value) {
+      clearDialogVisible.value = true
+    }
+  })
+} catch (e) {
+  console.log(e)
+}
 </script>
 
 <template>
@@ -151,8 +173,15 @@ EventsOn("log", (msg) => {
       <!--      </div>-->
 
       <div class="form-item">
-        <label>Token</label>
+        <label>有视频 Token</label>
-        <el-input v-model="token" placeholder="请输入Token" :disabled="isRunning" @change="writeToken()"/>
+        <el-input v-model="hasVideoToken" placeholder="有视频账号上传使用的 Token" :disabled="isRunning"
+                  @change="writeHasVideoToken()"/>
+      </div>
+
+      <div class="form-item">
+        <label>无视频 Token</label>
+        <el-input v-model="noVideoToken" placeholder="无视频账号上传使用的 Token" :disabled="isRunning"
+                  @change="writeNoVideoToken()"/>
       </div>
 
       <div class="form-item">
@@ -165,12 +194,14 @@ EventsOn("log", (msg) => {
 
       <div class="form-item">
         <label>同时处理文件数</label>
-        <el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning" @change="writeConcurrentFiles()"/>
+        <el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"
+                         @change="writeConcurrentFiles()"/>
       </div>
 
       <div class="form-item">
         <label>单文件上传线程</label>
-        <el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning" @change="writeUploadThreads()"/>
+        <el-input-number v-model="uploadThreads" :min="1" :max="500" :disabled="isRunning"
+                         @change="writeUploadThreads()"/>
       </div>
 
       <!--      <div class="form-item">-->
@@ -205,6 +236,11 @@ EventsOn("log", (msg) => {
       </div>
     </div>
   </div>
+
+  <ConfirmClearDialog
+      v-model:visible="clearDialogVisible"
+      :file-list="filesToClear"
+  />
 </template>
 
 <style scoped>

frontend/src/components/ConfirmClearDialog.vue

@@ -0,0 +1,124 @@
+<script lang="ts" setup>
+import {ref} from 'vue'
+import {WriteConfig} from "../../wailsjs/go/main/App";
+import {configModel} from "@/model.ts";
+import {EventsEmit} from "../../wailsjs/runtime";
+import {InfoFilled} from '@element-plus/icons-vue'
+
+const props = defineProps<{
+  visible: boolean
+  fileList: string[]
+}>()
+
+const emit = defineEmits<{
+  (e: 'update:visible', value: boolean): void
+  // (e: 'confirm', dontShowAgain: boolean): void
+}>()
+
+const dontShowAgain = ref(false)
+
+const handleClose = () => {
+  emit('update:visible', false)
+  EventsEmit('confirm-clear-files', false)
+}
+
+const handleConfirm = () => {
+  // emit('confirm', dontShowAgain.value)
+  emit('update:visible', false)
+  WriteConfig(configModel.ClearFilesNoPrompt, dontShowAgain.value)
+  EventsEmit('confirm-clear-files', true)
+}
+</script>
+
+<template>
+  <el-dialog
+      :model-value="visible"
+      width="600px"
+      :close-on-click-modal="false"
+      @update:model-value="(val: boolean) => emit('update:visible', val)"
+      @close="handleClose"
+      align-center
+  >
+    <template #header>
+      是否确认清空需要上传文件
+    </template>
+    <div class="hint-text">以下文件将会被清空并移动到 tmp 文件夹进行上传，您是否确认</div>
+    <div class="dialog-content">
+      <div class="file-list">
+        <div v-for="(file, index) in fileList" :key="index" class="file-item">
+          {{ file }}
+        </div>
+        <div v-if="fileList.length === 0" class="empty-text">暂无文件</div>
+      </div>
+    </div>
+    <template #footer>
+      <div class="dialog-footer">
+        <div class="dialog-footer-checkbox-container">
+          <el-checkbox v-model="dontShowAgain" label="下次清空文件不再弹出此弹窗确认"/>
+          <el-tooltip content="此弹窗会在首次运行时弹出，您可以选择下次不再弹出此弹窗">
+            <el-icon>
+              <InfoFilled/>
+            </el-icon>
+          </el-tooltip>
+        </div>
+
+        <div>
+          <el-button @click="handleClose">取消</el-button>
+          <el-button type="primary" @click="handleConfirm">确认</el-button>
+        </div>
+      </div>
+    </template>
+  </el-dialog>
+</template>
+
+<style scoped>
+.dialog-footer-checkbox-container {
+  display: flex;
+  align-items: center;
+  gap: 5px;
+}
+
+.dialog-footer-checkbox-container .el-icon {
+  color: #909399;
+}
+
+.dialog-content {
+  padding: 10px 0;
+}
+
+.hint-text {
+  font-size: 12px;
+  color: #909399;
+}
+
+.file-list {
+  max-height: 350px;
+  overflow-y: auto;
+  border: 1px solid #e4e7ed;
+  border-radius: 4px;
+  padding: 8px;
+}
+
+.file-item {
+  padding: 6px 8px;
+  font-size: 14px;
+  color: #606266;
+  border-bottom: 1px solid #f0f0f0;
+}
+
+.file-item:last-child {
+  border-bottom: none;
+}
+
+.empty-text {
+  text-align: center;
+  color: #909399;
+  padding: 20px 0;
+}
+
+.dialog-footer {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+</style>

frontend/src/model.ts

@@ -0,0 +1,11 @@
+export enum configModel {
+    Url = "url",
+    Token = "token",
+    HasVideoToken = "has-video-token",
+    NoVideoToken = "no-video-token",
+    ThreadCount = "thread-count",
+    HandleFileCount = "handle-file-count",
+    IsRunOnStart = "is-run-on-start",
+    CheckDir = "check-dir",
+    ClearFilesNoPrompt = "clear-files-no-prompt",
+}

frontend/wailsjs/go/models.ts

@@ -3,10 +3,13 @@ export namespace config {
 	export class Config {
 	    url: string;
 	    token: string;
+	    has_video_token: string;
+	    no_video_token: string;
 	    thread_count: number;
 	    handle_file_count: number;
 	    is_run_on_start: boolean;
 	    check_dir: string;
+	    clear_files_no_prompt: boolean;
 	
 	    static createFrom(source: any = {}) {
 	        return new Config(source);
@@ -16,10 +19,13 @@ export namespace config {
 	        if ('string' === typeof source) source = JSON.parse(source);
 	        this.url = source["url"];
 	        this.token = source["token"];
+	        this.has_video_token = source["has_video_token"];
+	        this.no_video_token = source["no_video_token"];
 	        this.thread_count = source["thread_count"];
 	        this.handle_file_count = source["handle_file_count"];
 	        this.is_run_on_start = source["is_run_on_start"];
 	        this.check_dir = source["check_dir"];
+	        this.clear_files_no_prompt = source["clear_files_no_prompt"];
 	    }
 	}
 

go.mod

@@ -5,6 +5,7 @@ go 1.26
 require (
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/spf13/viper v1.21.0
+	github.com/tidwall/gjson v1.14.2
 	github.com/wailsapp/wails/v2 v2.12.0
 	golang.org/x/sync v0.20.0
 )
@@ -37,6 +38,8 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tkrajina/go-reflector v0.5.8 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect

go.sum

@@ -81,6 +81,12 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tkrajina/go-reflector v0.5.8 h1:yPADHrwmUbMq4RGEyaOUpz2H90sRsETNVpjzo3DLVQQ=
 github.com/tkrajina/go-reflector v0.5.8/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

internal/api/api.go

@@ -7,8 +7,11 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
+
+	"github.com/tidwall/gjson"
 )
 
 var httpClient = &http.Client{
@@ -31,9 +34,9 @@ func init() {
 func InitConn() {
 	wg := &sync.WaitGroup{}
 
-	for i := 0; i < 10; i++ {
+	for range 10 {
 		wg.Go(func() {
-			for i := 0; i < 50; i++ {
+			for range 50 {
 				resp, err := httpClient.Get(config.APPConfig.Url + "/api/test")
 				if err != nil {
 					fmt.Println(err)
@@ -56,8 +59,31 @@ func UploadDataToServer(ctx context.Context, data string) error {
 		<-limit
 	}()
 
+	//根据账号是否有视频判断要存储的token
 	params := url.Values{}
-	params.Set("token", config.APPConfig.Token)
+	secUserId := strings.Split(data, "----")[1]
+	hasVideo, err := CheckHasVideo(secUserId)
+	if err != nil {
+		for err != nil {
+			fmt.Println("抖音请求api重试")
+			//定时器定时100毫秒
+			ticker := time.NewTicker(100 * time.Millisecond)
+
+			select {
+			case <-ctx.Done():
+				return err
+			case <-ticker.C:
+				ticker.Stop()
+				hasVideo, err = CheckHasVideo(secUserId)
+			}
+		}
+	}
+	if hasVideo {
+		params.Set("token", config.APPConfig.HasVideoToken)
+	} else {
+		params.Set("token", config.APPConfig.NoVideoToken)
+	}
+
 	params.Set("data", data)
 
 	//http://127.0.0.1:8080/api/data?token=123456&data=123456
@@ -82,3 +108,41 @@ func UploadDataToServer(ctx context.Context, data string) error {
 
 	return err
 }
+
+var douYinClient = &http.Client{}
+
+func CheckHasVideo(secUserId string) (bool, error) {
+	req, err := http.NewRequest(
+		"GET",
+		"https://imdesktop.douyin.com/aweme/v1/web/user/profile/other/?sec_user_id="+secUserId,
+		nil,
+	)
+
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+	req.Header.Add("User-Agent", "Apifox/1.0.0 (https://apifox.com)")
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Host", "imdesktop.douyin.com")
+	req.Header.Add("Connection", "keep-alive")
+
+	res, err := douYinClient.Do(req)
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+
+	if gjson.Get(string(body), "user.aweme_count").Int() > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}

internal/config/config.go

@@ -11,10 +11,13 @@ import (
 type Config struct {
 	Url                string `json:"url" mapstructure:"url"`
 	Token              string `json:"token" mapstructure:"token"`
+	HasVideoToken      string `json:"has_video_token" mapstructure:"has-video-token"`
+	NoVideoToken       string `json:"no_video_token" mapstructure:"no-video-token"`
 	ThreadCount        int    `json:"thread_count" mapstructure:"thread-count"`
 	HandleFileCount    int    `json:"handle_file_count" mapstructure:"handle-file-count"`
 	IsRunOnStart       bool   `json:"is_run_on_start" mapstructure:"is-run-on-start"`
 	CheckDir           string `json:"check_dir" mapstructure:"check-dir"`
+	ClearFilesNoPrompt bool   `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"`
 }
 
 var APPConfig Config
@@ -23,28 +26,38 @@ var configMu sync.Mutex
 const (
 	Url                = "url"
 	Token              = "token"
+	HasVideoToken      = "has-video-token"
+	NoVideoToken       = "no-video-token"
 	ThreadCount        = "thread-count"
 	HandleFileCount    = "handle-file-count"
 	IsRunOnStart       = "is-run-on-start"
 	CheckDir           = "check-dir"
+	ClearFilesNoPrompt = "clear-files-no-prompt"
 )
 
 func InitConfig() {
 	// 设置默认配置
 	defaultConfig := Config{
-		Url:             "http://127.0.0.1:8080",
+		//Url:                "http://127.0.0.1:8080",
+		Url:                "http://112.124.71.39:8080",
 		Token:              "",
+		HasVideoToken:      "1234",
+		NoVideoToken:       "5678",
 		ThreadCount:        10,
 		HandleFileCount:    25,
 		IsRunOnStart:       false,
 		CheckDir:           "",
+		ClearFilesNoPrompt: false,
 	}
 	viper.SetDefault(Url, defaultConfig.Url)
 	viper.SetDefault(Token, defaultConfig.Token)
+	viper.SetDefault(HasVideoToken, defaultConfig.HasVideoToken)
+	viper.SetDefault(NoVideoToken, defaultConfig.NoVideoToken)
 	viper.SetDefault(ThreadCount, defaultConfig.ThreadCount)
 	viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
 	viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)
 	viper.SetDefault(CheckDir, defaultConfig.CheckDir)
+	viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt)
 
 	//设置配置文件名和路径 ./config.toml
 	viper.AddConfigPath(".")

internal/uploader/uploader.go

@@ -7,6 +7,7 @@ import (
 	"dypid-client/internal/api"
 	"dypid-client/internal/config"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -20,7 +21,7 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-type Task struct {
+type fileInfo struct {
 	FilePath  string
 	FileLines int
 }
@@ -43,10 +44,12 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 	AddLog(logChan, `单文件上传线程: `+strconv.Itoa(config.APPConfig.ThreadCount))
 	AddLog(logChan, "===============================================")
 
+	//创建连接池
 	AddLog(logChan, "正在创建连接池（连接池可避免首次大量上传时出现网络错误）")
 	api.InitConn()
 	AddLog(logChan, "创建连接池完成，开始运行程序")
 
+	//清除进度
 	progress.Clear()
 
 	//推送上传进度
@@ -56,6 +59,8 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 			case <-ctx.Done():
 				return
 			default:
+			}
+
 			var pg []Progress
 			progress.Range(func(_, value any) bool {
 				pg = append(pg, value.(Progress))
@@ -65,12 +70,13 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 
 			time.Sleep(250 * time.Millisecond)
 		}
-		}
 	}()
 
 	//开启上传程序
 	for {
 		uploadData(ctx, logChan)
+
+		//延时1分钟运行
 		select {
 		case <-ctx.Done():
 			return
@@ -80,34 +86,94 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 }
 
 func uploadData(ctx context.Context, logChan *chan string) {
+	start := time.Now()
+
 	// 获取检测目录
-	var path = "./"
+	var checkPath = "./"
 	if config.APPConfig.CheckDir != "" {
-		path = config.APPConfig.CheckDir
+		checkPath = config.APPConfig.CheckDir
 	}
 
-	//获取文件列表
+	//要上传的文件路径字符串数组
-	files, err := getTxtFiles(path)
+	var files []string
+
+	//先检测tmp目录有没有残余文件
+	os.Mkdir("./tmp", os.ModePerm)
+	tmpFiles, err := getTxtFiles("./tmp")
+	if err != nil {
+		AddLog(logChan, "获取 tmp 文件列表失败："+err.Error())
+	}
+
+	//tmp有文件，优先上传（else：tmp没文件扫描指定文件夹，并复制文件到tmp）
+	if tmpFiles != nil {
+		AddLog(logChan, "当前 tmp 目录下还有未上传完成文件，将优先上传 tmp 目录文件")
+		files = tmpFiles
+	} else {
+		//tmp没文件，扫描指定文件夹
+		f, err := getTxtFiles(checkPath)
 		if err != nil {
 			AddLog(logChan, "获取文件列表失败："+err.Error())
 			return
 		}
-	if files == nil {
+
+		//指定文件夹没文件，退出函数
+		if f == nil {
 			return
 		}
 
-	start := time.Now()
+		//向用户提示要清空文件，获取确认
+		if config.APPConfig.ClearFilesNoPrompt == false {
+			//提示用户
+			wailsruntime.EventsEmit(ctx, "clear-files", f)
+
+			confirm := make(chan bool)
+
+			wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) {
+				confirm <- optionalData[0].(bool)
+			})
+
+			if <-confirm == false {
+				//取消上传
+				AddLog(logChan, "已取消上传，1分钟后再运行")
+				return
+			}
+		}
+
+		//复制文件到tmp目录
+		for _, p := range f {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			err := copyFile(p, "./tmp/"+filepath.Base(p))
+			if err != nil {
+				AddLog(logChan, "复制文件失败："+err.Error())
+			} else {
+				files = append(files, "./tmp/"+filepath.Base(p))
+				err := os.Truncate(p, 0)
+				if err != nil {
+					AddLog(logChan, "清空文件失败："+err.Error())
+				}
+			}
+		}
+	}
 
 	//检测到文件
 	//统计文件行数
-	fileLines := make(map[string]int)
+	var filesInfo = make(map[string]fileInfo)
-	AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
+
 	isAllEmpty := true
+
+	AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
 	for _, filePath := range files {
 		select {
 		case <-ctx.Done():
 			return
 		default:
+		}
+
 		file, err := os.Open(filePath)
 		if err != nil {
 			AddLog(logChan, "打开文件失败："+err.Error())
@@ -124,10 +190,14 @@ func uploadData(ctx context.Context, logChan *chan string) {
 		if lineCount == 0 {
 			continue
 		}
-			fileLines[filepath.Base(filePath)] = lineCount
+		filesInfo[filepath.Base(filePath)] = fileInfo{
-			isAllEmpty = false
+			FilePath:  filePath,
-			AddLog(logChan, fmt.Sprintf("%s 文件行数：%v", filepath.Base(filePath), lineCount))
+			FileLines: lineCount,
 		}
+
+		isAllEmpty = false
+
+		AddLog(logChan, fmt.Sprintf("%s 文件行数：%v", filepath.Base(filePath), lineCount))
 	}
 
 	if isAllEmpty {
@@ -135,62 +205,97 @@ func uploadData(ctx context.Context, logChan *chan string) {
 		return
 	}
 
+	//刷新文件上传进度
 	progress.Clear()
-
+	for fileName, info := range filesInfo {
-	//添加文件上传任务参数（文件路径，文件行数）
+		progress.Store(fileName,
-	var tasks []Task
+			Progress{
-	for fileName, lines := range fileLines {
+				FileName:   fileName,
-		tasks = append(tasks, Task{FilePath: path + "/" + fileName, FileLines: lines})
+				Total:      info.FileLines,
-		progress.Store(fileName, Progress{FileName: fileName, Total: lines, Uploaded: 0, Percentage: 0})
+				Uploaded:   0,
+				Percentage: 0,
+			},
+		)
 	}
 
 	// 使用 errgroup 控制同时处理的文件数，并开始上传文件任务
 	g, egctx := errgroup.WithContext(ctx)
-	g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
+	// 设置同时处理文件数
-	// 执行所有任务
+	g.SetLimit(config.APPConfig.HandleFileCount)
-	for _, task := range tasks {
+	// 执行文件上传任务参数（文件路径，文件行数）
+	for fileName, info := range filesInfo {
 		select {
 		case <-egctx.Done():
 			return
 		default:
+		}
+
 		g.Go(func() error {
 			select {
 			case <-egctx.Done():
 				return egctx.Err()
 			default:
-					AddLog(logChan, "正在上传文件："+filepath.Base(task.FilePath))
+			}
 
-					processFile(egctx, logChan, task.FilePath, task.FileLines)
+			AddLog(logChan, "正在上传文件："+fileName)
+
+			processFile(egctx, logChan, info.FilePath, info.FileLines)
 
 			select {
 			case <-egctx.Done():
 				return egctx.Err()
 			default:
-						//上传完成，清空文件
+			}
-						err := os.Truncate(task.FilePath, 0)
+
+			//上传完成，删除缓存文件
+			err := os.Remove(info.FilePath)
 			if err != nil {
-							AddLog(logChan, "清空文件失败："+err.Error())
+				AddLog(logChan, "删除缓存文件失败："+err.Error())
 			}
 			return nil
-					}
-				}
 		})
 	}
-	}
 
 	select {
 	case <-ctx.Done():
 		return
 	default:
+	}
+
 	// 等待所有任务完成
 	g.Wait()
 
 	AddLog(logChan, "所有任务执行完成！")
 	AddLog(logChan, fmt.Sprintf("上传完成，耗时：%s", time.Since(start).String()))
 }
+
+// copyFile 快速拷贝文件 src -> dst
+func copyFile(src, dst string) error {
+	// 打开源文件
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	// 创建目标文件
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	// 核心：最快拷贝，底层使用操作系统零拷贝技术
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return err
 	}
 
-// 获取目录中的所有txt文件
+	// 强制刷入磁盘，保证数据完整
+	return destFile.Sync()
+}
+
+// 获取目录中的所有txt文件（文件大小为0的不返回）
 func getTxtFiles(dir string) (txtFiles []string, err error) {
 	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -204,8 +309,10 @@ func getTxtFiles(dir string) (txtFiles []string, err error) {
 
 		// 检查文件扩展名是否为.txt
 		if strings.ToLower(filepath.Ext(path)) == ".txt" {
+			if info.Size() != 0 {
 				txtFiles = append(txtFiles, path)
 			}
+		}
 
 		return nil
 	})
@@ -227,17 +334,19 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 	lines := make(chan string, 200)
 	var countLine int32 = 0
 	// 创建指定个worker同时处理文件上传
-	for i := 0; i < config.APPConfig.ThreadCount; i++ {
+	for i := range config.APPConfig.ThreadCount {
 		select {
 		case <-ctx.Done():
 			close(lines)
 			return
 		default:
+		}
+
+		//创建处理进程
 		go func() {
 			processLines(ctx, logChan, &lines, i, filePath, &countLine)
 		}()
 	}
-	}
 
 	// 读取文件并发送到通道
 	scanner := bufio.NewScanner(file)
@@ -248,13 +357,15 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 				fmt.Println("panic:", f+":"+strconv.Itoa(l), r)
 			}
 		}()
+
 		for scanner.Scan() {
 			select {
 			case <-ctx.Done():
 				return
 			default:
-				lines <- scanner.Text()
 			}
+
+			lines <- scanner.Text()
 		}
 	}()
 
@@ -265,6 +376,8 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 			close(lines) //关闭processLines中的上传线程
 			return
 		default:
+		}
+
 		progress.Store(filepath.Base(filePath),
 			Progress{
 				FileName:   filepath.Base(filePath),
@@ -273,7 +386,7 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
 				Percentage: int(float64(countLine) / float64(fileLines) * 100),
 			},
 		)
-		}
+		time.Sleep(50 * time.Millisecond)
 	}
 	//上传完成，进度设为100
 	progress.Store(filepath.Base(filePath),
@@ -301,8 +414,11 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
 		case <-ctx.Done():
 			return
 		default:
+		}
+
 		// 跳过空行
 		if strings.TrimSpace(line) == "" {
+			atomic.AddInt32(countLine, 1)
 			continue
 		}
 		// 上传数据
@@ -312,7 +428,6 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
 		atomic.AddInt32(countLine, 1)
 	}
 }
-}
 
 // AddLog 添加日志
 func AddLog(logChan *chan string, message string) {

main.go

@@ -30,7 +30,7 @@ func main() {
 		},
 		BackgroundColour: &options.RGBA{R: 27, G: 38, B: 54, A: 1},
 		OnStartup:        app.startup,
-		Bind: []interface{}{
+		Bind: []any{
 			app,
 		},
 	})