From 8ffa0531d6e13c192881ee6b9495bdb9b630f8b9 Mon Sep 17 00:00:00 2001 From: YGXB_net Date: Fri, 1 May 2026 21:01:01 +0800 Subject: [PATCH] =?UTF-8?q?feat(uploader):=20=E6=B7=BB=E5=8A=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=8A=E4=BC=A0=E9=85=8D=E7=BD=AE=E5=92=8C=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 ClearFilesNoPrompt 配置项用于控制是否提示清空文件 - 实现临时文件目录(tmp)管理，优先处理未上传完的文件 - 添加文件复制功能，支持快速零拷贝技术 - 实现文件清空提示机制，支持用户确认操作 - 优化文件上传流程，添加进度跟踪和状态更新 - 过滤掉大小为0的文件，避免无效上传 - 修改数据结构名称提升代码可读性 --- internal/config/config.go | 40 ++++---- internal/uploader/uploader.go | 170 +++++++++++++++++++++++++++------- 2 files changed, 158 insertions(+), 52 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 306a134..a8f2087 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,35 +9,38 @@ import ( ) type Config struct { - Url string `json:"url" mapstructure:"url"` - Token string `json:"token" mapstructure:"token"` - ThreadCount int `json:"thread_count" mapstructure:"thread-count"` - HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"` - IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"` - CheckDir string `json:"check_dir" mapstructure:"check-dir"` + Url string `json:"url" mapstructure:"url"` + Token string `json:"token" mapstructure:"token"` + ThreadCount int `json:"thread_count" mapstructure:"thread-count"` + HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"` + IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"` + CheckDir string `json:"check_dir" mapstructure:"check-dir"` + ClearFilesNoPrompt bool `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"` } var APPConfig Config var configMu sync.Mutex const ( - Url = "url" - Token = "token" - ThreadCount = "thread-count" - HandleFileCount = "handle-file-count" - IsRunOnStart = "is-run-on-start" - CheckDir = "check-dir" + Url = "url" + Token = "token" + ThreadCount = "thread-count" + HandleFileCount = "handle-file-count" + IsRunOnStart = "is-run-on-start" + CheckDir = "check-dir" + ClearFilesNoPrompt = "clear-files-no-prompt" ) func InitConfig() { // 设置默认配置 defaultConfig := Config{ - Url: "http://127.0.0.1:8080", - Token: "", - ThreadCount: 10, - HandleFileCount: 25, - IsRunOnStart: false, - CheckDir: "", + Url: "http://127.0.0.1:8080", + Token: "", + ThreadCount: 10, + HandleFileCount: 25, + IsRunOnStart: false, + CheckDir: "", + ClearFilesNoPrompt: false, } viper.SetDefault(Url, defaultConfig.Url) viper.SetDefault(Token, defaultConfig.Token) @@ -45,6 +48,7 @@ func InitConfig() { viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount) viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart) viper.SetDefault(CheckDir, defaultConfig.CheckDir) + viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt) //设置配置文件名和路径 ./config.toml viper.AddConfigPath(".") diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index 474ff39..97c87d3 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -7,6 +7,7 @@ import ( "dypid-client/internal/api" "dypid-client/internal/config" "fmt" + "io" "os" "path/filepath" "runtime" @@ -20,7 +21,7 @@ import ( "golang.org/x/sync/errgroup" ) -type Task struct { +type fileInfo struct { FilePath string FileLines int } @@ -80,28 +81,92 @@ func StartUpload(ctx context.Context, logChan *chan string) { } func uploadData(ctx context.Context, logChan *chan string) { - // 获取检测目录 - var path = "./" - if config.APPConfig.CheckDir != "" { - path = config.APPConfig.CheckDir - } - - //获取文件列表 - files, err := getTxtFiles(path) - if err != nil { - AddLog(logChan, "获取文件列表失败："+err.Error()) - return - } - if files == nil { - return - } - start := time.Now() + // 获取检测目录 + var checkPath = "./" + if config.APPConfig.CheckDir != "" { + checkPath = config.APPConfig.CheckDir + } + + //要上传的文件路径字符串数组 + var files []string + + //先检测tmp目录有没有残余文件 + os.Mkdir("./tmp", os.ModePerm) + tmpFiles, err := getTxtFiles("./tmp") + if err != nil { + AddLog(logChan, "获取 tmp 文件列表失败："+err.Error()) + } + + //tmp有文件，优先上传（else：tmp没文件扫描指定文件夹，并复制文件到tmp） + if tmpFiles != nil { + AddLog(logChan, "当前 tmp 目录下还有未上传完成文件，将优先上传 tmp 目录文件") + files = tmpFiles + } else { + //tmp没文件，扫描指定文件夹 + f, err := getTxtFiles(checkPath) + if err != nil { + AddLog(logChan, "获取文件列表失败："+err.Error()) + return + } + + //指定文件夹没文件，退出函数 + if f == nil { + return + } + + //是否向用户提示清空文件，并复制文件到tmp + if config.APPConfig.ClearFilesNoPrompt { + //不用提示直接复制文件到tmp + for _, p := range f { + err := copyFile(p, "./tmp/"+filepath.Base(p)) + if err != nil { + AddLog(logChan, "复制文件失败："+err.Error()) + } else { + files = append(files, "./tmp/"+filepath.Base(p)) + err := os.Truncate(p, 0) + if err != nil { + AddLog(logChan, "清空文件失败："+err.Error()) + } + } + } + } else { + //提示用户 + wailsruntime.EventsEmit(ctx, "clear-files", f) + + confirm := make(chan bool) + + wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) { + confirm <- optionalData[0].(bool) + }) + + if <-confirm { + for _, p := range f { + err := copyFile(p, "./tmp/"+filepath.Base(p)) + if err != nil { + AddLog(logChan, "复制文件失败："+err.Error()) + } else { + files = append(files, "./tmp/"+filepath.Base(p)) + err := os.Truncate(p, 0) + if err != nil { + AddLog(logChan, "清空文件失败："+err.Error()) + } + } + } + } else { + AddLog(logChan, "已取消上传，1分钟后再运行") + return + } + } + } + //检测到文件 //统计文件行数 - fileLines := make(map[string]int) + var fInfo = make(map[string]fileInfo) + AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files))) + isAllEmpty := true for _, filePath := range files { select { @@ -124,8 +189,13 @@ func uploadData(ctx context.Context, logChan *chan string) { if lineCount == 0 { continue } - fileLines[filepath.Base(filePath)] = lineCount + fInfo[filepath.Base(filePath)] = fileInfo{ + FilePath: filePath, + FileLines: lineCount, + } + isAllEmpty = false + AddLog(logChan, fmt.Sprintf("%s 文件行数：%v", filepath.Base(filePath), lineCount)) } } @@ -135,20 +205,24 @@ func uploadData(ctx context.Context, logChan *chan string) { return } + //刷新文件上传进度 progress.Clear() - - //添加文件上传任务参数（文件路径，文件行数） - var tasks []Task - for fileName, lines := range fileLines { - tasks = append(tasks, Task{FilePath: path + "/" + fileName, FileLines: lines}) - progress.Store(fileName, Progress{FileName: fileName, Total: lines, Uploaded: 0, Percentage: 0}) + for fileName, info := range fInfo { + progress.Store(fileName, + Progress{ + FileName: fileName, + Total: info.FileLines, + Uploaded: 0, + Percentage: 0, + }, + ) } // 使用 errgroup 控制同时处理的文件数，并开始上传文件任务 g, egctx := errgroup.WithContext(ctx) g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数 - // 执行所有任务 - for _, task := range tasks { + // 执行文件上传任务参数（文件路径，文件行数） + for fileName, info := range fInfo { select { case <-egctx.Done(): return @@ -158,18 +232,18 @@ func uploadData(ctx context.Context, logChan *chan string) { case <-egctx.Done(): return egctx.Err() default: - AddLog(logChan, "正在上传文件："+filepath.Base(task.FilePath)) + AddLog(logChan, "正在上传文件："+fileName) - processFile(egctx, logChan, task.FilePath, task.FileLines) + processFile(egctx, logChan, info.FilePath, info.FileLines) select { case <-egctx.Done(): return egctx.Err() default: - //上传完成，清空文件 - err := os.Truncate(task.FilePath, 0) + //上传完成，删除缓存文件 + err := os.Remove(info.FilePath) if err != nil { - AddLog(logChan, "清空文件失败："+err.Error()) + AddLog(logChan, "删除缓存文件失败："+err.Error()) } return nil } @@ -190,7 +264,33 @@ func uploadData(ctx context.Context, logChan *chan string) { } } -// 获取目录中的所有txt文件 +// copyFile 快速拷贝文件 src -> dst +func copyFile(src, dst string) error { + // 打开源文件 + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + // 创建目标文件 + destFile, err := os.Create(dst) + if err != nil { + return err + } + defer destFile.Close() + + // 核心：最快拷贝，底层使用操作系统零拷贝技术 + _, err = io.Copy(destFile, sourceFile) + if err != nil { + return err + } + + // 强制刷入磁盘，保证数据完整 + return destFile.Sync() +} + +// 获取目录中的所有txt文件（文件大小为0的不返回） func getTxtFiles(dir string) (txtFiles []string, err error) { err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -204,7 +304,9 @@ func getTxtFiles(dir string) (txtFiles []string, err error) { // 检查文件扩展名是否为.txt if strings.ToLower(filepath.Ext(path)) == ".txt" { - txtFiles = append(txtFiles, path) + if info.Size() != 0 { + txtFiles = append(txtFiles, path) + } } return nil