Compare commits

..

17 Commits

Author SHA1 Message Date
ygxbnet b100fc9b32 fix(upload): 修正上传文件清空确认对话框文本
构建上传工具 / build (push) Successful in 1m51s
- 修正了标题文本"是否确认清空需要上传文件"
- 修正了提示文本中的"tmp"文件夹显示格式
- 更新了确认对话框的内容描述文本
2026-05-01 22:51:12 +08:00
ygxbnet d426c16104 refactor(uploader): 优化上传器代码结构和上下文取消处理
- 在进度更新循环中添加上下文取消检查点
- 在文件复制操作前添加上下文取消检查点
- 重构代码缩进和括号位置以提高可读性
- 优化 goroutine 中的上下文取消处理逻辑
- 统一代码块的括号格式和缩进风格
2026-05-01 22:47:39 +08:00
ygxbnet 993814cdfa refactor(uploader): 优化文件信息统计逻辑
- 将变量名 fInfo 重命名为 filesInfo 以提高可读性
- 调整代码顺序,将 AddLog 调用移到变量声明后
- 统一使用新变量名在所有相关位置进行引用
- 移动 g.SetLimit 注释位置以提高代码可读性
2026-05-01 22:37:45 +08:00
ygxbnet 8ffa0531d6 feat(uploader): 添加文件上传配置和临时文件处理功能
构建上传工具 / build (push) Successful in 1m10s
- 新增 ClearFilesNoPrompt 配置项用于控制是否提示清空文件
- 实现临时文件目录(tmp)管理,优先处理未上传完的文件
- 添加文件复制功能,支持快速零拷贝技术
- 实现文件清空提示机制,支持用户确认操作
- 优化文件上传流程,添加进度跟踪和状态更新
- 过滤掉大小为0的文件,避免无效上传
- 修改数据结构名称提升代码可读性
2026-05-01 21:01:01 +08:00
ygxbnet 3f6e999783 feat(app): 添加文件清空确认对话框功能
- 引入 ConfirmClearDialog 组件用于清空文件确认
- 添加 clear-files-no-prompt 配置项控制是否显示确认弹窗
- 实现清空文件列表显示和确认逻辑
- 集成 Element Plus 图标组件库
- 优化日志输出格式增加空格分隔
- 重构配置写入方法使用统一的 configModel 枚举
- 添加事件监听处理清空文件操作
- 实现勾选不再提示选项并保存配置
2026-05-01 20:45:17 +08:00
ygxbnet 03e4e6f45b fix(uploader): 修复上传器逻辑和UI提示问题
构建上传工具 / build (push) Successful in 1m23s
- 修复目录选择为空时不更改配置的逻辑
- 将警告提示改为错误提示以提高用户体验
- 优化停止运行时的成功提示
- 增加进度列表的最大高度以显示更多内容
- 调整定时器逻辑顺序避免提前退出
- 移除重复的任务等待错误处理
- 优化代码结构和空行格式
2026-04-30 21:24:30 +08:00
ygxbnet 6bd82024d9 refactor(App): 移除服务器地址和自动启动配置功能
- 注释掉 serverUrl 相关的响应式变量定义
- 注释掉 autoStart 相关的响应式变量定义
- 移除进度列表中的测试数据注释
- 注释掉服务器地址验证逻辑
- 注释掉服务器地址和自动启动的写入配置函数
- 注释掉配置加载中的相关字段赋值
- 在模板中隐藏服务器地址输入框和自动启动复选框组件
2026-04-30 20:39:08 +08:00
ygxbnet 6952c33f16 refactor(ci): 重命名构建工作流文件
构建上传工具 / build (push) Successful in 2m56s
- 将 .gitea/workflows/build_tool.yaml 重命名为 .gitea/workflows/build.yaml
- 更新工作流中的作业名称从 build-tool 到 build
- 保持相同的触发条件和运行环境配置
2026-04-29 14:07:22 +08:00
ygxbnet 3d2b3469cc refactor(uploader): 重命名文件上传函数并优化代码结构
构建上传工具 / build-tool (push) Successful in 1m59s
- 将 StartLooking 函数重命名为 StartUpload 以更准确反映功能
- 修复构建脚本中的版本标记 ldflags 格式问题
- 更新应用标题格式,在版本号前添加冒号分隔符
- 优化上传进度循环逻辑,调整代码执行顺序
- 添加注释说明上传程序启动和文件处理功能
- 清理代码中的多余空行和变量声明
2026-04-29 14:01:50 +08:00
ygxbnet a528d6a877 fix(uploader): 修复上传器上下文取消时的资源泄露
构建上传工具 / build-tool (push) Successful in 1m19s
- 在上下文取消时正确关闭 lines channel
- 防止 goroutine 阻塞导致的内存泄露
- 确保所有资源在程序退出时正确释放
2026-04-28 15:44:48 +08:00
ygxbnet 73a7d26816 refactor(uploader): 优化文件上传并发处理逻辑
构建上传工具 / build-tool (push) Successful in 1m46s
- 将初始化连接池的并发方式改为waitgroup控制的goroutine池
- 调整文件处理时的channel缓冲区大小从100增加到200
- 移除不必要的sync.WaitGroup变量声明
- 修改进度计算逻辑,确保上传完成时进度显示为100%
- 添加对processLines函数的功能注释
- 优化上下文取消时的资源清理流程,及时关闭channel
2026-04-28 15:43:02 +08:00
ygxbnet 1cac9e9013 fix(build): 修复版本号链接参数格式问题
- 修正了 ldflags 中版本号格式导致的编译错误
- 移除了版本字符串中的意外空格
- 确保版本号正确传递给主程序包变量
2026-04-28 15:08:21 +08:00
ygxbnet 7f0e4fe607 perf(api): 提高并发请求限制并添加配置写入日志
构建上传工具 / build-tool (push) Successful in 1m40s
- 将并发请求限制从10提升到500
- 在配置写入时添加调试日志输出
2026-04-28 15:03:13 +08:00
ygxbnet 199bd43b00 feat(api): 优化HTTP连接池和并发控制
- 增加IdleConnTimeout从30秒到30分钟
- 添加并发请求限制通道,最大同时请求数为10
- 实现InitConn函数用于预创建连接池
- 在UploadDataToServer中添加请求限流控制
- 优化资源清理逻辑,使用defer确保响应体关闭
- 重命名runtime包别名以避免冲突
- 在uploader中添加连接池初始化日志
- 添加panic恢复机制和错误处理
2026-04-28 15:00:15 +08:00
ygxbnet 4addc29b2c refactor(config): 添加配置写入的并发安全锁机制
- 引入 sync.Mutex 确保配置访问的线程安全性
- 在 WriteConfig 函数中实现读写锁定机制
- 防止多协程同时修改配置导致的数据竞争问题
2026-04-28 14:59:21 +08:00
ygxbnet 7face117f3 refactor(App): 解决程序运行时重复写入配置
- 移除 watch 监听器,改用事件驱动方式保存配置
- 添加 writeServerUrl、writeToken 等配置保存方法
- 在表单项上绑定 change 事件触发配置保存
- 移除不再使用的 nextTick 导入
- 统一配置保存逻辑到独立函数中
2026-04-28 14:53:19 +08:00
ygxbnet 602c4c8546 fix(uploader): 修复上传进度初始化逻辑
构建上传工具 / build-tool (push) Successful in 1m1s
- 在处理文件前先清除之前的进度记录
- 将循环变量名从 k, v 更改为 fileName, lines 提高可读性
- 移动进度初始化位置确保每个文件都有正确的进度跟踪
- 删除重复的进度清除操作避免潜在的数据丢失问题
2026-04-28 01:35:16 +08:00
13 changed files with 579 additions and 213 deletions
@@ -2,7 +2,7 @@ name: 构建上传工具
on: [ push ]
jobs:
build-tool:
build:
env:
RUNNER_TOOL_CACHE: /toolcache
runs-on: ubuntu-latest
@@ -52,9 +52,11 @@ jobs:
- name: 构建上传工具
run: |
set -x
git_hash=$(git rev-parse --short "$GITHUB_SHA")
build_date=$(TZ=Asia/Shanghai date +"%Y%m%d%H%M")
wails build \
-platform windows/amd64 \
-ldflags "-X 'main.version=$(TZ=Asia/Shanghai date +"%m%d%H%M")'" \
-ldflags "-X 'main.version=$build_date - $git_hash'" \
-o 上传工具.exe
- name: 上传构建文件
+8 -7
View File
@@ -45,12 +45,12 @@ func (a *App) startup(ctx context.Context) {
}()
//在程序启动时运行上传程序
if config.APPConfig.IsRunOnStart {
time.Sleep(time.Second)
a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
go uploader.StartLooking(a.uploaderCTX, &a.logChan, config.APPConfig.CheckDir)
a.isRun = true
}
//if config.APPConfig.IsRunOnStart {
// time.Sleep(time.Second)
// a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
// go uploader.StartUpload(a.uploaderCTX, &a.logChan)
// a.isRun = true
//}
}
// SelectPath 打开选择路径弹框
@@ -65,6 +65,7 @@ func (a *App) GetConfig() config.Config {
}
func (a *App) WriteConfig(key string, value any) {
fmt.Println("写入配置:", key, value)
config.WriteConfig(key, value)
}
@@ -73,7 +74,7 @@ func (a *App) StartUpload() {
return
}
a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
go uploader.StartLooking(a.uploaderCTX, &a.logChan, config.APPConfig.CheckDir)
go uploader.StartUpload(a.uploaderCTX, &a.logChan)
a.isRun = true
}
+1
View File
@@ -11,6 +11,7 @@
"type-check": "vue-tsc --build"
},
"dependencies": {
"@element-plus/icons-vue": "^2.3.2",
"element-plus": "^2.13.7",
"vue": "^3.5.32"
},
+1 -1
View File
@@ -1 +1 @@
aa914e6b4676ee4621ced7ad6d81c58c
05225657934ff66d822c925754c951bf
+3
View File
@@ -8,6 +8,9 @@ importers:
.:
dependencies:
'@element-plus/icons-vue':
specifier: ^2.3.2
version: 2.3.2(vue@3.5.33(typescript@6.0.3))
element-plus:
specifier: ^2.13.7
version: 2.13.7(typescript@6.0.3)(vue@3.5.33(typescript@6.0.3))
+87 -61
View File
@@ -1,24 +1,29 @@
<script lang="ts" setup>
import {ref, nextTick, watch, computed} from 'vue'
import {ElMessage} from 'element-plus'
import {ElMessageBox} from 'element-plus'
import {SelectPath, GetConfig, WriteConfig, StartUpload, StopUpload} from '../wailsjs/go/main/App';
import {computed, nextTick, ref} from 'vue'
import {ElMessage, ElMessageBox} from 'element-plus'
import ConfirmClearDialog from './components/ConfirmClearDialog.vue'
import {GetConfig, SelectPath, StartUpload, StopUpload, WriteConfig} from '../wailsjs/go/main/App';
import {config} from "../wailsjs/go/models.ts";
import Config = config.Config;
import {EventsOn, LogPrint} from "../wailsjs/runtime";
import {configModel} from "@/model.ts";
import Config = config.Config;
const serverUrl = ref('')
// const serverUrl = ref('')
const token = ref('')
const checkDir = ref('')
const concurrentFiles = ref(1)
const uploadThreads = ref(1)
const autoStart = ref(false)
// const autoStart = ref(false)
const isRunning = ref(false)
const logOutput = ref<string[]>([])
const logContentRef = ref<HTMLElement>()
const logRoll = ref(true)
const clearDialogVisible = ref(false)
const filesToClear = ref<string[]>([])
const noPromptClear = ref(false)
interface FileProgress {
name: string
uploaded: number
@@ -26,9 +31,8 @@ interface FileProgress {
percentage: number
}
const progressList = ref<FileProgress[]>([
// {name: '测试文件1.txt', uploaded: 100, total: 500, percentage: 20},
])
// {name: '测试文件1.txt', uploaded: 100, total: 500, percentage: 20},
const progressList = ref<FileProgress[]>([])
const sortedProgressList = computed(() => {
return [...progressList.value].sort((a, b) => {
@@ -42,7 +46,7 @@ const sortedProgressList = computed(() => {
})
const addLog = (msg: string) => {
logOutput.value.push(`[${new Date().toLocaleString()}]` + msg)
logOutput.value.push(`[${new Date().toLocaleString()}] ` + msg)
nextTick(() => {
if (logContentRef.value && logRoll.value) {
logContentRef.value.scrollTop = logContentRef.value.scrollHeight
@@ -53,21 +57,25 @@ const addLog = (msg: string) => {
const selectDirectory = () => {
// ElMessage.info('请手动输入检测目录路径')
SelectPath().then((path) => {
checkDir.value = path
if (path) {
checkDir.value = path
} else {
ElMessage.warning('未选择目录,不更改配置')
}
})
}
const startRun = () => {
if (!serverUrl.value) {
ElMessage.warning('请输入服务器地址')
return
}
// if (!serverUrl.value) {
// ElMessage.warning('请输入服务器地址')
// return
// }
if (!token.value) {
ElMessage.warning('请输入Token')
ElMessage.error('请输入Token')
return
}
if (!checkDir.value) {
ElMessage.warning('请选择检测目录')
ElMessage.error('请选择检测目录')
return
}
@@ -77,7 +85,7 @@ const startRun = () => {
const stopRun = () => {
addLog(`正在停止运行`)
StopUpload().then(() => {
ElMessage.info('已停止运行')
ElMessage.success('已停止运行')
})
}
@@ -93,15 +101,35 @@ const clearLog = () => {
})
}
// const writeServerUrl =() => {
// WriteConfig("url", serverUrl.value)
// }
const writeToken = () => {
WriteConfig(configModel.Token, token.value)
}
const writeCheckDir = () => {
WriteConfig(configModel.CheckDir, checkDir.value)
}
const writeConcurrentFiles = () => {
WriteConfig(configModel.HandleFileCount, concurrentFiles.value)
}
const writeUploadThreads = () => {
WriteConfig(configModel.ThreadCount, uploadThreads.value)
}
// const writeAutoStart = () => {
// WriteConfig("is-run-on-start", autoStart.value)
// }
// 加载配置
try {
GetConfig().then((config: Config) => {
serverUrl.value = config.url
// serverUrl.value = config.url
token.value = config.token
checkDir.value = config.check_dir
concurrentFiles.value = config.handle_file_count
uploadThreads.value = config.thread_count
autoStart.value = config.is_run_on_start
// autoStart.value = config.is_run_on_start
noPromptClear.value = config.clear_files_no_prompt
LogPrint(`[${new Date().toLocaleString()}] 配置已加载`)
})
@@ -109,70 +137,63 @@ try {
console.log(e)
}
watch(serverUrl, () => {
WriteConfig("url", serverUrl.value)
})
watch(token, () => {
WriteConfig("token", token.value)
})
watch(checkDir, () => {
WriteConfig("check-dir", checkDir.value)
})
watch(concurrentFiles, () => {
WriteConfig("handle-file-count", concurrentFiles.value)
})
watch(uploadThreads, () => {
WriteConfig("thread-count", uploadThreads.value)
})
watch(autoStart, () => {
WriteConfig("is-run-on-start", autoStart.value)
})
EventsOn("is-run", (run) => {
isRunning.value = run
})
EventsOn("progress", (progress) => {
progressList.value = progress
})
EventsOn("log", (msg) => {
addLog(msg)
})
try {
EventsOn("is-run", (run) => {
isRunning.value = run
})
EventsOn("progress", (progress) => {
progressList.value = progress
})
EventsOn("log", (msg) => {
addLog(msg)
})
EventsOn("clear-files", (files) => {
filesToClear.value = files
if (!noPromptClear.value) {
clearDialogVisible.value = true
}
})
} catch (e) {
console.log(e)
}
</script>
<template>
<div class="container">
<div class="left-panel">
<div class="form-item">
<label>服务器地址</label>
<el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning"/>
</div>
<!-- <div class="form-item">-->
<!-- <label>服务器地址</label>-->
<!-- <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
<!-- </div>-->
<div class="form-item">
<label>Token</label>
<el-input v-model="token" placeholder="请输入Token" :disabled="isRunning"/>
<el-input v-model="token" placeholder="请输入Token" :disabled="isRunning" @change="writeToken()"/>
</div>
<div class="form-item">
<label>检测目录</label>
<div class="dir-input">
<el-input v-model="checkDir" placeholder="请选择检测目录" :disabled="isRunning"/>
<el-input v-model="checkDir" placeholder="请选择检测目录" :disabled="isRunning" @change="writeCheckDir()"/>
<el-button @click="selectDirectory" :disabled="isRunning">选择目录</el-button>
</div>
</div>
<div class="form-item">
<label>同时处理文件数</label>
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"/>
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"
@change="writeConcurrentFiles()"/>
</div>
<div class="form-item">
<label>单文件上传线程</label>
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"/>
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"
@change="writeUploadThreads()"/>
</div>
<div class="form-item">
<el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning"/>
</div>
<!-- <div class="form-item">-->
<!-- <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
<!-- </div>-->
<div class="form-item">
<label>上传进度</label>
@@ -202,6 +223,11 @@ EventsOn("log", (msg) => {
</div>
</div>
</div>
<ConfirmClearDialog
v-model:visible="clearDialogVisible"
:file-list="filesToClear"
/>
</template>
<style scoped>
@@ -253,7 +279,7 @@ EventsOn("log", (msg) => {
.progress-list {
margin-top: 12px;
max-height: 160px;
max-height: 250px;
overflow-y: auto;
display: flex;
flex-direction: column;
@@ -0,0 +1,124 @@
<script lang="ts" setup>
import {ref} from 'vue'
import {WriteConfig} from "../../wailsjs/go/main/App";
import {configModel} from "@/model.ts";
import {EventsEmit} from "../../wailsjs/runtime";
import {InfoFilled} from '@element-plus/icons-vue'
const props = defineProps<{
visible: boolean
fileList: string[]
}>()
const emit = defineEmits<{
(e: 'update:visible', value: boolean): void
// (e: 'confirm', dontShowAgain: boolean): void
}>()
const dontShowAgain = ref(false)
const handleClose = () => {
emit('update:visible', false)
EventsEmit('confirm-clear-files', false)
}
const handleConfirm = () => {
// emit('confirm', dontShowAgain.value)
emit('update:visible', false)
WriteConfig(configModel.ClearFilesNoPrompt, dontShowAgain.value)
EventsEmit('confirm-clear-files', true)
}
</script>
<template>
<el-dialog
:model-value="visible"
width="600px"
:close-on-click-modal="false"
@update:model-value="(val: boolean) => emit('update:visible', val)"
@close="handleClose"
align-center
>
<template #header>
是否确认清空需要上传文件
</template>
<div class="hint-text">以下文件将会被清空并移动到 tmp 文件夹进行上传,您是否确认</div>
<div class="dialog-content">
<div class="file-list">
<div v-for="(file, index) in fileList" :key="index" class="file-item">
{{ file }}
</div>
<div v-if="fileList.length === 0" class="empty-text">暂无文件</div>
</div>
</div>
<template #footer>
<div class="dialog-footer">
<div class="dialog-footer-checkbox-container">
<el-checkbox v-model="dontShowAgain" label="下次清空文件不再弹出此弹窗确认"/>
<el-tooltip content="此弹窗会在首次运行时弹出您可以选择下次不再弹出此弹窗">
<el-icon>
<InfoFilled/>
</el-icon>
</el-tooltip>
</div>
<div>
<el-button @click="handleClose">取消</el-button>
<el-button type="primary" @click="handleConfirm">确认</el-button>
</div>
</div>
</template>
</el-dialog>
</template>
<style scoped>
.dialog-footer-checkbox-container {
display: flex;
align-items: center;
gap: 5px;
}
.dialog-footer-checkbox-container .el-icon {
color: #909399;
}
.dialog-content {
padding: 10px 0;
}
.hint-text {
font-size: 12px;
color: #909399;
}
.file-list {
max-height: 350px;
overflow-y: auto;
border: 1px solid #e4e7ed;
border-radius: 4px;
padding: 8px;
}
.file-item {
padding: 6px 8px;
font-size: 14px;
color: #606266;
border-bottom: 1px solid #f0f0f0;
}
.file-item:last-child {
border-bottom: none;
}
.empty-text {
text-align: center;
color: #909399;
padding: 20px 0;
}
.dialog-footer {
display: flex;
justify-content: space-between;
align-items: center;
}
</style>
+9
View File
@@ -0,0 +1,9 @@
export enum configModel {
Url = "url",
Token = "token",
ThreadCount = "thread-count",
HandleFileCount = "handle-file-count",
IsRunOnStart = "is-run-on-start",
CheckDir = "check-dir",
ClearFilesNoPrompt = "clear-files-no-prompt",
}
+2
View File
@@ -7,6 +7,7 @@ export namespace config {
handle_file_count: number;
is_run_on_start: boolean;
check_dir: string;
clear_files_no_prompt: boolean;
static createFrom(source: any = {}) {
return new Config(source);
@@ -20,6 +21,7 @@ export namespace config {
this.handle_file_count = source["handle_file_count"];
this.is_run_on_start = source["is_run_on_start"];
this.check_dir = source["check_dir"];
this.clear_files_no_prompt = source["clear_files_no_prompt"];
}
}
+42 -3
View File
@@ -3,9 +3,11 @@ package api
import (
"context"
"dypid-client/internal/config"
"fmt"
"io"
"net/http"
"net/url"
"sync"
"time"
)
@@ -13,12 +15,47 @@ var httpClient = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 500,
MaxIdleConnsPerHost: 500,
IdleConnTimeout: 30 * time.Second,
IdleConnTimeout: 30 * time.Minute,
},
Timeout: 30 * time.Second,
}
var limit chan struct{}
func init() {
//限制同时请求数为500
limit = make(chan struct{}, 500)
}
// InitConn 创建连接池
func InitConn() {
wg := &sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Go(func() {
for i := 0; i < 50; i++ {
resp, err := httpClient.Get(config.APPConfig.Url + "/api/test")
if err != nil {
fmt.Println(err)
return
}
defer func() {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
}
})
}
wg.Wait()
}
func UploadDataToServer(ctx context.Context, data string) error {
limit <- struct{}{}
defer func() {
<-limit
}()
params := url.Values{}
params.Set("token", config.APPConfig.Token)
params.Set("data", data)
@@ -38,8 +75,10 @@ func UploadDataToServer(ctx context.Context, data string) error {
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
defer func() {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
return err
}
+26 -18
View File
@@ -2,40 +2,45 @@ package config
import (
"fmt"
"sync"
"github.com/fsnotify/fsnotify"
"github.com/spf13/viper"
)
type Config struct {
Url string `json:"url" mapstructure:"url"`
Token string `json:"token" mapstructure:"token"`
ThreadCount int `json:"thread_count" mapstructure:"thread-count"`
HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"`
IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"`
CheckDir string `json:"check_dir" mapstructure:"check-dir"`
Url string `json:"url" mapstructure:"url"`
Token string `json:"token" mapstructure:"token"`
ThreadCount int `json:"thread_count" mapstructure:"thread-count"`
HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"`
IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"`
CheckDir string `json:"check_dir" mapstructure:"check-dir"`
ClearFilesNoPrompt bool `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"`
}
var APPConfig Config
var configMu sync.Mutex
const (
Url = "url"
Token = "token"
ThreadCount = "thread-count"
HandleFileCount = "handle-file-count"
IsRunOnStart = "is-run-on-start"
CheckDir = "check-dir"
Url = "url"
Token = "token"
ThreadCount = "thread-count"
HandleFileCount = "handle-file-count"
IsRunOnStart = "is-run-on-start"
CheckDir = "check-dir"
ClearFilesNoPrompt = "clear-files-no-prompt"
)
func InitConfig() {
// 设置默认配置
defaultConfig := Config{
Url: "http://127.0.0.1:8080",
Token: "",
ThreadCount: 10,
HandleFileCount: 25,
IsRunOnStart: false,
CheckDir: "",
Url: "http://127.0.0.1:8080",
Token: "",
ThreadCount: 10,
HandleFileCount: 25,
IsRunOnStart: false,
CheckDir: "",
ClearFilesNoPrompt: false,
}
viper.SetDefault(Url, defaultConfig.Url)
viper.SetDefault(Token, defaultConfig.Token)
@@ -43,6 +48,7 @@ func InitConfig() {
viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)
viper.SetDefault(CheckDir, defaultConfig.CheckDir)
viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt)
//设置配置文件名和路径 ./config.toml
viper.AddConfigPath(".")
@@ -70,6 +76,8 @@ func InitConfig() {
}
func WriteConfig(key string, value any) {
configMu.Lock()
viper.Set(key, value)
viper.WriteConfig()
configMu.Unlock()
}
+271 -120
View File
@@ -7,21 +7,21 @@ import (
"dypid-client/internal/api"
"dypid-client/internal/config"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/wailsapp/wails/v2/pkg/runtime"
wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime"
"golang.org/x/sync/errgroup"
)
var progress sync.Map
type Task struct {
type fileInfo struct {
FilePath string
FileLines int
}
@@ -33,7 +33,9 @@ type Progress struct {
Percentage int `json:"percentage"`
}
func StartLooking(ctx context.Context, logChan *chan string, lookingPath string) {
var progress sync.Map
func StartUpload(ctx context.Context, logChan *chan string) {
AddLog(logChan, "===============================================")
AddLog(logChan, `服务器: `+config.APPConfig.Url)
AddLog(logChan, `Token: `+config.APPConfig.Token)
@@ -42,7 +44,12 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
AddLog(logChan, `单文件上传线程: `+strconv.Itoa(config.APPConfig.ThreadCount))
AddLog(logChan, "===============================================")
AddLog(logChan, "正在创建连接池(连接池可避免首次大量上传时出现网络错误)")
api.InitConn()
AddLog(logChan, "创建连接池完成,开始运行程序")
progress.Clear()
//推送上传进度
go func() {
for {
@@ -50,153 +57,276 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
case <-ctx.Done():
return
default:
time.Sleep(250 * time.Millisecond)
var pg []Progress
progress.Range(func(key, value any) bool {
p := value.(Progress)
pg = append(pg, p)
return true
})
runtime.EventsEmit(ctx, "progress", pg)
}
var pg []Progress
progress.Range(func(_, value any) bool {
pg = append(pg, value.(Progress))
return true
})
wailsruntime.EventsEmit(ctx, "progress", pg)
time.Sleep(250 * time.Millisecond)
}
}()
//开启上传程序
for {
uploadData(ctx, logChan, lookingPath)
uploadData(ctx, logChan)
select {
case <-time.After(time.Minute):
case <-ctx.Done():
return
case <-time.After(time.Minute):
}
}
}
func uploadData(ctx context.Context, logChan *chan string, lookingPath string) {
var path = "./"
if lookingPath != "" {
path = lookingPath
}
//获取文件列表
files, err := getTxtFiles(path)
if err != nil {
AddLog(logChan, "获取文件列表失败:"+err.Error())
return
}
if files == nil {
return
}
func uploadData(ctx context.Context, logChan *chan string) {
start := time.Now()
// 获取检测目录
var checkPath = "./"
if config.APPConfig.CheckDir != "" {
checkPath = config.APPConfig.CheckDir
}
//要上传的文件路径字符串数组
var files []string
//先检测tmp目录有没有残余文件
os.Mkdir("./tmp", os.ModePerm)
tmpFiles, err := getTxtFiles("./tmp")
if err != nil {
AddLog(logChan, "获取 tmp 文件列表失败:"+err.Error())
}
//tmp有文件,优先上传(else:tmp没文件扫描指定文件夹,并复制文件到tmp)
if tmpFiles != nil {
AddLog(logChan, "当前 tmp 目录下还有未上传完成文件,将优先上传 tmp 目录文件")
files = tmpFiles
} else {
//tmp没文件,扫描指定文件夹
f, err := getTxtFiles(checkPath)
if err != nil {
AddLog(logChan, "获取文件列表失败:"+err.Error())
return
}
//指定文件夹没文件,退出函数
if f == nil {
return
}
//是否向用户提示清空文件,并复制文件到tmp
if config.APPConfig.ClearFilesNoPrompt {
//不用提示直接复制文件到tmp
for _, p := range f {
select {
case <-ctx.Done():
return
default:
}
err := copyFile(p, "./tmp/"+filepath.Base(p))
if err != nil {
AddLog(logChan, "复制文件失败:"+err.Error())
} else {
files = append(files, "./tmp/"+filepath.Base(p))
err := os.Truncate(p, 0)
if err != nil {
AddLog(logChan, "清空文件失败:"+err.Error())
}
}
}
} else {
//提示用户
wailsruntime.EventsEmit(ctx, "clear-files", f)
confirm := make(chan bool)
wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) {
confirm <- optionalData[0].(bool)
})
if <-confirm {
for _, p := range f {
select {
case <-ctx.Done():
return
default:
}
err := copyFile(p, "./tmp/"+filepath.Base(p))
if err != nil {
AddLog(logChan, "复制文件失败:"+err.Error())
} else {
files = append(files, "./tmp/"+filepath.Base(p))
err := os.Truncate(p, 0)
if err != nil {
AddLog(logChan, "清空文件失败:"+err.Error())
}
}
}
} else {
AddLog(logChan, "已取消上传,1分钟后再运行")
return
}
}
}
//检测到文件
//统计文件行数
fileLines := make(map[string]int)
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
var filesInfo = make(map[string]fileInfo)
isAllEmpty := true
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
for _, filePath := range files {
select {
case <-ctx.Done():
return
default:
file, err := os.Open(filePath)
if err != nil {
AddLog(logChan, "打开文件失败:"+err.Error())
}
// 使用 bufio.Scanner 逐行读取
scanner := bufio.NewScanner(file)
lineCount := 0
for scanner.Scan() {
lineCount++
}
err = file.Close()
if lineCount == 0 {
continue
}
fileLines[filepath.Base(filePath)] = lineCount
isAllEmpty = false
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
progress.Store(filepath.Base(filePath), Progress{FileName: filepath.Base(filePath), Total: lineCount, Uploaded: 0, Percentage: 0})
}
file, err := os.Open(filePath)
if err != nil {
AddLog(logChan, "打开文件失败:"+err.Error())
}
// 使用 bufio.Scanner 逐行读取
scanner := bufio.NewScanner(file)
lineCount := 0
for scanner.Scan() {
lineCount++
}
err = file.Close()
if lineCount == 0 {
continue
}
filesInfo[filepath.Base(filePath)] = fileInfo{
FilePath: filePath,
FileLines: lineCount,
}
isAllEmpty = false
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
}
if isAllEmpty {
AddLog(logChan, "所有文件都为空,不进行上传")
return
}
//添加文件上传任务参数(文件路径,文件行数)
var tasks []Task
for k, v := range fileLines {
tasks = append(tasks, Task{FilePath: path + "/" + k, FileLines: v})
//刷新文件上传进度
progress.Clear()
for fileName, info := range filesInfo {
progress.Store(fileName,
Progress{
FileName: fileName,
Total: info.FileLines,
Uploaded: 0,
Percentage: 0,
},
)
}
// 使用 errgroup 控制同时处理的文件数,并开始上传文件任务
g, egctx := errgroup.WithContext(ctx)
g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
// 执行所有任务
for _, task := range tasks {
// 设置同时处理文件数
g.SetLimit(config.APPConfig.HandleFileCount)
// 执行文件上传任务参数(文件路径,文件行数)
for fileName, info := range filesInfo {
select {
case <-egctx.Done():
return
default:
g.Go(func() error {
select {
case <-egctx.Done():
return egctx.Err()
default:
AddLog(logChan, "正在上传文件:"+filepath.Base(task.FilePath))
processFile(egctx, logChan, task.FilePath, task.FileLines)
select {
case <-egctx.Done():
return egctx.Err()
default:
//上传完成,清空文件
err := os.Truncate(task.FilePath, 0)
if err != nil {
AddLog(logChan, "清空文件失败:"+err.Error())
}
return nil
}
}
})
}
g.Go(func() error {
select {
case <-egctx.Done():
return egctx.Err()
default:
}
AddLog(logChan, "正在上传文件:"+fileName)
processFile(egctx, logChan, info.FilePath, info.FileLines)
select {
case <-egctx.Done():
return egctx.Err()
default:
}
//上传完成,删除缓存文件
err := os.Remove(info.FilePath)
if err != nil {
AddLog(logChan, "删除缓存文件失败:"+err.Error())
}
return nil
})
}
select {
case <-ctx.Done():
return
default:
// 等待所有任务完成
if err := g.Wait(); err != nil {
AddLog(logChan, fmt.Sprintf("任务执行出错: %v", err))
} else {
AddLog(logChan, "所有任务执行完成!")
}
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
progress.Clear()
}
// 等待所有任务完成
g.Wait()
AddLog(logChan, "所有任务执行完成!")
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
}
// 获取目录中的所有txt文件
// copyFile 快速拷贝文件 src -> dst
func copyFile(src, dst string) error {
// 打开源文件
sourceFile, err := os.Open(src)
if err != nil {
return err
}
defer sourceFile.Close()
// 创建目标文件
destFile, err := os.Create(dst)
if err != nil {
return err
}
defer destFile.Close()
// 核心:最快拷贝,底层使用操作系统零拷贝技术
_, err = io.Copy(destFile, sourceFile)
if err != nil {
return err
}
// 强制刷入磁盘,保证数据完整
return destFile.Sync()
}
// 获取目录中的所有txt文件(文件大小为0的不返回)
func getTxtFiles(dir string) (txtFiles []string, err error) {
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// 只处理普通文件,跳过目录
// 跳过目录,只处理普通文件
if !info.Mode().IsRegular() {
return nil
}
// 检查文件扩展名是否为.txt
if strings.ToLower(filepath.Ext(path)) == ".txt" {
txtFiles = append(txtFiles, path)
if info.Size() != 0 {
txtFiles = append(txtFiles, path)
}
}
return nil
@@ -205,8 +335,8 @@ func getTxtFiles(dir string) (txtFiles []string, err error) {
return txtFiles, err
}
// processFile 处理每个文件
func processFile(ctx context.Context, logChan *chan string, filePath string, fileLines int) {
var wg sync.WaitGroup
// 打开文件
file, err := os.Open(filePath)
if err != nil {
@@ -216,51 +346,70 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
defer file.Close()
// 创建行通道
lines := make(chan string, 100)
lines := make(chan string, 200)
var countLine int32 = 0
// 创建指定个worker同时处理文件上传
for i := 0; i < config.APPConfig.ThreadCount; i++ {
select {
case <-ctx.Done():
close(lines)
return
default:
wg.Go(func() {
processLines(ctx, logChan, &lines, i, filePath, &countLine)
})
}
go func() {
processLines(ctx, logChan, &lines, i, filePath, &countLine)
}()
}
// 读取文件并发送到通道
scanner := bufio.NewScanner(file)
go func() {
defer func() {
if r := recover(); r != nil {
_, f, l, _ := runtime.Caller(0)
fmt.Println("panic:", f+":"+strconv.Itoa(l), r)
}
}()
for scanner.Scan() {
select {
case <-ctx.Done():
return
default:
lines <- scanner.Text()
}
lines <- scanner.Text()
}
}()
// 等待所有行处理完成并推送进度
for int(countLine) != fileLines {
select {
case <-ctx.Done():
close(lines)
wg.Wait()
close(lines) //关闭processLines中的上传线程
return
default:
progress.Store(filepath.Base(filePath),
Progress{FileName: filepath.Base(filePath),
Total: fileLines, Uploaded: int(countLine),
Percentage: int(float64(countLine)/float64(fileLines)*100) + 1,
})
time.Sleep(500 * time.Millisecond)
}
}
close(lines)
wg.Wait()
progress.Store(filepath.Base(filePath),
Progress{
FileName: filepath.Base(filePath),
Total: fileLines,
Uploaded: int(countLine),
Percentage: int(float64(countLine) / float64(fileLines) * 100),
},
)
}
//上传完成,进度设为100
progress.Store(filepath.Base(filePath),
Progress{
FileName: filepath.Base(filePath),
Total: fileLines,
Uploaded: int(countLine),
Percentage: 100,
},
)
close(lines) //关闭processLines中的上传线程
if err := scanner.Err(); err != nil {
AddLog(logChan, fmt.Sprintf("读取文件 %s 错误: %v", filePath, err))
@@ -270,22 +419,24 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
AddLog(logChan, fmt.Sprintf("文件【%s】处理完成,共处理 %d 行数据", filepath.Base(filePath), countLine))
}
// processLines 处理接受到的每一行数据并上传(chan 管道接受数据)
func processLines(ctx context.Context, logChan *chan string, lines *chan string, workerID int, filePath string, countLine *int32) {
for line := range *lines {
select {
case <-ctx.Done():
return
default:
// 跳过空行
if strings.TrimSpace(line) == "" {
continue
}
// 上传数据
if err := api.UploadDataToServer(ctx, line); err != nil {
AddLog(logChan, fmt.Sprintf("Worker %d (文件 %s): 上传失败: %v", workerID, filepath.Base(filePath), err))
}
atomic.AddInt32(countLine, 1)
}
// 跳过空行
if strings.TrimSpace(line) == "" {
continue
}
// 上传数据
if err := api.UploadDataToServer(ctx, line); err != nil {
AddLog(logChan, fmt.Sprintf("Worker %d (文件 %s): 上传失败: %v", workerID, filepath.Base(filePath), err))
}
atomic.AddInt32(countLine, 1)
}
}
+1 -1
View File
@@ -22,7 +22,7 @@ func main() {
// Create application with options
err := wails.Run(&options.App{
Title: "dypid-client - 版本" + version,
Title: "dypid-client - 版本" + version,
Width: 1024,
Height: 768,
AssetServer: &assetserver.Options{