Compare commits
16 Commits
602c4c8546
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| b100fc9b32 | |||
| d426c16104 | |||
| 993814cdfa | |||
| 8ffa0531d6 | |||
| 3f6e999783 | |||
| 03e4e6f45b | |||
| 6bd82024d9 | |||
| 6952c33f16 | |||
| 3d2b3469cc | |||
| a528d6a877 | |||
| 73a7d26816 | |||
| 1cac9e9013 | |||
| 7f0e4fe607 | |||
| 199bd43b00 | |||
| 4addc29b2c | |||
| 7face117f3 |
@@ -2,7 +2,7 @@ name: 构建上传工具
|
||||
on: [ push ]
|
||||
|
||||
jobs:
|
||||
build-tool:
|
||||
build:
|
||||
env:
|
||||
RUNNER_TOOL_CACHE: /toolcache
|
||||
runs-on: ubuntu-latest
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
build_date=$(TZ=Asia/Shanghai date +"%Y%m%d%H%M")
|
||||
wails build \
|
||||
-platform windows/amd64 \
|
||||
-ldflags "-X 'main.version=$build_date-$git_hash'" \
|
||||
-ldflags "-X 'main.version=$build_date - $git_hash'" \
|
||||
-o 上传工具.exe
|
||||
|
||||
- name: 上传构建文件
|
||||
@@ -45,12 +45,12 @@ func (a *App) startup(ctx context.Context) {
|
||||
}()
|
||||
|
||||
//在程序启动时运行上传程序
|
||||
if config.APPConfig.IsRunOnStart {
|
||||
time.Sleep(time.Second)
|
||||
a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
|
||||
go uploader.StartLooking(a.uploaderCTX, &a.logChan, config.APPConfig.CheckDir)
|
||||
a.isRun = true
|
||||
}
|
||||
//if config.APPConfig.IsRunOnStart {
|
||||
// time.Sleep(time.Second)
|
||||
// a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
|
||||
// go uploader.StartUpload(a.uploaderCTX, &a.logChan)
|
||||
// a.isRun = true
|
||||
//}
|
||||
}
|
||||
|
||||
// SelectPath 打开选择路径弹框
|
||||
@@ -65,6 +65,7 @@ func (a *App) GetConfig() config.Config {
|
||||
}
|
||||
|
||||
func (a *App) WriteConfig(key string, value any) {
|
||||
fmt.Println("写入配置:", key, value)
|
||||
config.WriteConfig(key, value)
|
||||
}
|
||||
|
||||
@@ -73,7 +74,7 @@ func (a *App) StartUpload() {
|
||||
return
|
||||
}
|
||||
a.uploaderCTX, a.uploaderCancel = context.WithCancel(a.ctx)
|
||||
go uploader.StartLooking(a.uploaderCTX, &a.logChan, config.APPConfig.CheckDir)
|
||||
go uploader.StartUpload(a.uploaderCTX, &a.logChan)
|
||||
a.isRun = true
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"type-check": "vue-tsc --build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.3.2",
|
||||
"element-plus": "^2.13.7",
|
||||
"vue": "^3.5.32"
|
||||
},
|
||||
|
||||
@@ -1 +1 @@
|
||||
aa914e6b4676ee4621ced7ad6d81c58c
|
||||
05225657934ff66d822c925754c951bf
|
||||
Generated
+3
@@ -8,6 +8,9 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@element-plus/icons-vue':
|
||||
specifier: ^2.3.2
|
||||
version: 2.3.2(vue@3.5.33(typescript@6.0.3))
|
||||
element-plus:
|
||||
specifier: ^2.13.7
|
||||
version: 2.13.7(typescript@6.0.3)(vue@3.5.33(typescript@6.0.3))
|
||||
|
||||
+83
-57
@@ -1,24 +1,29 @@
|
||||
<script lang="ts" setup>
|
||||
import {ref, nextTick, watch, computed} from 'vue'
|
||||
import {ElMessage} from 'element-plus'
|
||||
import {ElMessageBox} from 'element-plus'
|
||||
import {SelectPath, GetConfig, WriteConfig, StartUpload, StopUpload} from '../wailsjs/go/main/App';
|
||||
import {computed, nextTick, ref} from 'vue'
|
||||
import {ElMessage, ElMessageBox} from 'element-plus'
|
||||
import ConfirmClearDialog from './components/ConfirmClearDialog.vue'
|
||||
import {GetConfig, SelectPath, StartUpload, StopUpload, WriteConfig} from '../wailsjs/go/main/App';
|
||||
import {config} from "../wailsjs/go/models.ts";
|
||||
import Config = config.Config;
|
||||
import {EventsOn, LogPrint} from "../wailsjs/runtime";
|
||||
import {configModel} from "@/model.ts";
|
||||
import Config = config.Config;
|
||||
|
||||
const serverUrl = ref('')
|
||||
// const serverUrl = ref('')
|
||||
const token = ref('')
|
||||
const checkDir = ref('')
|
||||
const concurrentFiles = ref(1)
|
||||
const uploadThreads = ref(1)
|
||||
const autoStart = ref(false)
|
||||
// const autoStart = ref(false)
|
||||
|
||||
const isRunning = ref(false)
|
||||
const logOutput = ref<string[]>([])
|
||||
const logContentRef = ref<HTMLElement>()
|
||||
const logRoll = ref(true)
|
||||
|
||||
const clearDialogVisible = ref(false)
|
||||
const filesToClear = ref<string[]>([])
|
||||
const noPromptClear = ref(false)
|
||||
|
||||
interface FileProgress {
|
||||
name: string
|
||||
uploaded: number
|
||||
@@ -26,9 +31,8 @@ interface FileProgress {
|
||||
percentage: number
|
||||
}
|
||||
|
||||
const progressList = ref<FileProgress[]>([
|
||||
// {name: '测试文件1.txt', uploaded: 100, total: 500, percentage: 20},
|
||||
])
|
||||
// {name: '测试文件1.txt', uploaded: 100, total: 500, percentage: 20},
|
||||
const progressList = ref<FileProgress[]>([])
|
||||
|
||||
const sortedProgressList = computed(() => {
|
||||
return [...progressList.value].sort((a, b) => {
|
||||
@@ -42,7 +46,7 @@ const sortedProgressList = computed(() => {
|
||||
})
|
||||
|
||||
const addLog = (msg: string) => {
|
||||
logOutput.value.push(`[${new Date().toLocaleString()}]` + msg)
|
||||
logOutput.value.push(`[${new Date().toLocaleString()}] ` + msg)
|
||||
nextTick(() => {
|
||||
if (logContentRef.value && logRoll.value) {
|
||||
logContentRef.value.scrollTop = logContentRef.value.scrollHeight
|
||||
@@ -53,21 +57,25 @@ const addLog = (msg: string) => {
|
||||
const selectDirectory = () => {
|
||||
// ElMessage.info('请手动输入检测目录路径')
|
||||
SelectPath().then((path) => {
|
||||
if (path) {
|
||||
checkDir.value = path
|
||||
} else {
|
||||
ElMessage.warning('未选择目录,不更改配置')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const startRun = () => {
|
||||
if (!serverUrl.value) {
|
||||
ElMessage.warning('请输入服务器地址')
|
||||
return
|
||||
}
|
||||
// if (!serverUrl.value) {
|
||||
// ElMessage.warning('请输入服务器地址')
|
||||
// return
|
||||
// }
|
||||
if (!token.value) {
|
||||
ElMessage.warning('请输入Token')
|
||||
ElMessage.error('请输入Token')
|
||||
return
|
||||
}
|
||||
if (!checkDir.value) {
|
||||
ElMessage.warning('请选择检测目录')
|
||||
ElMessage.error('请选择检测目录')
|
||||
return
|
||||
}
|
||||
|
||||
@@ -77,7 +85,7 @@ const startRun = () => {
|
||||
const stopRun = () => {
|
||||
addLog(`正在停止运行`)
|
||||
StopUpload().then(() => {
|
||||
ElMessage.info('已停止运行')
|
||||
ElMessage.success('已停止运行')
|
||||
})
|
||||
}
|
||||
|
||||
@@ -93,15 +101,35 @@ const clearLog = () => {
|
||||
})
|
||||
}
|
||||
|
||||
// const writeServerUrl =() => {
|
||||
// WriteConfig("url", serverUrl.value)
|
||||
// }
|
||||
const writeToken = () => {
|
||||
WriteConfig(configModel.Token, token.value)
|
||||
}
|
||||
const writeCheckDir = () => {
|
||||
WriteConfig(configModel.CheckDir, checkDir.value)
|
||||
}
|
||||
const writeConcurrentFiles = () => {
|
||||
WriteConfig(configModel.HandleFileCount, concurrentFiles.value)
|
||||
}
|
||||
const writeUploadThreads = () => {
|
||||
WriteConfig(configModel.ThreadCount, uploadThreads.value)
|
||||
}
|
||||
// const writeAutoStart = () => {
|
||||
// WriteConfig("is-run-on-start", autoStart.value)
|
||||
// }
|
||||
|
||||
// 加载配置
|
||||
try {
|
||||
GetConfig().then((config: Config) => {
|
||||
serverUrl.value = config.url
|
||||
// serverUrl.value = config.url
|
||||
token.value = config.token
|
||||
checkDir.value = config.check_dir
|
||||
concurrentFiles.value = config.handle_file_count
|
||||
uploadThreads.value = config.thread_count
|
||||
autoStart.value = config.is_run_on_start
|
||||
// autoStart.value = config.is_run_on_start
|
||||
noPromptClear.value = config.clear_files_no_prompt
|
||||
|
||||
LogPrint(`[${new Date().toLocaleString()}] 配置已加载`)
|
||||
})
|
||||
@@ -109,70 +137,63 @@ try {
|
||||
console.log(e)
|
||||
}
|
||||
|
||||
watch(serverUrl, () => {
|
||||
WriteConfig("url", serverUrl.value)
|
||||
})
|
||||
watch(token, () => {
|
||||
WriteConfig("token", token.value)
|
||||
})
|
||||
watch(checkDir, () => {
|
||||
WriteConfig("check-dir", checkDir.value)
|
||||
})
|
||||
watch(concurrentFiles, () => {
|
||||
WriteConfig("handle-file-count", concurrentFiles.value)
|
||||
})
|
||||
watch(uploadThreads, () => {
|
||||
WriteConfig("thread-count", uploadThreads.value)
|
||||
})
|
||||
watch(autoStart, () => {
|
||||
WriteConfig("is-run-on-start", autoStart.value)
|
||||
})
|
||||
|
||||
EventsOn("is-run", (run) => {
|
||||
try {
|
||||
EventsOn("is-run", (run) => {
|
||||
isRunning.value = run
|
||||
})
|
||||
EventsOn("progress", (progress) => {
|
||||
})
|
||||
EventsOn("progress", (progress) => {
|
||||
progressList.value = progress
|
||||
})
|
||||
EventsOn("log", (msg) => {
|
||||
})
|
||||
EventsOn("log", (msg) => {
|
||||
addLog(msg)
|
||||
})
|
||||
})
|
||||
EventsOn("clear-files", (files) => {
|
||||
filesToClear.value = files
|
||||
if (!noPromptClear.value) {
|
||||
clearDialogVisible.value = true
|
||||
}
|
||||
})
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="container">
|
||||
<div class="left-panel">
|
||||
<div class="form-item">
|
||||
<label>服务器地址</label>
|
||||
<el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning"/>
|
||||
</div>
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <label>服务器地址</label>-->
|
||||
<!-- <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
|
||||
<!-- </div>-->
|
||||
|
||||
<div class="form-item">
|
||||
<label>Token</label>
|
||||
<el-input v-model="token" placeholder="请输入Token" :disabled="isRunning"/>
|
||||
<el-input v-model="token" placeholder="请输入Token" :disabled="isRunning" @change="writeToken()"/>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label>检测目录</label>
|
||||
<div class="dir-input">
|
||||
<el-input v-model="checkDir" placeholder="请选择检测目录" :disabled="isRunning"/>
|
||||
<el-input v-model="checkDir" placeholder="请选择检测目录" :disabled="isRunning" @change="writeCheckDir()"/>
|
||||
<el-button @click="selectDirectory" :disabled="isRunning">选择目录</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label>同时处理文件数</label>
|
||||
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"/>
|
||||
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"
|
||||
@change="writeConcurrentFiles()"/>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label>单文件上传线程</label>
|
||||
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"/>
|
||||
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"
|
||||
@change="writeUploadThreads()"/>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning"/>
|
||||
</div>
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
|
||||
<!-- </div>-->
|
||||
|
||||
<div class="form-item">
|
||||
<label>上传进度</label>
|
||||
@@ -202,6 +223,11 @@ EventsOn("log", (msg) => {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ConfirmClearDialog
|
||||
v-model:visible="clearDialogVisible"
|
||||
:file-list="filesToClear"
|
||||
/>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
@@ -253,7 +279,7 @@ EventsOn("log", (msg) => {
|
||||
|
||||
.progress-list {
|
||||
margin-top: 12px;
|
||||
max-height: 160px;
|
||||
max-height: 250px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
<script lang="ts" setup>
|
||||
import {ref} from 'vue'
|
||||
import {WriteConfig} from "../../wailsjs/go/main/App";
|
||||
import {configModel} from "@/model.ts";
|
||||
import {EventsEmit} from "../../wailsjs/runtime";
|
||||
import {InfoFilled} from '@element-plus/icons-vue'
|
||||
|
||||
const props = defineProps<{
|
||||
visible: boolean
|
||||
fileList: string[]
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'update:visible', value: boolean): void
|
||||
// (e: 'confirm', dontShowAgain: boolean): void
|
||||
}>()
|
||||
|
||||
const dontShowAgain = ref(false)
|
||||
|
||||
const handleClose = () => {
|
||||
emit('update:visible', false)
|
||||
EventsEmit('confirm-clear-files', false)
|
||||
}
|
||||
|
||||
const handleConfirm = () => {
|
||||
// emit('confirm', dontShowAgain.value)
|
||||
emit('update:visible', false)
|
||||
WriteConfig(configModel.ClearFilesNoPrompt, dontShowAgain.value)
|
||||
EventsEmit('confirm-clear-files', true)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<el-dialog
|
||||
:model-value="visible"
|
||||
width="600px"
|
||||
:close-on-click-modal="false"
|
||||
@update:model-value="(val: boolean) => emit('update:visible', val)"
|
||||
@close="handleClose"
|
||||
align-center
|
||||
>
|
||||
<template #header>
|
||||
是否确认清空需要上传文件
|
||||
</template>
|
||||
<div class="hint-text">以下文件将会被清空并移动到 tmp 文件夹进行上传,您是否确认</div>
|
||||
<div class="dialog-content">
|
||||
<div class="file-list">
|
||||
<div v-for="(file, index) in fileList" :key="index" class="file-item">
|
||||
{{ file }}
|
||||
</div>
|
||||
<div v-if="fileList.length === 0" class="empty-text">暂无文件</div>
|
||||
</div>
|
||||
</div>
|
||||
<template #footer>
|
||||
<div class="dialog-footer">
|
||||
<div class="dialog-footer-checkbox-container">
|
||||
<el-checkbox v-model="dontShowAgain" label="下次清空文件不再弹出此弹窗确认"/>
|
||||
<el-tooltip content="此弹窗会在首次运行时弹出,您可以选择下次不再弹出此弹窗">
|
||||
<el-icon>
|
||||
<InfoFilled/>
|
||||
</el-icon>
|
||||
</el-tooltip>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<el-button @click="handleClose">取消</el-button>
|
||||
<el-button type="primary" @click="handleConfirm">确认</el-button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.dialog-footer-checkbox-container {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.dialog-footer-checkbox-container .el-icon {
|
||||
color: #909399;
|
||||
}
|
||||
|
||||
.dialog-content {
|
||||
padding: 10px 0;
|
||||
}
|
||||
|
||||
.hint-text {
|
||||
font-size: 12px;
|
||||
color: #909399;
|
||||
}
|
||||
|
||||
.file-list {
|
||||
max-height: 350px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #e4e7ed;
|
||||
border-radius: 4px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.file-item {
|
||||
padding: 6px 8px;
|
||||
font-size: 14px;
|
||||
color: #606266;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
.file-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.empty-text {
|
||||
text-align: center;
|
||||
color: #909399;
|
||||
padding: 20px 0;
|
||||
}
|
||||
|
||||
.dialog-footer {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,9 @@
|
||||
export enum configModel {
|
||||
Url = "url",
|
||||
Token = "token",
|
||||
ThreadCount = "thread-count",
|
||||
HandleFileCount = "handle-file-count",
|
||||
IsRunOnStart = "is-run-on-start",
|
||||
CheckDir = "check-dir",
|
||||
ClearFilesNoPrompt = "clear-files-no-prompt",
|
||||
}
|
||||
@@ -7,6 +7,7 @@ export namespace config {
|
||||
handle_file_count: number;
|
||||
is_run_on_start: boolean;
|
||||
check_dir: string;
|
||||
clear_files_no_prompt: boolean;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new Config(source);
|
||||
@@ -20,6 +21,7 @@ export namespace config {
|
||||
this.handle_file_count = source["handle_file_count"];
|
||||
this.is_run_on_start = source["is_run_on_start"];
|
||||
this.check_dir = source["check_dir"];
|
||||
this.clear_files_no_prompt = source["clear_files_no_prompt"];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+40
-1
@@ -3,9 +3,11 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"dypid-client/internal/config"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -13,12 +15,47 @@ var httpClient = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 500,
|
||||
MaxIdleConnsPerHost: 500,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
IdleConnTimeout: 30 * time.Minute,
|
||||
},
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
var limit chan struct{}
|
||||
|
||||
func init() {
|
||||
//限制同时请求数为500
|
||||
limit = make(chan struct{}, 500)
|
||||
}
|
||||
|
||||
// InitConn 创建连接池
|
||||
func InitConn() {
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Go(func() {
|
||||
for i := 0; i < 50; i++ {
|
||||
resp, err := httpClient.Get(config.APPConfig.Url + "/api/test")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func UploadDataToServer(ctx context.Context, data string) error {
|
||||
limit <- struct{}{}
|
||||
defer func() {
|
||||
<-limit
|
||||
}()
|
||||
|
||||
params := url.Values{}
|
||||
params.Set("token", config.APPConfig.Token)
|
||||
params.Set("data", data)
|
||||
@@ -38,8 +75,10 @@ func UploadDataToServer(ctx context.Context, data string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/spf13/viper"
|
||||
@@ -14,9 +15,11 @@ type Config struct {
|
||||
HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"`
|
||||
IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"`
|
||||
CheckDir string `json:"check_dir" mapstructure:"check-dir"`
|
||||
ClearFilesNoPrompt bool `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"`
|
||||
}
|
||||
|
||||
var APPConfig Config
|
||||
var configMu sync.Mutex
|
||||
|
||||
const (
|
||||
Url = "url"
|
||||
@@ -25,6 +28,7 @@ const (
|
||||
HandleFileCount = "handle-file-count"
|
||||
IsRunOnStart = "is-run-on-start"
|
||||
CheckDir = "check-dir"
|
||||
ClearFilesNoPrompt = "clear-files-no-prompt"
|
||||
)
|
||||
|
||||
func InitConfig() {
|
||||
@@ -36,6 +40,7 @@ func InitConfig() {
|
||||
HandleFileCount: 25,
|
||||
IsRunOnStart: false,
|
||||
CheckDir: "",
|
||||
ClearFilesNoPrompt: false,
|
||||
}
|
||||
viper.SetDefault(Url, defaultConfig.Url)
|
||||
viper.SetDefault(Token, defaultConfig.Token)
|
||||
@@ -43,6 +48,7 @@ func InitConfig() {
|
||||
viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
|
||||
viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)
|
||||
viper.SetDefault(CheckDir, defaultConfig.CheckDir)
|
||||
viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt)
|
||||
|
||||
//设置配置文件名和路径 ./config.toml
|
||||
viper.AddConfigPath(".")
|
||||
@@ -70,6 +76,8 @@ func InitConfig() {
|
||||
}
|
||||
|
||||
func WriteConfig(key string, value any) {
|
||||
configMu.Lock()
|
||||
viper.Set(key, value)
|
||||
viper.WriteConfig()
|
||||
configMu.Unlock()
|
||||
}
|
||||
|
||||
+218
-67
@@ -7,21 +7,21 @@ import (
|
||||
"dypid-client/internal/api"
|
||||
"dypid-client/internal/config"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var progress sync.Map
|
||||
|
||||
type Task struct {
|
||||
type fileInfo struct {
|
||||
FilePath string
|
||||
FileLines int
|
||||
}
|
||||
@@ -33,7 +33,9 @@ type Progress struct {
|
||||
Percentage int `json:"percentage"`
|
||||
}
|
||||
|
||||
func StartLooking(ctx context.Context, logChan *chan string, lookingPath string) {
|
||||
var progress sync.Map
|
||||
|
||||
func StartUpload(ctx context.Context, logChan *chan string) {
|
||||
AddLog(logChan, "===============================================")
|
||||
AddLog(logChan, `服务器: `+config.APPConfig.Url)
|
||||
AddLog(logChan, `Token: `+config.APPConfig.Token)
|
||||
@@ -42,7 +44,12 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
|
||||
AddLog(logChan, `单文件上传线程: `+strconv.Itoa(config.APPConfig.ThreadCount))
|
||||
AddLog(logChan, "===============================================")
|
||||
|
||||
AddLog(logChan, "正在创建连接池(连接池可避免首次大量上传时出现网络错误)")
|
||||
api.InitConn()
|
||||
AddLog(logChan, "创建连接池完成,开始运行程序")
|
||||
|
||||
progress.Clear()
|
||||
|
||||
//推送上传进度
|
||||
go func() {
|
||||
for {
|
||||
@@ -50,60 +57,143 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
|
||||
var pg []Progress
|
||||
progress.Range(func(key, value any) bool {
|
||||
p := value.(Progress)
|
||||
pg = append(pg, p)
|
||||
progress.Range(func(_, value any) bool {
|
||||
pg = append(pg, value.(Progress))
|
||||
return true
|
||||
})
|
||||
runtime.EventsEmit(ctx, "progress", pg)
|
||||
}
|
||||
wailsruntime.EventsEmit(ctx, "progress", pg)
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
|
||||
//开启上传程序
|
||||
for {
|
||||
uploadData(ctx, logChan, lookingPath)
|
||||
uploadData(ctx, logChan)
|
||||
|
||||
select {
|
||||
case <-time.After(time.Minute):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Minute):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func uploadData(ctx context.Context, logChan *chan string, lookingPath string) {
|
||||
var path = "./"
|
||||
if lookingPath != "" {
|
||||
path = lookingPath
|
||||
func uploadData(ctx context.Context, logChan *chan string) {
|
||||
start := time.Now()
|
||||
|
||||
// 获取检测目录
|
||||
var checkPath = "./"
|
||||
if config.APPConfig.CheckDir != "" {
|
||||
checkPath = config.APPConfig.CheckDir
|
||||
}
|
||||
|
||||
//获取文件列表
|
||||
files, err := getTxtFiles(path)
|
||||
//要上传的文件路径字符串数组
|
||||
var files []string
|
||||
|
||||
//先检测tmp目录有没有残余文件
|
||||
os.Mkdir("./tmp", os.ModePerm)
|
||||
tmpFiles, err := getTxtFiles("./tmp")
|
||||
if err != nil {
|
||||
AddLog(logChan, "获取 tmp 文件列表失败:"+err.Error())
|
||||
}
|
||||
|
||||
//tmp有文件,优先上传(else:tmp没文件扫描指定文件夹,并复制文件到tmp)
|
||||
if tmpFiles != nil {
|
||||
AddLog(logChan, "当前 tmp 目录下还有未上传完成文件,将优先上传 tmp 目录文件")
|
||||
files = tmpFiles
|
||||
} else {
|
||||
//tmp没文件,扫描指定文件夹
|
||||
f, err := getTxtFiles(checkPath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "获取文件列表失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if files == nil {
|
||||
|
||||
//指定文件夹没文件,退出函数
|
||||
if f == nil {
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
//是否向用户提示清空文件,并复制文件到tmp
|
||||
if config.APPConfig.ClearFilesNoPrompt {
|
||||
//不用提示直接复制文件到tmp
|
||||
for _, p := range f {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
||||
if err != nil {
|
||||
AddLog(logChan, "复制文件失败:"+err.Error())
|
||||
} else {
|
||||
files = append(files, "./tmp/"+filepath.Base(p))
|
||||
err := os.Truncate(p, 0)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//提示用户
|
||||
wailsruntime.EventsEmit(ctx, "clear-files", f)
|
||||
|
||||
confirm := make(chan bool)
|
||||
|
||||
wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) {
|
||||
confirm <- optionalData[0].(bool)
|
||||
})
|
||||
|
||||
if <-confirm {
|
||||
for _, p := range f {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
||||
if err != nil {
|
||||
AddLog(logChan, "复制文件失败:"+err.Error())
|
||||
} else {
|
||||
files = append(files, "./tmp/"+filepath.Base(p))
|
||||
err := os.Truncate(p, 0)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
AddLog(logChan, "已取消上传,1分钟后再运行")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//检测到文件
|
||||
//统计文件行数
|
||||
fileLines := make(map[string]int)
|
||||
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
||||
var filesInfo = make(map[string]fileInfo)
|
||||
|
||||
isAllEmpty := true
|
||||
|
||||
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
||||
for _, filePath := range files {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "打开文件失败:"+err.Error())
|
||||
}
|
||||
|
||||
// 使用 bufio.Scanner 逐行读取
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineCount := 0
|
||||
@@ -115,89 +205,129 @@ func uploadData(ctx context.Context, logChan *chan string, lookingPath string) {
|
||||
if lineCount == 0 {
|
||||
continue
|
||||
}
|
||||
fileLines[filepath.Base(filePath)] = lineCount
|
||||
filesInfo[filepath.Base(filePath)] = fileInfo{
|
||||
FilePath: filePath,
|
||||
FileLines: lineCount,
|
||||
}
|
||||
|
||||
isAllEmpty = false
|
||||
|
||||
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
|
||||
}
|
||||
}
|
||||
|
||||
if isAllEmpty {
|
||||
AddLog(logChan, "所有文件都为空,不进行上传")
|
||||
return
|
||||
}
|
||||
|
||||
//刷新文件上传进度
|
||||
progress.Clear()
|
||||
//添加文件上传任务参数(文件路径,文件行数)
|
||||
var tasks []Task
|
||||
for fileName, lines := range fileLines {
|
||||
tasks = append(tasks, Task{FilePath: path + "/" + fileName, FileLines: lines})
|
||||
progress.Store(fileName, Progress{FileName: fileName, Total: lines, Uploaded: 0, Percentage: 0})
|
||||
for fileName, info := range filesInfo {
|
||||
progress.Store(fileName,
|
||||
Progress{
|
||||
FileName: fileName,
|
||||
Total: info.FileLines,
|
||||
Uploaded: 0,
|
||||
Percentage: 0,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// 使用 errgroup 控制同时处理的文件数,并开始上传文件任务
|
||||
g, egctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
|
||||
// 执行所有任务
|
||||
for _, task := range tasks {
|
||||
// 设置同时处理文件数
|
||||
g.SetLimit(config.APPConfig.HandleFileCount)
|
||||
// 执行文件上传任务参数(文件路径,文件行数)
|
||||
for fileName, info := range filesInfo {
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
AddLog(logChan, "正在上传文件:"+filepath.Base(task.FilePath))
|
||||
}
|
||||
|
||||
processFile(egctx, logChan, task.FilePath, task.FileLines)
|
||||
AddLog(logChan, "正在上传文件:"+fileName)
|
||||
|
||||
processFile(egctx, logChan, info.FilePath, info.FileLines)
|
||||
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
//上传完成,清空文件
|
||||
err := os.Truncate(task.FilePath, 0)
|
||||
}
|
||||
|
||||
//上传完成,删除缓存文件
|
||||
err := os.Remove(info.FilePath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
AddLog(logChan, "删除缓存文件失败:"+err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
// 等待所有任务完成
|
||||
if err := g.Wait(); err != nil {
|
||||
AddLog(logChan, fmt.Sprintf("任务执行出错: %v", err))
|
||||
} else {
|
||||
AddLog(logChan, "所有任务执行完成!")
|
||||
}
|
||||
|
||||
// 等待所有任务完成
|
||||
g.Wait()
|
||||
|
||||
AddLog(logChan, "所有任务执行完成!")
|
||||
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
|
||||
}
|
||||
}
|
||||
|
||||
// 获取目录中的所有txt文件
|
||||
// copyFile 快速拷贝文件 src -> dst
|
||||
func copyFile(src, dst string) error {
|
||||
// 打开源文件
|
||||
sourceFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
// 创建目标文件
|
||||
destFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
// 核心:最快拷贝,底层使用操作系统零拷贝技术
|
||||
_, err = io.Copy(destFile, sourceFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 强制刷入磁盘,保证数据完整
|
||||
return destFile.Sync()
|
||||
}
|
||||
|
||||
// 获取目录中的所有txt文件(文件大小为0的不返回)
|
||||
func getTxtFiles(dir string) (txtFiles []string, err error) {
|
||||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 只处理普通文件,跳过目录
|
||||
// 跳过目录,只处理普通文件
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查文件扩展名是否为.txt
|
||||
if strings.ToLower(filepath.Ext(path)) == ".txt" {
|
||||
if info.Size() != 0 {
|
||||
txtFiles = append(txtFiles, path)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
@@ -205,8 +335,8 @@ func getTxtFiles(dir string) (txtFiles []string, err error) {
|
||||
return txtFiles, err
|
||||
}
|
||||
|
||||
// processFile 处理每个文件
|
||||
func processFile(ctx context.Context, logChan *chan string, filePath string, fileLines int) {
|
||||
var wg sync.WaitGroup
|
||||
// 打开文件
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
@@ -216,51 +346,70 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
||||
defer file.Close()
|
||||
|
||||
// 创建行通道
|
||||
lines := make(chan string, 100)
|
||||
lines := make(chan string, 200)
|
||||
var countLine int32 = 0
|
||||
// 创建指定个worker同时处理文件上传
|
||||
for i := 0; i < config.APPConfig.ThreadCount; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(lines)
|
||||
return
|
||||
default:
|
||||
wg.Go(func() {
|
||||
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
||||
})
|
||||
}
|
||||
|
||||
go func() {
|
||||
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
||||
}()
|
||||
}
|
||||
|
||||
// 读取文件并发送到通道
|
||||
scanner := bufio.NewScanner(file)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_, f, l, _ := runtime.Caller(0)
|
||||
fmt.Println("panic:", f+":"+strconv.Itoa(l), r)
|
||||
}
|
||||
}()
|
||||
for scanner.Scan() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
lines <- scanner.Text()
|
||||
}
|
||||
|
||||
lines <- scanner.Text()
|
||||
}
|
||||
}()
|
||||
|
||||
// 等待所有行处理完成并推送进度
|
||||
for int(countLine) != fileLines {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(lines)
|
||||
wg.Wait()
|
||||
close(lines) //关闭processLines中的上传线程
|
||||
return
|
||||
default:
|
||||
progress.Store(filepath.Base(filePath),
|
||||
Progress{FileName: filepath.Base(filePath),
|
||||
Total: fileLines, Uploaded: int(countLine),
|
||||
Percentage: int(float64(countLine)/float64(fileLines)*100) + 1,
|
||||
})
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
close(lines)
|
||||
wg.Wait()
|
||||
progress.Store(filepath.Base(filePath),
|
||||
Progress{
|
||||
FileName: filepath.Base(filePath),
|
||||
Total: fileLines,
|
||||
Uploaded: int(countLine),
|
||||
Percentage: int(float64(countLine) / float64(fileLines) * 100),
|
||||
},
|
||||
)
|
||||
}
|
||||
//上传完成,进度设为100
|
||||
progress.Store(filepath.Base(filePath),
|
||||
Progress{
|
||||
FileName: filepath.Base(filePath),
|
||||
Total: fileLines,
|
||||
Uploaded: int(countLine),
|
||||
Percentage: 100,
|
||||
},
|
||||
)
|
||||
close(lines) //关闭processLines中的上传线程
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
AddLog(logChan, fmt.Sprintf("读取文件 %s 错误: %v", filePath, err))
|
||||
@@ -270,12 +419,15 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
||||
AddLog(logChan, fmt.Sprintf("文件【%s】处理完成,共处理 %d 行数据", filepath.Base(filePath), countLine))
|
||||
}
|
||||
|
||||
// processLines 处理接受到的每一行数据并上传(chan 管道接受数据)
|
||||
func processLines(ctx context.Context, logChan *chan string, lines *chan string, workerID int, filePath string, countLine *int32) {
|
||||
for line := range *lines {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// 跳过空行
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
@@ -286,7 +438,6 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
|
||||
}
|
||||
atomic.AddInt32(countLine, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddLog 添加日志
|
||||
|
||||
Reference in New Issue
Block a user