Compare commits
5 Commits
03e4e6f45b
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| b100fc9b32 | |||
| d426c16104 | |||
| 993814cdfa | |||
| 8ffa0531d6 | |||
| 3f6e999783 |
@@ -11,6 +11,7 @@
|
||||
"type-check": "vue-tsc --build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.3.2",
|
||||
"element-plus": "^2.13.7",
|
||||
"vue": "^3.5.32"
|
||||
},
|
||||
|
||||
@@ -1 +1 @@
|
||||
aa914e6b4676ee4621ced7ad6d81c58c
|
||||
05225657934ff66d822c925754c951bf
|
||||
Generated
+3
@@ -8,6 +8,9 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@element-plus/icons-vue':
|
||||
specifier: ^2.3.2
|
||||
version: 2.3.2(vue@3.5.33(typescript@6.0.3))
|
||||
element-plus:
|
||||
specifier: ^2.13.7
|
||||
version: 2.13.7(typescript@6.0.3)(vue@3.5.33(typescript@6.0.3))
|
||||
|
||||
+55
-32
@@ -1,11 +1,12 @@
|
||||
<script lang="ts" setup>
|
||||
import {ref, nextTick, computed} from 'vue'
|
||||
import {ElMessage} from 'element-plus'
|
||||
import {ElMessageBox} from 'element-plus'
|
||||
import {SelectPath, GetConfig, WriteConfig, StartUpload, StopUpload} from '../wailsjs/go/main/App';
|
||||
import {computed, nextTick, ref} from 'vue'
|
||||
import {ElMessage, ElMessageBox} from 'element-plus'
|
||||
import ConfirmClearDialog from './components/ConfirmClearDialog.vue'
|
||||
import {GetConfig, SelectPath, StartUpload, StopUpload, WriteConfig} from '../wailsjs/go/main/App';
|
||||
import {config} from "../wailsjs/go/models.ts";
|
||||
import Config = config.Config;
|
||||
import {EventsOn, LogPrint} from "../wailsjs/runtime";
|
||||
import {configModel} from "@/model.ts";
|
||||
import Config = config.Config;
|
||||
|
||||
// const serverUrl = ref('')
|
||||
const token = ref('')
|
||||
@@ -19,6 +20,10 @@ const logOutput = ref<string[]>([])
|
||||
const logContentRef = ref<HTMLElement>()
|
||||
const logRoll = ref(true)
|
||||
|
||||
const clearDialogVisible = ref(false)
|
||||
const filesToClear = ref<string[]>([])
|
||||
const noPromptClear = ref(false)
|
||||
|
||||
interface FileProgress {
|
||||
name: string
|
||||
uploaded: number
|
||||
@@ -41,7 +46,7 @@ const sortedProgressList = computed(() => {
|
||||
})
|
||||
|
||||
const addLog = (msg: string) => {
|
||||
logOutput.value.push(`[${new Date().toLocaleString()}]` + msg)
|
||||
logOutput.value.push(`[${new Date().toLocaleString()}] ` + msg)
|
||||
nextTick(() => {
|
||||
if (logContentRef.value && logRoll.value) {
|
||||
logContentRef.value.scrollTop = logContentRef.value.scrollHeight
|
||||
@@ -52,9 +57,9 @@ const addLog = (msg: string) => {
|
||||
const selectDirectory = () => {
|
||||
// ElMessage.info('请手动输入检测目录路径')
|
||||
SelectPath().then((path) => {
|
||||
if (path){
|
||||
if (path) {
|
||||
checkDir.value = path
|
||||
}else {
|
||||
} else {
|
||||
ElMessage.warning('未选择目录,不更改配置')
|
||||
}
|
||||
})
|
||||
@@ -99,17 +104,17 @@ const clearLog = () => {
|
||||
// const writeServerUrl =() => {
|
||||
// WriteConfig("url", serverUrl.value)
|
||||
// }
|
||||
const writeToken =() => {
|
||||
WriteConfig("token", token.value)
|
||||
const writeToken = () => {
|
||||
WriteConfig(configModel.Token, token.value)
|
||||
}
|
||||
const writeCheckDir = () => {
|
||||
WriteConfig("check-dir", checkDir.value)
|
||||
WriteConfig(configModel.CheckDir, checkDir.value)
|
||||
}
|
||||
const writeConcurrentFiles = () => {
|
||||
WriteConfig("handle-file-count", concurrentFiles.value)
|
||||
WriteConfig(configModel.HandleFileCount, concurrentFiles.value)
|
||||
}
|
||||
const writeUploadThreads =() => {
|
||||
WriteConfig("thread-count", uploadThreads.value)
|
||||
const writeUploadThreads = () => {
|
||||
WriteConfig(configModel.ThreadCount, uploadThreads.value)
|
||||
}
|
||||
// const writeAutoStart = () => {
|
||||
// WriteConfig("is-run-on-start", autoStart.value)
|
||||
@@ -124,6 +129,7 @@ try {
|
||||
concurrentFiles.value = config.handle_file_count
|
||||
uploadThreads.value = config.thread_count
|
||||
// autoStart.value = config.is_run_on_start
|
||||
noPromptClear.value = config.clear_files_no_prompt
|
||||
|
||||
LogPrint(`[${new Date().toLocaleString()}] 配置已加载`)
|
||||
})
|
||||
@@ -131,24 +137,34 @@ try {
|
||||
console.log(e)
|
||||
}
|
||||
|
||||
EventsOn("is-run", (run) => {
|
||||
isRunning.value = run
|
||||
})
|
||||
EventsOn("progress", (progress) => {
|
||||
progressList.value = progress
|
||||
})
|
||||
EventsOn("log", (msg) => {
|
||||
addLog(msg)
|
||||
})
|
||||
try {
|
||||
EventsOn("is-run", (run) => {
|
||||
isRunning.value = run
|
||||
})
|
||||
EventsOn("progress", (progress) => {
|
||||
progressList.value = progress
|
||||
})
|
||||
EventsOn("log", (msg) => {
|
||||
addLog(msg)
|
||||
})
|
||||
EventsOn("clear-files", (files) => {
|
||||
filesToClear.value = files
|
||||
if (!noPromptClear.value) {
|
||||
clearDialogVisible.value = true
|
||||
}
|
||||
})
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="container">
|
||||
<div class="left-panel">
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <label>服务器地址</label>-->
|
||||
<!-- <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
|
||||
<!-- </div>-->
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <label>服务器地址</label>-->
|
||||
<!-- <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
|
||||
<!-- </div>-->
|
||||
|
||||
<div class="form-item">
|
||||
<label>Token</label>
|
||||
@@ -165,17 +181,19 @@ EventsOn("log", (msg) => {
|
||||
|
||||
<div class="form-item">
|
||||
<label>同时处理文件数</label>
|
||||
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning" @change="writeConcurrentFiles()"/>
|
||||
<el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"
|
||||
@change="writeConcurrentFiles()"/>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label>单文件上传线程</label>
|
||||
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning" @change="writeUploadThreads()"/>
|
||||
<el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"
|
||||
@change="writeUploadThreads()"/>
|
||||
</div>
|
||||
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
|
||||
<!-- </div>-->
|
||||
<!-- <div class="form-item">-->
|
||||
<!-- <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
|
||||
<!-- </div>-->
|
||||
|
||||
<div class="form-item">
|
||||
<label>上传进度</label>
|
||||
@@ -205,6 +223,11 @@ EventsOn("log", (msg) => {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ConfirmClearDialog
|
||||
v-model:visible="clearDialogVisible"
|
||||
:file-list="filesToClear"
|
||||
/>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
<script lang="ts" setup>
|
||||
import {ref} from 'vue'
|
||||
import {WriteConfig} from "../../wailsjs/go/main/App";
|
||||
import {configModel} from "@/model.ts";
|
||||
import {EventsEmit} from "../../wailsjs/runtime";
|
||||
import {InfoFilled} from '@element-plus/icons-vue'
|
||||
|
||||
const props = defineProps<{
|
||||
visible: boolean
|
||||
fileList: string[]
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'update:visible', value: boolean): void
|
||||
// (e: 'confirm', dontShowAgain: boolean): void
|
||||
}>()
|
||||
|
||||
const dontShowAgain = ref(false)
|
||||
|
||||
const handleClose = () => {
|
||||
emit('update:visible', false)
|
||||
EventsEmit('confirm-clear-files', false)
|
||||
}
|
||||
|
||||
const handleConfirm = () => {
|
||||
// emit('confirm', dontShowAgain.value)
|
||||
emit('update:visible', false)
|
||||
WriteConfig(configModel.ClearFilesNoPrompt, dontShowAgain.value)
|
||||
EventsEmit('confirm-clear-files', true)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<el-dialog
|
||||
:model-value="visible"
|
||||
width="600px"
|
||||
:close-on-click-modal="false"
|
||||
@update:model-value="(val: boolean) => emit('update:visible', val)"
|
||||
@close="handleClose"
|
||||
align-center
|
||||
>
|
||||
<template #header>
|
||||
是否确认清空需要上传文件
|
||||
</template>
|
||||
<div class="hint-text">以下文件将会被清空并移动到 tmp 文件夹进行上传,您是否确认</div>
|
||||
<div class="dialog-content">
|
||||
<div class="file-list">
|
||||
<div v-for="(file, index) in fileList" :key="index" class="file-item">
|
||||
{{ file }}
|
||||
</div>
|
||||
<div v-if="fileList.length === 0" class="empty-text">暂无文件</div>
|
||||
</div>
|
||||
</div>
|
||||
<template #footer>
|
||||
<div class="dialog-footer">
|
||||
<div class="dialog-footer-checkbox-container">
|
||||
<el-checkbox v-model="dontShowAgain" label="下次清空文件不再弹出此弹窗确认"/>
|
||||
<el-tooltip content="此弹窗会在首次运行时弹出,您可以选择下次不再弹出此弹窗">
|
||||
<el-icon>
|
||||
<InfoFilled/>
|
||||
</el-icon>
|
||||
</el-tooltip>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<el-button @click="handleClose">取消</el-button>
|
||||
<el-button type="primary" @click="handleConfirm">确认</el-button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.dialog-footer-checkbox-container {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.dialog-footer-checkbox-container .el-icon {
|
||||
color: #909399;
|
||||
}
|
||||
|
||||
.dialog-content {
|
||||
padding: 10px 0;
|
||||
}
|
||||
|
||||
.hint-text {
|
||||
font-size: 12px;
|
||||
color: #909399;
|
||||
}
|
||||
|
||||
.file-list {
|
||||
max-height: 350px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #e4e7ed;
|
||||
border-radius: 4px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.file-item {
|
||||
padding: 6px 8px;
|
||||
font-size: 14px;
|
||||
color: #606266;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
.file-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.empty-text {
|
||||
text-align: center;
|
||||
color: #909399;
|
||||
padding: 20px 0;
|
||||
}
|
||||
|
||||
.dialog-footer {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,9 @@
|
||||
export enum configModel {
|
||||
Url = "url",
|
||||
Token = "token",
|
||||
ThreadCount = "thread-count",
|
||||
HandleFileCount = "handle-file-count",
|
||||
IsRunOnStart = "is-run-on-start",
|
||||
CheckDir = "check-dir",
|
||||
ClearFilesNoPrompt = "clear-files-no-prompt",
|
||||
}
|
||||
@@ -7,6 +7,7 @@ export namespace config {
|
||||
handle_file_count: number;
|
||||
is_run_on_start: boolean;
|
||||
check_dir: string;
|
||||
clear_files_no_prompt: boolean;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new Config(source);
|
||||
@@ -20,6 +21,7 @@ export namespace config {
|
||||
this.handle_file_count = source["handle_file_count"];
|
||||
this.is_run_on_start = source["is_run_on_start"];
|
||||
this.check_dir = source["check_dir"];
|
||||
this.clear_files_no_prompt = source["clear_files_no_prompt"];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+22
-18
@@ -9,35 +9,38 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Url string `json:"url" mapstructure:"url"`
|
||||
Token string `json:"token" mapstructure:"token"`
|
||||
ThreadCount int `json:"thread_count" mapstructure:"thread-count"`
|
||||
HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"`
|
||||
IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"`
|
||||
CheckDir string `json:"check_dir" mapstructure:"check-dir"`
|
||||
Url string `json:"url" mapstructure:"url"`
|
||||
Token string `json:"token" mapstructure:"token"`
|
||||
ThreadCount int `json:"thread_count" mapstructure:"thread-count"`
|
||||
HandleFileCount int `json:"handle_file_count" mapstructure:"handle-file-count"`
|
||||
IsRunOnStart bool `json:"is_run_on_start" mapstructure:"is-run-on-start"`
|
||||
CheckDir string `json:"check_dir" mapstructure:"check-dir"`
|
||||
ClearFilesNoPrompt bool `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"`
|
||||
}
|
||||
|
||||
var APPConfig Config
|
||||
var configMu sync.Mutex
|
||||
|
||||
const (
|
||||
Url = "url"
|
||||
Token = "token"
|
||||
ThreadCount = "thread-count"
|
||||
HandleFileCount = "handle-file-count"
|
||||
IsRunOnStart = "is-run-on-start"
|
||||
CheckDir = "check-dir"
|
||||
Url = "url"
|
||||
Token = "token"
|
||||
ThreadCount = "thread-count"
|
||||
HandleFileCount = "handle-file-count"
|
||||
IsRunOnStart = "is-run-on-start"
|
||||
CheckDir = "check-dir"
|
||||
ClearFilesNoPrompt = "clear-files-no-prompt"
|
||||
)
|
||||
|
||||
func InitConfig() {
|
||||
// 设置默认配置
|
||||
defaultConfig := Config{
|
||||
Url: "http://127.0.0.1:8080",
|
||||
Token: "",
|
||||
ThreadCount: 10,
|
||||
HandleFileCount: 25,
|
||||
IsRunOnStart: false,
|
||||
CheckDir: "",
|
||||
Url: "http://127.0.0.1:8080",
|
||||
Token: "",
|
||||
ThreadCount: 10,
|
||||
HandleFileCount: 25,
|
||||
IsRunOnStart: false,
|
||||
CheckDir: "",
|
||||
ClearFilesNoPrompt: false,
|
||||
}
|
||||
viper.SetDefault(Url, defaultConfig.Url)
|
||||
viper.SetDefault(Token, defaultConfig.Token)
|
||||
@@ -45,6 +48,7 @@ func InitConfig() {
|
||||
viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
|
||||
viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)
|
||||
viper.SetDefault(CheckDir, defaultConfig.CheckDir)
|
||||
viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt)
|
||||
|
||||
//设置配置文件名和路径 ./config.toml
|
||||
viper.AddConfigPath(".")
|
||||
|
||||
+231
-105
@@ -7,6 +7,7 @@ import (
|
||||
"dypid-client/internal/api"
|
||||
"dypid-client/internal/config"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
@@ -20,7 +21,7 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type Task struct {
|
||||
type fileInfo struct {
|
||||
FilePath string
|
||||
FileLines int
|
||||
}
|
||||
@@ -56,21 +57,23 @@ func StartUpload(ctx context.Context, logChan *chan string) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
var pg []Progress
|
||||
progress.Range(func(_, value any) bool {
|
||||
pg = append(pg, value.(Progress))
|
||||
return true
|
||||
})
|
||||
wailsruntime.EventsEmit(ctx, "progress", pg)
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
|
||||
var pg []Progress
|
||||
progress.Range(func(_, value any) bool {
|
||||
pg = append(pg, value.(Progress))
|
||||
return true
|
||||
})
|
||||
wailsruntime.EventsEmit(ctx, "progress", pg)
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
|
||||
//开启上传程序
|
||||
for {
|
||||
uploadData(ctx, logChan)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -80,54 +83,136 @@ func StartUpload(ctx context.Context, logChan *chan string) {
|
||||
}
|
||||
|
||||
func uploadData(ctx context.Context, logChan *chan string) {
|
||||
// 获取检测目录
|
||||
var path = "./"
|
||||
if config.APPConfig.CheckDir != "" {
|
||||
path = config.APPConfig.CheckDir
|
||||
}
|
||||
|
||||
//获取文件列表
|
||||
files, err := getTxtFiles(path)
|
||||
if err != nil {
|
||||
AddLog(logChan, "获取文件列表失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if files == nil {
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
// 获取检测目录
|
||||
var checkPath = "./"
|
||||
if config.APPConfig.CheckDir != "" {
|
||||
checkPath = config.APPConfig.CheckDir
|
||||
}
|
||||
|
||||
//要上传的文件路径字符串数组
|
||||
var files []string
|
||||
|
||||
//先检测tmp目录有没有残余文件
|
||||
os.Mkdir("./tmp", os.ModePerm)
|
||||
tmpFiles, err := getTxtFiles("./tmp")
|
||||
if err != nil {
|
||||
AddLog(logChan, "获取 tmp 文件列表失败:"+err.Error())
|
||||
}
|
||||
|
||||
//tmp有文件,优先上传(else:tmp没文件扫描指定文件夹,并复制文件到tmp)
|
||||
if tmpFiles != nil {
|
||||
AddLog(logChan, "当前 tmp 目录下还有未上传完成文件,将优先上传 tmp 目录文件")
|
||||
files = tmpFiles
|
||||
} else {
|
||||
//tmp没文件,扫描指定文件夹
|
||||
f, err := getTxtFiles(checkPath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "获取文件列表失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
//指定文件夹没文件,退出函数
|
||||
if f == nil {
|
||||
return
|
||||
}
|
||||
|
||||
//是否向用户提示清空文件,并复制文件到tmp
|
||||
if config.APPConfig.ClearFilesNoPrompt {
|
||||
//不用提示直接复制文件到tmp
|
||||
for _, p := range f {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
||||
if err != nil {
|
||||
AddLog(logChan, "复制文件失败:"+err.Error())
|
||||
} else {
|
||||
files = append(files, "./tmp/"+filepath.Base(p))
|
||||
err := os.Truncate(p, 0)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//提示用户
|
||||
wailsruntime.EventsEmit(ctx, "clear-files", f)
|
||||
|
||||
confirm := make(chan bool)
|
||||
|
||||
wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) {
|
||||
confirm <- optionalData[0].(bool)
|
||||
})
|
||||
|
||||
if <-confirm {
|
||||
for _, p := range f {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
||||
if err != nil {
|
||||
AddLog(logChan, "复制文件失败:"+err.Error())
|
||||
} else {
|
||||
files = append(files, "./tmp/"+filepath.Base(p))
|
||||
err := os.Truncate(p, 0)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
AddLog(logChan, "已取消上传,1分钟后再运行")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//检测到文件
|
||||
//统计文件行数
|
||||
fileLines := make(map[string]int)
|
||||
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
||||
var filesInfo = make(map[string]fileInfo)
|
||||
|
||||
isAllEmpty := true
|
||||
|
||||
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
||||
for _, filePath := range files {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "打开文件失败:"+err.Error())
|
||||
}
|
||||
|
||||
// 使用 bufio.Scanner 逐行读取
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineCount := 0
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
}
|
||||
err = file.Close()
|
||||
|
||||
if lineCount == 0 {
|
||||
continue
|
||||
}
|
||||
fileLines[filepath.Base(filePath)] = lineCount
|
||||
isAllEmpty = false
|
||||
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "打开文件失败:"+err.Error())
|
||||
}
|
||||
|
||||
// 使用 bufio.Scanner 逐行读取
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineCount := 0
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
}
|
||||
err = file.Close()
|
||||
|
||||
if lineCount == 0 {
|
||||
continue
|
||||
}
|
||||
filesInfo[filepath.Base(filePath)] = fileInfo{
|
||||
FilePath: filePath,
|
||||
FileLines: lineCount,
|
||||
}
|
||||
|
||||
isAllEmpty = false
|
||||
|
||||
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
|
||||
}
|
||||
|
||||
if isAllEmpty {
|
||||
@@ -135,62 +220,97 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
||||
return
|
||||
}
|
||||
|
||||
//刷新文件上传进度
|
||||
progress.Clear()
|
||||
|
||||
//添加文件上传任务参数(文件路径,文件行数)
|
||||
var tasks []Task
|
||||
for fileName, lines := range fileLines {
|
||||
tasks = append(tasks, Task{FilePath: path + "/" + fileName, FileLines: lines})
|
||||
progress.Store(fileName, Progress{FileName: fileName, Total: lines, Uploaded: 0, Percentage: 0})
|
||||
for fileName, info := range filesInfo {
|
||||
progress.Store(fileName,
|
||||
Progress{
|
||||
FileName: fileName,
|
||||
Total: info.FileLines,
|
||||
Uploaded: 0,
|
||||
Percentage: 0,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// 使用 errgroup 控制同时处理的文件数,并开始上传文件任务
|
||||
g, egctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
|
||||
// 执行所有任务
|
||||
for _, task := range tasks {
|
||||
// 设置同时处理文件数
|
||||
g.SetLimit(config.APPConfig.HandleFileCount)
|
||||
// 执行文件上传任务参数(文件路径,文件行数)
|
||||
for fileName, info := range filesInfo {
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return
|
||||
default:
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
AddLog(logChan, "正在上传文件:"+filepath.Base(task.FilePath))
|
||||
|
||||
processFile(egctx, logChan, task.FilePath, task.FileLines)
|
||||
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
//上传完成,清空文件
|
||||
err := os.Truncate(task.FilePath, 0)
|
||||
if err != nil {
|
||||
AddLog(logChan, "清空文件失败:"+err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
AddLog(logChan, "正在上传文件:"+fileName)
|
||||
|
||||
processFile(egctx, logChan, info.FilePath, info.FileLines)
|
||||
|
||||
select {
|
||||
case <-egctx.Done():
|
||||
return egctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
//上传完成,删除缓存文件
|
||||
err := os.Remove(info.FilePath)
|
||||
if err != nil {
|
||||
AddLog(logChan, "删除缓存文件失败:"+err.Error())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
// 等待所有任务完成
|
||||
g.Wait()
|
||||
|
||||
AddLog(logChan, "所有任务执行完成!")
|
||||
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
|
||||
}
|
||||
|
||||
// 等待所有任务完成
|
||||
g.Wait()
|
||||
|
||||
AddLog(logChan, "所有任务执行完成!")
|
||||
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
|
||||
}
|
||||
|
||||
// 获取目录中的所有txt文件
|
||||
// copyFile 快速拷贝文件 src -> dst
|
||||
func copyFile(src, dst string) error {
|
||||
// 打开源文件
|
||||
sourceFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
// 创建目标文件
|
||||
destFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
// 核心:最快拷贝,底层使用操作系统零拷贝技术
|
||||
_, err = io.Copy(destFile, sourceFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 强制刷入磁盘,保证数据完整
|
||||
return destFile.Sync()
|
||||
}
|
||||
|
||||
// 获取目录中的所有txt文件(文件大小为0的不返回)
|
||||
func getTxtFiles(dir string) (txtFiles []string, err error) {
|
||||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
@@ -204,7 +324,9 @@ func getTxtFiles(dir string) (txtFiles []string, err error) {
|
||||
|
||||
// 检查文件扩展名是否为.txt
|
||||
if strings.ToLower(filepath.Ext(path)) == ".txt" {
|
||||
txtFiles = append(txtFiles, path)
|
||||
if info.Size() != 0 {
|
||||
txtFiles = append(txtFiles, path)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -233,10 +355,11 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
||||
close(lines)
|
||||
return
|
||||
default:
|
||||
go func() {
|
||||
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
||||
}()
|
||||
}
|
||||
|
||||
// 读取文件并发送到通道
|
||||
@@ -253,8 +376,9 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
lines <- scanner.Text()
|
||||
}
|
||||
|
||||
lines <- scanner.Text()
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -265,15 +389,16 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
||||
close(lines) //关闭processLines中的上传线程
|
||||
return
|
||||
default:
|
||||
progress.Store(filepath.Base(filePath),
|
||||
Progress{
|
||||
FileName: filepath.Base(filePath),
|
||||
Total: fileLines,
|
||||
Uploaded: int(countLine),
|
||||
Percentage: int(float64(countLine) / float64(fileLines) * 100),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
progress.Store(filepath.Base(filePath),
|
||||
Progress{
|
||||
FileName: filepath.Base(filePath),
|
||||
Total: fileLines,
|
||||
Uploaded: int(countLine),
|
||||
Percentage: int(float64(countLine) / float64(fileLines) * 100),
|
||||
},
|
||||
)
|
||||
}
|
||||
//上传完成,进度设为100
|
||||
progress.Store(filepath.Base(filePath),
|
||||
@@ -301,16 +426,17 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
// 跳过空行
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
// 上传数据
|
||||
if err := api.UploadDataToServer(ctx, line); err != nil {
|
||||
AddLog(logChan, fmt.Sprintf("Worker %d (文件 %s): 上传失败: %v", workerID, filepath.Base(filePath), err))
|
||||
}
|
||||
atomic.AddInt32(countLine, 1)
|
||||
}
|
||||
|
||||
// 跳过空行
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
// 上传数据
|
||||
if err := api.UploadDataToServer(ctx, line); err != nil {
|
||||
AddLog(logChan, fmt.Sprintf("Worker %d (文件 %s): 上传失败: %v", workerID, filepath.Base(filePath), err))
|
||||
}
|
||||
atomic.AddInt32(countLine, 1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user