|
|
@@ -57,6 +57,8 @@ func StartUpload(ctx context.Context, logChan *chan string) {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
var pg []Progress
|
|
|
|
var pg []Progress
|
|
|
|
progress.Range(func(_, value any) bool {
|
|
|
|
progress.Range(func(_, value any) bool {
|
|
|
|
pg = append(pg, value.(Progress))
|
|
|
|
pg = append(pg, value.(Progress))
|
|
|
@@ -66,12 +68,12 @@ func StartUpload(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
time.Sleep(250 * time.Millisecond)
|
|
|
|
time.Sleep(250 * time.Millisecond)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
//开启上传程序
|
|
|
|
//开启上传程序
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
uploadData(ctx, logChan)
|
|
|
|
uploadData(ctx, logChan)
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
@@ -120,6 +122,12 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
if config.APPConfig.ClearFilesNoPrompt {
|
|
|
|
if config.APPConfig.ClearFilesNoPrompt {
|
|
|
|
//不用提示直接复制文件到tmp
|
|
|
|
//不用提示直接复制文件到tmp
|
|
|
|
for _, p := range f {
|
|
|
|
for _, p := range f {
|
|
|
|
|
|
|
|
select {
|
|
|
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
|
|
|
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
AddLog(logChan, "复制文件失败:"+err.Error())
|
|
|
|
AddLog(logChan, "复制文件失败:"+err.Error())
|
|
|
@@ -143,6 +151,12 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
if <-confirm {
|
|
|
|
if <-confirm {
|
|
|
|
for _, p := range f {
|
|
|
|
for _, p := range f {
|
|
|
|
|
|
|
|
select {
|
|
|
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
|
|
|
err := copyFile(p, "./tmp/"+filepath.Base(p))
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
AddLog(logChan, "复制文件失败:"+err.Error())
|
|
|
|
AddLog(logChan, "复制文件失败:"+err.Error())
|
|
|
@@ -163,16 +177,18 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
//检测到文件
|
|
|
|
//检测到文件
|
|
|
|
//统计文件行数
|
|
|
|
//统计文件行数
|
|
|
|
var fInfo = make(map[string]fileInfo)
|
|
|
|
var filesInfo = make(map[string]fileInfo)
|
|
|
|
|
|
|
|
|
|
|
|
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
isAllEmpty := true
|
|
|
|
isAllEmpty := true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
|
|
|
|
for _, filePath := range files {
|
|
|
|
for _, filePath := range files {
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
file, err := os.Open(filePath)
|
|
|
|
file, err := os.Open(filePath)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
AddLog(logChan, "打开文件失败:"+err.Error())
|
|
|
|
AddLog(logChan, "打开文件失败:"+err.Error())
|
|
|
@@ -189,7 +205,7 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
if lineCount == 0 {
|
|
|
|
if lineCount == 0 {
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fInfo[filepath.Base(filePath)] = fileInfo{
|
|
|
|
filesInfo[filepath.Base(filePath)] = fileInfo{
|
|
|
|
FilePath: filePath,
|
|
|
|
FilePath: filePath,
|
|
|
|
FileLines: lineCount,
|
|
|
|
FileLines: lineCount,
|
|
|
|
}
|
|
|
|
}
|
|
|
@@ -198,7 +214,6 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
|
|
|
|
AddLog(logChan, fmt.Sprintf("%s 文件行数:%v", filepath.Base(filePath), lineCount))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if isAllEmpty {
|
|
|
|
if isAllEmpty {
|
|
|
|
AddLog(logChan, "所有文件都为空,不进行上传")
|
|
|
|
AddLog(logChan, "所有文件都为空,不进行上传")
|
|
|
@@ -207,7 +222,7 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
//刷新文件上传进度
|
|
|
|
//刷新文件上传进度
|
|
|
|
progress.Clear()
|
|
|
|
progress.Clear()
|
|
|
|
for fileName, info := range fInfo {
|
|
|
|
for fileName, info := range filesInfo {
|
|
|
|
progress.Store(fileName,
|
|
|
|
progress.Store(fileName,
|
|
|
|
Progress{
|
|
|
|
Progress{
|
|
|
|
FileName: fileName,
|
|
|
|
FileName: fileName,
|
|
|
@@ -220,18 +235,23 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
|
|
|
|
|
|
|
|
// 使用 errgroup 控制同时处理的文件数,并开始上传文件任务
|
|
|
|
// 使用 errgroup 控制同时处理的文件数,并开始上传文件任务
|
|
|
|
g, egctx := errgroup.WithContext(ctx)
|
|
|
|
g, egctx := errgroup.WithContext(ctx)
|
|
|
|
g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
|
|
|
|
// 设置同时处理文件数
|
|
|
|
|
|
|
|
g.SetLimit(config.APPConfig.HandleFileCount)
|
|
|
|
// 执行文件上传任务参数(文件路径,文件行数)
|
|
|
|
// 执行文件上传任务参数(文件路径,文件行数)
|
|
|
|
for fileName, info := range fInfo {
|
|
|
|
for fileName, info := range filesInfo {
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-egctx.Done():
|
|
|
|
case <-egctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
g.Go(func() error {
|
|
|
|
g.Go(func() error {
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-egctx.Done():
|
|
|
|
case <-egctx.Done():
|
|
|
|
return egctx.Err()
|
|
|
|
return egctx.Err()
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
AddLog(logChan, "正在上传文件:"+fileName)
|
|
|
|
AddLog(logChan, "正在上传文件:"+fileName)
|
|
|
|
|
|
|
|
|
|
|
|
processFile(egctx, logChan, info.FilePath, info.FileLines)
|
|
|
|
processFile(egctx, logChan, info.FilePath, info.FileLines)
|
|
|
@@ -240,29 +260,29 @@ func uploadData(ctx context.Context, logChan *chan string) {
|
|
|
|
case <-egctx.Done():
|
|
|
|
case <-egctx.Done():
|
|
|
|
return egctx.Err()
|
|
|
|
return egctx.Err()
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//上传完成,删除缓存文件
|
|
|
|
//上传完成,删除缓存文件
|
|
|
|
err := os.Remove(info.FilePath)
|
|
|
|
err := os.Remove(info.FilePath)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
AddLog(logChan, "删除缓存文件失败:"+err.Error())
|
|
|
|
AddLog(logChan, "删除缓存文件失败:"+err.Error())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 等待所有任务完成
|
|
|
|
// 等待所有任务完成
|
|
|
|
g.Wait()
|
|
|
|
g.Wait()
|
|
|
|
|
|
|
|
|
|
|
|
AddLog(logChan, "所有任务执行完成!")
|
|
|
|
AddLog(logChan, "所有任务执行完成!")
|
|
|
|
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
|
|
|
|
AddLog(logChan, fmt.Sprintf("上传完成,耗时:%s", time.Since(start).String()))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// copyFile 快速拷贝文件 src -> dst
|
|
|
|
// copyFile 快速拷贝文件 src -> dst
|
|
|
|
func copyFile(src, dst string) error {
|
|
|
|
func copyFile(src, dst string) error {
|
|
|
@@ -335,11 +355,12 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
|
|
|
close(lines)
|
|
|
|
close(lines)
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
go func() {
|
|
|
|
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
|
|
|
processLines(ctx, logChan, &lines, i, filePath, &countLine)
|
|
|
|
}()
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// 读取文件并发送到通道
|
|
|
|
// 读取文件并发送到通道
|
|
|
|
scanner := bufio.NewScanner(file)
|
|
|
|
scanner := bufio.NewScanner(file)
|
|
|
@@ -355,8 +376,9 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
lines <- scanner.Text()
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lines <- scanner.Text()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
@@ -367,6 +389,8 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
|
|
|
close(lines) //关闭processLines中的上传线程
|
|
|
|
close(lines) //关闭processLines中的上传线程
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
progress.Store(filepath.Base(filePath),
|
|
|
|
progress.Store(filepath.Base(filePath),
|
|
|
|
Progress{
|
|
|
|
Progress{
|
|
|
|
FileName: filepath.Base(filePath),
|
|
|
|
FileName: filepath.Base(filePath),
|
|
|
@@ -376,7 +400,6 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
|
|
|
|
},
|
|
|
|
},
|
|
|
|
)
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//上传完成,进度设为100
|
|
|
|
//上传完成,进度设为100
|
|
|
|
progress.Store(filepath.Base(filePath),
|
|
|
|
progress.Store(filepath.Base(filePath),
|
|
|
|
Progress{
|
|
|
|
Progress{
|
|
|
@@ -403,6 +426,8 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 跳过空行
|
|
|
|
// 跳过空行
|
|
|
|
if strings.TrimSpace(line) == "" {
|
|
|
|
if strings.TrimSpace(line) == "" {
|
|
|
|
continue
|
|
|
|
continue
|
|
|
@@ -414,7 +439,6 @@ func processLines(ctx context.Context, logChan *chan string, lines *chan string,
|
|
|
|
atomic.AddInt32(countLine, 1)
|
|
|
|
atomic.AddInt32(countLine, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// AddLog 添加日志
|
|
|
|
// AddLog 添加日志
|
|
|
|
func AddLog(logChan *chan string, message string) {
|
|
|
|
func AddLog(logChan *chan string, message string) {
|
|
|
|