feat(api): 优化HTTP连接池和并发控制

- 增加IdleConnTimeout从30秒到30分钟
- 添加并发请求限制通道,最大同时请求数为10
- 实现InitConn函数用于预创建连接池
- 在UploadDataToServer中添加请求限流控制
- 优化资源清理逻辑,使用defer确保响应体关闭
- 重命名runtime包别名以避免冲突
- 在uploader中添加连接池初始化日志
- 添加panic恢复机制和错误处理
This commit is contained in:
2026-04-28 15:00:15 +08:00
parent 4addc29b2c
commit 199bd43b00
2 changed files with 47 additions and 7 deletions
+33 -3
View File
@@ -3,6 +3,7 @@ package api
import (
"context"
"dypid-client/internal/config"
"fmt"
"io"
"net/http"
"net/url"
@@ -13,12 +14,39 @@ var httpClient = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 500,
MaxIdleConnsPerHost: 500,
IdleConnTimeout: 30 * time.Second,
IdleConnTimeout: 30 * time.Minute,
},
Timeout: 30 * time.Second,
}
var limit chan struct{}
func init() {
//限制同时请求数为500
limit = make(chan struct{}, 10)
}
// InitConn 创建连接池
func InitConn() {
for i := 0; i < 200; i++ {
resp, err := httpClient.Get(config.APPConfig.Url + "/api/test")
if err != nil {
fmt.Println(err)
return
}
defer func() {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
}
}
func UploadDataToServer(ctx context.Context, data string) error {
limit <- struct{}{}
defer func() {
<-limit
}()
params := url.Values{}
params.Set("token", config.APPConfig.Token)
params.Set("data", data)
@@ -38,8 +66,10 @@ func UploadDataToServer(ctx context.Context, data string) error {
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
defer func() {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
return err
}
+14 -4
View File
@@ -9,13 +9,14 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/wailsapp/wails/v2/pkg/runtime"
wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime"
"golang.org/x/sync/errgroup"
)
@@ -42,6 +43,10 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
AddLog(logChan, `单文件上传线程: `+strconv.Itoa(config.APPConfig.ThreadCount))
AddLog(logChan, "===============================================")
AddLog(logChan, "正在创建连接池(连接池可避免首次大量上传时出现网络错误)")
api.InitConn()
AddLog(logChan, "创建连接池完成,开始运行程序")
progress.Clear()
//推送上传进度
go func() {
@@ -58,7 +63,7 @@ func StartLooking(ctx context.Context, logChan *chan string, lookingPath string)
pg = append(pg, p)
return true
})
runtime.EventsEmit(ctx, "progress", pg)
wailsruntime.EventsEmit(ctx, "progress", pg)
}
}
}()
@@ -217,6 +222,7 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
// 创建行通道
lines := make(chan string, 100)
defer close(lines)
var countLine int32 = 0
// 创建指定个worker同时处理文件上传
for i := 0; i < config.APPConfig.ThreadCount; i++ {
@@ -233,6 +239,12 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
// 读取文件并发送到通道
scanner := bufio.NewScanner(file)
go func() {
defer func() {
if r := recover(); r != nil {
_, f, l, _ := runtime.Caller(0)
fmt.Println("panic:", f+":"+strconv.Itoa(l), r)
}
}()
for scanner.Scan() {
select {
case <-ctx.Done():
@@ -246,7 +258,6 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
for int(countLine) != fileLines {
select {
case <-ctx.Done():
close(lines)
wg.Wait()
return
default:
@@ -259,7 +270,6 @@ func processFile(ctx context.Context, logChan *chan string, filePath string, fil
}
}
close(lines)
wg.Wait()
if err := scanner.Err(); err != nil {