Compare commits
5 Commits
a505f2ddc9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 819a2eb8ec | |||
| ea9ecb770d | |||
| 3476d66c0d | |||
| 1f0158dd76 | |||
| f35fad442a |
@@ -1,37 +0,0 @@
|
||||
name: 构建上传工具
|
||||
on: [ push ]
|
||||
|
||||
jobs:
|
||||
build-tool:
|
||||
env:
|
||||
RUNNER_TOOL_CACHE: /toolcache
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 检出代码
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装Go(镜像)
|
||||
run: |
|
||||
# 使用国内镜像下载 Go
|
||||
wget https://mirrors.aliyun.com/golang/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
|
||||
# 解压并设置环境变量
|
||||
sudo rm -rf /usr/local/go
|
||||
sudo tar -C /usr/local -xzf go.tar.gz
|
||||
echo "/usr/local/go/bin" >> $GITHUB_PATH
|
||||
env:
|
||||
GOROOT: /usr/local/go
|
||||
|
||||
- name: 构建上传工具
|
||||
run: |
|
||||
go env -w CGO_ENABLED=0 \
|
||||
&& go env -w GOARCH=amd64 \
|
||||
&& go env -w GOOS=windows \
|
||||
&& go mod tidy \
|
||||
&& cd ./tool \
|
||||
&& go build -o 上传工具.exe
|
||||
|
||||
- name: 上传构建文件
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: 上传工具
|
||||
path: tool/上传工具.exe
|
||||
41
api/api.go
Normal file
41
api/api.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"dypid/internal/controller"
|
||||
"embed"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func RegRoutes(r *gin.Engine) {
|
||||
g := r.Group("/api") //初始化路由组 /api/xxxx
|
||||
{
|
||||
g.GET("/token", controller.ListTokenHandler) //获取token列表
|
||||
g.POST("/token", controller.CreateTokenHandler) //创建token
|
||||
g.PUT("/token", controller.UpdateTokenHandler) //更新token
|
||||
g.DELETE("/token", controller.DeleteTokenHandler) //删除token
|
||||
g.GET("/token/info", controller.GetTokenInfoHandler) //获取token信息
|
||||
g.DELETE("/token/info", controller.DeleteTokenInfoHandler) //删除token数据库
|
||||
}
|
||||
{
|
||||
g.GET("/data", controller.ReadDataHandler) //获取数据
|
||||
g.POST("/data", controller.WriteDataHandler) //写入数据
|
||||
}
|
||||
}
|
||||
|
||||
func RegWebService(r *gin.Engine, webDir embed.FS) {
|
||||
assets, _ := fs.Sub(webDir, "web/dist/assets")
|
||||
r.StaticFS("/assets", http.FS(assets))
|
||||
|
||||
icon, _ := fs.ReadFile(webDir, "web/dist/favicon.ico")
|
||||
r.GET("/favicon.ico", func(c *gin.Context) {
|
||||
c.Data(200, "image/x-icon", icon)
|
||||
})
|
||||
|
||||
indexHtml, _ := fs.ReadFile(webDir, "web/dist/index.html")
|
||||
r.NoRoute(func(c *gin.Context) {
|
||||
c.Data(200, "text/html; charset=utf-8", indexHtml)
|
||||
})
|
||||
}
|
||||
2
go.mod
2
go.mod
@@ -3,7 +3,6 @@ module dypid
|
||||
go 1.25
|
||||
|
||||
require (
|
||||
github.com/fsnotify/fsnotify v1.8.0
|
||||
github.com/gin-contrib/cors v1.7.6
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/redis/go-redis/v9 v9.12.1
|
||||
@@ -16,6 +15,7 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/fsnotify/fsnotify v1.8.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"dypid/db"
|
||||
"dypid/global"
|
||||
"dypid/internal/db"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -1,8 +1,8 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"dypid/db"
|
||||
"dypid/global"
|
||||
"dypid/internal/db"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
38
main.go
38
main.go
@@ -1,13 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"dypid/api"
|
||||
"dypid/config"
|
||||
"dypid/controller"
|
||||
"dypid/db"
|
||||
"dypid/internal/db"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-contrib/cors"
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -23,35 +21,13 @@ func main() {
|
||||
//初始化一个http服务对象
|
||||
gin.SetMode(config.APPConfig.RunMode)
|
||||
r := gin.Default()
|
||||
|
||||
//跨域设置
|
||||
r.Use(cors.Default())
|
||||
|
||||
//Vue网站服务
|
||||
assets, _ := fs.Sub(webDir, "web/dist/assets")
|
||||
r.StaticFS("/assets", http.FS(assets))
|
||||
icon, _ := fs.ReadFile(webDir, "web/dist/favicon.ico")
|
||||
r.GET("/favicon.ico", func(c *gin.Context) {
|
||||
c.Data(200, "image/x-icon", icon)
|
||||
})
|
||||
indexHtml, _ := fs.ReadFile(webDir, "web/dist/index.html")
|
||||
r.NoRoute(func(c *gin.Context) {
|
||||
c.Data(200, "text/html; charset=utf-8", indexHtml)
|
||||
})
|
||||
|
||||
//API接口
|
||||
g := r.Group("/api") //初始化路由组 /api/xxxx
|
||||
{
|
||||
g.GET("/token", controller.ListTokenHandler) //获取token列表
|
||||
g.POST("/token", controller.CreateTokenHandler) //创建token
|
||||
g.PUT("/token", controller.UpdateTokenHandler) //更新token
|
||||
g.DELETE("/token", controller.DeleteTokenHandler) //删除token
|
||||
g.GET("/token/info", controller.GetTokenInfoHandler) //获取token信息
|
||||
g.DELETE("/token/info", controller.DeleteTokenInfoHandler) //删除token数据库
|
||||
}
|
||||
{
|
||||
g.GET("/data", controller.ReadDataHandler) //获取数据
|
||||
g.POST("/data", controller.WriteDataHandler) //写入数据
|
||||
}
|
||||
//注册网页服务(Vue)
|
||||
api.RegWebService(r, webDir)
|
||||
//注册API接口
|
||||
api.RegRoutes(r)
|
||||
|
||||
// 监听并在 0.0.0.0:8080 上启动服务
|
||||
fmt.Printf("服务器正在运行:http://%s\n", config.APPConfig.Host)
|
||||
|
||||
2
tool/.gitignore
vendored
2
tool/.gitignore
vendored
@@ -1,2 +0,0 @@
|
||||
./upload
|
||||
config.toml
|
||||
@@ -1,179 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
var httpClient = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 200,
|
||||
MaxIdleConnsPerHost: 100,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
},
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
func main() {
|
||||
initConfig()
|
||||
|
||||
//检测./upload
|
||||
fmt.Println("程序启动成功,正在检测txt文件")
|
||||
for {
|
||||
files, err := getTxtFiles("./")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
if files != nil {
|
||||
start := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
for _, filePath := range files {
|
||||
fmt.Println("正在上传文件:", filePath)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
processFile(filePath)
|
||||
err := os.Truncate(filePath, 0)
|
||||
if err != nil {
|
||||
fmt.Println("清空文件失败:", err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
fmt.Printf("上传完成,耗时:%s\n", time.Since(start))
|
||||
}
|
||||
time.Sleep(time.Minute)
|
||||
}
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
//程序配置
|
||||
viper.SetDefault("url", "http://localhost:8080")
|
||||
viper.SetDefault("token", "")
|
||||
viper.SetDefault("thread-count", 10)
|
||||
//设置配置文件名和路径 ./config.toml
|
||||
viper.AddConfigPath(".")
|
||||
viper.SetConfigName("config")
|
||||
viper.SetConfigType("toml")
|
||||
viper.SafeWriteConfig() //安全写入默认配置
|
||||
//读取配置文件
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
fmt.Errorf("无法读取配置文件: %w", err)
|
||||
}
|
||||
viper.WatchConfig()
|
||||
viper.OnConfigChange(func(e fsnotify.Event) {
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
fmt.Errorf("无法读取配置文件: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func uploadDataToServer(data string) error {
|
||||
params := url.Values{}
|
||||
params.Set("token", viper.GetString("token"))
|
||||
params.Set("data", data)
|
||||
|
||||
resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp != nil {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// 获取目录中的所有txt文件
|
||||
func getTxtFiles(dir string) (txtFiles []string, err error) {
|
||||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 只处理普通文件,跳过目录
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查文件扩展名是否为.txt
|
||||
if strings.ToLower(filepath.Ext(path)) == ".txt" {
|
||||
txtFiles = append(txtFiles, path)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return txtFiles, err
|
||||
}
|
||||
|
||||
func processFile(filePath string) {
|
||||
var wg sync.WaitGroup
|
||||
// 打开文件
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
fmt.Printf("无法打开文件 %s: %v\n", filePath, err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 创建行通道
|
||||
lines := make(chan string, 100)
|
||||
|
||||
// 创建10个worker处理文件上传
|
||||
for i := 0; i < viper.GetInt("thread-count"); i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
processLines(lines, workerID, filePath)
|
||||
}(i)
|
||||
}
|
||||
|
||||
// 读取文件并发送到通道
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineCount := 0
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
lines <- line
|
||||
lineCount++
|
||||
if lineCount%10000 == 0 {
|
||||
fmt.Printf("文件【%s】处理进度:%v%%\n", filePath, float64(lineCount)/40000*100)
|
||||
}
|
||||
}
|
||||
|
||||
close(lines)
|
||||
wg.Wait()
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("文件【%s】处理完成,共处理 %d 行数据\n", filePath, lineCount)
|
||||
}
|
||||
|
||||
func processLines(lines <-chan string, workerID int, filePath string) {
|
||||
for line := range lines {
|
||||
// 跳过空行
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
// 上传数据
|
||||
if err := uploadDataToServer(line); err != nil {
|
||||
fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user