feat: 添加文件上传工具及CI/CD配置

2025-10-11 12:34:06 +08:00
parent cd068268b0
commit 0407102d70
5 changed files with 360 additions and 0 deletions
--- a/.gitea/workflows/build_tool.yaml
+++ b/.gitea/workflows/build_tool.yaml
@@ -0,0 +1,37 @@
+name: 构建上传工具
+on: [ push ]
+
+jobs:
+  build-tool:
+    env:
+      RUNNER_TOOL_CACHE: /toolcache
+    runs-on: ubuntu-latest
+    steps:
+      - name: 检出代码
+        uses: actions/checkout@v4
+
+      - name: 安装Go（镜像）
+        run: |
+          # 使用国内镜像下载 Go
+          wget https://mirrors.aliyun.com/golang/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
+          # 解压并设置环境变量
+          sudo rm -rf /usr/local/go
+          sudo tar -C /usr/local -xzf go.tar.gz
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+        env:
+          GOROOT: /usr/local/go
+
+      - name: 构建上传工具
+        run: |
+          go env -w CGO_ENABLED=0 \
+          && go env -w GOARCH=amd64 \
+          && go env -w GOOS=windows \
+          && go mod tidy \
+          && cd ./tool \
+          && go build -o 上传工具.exe
+
+      - name: 上传构建文件
+        uses: actions/upload-artifact@v3
+        with:
+          name: 上传工具
+          path: tool/上传工具.exe
--- a/api/api.go
+++ b/api/api.go
@@ -0,0 +1,25 @@
+package api
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+func uploadDataToServer(httpClient *http.Client, data string) error {
+	params := url.Values{}
+	params.Set("token", viper.GetString("token"))
+	params.Set("data", data)
+
+	resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
+	if err != nil {
+		return err
+	}
+	if resp != nil {
+		_, _ = io.Copy(io.Discard, resp.Body)
+	}
+	return err
+}
--- a/go.mod
+++ b/go.mod
@@ -0,0 +1,23 @@
+module dypid-client
+
+go 1.25.1
+
+require (
+	github.com/fsnotify/fsnotify v1.9.0
+	github.com/spf13/viper v1.21.0
+	golang.org/x/sync v0.17.0
+)
+
+require (
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+)
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,49 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
+github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
+github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
+github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/main.go
+++ b/main.go
@@ -0,0 +1,226 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/viper"
+	"golang.org/x/sync/errgroup"
+)
+
+var httpClient = &http.Client{
+	Transport: &http.Transport{
+		MaxIdleConns:        200,
+		MaxIdleConnsPerHost: 100,
+		IdleConnTimeout:     30 * time.Second,
+	},
+	Timeout: 30 * time.Second,
+}
+
+type Task struct {
+	FilePath  string
+	FileLines int
+}
+
+func main() {
+	initConfig()
+
+	//检测./upload
+	fmt.Println("程序启动成功，正在检测txt文件")
+	for {
+		files, err := getTxtFiles("./")
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		if files != nil {
+			start := time.Now()
+
+			fileLines := make(map[string]int)
+			fmt.Println("正在统计", len(files), "个文件行数")
+			for _, filePath := range files {
+				file, err := os.Open(filePath)
+				if err != nil {
+					fmt.Println("打开文件失败：", err)
+				}
+				// 使用 bufio.Scanner 逐行读取
+				scanner := bufio.NewScanner(file)
+				lineCount := 0
+				for scanner.Scan() {
+					lineCount++
+				}
+				file.Close()
+				if lineCount == 0 {
+					continue
+				}
+				fileLines[filepath.Base(filePath)] = lineCount
+				fmt.Println(filepath.Base(filePath), "文件行数：", lineCount)
+
+			}
+
+			var tasks []Task
+			for k, v := range fileLines {
+				tasks = append(tasks, Task{FilePath: k, FileLines: v})
+			}
+
+			// 使用errgroup控制并发
+			g, ctx := errgroup.WithContext(context.Background())
+			g.SetLimit(50) // 设置最大并发数为50
+			// 执行所有任务
+			for _, task := range tasks {
+				task := task // 创建局部变量
+				g.Go(func() error {
+					select {
+					case <-ctx.Done():
+						return ctx.Err()
+					default:
+						fmt.Println("正在上传文件:", filepath.Base(task.FilePath))
+						processFile(task.FilePath, task.FileLines)
+						err := os.Truncate(task.FilePath, 0)
+						if err != nil {
+							fmt.Println("清空文件失败：", err)
+						}
+						return nil
+					}
+				})
+			}
+
+			// 等待所有任务完成
+			if err := g.Wait(); err != nil {
+				fmt.Printf("任务执行出错: %v\n", err)
+			} else {
+				fmt.Println("所有任务执行完成！")
+			}
+
+			fmt.Printf("上传完成，耗时：%s\n", time.Since(start))
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func initConfig() {
+	//程序配置
+	viper.SetDefault("url", "http://localhost:8080")
+	viper.SetDefault("token", "")
+	viper.SetDefault("thread-count", 10)
+	//设置配置文件名和路径 ./config.toml
+	viper.AddConfigPath(".")
+	viper.SetConfigName("config")
+	viper.SetConfigType("toml")
+	viper.SafeWriteConfig() //安全写入默认配置
+	//读取配置文件
+	if err := viper.ReadInConfig(); err != nil {
+		fmt.Errorf("无法读取配置文件: %w", err)
+	}
+	viper.WatchConfig()
+	viper.OnConfigChange(func(e fsnotify.Event) {
+		if err := viper.ReadInConfig(); err != nil {
+			fmt.Errorf("无法读取配置文件: %w", err)
+		}
+	})
+}
+
+func uploadDataToServer(data string) error {
+	params := url.Values{}
+	params.Set("token", viper.GetString("token"))
+	params.Set("data", data)
+
+	resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
+	if err != nil {
+		return err
+	}
+	if resp != nil {
+		_, _ = io.Copy(io.Discard, resp.Body)
+	}
+	return err
+}
+
+// 获取目录中的所有txt文件
+func getTxtFiles(dir string) (txtFiles []string, err error) {
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// 只处理普通文件，跳过目录
+		if !info.Mode().IsRegular() {
+			return nil
+		}
+
+		// 检查文件扩展名是否为.txt
+		if strings.ToLower(filepath.Ext(path)) == ".txt" {
+			txtFiles = append(txtFiles, path)
+		}
+
+		return nil
+	})
+
+	return txtFiles, err
+}
+
+func processFile(filePath string, fileLines int) {
+	var wg sync.WaitGroup
+	// 打开文件
+	file, err := os.Open(filePath)
+	if err != nil {
+		fmt.Printf("无法打开文件 %s: %v\n", filePath, err)
+		return
+	}
+	defer file.Close()
+
+	// 创建行通道
+	lines := make(chan string, 100)
+
+	// 创建10个worker处理文件上传
+	for i := 0; i < viper.GetInt("thread-count"); i++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			processLines(lines, workerID, filePath)
+		}(i)
+	}
+
+	// 读取文件并发送到通道
+	scanner := bufio.NewScanner(file)
+	lineCount := 0
+	for scanner.Scan() {
+		lines <- scanner.Text()
+		lineCount++
+		if lineCount%10000 == 0 {
+			fmt.Printf("文件【%s】处理进度：%.2f%%\n", filePath, float64(lineCount)/float64(fileLines)*100)
+		}
+	}
+
+	close(lines)
+	wg.Wait()
+
+	if err := scanner.Err(); err != nil {
+		fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
+		return
+	}
+
+	fmt.Printf("文件【%s】处理完成，共处理 %d 行数据\n", filePath, lineCount)
+}
+
+func processLines(lines <-chan string, workerID int, filePath string) {
+	for line := range lines {
+		// 跳过空行
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		// 上传数据
+		if err := uploadDataToServer(line); err != nil {
+			fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err)
+		}
+	}
+}