diff --git a/DockerFile b/DockerFile index 8121c44..4dc1a52 100644 --- a/DockerFile +++ b/DockerFile @@ -1,4 +1,4 @@ -FROM golang:1.21 as BUILDER +FROM golang:1.24 as BUILDER WORKDIR /home COPY . . diff --git a/auth/client.go b/auth/client.go index aa3edf5..8d4e71c 100644 --- a/auth/client.go +++ b/auth/client.go @@ -1,12 +1,14 @@ package auth import ( + "context" "encoding/json" "errors" "fmt" "io" "net/http" "strings" + "time" "github.com/sirupsen/logrus" ) @@ -16,14 +18,20 @@ type Client struct { // getParsedResponse gets response data from gitee func getParsedResponse(method, path string, header http.Header, body io.Reader, obj interface{}) error { + client := &http.Client{ + Timeout: 30 * time.Second, // 整个请求的最大超时时间 + } req, err := http.NewRequest(method, path, body) if err != nil { panic(err) } req.Header = header - response, err := http.DefaultClient.Do(req) + response, err := client.Do(req) if err != nil { - panic(err) + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("请求超时: %w", err) + } + return fmt.Errorf("请求执行失败: %w", err) } defer response.Body.Close() diff --git a/auth/gitee.go b/auth/gitee.go index 8a9729f..ab172a7 100644 --- a/auth/gitee.go +++ b/auth/gitee.go @@ -22,11 +22,13 @@ var ( clientId string clientSecret string defaultToken string + defaultGiteCodeToken string + gitCodeSwitch bool openEulerAccountParam batch.OpenEulerAccountParam ) var ( - allowedRepos = []string{"openeuler", "src-openeuler", "lfs-org"} + allowedRepos = []string{"openeuler", "src-openeuler", "lfs-org", "openeuler-test"} uploadPermissions = []string{"admin", "developer"} downloadPermissions = []string{"admin", "developer", "read"} ) @@ -43,6 +45,7 @@ const ( contentType = "Content-Type" authorization = "Authorization" acceptEncoding = "Accept-Encoding" + headerContentType = "application/json;charset=UTF-8" formatLogString = "%s | %s" openEulerGetUserInfo = "/oneid/manager/personal/center/user" appendPathAccessToken = "?access_token=" @@ -101,7 +104,15 @@ func Init(cfg *config.Config) error { return errors.New("default token required") } } + defaultGiteCodeToken = cfg.DefaultGitCodeToken + if defaultGiteCodeToken == "" { + defaultGiteCodeToken = os.Getenv("GITE_CODE_TOKEN") + if defaultGiteCodeToken == "" { + return errors.New("defaultGiteCode token required") + } + } + gitCodeSwitch = cfg.GitCodeSwitch return nil } @@ -133,7 +144,10 @@ func InitOpenEulerParam(cfg *config.Config) error { func GiteeAuth() func(UserInRepo) error { return func(userInRepo UserInRepo) error { - if userInRepo.Password != "" { + if gitCodeSwitch { + userInRepo.Token = userInRepo.Password + } + if userInRepo.Password != "" && !gitCodeSwitch { token, err := getToken(userInRepo.Username, userInRepo.Password) if err != nil { userInRepo.Token = userInRepo.Password @@ -163,13 +177,30 @@ func CheckRepoOwner(userInRepo UserInRepo) (Repo, error) { } else { path += defaultToken } - headers := http.Header{contentType: []string{"application/json;charset=UTF-8"}} + headers := http.Header{contentType: []string{headerContentType}} repo := new(Repo) err := getParsedResponse("GET", path, headers, nil, &repo) - if err != nil { + gitCodePath := fmt.Sprintf( + "https://api.gitcode.com/api/v5/repos/%s/%s%s", + userInRepo.Owner, + userInRepo.Repo, + appendPathAccessToken, + ) + if userInRepo.Token != "" { + gitCodePath += userInRepo.Token + } else { + gitCodePath += defaultGiteCodeToken + } + headersGitCode := http.Header{contentType: []string{headerContentType}} + gitCodeRepo := new(Repo) + gitCodeErr := getParsedResponse("GET", gitCodePath, headersGitCode, nil, &gitCodeRepo) + if err != nil && gitCodeErr != nil { msg := err.Error() + ": check repo_id failed" return *repo, errors.New(msg) } + if gitCodeErr == nil { + repo = gitCodeRepo + } for _, allowedRepo := range allowedRepos { if strings.Split(repo.Fullname, "/")[0] == allowedRepo { return *repo, nil @@ -224,12 +255,29 @@ func VerifyUser(userInRepo UserInRepo) error { } else { path += defaultToken } - headers := http.Header{contentType: []string{"application/json;charset=UTF-8"}} + if gitCodeSwitch { + path = fmt.Sprintf( + "https://api.gitcode.com/api/v5/repos/%s/%s/collaborators/%s/permission%s", + userInRepo.Owner, + userInRepo.Repo, + userInRepo.Username, + appendPathAccessToken, + ) + if userInRepo.Token != "" { + path += userInRepo.Token + } else { + path += defaultGiteCodeToken + } + } + headers := http.Header{contentType: []string{headerContentType}} giteeUser := new(giteeUser) err := getParsedResponse("GET", path, headers, nil, &giteeUser) if err != nil { if userInRepo.Operation == "delete" { msg := err.Error() + ": 删除权限校验失败,用户使用的gitee token错误或已经过期,请重新使用gitee登录" + if gitCodeSwitch { + msg = err.Error() + ": 删除权限校验失败,用户使用的gitCode token错误或已经过期,请重新使用gitCode登录" + } return errors.New(msg) } else { msg := err.Error() + ": verify user permission failed" @@ -407,6 +455,12 @@ func GetLFSMapping(userInRepo UserInRepo, pythonScriptPath ...string) (map[strin repo := userInRepo.Repo username := userInRepo.Username token := userInRepo.Token + var platform string + if gitCodeSwitch { + platform = "gitcode" + } else { + platform = "gitee" + } // 确定Python脚本路径 scriptPath, err := resolveScriptPath(pythonScriptPath...) @@ -424,6 +478,7 @@ func GetLFSMapping(userInRepo UserInRepo, pythonScriptPath ...string) (map[strin // 构建并执行命令 cmd := exec.Command("python3") cmd.Args = append(cmd.Args, scriptPath) + cmd.Args = append(cmd.Args, platform) cmd.Args = append(cmd.Args, owner) cmd.Args = append(cmd.Args, repo) cmd.Args = append(cmd.Args, outputFile) diff --git a/auth/gitee_test.go b/auth/gitee_test.go index 8143509..e7f6642 100644 --- a/auth/gitee_test.go +++ b/auth/gitee_test.go @@ -26,9 +26,10 @@ func (s *SuiteGitee) SetupSuite() { s.UserName = "user" s.Password = "wrong_pwd" s.cfg = config.Config{ - ClientId: "clientId", - ClientSecret: "clientSecret", - DefaultToken: "defaultToken", + ClientId: "clientId", + ClientSecret: "clientSecret", + DefaultToken: "defaultToken", + DefaultGitCodeToken: "defaultGiteCode", OpenEulerAccountConfig: config.OpenEulerAccountConfig{ AppId: "appId", UrlPath: "urlPath", diff --git a/config/config.go b/config/config.go index 4930090..035e279 100644 --- a/config/config.go +++ b/config/config.go @@ -16,11 +16,13 @@ type Config struct { CdnDomain string `json:"CDN_DOMAIN"` ObsRegion string `json:"OBS_REGION"` DefaultToken string `json:"DEFAULT_TOKEN"` + DefaultGitCodeToken string `json:"DEFAULT_GIT_CODE_TOKEN"` ValidateConfig ValidateConfig `json:"VALIDATE_REGEXP"` ObsAccessKeyId string `json:"OBS_ACCESS_KEY_ID"` ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"` OpenEulerAccountConfig OpenEulerAccountConfig `json:"OPENEULER_ACCOUNT_PARAM"` DBConfig DBConfig `json:"DATABASE"` + GitCodeSwitch bool `json:"GIT_CODE_SWITCH" default:"false"` } type ValidateConfig struct { diff --git a/docs/BasicGuide.md b/docs/BasicGuide.md index f612aa5..0a7231a 100644 --- a/docs/BasicGuide.md +++ b/docs/BasicGuide.md @@ -53,18 +53,19 @@ Git LFS initialized. ``` [lfs] - url = https://artifacts.openeuler.openatom.cn/{owner}/{repo} + url = https://artlfs.openeuler.openatom.cn/{owner}/{repo} ``` - 或者通过命令行设置仓库中LFS远程地址: ``` -git config --local lfs.url https://artifacts.openeuler.openatom.cn/{owner}/{repo} +git config --local lfs.url https://artlfs.openeuler.openatom.cn/{owner}/{repo} ``` > - 当存在.lfsconfig文件时,使用命令行进行LFS远程地址设置的优先级将高于.lfsconfig文件。 > - 在fork一个已经使用第三方LFS服务服务作为LFS远程服务的仓库后,需要需手动使用上述命令设置仓库中LFS远程地址,否则可能会出现权限校验问题,**错误代码401**。 > - url中{owner}/{repo}替换为实际的仓库路径,注意仓库路径的大小写。 +> - Artlfs服务在迁移到AtomGit平台后,需要使用账号-token的方式进行身份验证。 - 选择要用LFS追踪的文件 diff --git a/docs/QuickStart.md b/docs/QuickStart.md index ccf0b7f..9507bf6 100644 --- a/docs/QuickStart.md +++ b/docs/QuickStart.md @@ -12,17 +12,18 @@ ``` [lfs] - url = https://artifacts.openeuler.openatom.cn/{owner}/{repo} + url = https://artlfs.openeuler.openatom.cn/{owner}/{repo} ``` - 或者通过命令行设置仓库中LFS远程地址: ``` -$ git config --local lfs.url https://artifacts.openeuler.openatom.cn/{owner}/{repo} +$ git config --local lfs.url https://artlfs.openeuler.openatom.cn/{owner}/{repo} ``` > 当存在.lfsconfig文件时,使用命令行进行LFS远程地址设置的优先级将高于.lfsconfig文件。 -> url中{owner}/{repo}替换为实际的仓库路径,如:openeuler/lfs。由于Gitee默认会将仓库路径中的大写转化为小写,请确认仓库路径的大小写。 +> url中{owner}/{repo}替换为实际的仓库路径,如:openeuler/lfs。由于Gitee默认会将仓库路径中的大写转化为小写,请确认仓库路径的大小写。 +> Artlfs服务在迁移到AtomGit平台后,需要使用账号-token的方式进行身份验证。 ## 第三方LFS服务与Gitee的使用差异 @@ -31,7 +32,7 @@ $ git config --local lfs.url https://artifacts.openeuler.openatom.cn/{owner}/{re - 当您fork一个仓库:将fork仓库克隆到本地后,需手动使用如下命令修改本地仓库的lfs配置: ``` - $ git config --local lfs.url https://artifacts.openeuler.openatom.cn/{owner}/{repo} + $ git config --local lfs.url https://artlfs.openeuler.openatom.cn/{owner}/{repo} ``` - 当您使用ssh协议进行克隆或推送:克隆或推送大文件时仍需输入用户名和密码进行认证。 diff --git a/go.mod b/go.mod index 6233521..832ad2d 100644 --- a/go.mod +++ b/go.mod @@ -1,36 +1,36 @@ module github.com/metalogical/BigFiles -go 1.21 +go 1.24.0 -toolchain go1.21.4 +toolchain go1.24.11 require ( bou.ke/monkey v1.0.2 github.com/go-chi/chi v4.1.2+incompatible - gorm.io/gorm v1.25.12 - sigs.k8s.io/yaml v1.4.0 + gorm.io/driver/mysql v1.6.0 + gorm.io/gorm v1.31.1 + sigs.k8s.io/yaml v1.6.0 ) require ( filippo.io/edwards25519 v1.1.0 // indirect - github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect - github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - gorm.io/driver/mysql v1.5.7 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( - github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.9+incompatible github.com/sirupsen/logrus v1.9.3 - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect ) diff --git a/go.sum b/go.sum index 8d3c04b..bf6ade0 100644 --- a/go.sum +++ b/go.sum @@ -2,25 +2,24 @@ bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= -github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= +github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible h1:XQVXdk+WAJ4fSNB6mMRuYNvFWou7BZs6SZB925hPrnk= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.9+incompatible h1:T9+wBrjfJUrWKppRwXhDNjf6vAJy7DfZYWgkjNbxkIU= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -29,13 +28,27 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= +go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -43,8 +56,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= -gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= +gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/main.go b/main.go index 7a86b3b..f2a6530 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func gatherOptions(fs *flag.FlagSet, args ...string) (options, error) { } func initConfig(cfg *config.Config) { - if err := server.Init(cfg.ValidateConfig); err != nil { + if err := server.Init(cfg); err != nil { logrus.Errorf("load ValidateConfig, err:%s", err.Error()) return } diff --git a/scripts/lfsNameQuery.py b/scripts/lfsNameQuery.py index 59a469a..62a2628 100644 --- a/scripts/lfsNameQuery.py +++ b/scripts/lfsNameQuery.py @@ -5,15 +5,39 @@ import shutil from urllib.parse import quote_plus +# 平台配置映射 +PLATFORM_CONFIGS = { + "gitee": { + "domain": "gitee.com", + "auth_method": "token" # 或 "username_password" + }, + "gitcode": { + "domain": "gitcode.com", + "auth_method": "token" # GitCode必须使用Token认证[6](@ref) + } +} + +def clone_repo_skip_lfs(platform, owner, repo, username=None, token=None, target_dir="temp_repo"): + """克隆仓库并强制跳过LFS文件下载,支持多平台""" + if platform not in PLATFORM_CONFIGS: + raise ValueError(f"不支持的平台: {platform},支持的平台: {list(PLATFORM_CONFIGS.keys())}") + + config = PLATFORM_CONFIGS[platform] + domain = config["domain"] -def clone_repo_skip_lfs(gitee_owner, gitee_repo, username=None, token=None, target_dir="temp_repo"): - """克隆仓库并强制跳过LFS文件下载""" + # 构建认证URL if username and token: encoded_username = quote_plus(username) encoded_token = quote_plus(token) - repo_url = f"https://{encoded_username}:{encoded_token}@gitee.com/{gitee_owner}/{gitee_repo}.git" + repo_url = f"https://{encoded_username}:{encoded_token}@{domain}/{owner}/{repo}.git" + elif token and config["auth_method"] == "token": + # GitCode推荐方式:使用用户名+Token[6](@ref) + encoded_token = quote_plus(token) + # 假设username作为GitCode用户名 + username = username or "gitcode_user" + repo_url = f"https://{username}:{encoded_token}@{domain}/{owner}/{repo}.git" else: - repo_url = f"https://gitee.com/{gitee_owner}/{gitee_repo}.git" + repo_url = f"https://{domain}/{owner}/{repo}.git" force_remove(target_dir) @@ -24,6 +48,8 @@ def clone_repo_skip_lfs(gitee_owner, gitee_repo, username=None, token=None, targ "GIT_CLONE_PROTECTION_ACTIVE": "false" }) + print(f"正在克隆 {platform} 仓库: {repo_url.replace(encoded_token, '***') if token else repo_url}") + subprocess.run( ["git", "clone", repo_url, target_dir], env=env, @@ -34,10 +60,15 @@ def clone_repo_skip_lfs(gitee_owner, gitee_repo, username=None, token=None, targ ) return target_dir except subprocess.CalledProcessError as e: - error_msg = "克隆失败: " + error_msg = f"{platform}克隆失败: " if username and token: error_msg += "认证失败或" error_msg += e.stderr.strip() if e.stderr else '未知错误' + + # 平台特定的错误提示 + if platform == "gitcode": + error_msg += "\nGitCode提示: 请确认使用Token认证而非密码[6](@ref)" + force_remove(target_dir) raise RuntimeError(error_msg) @@ -123,15 +154,19 @@ def force_remove(path): os.system(f'rm -rf "{path}"' if os.name != 'nt' else f'rd /s /q "{path}"') -def main(owner, repo, output_file="lfs_mapping.json", username=None, token=None): +def main(platform, owner, repo, output_file="lfs_mapping.json", username=None, token=None): + """主函数,支持平台参数""" try: - repo_dir = clone_repo_skip_lfs(owner, repo, username, token) + if platform == "gitcode" and not token: + print("警告: GitCode强烈建议使用Token认证而非密码[6](@ref)") + + repo_dir = clone_repo_skip_lfs(platform, owner, repo, username, token) mapping = get_all_branches_lfs_mapping(repo_dir) with open(output_file, "w", encoding="utf-8") as f: json.dump(mapping, f, indent=2, ensure_ascii=False) - print(f"结果已保存到 {output_file}") + print(f"{platform}平台结果已保存到 {output_file}") return True except Exception as e: print(f"错误: {str(e)}", file=sys.stderr) @@ -140,18 +175,19 @@ def main(owner, repo, output_file="lfs_mapping.json", username=None, token=None) if 'repo_dir' in locals(): force_remove(repo_dir) - if __name__ == "__main__": - if len(sys.argv) < 3: - print("用法: python lfsNameQuery.py [output_file] [username] [token]") + if len(sys.argv) < 4: + print("用法: python lfsNameQuery.py [output_file] [username] [token]") + print("平台支持: gitee, gitcode") sys.exit(1) args = { - "owner": sys.argv[1], - "repo": sys.argv[2], - "output_file": sys.argv[3] if len(sys.argv) > 3 else "lfs_mapping.json", - "username": sys.argv[4] if len(sys.argv) > 4 else None, - "token": sys.argv[5] if len(sys.argv) > 5 else None + "platform": sys.argv[1], # 新增平台参数 + "owner": sys.argv[2], + "repo": sys.argv[3], + "output_file": sys.argv[4] if len(sys.argv) > 4 else "lfs_mapping.json", + "username": sys.argv[5] if len(sys.argv) > 5 else None, + "token": sys.argv[6] if len(sys.argv) > 6 else None } - sys.exit(0 if main( ** args) else 1) + sys.exit(0 if main(**args) else 1) diff --git a/server/server.go b/server/server.go index cb4727e..acf072f 100644 --- a/server/server.go +++ b/server/server.go @@ -170,6 +170,10 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { } func addMetaData(req batch.Request, w http.ResponseWriter, userInRepo auth.UserInRepo) { + platform := "gitee" + if gitCodeSwitch { + platform = "atomGit" + } // 添加元数据 if req.Operation == "upload" { for _, object := range req.Objects { @@ -179,7 +183,7 @@ func addMetaData(req batch.Request, w http.ResponseWriter, userInRepo auth.UserI Oid: object.OID, Size: object.Size, Exist: 2, // 默认设置为2 - Platform: "gitee", // 默认平台 + Platform: platform, // 默认平台 Operator: userInRepo.Username, // 操作人 } @@ -500,8 +504,13 @@ func (s *server) getLfsFiles(owner, repo, platform string, page, limit int) ([]d var files []db.LfsObj query := db.Db.Model(&db.LfsObj{}). - Where("owner = ? AND repo = ? AND platform = ? AND exist = 1", owner, repo, platform). - Order("create_time DESC"). + Where("owner = ? AND repo = ? AND exist = 1", owner, repo) + + if platform != "" { + query = query.Where("platform = ?", platform) + } + + query = query.Order("create_time DESC"). Limit(limit). Offset((page - 1) * limit) @@ -583,10 +592,15 @@ func checkOidFileName() { logrus.Errorf("fetch repo list failed: %v", err) return } + token := giteeDefaultToken + if gitCodeSwitch { + token = atomGiteDefaultToken + } for _, repo := range repoList { userInRepo := auth.UserInRepo{ Repo: repo.Repo, - Owner: repo.Owner} + Owner: repo.Owner, + Token: token} logrus.Infof("checkOidFileName owner:%v repo:%v", repo.Owner, repo.Repo) checkRepoOidName(userInRepo) diff --git a/server/validate.go b/server/validate.go index 5decc4f..82c3b52 100644 --- a/server/validate.go +++ b/server/validate.go @@ -15,27 +15,34 @@ type validateConfig struct { var validatecfg validateConfig var Webhook_key string - -func Init(cfg config.ValidateConfig) error { +var gitCodeSwitch bool +var giteeDefaultToken string +var atomGiteDefaultToken string + +func Init(cfg *config.Config) error { + validateConfig := cfg.ValidateConfig + gitCodeSwitch = cfg.GitCodeSwitch + giteeDefaultToken = cfg.DefaultToken + atomGiteDefaultToken = cfg.DefaultGitCodeToken var err error - Webhook_key = cfg.WebhookKey + Webhook_key = validateConfig.WebhookKey - validatecfg.ownerRegexp, err = regexp.Compile(cfg.OwnerRegexp) + validatecfg.ownerRegexp, err = regexp.Compile(validateConfig.OwnerRegexp) if err != nil { return fmt.Errorf("failed to compile owner regexp: %w", err) } - validatecfg.reponameRegexp, err = regexp.Compile(cfg.RepoNameRegexp) + validatecfg.reponameRegexp, err = regexp.Compile(validateConfig.RepoNameRegexp) if err != nil { return fmt.Errorf("failed to compile repo name regexp: %w", err) } - validatecfg.usernameRegexp, err = regexp.Compile(cfg.UsernameRegexp) + validatecfg.usernameRegexp, err = regexp.Compile(validateConfig.UsernameRegexp) if err != nil { return fmt.Errorf("failed to compile username regexp: %w", err) } - validatecfg.passwordRegexp, err = regexp.Compile(cfg.PasswordRegexp) + validatecfg.passwordRegexp, err = regexp.Compile(validateConfig.PasswordRegexp) if err != nil { return fmt.Errorf("failed to compile password regexp: %w", err) } diff --git a/server/validate_test.go b/server/validate_test.go index 6f696f4..388158d 100644 --- a/server/validate_test.go +++ b/server/validate_test.go @@ -8,50 +8,59 @@ import ( func TestInit(t *testing.T) { tests := []struct { name string - args config.ValidateConfig + args *config.Config wantErr bool }{ { name: "compile owner regexp failed", - args: config.ValidateConfig{ - OwnerRegexp: `^[\\-?]$`, + args: &config.Config{ + ValidateConfig: config.ValidateConfig{ + OwnerRegexp: `^[\\-?]$`}, }, wantErr: true, }, { name: "compile repo regexp failed", - args: config.ValidateConfig{ - OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - RepoNameRegexp: `^[\\-?]$`, + args: &config.Config{ + ValidateConfig: config.ValidateConfig{ + OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + RepoNameRegexp: `^[\\-?]$`, + }, }, wantErr: true, }, { name: "compile username regexp failed", - args: config.ValidateConfig{ - OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, - UsernameRegexp: `^[\\-?]$`, + args: &config.Config{ + ValidateConfig: config.ValidateConfig{ + OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, + UsernameRegexp: `^[\\-?]$`, + }, }, wantErr: true, }, { name: "compile password regexp failed", - args: config.ValidateConfig{ - OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, - UsernameRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - PasswordRegexp: `^[\\-?]$`, + args: &config.Config{ + ValidateConfig: config.ValidateConfig{ + OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, + UsernameRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + PasswordRegexp: `^[\\-?]$`, + }, }, wantErr: true, }, { name: "compile regexp success", - args: config.ValidateConfig{ - OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, - UsernameRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, - PasswordRegexp: `^[a-zA-Z0-9!@_#$%^&*()-=+,?.,]*$`, + args: &config.Config{ + ValidateConfig: config.ValidateConfig{ + OwnerRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + RepoNameRegexp: `^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$`, + UsernameRegexp: `^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$`, + PasswordRegexp: `^[a-zA-Z0-9!@_#$%^&*()-=+,?.,]*$`, + }, }, wantErr: false, }, diff --git a/typos.toml b/typos.toml index 9a1e9b4..a3c872b 100644 --- a/typos.toml +++ b/typos.toml @@ -1,14 +1,14 @@ [files] extend-exclude = [ -"*.yaml", -"*.yml", -"*.md", -"*.sh", -"*.tpl", -"go.mod", -"go.sum", -"tests/", -"api/", + "*.yaml", + "*.yml", + "*.md", + "*.sh", + "*.tpl", + "go.mod", + "go.sum", + "tests/", + "api/", ] ignore-hidden = true ignore-files = true @@ -19,11 +19,8 @@ ignore-parent = true [default] locale = "en" -extend-glob = [] -extend-ignore-identifiers = [] extend-ignore-words-re = [] extend-ignore-re = [] - [default.extend-identifiers] cann_version = "cann_version" CannVersion = "CannVersion"