From e35f8e15a67e8268542d2442dac32993b6944043 Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Sun, 5 Feb 2023 02:35:04 +0900 Subject: [PATCH 01/33] add default user visibility to cli command "admin user create" (#22750) Fixes https://github.com/go-gitea/gitea/issues/22523 --- cmd/admin.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/admin.go b/cmd/admin.go index 8f8c68f981..eafed24bdd 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -578,12 +578,16 @@ func runCreateUser(c *cli.Context) error { restricted = util.OptionalBoolOf(c.Bool("restricted")) } + // default user visibility in app.ini + visibility := setting.Service.DefaultUserVisibilityMode + u := &user_model.User{ Name: username, Email: c.String("email"), Passwd: password, IsAdmin: c.Bool("admin"), MustChangePassword: changePassword, + Visibility: visibility, } overwriteDefault := &user_model.CreateUserOverwriteOptions{ From 7baeb9c52a69eb6f7e0973986f2a6bebdd6352d0 Mon Sep 17 00:00:00 2001 From: ByLCY Date: Sun, 5 Feb 2023 15:29:03 +0800 Subject: [PATCH 02/33] Add new captcha: cloudflare turnstile (#22369) Added a new captcha(cloudflare turnstile) and its corresponding document. Cloudflare turnstile official instructions are here: https://developers.cloudflare.com/turnstile Signed-off-by: ByLCY Co-authored-by: Lunny Xiao Co-authored-by: Jason Song --- custom/conf/app.example.ini | 6 +- .../doc/advanced/config-cheat-sheet.en-us.md | 4 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 11 +++ modules/context/captcha.go | 11 ++- modules/setting/service.go | 4 + modules/setting/setting.go | 1 + modules/turnstile/turnstile.go | 92 +++++++++++++++++++ templates/base/footer.tmpl | 7 +- templates/user/auth/captcha.tmpl | 10 +- web_src/js/features/captcha.js | 51 ++++++++++ web_src/js/features/mcaptcha.js | 16 ---- web_src/js/index.js | 4 +- web_src/less/_form.less | 14 ++- 13 files changed, 199 insertions(+), 32 deletions(-) create mode 100644 modules/turnstile/turnstile.go create mode 100644 web_src/js/features/captcha.js delete mode 100644 web_src/js/features/mcaptcha.js diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 5a1edf9fbb..04ba2dc67a 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -765,7 +765,7 @@ ROUTER = console ;; Enable this to require captcha validation for login ;REQUIRE_CAPTCHA_FOR_LOGIN = false ;; -;; Type of captcha you want to use. Options: image, recaptcha, hcaptcha, mcaptcha. +;; Type of captcha you want to use. Options: image, recaptcha, hcaptcha, mcaptcha, cfturnstile. ;CAPTCHA_TYPE = image ;; ;; Change this to use recaptcha.net or other recaptcha service @@ -787,6 +787,10 @@ ROUTER = console ;MCAPTCHA_SECRET = ;MCAPTCHA_SITEKEY = ;; +;; Go to https://dash.cloudflare.com/?to=/:account/turnstile to sign up for a key +;CF_TURNSTILE_SITEKEY = +;CF_TURNSTILE_SECRET = +;; ;; Default value for KeepEmailPrivate ;; Each new user will get the value of this setting copied into their profile ;DEFAULT_KEEP_EMAIL_PRIVATE = false diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 67ca7a5166..9254962dc5 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -643,7 +643,7 @@ Certain queues have defaults that override the defaults set in `[queue]` (this o - `REQUIRE_CAPTCHA_FOR_LOGIN`: **false**: Enable this to require captcha validation for login. You also must enable `ENABLE_CAPTCHA`. - `REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA`: **false**: Enable this to force captcha validation even for External Accounts (i.e. GitHub, OpenID Connect, etc). You also must enable `ENABLE_CAPTCHA`. -- `CAPTCHA_TYPE`: **image**: \[image, recaptcha, hcaptcha, mcaptcha\] +- `CAPTCHA_TYPE`: **image**: \[image, recaptcha, hcaptcha, mcaptcha, cfturnstile\] - `RECAPTCHA_SECRET`: **""**: Go to https://www.google.com/recaptcha/admin to get a secret for recaptcha. - `RECAPTCHA_SITEKEY`: **""**: Go to https://www.google.com/recaptcha/admin to get a sitekey for recaptcha. - `RECAPTCHA_URL`: **https://www.google.com/recaptcha/**: Set the recaptcha url - allows the use of recaptcha net. @@ -652,6 +652,8 @@ Certain queues have defaults that override the defaults set in `[queue]` (this o - `MCAPTCHA_SECRET`: **""**: Go to your mCaptcha instance to get a secret for mCaptcha. - `MCAPTCHA_SITEKEY`: **""**: Go to your mCaptcha instance to get a sitekey for mCaptcha. - `MCAPTCHA_URL` **https://demo.mcaptcha.org/**: Set the mCaptcha URL. +- `CF_TURNSTILE_SECRET` **""**: Go to https://dash.cloudflare.com/?to=/:account/turnstile to get a secret for cloudflare turnstile. +- `CF_TURNSTILE_SITEKEY` **""**: Go to https://dash.cloudflare.com/?to=/:account/turnstile to get a sitekey for cloudflare turnstile. - `DEFAULT_KEEP_EMAIL_PRIVATE`: **false**: By default set users to keep their email address private. - `DEFAULT_ALLOW_CREATE_ORGANIZATION`: **true**: Allow new users to create organizations by default. - `DEFAULT_USER_IS_RESTRICTED`: **false**: Give new users restricted permissions by default diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index f10b6258c8..2598f16a14 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -147,6 +147,17 @@ menu: - `ENABLE_REVERSE_PROXY_AUTO_REGISTRATION`: 允许通过反向认证做自动注册。 - `ENABLE_CAPTCHA`: **false**: 注册时使用图片验证码。 - `REQUIRE_CAPTCHA_FOR_LOGIN`: **false**: 登录时需要图片验证码。需要同时开启 `ENABLE_CAPTCHA`。 +- `CAPTCHA_TYPE`: **image**: \[image, recaptcha, hcaptcha, mcaptcha, cfturnstile\]，人机验证类型，分别表示图片认证、 recaptcha 、 hcaptcha 、mcaptcha 、和 cloudlfare 的 turnstile。 +- `RECAPTCHA_SECRET`: **""**: recaptcha 服务的密钥，可在 https://www.google.com/recaptcha/admin 获取。 +- `RECAPTCHA_SITEKEY`: **""**: recaptcha 服务的网站密钥 ，可在 https://www.google.com/recaptcha/admin 获取。 +- `RECAPTCHA_URL`: **https://www.google.com/recaptcha/**: 设置 recaptcha 的 url 。 +- `HCAPTCHA_SECRET`: **""**: hcaptcha 服务的密钥，可在 https://www.hcaptcha.com/ 获取。 +- `HCAPTCHA_SITEKEY`: **""**: hcaptcha 服务的网站密钥，可在 https://www.hcaptcha.com/ 获取。 +- `MCAPTCHA_SECRET`: **""**: mCaptcha 服务的密钥。 +- `MCAPTCHA_SITEKEY`: **""**: mCaptcha 服务的网站密钥。 +- `MCAPTCHA_URL` **https://demo.mcaptcha.org/**: 设置 remCaptchacaptcha 的 url 。 +- `CF_TURNSTILE_SECRET` **""**: cloudlfare turnstile 服务的密钥，可在 https://dash.cloudflare.com/?to=/:account/turnstile 获取。 +- `CF_TURNSTILE_SITEKEY` **""**: cloudlfare turnstile 服务的网站密钥 ，可在 https://www.google.com/recaptcha/admin 获取。 ### Service - Expore (`service.explore`) diff --git a/modules/context/captcha.go b/modules/context/captcha.go index 735613504c..07232e9390 100644 --- a/modules/context/captcha.go +++ b/modules/context/captcha.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/mcaptcha" "code.gitea.io/gitea/modules/recaptcha" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/turnstile" "gitea.com/go-chi/captcha" ) @@ -47,12 +48,14 @@ func SetCaptchaData(ctx *Context) { ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey ctx.Data["McaptchaSitekey"] = setting.Service.McaptchaSitekey ctx.Data["McaptchaURL"] = setting.Service.McaptchaURL + ctx.Data["CfTurnstileSitekey"] = setting.Service.CfTurnstileSitekey } const ( - gRecaptchaResponseField = "g-recaptcha-response" - hCaptchaResponseField = "h-captcha-response" - mCaptchaResponseField = "m-captcha-response" + gRecaptchaResponseField = "g-recaptcha-response" + hCaptchaResponseField = "h-captcha-response" + mCaptchaResponseField = "m-captcha-response" + cfTurnstileResponseField = "cf-turnstile-response" ) // VerifyCaptcha verifies Captcha data @@ -73,6 +76,8 @@ func VerifyCaptcha(ctx *Context, tpl base.TplName, form interface{}) { valid, err = hcaptcha.Verify(ctx, ctx.Req.Form.Get(hCaptchaResponseField)) case setting.MCaptcha: valid, err = mcaptcha.Verify(ctx, ctx.Req.Form.Get(mCaptchaResponseField)) + case setting.CfTurnstile: + valid, err = turnstile.Verify(ctx, ctx.Req.Form.Get(cfTurnstileResponseField)) default: ctx.ServerError("Unknown Captcha Type", fmt.Errorf("Unknown Captcha Type: %s", setting.Service.CaptchaType)) return diff --git a/modules/setting/service.go b/modules/setting/service.go index 7b4bfc5c7b..1d33ac6bce 100644 --- a/modules/setting/service.go +++ b/modules/setting/service.go @@ -46,6 +46,8 @@ var Service = struct { RecaptchaSecret string RecaptchaSitekey string RecaptchaURL string + CfTurnstileSecret string + CfTurnstileSitekey string HcaptchaSecret string HcaptchaSitekey string McaptchaSecret string @@ -137,6 +139,8 @@ func newService() { Service.RecaptchaSecret = sec.Key("RECAPTCHA_SECRET").MustString("") Service.RecaptchaSitekey = sec.Key("RECAPTCHA_SITEKEY").MustString("") Service.RecaptchaURL = sec.Key("RECAPTCHA_URL").MustString("https://www.google.com/recaptcha/") + Service.CfTurnstileSecret = sec.Key("CF_TURNSTILE_SECRET").MustString("") + Service.CfTurnstileSitekey = sec.Key("CF_TURNSTILE_SITEKEY").MustString("") Service.HcaptchaSecret = sec.Key("HCAPTCHA_SECRET").MustString("") Service.HcaptchaSitekey = sec.Key("HCAPTCHA_SITEKEY").MustString("") Service.McaptchaURL = sec.Key("MCAPTCHA_URL").MustString("https://demo.mcaptcha.org/") diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 23cd90553e..a68a46f7ad 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -61,6 +61,7 @@ const ( ReCaptcha = "recaptcha" HCaptcha = "hcaptcha" MCaptcha = "mcaptcha" + CfTurnstile = "cfturnstile" ) // settings diff --git a/modules/turnstile/turnstile.go b/modules/turnstile/turnstile.go new file mode 100644 index 0000000000..38d0233446 --- /dev/null +++ b/modules/turnstile/turnstile.go @@ -0,0 +1,92 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package turnstile + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/setting" +) + +// Response is the structure of JSON returned from API +type Response struct { + Success bool `json:"success"` + ChallengeTS string `json:"challenge_ts"` + Hostname string `json:"hostname"` + ErrorCodes []ErrorCode `json:"error-codes"` + Action string `json:"login"` + Cdata string `json:"cdata"` +} + +// Verify calls Cloudflare Turnstile API to verify token +func Verify(ctx context.Context, response string) (bool, error) { + // Cloudflare turnstile official access instruction address: https://developers.cloudflare.com/turnstile/get-started/server-side-validation/ + post := url.Values{ + "secret": {setting.Service.CfTurnstileSecret}, + "response": {response}, + } + // Basically a copy of http.PostForm, but with a context + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + "https://challenges.cloudflare.com/turnstile/v0/siteverify", strings.NewReader(post.Encode())) + if err != nil { + return false, fmt.Errorf("Failed to create CAPTCHA request: %w", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, fmt.Errorf("Failed to send CAPTCHA response: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("Failed to read CAPTCHA response: %w", err) + } + + var jsonResponse Response + if err := json.Unmarshal(body, &jsonResponse); err != nil { + return false, fmt.Errorf("Failed to parse CAPTCHA response: %w", err) + } + + var respErr error + if len(jsonResponse.ErrorCodes) > 0 { + respErr = jsonResponse.ErrorCodes[0] + } + return jsonResponse.Success, respErr +} + +// ErrorCode is a reCaptcha error +type ErrorCode string + +// String fulfills the Stringer interface +func (e ErrorCode) String() string { + switch e { + case "missing-input-secret": + return "The secret parameter was not passed." + case "invalid-input-secret": + return "The secret parameter was invalid or did not exist." + case "missing-input-response": + return "The response parameter was not passed." + case "invalid-input-response": + return "The response parameter is invalid or has expired." + case "bad-request": + return "The request was rejected because it was malformed." + case "timeout-or-duplicate": + return "The response parameter has already been validated before." + case "internal-error": + return "An internal error happened while validating the response. The request can be retried." + } + return string(e) +} + +// Error fulfills the error interface +func (e ErrorCode) Error() string { + return e.String() +} diff --git a/templates/base/footer.tmpl b/templates/base/footer.tmpl index 4d3e08a597..e3cac806a4 100644 --- a/templates/base/footer.tmpl +++ b/templates/base/footer.tmpl @@ -16,10 +16,13 @@ {{if .EnableCaptcha}} {{if eq .CaptchaType "recaptcha"}} - + {{end}} {{if eq .CaptchaType "hcaptcha"}} - + + {{end}} + {{if eq .CaptchaType "cfturnstile"}} + {{end}} {{end}} diff --git a/templates/user/auth/captcha.tmpl b/templates/user/auth/captcha.tmpl index 87b22a0720..a794c8f543 100644 --- a/templates/user/auth/captcha.tmpl +++ b/templates/user/auth/captcha.tmpl @@ -9,16 +9,20 @@ {{else if eq .CaptchaType "recaptcha"}}
-
+
{{else if eq .CaptchaType "hcaptcha"}}
-
+
{{else if eq .CaptchaType "mcaptcha"}}
{{.locale.Tr "captcha"}}
-
+
+
+{{else if eq .CaptchaType "cfturnstile"}} +
+
{{end}}{{end}} diff --git a/web_src/js/features/captcha.js b/web_src/js/features/captcha.js new file mode 100644 index 0000000000..3da5dbda41 --- /dev/null +++ b/web_src/js/features/captcha.js @@ -0,0 +1,51 @@ +import {isDarkTheme} from '../utils.js'; + +export async function initCaptcha() { + const captchaEl = document.querySelector('#captcha'); + if (!captchaEl) return; + + const siteKey = captchaEl.getAttribute('data-sitekey'); + const isDark = isDarkTheme(); + + const params = { + sitekey: siteKey, + theme: isDark ? 'dark' : 'light' + }; + + switch (captchaEl.getAttribute('data-captcha-type')) { + case 'g-recaptcha': { + if (window.grecaptcha) { + window.grecaptcha.ready(() => { + window.grecaptcha.render(captchaEl, params); + }); + } + break; + } + case 'cf-turnstile': { + if (window.turnstile) { + window.turnstile.render(captchaEl, params); + } + break; + } + case 'h-captcha': { + if (window.hcaptcha) { + window.hcaptcha.render(captchaEl, params); + } + break; + } + case 'm-captcha': { + const {default: mCaptcha} = await import(/* webpackChunkName: "mcaptcha-vanilla-glue" */'@mcaptcha/vanilla-glue'); + mCaptcha.INPUT_NAME = 'm-captcha-response'; + const instanceURL = captchaEl.getAttribute('data-instance-url'); + + mCaptcha.default({ + siteKey: { + instanceUrl: new URL(instanceURL), + key: siteKey, + } + }); + break; + } + default: + } +} diff --git a/web_src/js/features/mcaptcha.js b/web_src/js/features/mcaptcha.js deleted file mode 100644 index 725e2e28ac..0000000000 --- a/web_src/js/features/mcaptcha.js +++ /dev/null @@ -1,16 +0,0 @@ -export async function initMcaptcha() { - const mCaptchaEl = document.querySelector('.m-captcha'); - if (!mCaptchaEl) return; - - const {default: mCaptcha} = await import(/* webpackChunkName: "mcaptcha-vanilla-glue" */'@mcaptcha/vanilla-glue'); - mCaptcha.INPUT_NAME = 'm-captcha-response'; - const siteKey = mCaptchaEl.getAttribute('data-sitekey'); - const instanceURL = mCaptchaEl.getAttribute('data-instance-url'); - - mCaptcha.default({ - siteKey: { - instanceUrl: new URL(instanceURL), - key: siteKey, - } - }); -} diff --git a/web_src/js/index.js b/web_src/js/index.js index 14483f3fa2..74d80776b5 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -88,8 +88,8 @@ import {initCommonOrganization} from './features/common-organization.js'; import {initRepoWikiForm} from './features/repo-wiki.js'; import {initRepoCommentForm, initRepository} from './features/repo-legacy.js'; import {initFormattingReplacements} from './features/formatting.js'; -import {initMcaptcha} from './features/mcaptcha.js'; import {initCopyContent} from './features/copycontent.js'; +import {initCaptcha} from './features/captcha.js'; import {initRepositoryActionView} from './components/RepoActionView.vue'; // Run time-critical code as soon as possible. This is safe to do because this @@ -191,7 +191,7 @@ $(document).ready(() => { initRepositoryActionView(); initCommitStatuses(); - initMcaptcha(); + initCaptcha(); initUserAuthLinkAccountView(); initUserAuthOauth2(); diff --git a/web_src/less/_form.less b/web_src/less/_form.less index 3d2ec9fb8a..1a1c1678f8 100644 --- a/web_src/less/_form.less +++ b/web_src/less/_form.less @@ -220,18 +220,24 @@ textarea:focus, } @media @mediaMdAndUp { - .g-recaptcha, - .h-captcha { + .g-recaptcha-style, + .h-captcha-style { margin: 0 auto !important; width: 304px; padding-left: 30px; + + iframe { + border-radius: 5px !important; + width: 302px !important; + height: 76px !important; + } } } @media (max-height: 575px) { #rc-imageselect, - .g-recaptcha, - .h-captcha { + .g-recaptcha-style, + .h-captcha-style { transform: scale(.77); transform-origin: 0 0; } From df789d962b3d568a77773dd53f98eb37c8bd1f6b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 5 Feb 2023 11:12:31 +0100 Subject: [PATCH 03/33] Add Cargo package registry (#21888) This PR implements a [Cargo registry](https://doc.rust-lang.org/cargo/) to manage Rust packages. This package type was a little bit more complicated because Cargo needs an additional Git repository to store its package index. Screenshots: ![grafik](https://user-images.githubusercontent.com/1666336/203102004-08d812ac-c066-4969-9bda-2fed818554eb.png) ![grafik](https://user-images.githubusercontent.com/1666336/203102141-d9970f14-dca6-4174-b17a-50ba1bd79087.png) ![grafik](https://user-images.githubusercontent.com/1666336/203102244-dc05743b-78b6-4d97-998e-ef76341a978f.png) --------- Co-authored-by: Lunny Xiao --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + docs/content/doc/packages/cargo.en-us.md | 109 ++++++ docs/content/doc/packages/overview.en-us.md | 1 + models/packages/descriptor.go | 3 + models/packages/package.go | 6 + models/packages/package_property.go | 6 + modules/packages/cargo/parser.go | 169 +++++++++ modules/packages/cargo/parser_test.go | 86 +++++ modules/repository/create.go | 1 + modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 14 + public/img/svg/gitea-cargo.svg | 1 + routers/api/packages/api.go | 15 + routers/api/packages/cargo/cargo.go | 281 +++++++++++++++ routers/api/v1/packages/package.go | 2 +- routers/web/org/setting_packages.go | 20 + routers/web/shared/packages/packages.go | 22 ++ routers/web/user/setting/packages.go | 18 + routers/web/web.go | 8 + services/cron/tasks_basic.go | 4 +- services/forms/package_form.go | 2 +- services/packages/cargo/index.go | 290 +++++++++++++++ services/packages/cleanup/cleanup.go | 154 ++++++++ services/packages/packages.go | 121 +------ templates/org/settings/packages.tmpl | 1 + templates/package/content/cargo.tmpl | 62 ++++ templates/package/metadata/cargo.tmpl | 7 + templates/package/shared/cargo.tmpl | 24 ++ templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + templates/user/settings/packages.tmpl | 1 + tests/integration/api_packages_cargo_test.go | 341 ++++++++++++++++++ tests/integration/api_packages_test.go | 5 +- web_src/svg/gitea-cargo.svg | 3 + 35 files changed, 1660 insertions(+), 125 deletions(-) create mode 100644 docs/content/doc/packages/cargo.en-us.md create mode 100644 modules/packages/cargo/parser.go create mode 100644 modules/packages/cargo/parser_test.go create mode 100644 public/img/svg/gitea-cargo.svg create mode 100644 routers/api/packages/cargo/cargo.go create mode 100644 services/packages/cargo/index.go create mode 100644 services/packages/cleanup/cleanup.go create mode 100644 templates/package/content/cargo.tmpl create mode 100644 templates/package/metadata/cargo.tmpl create mode 100644 templates/package/shared/cargo.tmpl create mode 100644 tests/integration/api_packages_cargo_test.go create mode 100644 web_src/svg/gitea-cargo.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 04ba2dc67a..0f1f18646b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2458,6 +2458,8 @@ ROUTER = console ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_TOTAL_OWNER_SIZE = -1 +;; Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CARGO = -1 ;; Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_COMPOSER = -1 ;; Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 9254962dc5..c9116edc66 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1213,6 +1213,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` - `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maximum count of package versions a single owner can have (`-1` means no limits) - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CARGO`: **-1**: Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/packages/cargo.en-us.md b/docs/content/doc/packages/cargo.en-us.md new file mode 100644 index 0000000000..1f90d939d1 --- /dev/null +++ b/docs/content/doc/packages/cargo.en-us.md @@ -0,0 +1,109 @@ +--- +date: "2022-11-20T00:00:00+00:00" +title: "Cargo Packages Repository" +slug: "packages/cargo" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Cargo" + weight: 5 + identifier: "cargo" +--- + +# Cargo Packages Repository + +Publish [Cargo](https://doc.rust-lang.org/stable/cargo/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Cargo package registry, you need [Rust and Cargo](https://www.rust-lang.org/tools/install). + +Cargo stores informations about the available packages in a package index stored in a git repository. +This repository is needed to work with the registry. +The following section describes how to create it. + +## Index Repository + +Cargo stores informations about the available packages in a package index stored in a git repository. +In Gitea this repository has the special name `_cargo-index`. +After a package was uploaded, its metadata is automatically written to the index. +The content of this repository should not be manually modified. + +The user or organization package settings page allows to create the index repository along with the configuration file. +If needed this action will rewrite the configuration file. +This can be useful if for example the Gitea instance domain was changed. + +If the case arises where the packages stored in Gitea and the information in the index repository are out of sync, the settings page allows to rebuild the index repository. +This action iterates all packages in the registry and writes their information to the index. +If there are lot of packages this process may take some time. + +## Configuring the package registry + +To register the package registry the Cargo configuration must be updated. +Add the following text to the configuration file located in the current users home directory (for example `~/.cargo/config.toml`): + +``` +[registry] +default = "gitea" + +[registries.gitea] +index = "https://gitea.example.com/{owner}/_cargo-index.git" + +[net] +git-fetch-with-cli = true +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +If the registry is private or you want to publish new packages, you have to configure your credentials. +Add the credentials section to the credentials file located in the current users home directory (for example `~/.cargo/credentials.toml`): + +``` +[registries.gitea] +token = "Bearer {token}" +``` + +| Parameter | Description | +| --------- | ----------- | +| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) | + +## Publish a package + +Publish a package by running the following command in your project: + +```shell +cargo publish +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +cargo add {package_name} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `package_name` | The package name. | + +## Supported commands + +``` +cargo publish +cargo add +cargo install +cargo yank +cargo unyank +cargo search +``` diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 9a736c1e56..b3ccb73c19 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -26,6 +26,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | +| [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Conda]({{< relref "doc/packages/conda.en-us.md" >}}) | - | `conda` | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 3b36ee2266..40010eb720 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/conda" @@ -129,6 +130,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc var metadata interface{} switch p.Type { + case TypeCargo: + metadata = &cargo.Metadata{} case TypeComposer: metadata = &composer.Metadata{} case TypeConan: diff --git a/models/packages/package.go b/models/packages/package.go index 0015953d81..b6b033cc95 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -30,6 +30,7 @@ type Type string // List of supported packages const ( + TypeCargo Type = "cargo" TypeComposer Type = "composer" TypeConan Type = "conan" TypeConda Type = "conda" @@ -46,6 +47,7 @@ const ( ) var TypeList = []Type{ + TypeCargo, TypeComposer, TypeConan, TypeConda, @@ -64,6 +66,8 @@ var TypeList = []Type{ // Name gets the name of the package type func (pt Type) Name() string { switch pt { + case TypeCargo: + return "Cargo" case TypeComposer: return "Composer" case TypeConan: @@ -97,6 +101,8 @@ func (pt Type) Name() string { // SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { + case TypeCargo: + return "gitea-cargo" case TypeComposer: return "gitea-composer" case TypeConan: diff --git a/models/packages/package_property.go b/models/packages/package_property.go index 1b7f253d56..e03b12c9df 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -58,6 +58,12 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) } +// UpdateProperty updates a property +func UpdateProperty(ctx context.Context, pp *PackageProperty) error { + _, err := db.GetEngine(ctx).ID(pp.ID).Update(pp) + return err +} + // DeleteAllProperties deletes all properties of a ref func DeleteAllProperties(ctx context.Context, refType PropertyType, refID int64) error { _, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Delete(&PackageProperty{}) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go new file mode 100644 index 0000000000..36cd44df84 --- /dev/null +++ b/modules/packages/cargo/parser.go @@ -0,0 +1,169 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "encoding/binary" + "errors" + "io" + "regexp" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +const PropertyYanked = "cargo.yanked" + +var ( + ErrInvalidName = errors.New("package name is invalid") + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Cargo package +type Package struct { + Name string + Version string + Metadata *Metadata + Content io.Reader + ContentSize int64 +} + +// Metadata represents the metadata of a Cargo package +type Metadata struct { + Dependencies []*Dependency `json:"dependencies,omitempty"` + Features map[string][]string `json:"features,omitempty"` + Authors []string `json:"authors,omitempty"` + Description string `json:"description,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Categories []string `json:"categories,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Links string `json:"links,omitempty"` +} + +type Dependency struct { + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + Package *string `json:"package"` +} + +var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) + +// ParsePackage reads the metadata and content of a package +func ParsePackage(r io.Reader) (*Package, error) { + var size uint32 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p, err := parsePackage(io.LimitReader(r, int64(size))) + if err != nil { + return nil, err + } + + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p.Content = io.LimitReader(r, int64(size)) + p.ContentSize = int64(size) + + return p, nil +} + +func parsePackage(r io.Reader) (*Package, error) { + var meta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + ExplicitNameInToml string `json:"explicit_name_in_toml"` + } `json:"deps"` + Features map[string][]string `json:"features"` + Authors []string `json:"authors"` + Description string `json:"description"` + Documentation string `json:"documentation"` + Homepage string `json:"homepage"` + Readme string `json:"readme"` + ReadmeFile string `json:"readme_file"` + Keywords []string `json:"keywords"` + Categories []string `json:"categories"` + License string `json:"license"` + LicenseFile string `json:"license_file"` + Repository string `json:"repository"` + Links string `json:"links"` + } + if err := json.NewDecoder(r).Decode(&meta); err != nil { + return nil, err + } + + if !nameMatch.MatchString(meta.Name) { + return nil, ErrInvalidName + } + + if _, err := version.NewSemver(meta.Vers); err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + if !validation.IsValidURL(meta.Documentation) { + meta.Documentation = "" + } + if !validation.IsValidURL(meta.Repository) { + meta.Repository = "" + } + + dependencies := make([]*Dependency, 0, len(meta.Deps)) + for _, dep := range meta.Deps { + dependencies = append(dependencies, &Dependency{ + Name: dep.Name, + Req: dep.VersionReq, + Features: dep.Features, + Optional: dep.Optional, + DefaultFeatures: dep.DefaultFeatures, + Target: dep.Target, + Kind: dep.Kind, + Registry: dep.Registry, + }) + } + + return &Package{ + Name: meta.Name, + Version: meta.Vers, + Metadata: &Metadata{ + Dependencies: dependencies, + Features: meta.Features, + Authors: meta.Authors, + Description: meta.Description, + DocumentationURL: meta.Documentation, + ProjectURL: meta.Homepage, + Readme: meta.Readme, + Keywords: meta.Keywords, + Categories: meta.Categories, + License: meta.License, + RepositoryURL: meta.Repository, + Links: meta.Links, + }, + }, nil +} diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go new file mode 100644 index 0000000000..2230a5b499 --- /dev/null +++ b/modules/packages/cargo/parser_test.go @@ -0,0 +1,86 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + description = "Package Description" + author = "KN4CK3R" + homepage = "https://gitea.io/" + license = "MIT" +) + +func TestParsePackage(t *testing.T) { + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + description + `", + "authors": ["` + author + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0" + } + ], + "homepage":"` + homepage + `", + "license":"` + license + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} { + data := createPackage(name, "1.0.0") + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} { + data := createPackage("test", version) + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage("test", "1.0.0") + + cp, err := ParsePackage(data) + assert.NotNil(t, cp) + assert.NoError(t, err) + + assert.Equal(t, "test", cp.Name) + assert.Equal(t, "1.0.0", cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Equal(t, []string{author}, cp.Metadata.Authors) + assert.Len(t, cp.Metadata.Dependencies, 1) + assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name) + assert.Equal(t, homepage, cp.Metadata.ProjectURL) + assert.Equal(t, license, cp.Metadata.License) + content, _ := io.ReadAll(cp.Content) + assert.Equal(t, "test", string(content)) + }) +} diff --git a/modules/repository/create.go b/modules/repository/create.go index 7bcda0fe45..b9a72ad573 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -211,6 +211,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, IsMirror: opts.IsMirror, + DefaultBranch: opts.DefaultBranch, } var rollbackRepo *repo_model.Repository diff --git a/modules/setting/packages.go b/modules/setting/packages.go index d0cd80aa03..190c17dd8f 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -25,6 +25,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 + LimitSizeCargo int64 LimitSizeComposer int64 LimitSizeConan int64 LimitSizeConda int64 @@ -65,6 +66,7 @@ func newPackages() { } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") + Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f384056613..bc2e8cb91c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3152,6 +3152,11 @@ versions.on = on versions.view_all = View all dependency.id = ID dependency.version = Version +cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): +cargo.install = To install the package using Cargo, run the following command: +cargo.documentation = For more information on the Cargo registry, see the documentation. +cargo.details.repository_site = Repository Site +cargo.details.documentation_site = Documentation Site composer.registry = Setup this registry in your ~/.composer/config.json file: composer.install = To install the package using Composer, run the following command: composer.documentation = For more information on the Composer registry, see the documentation. @@ -3228,6 +3233,15 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cargo.title = Cargo Registry Index +owner.settings.cargo.initialize = Initialize Index +owner.settings.cargo.initialize.description = To use the Cargo registry a special index git repository is needed. Here you can (re)create it with the required config. +owner.settings.cargo.initialize.error = Failed to initialize Cargo index: %v +owner.settings.cargo.initialize.success = The Cargo index was successfully created. +owner.settings.cargo.rebuild = Rebuild Index +owner.settings.cargo.rebuild.description = If the index is out of sync with the cargo packages stored you can rebuild it here. +owner.settings.cargo.rebuild.error = Failed to rebuild Cargo index: %v +owner.settings.cargo.rebuild.success = The Cargo index was successfully rebuild. owner.settings.cleanuprules.title = Manage Cleanup Rules owner.settings.cleanuprules.add = Add Cleanup Rule owner.settings.cleanuprules.edit = Edit Cleanup Rule diff --git a/public/img/svg/gitea-cargo.svg b/public/img/svg/gitea-cargo.svg new file mode 100644 index 0000000000..91d53941ca --- /dev/null +++ b/public/img/svg/gitea-cargo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 7a07fea815..8ec9ae9bf7 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/conda" @@ -71,6 +72,20 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) r.Group("/{username}", func() { + r.Group("/cargo", func() { + r.Group("/api/v1/crates", func() { + r.Get("", cargo.SearchPackages) + r.Put("/new", reqPackageAccess(perm.AccessModeWrite), cargo.UploadPackage) + r.Group("/{package}", func() { + r.Group("/{version}", func() { + r.Get("/download", cargo.DownloadPackageFile) + r.Delete("/yank", reqPackageAccess(perm.AccessModeWrite), cargo.YankPackage) + r.Put("/unyank", reqPackageAccess(perm.AccessModeWrite), cargo.UnyankPackage) + }) + r.Get("/owners", cargo.ListOwners) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/composer", func() { r.Get("/packages.json", composer.ServiceIndex) r.Get("/search.json", composer.SearchPackages) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go new file mode 100644 index 0000000000..e0bf5da13a --- /dev/null +++ b/routers/api/packages/cargo/cargo.go @@ -0,0 +1,281 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + "code.gitea.io/gitea/services/convert" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#web-api +type StatusResponse struct { + OK bool `json:"ok"` + Errors []StatusMessage `json:"errors,omitempty"` +} + +type StatusMessage struct { + Message string `json:"detail"` +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, StatusResponse{ + OK: false, + Errors: []StatusMessage{ + { + Message: message, + }, + }, + }) + }) +} + +type SearchResult struct { + Crates []*SearchResultCrate `json:"crates"` + Meta SearchResultMeta `json:"meta"` +} + +type SearchResultCrate struct { + Name string `json:"name"` + LatestVersion string `json:"max_version"` + Description string `json:"description"` +} + +type SearchResultMeta struct { + Total int64 `json:"total"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#search +func SearchPackages(ctx *context.Context) { + page := ctx.FormInt("page") + if page < 1 { + page = 1 + } + perPage := ctx.FormInt("per_page") + paginator := db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(perPage), + } + + pvs, total, err := packages_model.SearchLatestVersions( + ctx, + &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeCargo, + Name: packages_model.SearchValue{Value: ctx.FormTrim("q")}, + IsInternal: util.OptionalBoolFalse, + Paginator: &paginator, + }, + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + crates := make([]*SearchResultCrate, 0, len(pvs)) + for _, pd := range pds { + crates = append(crates, &SearchResultCrate{ + Name: pd.Package.Name, + LatestVersion: pd.Version.Version, + Description: pd.Metadata.(*cargo_module.Metadata).Description, + }) + } + + ctx.JSON(http.StatusOK, SearchResult{ + Crates: crates, + Meta: SearchResultMeta{ + Total: total, + }, + }) +} + +type Owners struct { + Users []OwnerUser `json:"users"` +} + +type OwnerUser struct { + ID int64 `json:"id"` + Login string `json:"login"` + Name string `json:"name"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#owners-list +func ListOwners(ctx *context.Context) { + ctx.JSON(http.StatusOK, Owners{ + Users: []OwnerUser{ + { + ID: ctx.Package.Owner.ID, + Login: ctx.Package.Owner.Name, + Name: ctx.Package.Owner.DisplayName(), + }, + }, + }) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: ctx.Params("package"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", ctx.Params("package"), ctx.Params("version"))), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#publish +func UploadPackage(ctx *context.Context) { + defer ctx.Req.Body.Close() + + cp, err := cargo_module.ParsePackage(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() != cp.ContentSize { + apiError(ctx, http.StatusBadRequest, "invalid content size") + return + } + + pv, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: cp.Name, + Version: cp.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: cp.Metadata, + VersionProperties: map[string]string{ + cargo_module.PropertyYanked: strconv.FormatBool(false), + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + log.Error("Rollback creation of package version: %v", err) + } + + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#yank +func YankPackage(ctx *context.Context) { + yankPackage(ctx, true) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#unyank +func UnyankPackage(ctx *context.Context) { + yankPackage(ctx, false) +} + +func yankPackage(ctx *context.Context, yank bool) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"), ctx.Params("version")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, cargo_module.PropertyYanked) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pps) == 0 { + apiError(ctx, http.StatusInternalServerError, "Property not found") + return + } + + pp := pps[0] + pp.Value = strconv.FormatBool(yank) + + if err := packages_model.UpdateProperty(ctx, pp); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 5ffefc4862..2a20d66d1e 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] + // enum: [cargo, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go index 80135ca2d0..21d25bd90a 100644 --- a/routers/web/org/setting_packages.go +++ b/routers/web/org/setting_packages.go @@ -84,3 +84,23 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go index b9aa40bdd2..30c25374d1 100644 --- a/routers/web/shared/packages/packages.go +++ b/routers/web/shared/packages/packages.go @@ -13,9 +13,11 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/services/forms" + cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" ) @@ -223,3 +225,23 @@ func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *pack return nil } + +func InitializeCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.InitializeIndexRepository(ctx, owner, owner) + if err != nil { + log.Error("InitializeIndexRepository failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.initialize.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.initialize.success")) + } +} + +func RebuildCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.RebuildIndex(ctx, owner, owner) + if err != nil { + log.Error("RebuildIndex failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.rebuild.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.rebuild.success")) + } +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go index f6f7195adf..b3f8a3e41d 100644 --- a/routers/web/user/setting/packages.go +++ b/routers/web/user/setting/packages.go @@ -77,3 +77,21 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} diff --git a/routers/web/web.go b/routers/web/web.go index b7128fc3a9..a024c0ac37 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -468,6 +468,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", user_setting.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", user_setting.InitializeCargoIndex) + m.Post("/rebuild", user_setting.RebuildCargoIndex) + }) }, packagesEnabled) m.Group("/secrets", func() { m.Get("", user_setting.Secrets) @@ -818,6 +822,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", org.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", org.InitializeCargoIndex) + m.Post("/rebuild", org.RebuildCargoIndex) + }) }, packagesEnabled) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index aad0e39591..2e6560ec0c 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -16,7 +16,7 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" - packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" ) @@ -152,7 +152,7 @@ func registerCleanupPackages() { OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { realConfig := config.(*OlderThanConfig) - return packages_service.Cleanup(ctx, realConfig.OlderThan) + return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan) }) } diff --git a/services/forms/package_form.go b/services/forms/package_form.go index e78e64ef7e..558ed54b65 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + Type string `binding:"Required;In(cargo,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go new file mode 100644 index 0000000000..e58a472816 --- /dev/null +++ b/services/packages/cargo/index.go @@ -0,0 +1,290 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "strconv" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + files_service "code.gitea.io/gitea/services/repository/files" +) + +const ( + IndexRepositoryName = "_cargo-index" + ConfigFileName = "config.json" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#index-format + +func BuildPackagePath(name string) string { + switch len(name) { + case 0: + panic("Cargo package name can not be empty") + case 1: + return path.Join("1", name) + case 2: + return path.Join("2", name) + case 3: + return path.Join("3", string(name[0]), name) + default: + return path.Join(name[0:2], name[2:4], name) + } +} + +func InitializeIndexRepository(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + if err := createOrUpdateConfigFile(ctx, repo, doer, owner); err != nil { + return fmt.Errorf("createOrUpdateConfigFile: %w", err) + } + + return nil +} + +func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeCargo) + if err != nil { + return fmt.Errorf("GetPackagesByType: %w", err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Rebuild Cargo Index", + func(t *files_service.TemporaryUploadRepository) error { + // Remove all existing content but the Cargo config + files, err := t.LsFiles() + if err != nil { + return err + } + for i, file := range files { + if file == ConfigFileName { + files[i] = files[len(files)-1] + files = files[:len(files)-1] + break + } + } + if err := t.RemoveFilesFromIndex(files...); err != nil { + return err + } + + // Add all packages + for _, p := range ps { + if err := addOrUpdatePackageIndex(ctx, t, p); err != nil { + return err + } + } + + return nil + }, + ) +} + +func AddOrUpdatePackageIndex(ctx context.Context, doer, owner *user_model.User, packageID int64) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + p, err := packages_model.GetPackageByID(ctx, packageID) + if err != nil { + return fmt.Errorf("GetPackageByID[%d]: %w", packageID, err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Update "+p.Name, + func(t *files_service.TemporaryUploadRepository) error { + return addOrUpdatePackageIndex(ctx, t, p) + }, + ) +} + +type IndexVersionEntry struct { + Name string `json:"name"` + Version string `json:"vers"` + Dependencies []*cargo_module.Dependency `json:"deps"` + FileChecksum string `json:"cksum"` + Features map[string][]string `json:"features"` + Yanked bool `json:"yanked"` + Links string `json:"links,omitempty"` +} + +func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + Sort: packages_model.SortVersionAsc, + }) + if err != nil { + return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) + } + if len(pvs) == 0 { + return nil + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) + } + + var b bytes.Buffer + for _, pd := range pds { + metadata := pd.Metadata.(*cargo_module.Metadata) + + dependencies := metadata.Dependencies + if dependencies == nil { + dependencies = make([]*cargo_module.Dependency, 0) + } + + features := metadata.Features + if features == nil { + features = make(map[string][]string) + } + + yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked)) + entry, err := json.Marshal(&IndexVersionEntry{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Dependencies: dependencies, + FileChecksum: pd.Files[0].Blob.HashSHA256, + Features: features, + Yanked: yanked, + Links: metadata.Links, + }) + if err != nil { + return err + } + + b.Write(entry) + b.WriteString("\n") + } + + return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) +} + +func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner.Name, IndexRepositoryName) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + repo, err = repo_module.CreateRepository(doer, owner, repo_module.CreateRepoOptions{ + Name: IndexRepositoryName, + }) + if err != nil { + return nil, fmt.Errorf("CreateRepository: %w", err) + } + } else { + return nil, fmt.Errorf("GetRepositoryByOwnerAndName: %w", err) + } + } + + return repo, nil +} + +type Config struct { + DownloadURL string `json:"dl"` + APIURL string `json:"api"` +} + +func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { + return alterRepositoryContent( + ctx, + doer, + repo, + "Initialize Cargo Config", + func(t *files_service.TemporaryUploadRepository) error { + var b bytes.Buffer + err := json.NewEncoder(&b).Encode(Config{ + DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", + APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", + }) + if err != nil { + return err + } + + return writeObjectToIndex(t, ConfigFileName, &b) + }, + ) +} + +// This is a shorter version of CreateOrUpdateRepoFile which allows to perform multiple actions on a git repository +func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, commitMessage string, fn func(*files_service.TemporaryUploadRepository) error) error { + t, err := files_service.NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return err + } + defer t.Close() + + var lastCommitID string + if err := t.Clone(repo.DefaultBranch); err != nil { + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return err + } + if err := t.Init(); err != nil { + return err + } + } else { + if err := t.SetDefaultIndex(); err != nil { + return err + } + + commit, err := t.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return err + } + + lastCommitID = commit.ID.String() + } + + if err := fn(t); err != nil { + return err + } + + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + now := time.Now() + commitHash, err := t.CommitTreeWithDate(lastCommitID, doer, doer, treeHash, commitMessage, false, now, now) + if err != nil { + return err + } + + return t.Push(doer, commitHash, repo.DefaultBranch) +} + +func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { + hash, err := t.HashObject(r) + if err != nil { + return err + } + + return t.AddObjectToIndex("100644", hash, path) +} diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go new file mode 100644 index 0000000000..2d62a028a4 --- /dev/null +++ b/services/packages/cleanup/cleanup.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + container_service "code.gitea.io/gitea/services/packages/container" +) + +// Cleanup removes expired package data +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(taskCtx) + if err != nil { + return err + } + defer committer.Close() + + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-taskCtx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } + + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + versionDeleted := false + for _, pv := range pvs { + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + + versionDeleted = true + } + + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) + } + if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err) + } + } + } + } + return nil + }) + if err != nil { + return err + } + + if err := container_service.Cleanup(ctx, olderThan); err != nil { + return err + } + + ps, err := packages_model.FindUnreferencedPackages(ctx) + if err != nil { + return err + } + for _, p := range ps { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { + return err + } + if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { + return err + } + } + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) + if err != nil { + return err + } + + for _, pb := range pbs { + if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + contentStore := packages_module.NewContentStore() + for _, pb := range pbs { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob [%v]: %v", pb.ID, err) + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 9e52cb1450..f502840755 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -22,7 +21,6 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - container_service "code.gitea.io/gitea/services/packages/container" ) var ( @@ -335,6 +333,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p var typeSpecificSize int64 switch packageType { + case packages_model.TypeCargo: + typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeComposer: typeSpecificSize = setting.Packages.LimitSizeComposer case packages_model.TypeConan: @@ -448,123 +448,6 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro return packages_model.DeleteFileByID(ctx, pf.ID) } -// Cleanup removes expired package data -func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(taskCtx) - if err != nil { - return err - } - defer committer.Close() - - err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { - select { - case <-taskCtx.Done(): - return db.ErrCancelledf("While processing package cleanup rules") - default: - } - - if err := pcr.CompiledPattern(); err != nil { - return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) - } - - olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) - - packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) - } - - for _, p := range packages { - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - PackageID: p.ID, - IsInternal: util.OptionalBoolFalse, - Sort: packages_model.SortCreatedDesc, - Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), - }) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) - } - for _, pv := range pvs { - if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) - } else if skip { - log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) - continue - } - - toMatch := pv.LowerVersion - if pcr.MatchFullName { - toMatch = p.LowerName + "/" + pv.LowerVersion - } - - if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) - continue - } - if pv.CreatedUnix.AsLocalTime().After(olderThan) { - log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) - continue - } - if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) - continue - } - - log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) - - if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) - } - } - } - return nil - }) - if err != nil { - return err - } - - if err := container_service.Cleanup(ctx, olderThan); err != nil { - return err - } - - ps, err := packages_model.FindUnreferencedPackages(ctx) - if err != nil { - return err - } - for _, p := range ps { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { - return err - } - if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { - return err - } - } - - pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) - if err != nil { - return err - } - - for _, pb := range pbs { - if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { - return err - } - } - - if err := committer.Commit(); err != nil { - return err - } - - contentStore := packages_module.NewContentStore() - for _, pb := range pbs { - if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { - log.Error("Error deleting package blob [%v]: %v", pb.ID, err) - } - } - - return nil -} - // GetFileStreamByPackageNameAndVersion returns the content of the specific package file func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl index 412a9813c2..04e5d45b50 100644 --- a/templates/org/settings/packages.tmpl +++ b/templates/org/settings/packages.tmpl @@ -7,6 +7,7 @@
{{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} + {{template "package/shared/cargo" .}}
diff --git a/templates/package/content/cargo.tmpl b/templates/package/content/cargo.tmpl new file mode 100644 index 0000000000..54c40a5b0d --- /dev/null +++ b/templates/package/content/cargo.tmpl @@ -0,0 +1,62 @@ +{{if eq .PackageDescriptor.Package.Type "cargo"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
[registry]
+default = "gitea"
+
+[registries.gitea]
+index = "{{AppUrl}}{{.PackageDescriptor.Owner.Name}}/_cargo-index.git"
+
+[net]
+git-fetch-with-cli = true
+
+
+ + 
cargo add {{.PackageDescriptor.Package.Name}}@{{.PackageDescriptor.Version.Version}}
+
+
+ +
+
+
+ + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}} +

{{.locale.Tr "packages.about"}}

+ {{if .PackageDescriptor.Metadata.Description}}
{{.PackageDescriptor.Metadata.Description}}
{{end}} + {{if .PackageDescriptor.Metadata.Readme}}
{{RenderMarkdownToHtml .PackageDescriptor.Metadata.Readme}}
{{end}} + {{end}} + + {{if .PackageDescriptor.Metadata.Dependencies}} +

{{.locale.Tr "packages.dependencies"}}

+
+ + + + + + + + + {{range .PackageDescriptor.Metadata.Dependencies}} + + + + + {{end}} + +
{{.locale.Tr "packages.dependency.id"}}{{.locale.Tr "packages.dependency.version"}}
{{.Name}}{{.Req}}
+
+ {{end}} + + {{if .PackageDescriptor.Metadata.Keywords}} +

{{.locale.Tr "packages.keywords"}}

+
+ {{range .PackageDescriptor.Metadata.Keywords}} + {{.}} + {{end}} +
+ {{end}} +{{end}} diff --git a/templates/package/metadata/cargo.tmpl b/templates/package/metadata/cargo.tmpl new file mode 100644 index 0000000000..68232d4832 --- /dev/null +++ b/templates/package/metadata/cargo.tmpl @@ -0,0 +1,7 @@ +{{if eq .PackageDescriptor.Package.Type "cargo"}} + {{range .PackageDescriptor.Metadata.Authors}}
{{svg "octicon-person" 16 "mr-3"}} {{.}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.cargo.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.cargo.details.documentation_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
{{end}} +{{end}} diff --git a/templates/package/shared/cargo.tmpl b/templates/package/shared/cargo.tmpl new file mode 100644 index 0000000000..b9c2ef4d2b --- /dev/null +++ b/templates/package/shared/cargo.tmpl @@ -0,0 +1,24 @@ +

+ {{.locale.Tr "packages.owner.settings.cargo.title"}} +

+
+
+
+ +
+
+ {{.CsrfTokenHtml}} + +
+
+ +
+
+ {{.CsrfTokenHtml}} + +
+
+ +
+
+
diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 4611cdb8d8..6209e6cb02 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
+ {{template "package/content/cargo" .}} {{template "package/content/composer" .}} {{template "package/content/conan" .}} {{template "package/content/conda" .}} @@ -43,6 +44,7 @@ {{end}}
{{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
+ {{template "package/metadata/cargo" .}} {{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} {{template "package/metadata/conda" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 5a3f2c2bb9..d6e3009b35 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2100,6 +2100,7 @@ }, { "enum": [ + "cargo", "composer", "conan", "conda", diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl index 9aae14f934..472d84327a 100644 --- a/templates/user/settings/packages.tmpl +++ b/templates/user/settings/packages.tmpl @@ -4,6 +4,7 @@
{{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} + {{template "package/shared/cargo" .}}
{{template "base/footer" .}} diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go new file mode 100644 index 0000000000..0c542eaf1e --- /dev/null +++ b/tests/integration/api_packages_cargo_test.go @@ -0,0 +1,341 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net/http" + neturl "net/url" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/setting" + cargo_router "code.gitea.io/gitea/routers/api/packages/cargo" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageCargo(t *testing.T) { + onGiteaRun(t, testPackageCargo) +} + +func testPackageCargo(t *testing.T, _ *neturl.URL) { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "cargo-package" + packageVersion := "1.0.3" + packageDescription := "Package Description" + packageAuthor := "KN4CK3R" + packageHomepage := "https://gitea.io/" + packageLicense := "MIT" + + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + packageDescription + `", + "authors": ["` + packageAuthor + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0", + "registry": "https://gitea.io/user/_cargo-index", + "kind": "normal", + "default_features": true + } + ], + "homepage":"` + packageHomepage + `", + "license":"` + packageLicense + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + err := cargo_service.InitializeIndexRepository(db.DefaultContext, user, user) + assert.NoError(t, err) + + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName) + assert.NotNil(t, repo) + assert.NoError(t, err) + + readGitContent := func(t *testing.T, path string) string { + gitRepo, err := git.OpenRepository(db.DefaultContext, repo.RepoPath()) + assert.NoError(t, err) + defer gitRepo.Close() + + commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + assert.NoError(t, err) + + blob, err := commit.GetBlobByPath(path) + assert.NoError(t, err) + + content, err := blob.GetBlobContent() + assert.NoError(t, err) + + return content + } + + root := fmt.Sprintf("%sapi/packages/%s/cargo", setting.AppURL, user.Name) + url := fmt.Sprintf("%s/api/v1/crates", root) + + t.Run("Index", func(t *testing.T) { + t.Run("Config", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.ConfigFileName) + + var config cargo_service.Config + err := json.Unmarshal([]byte(content), &config) + assert.NoError(t, err) + + assert.Equal(t, url, config.DownloadURL) + assert.Equal(t, root, config.APIURL) + }) + }) + + t.Run("Upload", func(t *testing.T) { + t.Run("InvalidNameOrVersion", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := createPackage("0test", "1.0.0") + + req := NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusBadRequest) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + + content = createPackage("test", "-1.0.0") + + req = NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusBadRequest) + + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + }) + + t.Run("InvalidContent", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + metadata := `{"name":"test","vers":"1.0.0"}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("te") + + req := NewRequestWithBody(t, "PUT", url+"/new", &buf) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Valid", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeCargo) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &cargo_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s-%s.crate", packageName, packageVersion), pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.EqualValues(t, 4, pb.Size) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + t.Run("Index", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) + + t.Run("Rebuild", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) + assert.NoError(t, err) + + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + }) + }) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 0, pv.DownloadCount) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/download", url, neturl.PathEscape(packageName), neturl.PathEscape(pv.Version))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "test", resp.Body.String()) + + pv, err = packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 1, pv.DownloadCount) + }) + + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + cases := []struct { + Query string + Page int + PerPage int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 1, 10, 1, 1}, + {"cargo", 1, 0, 1, 1}, + {"cargo", 1, 10, 1, 1}, + {"cargo", 2, 10, 1, 0}, + {"test", 0, 10, 0, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s?q=%s&page=%d&per_page=%d", url, c.Query, c.Page, c.PerPage)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result cargo_router.SearchResult + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Meta.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Crates, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + + t.Run("Yank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/yank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.True(t, entry.Yanked) + }) + + t.Run("Unyank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "PUT", fmt.Sprintf("%s/%s/%s/unyank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.False(t, entry.Yanked) + }) + + t.Run("ListOwners", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/owners", url, neturl.PathEscape(packageName))) + resp := MakeRequest(t, req, http.StatusOK) + + var owners cargo_router.Owners + DecodeJSON(t, resp, &owners) + + assert.Len(t, owners.Users, 1) + assert.Equal(t, user.ID, owners.Users[0].ID) + assert.Equal(t, user.Name, owners.Users[0].Login) + assert.Equal(t, user.DisplayName(), owners.Users[0].Name) + }) +} diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index a6c335eb7e..39852e212c 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -246,7 +247,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -383,7 +384,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions { diff --git a/web_src/svg/gitea-cargo.svg b/web_src/svg/gitea-cargo.svg new file mode 100644 index 0000000000..dbec107ad0 --- /dev/null +++ b/web_src/svg/gitea-cargo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file From c18a62279a54f1ba0dc0293dddbe197f527fbf00 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sun, 5 Feb 2023 19:57:38 +0800 Subject: [PATCH 04/33] Fix time to NotifyPullRequestSynchronized (#22650) Should call `PushToBaseRepo` before `notification.NotifyPullRequestSynchronized`. Or the notifier will get an old commit when reading branch `pull/xxx/head`. Found by ~#21937~ #22679. Co-authored-by: Lunny Xiao --- models/issues/pull_list.go | 13 ++++++++++++- services/pull/pull.go | 36 ++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/models/issues/pull_list.go b/models/issues/pull_list.go index 007c2fd903..68f62ffc43 100644 --- a/models/issues/pull_list.go +++ b/models/issues/pull_list.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" "xorm.io/xorm" ) @@ -175,7 +176,17 @@ func (prs PullRequestList) loadAttributes(ctx context.Context) error { } for _, pr := range prs { pr.Issue = set[pr.IssueID] - pr.Issue.PullRequest = pr // panic here means issueIDs and prs are not in sync + /* + Old code: + pr.Issue.PullRequest = pr // panic here means issueIDs and prs are not in sync + + It's worth panic because it's almost impossible to happen under normal use. + But in integration testing, an asynchronous task could read a database that has been reset. + So returning an error would make more sense, let the caller has a choice to ignore it. + */ + if pr.Issue == nil { + return fmt.Errorf("issues and prs may be not in sync: cannot find issue %v for pr %v: %w", pr.IssueID, pr.ID, util.ErrNotExist) + } } return nil } diff --git a/services/pull/pull.go b/services/pull/pull.go index 317875d211..0d260c93b1 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -263,6 +263,24 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, return } + for _, pr := range prs { + log.Trace("Updating PR[%d]: composing new test task", pr.ID) + if pr.Flow == issues_model.PullRequestFlowGithub { + if err := PushToBaseRepo(ctx, pr); err != nil { + log.Error("PushToBaseRepo: %v", err) + continue + } + } else { + continue + } + + AddToTaskQueue(pr) + comment, err := CreatePushPullComment(ctx, doer, pr, oldCommitID, newCommitID) + if err == nil && comment != nil { + notification.NotifyPullRequestPushCommits(ctx, doer, pr, comment) + } + } + if isSync { requests := issues_model.PullRequestList(prs) if err = requests.LoadAttributes(); err != nil { @@ -303,24 +321,6 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, } } - for _, pr := range prs { - log.Trace("Updating PR[%d]: composing new test task", pr.ID) - if pr.Flow == issues_model.PullRequestFlowGithub { - if err := PushToBaseRepo(ctx, pr); err != nil { - log.Error("PushToBaseRepo: %v", err) - continue - } - } else { - continue - } - - AddToTaskQueue(pr) - comment, err := CreatePushPullComment(ctx, doer, pr, oldCommitID, newCommitID) - if err == nil && comment != nil { - notification.NotifyPullRequestPushCommits(ctx, doer, pr, comment) - } - } - log.Trace("AddTestPullRequestTask [base_repo_id: %d, base_branch: %s]: finding pull requests", repoID, branch) prs, err = issues_model.GetUnmergedPullRequestsByBaseInfo(repoID, branch) if err != nil { From d74d16a4b1c79b5b1fc58998aa38bf299cee10ec Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sun, 5 Feb 2023 22:24:43 +0800 Subject: [PATCH 05/33] Set PR for issue when load attributes for PRs (#22766) A missing patch for #22650. --- models/issues/pull_list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/issues/pull_list.go b/models/issues/pull_list.go index 68f62ffc43..f4efd916c8 100644 --- a/models/issues/pull_list.go +++ b/models/issues/pull_list.go @@ -187,6 +187,7 @@ func (prs PullRequestList) loadAttributes(ctx context.Context) error { if pr.Issue == nil { return fmt.Errorf("issues and prs may be not in sync: cannot find issue %v for pr %v: %w", pr.IssueID, pr.ID, util.ErrNotExist) } + pr.Issue.PullRequest = pr } return nil } From c07199f9ab4ac5a1ecbfd27616ad44bf35d00fc1 Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Mon, 6 Feb 2023 01:06:26 +0900 Subject: [PATCH 06/33] remove update language in ProfilePost (#22748) Fixes https://github.com/go-gitea/gitea/issues/22703 Change language has been moved to `UpdateUserLang` Co-authored-by: Lunny Xiao --- routers/web/user/setting/profile.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/routers/web/user/setting/profile.go b/routers/web/user/setting/profile.go index e01f3cdeea..f0f053a514 100644 --- a/routers/web/user/setting/profile.go +++ b/routers/web/user/setting/profile.go @@ -137,11 +137,8 @@ func ProfilePost(ctx *context.Context) { return } - // Update the language to the one we just set - middleware.SetLocaleCookie(ctx.Resp, ctx.Doer.Language, 0) - log.Trace("User settings updated: %s", ctx.Doer.Name) - ctx.Flash.Success(translation.NewLocale(ctx.Doer.Language).Tr("settings.update_profile_success")) + ctx.Flash.Success(ctx.Tr("settings.update_profile_success")) ctx.Redirect(setting.AppSubURL + "/user/settings") } From ff18d1744273d093d854f548662a0c204f220c16 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Sun, 5 Feb 2023 16:42:48 -0500 Subject: [PATCH 07/33] use drone secrets for s3 config (#22770) --- .drone.yml | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/.drone.yml b/.drone.yml index f9da8f9743..dc8423875a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -763,10 +763,16 @@ steps: image: woodpeckerci/plugin-s3:latest pull: always settings: - acl: public-read - bucket: gitea-artifacts - endpoint: https://ams3.digitaloceanspaces.com - path_style: true + acl: + from_secret: aws_s3_acl + region: + from_secret: aws_s3_region + bucket: + from_secret: aws_s3_bucket + endpoint: + from_secret: aws_s3_endpoint + path_style: + from_secret: aws_s3_path_style source: "dist/release/*" strip_prefix: dist/release/ target: "/gitea/${DRONE_BRANCH##release/v}" @@ -784,10 +790,16 @@ steps: - name: release-main image: woodpeckerci/plugin-s3:latest settings: - acl: public-read - bucket: gitea-artifacts - endpoint: https://ams3.digitaloceanspaces.com - path_style: true + acl: + from_secret: aws_s3_acl + region: + from_secret: aws_s3_region + bucket: + from_secret: aws_s3_bucket + endpoint: + from_secret: aws_s3_endpoint + path_style: + from_secret: aws_s3_path_style source: "dist/release/*" strip_prefix: dist/release/ target: /gitea/main @@ -886,10 +898,16 @@ steps: image: woodpeckerci/plugin-s3:latest pull: always settings: - acl: public-read - bucket: gitea-artifacts - endpoint: https://ams3.digitaloceanspaces.com - path_style: true + acl: + from_secret: aws_s3_acl + region: + from_secret: aws_s3_region + bucket: + from_secret: aws_s3_bucket + endpoint: + from_secret: aws_s3_endpoint + path_style: + from_secret: aws_s3_path_style source: "dist/release/*" strip_prefix: dist/release/ target: "/gitea/${DRONE_TAG##v}" From d987ac6bf1d78b3a9bbd213e73b871ebc687acb2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 6 Feb 2023 02:49:21 +0100 Subject: [PATCH 08/33] Add Chef package registry (#22554) This PR implements a [Chef registry](https://chef.io/) to manage cookbooks. This package type was a bit complicated because Chef uses RSA signed requests as authentication with the registry. ![grafik](https://user-images.githubusercontent.com/1666336/213747995-46819fd8-c3d6-45a2-afd4-a4c3c8505a4a.png) ![grafik](https://user-images.githubusercontent.com/1666336/213748145-d01c9e81-d4dd-41e3-a3cc-8241862c3166.png) Co-authored-by: Lunny Xiao --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + docs/content/doc/packages/chef.en-us.md | 96 +++ docs/content/doc/packages/overview.en-us.md | 1 + models/packages/descriptor.go | 3 + models/packages/package.go | 6 + modules/activitypub/user_settings.go | 5 +- modules/packages/chef/metadata.go | 134 +++++ modules/packages/chef/metadata_test.go | 92 +++ modules/setting/packages.go | 2 + modules/{activitypub => util}/keypair.go | 10 +- modules/{activitypub => util}/keypair_test.go | 6 +- options/locale/locale_en-US.ini | 12 +- public/img/svg/gitea-chef.svg | 1 + routers/api/packages/api.go | 21 + routers/api/packages/chef/auth.go | 270 +++++++++ routers/api/packages/chef/chef.go | 404 +++++++++++++ routers/api/v1/packages/package.go | 2 +- routers/web/user/setting/packages.go | 22 + routers/web/web.go | 1 + services/forms/package_form.go | 2 +- services/packages/packages.go | 2 + templates/package/content/chef.tmpl | 48 ++ templates/package/metadata/chef.tmpl | 5 + templates/package/metadata/container.tmpl | 4 +- templates/package/metadata/pub.tmpl | 4 +- templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + templates/user/settings/packages.tmpl | 18 + tests/integration/api_packages_chef_test.go | 560 ++++++++++++++++++ web_src/svg/gitea-chef.svg | 20 + 31 files changed, 1737 insertions(+), 20 deletions(-) create mode 100644 docs/content/doc/packages/chef.en-us.md create mode 100644 modules/packages/chef/metadata.go create mode 100644 modules/packages/chef/metadata_test.go rename modules/{activitypub => util}/keypair.go (76%) rename modules/{activitypub => util}/keypair_test.go (93%) create mode 100644 public/img/svg/gitea-chef.svg create mode 100644 routers/api/packages/chef/auth.go create mode 100644 routers/api/packages/chef/chef.go create mode 100644 templates/package/content/chef.tmpl create mode 100644 templates/package/metadata/chef.tmpl create mode 100644 tests/integration/api_packages_chef_test.go create mode 100644 web_src/svg/gitea-chef.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 0f1f18646b..b478785a07 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2460,6 +2460,8 @@ ROUTER = console ;LIMIT_TOTAL_OWNER_SIZE = -1 ;; Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CARGO = -1 +;; Maximum size of a Chef upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CHEF = -1 ;; Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_COMPOSER = -1 ;; Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index c9116edc66..04344b15dc 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1214,6 +1214,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maximum count of package versions a single owner can have (`-1` means no limits) - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CARGO`: **-1**: Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CHEF`: **-1**: Maximum size of a Chef upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/packages/chef.en-us.md b/docs/content/doc/packages/chef.en-us.md new file mode 100644 index 0000000000..ecc774d792 --- /dev/null +++ b/docs/content/doc/packages/chef.en-us.md @@ -0,0 +1,96 @@ +--- +date: "2023-01-20T00:00:00+00:00" +title: "Chef Packages Repository" +slug: "packages/chef" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Chef" + weight: 5 + identifier: "chef" +--- + +# Chef Packages Repository + +Publish [Chef](https://chef.io/) cookbooks for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Chef package registry, you have to use [`knife`](https://docs.chef.io/workstation/knife/). + +## Authentication + +The Chef package registry does not use an username:password authentication but signed requests with a private:public key pair. +Visit the package owner settings page to create the necessary key pair. +Only the public key is stored inside Gitea. if you loose access to the private key you must re-generate the key pair. +[Configure `knife`](https://docs.chef.io/workstation/knife_setup/) to use the downloaded private key with your Gitea username as `client_name`. + +## Configure the package registry + +To [configure `knife`](https://docs.chef.io/workstation/knife_setup/) to use the Gitea package registry add the url to the `~/.chef/config.rb` file. + +``` +knife[:supermarket_site] = 'https://gitea.example.com/api/packages/{owner}/chef' +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +## Publish a package + +To publish a Chef package execute the following command: + +```shell +knife supermarket share {package_name} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `package_name` | The package name. | + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +knife supermarket install {package_name} +``` + +Optional you can specify the package version: + +```shell +knife supermarket install {package_name} {package_version} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `package_name` | The package name. | +| `package_version` | The package version. | + +## Delete a package + +If you want to remove a package from the registry, execute the following command: + +```shell +knife supermarket unshare {package_name} +``` + +Optional you can specify the package version: + +```shell +knife supermarket unshare {package_name}/versions/{package_version} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `package_name` | The package name. | +| `package_version` | The package version. | diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index b3ccb73c19..1199d9ede0 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -27,6 +27,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | | [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` | +| [Chef]({{< relref "doc/packages/chef.en-us.md" >}}) | - | `knife` | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Conda]({{< relref "doc/packages/conda.en-us.md" >}}) | - | `conda` | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 40010eb720..96f34a22ee 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -12,6 +12,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/conda" @@ -132,6 +133,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc switch p.Type { case TypeCargo: metadata = &cargo.Metadata{} + case TypeChef: + metadata = &chef.Metadata{} case TypeComposer: metadata = &composer.Metadata{} case TypeConan: diff --git a/models/packages/package.go b/models/packages/package.go index b6b033cc95..32f30fab9b 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -31,6 +31,7 @@ type Type string // List of supported packages const ( TypeCargo Type = "cargo" + TypeChef Type = "chef" TypeComposer Type = "composer" TypeConan Type = "conan" TypeConda Type = "conda" @@ -48,6 +49,7 @@ const ( var TypeList = []Type{ TypeCargo, + TypeChef, TypeComposer, TypeConan, TypeConda, @@ -68,6 +70,8 @@ func (pt Type) Name() string { switch pt { case TypeCargo: return "Cargo" + case TypeChef: + return "Chef" case TypeComposer: return "Composer" case TypeConan: @@ -103,6 +107,8 @@ func (pt Type) SVGName() string { switch pt { case TypeCargo: return "gitea-cargo" + case TypeChef: + return "gitea-chef" case TypeComposer: return "gitea-composer" case TypeConan: diff --git a/modules/activitypub/user_settings.go b/modules/activitypub/user_settings.go index ec5fa59842..2d156c17e6 100644 --- a/modules/activitypub/user_settings.go +++ b/modules/activitypub/user_settings.go @@ -5,8 +5,11 @@ package activitypub import ( user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" ) +const rsaBits = 2048 + // GetKeyPair function returns a user's private and public keys func GetKeyPair(user *user_model.User) (pub, priv string, err error) { var settings map[string]*user_model.Setting @@ -14,7 +17,7 @@ func GetKeyPair(user *user_model.User) (pub, priv string, err error) { if err != nil { return } else if len(settings) == 0 { - if priv, pub, err = GenerateKeyPair(); err != nil { + if priv, pub, err = util.GenerateKeyPair(rsaBits); err != nil { return } if err = user_model.SetUserSetting(user.ID, user_model.UserActivityPubPrivPem, priv); err != nil { diff --git a/modules/packages/chef/metadata.go b/modules/packages/chef/metadata.go new file mode 100644 index 0000000000..a1c91870c2 --- /dev/null +++ b/modules/packages/chef/metadata.go @@ -0,0 +1,134 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "archive/tar" + "compress/gzip" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" +) + +const ( + KeyBits = 4096 + SettingPublicPem = "chef.public_pem" +) + +var ( + ErrMissingMetadataFile = util.NewInvalidArgumentErrorf("metadata.json file is missing") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") + + namePattern = regexp.MustCompile(`\A\S+\z`) + versionPattern = regexp.MustCompile(`\A\d+\.\d+(?:\.\d+)?\z`) +) + +// Package represents a Chef package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a Chef package +type Metadata struct { + Description string `json:"description,omitempty"` + LongDescription string `json:"long_description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` +} + +type chefMetadata struct { + Name string `json:"name"` + Description string `json:"description"` + LongDescription string `json:"long_description"` + Maintainer string `json:"maintainer"` + MaintainerEmail string `json:"maintainer_email"` + License string `json:"license"` + Platforms map[string]string `json:"platforms"` + Dependencies map[string]string `json:"dependencies"` + Providing map[string]string `json:"providing"` + Recipes map[string]string `json:"recipes"` + Version string `json:"version"` + SourceURL string `json:"source_url"` + IssuesURL string `json:"issues_url"` + Privacy bool `json:"privacy"` + ChefVersions [][]string `json:"chef_versions"` + Gems [][]string `json:"gems"` + EagerLoadLibraries bool `json:"eager_load_libraries"` +} + +// ParsePackage parses the Chef package file +func ParsePackage(r io.Reader) (*Package, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if strings.Count(hd.Name, "/") != 1 { + continue + } + + if hd.FileInfo().Name() == "metadata.json" { + return ParseChefMetadata(tr) + } + } + + return nil, ErrMissingMetadataFile +} + +// ParseChefMetadata parses a metadata.json file to retrieve the metadata of a Chef package +func ParseChefMetadata(r io.Reader) (*Package, error) { + var cm chefMetadata + if err := json.NewDecoder(r).Decode(&cm); err != nil { + return nil, err + } + + if !namePattern.MatchString(cm.Name) { + return nil, ErrInvalidName + } + + if !versionPattern.MatchString(cm.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(cm.SourceURL) { + cm.SourceURL = "" + } + + return &Package{ + Name: cm.Name, + Version: cm.Version, + Metadata: &Metadata{ + Description: cm.Description, + LongDescription: cm.LongDescription, + Author: cm.Maintainer, + License: cm.License, + RepositoryURL: cm.SourceURL, + Dependencies: cm.Dependencies, + }, + }, nil +} diff --git a/modules/packages/chef/metadata_test.go b/modules/packages/chef/metadata_test.go new file mode 100644 index 0000000000..6def4162a9 --- /dev/null +++ b/modules/packages/chef/metadata_test.go @@ -0,0 +1,92 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + packageAuthor = "KN4CK3R" + packageDescription = "Package Description" + packageRepositoryURL = "https://gitea.io/gitea/gitea" +) + +func TestParsePackage(t *testing.T) { + t.Run("MissingMetadataFile", func(t *testing.T) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + tw.Close() + zw.Close() + + p, err := ParsePackage(&buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrMissingMetadataFile) + }) + + t.Run("Valid", func(t *testing.T) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + + content := `{"name":"` + packageName + `","version":"` + packageVersion + `"}` + + hdr := &tar.Header{ + Name: packageName + "/metadata.json", + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write([]byte(content)) + + tw.Close() + zw.Close() + + p, err := ParsePackage(&buf) + assert.NoError(t, err) + assert.NotNil(t, p) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.NotNil(t, p.Metadata) + }) +} + +func TestParseChefMetadata(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{" test", "test "} { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"` + name + `","version":"1.0.0"}`)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"1", "1.2.3.4", "1.0.0 "} { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"test","version":"` + version + `"}`)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"` + packageName + `","version":"` + packageVersion + `","description":"` + packageDescription + `","maintainer":"` + packageAuthor + `","source_url":"` + packageRepositoryURL + `"}`)) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, packageRepositoryURL, p.Metadata.RepositoryURL) + }) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 190c17dd8f..84da4eb53e 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -26,6 +26,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 LimitSizeCargo int64 + LimitSizeChef int64 LimitSizeComposer int64 LimitSizeConan int64 LimitSizeConda int64 @@ -67,6 +68,7 @@ func newPackages() { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") + Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") diff --git a/modules/activitypub/keypair.go b/modules/util/keypair.go similarity index 76% rename from modules/activitypub/keypair.go rename to modules/util/keypair.go index 299bdc43e3..5a3ce715a4 100644 --- a/modules/activitypub/keypair.go +++ b/modules/util/keypair.go @@ -1,7 +1,7 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package activitypub +package util import ( "crypto/rand" @@ -10,11 +10,9 @@ import ( "encoding/pem" ) -const rsaBits = 2048 - -// GenerateKeyPair generates a public and private keypair for signing actions by users for activitypub purposes -func GenerateKeyPair() (string, string, error) { - priv, _ := rsa.GenerateKey(rand.Reader, rsaBits) +// GenerateKeyPair generates a public and private keypair +func GenerateKeyPair(bits int) (string, string, error) { + priv, _ := rsa.GenerateKey(rand.Reader, bits) privPem, err := pemBlockForPriv(priv) if err != nil { return "", "", err diff --git a/modules/activitypub/keypair_test.go b/modules/util/keypair_test.go similarity index 93% rename from modules/activitypub/keypair_test.go rename to modules/util/keypair_test.go index 888254c9da..c6f68c845a 100644 --- a/modules/activitypub/keypair_test.go +++ b/modules/util/keypair_test.go @@ -1,7 +1,7 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package activitypub +package util import ( "crypto" @@ -17,7 +17,7 @@ import ( ) func TestKeygen(t *testing.T) { - priv, pub, err := GenerateKeyPair() + priv, pub, err := GenerateKeyPair(2048) assert.NoError(t, err) assert.NotEmpty(t, priv) @@ -28,7 +28,7 @@ func TestKeygen(t *testing.T) { } func TestSignUsingKeys(t *testing.T) { - priv, pub, err := GenerateKeyPair() + priv, pub, err := GenerateKeyPair(2048) assert.NoError(t, err) privPem, _ := pem.Decode([]byte(priv)) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index bc2e8cb91c..a7506986f6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3145,6 +3145,8 @@ keywords = Keywords details = Details details.author = Author details.project_site = Project Site +details.repository_site = Repository Site +details.documentation_site = Documentation Site details.license = License assets = Assets versions = Versions @@ -3157,6 +3159,9 @@ cargo.install = To install the package using Cargo, run the following command: cargo.documentation = For more information on the Cargo registry, see the documentation. cargo.details.repository_site = Repository Site cargo.details.documentation_site = Documentation Site +chef.registry = Setup this registry in your ~/.chef/config.rb file: +chef.install = To install the package, run the following command: +chef.documentation = For more information on the Chef registry, see the documentation. composer.registry = Setup this registry in your ~/.composer/config.json file: composer.install = To install the package using Composer, run the following command: composer.documentation = For more information on the Composer registry, see the documentation. @@ -3173,8 +3178,6 @@ conda.details.repository_site = Repository Site conda.details.documentation_site = Documentation Site container.details.type = Image Type container.details.platform = Platform -container.details.repository_site = Repository Site -container.details.documentation_site = Documentation Site container.pull = Pull the image from the command line: container.digest = Digest: container.documentation = For more information on the Container registry, see the documentation. @@ -3208,8 +3211,6 @@ npm.dependencies.optional = Optional Dependencies npm.details.tag = Tag pub.install = To install the package using Dart, run the following command: pub.documentation = For more information on the Pub registry, see the documentation. -pub.details.repository_site = Repository Site -pub.details.documentation_site = Documentation Site pypi.requires = Requires Python pypi.install = To install the package using pip, run the following command: pypi.documentation = For more information on the PyPI registry, see the documentation. @@ -3262,6 +3263,9 @@ owner.settings.cleanuprules.remove.days = Remove versions older than owner.settings.cleanuprules.remove.pattern = Remove versions matching owner.settings.cleanuprules.success.update = Cleanup rule has been updated. owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted. +owner.settings.chef.title = Chef Registry +owner.settings.chef.keypair = Generate key pair +owner.settings.chef.keypair.description = Generate a key pair used to authenticate against the Chef registry. The previous key can not be used afterwards. [secrets] secrets = Secrets diff --git a/public/img/svg/gitea-chef.svg b/public/img/svg/gitea-chef.svg new file mode 100644 index 0000000000..8f1cd6165c --- /dev/null +++ b/public/img/svg/gitea-chef.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 8ec9ae9bf7..9f77367d6f 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/cargo" + "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/conda" @@ -54,6 +55,7 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { &auth.Basic{}, &nuget.Auth{}, &conan.Auth{}, + &chef.Auth{}, } if setting.Service.EnableReverseProxyAuth { authMethods = append(authMethods, &auth.ReverseProxy{}) @@ -86,6 +88,25 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/chef", func() { + r.Group("/api/v1", func() { + r.Get("/universe", chef.PackagesUniverse) + r.Get("/search", chef.EnumeratePackages) + r.Group("/cookbooks", func() { + r.Get("", chef.EnumeratePackages) + r.Post("", reqPackageAccess(perm.AccessModeWrite), chef.UploadPackage) + r.Group("/{name}", func() { + r.Get("", chef.PackageMetadata) + r.Group("/versions/{version}", func() { + r.Get("", chef.PackageVersionMetadata) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), chef.DeletePackageVersion) + r.Get("/download", chef.DownloadPackage) + }) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), chef.DeletePackage) + }) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/composer", func() { r.Get("/packages.json", composer.ServiceIndex) r.Get("/search.json", composer.SearchPackages) diff --git a/routers/api/packages/chef/auth.go b/routers/api/packages/chef/auth.go new file mode 100644 index 0000000000..69f7b763ab --- /dev/null +++ b/routers/api/packages/chef/auth.go @@ -0,0 +1,270 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "crypto" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "hash" + "math/big" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" + + user_model "code.gitea.io/gitea/models/user" + chef_module "code.gitea.io/gitea/modules/packages/chef" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/auth" +) + +const ( + maxTimeDifference = 10 * time.Minute +) + +var ( + algorithmPattern = regexp.MustCompile(`algorithm=(\w+)`) + versionPattern = regexp.MustCompile(`version=(\d+\.\d+)`) + authorizationPattern = regexp.MustCompile(`\AX-Ops-Authorization-(\d+)`) +) + +// Documentation: +// https://docs.chef.io/server/api_chef_server/#required-headers +// https://github.com/chef-boneyard/chef-rfc/blob/master/rfc065-sign-v1.3.md +// https://github.com/chef/mixlib-authentication/blob/bc8adbef833d4be23dc78cb23e6fe44b51ebc34f/lib/mixlib/authentication/signedheaderauth.rb + +type Auth struct{} + +func (a *Auth) Name() string { + return "chef" +} + +// Verify extracts the user from the signed request +// If the request is signed with the user private key the user is verified. +func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) { + u, err := getUserFromRequest(req) + if err != nil { + return nil, err + } + if u == nil { + return nil, nil + } + + pub, err := getUserPublicKey(u) + if err != nil { + return nil, err + } + + if err := verifyTimestamp(req); err != nil { + return nil, err + } + + version, err := getSignVersion(req) + if err != nil { + return nil, err + } + + if err := verifySignedHeaders(req, version, pub.(*rsa.PublicKey)); err != nil { + return nil, err + } + + return u, nil +} + +func getUserFromRequest(req *http.Request) (*user_model.User, error) { + username := req.Header.Get("X-Ops-Userid") + if username == "" { + return nil, nil + } + + return user_model.GetUserByName(req.Context(), username) +} + +func getUserPublicKey(u *user_model.User) (crypto.PublicKey, error) { + pubKey, err := user_model.GetSetting(u.ID, chef_module.SettingPublicPem) + if err != nil { + return nil, err + } + + pubPem, _ := pem.Decode([]byte(pubKey)) + + return x509.ParsePKIXPublicKey(pubPem.Bytes) +} + +func verifyTimestamp(req *http.Request) error { + hdr := req.Header.Get("X-Ops-Timestamp") + if hdr == "" { + return util.NewInvalidArgumentErrorf("X-Ops-Timestamp header missing") + } + + ts, err := time.Parse(time.RFC3339, hdr) + if err != nil { + return err + } + + diff := time.Now().UTC().Sub(ts) + if diff < 0 { + diff = -diff + } + + if diff > maxTimeDifference { + return fmt.Errorf("time difference") + } + + return nil +} + +func getSignVersion(req *http.Request) (string, error) { + hdr := req.Header.Get("X-Ops-Sign") + if hdr == "" { + return "", util.NewInvalidArgumentErrorf("X-Ops-Sign header missing") + } + + m := versionPattern.FindStringSubmatch(hdr) + if len(m) != 2 { + return "", util.NewInvalidArgumentErrorf("invalid X-Ops-Sign header") + } + + switch m[1] { + case "1.0", "1.1", "1.2", "1.3": + default: + return "", util.NewInvalidArgumentErrorf("unsupported version") + } + + version := m[1] + + m = algorithmPattern.FindStringSubmatch(hdr) + if len(m) == 2 && m[1] != "sha1" && !(m[1] == "sha256" && version == "1.3") { + return "", util.NewInvalidArgumentErrorf("unsupported algorithm") + } + + return version, nil +} + +func verifySignedHeaders(req *http.Request, version string, pub *rsa.PublicKey) error { + authorizationData, err := getAuthorizationData(req) + if err != nil { + return err + } + + checkData := buildCheckData(req, version) + + switch version { + case "1.3": + return verifyDataNew(authorizationData, checkData, pub, crypto.SHA256) + case "1.2": + return verifyDataNew(authorizationData, checkData, pub, crypto.SHA1) + default: + return verifyDataOld(authorizationData, checkData, pub) + } +} + +func getAuthorizationData(req *http.Request) ([]byte, error) { + valueList := make(map[int]string) + for k, vs := range req.Header { + if m := authorizationPattern.FindStringSubmatch(k); m != nil { + index, _ := strconv.Atoi(m[1]) + var v string + if len(vs) == 0 { + v = "" + } else { + v = vs[0] + } + valueList[index] = v + } + } + + tmp := make([]string, len(valueList)) + for k, v := range valueList { + if k > len(tmp) { + return nil, fmt.Errorf("invalid X-Ops-Authorization headers") + } + tmp[k-1] = v + } + + return base64.StdEncoding.DecodeString(strings.Join(tmp, "")) +} + +func buildCheckData(req *http.Request, version string) []byte { + username := req.Header.Get("X-Ops-Userid") + if version != "1.0" && version != "1.3" { + sum := sha1.Sum([]byte(username)) + username = base64.StdEncoding.EncodeToString(sum[:]) + } + + var data string + if version == "1.3" { + data = fmt.Sprintf( + "Method:%s\nPath:%s\nX-Ops-Content-Hash:%s\nX-Ops-Sign:version=%s\nX-Ops-Timestamp:%s\nX-Ops-UserId:%s\nX-Ops-Server-API-Version:%s", + req.Method, + path.Clean(req.URL.Path), + req.Header.Get("X-Ops-Content-Hash"), + version, + req.Header.Get("X-Ops-Timestamp"), + username, + req.Header.Get("X-Ops-Server-Api-Version"), + ) + } else { + sum := sha1.Sum([]byte(path.Clean(req.URL.Path))) + data = fmt.Sprintf( + "Method:%s\nHashed Path:%s\nX-Ops-Content-Hash:%s\nX-Ops-Timestamp:%s\nX-Ops-UserId:%s", + req.Method, + base64.StdEncoding.EncodeToString(sum[:]), + req.Header.Get("X-Ops-Content-Hash"), + req.Header.Get("X-Ops-Timestamp"), + username, + ) + } + + return []byte(data) +} + +func verifyDataNew(signature, data []byte, pub *rsa.PublicKey, algo crypto.Hash) error { + var h hash.Hash + if algo == crypto.SHA256 { + h = sha256.New() + } else { + h = sha1.New() + } + if _, err := h.Write(data); err != nil { + return err + } + + return rsa.VerifyPKCS1v15(pub, algo, h.Sum(nil), signature) +} + +func verifyDataOld(signature, data []byte, pub *rsa.PublicKey) error { + c := new(big.Int) + m := new(big.Int) + m.SetBytes(signature) + e := big.NewInt(int64(pub.E)) + c.Exp(m, e, pub.N) + + out := c.Bytes() + + skip := 0 + for i := 2; i < len(out); i++ { + if i+1 >= len(out) { + break + } + if out[i] == 0xFF && out[i+1] == 0 { + skip = i + 2 + break + } + } + + if !util.SliceEqual(out[skip:], data) { + return fmt.Errorf("could not verify signature") + } + + return nil +} diff --git a/routers/api/packages/chef/chef.go b/routers/api/packages/chef/chef.go new file mode 100644 index 0000000000..28d07dea47 --- /dev/null +++ b/routers/api/packages/chef/chef.go @@ -0,0 +1,404 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + chef_module "code.gitea.io/gitea/modules/packages/chef" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + type Error struct { + ErrorMessages []string `json:"error_messages"` + } + + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, Error{ + ErrorMessages: []string{message}, + }) + }) +} + +func PackagesUniverse(ctx *context.Context) { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeChef, + IsInternal: util.OptionalBoolFalse, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type VersionInfo struct { + LocationType string `json:"location_type"` + LocationPath string `json:"location_path"` + DownloadURL string `json:"download_url"` + Dependencies map[string]string `json:"dependencies"` + } + + baseURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/chef/api/v1" + + universe := make(map[string]map[string]*VersionInfo) + for _, pd := range pds { + if _, ok := universe[pd.Package.Name]; !ok { + universe[pd.Package.Name] = make(map[string]*VersionInfo) + } + universe[pd.Package.Name][pd.Version.Version] = &VersionInfo{ + LocationType: "opscode", + LocationPath: baseURL, + DownloadURL: fmt.Sprintf("%s/cookbooks/%s/versions/%s/download", baseURL, url.PathEscape(pd.Package.Name), pd.Version.Version), + Dependencies: pd.Metadata.(*chef_module.Metadata).Dependencies, + } + } + + ctx.JSON(http.StatusOK, universe) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_list.rb +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_search.rb +func EnumeratePackages(ctx *context.Context) { + opts := &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeChef, + Name: packages_model.SearchValue{Value: ctx.FormTrim("q")}, + IsInternal: util.OptionalBoolFalse, + Paginator: db.NewAbsoluteListOptions( + ctx.FormInt("start"), + ctx.FormInt("items"), + ), + } + + switch strings.ToLower(ctx.FormTrim("order")) { + case "recently_updated", "recently_added": + opts.Sort = packages_model.SortCreatedDesc + default: + opts.Sort = packages_model.SortNameAsc + } + + pvs, total, err := packages_model.SearchLatestVersions(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type Item struct { + CookbookName string `json:"cookbook_name"` + CookbookMaintainer string `json:"cookbook_maintainer"` + CookbookDescription string `json:"cookbook_description"` + Cookbook string `json:"cookbook"` + } + + type Result struct { + Start int `json:"start"` + Total int `json:"total"` + Items []*Item `json:"items"` + } + + baseURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/chef/api/v1/cookbooks/" + + items := make([]*Item, 0, len(pds)) + for _, pd := range pds { + metadata := pd.Metadata.(*chef_module.Metadata) + + items = append(items, &Item{ + CookbookName: pd.Package.Name, + CookbookMaintainer: metadata.Author, + CookbookDescription: metadata.Description, + Cookbook: baseURL + url.PathEscape(pd.Package.Name), + }) + } + + skip, _ := opts.Paginator.GetSkipTake() + + ctx.JSON(http.StatusOK, &Result{ + Start: skip, + Total: int(total), + Items: items, + }) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_show.rb +func PackageMetadata(ctx *context.Context) { + packageName := ctx.Params("name") + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeChef, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + type Result struct { + Name string `json:"name"` + Maintainer string `json:"maintainer"` + Description string `json:"description"` + Category string `json:"category"` + LatestVersion string `json:"latest_version"` + SourceURL string `json:"source_url"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Deprecated bool `json:"deprecated"` + Versions []string `json:"versions"` + } + + baseURL := fmt.Sprintf("%sapi/packages/%s/chef/api/v1/cookbooks/%s/versions/", setting.AppURL, ctx.Package.Owner.Name, url.PathEscape(packageName)) + + versions := make([]string, 0, len(pds)) + for _, pd := range pds { + versions = append(versions, baseURL+pd.Version.Version) + } + + latest := pds[len(pds)-1] + + metadata := latest.Metadata.(*chef_module.Metadata) + + ctx.JSON(http.StatusOK, &Result{ + Name: latest.Package.Name, + Maintainer: metadata.Author, + Description: metadata.Description, + LatestVersion: baseURL + latest.Version.Version, + SourceURL: metadata.RepositoryURL, + CreatedAt: latest.Version.CreatedUnix.AsLocalTime(), + UpdatedAt: latest.Version.CreatedUnix.AsLocalTime(), + Deprecated: false, + Versions: versions, + }) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_show.rb +func PackageVersionMetadata(ctx *context.Context) { + packageName := ctx.Params("name") + packageVersion := strings.ReplaceAll(ctx.Params("version"), "_", ".") // Chef calls this endpoint with "_" instead of "."?! + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeChef, packageName, packageVersion) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type Result struct { + Version string `json:"version"` + TarballFileSize int64 `json:"tarball_file_size"` + PublishedAt time.Time `json:"published_at"` + Cookbook string `json:"cookbook"` + File string `json:"file"` + License string `json:"license"` + Dependencies map[string]string `json:"dependencies"` + } + + baseURL := fmt.Sprintf("%sapi/packages/%s/chef/api/v1/cookbooks/%s", setting.AppURL, ctx.Package.Owner.Name, url.PathEscape(pd.Package.Name)) + + metadata := pd.Metadata.(*chef_module.Metadata) + + ctx.JSON(http.StatusOK, &Result{ + Version: pd.Version.Version, + TarballFileSize: pd.Files[0].Blob.Size, + PublishedAt: pd.Version.CreatedUnix.AsLocalTime(), + Cookbook: baseURL, + File: fmt.Sprintf("%s/versions/%s/download", baseURL, pd.Version.Version), + License: metadata.License, + Dependencies: metadata.Dependencies, + }) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_share.rb +func UploadPackage(ctx *context.Context) { + file, _, err := ctx.Req.FormFile("tarball") + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + defer file.Close() + + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pck, err := chef_module.ParsePackage(buf) + if err != nil { + if errors.Is(err, util.ErrInvalidArgument) { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeChef, + Name: pck.Name, + Version: pck.Version, + }, + Creator: ctx.Doer, + SemverCompatible: true, + Metadata: pck.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(pck.Version + ".tar.gz"), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusBadRequest, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.JSON(http.StatusCreated, make(map[any]any)) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_download.rb +func DownloadPackage(ctx *context.Context) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeChef, ctx.Params("name"), ctx.Params("version")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pf := pd.Files[0].File + + s, _, err := packages_service.GetPackageFileStream(ctx, pf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_unshare.rb +func DeletePackageVersion(ctx *context.Context) { + packageName := ctx.Params("name") + packageVersion := ctx.Params("version") + + err := packages_service.RemovePackageVersionByNameAndVersion( + ctx.Doer, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeChef, + Name: packageName, + Version: packageVersion, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.Status(http.StatusOK) +} + +// https://github.com/chef/chef/blob/main/knife/lib/chef/knife/supermarket_unshare.rb +func DeletePackage(ctx *context.Context) { + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeChef, ctx.Params("name")) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + for _, pv := range pvs { + if err := packages_service.RemovePackageVersion(ctx.Doer, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + ctx.Status(http.StatusOK) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 2a20d66d1e..ab077090d1 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [cargo, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] + // enum: [cargo, chef, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go index b3f8a3e41d..0d2eb14c20 100644 --- a/routers/web/user/setting/packages.go +++ b/routers/web/user/setting/packages.go @@ -5,10 +5,14 @@ package setting import ( "net/http" + "strings" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + chef_module "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" shared "code.gitea.io/gitea/routers/web/shared/packages" ) @@ -95,3 +99,21 @@ func RebuildCargoIndex(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/user/settings/packages") } + +func RegenerateChefKeyPair(ctx *context.Context) { + priv, pub, err := util.GenerateKeyPair(chef_module.KeyBits) + if err != nil { + ctx.ServerError("GenerateKeyPair", err) + return + } + + if err := user_model.SetUserSetting(ctx.Doer.ID, chef_module.SettingPublicPem, pub); err != nil { + ctx.ServerError("SetUserSetting", err) + return + } + + ctx.ServeContent(strings.NewReader(priv), &context.ServeHeaderOptions{ + ContentType: "application/x-pem-file", + Filename: ctx.Doer.Name + ".priv", + }) +} diff --git a/routers/web/web.go b/routers/web/web.go index a024c0ac37..6898956053 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -472,6 +472,7 @@ func RegisterRoutes(m *web.Route) { m.Post("/initialize", user_setting.InitializeCargoIndex) m.Post("/rebuild", user_setting.RebuildCargoIndex) }) + m.Post("/chef/regenerate_keypair", user_setting.RegenerateChefKeyPair) }, packagesEnabled) m.Group("/secrets", func() { m.Get("", user_setting.Secrets) diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 558ed54b65..b22ed47c77 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(cargo,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/packages.go b/services/packages/packages.go index f502840755..3abca7337c 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -335,6 +335,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p switch packageType { case packages_model.TypeCargo: typeSpecificSize = setting.Packages.LimitSizeCargo + case packages_model.TypeChef: + typeSpecificSize = setting.Packages.LimitSizeChef case packages_model.TypeComposer: typeSpecificSize = setting.Packages.LimitSizeComposer case packages_model.TypeConan: diff --git a/templates/package/content/chef.tmpl b/templates/package/content/chef.tmpl new file mode 100644 index 0000000000..f0f8de7bd5 --- /dev/null +++ b/templates/package/content/chef.tmpl @@ -0,0 +1,48 @@ +{{if eq .PackageDescriptor.Package.Type "chef"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
knife[:supermarket_site] = '{{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/chef'
+
+
+ + 
knife supermarket install {{.PackageDescriptor.Package.Name}} {{.PackageDescriptor.Version.Version}}
+
+
+ +
+
+
+ + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.LongDescription}} +

{{.locale.Tr "packages.about"}}

+
+ {{if .PackageDescriptor.Metadata.Description}}

{{.PackageDescriptor.Metadata.Description}}

{{end}} + {{if .PackageDescriptor.Metadata.LongDescription}}{{RenderMarkdownToHtml .PackageDescriptor.Metadata.LongDescription}}{{end}} +
+ {{end}} + + {{if .PackageDescriptor.Metadata.Dependencies}} +

{{.locale.Tr "packages.dependencies"}}

+
+ + + + + + + + + {{range $dependency, $version := .PackageDescriptor.Metadata.Dependencies}} + + + + + {{end}} + +
{{.locale.Tr "packages.dependency.id"}}{{.locale.Tr "packages.dependency.version"}}
{{$dependency}}{{$version}}
+
+ {{end}} +{{end}} diff --git a/templates/package/metadata/chef.tmpl b/templates/package/metadata/chef.tmpl new file mode 100644 index 0000000000..00fc0e6c34 --- /dev/null +++ b/templates/package/metadata/chef.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "chef"}} + {{if .PackageDescriptor.Metadata.Author}}
{{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
{{end}} +{{end}} diff --git a/templates/package/metadata/container.tmpl b/templates/package/metadata/container.tmpl index 1b13072c38..71df960f76 100644 --- a/templates/package/metadata/container.tmpl +++ b/templates/package/metadata/container.tmpl @@ -4,6 +4,6 @@ {{range .PackageDescriptor.Metadata.Authors}}
{{svg "octicon-person" 16 "mr-3"}} {{.}}
{{end}} {{if .PackageDescriptor.Metadata.Licenses}}
{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Licenses}}
{{end}} {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} - {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.container.details.repository_site"}}
{{end}} - {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.container.details.documentation_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.documentation_site"}}
{{end}} {{end}} diff --git a/templates/package/metadata/pub.tmpl b/templates/package/metadata/pub.tmpl index 8bd65b49d6..632a506551 100644 --- a/templates/package/metadata/pub.tmpl +++ b/templates/package/metadata/pub.tmpl @@ -1,5 +1,5 @@ {{if eq .PackageDescriptor.Package.Type "pub"}} {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} - {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.pub.details.repository_site"}}
{{end}} - {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.pub.details.documentation_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.documentation_site"}}
{{end}} {{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 6209e6cb02..a548d9e0b6 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -20,6 +20,7 @@
{{template "package/content/cargo" .}} + {{template "package/content/chef" .}} {{template "package/content/composer" .}} {{template "package/content/conan" .}} {{template "package/content/conda" .}} @@ -45,6 +46,7 @@
{{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
{{template "package/metadata/cargo" .}} + {{template "package/metadata/chef" .}} {{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} {{template "package/metadata/conda" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index d6e3009b35..812e626d2d 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2101,6 +2101,7 @@ { "enum": [ "cargo", + "chef", "composer", "conan", "conda", diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl index 472d84327a..1faddab5de 100644 --- a/templates/user/settings/packages.tmpl +++ b/templates/user/settings/packages.tmpl @@ -5,6 +5,24 @@ {{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} {{template "package/shared/cargo" .}} + +

+ {{.locale.Tr "packages.owner.settings.chef.title"}} +

+
+
+
+ +
+
+ {{.CsrfTokenHtml}} + +
+
+ +
+
+
{{template "base/footer" .}} diff --git a/tests/integration/api_packages_chef_test.go b/tests/integration/api_packages_chef_test.go new file mode 100644 index 0000000000..14baddca94 --- /dev/null +++ b/tests/integration/api_packages_chef_test.go @@ -0,0 +1,560 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "hash" + "math/big" + "mime/multipart" + "net/http" + "path" + "strings" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + chef_module "code.gitea.io/gitea/modules/packages/chef" + "code.gitea.io/gitea/modules/setting" + chef_router "code.gitea.io/gitea/routers/api/packages/chef" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageChef(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + privPem := `-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAtWp2PZz4TSU5A6ixw41HdbfBuGJwPuTtrsdoUf0DQ0/DJBNP +qOCBAgEu6ZdUqIbWJ5Da+nevjtncy5hENdi6XrXjyzlUxghMuXjE5SeLGpgfQvkq +bTkYaFpMe8PTzNeze3fei8+Eu6mzeb6g1GrqXznuPIc7bNss0w5iX9RiBM9dWPuX +onx9xSEy0LYqJm7yXmshNe1aRwkjG/y5C26BzBFnMKp9YRTua0DO1WqLNhcaRnda +lIFYouDNVTbwxSlYL16bZVoebqzZvLGrPvZJkPuCu6vH9brvOuYo0q8hLVNkBeXc +imRpsDjLhQYzEJjoMTbaiVGnjBky+PWNiofJnwIDAQABAoIBAQCotF1KxLt/ejr/ +9ROCh9JJXV3v6tL5GgkSPOv9Oq2bHgSZer/cixJNW+5VWd5nbiSe3K1WuJBw5pbW +Wj4sWORPiRRR+3mjQzqeS/nGJDTOwWJo9K8IrUzOVhLEEYLX/ksxaXJyT8PehFyb +vbNwdhCIB6ZNcXDItTWE+95twWJ5lxAIj2dNwZZni3UkwwjYnCnqFtvHCKOg0NH2 +RjQcFYmu3fncNeqLezUSdVyRyXxSCHsUdlYeX/e44StCnXdrmLUHlb2P27ZVdPGh +SW7qTUPpmJKekYiRPOpTLj+ZKXIsANkyWO+7dVtZLBm5bIyAsmp0W/DmK+wRsejj +alFbIsh5AoGBANJr7HSG695wkfn+kvu/V8qHbt+KDv4WjWHjGRsUqvxoHOUNkQmW +vZWdk4gjHYn1l+QHWmoOE3AgyqtCZ4bFILkZPLN/F8Mh3+r4B0Ac4biJJt7XGMNQ +Nv4wsk7TR7CCARsjO7GP1PT60hpjMvYmc1E36gNM7QIZE9jBE+L8eWYtAoGBANy2 +JOAWf+QeBlur6o9feH76cEmpQzUUq4Lj9mmnXgIirSsFoBnDb8VA6Ws+ltL9U9H2 +vaCoaTyi9twW9zWj+Ywg2mVR5nlSAPfdlTWS1GLUbDotlj5apc/lvnGuNlWzN+I4 +Tu64hhgBXqGvRZ0o7HzFodqRAkpVXp6CQCqBM7p7AoGAIgO0K3oL8t87ma/fTra1 +mFWgRJ5qogQ/Qo2VZ11F7ptd4GD7CxPE/cSFLsKOadi7fu75XJ994OhMGrcXSR/g +lEtSFqn6y15UdgU2FtUUX+I72FXo+Nmkqh5xFHDu68d4Kkzdv2xCvn81K3LRsByz +E3P4biQnQ+mN3cIIVu79KNkCgYEAm6uctrEn4y2KLn5DInyj8GuTZ2ELFhVOIzPG +SR7TH451tTJyiblezDHMcOfkWUx0IlN1zCr8jtgiZXmNQzg0erFxWKU7ebZtGGYh +J3g4dLx+2Unt/mzRJqFUgbnueOO/Nr+gbJ+ZdLUCmeeVohOLOTXrws0kYGl2Izab +K1+VrKECgYEAxQohoOegA0f4mofisXItbwwqTIX3bLpxBc4woa1sB4kjNrLo4slc +qtWZGVlRxwBvQUg0cYj+xtr5nyBdHLy0qwX/kMq4GqQnvW6NqsbrP3MjCZ8NX/Sj +A2W0jx50Hs/XNw6IZFLYgWVoOzCaD+jYFpHhzUZyQD6/rYhwhHrNQmU= +-----END RSA PRIVATE KEY-----` + + tmp, _ := pem.Decode([]byte(privPem)) + privKey, _ := x509.ParsePKCS1PrivateKey(tmp.Bytes) + + pubPem := `-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtWp2PZz4TSU5A6ixw41H +dbfBuGJwPuTtrsdoUf0DQ0/DJBNPqOCBAgEu6ZdUqIbWJ5Da+nevjtncy5hENdi6 +XrXjyzlUxghMuXjE5SeLGpgfQvkqbTkYaFpMe8PTzNeze3fei8+Eu6mzeb6g1Grq +XznuPIc7bNss0w5iX9RiBM9dWPuXonx9xSEy0LYqJm7yXmshNe1aRwkjG/y5C26B +zBFnMKp9YRTua0DO1WqLNhcaRndalIFYouDNVTbwxSlYL16bZVoebqzZvLGrPvZJ +kPuCu6vH9brvOuYo0q8hLVNkBeXcimRpsDjLhQYzEJjoMTbaiVGnjBky+PWNiofJ +nwIDAQAB +-----END PUBLIC KEY-----` + + err := user_model.SetUserSetting(user.ID, chef_module.SettingPublicPem, pubPem) + assert.NoError(t, err) + + t.Run("Authenticate", func(t *testing.T) { + auth := &chef_router.Auth{} + + t.Run("MissingUser", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "POST", "/dummy") + u, err := auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.NoError(t, err) + }) + + t.Run("NotExistingUser", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "POST", "/dummy") + req.Header.Set("X-Ops-Userid", "not-existing-user") + u, err := auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + }) + + t.Run("Timestamp", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "POST", "/dummy") + req.Header.Set("X-Ops-Userid", user.Name) + u, err := auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + req.Header.Set("X-Ops-Timestamp", "2023-01-01T00:00:00Z") + u, err = auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + }) + + t.Run("SigningVersion", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "POST", "/dummy") + req.Header.Set("X-Ops-Userid", user.Name) + req.Header.Set("X-Ops-Timestamp", time.Now().UTC().Format(time.RFC3339)) + u, err := auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + req.Header.Set("X-Ops-Sign", "version=none") + u, err = auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + req.Header.Set("X-Ops-Sign", "version=1.4") + u, err = auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + req.Header.Set("X-Ops-Sign", "version=1.0;algorithm=sha2") + u, err = auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + req.Header.Set("X-Ops-Sign", "version=1.0;algorithm=sha256") + u, err = auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + }) + + t.Run("SignedHeaders", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ts := time.Now().UTC().Format(time.RFC3339) + + req := NewRequest(t, "POST", "/dummy") + req.Header.Set("X-Ops-Userid", user.Name) + req.Header.Set("X-Ops-Timestamp", ts) + req.Header.Set("X-Ops-Sign", "version=1.0;algorithm=sha1") + req.Header.Set("X-Ops-Content-Hash", "unused") + req.Header.Set("X-Ops-Authorization-4", "dummy") + u, err := auth.Verify(req, nil, nil, nil) + assert.Nil(t, u) + assert.Error(t, err) + + signRequest := func(t *testing.T, req *http.Request, version string) { + username := req.Header.Get("X-Ops-Userid") + if version != "1.0" && version != "1.3" { + sum := sha1.Sum([]byte(username)) + username = base64.StdEncoding.EncodeToString(sum[:]) + } + + req.Header.Set("X-Ops-Sign", "version="+version) + + var data []byte + if version == "1.3" { + data = []byte(fmt.Sprintf( + "Method:%s\nPath:%s\nX-Ops-Content-Hash:%s\nX-Ops-Sign:version=%s\nX-Ops-Timestamp:%s\nX-Ops-UserId:%s\nX-Ops-Server-API-Version:%s", + req.Method, + path.Clean(req.URL.Path), + req.Header.Get("X-Ops-Content-Hash"), + version, + req.Header.Get("X-Ops-Timestamp"), + username, + req.Header.Get("X-Ops-Server-Api-Version"), + )) + } else { + sum := sha1.Sum([]byte(path.Clean(req.URL.Path))) + data = []byte(fmt.Sprintf( + "Method:%s\nHashed Path:%s\nX-Ops-Content-Hash:%s\nX-Ops-Timestamp:%s\nX-Ops-UserId:%s", + req.Method, + base64.StdEncoding.EncodeToString(sum[:]), + req.Header.Get("X-Ops-Content-Hash"), + req.Header.Get("X-Ops-Timestamp"), + username, + )) + } + + for k := range req.Header { + if strings.HasPrefix(k, "X-Ops-Authorization-") { + req.Header.Del(k) + } + } + + var signature []byte + if version == "1.3" || version == "1.2" { + var h hash.Hash + var ch crypto.Hash + if version == "1.3" { + h = sha256.New() + ch = crypto.SHA256 + } else { + h = sha1.New() + ch = crypto.SHA1 + } + h.Write(data) + + signature, _ = rsa.SignPKCS1v15(rand.Reader, privKey, ch, h.Sum(nil)) + } else { + c := new(big.Int).SetBytes(data) + m := new(big.Int).Exp(c, privKey.D, privKey.N) + + signature = m.Bytes() + } + + enc := base64.StdEncoding.EncodeToString(signature) + + const chunkSize = 60 + chunks := make([]string, 0, (len(enc)-1)/chunkSize+1) + currentLen := 0 + currentStart := 0 + for i := range enc { + if currentLen == chunkSize { + chunks = append(chunks, enc[currentStart:i]) + currentLen = 0 + currentStart = i + } + currentLen++ + } + chunks = append(chunks, enc[currentStart:]) + + for i, chunk := range chunks { + req.Header.Set(fmt.Sprintf("X-Ops-Authorization-%d", i+1), chunk) + } + } + + for _, v := range []string{"1.0", "1.1", "1.2", "1.3"} { + t.Run(v, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + signRequest(t, req, v) + u, err = auth.Verify(req, nil, nil, nil) + assert.NotNil(t, u) + assert.NoError(t, err) + }) + } + }) + }) + + packageName := "test" + packageVersion := "1.0.1" + packageDescription := "Test Description" + packageAuthor := "KN4CK3R" + + root := fmt.Sprintf("/api/packages/%s/chef/api/v1", user.Name) + + uploadPackage := func(t *testing.T, version string, expectedStatus int) { + var body bytes.Buffer + mpw := multipart.NewWriter(&body) + part, _ := mpw.CreateFormFile("tarball", fmt.Sprintf("%s.tar.gz", version)) + zw := gzip.NewWriter(part) + tw := tar.NewWriter(zw) + + content := `{"name":"` + packageName + `","version":"` + version + `","description":"` + packageDescription + `","maintainer":"` + packageAuthor + `"}` + + hdr := &tar.Header{ + Name: packageName + "/metadata.json", + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write([]byte(content)) + + tw.Close() + zw.Close() + mpw.Close() + + req := NewRequestWithBody(t, "POST", root+"/cookbooks", &body) + req.Header.Add("Content-Type", mpw.FormDataContentType()) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, expectedStatus) + } + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "POST", root+"/cookbooks", bytes.NewReader([]byte{})) + MakeRequest(t, req, http.StatusUnauthorized) + + uploadPackage(t, packageVersion, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeChef) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &chef_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s.tar.gz", packageVersion), pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + uploadPackage(t, packageVersion, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/cookbooks/%s/versions/%s/download", root, packageName, packageVersion)) + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Universe", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root+"/universe") + resp := MakeRequest(t, req, http.StatusOK) + + type VersionInfo struct { + LocationType string `json:"location_type"` + LocationPath string `json:"location_path"` + DownloadURL string `json:"download_url"` + Dependencies map[string]string `json:"dependencies"` + } + + var result map[string]map[string]*VersionInfo + DecodeJSON(t, resp, &result) + + assert.Len(t, result, 1) + assert.Contains(t, result, packageName) + + versions := result[packageName] + + assert.Len(t, versions, 1) + assert.Contains(t, versions, packageVersion) + + info := versions[packageVersion] + + assert.Equal(t, "opscode", info.LocationType) + assert.Equal(t, setting.AppURL+root[1:], info.LocationPath) + assert.Equal(t, fmt.Sprintf("%s%s/cookbooks/%s/versions/%s/download", setting.AppURL, root[1:], packageName, packageVersion), info.DownloadURL) + }) + + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + cases := []struct { + Query string + Start int + Items int + ExpectedTotal int + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 0, 10, 1, 1}, + {"gitea", 0, 10, 0, 0}, + {"test", 0, 10, 1, 1}, + {"test", 1, 10, 1, 0}, + } + + type Item struct { + CookbookName string `json:"cookbook_name"` + CookbookMaintainer string `json:"cookbook_maintainer"` + CookbookDescription string `json:"cookbook_description"` + Cookbook string `json:"cookbook"` + } + + type Result struct { + Start int `json:"start"` + Total int `json:"total"` + Items []*Item `json:"items"` + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/search?q=%s&start=%d&items=%d", root, c.Query, c.Start, c.Items)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result Result + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Items, c.ExpectedResults, "case %d: unexpected result count", i) + + if len(result.Items) == 1 { + item := result.Items[0] + assert.Equal(t, packageName, item.CookbookName) + assert.Equal(t, packageAuthor, item.CookbookMaintainer) + assert.Equal(t, packageDescription, item.CookbookDescription) + assert.Equal(t, fmt.Sprintf("%s%s/cookbooks/%s", setting.AppURL, root[1:], packageName), item.Cookbook) + } + } + }) + + t.Run("EnumeratePackages", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + cases := []struct { + Sort string + Start int + Items int + ExpectedTotal int + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 0, 10, 1, 1}, + {"RECENTLY_ADDED", 0, 10, 1, 1}, + {"RECENTLY_UPDATED", 0, 10, 1, 1}, + {"", 1, 10, 1, 0}, + } + + type Item struct { + CookbookName string `json:"cookbook_name"` + CookbookMaintainer string `json:"cookbook_maintainer"` + CookbookDescription string `json:"cookbook_description"` + Cookbook string `json:"cookbook"` + } + + type Result struct { + Start int `json:"start"` + Total int `json:"total"` + Items []*Item `json:"items"` + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/cookbooks?start=%d&items=%d&sort=%s", root, c.Start, c.Items, c.Sort)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result Result + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Items, c.ExpectedResults, "case %d: unexpected result count", i) + + if len(result.Items) == 1 { + item := result.Items[0] + assert.Equal(t, packageName, item.CookbookName) + assert.Equal(t, packageAuthor, item.CookbookMaintainer) + assert.Equal(t, packageDescription, item.CookbookDescription) + assert.Equal(t, fmt.Sprintf("%s%s/cookbooks/%s", setting.AppURL, root[1:], packageName), item.Cookbook) + } + } + }) + + t.Run("PackageMetadata", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/cookbooks/%s", root, packageName)) + resp := MakeRequest(t, req, http.StatusOK) + + type Result struct { + Name string `json:"name"` + Maintainer string `json:"maintainer"` + Description string `json:"description"` + Category string `json:"category"` + LatestVersion string `json:"latest_version"` + SourceURL string `json:"source_url"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Deprecated bool `json:"deprecated"` + Versions []string `json:"versions"` + } + + var result Result + DecodeJSON(t, resp, &result) + + versionURL := fmt.Sprintf("%s%s/cookbooks/%s/versions/%s", setting.AppURL, root[1:], packageName, packageVersion) + + assert.Equal(t, packageName, result.Name) + assert.Equal(t, packageAuthor, result.Maintainer) + assert.Equal(t, packageDescription, result.Description) + assert.Equal(t, versionURL, result.LatestVersion) + assert.False(t, result.Deprecated) + assert.ElementsMatch(t, []string{versionURL}, result.Versions) + }) + + t.Run("PackageVersionMetadata", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/cookbooks/%s/versions/%s", root, packageName, packageVersion)) + resp := MakeRequest(t, req, http.StatusOK) + + type Result struct { + Version string `json:"version"` + TarballFileSize int64 `json:"tarball_file_size"` + PublishedAt time.Time `json:"published_at"` + Cookbook string `json:"cookbook"` + File string `json:"file"` + License string `json:"license"` + Dependencies map[string]string `json:"dependencies"` + } + + var result Result + DecodeJSON(t, resp, &result) + + packageURL := fmt.Sprintf("%s%s/cookbooks/%s", setting.AppURL, root[1:], packageName) + + assert.Equal(t, packageVersion, result.Version) + assert.Equal(t, packageURL, result.Cookbook) + assert.Equal(t, fmt.Sprintf("%s/versions/%s/download", packageURL, packageVersion), result.File) + }) + + t.Run("Delete", func(t *testing.T) { + uploadPackage(t, "1.0.2", http.StatusCreated) + uploadPackage(t, "1.0.3", http.StatusCreated) + + t.Run("Version", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/cookbooks/%s/versions/%s", root, packageName, "1.0.2")) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/cookbooks/%s/versions/%s", root, packageName, "1.0.2")) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeChef, packageName, "1.0.2") + assert.Nil(t, pv) + assert.Error(t, err) + }) + + t.Run("Package", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/cookbooks/%s", root, packageName)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/cookbooks/%s", root, packageName)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeChef) + assert.NoError(t, err) + assert.Empty(t, pvs) + }) + }) +} diff --git a/web_src/svg/gitea-chef.svg b/web_src/svg/gitea-chef.svg new file mode 100644 index 0000000000..ce318c6263 --- /dev/null +++ b/web_src/svg/gitea-chef.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + From 50111c71c3d8feedb170a0122c97237f78d1751f Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 6 Feb 2023 10:23:17 +0800 Subject: [PATCH 09/33] Refactor legacy strange git operations (#22756) During the refactoring of the git module, I found there were some strange operations. This PR tries to fix 2 of them 1. The empty argument `--` in repo_attribute.go, which was introduced by #16773. It seems unnecessary because nothing else would be added later. 2. The complex git service logic in repo/http.go. * Before: the `hasAccess` only allow `service == "upload-pack" || service == "receive-pack"` * After: unrelated code is removed. No need to call ToTrustedCmdArgs anymore. Co-authored-by: Lunny Xiao --- modules/git/repo_attribute.go | 3 +- routers/web/repo/http.go | 76 +++++++++++++---------------------- 2 files changed, 28 insertions(+), 51 deletions(-) diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go index e7d5fb6806..2b34f117f7 100644 --- a/modules/git/repo_attribute.go +++ b/modules/git/repo_attribute.go @@ -135,8 +135,7 @@ func (c *CheckAttributeReader) Init(ctx context.Context) error { c.env = append(c.env, "GIT_FLUSH=1") - // The empty "--" comes from #16773 , and it seems unnecessary because nothing else would be added later. - c.cmd.AddDynamicArguments(c.Attributes...).AddArguments("--") + c.cmd.AddDynamicArguments(c.Attributes...) var err error diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 89c86e764e..e82b94b9e8 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -424,60 +424,40 @@ func (h *serviceHandler) sendFile(contentType, file string) { // one or more key=value pairs separated by colons var safeGitProtocolHeader = regexp.MustCompile(`^[0-9a-zA-Z]+=[0-9a-zA-Z]+(:[0-9a-zA-Z]+=[0-9a-zA-Z]+)*$`) -func getGitConfig(ctx gocontext.Context, option, dir string) string { - out, _, err := git.NewCommand(ctx, "config").AddDynamicArguments(option).RunStdString(&git.RunOpts{Dir: dir}) - if err != nil { - log.Error("%v - %s", err, out) +func prepareGitCmdWithAllowedService(service string, h *serviceHandler) (*git.Command, error) { + if service == "receive-pack" && h.cfg.ReceivePack { + return git.NewCommand(h.r.Context(), "receive-pack"), nil } - return out[0 : len(out)-1] + if service == "upload-pack" && h.cfg.UploadPack { + return git.NewCommand(h.r.Context(), "upload-pack"), nil + } + + return nil, fmt.Errorf("service %q is not allowed", service) } -func getConfigSetting(ctx gocontext.Context, service, dir string) bool { - service = strings.ReplaceAll(service, "-", "") - setting := getGitConfig(ctx, "http."+service, dir) - - if service == "uploadpack" { - return setting != "false" - } - - return setting == "true" -} - -func hasAccess(ctx gocontext.Context, service string, h serviceHandler, checkContentType bool) bool { - if checkContentType { - if h.r.Header.Get("Content-Type") != fmt.Sprintf("application/x-git-%s-request", service) { - return false - } - } - - if !(service == "upload-pack" || service == "receive-pack") { - return false - } - if service == "receive-pack" { - return h.cfg.ReceivePack - } - if service == "upload-pack" { - return h.cfg.UploadPack - } - - return getConfigSetting(ctx, service, h.dir) -} - -func serviceRPC(ctx gocontext.Context, h serviceHandler, service string) { +func serviceRPC(h *serviceHandler, service string) { defer func() { if err := h.r.Body.Close(); err != nil { log.Error("serviceRPC: Close: %v", err) } }() - if !hasAccess(ctx, service, h, true) { + expectedContentType := fmt.Sprintf("application/x-git-%s-request", service) + if h.r.Header.Get("Content-Type") != expectedContentType { + log.Error("Content-Type (%q) doesn't match expected: %q", h.r.Header.Get("Content-Type"), expectedContentType) + h.w.WriteHeader(http.StatusUnauthorized) + return + } + + cmd, err := prepareGitCmdWithAllowedService(service, h) + if err != nil { + log.Error("Failed to prepareGitCmdWithService: %v", err) h.w.WriteHeader(http.StatusUnauthorized) return } h.w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", service)) - var err error reqBody := h.r.Body // Handle GZIP. @@ -498,8 +478,7 @@ func serviceRPC(ctx gocontext.Context, h serviceHandler, service string) { } var stderr bytes.Buffer - // the service is generated by ourselves, so it's safe to trust it - cmd := git.NewCommand(h.r.Context(), git.ToTrustedCmdArgs([]string{service})...).AddArguments("--stateless-rpc").AddDynamicArguments(h.dir) + cmd.AddArguments("--stateless-rpc").AddDynamicArguments(h.dir) cmd.SetDescription(fmt.Sprintf("%s %s %s [repo_path: %s]", git.GitExecutable, service, "--stateless-rpc", h.dir)) if err := cmd.Run(&git.RunOpts{ Dir: h.dir, @@ -520,7 +499,7 @@ func serviceRPC(ctx gocontext.Context, h serviceHandler, service string) { func ServiceUploadPack(ctx *context.Context) { h := httpBase(ctx) if h != nil { - serviceRPC(ctx, *h, "upload-pack") + serviceRPC(h, "upload-pack") } } @@ -528,7 +507,7 @@ func ServiceUploadPack(ctx *context.Context) { func ServiceReceivePack(ctx *context.Context) { h := httpBase(ctx) if h != nil { - serviceRPC(ctx, *h, "receive-pack") + serviceRPC(h, "receive-pack") } } @@ -537,7 +516,7 @@ func getServiceType(r *http.Request) string { if !strings.HasPrefix(serviceType, "git-") { return "" } - return strings.Replace(serviceType, "git-", "", 1) + return strings.TrimPrefix(serviceType, "git-") } func updateServerInfo(ctx gocontext.Context, dir string) []byte { @@ -563,16 +542,15 @@ func GetInfoRefs(ctx *context.Context) { return } h.setHeaderNoCache() - if hasAccess(ctx, getServiceType(h.r), *h, false) { - service := getServiceType(h.r) - + service := getServiceType(h.r) + cmd, err := prepareGitCmdWithAllowedService(service, h) + if err == nil { if protocol := h.r.Header.Get("Git-Protocol"); protocol != "" && safeGitProtocolHeader.MatchString(protocol) { h.environ = append(h.environ, "GIT_PROTOCOL="+protocol) } h.environ = append(os.Environ(), h.environ...) - // the service is generated by ourselves, so we can trust it - refs, _, err := git.NewCommand(ctx, git.ToTrustedCmdArgs([]string{service})...).AddArguments("--stateless-rpc", "--advertise-refs", ".").RunStdBytes(&git.RunOpts{Env: h.environ, Dir: h.dir}) + refs, _, err := cmd.AddArguments("--stateless-rpc", "--advertise-refs", ".").RunStdBytes(&git.RunOpts{Env: h.environ, Dir: h.dir}) if err != nil { log.Error(fmt.Sprintf("%v - %s", err, string(refs))) } From c13eb8e6b326bf6008b29778f311d5e5f89d5541 Mon Sep 17 00:00:00 2001 From: Adi <39572144+Adito5393@users.noreply.github.com> Date: Mon, 6 Feb 2023 05:12:13 +0100 Subject: [PATCH 10/33] Add CLI option tenant ID for oauth2 source (#22769) Fixes #22713 --- cmd/admin.go | 11 +++++++++++ docs/content/doc/usage/command-line.en-us.md | 2 ++ 2 files changed, 13 insertions(+) diff --git a/cmd/admin.go b/cmd/admin.go index eafed24bdd..4e2dc2bf06 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -308,6 +308,11 @@ var ( Value: "false", Usage: "Use custom URLs for GitLab/GitHub OAuth endpoints", }, + cli.StringFlag{ + Name: "custom-tenant-id", + Value: "", + Usage: "Use custom Tenant ID for OAuth endpoints", + }, cli.StringFlag{ Name: "custom-auth-url", Value: "", @@ -829,6 +834,7 @@ func parseOAuth2Config(c *cli.Context) *oauth2.Source { AuthURL: c.String("custom-auth-url"), ProfileURL: c.String("custom-profile-url"), EmailURL: c.String("custom-email-url"), + Tenant: c.String("custom-tenant-id"), } } else { customURLMapping = nil @@ -938,6 +944,7 @@ func runUpdateOauth(c *cli.Context) error { customURLMapping.AuthURL = oAuth2Config.CustomURLMapping.AuthURL customURLMapping.ProfileURL = oAuth2Config.CustomURLMapping.ProfileURL customURLMapping.EmailURL = oAuth2Config.CustomURLMapping.EmailURL + customURLMapping.Tenant = oAuth2Config.CustomURLMapping.Tenant } if c.IsSet("use-custom-urls") && c.IsSet("custom-token-url") { customURLMapping.TokenURL = c.String("custom-token-url") @@ -955,6 +962,10 @@ func runUpdateOauth(c *cli.Context) error { customURLMapping.EmailURL = c.String("custom-email-url") } + if c.IsSet("use-custom-urls") && c.IsSet("custom-tenant-id") { + customURLMapping.Tenant = c.String("custom-tenant-id") + } + oAuth2Config.CustomURLMapping = customURLMapping source.Cfg = oAuth2Config diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index d9d397df31..f2e72d4fc0 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -124,6 +124,7 @@ Admin operations: - `--secret`: Client Secret. - `--auto-discover-url`: OpenID Connect Auto Discovery URL (only required when using OpenID Connect as provider). - `--use-custom-urls`: Use custom URLs for GitLab/GitHub OAuth endpoints. + - `--custom-tenant-id`: Use custom Tenant ID for OAuth endpoints. - `--custom-auth-url`: Use a custom Authorization URL (option for GitLab/GitHub). - `--custom-token-url`: Use a custom Token URL (option for GitLab/GitHub). - `--custom-profile-url`: Use a custom Profile URL (option for GitLab/GitHub). @@ -147,6 +148,7 @@ Admin operations: - `--secret`: Client Secret. - `--auto-discover-url`: OpenID Connect Auto Discovery URL (only required when using OpenID Connect as provider). - `--use-custom-urls`: Use custom URLs for GitLab/GitHub OAuth endpoints. + - `--custom-tenant-id`: Use custom Tenant ID for OAuth endpoints. - `--custom-auth-url`: Use a custom Authorization URL (option for GitLab/GitHub). - `--custom-token-url`: Use a custom Token URL (option for GitLab/GitHub). - `--custom-profile-url`: Use a custom Profile URL (option for GitLab/GitHub). From 361d8072747ae930695337b6217cb1b1773268ea Mon Sep 17 00:00:00 2001 From: John Olheiser Date: Mon, 6 Feb 2023 00:37:18 -0600 Subject: [PATCH 11/33] Update gogs upgrade information (#22777) It seems that migrating from Gogs `0.12.x` and above may require more work as time goes on and the projects continue to diverge. This PR updates the docs to make it more clear. Signed-off-by: jolheiser --- docs/content/doc/upgrade/from-gogs.en-us.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/content/doc/upgrade/from-gogs.en-us.md b/docs/content/doc/upgrade/from-gogs.en-us.md index a10fffdcd6..f0bfe96050 100644 --- a/docs/content/doc/upgrade/from-gogs.en-us.md +++ b/docs/content/doc/upgrade/from-gogs.en-us.md @@ -85,8 +85,10 @@ Then repeat the procedure, but this time using the [latest release](https://dl.g ## Upgrading from a more recent version of Gogs -Upgrading from a more recent version of Gogs is also possible, but requires a bit more work. -See [#4286](https://github.com/go-gitea/gitea/issues/4286). +Upgrading from a more recent version of Gogs (up to `0.11.x`) may also be possible, but will require a bit more work. +See [#4286](https://github.com/go-gitea/gitea/issues/4286), which includes various Gogs `0.11.x` versions. + +Upgrading from Gogs `0.12.x` and above will be increasingly more difficult as the projects diverge further apart in configuration and schema. ## Troubleshooting From f8c1e14a136c9702f459a36a1ea2e75fedd123af Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 6 Feb 2023 11:07:09 +0100 Subject: [PATCH 12/33] Use import of OCI structs (#22765) Fixes #22758 Otherwise we would need to rewrite the structs in `oci.go`. --- assets/go-licenses.json | 10 + go.mod | 2 + go.sum | 4 + modules/packages/container/metadata.go | 7 +- modules/packages/container/metadata_test.go | 6 +- modules/packages/container/oci/digest.go | 26 --- modules/packages/container/oci/mediatype.go | 35 ---- modules/packages/container/oci/oci.go | 190 ------------------ modules/packages/container/oci/reference.go | 16 -- routers/api/packages/container/container.go | 56 +++--- routers/api/packages/container/manifest.go | 44 ++-- services/packages/container/cleanup.go | 5 +- .../api_packages_container_test.go | 16 +- 13 files changed, 93 insertions(+), 324 deletions(-) delete mode 100644 modules/packages/container/oci/digest.go delete mode 100644 modules/packages/container/oci/mediatype.go delete mode 100644 modules/packages/container/oci/oci.go delete mode 100644 modules/packages/container/oci/reference.go diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 8789bd3f06..e78e7968ac 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -799,6 +799,16 @@ "path": "github.com/olivere/elastic/v7/uritemplates/LICENSE", "licenseText": "Copyright (c) 2013 Joshua Tacoma\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/opencontainers/go-digest", + "path": "github.com/opencontainers/go-digest/LICENSE", + "licenseText": "\n Apache License\n Version 2.0, January 2004\n https://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n Copyright 2019, 2020 OCI Contributors\n Copyright 2016 Docker, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/opencontainers/image-spec/specs-go", + "path": "github.com/opencontainers/image-spec/specs-go/LICENSE", + "licenseText": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n Copyright 2016 The Linux Foundation.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "github.com/pierrec/lz4/v4", "path": "github.com/pierrec/lz4/v4/LICENSE", diff --git a/go.mod b/go.mod index eb23bd9e32..3dc050b99a 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,8 @@ require ( github.com/niklasfasching/go-org v1.6.5 github.com/oliamb/cutter v0.2.2 github.com/olivere/elastic/v7 v7.0.32 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.14.0 diff --git a/go.sum b/go.sum index 864ddff1cd..03c2cc18c6 100644 --- a/go.sum +++ b/go.sum @@ -1009,6 +1009,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index c3946f38f1..6f62ab6a54 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -10,8 +10,9 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/container/helm" - "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/validation" + + oci "github.com/opencontainers/image-spec/specs-go/v1" ) const ( @@ -65,8 +66,8 @@ type Metadata struct { } // ParseImageConfig parses the metadata of an image config -func ParseImageConfig(mediaType oci.MediaType, r io.Reader) (*Metadata, error) { - if strings.EqualFold(string(mediaType), helm.ConfigMediaType) { +func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) { + if strings.EqualFold(mt, helm.ConfigMediaType) { return parseHelmConfig(r) } diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index f9ee478d63..5d8d3abfae 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -8,8 +8,8 @@ import ( "testing" "code.gitea.io/gitea/modules/packages/container/helm" - "code.gitea.io/gitea/modules/packages/container/oci" + oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" ) @@ -23,7 +23,7 @@ func TestParseImageConfig(t *testing.T) { configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` - metadata, err := ParseImageConfig(oci.MediaType(oci.MediaTypeImageManifest), strings.NewReader(configOCI)) + metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader(configOCI)) assert.NoError(t, err) assert.Equal(t, TypeOCI, metadata.Type) @@ -50,7 +50,7 @@ func TestParseImageConfig(t *testing.T) { configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}` - metadata, err = ParseImageConfig(oci.MediaType(helm.ConfigMediaType), strings.NewReader(configHelm)) + metadata, err = ParseImageConfig(helm.ConfigMediaType, strings.NewReader(configHelm)) assert.NoError(t, err) assert.Equal(t, TypeHelm, metadata.Type) diff --git a/modules/packages/container/oci/digest.go b/modules/packages/container/oci/digest.go deleted file mode 100644 index dd9cc0095c..0000000000 --- a/modules/packages/container/oci/digest.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package oci - -import ( - "regexp" - "strings" -) - -var digestPattern = regexp.MustCompile(`\Asha256:[a-f0-9]{64}\z`) - -type Digest string - -// Validate checks if the digest has a valid SHA256 signature -func (d Digest) Validate() bool { - return digestPattern.MatchString(string(d)) -} - -func (d Digest) Hash() string { - p := strings.SplitN(string(d), ":", 2) - if len(p) != 2 { - return "" - } - return p[1] -} diff --git a/modules/packages/container/oci/mediatype.go b/modules/packages/container/oci/mediatype.go deleted file mode 100644 index f9c3907e17..0000000000 --- a/modules/packages/container/oci/mediatype.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package oci - -import ( - "strings" -) - -const ( - MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json" - MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json" - MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" - MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" -) - -type MediaType string - -// IsValid tests if the media type is in the OCI or Docker namespace -func (m MediaType) IsValid() bool { - s := string(m) - return strings.HasPrefix(s, "application/vnd.docker.") || strings.HasPrefix(s, "application/vnd.oci.") -} - -// IsImageManifest tests if the media type is an image manifest -func (m MediaType) IsImageManifest() bool { - s := string(m) - return strings.EqualFold(s, MediaTypeDockerManifest) || strings.EqualFold(s, MediaTypeImageManifest) -} - -// IsImageIndex tests if the media type is an image index -func (m MediaType) IsImageIndex() bool { - s := string(m) - return strings.EqualFold(s, MediaTypeDockerManifestList) || strings.EqualFold(s, MediaTypeImageIndex) -} diff --git a/modules/packages/container/oci/oci.go b/modules/packages/container/oci/oci.go deleted file mode 100644 index 570d2e92c2..0000000000 --- a/modules/packages/container/oci/oci.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package oci - -import ( - "time" -) - -// https://github.com/opencontainers/image-spec/tree/main/specs-go/v1 - -// ImageConfig defines the execution parameters which should be used as a base when running a container using an image. -type ImageConfig struct { - // User defines the username or UID which the process in the container should run as. - User string `json:"User,omitempty"` - - // ExposedPorts a set of ports to expose from a container running this image. - ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"` - - // Env is a list of environment variables to be used in a container. - Env []string `json:"Env,omitempty"` - - // Entrypoint defines a list of arguments to use as the command to execute when the container starts. - Entrypoint []string `json:"Entrypoint,omitempty"` - - // Cmd defines the default arguments to the entrypoint of the container. - Cmd []string `json:"Cmd,omitempty"` - - // Volumes is a set of directories describing where the process is likely write data specific to a container instance. - Volumes map[string]struct{} `json:"Volumes,omitempty"` - - // WorkingDir sets the current working directory of the entrypoint process in the container. - WorkingDir string `json:"WorkingDir,omitempty"` - - // Labels contains arbitrary metadata for the container. - Labels map[string]string `json:"Labels,omitempty"` - - // StopSignal contains the system call signal that will be sent to the container to exit. - StopSignal string `json:"StopSignal,omitempty"` -} - -// RootFS describes a layer content addresses -type RootFS struct { - // Type is the type of the rootfs. - Type string `json:"type"` - - // DiffIDs is an array of layer content hashes, in order from bottom-most to top-most. - DiffIDs []string `json:"diff_ids"` -} - -// History describes the history of a layer. -type History struct { - // Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6. - Created *time.Time `json:"created,omitempty"` - - // CreatedBy is the command which created the layer. - CreatedBy string `json:"created_by,omitempty"` - - // Author is the author of the build point. - Author string `json:"author,omitempty"` - - // Comment is a custom message set when creating the layer. - Comment string `json:"comment,omitempty"` - - // EmptyLayer is used to mark if the history item created a filesystem diff. - EmptyLayer bool `json:"empty_layer,omitempty"` -} - -// Image is the JSON structure which describes some basic information about the image. -// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON. -type Image struct { - // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6. - Created *time.Time `json:"created,omitempty"` - - // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. - Author string `json:"author,omitempty"` - - // Architecture is the CPU architecture which the binaries in this image are built to run on. - Architecture string `json:"architecture"` - - // Variant is the variant of the specified CPU architecture which image binaries are intended to run on. - Variant string `json:"variant,omitempty"` - - // OS is the name of the operating system which the image is built to run on. - OS string `json:"os"` - - // OSVersion is an optional field specifying the operating system - // version, for example on Windows `10.0.14393.1066`. - OSVersion string `json:"os.version,omitempty"` - - // OSFeatures is an optional field specifying an array of strings, - // each listing a required OS feature (for example on Windows `win32k`). - OSFeatures []string `json:"os.features,omitempty"` - - // Config defines the execution parameters which should be used as a base when running a container using the image. - Config ImageConfig `json:"config,omitempty"` - - // RootFS references the layer content addresses used by the image. - RootFS RootFS `json:"rootfs"` - - // History describes the history of each layer. - History []History `json:"history,omitempty"` -} - -// Descriptor describes the disposition of targeted content. -// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype -// when marshalled to JSON. -type Descriptor struct { - // MediaType is the media type of the object this schema refers to. - MediaType MediaType `json:"mediaType,omitempty"` - - // Digest is the digest of the targeted content. - Digest Digest `json:"digest"` - - // Size specifies the size in bytes of the blob. - Size int64 `json:"size"` - - // URLs specifies a list of URLs from which this object MAY be downloaded - URLs []string `json:"urls,omitempty"` - - // Annotations contains arbitrary metadata relating to the targeted content. - Annotations map[string]string `json:"annotations,omitempty"` - - // Data is an embedding of the targeted content. This is encoded as a base64 - // string when marshalled to JSON (automatically, by encoding/json). If - // present, Data can be used directly to avoid fetching the targeted content. - Data []byte `json:"data,omitempty"` - - // Platform describes the platform which the image in the manifest runs on. - // - // This should only be used when referring to a manifest. - Platform *Platform `json:"platform,omitempty"` -} - -// Platform describes the platform which the image in the manifest runs on. -type Platform struct { - // Architecture field specifies the CPU architecture, for example - // `amd64` or `ppc64`. - Architecture string `json:"architecture"` - - // OS specifies the operating system, for example `linux` or `windows`. - OS string `json:"os"` - - // OSVersion is an optional field specifying the operating system - // version, for example on Windows `10.0.14393.1066`. - OSVersion string `json:"os.version,omitempty"` - - // OSFeatures is an optional field specifying an array of strings, - // each listing a required OS feature (for example on Windows `win32k`). - OSFeatures []string `json:"os.features,omitempty"` - - // Variant is an optional field specifying a variant of the CPU, for - // example `v7` to specify ARMv7 when architecture is `arm`. - Variant string `json:"variant,omitempty"` -} - -type SchemaMediaBase struct { - // SchemaVersion is the image manifest schema that this image follows - SchemaVersion int `json:"schemaVersion"` - - // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json` - MediaType MediaType `json:"mediaType,omitempty"` -} - -// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON. -type Manifest struct { - SchemaMediaBase - - // Config references a configuration object for a container, by digest. - // The referenced configuration object is a JSON blob that the runtime uses to set up the container. - Config Descriptor `json:"config"` - - // Layers is an indexed list of layers referenced by the manifest. - Layers []Descriptor `json:"layers"` - - // Annotations contains arbitrary metadata for the image manifest. - Annotations map[string]string `json:"annotations,omitempty"` -} - -// Index references manifests for various platforms. -// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON. -type Index struct { - SchemaMediaBase - - // Manifests references platform specific manifests. - Manifests []Descriptor `json:"manifests"` - - // Annotations contains arbitrary metadata for the image index. - Annotations map[string]string `json:"annotations,omitempty"` -} diff --git a/modules/packages/container/oci/reference.go b/modules/packages/container/oci/reference.go deleted file mode 100644 index 7ec399255d..0000000000 --- a/modules/packages/container/oci/reference.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package oci - -import ( - "regexp" -) - -var referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) - -type Reference string - -func (r Reference) Validate() bool { - return referencePattern.MatchString(string(r)) -} diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index c22cfb5009..883fe73cbd 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -22,19 +22,23 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" - "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" + + digest "github.com/opencontainers/go-digest" ) // maximum size of a container manifest // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests const maxManifestSize = 10 * 1024 * 1024 -var imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) +var ( + imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) + referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) +) type containerHeaders struct { Status int @@ -434,16 +438,16 @@ func CancelUploadBlob(ctx *context.Context) { } func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { - digest := ctx.Params("digest") + d := ctx.Params("digest") - if !oci.Digest(digest).Validate() { + if digest.Digest(d).Validate() != nil { return nil, container_model.ErrContainerBlobNotExist } return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.Params("image"), - Digest: digest, + Digest: d, }) } @@ -498,14 +502,14 @@ func GetBlob(ctx *context.Context) { // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs func DeleteBlob(ctx *context.Context) { - digest := ctx.Params("digest") + d := ctx.Params("digest") - if !oci.Digest(digest).Validate() { + if digest.Digest(d).Validate() != nil { apiErrorDefined(ctx, errBlobUnknown) return } - if err := deleteBlob(ctx.Package.Owner.ID, ctx.Params("image"), digest); err != nil { + if err := deleteBlob(ctx.Package.Owner.ID, ctx.Params("image"), d); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -520,15 +524,15 @@ func UploadManifest(ctx *context.Context) { reference := ctx.Params("reference") mci := &manifestCreationInfo{ - MediaType: oci.MediaType(ctx.Req.Header.Get("Content-Type")), + MediaType: ctx.Req.Header.Get("Content-Type"), Owner: ctx.Package.Owner, Creator: ctx.Doer, Image: ctx.Params("image"), Reference: reference, - IsTagged: !oci.Digest(reference).Validate(), + IsTagged: digest.Digest(reference).Validate() != nil, } - if mci.IsTagged && !oci.Reference(reference).Validate() { + if mci.IsTagged && !referencePattern.MatchString(reference) { apiErrorDefined(ctx, errManifestInvalid.WithMessage("Tag is invalid")) return } @@ -571,7 +575,7 @@ func UploadManifest(ctx *context.Context) { }) } -func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { +func getBlobSearchOptionsFromContext(ctx *context.Context) (*container_model.BlobSearchOptions, error) { reference := ctx.Params("reference") opts := &container_model.BlobSearchOptions{ @@ -579,14 +583,24 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe Image: ctx.Params("image"), IsManifest: true, } - if oci.Digest(reference).Validate() { + + if digest.Digest(reference).Validate() == nil { opts.Digest = reference - } else if oci.Reference(reference).Validate() { + } else if referencePattern.MatchString(reference) { opts.Tag = reference } else { return nil, container_model.ErrContainerBlobNotExist } + return opts, nil +} + +func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { + opts, err := getBlobSearchOptionsFromContext(ctx) + if err != nil { + return nil, err + } + return workaroundGetContainerBlob(ctx, opts) } @@ -643,18 +657,8 @@ func GetManifest(ctx *context.Context) { // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-tags // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-manifests func DeleteManifest(ctx *context.Context) { - reference := ctx.Params("reference") - - opts := &container_model.BlobSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Image: ctx.Params("image"), - IsManifest: true, - } - if oci.Digest(reference).Validate() { - opts.Digest = reference - } else if oci.Reference(reference).Validate() { - opts.Tag = reference - } else { + opts, err := getBlobSearchOptionsFromContext(ctx) + if err != nil { apiErrorDefined(ctx, errManifestUnknown) return } diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index 491fb70639..6167d00f31 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -19,14 +19,28 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" - "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" + + digest "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" ) +func isValidMediaType(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.") +} + +func isImageManifestMediaType(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json") +} + +func isImageIndexMediaType(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json") +} + // manifestCreationInfo describes a manifest to create type manifestCreationInfo struct { - MediaType oci.MediaType + MediaType string Owner *user_model.User Creator *user_model.User Image string @@ -36,12 +50,12 @@ type manifestCreationInfo struct { } func processManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { - var schema oci.SchemaMediaBase - if err := json.NewDecoder(buf).Decode(&schema); err != nil { + var index oci.Index + if err := json.NewDecoder(buf).Decode(&index); err != nil { return "", err } - if schema.SchemaVersion != 2 { + if index.SchemaVersion != 2 { return "", errUnsupported.WithMessage("Schema version is not supported") } @@ -49,17 +63,17 @@ func processManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffe return "", err } - if !mci.MediaType.IsValid() { - mci.MediaType = schema.MediaType - if !mci.MediaType.IsValid() { + if !isValidMediaType(mci.MediaType) { + mci.MediaType = index.MediaType + if !isValidMediaType(mci.MediaType) { return "", errManifestInvalid.WithMessage("MediaType not recognized") } } - if mci.MediaType.IsImageManifest() { + if isImageManifestMediaType(mci.MediaType) { d, err := processImageManifest(mci, buf) return d, err - } else if mci.MediaType.IsImageIndex() { + } else if isImageIndexMediaType(mci.MediaType) { d, err := processImageManifestIndex(mci, buf) return d, err } @@ -204,7 +218,7 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H } for _, manifest := range index.Manifests { - if !manifest.MediaType.IsImageManifest() { + if !isImageManifestMediaType(manifest.MediaType) { return errManifestInvalid } @@ -348,8 +362,8 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met } type blobReference struct { - Digest oci.Digest - MediaType oci.MediaType + Digest digest.Digest + MediaType string Name string File *packages_model.PackageFileDescriptor ExpectedSize int64 @@ -383,7 +397,7 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package } props := map[string]string{ - container_module.PropertyMediaType: string(ref.MediaType), + container_module.PropertyMediaType: ref.MediaType, container_module.PropertyDigest: string(ref.Digest), } for name, value := range props { @@ -428,7 +442,7 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack manifestDigest := digestFromHashSummer(buf) err = createFileFromBlobReference(ctx, pv, nil, &blobReference{ - Digest: oci.Digest(manifestDigest), + Digest: digest.Digest(manifestDigest), MediaType: mci.MediaType, Name: container_model.ManifestFilename, File: &packages_model.PackageFileDescriptor{Blob: pb}, diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index d6d4d152c8..1a9ef26391 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -10,8 +10,9 @@ import ( packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" container_module "code.gitea.io/gitea/modules/packages/container" - "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/util" + + digest "github.com/opencontainers/go-digest" ) // Cleanup removes expired container data @@ -87,7 +88,7 @@ func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule } // Check if the version is a digest (or untagged) - if oci.Digest(pv.LowerVersion).Validate() { + if digest.Digest(pv.LowerVersion).Validate() == nil { // Check if there is another manifest referencing this version has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, diff --git a/tests/integration/api_packages_container_test.go b/tests/integration/api_packages_container_test.go index 39297c7d94..3d9319f370 100644 --- a/tests/integration/api_packages_container_test.go +++ b/tests/integration/api_packages_container_test.go @@ -20,11 +20,11 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" container_module "code.gitea.io/gitea/modules/packages/container" - "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" + oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" ) @@ -66,13 +66,13 @@ func TestPackageContainer(t *testing.T) { configContent := `{"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/true"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"container":"b89fe92a887d55c0961f02bdfbfd8ac3ddf66167db374770d2d9e9fab3311510","container_config":{"Hostname":"b89fe92a887d","Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/true\"]"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"created":"2022-01-01T00:00:00.000000000Z","docker_version":"20.10.12","history":[{"created":"2022-01-01T00:00:00.000000000Z","created_by":"/bin/sh -c #(nop) COPY file:0e7589b0c800daaf6fa460d2677101e4676dd9491980210cb345480e513f3602 in /true "},{"created":"2022-01-01T00:00:00.000000001Z","created_by":"/bin/sh -c #(nop) CMD [\"/true\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:0ff3b91bdf21ecdf2f2f3d4372c2098a14dbe06cd678e8f0a85fd4902d00e2e2"]}}` manifestDigest := "sha256:4f10484d1c1bb13e3956b4de1cd42db8e0f14a75be1617b60f2de3cd59c803c6" - manifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeDockerManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}` + manifestContent := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}` untaggedManifestDigest := "sha256:4305f5f5572b9a426b88909b036e52ee3cf3d7b9c1b01fac840e90747f56623d" untaggedManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}` indexManifestDigest := "sha256:bab112d6efb9e7f221995caaaa880352feb5bd8b1faf52fae8d12c113aa123ec" - indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","manifests":[{"mediaType":"` + oci.MediaTypeDockerManifest + `","digest":"` + manifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}},{"mediaType":"` + oci.MediaTypeImageManifest + `","digest":"` + untaggedManifestDigest + `","platform":{"os":"linux","architecture":"arm64","variant":"v8"}}]}` + indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"` + manifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}},{"mediaType":"` + oci.MediaTypeImageManifest + `","digest":"` + untaggedManifestDigest + `","platform":{"os":"linux","architecture":"arm64","variant":"v8"}}]}` anonymousToken := "" userToken := "" @@ -296,12 +296,12 @@ func TestPackageContainer(t *testing.T) { req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) addTokenAuthHeader(req, anonymousToken) - req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + req.Header.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") MakeRequest(t, req, http.StatusUnauthorized) req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) addTokenAuthHeader(req, userToken) - req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + req.Header.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") resp := MakeRequest(t, req, http.StatusCreated) assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest")) @@ -328,7 +328,7 @@ func TestPackageContainer(t *testing.T) { switch pfd.File.Name { case container_model.ManifestFilename: assert.True(t, pfd.File.IsLead) - assert.Equal(t, oci.MediaTypeDockerManifest, pfd.Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, "application/vnd.docker.distribution.manifest.v2+json", pfd.Properties.GetByName(container_module.PropertyMediaType)) assert.Equal(t, manifestDigest, pfd.Properties.GetByName(container_module.PropertyDigest)) case strings.Replace(configDigest, ":", "_", 1): assert.False(t, pfd.File.IsLead) @@ -354,7 +354,7 @@ func TestPackageContainer(t *testing.T) { // Overwrite existing tag should keep the download count req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) addTokenAuthHeader(req, userToken) - req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + req.Header.Set("Content-Type", oci.MediaTypeImageManifest) MakeRequest(t, req, http.StatusCreated) pv, err = packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, image, tag) @@ -389,7 +389,7 @@ func TestPackageContainer(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, fmt.Sprintf("%d", len(manifestContent)), resp.Header().Get("Content-Length")) - assert.Equal(t, oci.MediaTypeDockerManifest, resp.Header().Get("Content-Type")) + assert.Equal(t, oci.MediaTypeImageManifest, resp.Header().Get("Content-Type")) assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest")) assert.Equal(t, manifestContent, resp.Body.String()) }) From 189d5b7045b2200fa55c20707a39138cfb8cf0ed Mon Sep 17 00:00:00 2001 From: John Olheiser Date: Mon, 6 Feb 2023 10:20:20 -0600 Subject: [PATCH 13/33] Add repo adoption to FAQ (#22778) This should be a simple set of steps to achieve repo adoption. --- docs/content/doc/help/faq.en-us.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/content/doc/help/faq.en-us.md b/docs/content/doc/help/faq.en-us.md index 2fcf0be156..c8327de7fe 100644 --- a/docs/content/doc/help/faq.en-us.md +++ b/docs/content/doc/help/faq.en-us.md @@ -449,3 +449,14 @@ It is highly recommended to back-up your database before running these commands. If you are using Cloudflare, turn off the auto-minify option in the dashboard. `Speed` -> `Optimization` -> Uncheck `HTML` within the `Auto-Minify` settings. + +## How to adopt repositories from disk + +- Add your (bare) repositories to the correct spot for your configuration (`repository.ROOT`), ensuring they are in the correct layout `/[user]/[repo].git`. + - **Note:** the directory names must be lowercase. + - You can also check `/admin/config` for the repository root path. +- Ensure that the user/org exists that you want to adopt repositories for. +- As an admin, go to `/admin/repos/unadopted` and search. + - Users can also be given similar permissions via config [`ALLOW_ADOPTION_OF_UNADOPTED_REPOSITORIES`]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#repository" >}}). +- If the above steps are done correctly, you should be able to select repositories to adopt. + - If no repositories are found, enable [debug logging]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#repository" >}}) to check for any specific errors. From 769be877f2ce572b94b7c0a85853a2f7836075ac Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 7 Feb 2023 02:09:18 +0800 Subject: [PATCH 14/33] Use link in UI which returned a relative url but not html_url which contains an absolute url (#21986) partially fix #19345 This PR add some `Link` methods for different objects. The `Link` methods are not different from `HTMLURL`, they are lack of the absolute URL. And most of UI `HTMLURL` have been replaced to `Link` so that users can visit them from a different domain or IP. This PR also introduces a new javascript configuration `window.config.reqAppUrl` which is different from `appUrl` which is still an absolute url but the domain has been replaced to the current requested domain. --- models/activities/action.go | 55 ++++++++++++++++--- models/activities/action_test.go | 2 +- models/activities/notification.go | 16 ++++++ models/issues/comment.go | 33 ++++++++--- models/issues/issue.go | 2 +- models/issues/pull.go | 12 ++-- models/packages/descriptor.go | 2 +- models/project/project.go | 1 + models/repo/release.go | 5 ++ models/repo/repo.go | 2 +- modules/structs/repo.go | 1 + routers/web/feed/convert.go | 2 +- routers/web/repo/pull.go | 4 +- routers/web/repo/repo.go | 1 + templates/code/searchresults.tmpl | 6 +- templates/mail/issue/assigned.tmpl | 2 +- templates/mail/issue/default.tmpl | 6 +- templates/mail/release.tmpl | 4 +- templates/package/shared/list.tmpl | 4 +- templates/package/view.tmpl | 6 +- templates/repo/branch/list.tmpl | 8 +-- templates/repo/diff/box.tmpl | 4 +- templates/repo/diff/comment_form.tmpl | 4 +- templates/repo/editor/edit.tmpl | 4 +- templates/repo/header.tmpl | 2 +- templates/repo/issue/comment_tab.tmpl | 4 +- templates/repo/issue/view_content.tmpl | 2 +- .../repo/issue/view_content/comments.tmpl | 8 +-- .../repo/issue/view_content/context_menu.tmpl | 6 +- templates/repo/issue/view_content/pull.tmpl | 2 +- templates/repo/issue/view_title.tmpl | 8 +-- templates/repo/projects/view.tmpl | 2 +- templates/repo/release/new.tmpl | 2 +- templates/repo/view_file.tmpl | 5 +- templates/repo/wiki/new.tmpl | 4 +- templates/shared/issuelist.tmpl | 10 ++-- templates/swagger/v1_json.tmpl | 4 ++ templates/user/dashboard/repolist.tmpl | 2 +- .../user/notification/notification_div.tmpl | 6 +- web_src/js/features/clipboard.js | 7 ++- web_src/js/test/setup.js | 1 + web_src/js/utils.js | 7 +++ web_src/js/utils.test.js | 6 ++ 43 files changed, 191 insertions(+), 83 deletions(-) diff --git a/models/activities/action.go b/models/activities/action.go index 4baedbfe12..8e7492c008 100644 --- a/models/activities/action.go +++ b/models/activities/action.go @@ -223,18 +223,24 @@ func (a *Action) GetRepoAbsoluteLink() string { return setting.AppURL + url.PathEscape(a.GetRepoUserName()) + "/" + url.PathEscape(a.GetRepoName()) } -// GetCommentLink returns link to action comment. -func (a *Action) GetCommentLink() string { - return a.getCommentLink(db.DefaultContext) +// GetCommentHTMLURL returns link to action comment. +func (a *Action) GetCommentHTMLURL() string { + return a.getCommentHTMLURL(db.DefaultContext) } -func (a *Action) getCommentLink(ctx context.Context) string { +func (a *Action) loadComment(ctx context.Context) (err error) { + if a.CommentID == 0 || a.Comment != nil { + return nil + } + a.Comment, err = issues_model.GetCommentByID(ctx, a.CommentID) + return err +} + +func (a *Action) getCommentHTMLURL(ctx context.Context) string { if a == nil { return "#" } - if a.Comment == nil && a.CommentID != 0 { - a.Comment, _ = issues_model.GetCommentByID(ctx, a.CommentID) - } + _ = a.loadComment(ctx) if a.Comment != nil { return a.Comment.HTMLURL() } @@ -260,6 +266,41 @@ func (a *Action) getCommentLink(ctx context.Context) string { return issue.HTMLURL() } +// GetCommentLink returns link to action comment. +func (a *Action) GetCommentLink() string { + return a.getCommentLink(db.DefaultContext) +} + +func (a *Action) getCommentLink(ctx context.Context) string { + if a == nil { + return "#" + } + _ = a.loadComment(ctx) + if a.Comment != nil { + return a.Comment.Link() + } + if len(a.GetIssueInfos()) == 0 { + return "#" + } + // Return link to issue + issueIDString := a.GetIssueInfos()[0] + issueID, err := strconv.ParseInt(issueIDString, 10, 64) + if err != nil { + return "#" + } + + issue, err := issues_model.GetIssueByID(ctx, issueID) + if err != nil { + return "#" + } + + if err = issue.LoadRepo(ctx); err != nil { + return "#" + } + + return issue.Link() +} + // GetBranch returns the action's repository branch. func (a *Action) GetBranch() string { return strings.TrimPrefix(a.RefName, git.BranchPrefix) diff --git a/models/activities/action_test.go b/models/activities/action_test.go index 29312bd482..f37e58f685 100644 --- a/models/activities/action_test.go +++ b/models/activities/action_test.go @@ -36,7 +36,7 @@ func TestAction_GetRepoLink(t *testing.T) { expected := path.Join(setting.AppSubURL, owner.Name, repo.Name) assert.Equal(t, expected, action.GetRepoLink()) assert.Equal(t, repo.HTMLURL(), action.GetRepoAbsoluteLink()) - assert.Equal(t, comment.HTMLURL(), action.GetCommentLink()) + assert.Equal(t, comment.HTMLURL(), action.GetCommentHTMLURL()) } func TestGetFeeds(t *testing.T) { diff --git a/models/activities/notification.go b/models/activities/notification.go index f153eb0589..75276a0443 100644 --- a/models/activities/notification.go +++ b/models/activities/notification.go @@ -459,6 +459,22 @@ func (n *Notification) HTMLURL() string { return "" } +// Link formats a relative URL-string to the notification +func (n *Notification) Link() string { + switch n.Source { + case NotificationSourceIssue, NotificationSourcePullRequest: + if n.Comment != nil { + return n.Comment.Link() + } + return n.Issue.Link() + case NotificationSourceCommit: + return n.Repository.Link() + "/commit/" + url.PathEscape(n.CommitID) + case NotificationSourceRepository: + return n.Repository.Link() + } + return "" +} + // APIURL formats a URL-string to the notification func (n *Notification) APIURL() string { return setting.AppURL + "api/v1/notifications/threads/" + strconv.FormatInt(n.ID, 10) diff --git a/models/issues/comment.go b/models/issues/comment.go index 9ad538fcc6..c935e4ac91 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -391,21 +391,40 @@ func (c *Comment) HTMLURL() string { log.Error("loadRepo(%d): %v", c.Issue.RepoID, err) return "" } + return c.Issue.HTMLURL() + c.hashLink() +} + +// Link formats a relative URL-string to the issue-comment +func (c *Comment) Link() string { + err := c.LoadIssue(db.DefaultContext) + if err != nil { // Silently dropping errors :unamused: + log.Error("LoadIssue(%d): %v", c.IssueID, err) + return "" + } + err = c.Issue.LoadRepo(db.DefaultContext) + if err != nil { // Silently dropping errors :unamused: + log.Error("loadRepo(%d): %v", c.Issue.RepoID, err) + return "" + } + return c.Issue.Link() + c.hashLink() +} + +func (c *Comment) hashLink() string { if c.Type == CommentTypeCode { if c.ReviewID == 0 { - return fmt.Sprintf("%s/files#%s", c.Issue.HTMLURL(), c.HashTag()) + return "/files#" + c.HashTag() } if c.Review == nil { if err := c.LoadReview(); err != nil { log.Warn("LoadReview(%d): %v", c.ReviewID, err) - return fmt.Sprintf("%s/files#%s", c.Issue.HTMLURL(), c.HashTag()) + return "/files#" + c.HashTag() } } if c.Review.Type <= ReviewTypePending { - return fmt.Sprintf("%s/files#%s", c.Issue.HTMLURL(), c.HashTag()) + return "/files#" + c.HashTag() } } - return fmt.Sprintf("%s#%s", c.Issue.HTMLURL(), c.HashTag()) + return "#" + c.HashTag() } // APIURL formats a API-string to the issue-comment @@ -708,8 +727,8 @@ func (c *Comment) UnsignedLine() uint64 { return uint64(c.Line) } -// CodeCommentURL returns the url to a comment in code -func (c *Comment) CodeCommentURL() string { +// CodeCommentLink returns the url to a comment in code +func (c *Comment) CodeCommentLink() string { err := c.LoadIssue(db.DefaultContext) if err != nil { // Silently dropping errors :unamused: log.Error("LoadIssue(%d): %v", c.IssueID, err) @@ -720,7 +739,7 @@ func (c *Comment) CodeCommentURL() string { log.Error("loadRepo(%d): %v", c.Issue.RepoID, err) return "" } - return fmt.Sprintf("%s/files#%s", c.Issue.HTMLURL(), c.HashTag()) + return fmt.Sprintf("%s/files#%s", c.Issue.Link(), c.HashTag()) } // LoadPushCommits Load push commits diff --git a/models/issues/issue.go b/models/issues/issue.go index 3ddc799270..62b936331b 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -419,7 +419,7 @@ func (issue *Issue) HTMLURL() string { return fmt.Sprintf("%s/%s/%d", issue.Repo.HTMLURL(), path, issue.Index) } -// Link returns the Link URL to this issue. +// Link returns the issue's relative URL. func (issue *Issue) Link() string { var path string if issue.IsPull { diff --git a/models/issues/pull.go b/models/issues/pull.go index 044fb5fa04..3f8b0bc7ac 100644 --- a/models/issues/pull.go +++ b/models/issues/pull.go @@ -759,8 +759,8 @@ func GetPullRequestsByHeadBranch(ctx context.Context, headBranch string, headRep return prs, nil } -// GetBaseBranchHTMLURL returns the HTML URL of the base branch -func (pr *PullRequest) GetBaseBranchHTMLURL() string { +// GetBaseBranchLink returns the relative URL of the base branch +func (pr *PullRequest) GetBaseBranchLink() string { if err := pr.LoadBaseRepo(db.DefaultContext); err != nil { log.Error("LoadBaseRepo: %v", err) return "" @@ -768,11 +768,11 @@ func (pr *PullRequest) GetBaseBranchHTMLURL() string { if pr.BaseRepo == nil { return "" } - return pr.BaseRepo.HTMLURL() + "/src/branch/" + util.PathEscapeSegments(pr.BaseBranch) + return pr.BaseRepo.Link() + "/src/branch/" + util.PathEscapeSegments(pr.BaseBranch) } -// GetHeadBranchHTMLURL returns the HTML URL of the head branch -func (pr *PullRequest) GetHeadBranchHTMLURL() string { +// GetHeadBranchLink returns the relative URL of the head branch +func (pr *PullRequest) GetHeadBranchLink() string { if pr.Flow == PullRequestFlowAGit { return "" } @@ -784,7 +784,7 @@ func (pr *PullRequest) GetHeadBranchHTMLURL() string { if pr.HeadRepo == nil { return "" } - return pr.HeadRepo.HTMLURL() + "/src/branch/" + util.PathEscapeSegments(pr.HeadBranch) + return pr.HeadRepo.Link() + "/src/branch/" + util.PathEscapeSegments(pr.HeadBranch) } // UpdateAllowEdits update if PR can be edited from maintainers diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 96f34a22ee..f4be21e74e 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -65,7 +65,7 @@ type PackageFileDescriptor struct { // PackageWebLink returns the package web link func (pd *PackageDescriptor) PackageWebLink() string { - return fmt.Sprintf("%s/-/packages/%s/%s", pd.Owner.HTMLURL(), string(pd.Package.Type), url.PathEscape(pd.Package.LowerName)) + return fmt.Sprintf("%s/-/packages/%s/%s", pd.Owner.HomeLink(), string(pd.Package.Type), url.PathEscape(pd.Package.LowerName)) } // FullWebLink returns the package version web link diff --git a/models/project/project.go b/models/project/project.go index 273823ac9d..9074fd0c15 100644 --- a/models/project/project.go +++ b/models/project/project.go @@ -116,6 +116,7 @@ func (p *Project) LoadRepo(ctx context.Context) (err error) { return err } +// Link returns the project's relative URL. func (p *Project) Link() string { if p.OwnerID > 0 { err := p.LoadOwner(db.DefaultContext) diff --git a/models/repo/release.go b/models/repo/release.go index 08b429f5e1..abf91bc4bb 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -130,6 +130,11 @@ func (r *Release) HTMLURL() string { return r.Repo.HTMLURL() + "/releases/tag/" + util.PathEscapeSegments(r.TagName) } +// Link the relative url for a release on the web UI. release must have attributes loaded +func (r *Release) Link() string { + return r.Repo.Link() + "/releases/tag/" + util.PathEscapeSegments(r.TagName) +} + // IsReleaseExist returns true if release with given tag name already exists. func IsReleaseExist(ctx context.Context, repoID int64, tagName string) (bool, error) { if len(tagName) == 0 { diff --git a/models/repo/repo.go b/models/repo/repo.go index 06ec34ed63..5d3753620d 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -480,7 +480,7 @@ func (repo *Repository) RepoPath() string { return RepoPath(repo.OwnerName, repo.Name) } -// Link returns the repository link +// Link returns the repository relative url func (repo *Repository) Link() string { return setting.AppSubURL + "/" + url.PathEscape(repo.OwnerName) + "/" + url.PathEscape(repo.Name) } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 16f3d9dd26..ee4bec4df7 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -63,6 +63,7 @@ type Repository struct { Language string `json:"language"` LanguagesURL string `json:"languages_url"` HTMLURL string `json:"html_url"` + Link string `json:"link"` SSHURL string `json:"ssh_url"` CloneURL string `json:"clone_url"` OriginalURL string `json:"original_url"` diff --git a/routers/web/feed/convert.go b/routers/web/feed/convert.go index 7c375a085f..76dc769c65 100644 --- a/routers/web/feed/convert.go +++ b/routers/web/feed/convert.go @@ -73,7 +73,7 @@ func feedActionsToFeedItems(ctx *context.Context, actions activities_model.Actio var content, desc, title string - link := &feeds.Link{Href: act.GetCommentLink()} + link := &feeds.Link{Href: act.GetCommentHTMLURL()} // title title = act.ActUser.DisplayName() + " " diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 11d336d4ec..ad17005d90 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -339,8 +339,8 @@ func setMergeTarget(ctx *context.Context, pull *issues_model.PullRequest) { ctx.Data["HeadTarget"] = pull.MustHeadUserName(ctx) + "/" + pull.HeadRepo.Name + ":" + pull.HeadBranch } ctx.Data["BaseTarget"] = pull.BaseBranch - ctx.Data["HeadBranchHTMLURL"] = pull.GetHeadBranchHTMLURL() - ctx.Data["BaseBranchHTMLURL"] = pull.GetBaseBranchHTMLURL() + ctx.Data["HeadBranchLink"] = pull.GetHeadBranchLink() + ctx.Data["BaseBranchLink"] = pull.GetBaseBranchLink() } // PrepareMergedViewPullInfo show meta information for a merged pull request view page diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index f9c67f170b..0a51dfa733 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -569,6 +569,7 @@ func SearchRepo(ctx *context.Context) { Mirror: repo.IsMirror, Stars: repo.NumStars, HTMLURL: repo.HTMLURL(), + Link: repo.Link(), Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, } } diff --git a/templates/code/searchresults.tmpl b/templates/code/searchresults.tmpl index e21a50e1f1..f9b17aee41 100644 --- a/templates/code/searchresults.tmpl +++ b/templates/code/searchresults.tmpl @@ -13,13 +13,13 @@

- {{$repo.FullName}} + {{$repo.FullName}} {{if $repo.IsArchived}} {{$.locale.Tr "repo.desc.archived"}} {{end}} - {{.Filename}} - {{$.locale.Tr "repo.diff.view_file"}} + {{$.locale.Tr "repo.diff.view_file"}}

@@ -28,7 +28,7 @@ {{range .LineNumbers}} - {{.}} + {{.}} {{end}} {{.FormattedLines | Safe}} diff --git a/templates/mail/issue/assigned.tmpl b/templates/mail/issue/assigned.tmpl index 05bed69022..232a41b56f 100644 --- a/templates/mail/issue/assigned.tmpl +++ b/templates/mail/issue/assigned.tmpl @@ -8,7 +8,7 @@ {{.Subject}} -{{$repo_url := printf "%s" (Escape .Issue.Repo.HTMLURL) (Escape .Issue.Repo.FullName)}} +{{$repo_url := printf "%s" (Escape .Issue.Repo.Link) (Escape .Issue.Repo.FullName)}} {{$link := printf "#%d" (Escape .Link) .Issue.Index}}

diff --git a/templates/mail/issue/default.tmpl b/templates/mail/issue/default.tmpl index 64dbb3df68..3bda408a05 100644 --- a/templates/mail/issue/default.tmpl +++ b/templates/mail/issue/default.tmpl @@ -20,11 +20,11 @@ {{if eq .ActionName "push"}}

{{if .Comment.IsForcePush}} - {{$oldCommitUrl := printf "%s/commit/%s" .Comment.Issue.PullRequest.BaseRepo.HTMLURL .Comment.OldCommit}} + {{$oldCommitUrl := printf "%s/commit/%s" .Comment.Issue.PullRequest.BaseRepo.Link .Comment.OldCommit}} {{$oldShortSha := ShortSha .Comment.OldCommit}} {{$oldCommitLink := printf "%[2]s" (Escape $oldCommitUrl) (Escape $oldShortSha)}} - {{$newCommitUrl := printf "%s/commit/%s" .Comment.Issue.PullRequest.BaseRepo.HTMLURL .Comment.NewCommit}} + {{$newCommitUrl := printf "%s/commit/%s" .Comment.Issue.PullRequest.BaseRepo.Link .Comment.NewCommit}} {{$newShortSha := ShortSha .Comment.NewCommit}} {{$newCommitLink := printf "%[2]s" (Escape $newCommitUrl) (Escape $newShortSha)}} @@ -72,7 +72,7 @@

    {{range .Comment.Commits}}
  • - + {{ShortSha .ID.String}} - {{.Summary}}
    • diff --git a/templates/mail/release.tmpl b/templates/mail/release.tmpl index b2acdce8b2..4250433523 100644 --- a/templates/mail/release.tmpl +++ b/templates/mail/release.tmpl @@ -11,8 +11,8 @@ -{{$release_url := printf "%s" (.Release.HTMLURL | Escape) (.Release.TagName | Escape)}} -{{$repo_url := printf "%s" (.Release.Repo.HTMLURL | Escape) (.Release.Repo.FullName | Escape)}} +{{$release_url := printf "%s" (.Release.Link | Escape) (.Release.TagName | Escape)}} +{{$repo_url := printf "%s" (.Release.Repo.Link | Escape) (.Release.Repo.FullName | Escape)}}

    {{.locale.Tr "mail.release.new.text" .Release.Publisher.Name $release_url $repo_url | Str2html}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index ec2e88c854..01e0cb49a6 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -28,7 +28,7 @@ {{$hasRepositoryAccess = index $.RepositoryAccessMap .Repository.ID}} {{end}} {{if $hasRepositoryAccess}} - {{$.locale.Tr "packages.published_by_in" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) .Repository.HTMLURL (.Repository.FullName | Escape) | Safe}} + {{$.locale.Tr "packages.published_by_in" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) .Repository.Link (.Repository.FullName | Escape) | Safe}} {{else}} {{$.locale.Tr "packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} {{end}} @@ -41,7 +41,7 @@ {{svg "octicon-package" 32}}

    {{.locale.Tr "packages.empty"}}

    {{if and .Repository .CanWritePackages}} - {{$packagesUrl := URLJoin .Owner.HTMLURL "-" "packages"}} + {{$packagesUrl := URLJoin .Owner.HomeLink "-" "packages"}}

    {{.locale.Tr "packages.empty.repo" $packagesUrl | Safe}}

    {{end}}

    {{.locale.Tr "packages.empty.documentation" | Safe}}

    diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index a548d9e0b6..2b32139681 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -11,7 +11,7 @@
    {{$timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}} {{if .HasRepositoryAccess}} - {{.locale.Tr "packages.published_by_in" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) .PackageDescriptor.Repository.HTMLURL (.PackageDescriptor.Repository.FullName | Escape) | Safe}} + {{.locale.Tr "packages.published_by_in" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) .PackageDescriptor.Repository.Link (.PackageDescriptor.Repository.FullName | Escape) | Safe}} {{else}} {{.locale.Tr "packages.published_by" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) | Safe}} {{end}} @@ -41,7 +41,7 @@
    {{svg .PackageDescriptor.Package.Type.SVGName 16 "mr-3"}} {{.PackageDescriptor.Package.Type.Name}}
    {{if .HasRepositoryAccess}} -
    {{svg "octicon-repo" 16 "mr-3"}} {{.PackageDescriptor.Repository.FullName}}
    +
    {{svg "octicon-repo" 16 "mr-3"}} {{.PackageDescriptor.Repository.FullName}}
    {{end}}
    {{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    @@ -91,7 +91,7 @@
    {{if .HasRepositoryAccess}} -
    {{svg "octicon-issue-opened" 16 "mr-3"}} {{.locale.Tr "repo.issues"}}
    +
    {{svg "octicon-issue-opened" 16 "mr-3"}} {{.locale.Tr "repo.issues"}}
    {{end}} {{if .CanWritePackages}}
    {{svg "octicon-tools" 16 "mr-3"}} {{.locale.Tr "repo.settings"}}
    diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index ee844299ab..1a68c90e51 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -96,13 +96,13 @@ {{end}} {{else}} - {{if not .LatestPullRequest.IsSameRepo}}{{.LatestPullRequest.BaseRepo.FullName}}{{end}}#{{.LatestPullRequest.Issue.Index}} + {{if not .LatestPullRequest.IsSameRepo}}{{.LatestPullRequest.BaseRepo.FullName}}{{end}}#{{.LatestPullRequest.Issue.Index}} {{if .LatestPullRequest.HasMerged}} - {{svg "octicon-git-merge" 16 "mr-2"}}{{$.locale.Tr "repo.pulls.merged"}} + {{svg "octicon-git-merge" 16 "mr-2"}}{{$.locale.Tr "repo.pulls.merged"}} {{else if .LatestPullRequest.Issue.IsClosed}} - {{svg "octicon-git-pull-request" 16 "mr-2"}}{{$.locale.Tr "repo.issues.closed_title"}} + {{svg "octicon-git-pull-request" 16 "mr-2"}}{{$.locale.Tr "repo.issues.closed_title"}} {{else}} - {{svg "octicon-git-pull-request" 16 "mr-2"}}{{$.locale.Tr "repo.issues.open_title"}} + {{svg "octicon-git-pull-request" 16 "mr-2"}}{{$.locale.Tr "repo.issues.open_title"}} {{end}} {{end}} diff --git a/templates/repo/diff/box.tmpl b/templates/repo/diff/box.tmpl index 798a7eae14..4bc211f931 100644 --- a/templates/repo/diff/box.tmpl +++ b/templates/repo/diff/box.tmpl @@ -151,7 +151,7 @@ {{end}}
    {{else}} - +
    {{if $.IsSplitStyle}} {{template "repo/diff/section_split" dict "file" . "root" $}} {{else}} @@ -191,7 +191,7 @@
    {{$.locale.Tr "write"}} - {{$.locale.Tr "preview"}} + {{$.locale.Tr "preview"}}
    diff --git a/templates/repo/diff/comment_form.tmpl b/templates/repo/diff/comment_form.tmpl index 9325c754ce..f51e840721 100644 --- a/templates/repo/diff/comment_form.tmpl +++ b/templates/repo/diff/comment_form.tmpl @@ -1,5 +1,5 @@ {{if and $.root.SignedUserID (not $.Repository.IsArchived)}} -
    + {{$.root.CsrfTokenHtml}} @@ -11,7 +11,7 @@
    {{$.root.locale.Tr "write"}} - {{$.root.locale.Tr "preview"}} + {{$.root.locale.Tr "preview"}}
    diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index 3bc9da99b0..c4a48af9f5 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -31,13 +31,13 @@
    {{svg "octicon-code"}} {{if .IsNewFile}}{{.locale.Tr "repo.editor.new_file"}}{{else}}{{.locale.Tr "repo.editor.edit_file"}}{{end}} {{if not .IsNewFile}} - {{svg "octicon-eye"}} {{.locale.Tr "preview"}} + {{svg "octicon-eye"}} {{.locale.Tr "preview"}} {{svg "octicon-diff"}} {{.locale.Tr "repo.editor.preview_changes"}} {{end}}
    diff --git a/templates/repo/issue/view_content.tmpl b/templates/repo/issue/view_content.tmpl index 9c5f9a4779..7b318d51f1 100644 --- a/templates/repo/issue/view_content.tmpl +++ b/templates/repo/issue/view_content.tmpl @@ -198,7 +198,7 @@
    {{$.locale.Tr "write"}} - {{$.locale.Tr "preview"}} + {{$.locale.Tr "preview"}}
    diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 3012b09d58..39fbc638ce 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -123,7 +123,7 @@ {{template "shared/user/avatarlink" .Poster}} {{template "shared/user/authorlink" .Poster}} - {{$link := printf "%s/commit/%s" $.Repository.HTMLURL ($.Issue.PullRequest.MergedCommitID|PathEscape)}} + {{$link := printf "%s/commit/%s" $.Repository.Link ($.Issue.PullRequest.MergedCommitID|PathEscape)}} {{if eq $.Issue.PullRequest.Status 3}} {{$.locale.Tr "repo.issues.manually_pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} {{else}} @@ -329,7 +329,7 @@
    {{svg "octicon-plus"}} - + {{if eq .DependentIssue.RepoID .Issue.RepoID}} #{{.DependentIssue.Index}} {{.DependentIssue.Title}} {{else}} @@ -352,7 +352,7 @@
    {{svg "octicon-trash"}} - + {{if eq .DependentIssue.RepoID .Issue.RepoID}} #{{.DependentIssue.Index}} {{.DependentIssue.Title}} {{else}} @@ -476,7 +476,7 @@ {{$resolveDoer := (index $comms 0).ResolveDoer}} {{$isNotPending := (not (eq (index $comms 0).Review.Type 0))}}
    - {{$filename}} + {{$filename}} {{if $invalid}} {{$.locale.Tr "repo.issues.review.outdated"}} diff --git a/templates/repo/issue/view_content/context_menu.tmpl b/templates/repo/issue/view_content/context_menu.tmpl index 7459623a54..b96fd86af5 100644 --- a/templates/repo/issue/view_content/context_menu.tmpl +++ b/templates/repo/issue/view_content/context_menu.tmpl @@ -6,11 +6,11 @@
    {{$referenceUrl := ""}} {{if .issue}} - {{$referenceUrl = Printf "%s#%s" .ctx.Issue.HTMLURL .item.HashTag}} + {{$referenceUrl = Printf "%s#%s" .ctx.Issue.Link .item.HashTag}} {{else}} - {{$referenceUrl = Printf "%s/files#%s" .ctx.Issue.HTMLURL .item.HashTag}} + {{$referenceUrl = Printf "%s/files#%s" .ctx.Issue.Link .item.HashTag}} {{end}} -
    {{.ctx.locale.Tr "repo.issues.context.copy_link"}}
    +
    {{.ctx.locale.Tr "repo.issues.context.copy_link"}}
    {{.ctx.locale.Tr "repo.issues.context.quote_reply"}}
    {{if not .ctx.UnitIssuesGlobalDisabled}}
    {{.ctx.locale.Tr "repo.issues.context.reference_issue"}}
    diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index dc671eb6d6..c68a150832 100644 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -133,7 +133,7 @@ {{if .Issue.PullRequest.HasMerged}}
    {{if .Issue.PullRequest.MergedCommitID}} - {{$link := printf "%s/commit/%s" $.Repository.HTMLURL (.Issue.PullRequest.MergedCommitID|PathEscape)}} + {{$link := printf "%s/commit/%s" $.Repository.Link (.Issue.PullRequest.MergedCommitID|PathEscape)}} {{if eq $.Issue.PullRequest.Status 3}} {{$.locale.Tr "repo.pulls.manually_merged_as" ($link|Escape) (ShortSha .Issue.PullRequest.MergedCommitID) | Safe}} {{else}} diff --git a/templates/repo/issue/view_title.tmpl b/templates/repo/issue/view_title.tmpl index 58f4722e28..74fe1ff248 100644 --- a/templates/repo/issue/view_title.tmpl +++ b/templates/repo/issue/view_title.tmpl @@ -35,13 +35,13 @@ {{if .Issue.IsPull}} {{$headHref := .HeadTarget|Escape}} - {{if .HeadBranchHTMLURL}} - {{$headHref = printf "%s" (.HeadBranchHTMLURL | Escape) $headHref}} + {{if .HeadBranchLink}} + {{$headHref = printf "%s" (.HeadBranchLink | Escape) $headHref}} {{end}} {{$headHref = printf "%s %s" $headHref (.locale.Tr "copy_branch") (.HeadTarget | Escape) (svg "octicon-copy" 14)}} {{$baseHref := .BaseTarget|Escape}} - {{if .BaseBranchHTMLURL}} - {{$baseHref = printf "%s" (.BaseBranchHTMLURL | Escape) $baseHref}} + {{if .BaseBranchLink}} + {{$baseHref = printf "%s" (.BaseBranchLink | Escape) $baseHref}} {{end}} {{if .Issue.PullRequest.HasMerged}} {{$mergedStr:= TimeSinceUnix .Issue.PullRequest.MergedUnix $.locale}} diff --git a/templates/repo/projects/view.tmpl b/templates/repo/projects/view.tmpl index 1c09aade35..63d2727b63 100644 --- a/templates/repo/projects/view.tmpl +++ b/templates/repo/projects/view.tmpl @@ -242,7 +242,7 @@ {{end}}
    {{range .Assignees}} - {{avatar . 28 "mini mr-3"}} + {{avatar . 28 "mini mr-3"}} {{end}}
    diff --git a/templates/repo/release/new.tmpl b/templates/repo/release/new.tmpl index b4b7186a9a..738b3bd174 100644 --- a/templates/repo/release/new.tmpl +++ b/templates/repo/release/new.tmpl @@ -48,7 +48,7 @@
    {{$.locale.Tr "write"}} - {{$.locale.Tr "preview"}} + {{$.locale.Tr "preview"}}
    diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 711a37461e..85ff0120b2 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -110,10 +110,11 @@
    {{if $.Permission.CanRead $.UnitTypeIssues}} + {{/* FIXME: Here we use HTMLURL but not link, see https://github.com/go-gitea/gitea/pull/21986/files#r1096532186*/}} {{.locale.Tr "repo.issues.context.reference_issue"}} {{end}} - {{.locale.Tr "repo.view_git_blame"}} - {{.locale.Tr "repo.file_copy_permalink"}} + {{.locale.Tr "repo.view_git_blame"}} + {{.locale.Tr "repo.file_copy_permalink"}}
    {{end}} {{end}} diff --git a/templates/repo/wiki/new.tmpl b/templates/repo/wiki/new.tmpl index 194e40122b..9e9de99022 100644 --- a/templates/repo/wiki/new.tmpl +++ b/templates/repo/wiki/new.tmpl @@ -21,11 +21,11 @@
    {{.locale.Tr "write"}} - {{$.locale.Tr "preview"}} + {{$.locale.Tr "preview"}}
    - +
    diff --git a/templates/shared/issuelist.tmpl b/templates/shared/issuelist.tmpl index 93d54930ff..9e3f605456 100644 --- a/templates/shared/issuelist.tmpl +++ b/templates/shared/issuelist.tmpl @@ -34,7 +34,7 @@
    - {{RenderEmoji .Title | RenderCodeBlock}} + {{RenderEmoji .Title | RenderCodeBlock}} {{if .IsPull}} {{if (index $.CommitStatuses .PullRequest.ID)}} {{template "repo/commit_statuses" dict "Status" (index $.CommitLastStatus .PullRequest.ID) "Statuses" (index $.CommitStatuses .PullRequest.ID) "root" $}} @@ -47,7 +47,7 @@
    - + {{if eq $.listType "dashboard"}} {{.Repo.FullName}}#{{.Index}} {{else}} @@ -65,7 +65,7 @@ {{if .IsPull}}
    - + {{/* inline to remove the spaces between spans */}} {{if ne .RepoID .PullRequest.BaseRepoID}}{{.PullRequest.BaseRepo.OwnerName}}:{{end}}{{.PullRequest.BaseBranch}} @@ -73,7 +73,7 @@ {{svg "gitea-double-chevron-left" 12 "mx-1"}} {{if .PullRequest.HeadRepo}}
    - + {{/* inline to remove the spaces between spans */}} {{if ne .RepoID .PullRequest.HeadRepoID}}{{.PullRequest.HeadRepo.OwnerName}}:{{end}}{{.PullRequest.HeadBranch}} @@ -159,7 +159,7 @@
    {{if .NumComments}} - + {{svg "octicon-comment" 16 "mr-2"}}{{.NumComments}} {{end}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 812e626d2d..5b07965d2a 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -19098,6 +19098,10 @@ "type": "string", "x-go-name": "LanguagesURL" }, + "link": { + "type": "string", + "x-go-name": "Link" + }, "mirror": { "type": "boolean", "x-go-name": "Mirror" diff --git a/templates/user/dashboard/repolist.tmpl b/templates/user/dashboard/repolist.tmpl index 9d0ea01ec5..8fb5763f3a 100644 --- a/templates/user/dashboard/repolist.tmpl +++ b/templates/user/dashboard/repolist.tmpl @@ -127,7 +127,7 @@
    • - +
      ${repo.full_name}
      diff --git a/templates/user/notification/notification_div.tmpl b/templates/user/notification/notification_div.tmpl index 702dd419a3..0ea001ff3d 100644 --- a/templates/user/notification/notification_div.tmpl +++ b/templates/user/notification/notification_div.tmpl @@ -35,7 +35,7 @@ {{$issue := .Issue}} {{$repo := .Repository}} - + {{if eq .Status 3}} {{svg "octicon-pin"}} {{else if not $issue}} @@ -58,8 +58,8 @@ {{end}} {{end}} - -       + + {{if $issue}} #{{$issue.Index}} - {{$issue.Title}} {{else}} diff --git a/web_src/js/features/clipboard.js b/web_src/js/features/clipboard.js index f8486cdc6c..07c439504e 100644 --- a/web_src/js/features/clipboard.js +++ b/web_src/js/features/clipboard.js @@ -1,4 +1,5 @@ import {showTemporaryTooltip} from '../modules/tippy.js'; +import {toAbsoluteUrl} from '../utils.js'; const {copy_success, copy_error} = window.config.i18n; @@ -50,7 +51,11 @@ export function initGlobalCopyToClipboardListener() { // in case , so we just search // up to 3 levels for performance for (let i = 0; i < 3 && target; i++) { - const text = target.getAttribute('data-clipboard-text') || document.querySelector(target.getAttribute('data-clipboard-target'))?.value; + let txt = target.getAttribute('data-clipboard-text'); + if (txt && target.getAttribute('data-clipboard-text-type') === 'url') { + txt = toAbsoluteUrl(txt); + } + const text = txt || document.querySelector(target.getAttribute('data-clipboard-target'))?.value; if (text) { e.preventDefault(); diff --git a/web_src/js/test/setup.js b/web_src/js/test/setup.js index 7c208eb0d2..e0e2c71e29 100644 --- a/web_src/js/test/setup.js +++ b/web_src/js/test/setup.js @@ -2,4 +2,5 @@ window.config = { csrfToken: 'test-csrf-token-123456', pageData: {}, i18n: {}, + appSubUrl: '', }; diff --git a/web_src/js/utils.js b/web_src/js/utils.js index 01c076aeba..b9cd69e15a 100644 --- a/web_src/js/utils.js +++ b/web_src/js/utils.js @@ -133,3 +133,10 @@ export function convertImage(blob, mime) { } }); } + +export function toAbsoluteUrl(relUrl) { + if (relUrl.startsWith('http://') || relUrl.startsWith('https://')) { + return relUrl; + } + return `${window.location.origin}${relUrl}`; +} diff --git a/web_src/js/utils.test.js b/web_src/js/utils.test.js index 1df0caa211..4efe916231 100644 --- a/web_src/js/utils.test.js +++ b/web_src/js/utils.test.js @@ -2,6 +2,7 @@ import {expect, test} from 'vitest'; import { basename, extname, isObject, uniq, stripTags, joinPaths, parseIssueHref, prettyNumber, parseUrl, translateMonth, translateDay, blobToDataURI, + toAbsoluteUrl, } from './utils.js'; test('basename', () => { @@ -136,3 +137,8 @@ test('blobToDataURI', async () => { const blob = new Blob([JSON.stringify({test: true})], {type: 'application/json'}); expect(await blobToDataURI(blob)).toEqual('data:application/json;base64,eyJ0ZXN0Ijp0cnVlfQ=='); }); + +test('toAbsoluteUrl', () => { + expect(toAbsoluteUrl('')).toEqual('http://localhost:3000'); + expect(toAbsoluteUrl('/user/repo')).toEqual('http://localhost:3000/user/repo'); +}); From 3ae78bc0a9408ca2b8ed5a1ffafdcf66639449e5 Mon Sep 17 00:00:00 2001 From: Peyton Duncan <11335266+Helithumper@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:05:59 -0800 Subject: [PATCH 15/33] Grammar fix (#22790) Noticed a minor grammatical error. --- docs/content/doc/installation/with-docker-rootless.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/installation/with-docker-rootless.en-us.md b/docs/content/doc/installation/with-docker-rootless.en-us.md index 028d81ab91..7e1a0e6fe0 100644 --- a/docs/content/doc/installation/with-docker-rootless.en-us.md +++ b/docs/content/doc/installation/with-docker-rootless.en-us.md @@ -19,7 +19,7 @@ Gitea provides automatically updated Docker images within its Docker Hub organiz possible to always use the latest stable tag or to use another service that handles updating Docker images. -The rootless image use Gitea internal SSH to provide Git protocol and doesn't support OpenSSH. +The rootless image uses Gitea internal SSH to provide Git protocol and doesn't support OpenSSH. This reference setup guides users through the setup based on `docker-compose`, but the installation of `docker-compose` is out of scope of this documentation. To install `docker-compose` itself, follow From d5fa2e7510844ad092804b7c11bf6847f7c4e45d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 7 Feb 2023 09:18:52 +0800 Subject: [PATCH 16/33] Fix restore repo bug, clarify the problem of ForeignIndex (#22776) Fix #22581 TLDR: #18446 made a mess with ForeignIndex and triggered a design flaw/bug of #16356, then a quick patch #21271 helped #18446, then the the bug was re-triggered by #21721 . Related: * #16356 * BasicIssueContext https://github.com/go-gitea/gitea/pull/16356/files#diff-7938eb670d42a5ead6b08121e16aa4537a4d716c1cf37923c70470020fb9d036R16-R27 * #18446 * If some issues were dumped without ForeignIndex, then they would be imported as ForeignIndex=0 https://github.com/go-gitea/gitea/pull/18446/files#diff-1624a3e715d8fc70edf2db1630642b7d6517f8c359cc69d58c3958b34ba4ce5eR38-R39 * #21271 * It patched the above bug (somewhat), made the issues without ForeignIndex could have the same value as LocalIndex * #21721 * It re-triggered the zero-ForeignIndex bug. ps: I am not sure whether the changes in `GetForeignIndex` are ideal (at least, now it has almost the same behavior as BasicIssueContext in #16356), it's just a quick fix. Feel free to edit on this PR directly or replace it. Co-authored-by: zeripath --- modules/migration/comment.go | 3 +-- modules/migration/issue.go | 13 +++++++++++-- modules/migration/review.go | 12 +++++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/modules/migration/comment.go b/modules/migration/comment.go index f994e972ed..92ce30e302 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -8,8 +8,7 @@ import "time" // Commentable can be commented upon type Commentable interface { - GetLocalIndex() int64 - GetForeignIndex() int64 + Reviewable GetContext() DownloaderContext } diff --git a/modules/migration/issue.go b/modules/migration/issue.go index 7cb9f84b0d..3d1d1b4e0d 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -34,6 +34,15 @@ func (issue *Issue) GetExternalName() string { return issue.PosterName } // GetExternalID ExternalUserMigrated interface func (issue *Issue) GetExternalID() int64 { return issue.PosterID } -func (issue *Issue) GetLocalIndex() int64 { return issue.Number } -func (issue *Issue) GetForeignIndex() int64 { return issue.ForeignIndex } +func (issue *Issue) GetLocalIndex() int64 { return issue.Number } + +func (issue *Issue) GetForeignIndex() int64 { + // see the comment of Reviewable.GetForeignIndex + // if there is no ForeignIndex, then use LocalIndex + if issue.ForeignIndex == 0 { + return issue.Number + } + return issue.ForeignIndex +} + func (issue *Issue) GetContext() DownloaderContext { return issue.Context } diff --git a/modules/migration/review.go b/modules/migration/review.go index a420c130c7..79e821b2e1 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -8,6 +8,16 @@ import "time" // Reviewable can be reviewed type Reviewable interface { GetLocalIndex() int64 + + // GetForeignIndex presents the foreign index, which could be misused: + // For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it: + // * if site-A exports files by using its LocalIndex + // * from site-A's view, LocalIndex is site-A's IssueIndex while ForeignIndex is site-B's IssueIndex + // * but from site-B's view, LocalIndex is site-B's IssueIndex while ForeignIndex is site-A's IssueIndex + // + // So the exporting/importing must be paired, but the meaning of them looks confusing then: + // * either site-A and site-B both use LocalIndex during dumping/restoring + // * or site-A and site-B both use ForeignIndex GetForeignIndex() int64 } @@ -37,7 +47,7 @@ type Review struct { // GetExternalName ExternalUserMigrated interface func (r *Review) GetExternalName() string { return r.ReviewerName } -// ExternalID ExternalUserMigrated interface +// GetExternalID ExternalUserMigrated interface func (r *Review) GetExternalID() int64 { return r.ReviewerID } // ReviewComment represents a review comment From 2c6cc0b8c982b3d49a5b208f75e15b2269584312 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 8 Feb 2023 00:08:44 +0800 Subject: [PATCH 17/33] Fix links for the menus in the view file page (#22795) --- templates/repo/view_file.tmpl | 3 +-- web_src/js/features/repo-code.js | 14 ++++++++------ web_src/js/utils.js | 14 ++++++++++---- web_src/js/utils.test.js | 5 +++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 85ff0120b2..e92aeb0702 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -110,8 +110,7 @@
      {{if $.Permission.CanRead $.UnitTypeIssues}} - {{/* FIXME: Here we use HTMLURL but not link, see https://github.com/go-gitea/gitea/pull/21986/files#r1096532186*/}} - {{.locale.Tr "repo.issues.context.reference_issue"}} + {{.locale.Tr "repo.issues.context.reference_issue"}} {{end}} {{.locale.Tr "repo.view_git_blame"}} {{.locale.Tr "repo.file_copy_permalink"}} diff --git a/web_src/js/features/repo-code.js b/web_src/js/features/repo-code.js index 083a17bf21..1c59913132 100644 --- a/web_src/js/features/repo-code.js +++ b/web_src/js/features/repo-code.js @@ -3,6 +3,7 @@ import {svg} from '../svg.js'; import {invertFileFolding} from './file-fold.js'; import {createTippy} from '../modules/tippy.js'; import {copyToClipboard} from './clipboard.js'; +import {toAbsoluteUrl} from '../utils.js'; export const singleAnchorRegex = /^#(L|n)([1-9][0-9]*)$/; export const rangeAnchorRegex = /^#(L[1-9][0-9]*)-(L[1-9][0-9]*)$/; @@ -19,17 +20,18 @@ function selectRange($list, $select, $from) { $list.removeClass('active'); // add hashchange to permalink - const $issue = $('a.ref-in-new-issue'); + const $refInNewIssue = $('a.ref-in-new-issue'); const $copyPermalink = $('a.copy-line-permalink'); const $viewGitBlame = $('a.view_git_blame'); const updateIssueHref = function (anchor) { - if ($issue.length === 0) { + if ($refInNewIssue.length === 0) { return; } - let href = $issue.attr('href'); - href = `${href.replace(/%23L\d+$|%23L\d+-L\d+$/, '')}%23${anchor}`; - $issue.attr('href', href); + const urlIssueNew = $refInNewIssue.attr('data-url-issue-new'); + const urlParamBodyLink = $refInNewIssue.attr('data-url-param-body-link'); + const issueContent = `${toAbsoluteUrl(urlParamBodyLink)}#${anchor}`; // the default content for issue body + $refInNewIssue.attr('href', `${urlIssueNew}?body=${encodeURIComponent(issueContent)}`); }; const updateViewGitBlameFragment = function (anchor) { @@ -188,7 +190,7 @@ export function initRepoCodeView() { currentTarget.closest('tr').outerHTML = blob; }); $(document).on('click', '.copy-line-permalink', async (e) => { - const success = await copyToClipboard(e.currentTarget.getAttribute('data-url')); + const success = await copyToClipboard(toAbsoluteUrl(e.currentTarget.getAttribute('data-url'))); if (!success) return; document.querySelector('.code-line-button')?._tippy?.hide(); }); diff --git a/web_src/js/utils.js b/web_src/js/utils.js index b9cd69e15a..c7624404c7 100644 --- a/web_src/js/utils.js +++ b/web_src/js/utils.js @@ -134,9 +134,15 @@ export function convertImage(blob, mime) { }); } -export function toAbsoluteUrl(relUrl) { - if (relUrl.startsWith('http://') || relUrl.startsWith('https://')) { - return relUrl; +export function toAbsoluteUrl(url) { + if (url.startsWith('http://') || url.startsWith('https://')) { + return url; } - return `${window.location.origin}${relUrl}`; + if (url.startsWith('//')) { + return `${window.location.protocol}${url}`; // it's also a somewhat absolute URL (with the current scheme) + } + if (url && !url.startsWith('/')) { + throw new Error('unsupported url, it should either start with / or http(s)://'); + } + return `${window.location.origin}${url}`; } diff --git a/web_src/js/utils.test.js b/web_src/js/utils.test.js index 4efe916231..306acd34af 100644 --- a/web_src/js/utils.test.js +++ b/web_src/js/utils.test.js @@ -139,6 +139,11 @@ test('blobToDataURI', async () => { }); test('toAbsoluteUrl', () => { + expect(toAbsoluteUrl('//host/dir')).toEqual('http://host/dir'); + expect(toAbsoluteUrl('https://host/dir')).toEqual('https://host/dir'); + expect(toAbsoluteUrl('')).toEqual('http://localhost:3000'); expect(toAbsoluteUrl('/user/repo')).toEqual('http://localhost:3000/user/repo'); + + expect(() => toAbsoluteUrl('path')).toThrowError('unsupported'); }); From e8186f1c0f194ce3f63bed9a564002b80c0859c9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 8 Feb 2023 07:44:42 +0100 Subject: [PATCH 18/33] Map OIDC groups to Orgs/Teams (#21441) Fixes #19555 Test-Instructions: https://github.com/go-gitea/gitea/pull/21441#issuecomment-1419438000 This PR implements the mapping of user groups provided by OIDC providers to orgs teams in Gitea. The main part is a refactoring of the existing LDAP code to make it usable from different providers. Refactorings: - Moved the router auth code from module to service because of import cycles - Changed some model methods to take a `Context` parameter - Moved the mapping code from LDAP to a common location I've tested it with Keycloak but other providers should work too. The JSON mapping format is the same as for LDAP. ![grafik](https://user-images.githubusercontent.com/1666336/195634392-3fc540fc-b229-4649-99ac-91ae8e19df2d.png) --------- Co-authored-by: Lunny Xiao --- cmd/admin.go | 17 +++ docs/content/doc/usage/command-line.en-us.md | 2 + models/organization/org.go | 20 +-- models/organization/org_test.go | 16 +-- modules/auth/common.go | 22 +++ modules/context/api.go | 30 ---- modules/context/context.go | 32 ----- modules/context/org.go | 2 +- modules/repository/create_test.go | 4 +- modules/repository/repo.go | 2 +- modules/validation/binding.go | 24 +++- modules/web/middleware/binding.go | 2 + options/locale/locale_en-US.ini | 3 + routers/api/v1/api.go | 4 +- routers/api/v1/repo/fork.go | 2 +- routers/api/v1/repo/repo.go | 2 +- routers/web/admin/auths.go | 2 + routers/web/auth/linkaccount.go | 7 + routers/web/auth/oauth.go | 95 ++++++++---- routers/web/repo/issue_label.go | 2 +- routers/web/repo/setting.go | 2 +- routers/web/web.go | 2 +- services/auth/middleware.go | 60 ++++++++ .../auth/source/ldap/source_authenticate.go | 104 +++++++------- .../auth/source/ldap/source_group_sync.go | 94 ------------ services/auth/source/ldap/source_search.go | 136 ++++++------------ services/auth/source/ldap/source_sync.go | 11 +- services/auth/source/oauth2/source.go | 23 ++- services/auth/source/source_group_sync.go | 116 +++++++++++++++ services/forms/auth_form.go | 4 +- templates/admin/auth/edit.tmpl | 8 ++ templates/admin/auth/source/ldap.tmpl | 2 +- templates/admin/auth/source/oauth.tmpl | 8 ++ tests/integration/auth_ldap_test.go | 71 +++------ 34 files changed, 504 insertions(+), 427 deletions(-) create mode 100644 modules/auth/common.go create mode 100644 services/auth/middleware.go delete mode 100644 services/auth/source/ldap/source_group_sync.go create mode 100644 services/auth/source/source_group_sync.go diff --git a/cmd/admin.go b/cmd/admin.go index 4e2dc2bf06..318c212d08 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -372,6 +372,15 @@ var ( Value: "", Usage: "Group Claim value for restricted users", }, + cli.StringFlag{ + Name: "group-team-map", + Value: "", + Usage: "JSON mapping between groups and org teams", + }, + cli.BoolFlag{ + Name: "group-team-map-removal", + Usage: "Activate automatic team membership removal depending on groups", + }, } microcmdAuthUpdateOauth = cli.Command{ @@ -853,6 +862,8 @@ func parseOAuth2Config(c *cli.Context) *oauth2.Source { GroupClaimName: c.String("group-claim-name"), AdminGroup: c.String("admin-group"), RestrictedGroup: c.String("restricted-group"), + GroupTeamMap: c.String("group-team-map"), + GroupTeamMapRemoval: c.Bool("group-team-map-removal"), } } @@ -935,6 +946,12 @@ func runUpdateOauth(c *cli.Context) error { if c.IsSet("restricted-group") { oAuth2Config.RestrictedGroup = c.String("restricted-group") } + if c.IsSet("group-team-map") { + oAuth2Config.GroupTeamMap = c.String("group-team-map") + } + if c.IsSet("group-team-map-removal") { + oAuth2Config.GroupTeamMapRemoval = c.Bool("group-team-map-removal") + } // update custom URL mapping customURLMapping := &oauth2.CustomURLMapping{} diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index f2e72d4fc0..9b861a9da3 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -137,6 +137,8 @@ Admin operations: - `--group-claim-name`: Claim name providing group names for this source. (Optional) - `--admin-group`: Group Claim value for administrator users. (Optional) - `--restricted-group`: Group Claim value for restricted users. (Optional) + - `--group-team-map`: JSON mapping between groups and org teams. (Optional) + - `--group-team-map-removal`: Activate automatic team membership removal depending on groups. (Optional) - Examples: - `gitea admin auth add-oauth --name external-github --provider github --key OBTAIN_FROM_SOURCE --secret OBTAIN_FROM_SOURCE` - `update-oauth`: diff --git a/models/organization/org.go b/models/organization/org.go index 05eaead60b..852facf704 100644 --- a/models/organization/org.go +++ b/models/organization/org.go @@ -110,22 +110,14 @@ func (org *Organization) CanCreateOrgRepo(uid int64) (bool, error) { return CanCreateOrgRepo(db.DefaultContext, org.ID, uid) } -func (org *Organization) getTeam(ctx context.Context, name string) (*Team, error) { +// GetTeam returns named team of organization. +func (org *Organization) GetTeam(ctx context.Context, name string) (*Team, error) { return GetTeam(ctx, org.ID, name) } -// GetTeam returns named team of organization. -func (org *Organization) GetTeam(name string) (*Team, error) { - return org.getTeam(db.DefaultContext, name) -} - -func (org *Organization) getOwnerTeam(ctx context.Context) (*Team, error) { - return org.getTeam(ctx, OwnerTeamName) -} - // GetOwnerTeam returns owner team of organization. -func (org *Organization) GetOwnerTeam() (*Team, error) { - return org.getOwnerTeam(db.DefaultContext) +func (org *Organization) GetOwnerTeam(ctx context.Context) (*Team, error) { + return org.GetTeam(ctx, OwnerTeamName) } // FindOrgTeams returns all teams of a given organization @@ -342,7 +334,7 @@ func CreateOrganization(org *Organization, owner *user_model.User) (err error) { } // GetOrgByName returns organization by given name. -func GetOrgByName(name string) (*Organization, error) { +func GetOrgByName(ctx context.Context, name string) (*Organization, error) { if len(name) == 0 { return nil, ErrOrgNotExist{0, name} } @@ -350,7 +342,7 @@ func GetOrgByName(name string) (*Organization, error) { LowerName: strings.ToLower(name), Type: user_model.UserTypeOrganization, } - has, err := db.GetEngine(db.DefaultContext).Get(u) + has, err := db.GetEngine(ctx).Get(u) if err != nil { return nil, err } else if !has { diff --git a/models/organization/org_test.go b/models/organization/org_test.go index 0a38365924..cfa304d7b2 100644 --- a/models/organization/org_test.go +++ b/models/organization/org_test.go @@ -61,28 +61,28 @@ func TestUser_IsOrgMember(t *testing.T) { func TestUser_GetTeam(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3}) - team, err := org.GetTeam("team1") + team, err := org.GetTeam(db.DefaultContext, "team1") assert.NoError(t, err) assert.Equal(t, org.ID, team.OrgID) assert.Equal(t, "team1", team.LowerName) - _, err = org.GetTeam("does not exist") + _, err = org.GetTeam(db.DefaultContext, "does not exist") assert.True(t, organization.IsErrTeamNotExist(err)) nonOrg := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 2}) - _, err = nonOrg.GetTeam("team") + _, err = nonOrg.GetTeam(db.DefaultContext, "team") assert.True(t, organization.IsErrTeamNotExist(err)) } func TestUser_GetOwnerTeam(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3}) - team, err := org.GetOwnerTeam() + team, err := org.GetOwnerTeam(db.DefaultContext) assert.NoError(t, err) assert.Equal(t, org.ID, team.OrgID) nonOrg := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 2}) - _, err = nonOrg.GetOwnerTeam() + _, err = nonOrg.GetOwnerTeam(db.DefaultContext) assert.True(t, organization.IsErrTeamNotExist(err)) } @@ -115,15 +115,15 @@ func TestUser_GetMembers(t *testing.T) { func TestGetOrgByName(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - org, err := organization.GetOrgByName("user3") + org, err := organization.GetOrgByName(db.DefaultContext, "user3") assert.NoError(t, err) assert.EqualValues(t, 3, org.ID) assert.Equal(t, "user3", org.Name) - _, err = organization.GetOrgByName("user2") // user2 is an individual + _, err = organization.GetOrgByName(db.DefaultContext, "user2") // user2 is an individual assert.True(t, organization.IsErrOrgNotExist(err)) - _, err = organization.GetOrgByName("") // corner case + _, err = organization.GetOrgByName(db.DefaultContext, "") // corner case assert.True(t, organization.IsErrOrgNotExist(err)) } diff --git a/modules/auth/common.go b/modules/auth/common.go new file mode 100644 index 0000000000..77361f6561 --- /dev/null +++ b/modules/auth/common.go @@ -0,0 +1,22 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package auth + +import ( + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" +) + +func UnmarshalGroupTeamMapping(raw string) (map[string]map[string][]string, error) { + groupTeamMapping := make(map[string]map[string][]string) + if raw == "" { + return groupTeamMapping, nil + } + err := json.Unmarshal([]byte(raw), &groupTeamMapping) + if err != nil { + log.Error("Failed to unmarshal group team mapping: %v", err) + return nil, err + } + return groupTeamMapping, nil +} diff --git a/modules/context/api.go b/modules/context/api.go index 3f52c54d4c..3f938948ae 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -19,7 +19,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web/middleware" - auth_service "code.gitea.io/gitea/services/auth" ) // APIContext is a specific context for API service @@ -215,35 +214,6 @@ func (ctx *APIContext) CheckForOTP() { } } -// APIAuth converts auth_service.Auth as a middleware -func APIAuth(authMethod auth_service.Method) func(*APIContext) { - return func(ctx *APIContext) { - // Get user from session if logged in. - var err error - ctx.Doer, err = authMethod.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) - if err != nil { - ctx.Error(http.StatusUnauthorized, "APIAuth", err) - return - } - - if ctx.Doer != nil { - if ctx.Locale.Language() != ctx.Doer.Language { - ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) - } - ctx.IsBasicAuth = ctx.Data["AuthedMethod"].(string) == auth_service.BasicMethodName - ctx.IsSigned = true - ctx.Data["IsSigned"] = ctx.IsSigned - ctx.Data["SignedUser"] = ctx.Doer - ctx.Data["SignedUserID"] = ctx.Doer.ID - ctx.Data["SignedUserName"] = ctx.Doer.Name - ctx.Data["IsAdmin"] = ctx.Doer.IsAdmin - } else { - ctx.Data["SignedUserID"] = int64(0) - ctx.Data["SignedUserName"] = "" - } - } -} - // APIContexter returns apicontext as middleware func APIContexter() func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { diff --git a/modules/context/context.go b/modules/context/context.go index 84f40ce063..a2088217ff 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -36,7 +36,6 @@ import ( "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" - "code.gitea.io/gitea/services/auth" "gitea.com/go-chi/cache" "gitea.com/go-chi/session" @@ -659,37 +658,6 @@ func getCsrfOpts() CsrfOptions { } } -// Auth converts auth.Auth as a middleware -func Auth(authMethod auth.Method) func(*Context) { - return func(ctx *Context) { - var err error - ctx.Doer, err = authMethod.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) - if err != nil { - log.Error("Failed to verify user %v: %v", ctx.Req.RemoteAddr, err) - ctx.Error(http.StatusUnauthorized, "Verify") - return - } - if ctx.Doer != nil { - if ctx.Locale.Language() != ctx.Doer.Language { - ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) - } - ctx.IsBasicAuth = ctx.Data["AuthedMethod"].(string) == auth.BasicMethodName - ctx.IsSigned = true - ctx.Data["IsSigned"] = ctx.IsSigned - ctx.Data["SignedUser"] = ctx.Doer - ctx.Data["SignedUserID"] = ctx.Doer.ID - ctx.Data["SignedUserName"] = ctx.Doer.Name - ctx.Data["IsAdmin"] = ctx.Doer.IsAdmin - } else { - ctx.Data["SignedUserID"] = int64(0) - ctx.Data["SignedUserName"] = "" - - // ensure the session uid is deleted - _ = ctx.Session.Delete("uid") - } - } -} - // Contexter initializes a classic context for a request. func Contexter(ctx context.Context) func(next http.Handler) http.Handler { _, rnd := templates.HTMLRenderer(ctx) diff --git a/modules/context/org.go b/modules/context/org.go index ff3a5ae7ec..0add7f2c0c 100644 --- a/modules/context/org.go +++ b/modules/context/org.go @@ -80,7 +80,7 @@ func HandleOrgAssignment(ctx *Context, args ...bool) { orgName := ctx.Params(":org") var err error - ctx.Org.Organization, err = organization.GetOrgByName(orgName) + ctx.Org.Organization, err = organization.GetOrgByName(ctx, orgName) if err != nil { if organization.IsErrOrgNotExist(err) { redirectUserID, err := user_model.LookupUserRedirect(orgName) diff --git a/modules/repository/create_test.go b/modules/repository/create_test.go index 293071bdce..e620422bcb 100644 --- a/modules/repository/create_test.go +++ b/modules/repository/create_test.go @@ -49,7 +49,7 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { assert.NoError(t, organization.CreateOrganization(org, user), "CreateOrganization") // Check Owner team. - ownerTeam, err := org.GetOwnerTeam() + ownerTeam, err := org.GetOwnerTeam(db.DefaultContext) assert.NoError(t, err, "GetOwnerTeam") assert.True(t, ownerTeam.IncludesAllRepositories, "Owner team includes all repositories") @@ -63,7 +63,7 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { } } // Get fresh copy of Owner team after creating repos. - ownerTeam, err = org.GetOwnerTeam() + ownerTeam, err = org.GetOwnerTeam(db.DefaultContext) assert.NoError(t, err, "GetOwnerTeam") // Create teams and check repositories. diff --git a/modules/repository/repo.go b/modules/repository/repo.go index d1a70e7c15..c03e469990 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -57,7 +57,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repoPath := repo_model.RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { - t, err := organization.OrgFromUser(u).GetOwnerTeam() + t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) if err != nil { return nil, err } diff --git a/modules/validation/binding.go b/modules/validation/binding.go index ef0d01e80f..1f904979ff 100644 --- a/modules/validation/binding.go +++ b/modules/validation/binding.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/git" "gitea.com/go-chi/binding" @@ -17,15 +18,14 @@ import ( const ( // ErrGitRefName is git reference name error ErrGitRefName = "GitRefNameError" - // ErrGlobPattern is returned when glob pattern is invalid ErrGlobPattern = "GlobPattern" - // ErrRegexPattern is returned when a regex pattern is invalid ErrRegexPattern = "RegexPattern" - // ErrUsername is username error ErrUsername = "UsernameError" + // ErrInvalidGroupTeamMap is returned when a group team mapping is invalid + ErrInvalidGroupTeamMap = "InvalidGroupTeamMap" ) // AddBindingRules adds additional binding rules @@ -37,6 +37,7 @@ func AddBindingRules() { addRegexPatternRule() addGlobOrRegexPatternRule() addUsernamePatternRule() + addValidGroupTeamMapRule() } func addGitRefNameBindingRule() { @@ -167,6 +168,23 @@ func addUsernamePatternRule() { }) } +func addValidGroupTeamMapRule() { + binding.AddRule(&binding.Rule{ + IsMatch: func(rule string) bool { + return strings.HasPrefix(rule, "ValidGroupTeamMap") + }, + IsValid: func(errs binding.Errors, name string, val interface{}) (bool, binding.Errors) { + _, err := auth.UnmarshalGroupTeamMapping(fmt.Sprintf("%v", val)) + if err != nil { + errs.Add([]string{name}, ErrInvalidGroupTeamMap, err.Error()) + return false, errs + } + + return true, errs + }, + }) +} + func portOnly(hostport string) string { colon := strings.IndexByte(hostport, ':') if colon == -1 { diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 733f00a1d5..8b74a864d9 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -136,6 +136,8 @@ func Validate(errs binding.Errors, data map[string]interface{}, f Form, l transl data["ErrorMsg"] = trName + l.Tr("form.regex_pattern_error", errs[0].Message) case validation.ErrUsername: data["ErrorMsg"] = trName + l.Tr("form.username_error") + case validation.ErrInvalidGroupTeamMap: + data["ErrorMsg"] = trName + l.Tr("form.invalid_group_team_map_error", errs[0].Message) default: msg := errs[0].Classification if msg != "" && errs[0].Message != "" { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a7506986f6..f784b10c8d 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -477,6 +477,7 @@ include_error = ` must contain substring '%s'.` glob_pattern_error = ` glob pattern is invalid: %s.` regex_pattern_error = ` regex pattern is invalid: %s.` username_error = ` can only contain alphanumeric chars ('0-9','a-z','A-Z'), dash ('-'), underscore ('_') and dot ('.'). It cannot begin or end with non-alphanumeric chars, and consecutive non-alphanumeric chars are also forbidden.` +invalid_group_team_map_error = ` mapping is invalid: %s` unknown_error = Unknown error: captcha_incorrect = The CAPTCHA code is incorrect. password_not_match = The passwords do not match. @@ -2758,6 +2759,8 @@ auths.oauth2_required_claim_value_helper = Set this value to restrict login from auths.oauth2_group_claim_name = Claim name providing group names for this source. (Optional) auths.oauth2_admin_group = Group Claim value for administrator users. (Optional - requires claim name above) auths.oauth2_restricted_group = Group Claim value for restricted users. (Optional - requires claim name above) +auths.oauth2_map_group_to_team = Map claimed groups to Organization teams. (Optional - requires claim name above) +auths.oauth2_map_group_to_team_removal = Remove users from synchronized teams if user does not belong to corresponding group. auths.enable_auto_register = Enable Auto Registration auths.sspi_auto_create_users = Automatically create users auths.sspi_auto_create_users_helper = Allow SSPI auth method to automatically create new accounts for users that login for the first time diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 5f57977c29..1d2f8b18e0 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -507,7 +507,7 @@ func orgAssignment(args ...bool) func(ctx *context.APIContext) { var err error if assignOrg { - ctx.Org.Organization, err = organization.GetOrgByName(ctx.Params(":org")) + ctx.Org.Organization, err = organization.GetOrgByName(ctx, ctx.Params(":org")) if err != nil { if organization.IsErrOrgNotExist(err) { redirectUserID, err := user_model.LookupUserRedirect(ctx.Params(":org")) @@ -687,7 +687,7 @@ func Routes(ctx gocontext.Context) *web.Route { } // Get user from session if logged in. - m.Use(context.APIAuth(group)) + m.Use(auth.APIAuth(group)) m.Use(context.ToggleAPI(&context.ToggleOptions{ SignInRequired: setting.Service.RequireSignInView, diff --git a/routers/api/v1/repo/fork.go b/routers/api/v1/repo/fork.go index e4c7eb7041..5b564a8066 100644 --- a/routers/api/v1/repo/fork.go +++ b/routers/api/v1/repo/fork.go @@ -108,7 +108,7 @@ func CreateFork(ctx *context.APIContext) { if form.Organization == nil { forker = ctx.Doer } else { - org, err := organization.GetOrgByName(*form.Organization) + org, err := organization.GetOrgByName(ctx, *form.Organization) if err != nil { if organization.IsErrOrgNotExist(err) { ctx.Error(http.StatusUnprocessableEntity, "", err) diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index e0cae5f82c..1426d1dbcc 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -468,7 +468,7 @@ func CreateOrgRepo(ctx *context.APIContext) { // "403": // "$ref": "#/responses/forbidden" opt := web.GetForm(ctx).(*api.CreateRepoOption) - org, err := organization.GetOrgByName(ctx.Params(":org")) + org, err := organization.GetOrgByName(ctx, ctx.Params(":org")) if err != nil { if organization.IsErrOrgNotExist(err) { ctx.Error(http.StatusUnprocessableEntity, "", err) diff --git a/routers/web/admin/auths.go b/routers/web/admin/auths.go index 1bc166902c..8ce45720fe 100644 --- a/routers/web/admin/auths.go +++ b/routers/web/admin/auths.go @@ -204,6 +204,8 @@ func parseOAuth2Config(form forms.AuthenticationForm) *oauth2.Source { GroupClaimName: form.Oauth2GroupClaimName, RestrictedGroup: form.Oauth2RestrictedGroup, AdminGroup: form.Oauth2AdminGroup, + GroupTeamMap: form.Oauth2GroupTeamMap, + GroupTeamMapRemoval: form.Oauth2GroupTeamMapRemoval, } } diff --git a/routers/web/auth/linkaccount.go b/routers/web/auth/linkaccount.go index 6c409c6b9d..47a0daa06d 100644 --- a/routers/web/auth/linkaccount.go +++ b/routers/web/auth/linkaccount.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" auth_service "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/forms" @@ -267,5 +268,11 @@ func LinkAccountPostRegister(ctx *context.Context) { return } + source := authSource.Cfg.(*oauth2.Source) + if err := syncGroupsToTeams(ctx, source, &gothUser, u); err != nil { + ctx.ServerError("SyncGroupsToTeams", err) + return + } + handleSignIn(ctx, u, false) } diff --git a/routers/web/auth/oauth.go b/routers/web/auth/oauth.go index be60a0c73b..a11417da16 100644 --- a/routers/web/auth/oauth.go +++ b/routers/web/auth/oauth.go @@ -17,7 +17,9 @@ import ( "code.gitea.io/gitea/models/auth" org_model "code.gitea.io/gitea/models/organization" user_model "code.gitea.io/gitea/models/user" + auth_module "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -27,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web/middleware" auth_service "code.gitea.io/gitea/services/auth" + source_service "code.gitea.io/gitea/services/auth/source" "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/forms" @@ -963,12 +966,19 @@ func SignInOAuthCallback(ctx *context.Context) { IsActive: util.OptionalBoolOf(!setting.OAuth2Client.RegisterEmailConfirm), } - setUserGroupClaims(authSource, u, &gothUser) + source := authSource.Cfg.(*oauth2.Source) + + setUserAdminAndRestrictedFromGroupClaims(source, u, &gothUser) if !createAndHandleCreatedUser(ctx, base.TplName(""), nil, u, overwriteDefault, &gothUser, setting.OAuth2Client.AccountLinking != setting.OAuth2AccountLinkingDisabled) { // error already handled return } + + if err := syncGroupsToTeams(ctx, source, &gothUser, u); err != nil { + ctx.ServerError("SyncGroupsToTeams", err) + return + } } else { // no existing user is found, request attach or new account showLinkingLogin(ctx, gothUser) @@ -979,7 +989,7 @@ func SignInOAuthCallback(ctx *context.Context) { handleOAuth2SignIn(ctx, authSource, u, gothUser) } -func claimValueToStringSlice(claimValue interface{}) []string { +func claimValueToStringSet(claimValue interface{}) container.Set[string] { var groups []string switch rawGroup := claimValue.(type) { @@ -993,37 +1003,45 @@ func claimValueToStringSlice(claimValue interface{}) []string { str := fmt.Sprintf("%s", rawGroup) groups = strings.Split(str, ",") } - return groups + return container.SetOf(groups...) } -func setUserGroupClaims(loginSource *auth.Source, u *user_model.User, gothUser *goth.User) bool { - source := loginSource.Cfg.(*oauth2.Source) - if source.GroupClaimName == "" || (source.AdminGroup == "" && source.RestrictedGroup == "") { - return false +func syncGroupsToTeams(ctx *context.Context, source *oauth2.Source, gothUser *goth.User, u *user_model.User) error { + if source.GroupTeamMap != "" || source.GroupTeamMapRemoval { + groupTeamMapping, err := auth_module.UnmarshalGroupTeamMapping(source.GroupTeamMap) + if err != nil { + return err + } + + groups := getClaimedGroups(source, gothUser) + + if err := source_service.SyncGroupsToTeams(ctx, u, groups, groupTeamMapping, source.GroupTeamMapRemoval); err != nil { + return err + } } + return nil +} + +func getClaimedGroups(source *oauth2.Source, gothUser *goth.User) container.Set[string] { groupClaims, has := gothUser.RawData[source.GroupClaimName] if !has { - return false + return nil } - groups := claimValueToStringSlice(groupClaims) + return claimValueToStringSet(groupClaims) +} + +func setUserAdminAndRestrictedFromGroupClaims(source *oauth2.Source, u *user_model.User, gothUser *goth.User) bool { + groups := getClaimedGroups(source, gothUser) wasAdmin, wasRestricted := u.IsAdmin, u.IsRestricted if source.AdminGroup != "" { - u.IsAdmin = false + u.IsAdmin = groups.Contains(source.AdminGroup) } if source.RestrictedGroup != "" { - u.IsRestricted = false - } - - for _, g := range groups { - if source.AdminGroup != "" && g == source.AdminGroup { - u.IsAdmin = true - } else if source.RestrictedGroup != "" && g == source.RestrictedGroup { - u.IsRestricted = true - } + u.IsRestricted = groups.Contains(source.RestrictedGroup) } return wasAdmin != u.IsAdmin || wasRestricted != u.IsRestricted @@ -1070,6 +1088,15 @@ func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model needs2FA = err == nil } + oauth2Source := source.Cfg.(*oauth2.Source) + groupTeamMapping, err := auth_module.UnmarshalGroupTeamMapping(oauth2Source.GroupTeamMap) + if err != nil { + ctx.ServerError("UnmarshalGroupTeamMapping", err) + return + } + + groups := getClaimedGroups(oauth2Source, &gothUser) + // If this user is enrolled in 2FA and this source doesn't override it, // we can't sign the user in just yet. Instead, redirect them to the 2FA authentication page. if !needs2FA { @@ -1088,7 +1115,7 @@ func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model u.SetLastLogin() // Update GroupClaims - changed := setUserGroupClaims(source, u, &gothUser) + changed := setUserAdminAndRestrictedFromGroupClaims(oauth2Source, u, &gothUser) cols := []string{"last_login_unix"} if changed { cols = append(cols, "is_admin", "is_restricted") @@ -1099,6 +1126,13 @@ func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model return } + if oauth2Source.GroupTeamMap != "" || oauth2Source.GroupTeamMapRemoval { + if err := source_service.SyncGroupsToTeams(ctx, u, groups, groupTeamMapping, oauth2Source.GroupTeamMapRemoval); err != nil { + ctx.ServerError("SyncGroupsToTeams", err) + return + } + } + // update external user information if err := externalaccount.UpdateExternalUser(u, gothUser); err != nil { if !errors.Is(err, util.ErrNotExist) { @@ -1121,7 +1155,7 @@ func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model return } - changed := setUserGroupClaims(source, u, &gothUser) + changed := setUserAdminAndRestrictedFromGroupClaims(oauth2Source, u, &gothUser) if changed { if err := user_model.UpdateUserCols(ctx, u, "is_admin", "is_restricted"); err != nil { ctx.ServerError("UpdateUserCols", err) @@ -1129,6 +1163,13 @@ func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model } } + if oauth2Source.GroupTeamMap != "" || oauth2Source.GroupTeamMapRemoval { + if err := source_service.SyncGroupsToTeams(ctx, u, groups, groupTeamMapping, oauth2Source.GroupTeamMapRemoval); err != nil { + ctx.ServerError("SyncGroupsToTeams", err) + return + } + } + if err := updateSession(ctx, nil, map[string]interface{}{ // User needs to use 2FA, save data and redirect to 2FA page. "twofaUid": u.ID, @@ -1188,15 +1229,9 @@ func oAuth2UserLoginCallback(authSource *auth.Source, request *http.Request, res } if oauth2Source.RequiredClaimValue != "" { - groups := claimValueToStringSlice(claimInterface) - found := false - for _, group := range groups { - if group == oauth2Source.RequiredClaimValue { - found = true - break - } - } - if !found { + groups := claimValueToStringSet(claimInterface) + + if !groups.Contains(oauth2Source.RequiredClaimValue) { return nil, goth.User{}, user_model.ErrUserProhibitLogin{Name: gothUser.UserID} } } diff --git a/routers/web/repo/issue_label.go b/routers/web/repo/issue_label.go index 01421dc927..66e8920bd9 100644 --- a/routers/web/repo/issue_label.go +++ b/routers/web/repo/issue_label.go @@ -78,7 +78,7 @@ func RetrieveLabels(ctx *context.Context) { } ctx.Data["OrgLabels"] = orgLabels - org, err := organization.GetOrgByName(ctx.Repo.Owner.LowerName) + org, err := organization.GetOrgByName(ctx, ctx.Repo.Owner.LowerName) if err != nil { ctx.ServerError("GetOrgByName", err) return diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 2cc263e5bb..5c30795f22 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -1006,7 +1006,7 @@ func AddTeamPost(ctx *context.Context) { return } - team, err := organization.OrgFromUser(ctx.Repo.Owner).GetTeam(name) + team, err := organization.OrgFromUser(ctx.Repo.Owner).GetTeam(ctx, name) if err != nil { if organization.IsErrTeamNotExist(err) { ctx.Flash.Error(ctx.Tr("form.team_not_exist")) diff --git a/routers/web/web.go b/routers/web/web.go index 6898956053..88e27ad678 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -203,7 +203,7 @@ func Routes(ctx gocontext.Context) *web.Route { } // Get user from session if logged in. - common = append(common, context.Auth(group)) + common = append(common, auth_service.Auth(group)) // GetHead allows a HEAD request redirect to GET if HEAD method is not defined for that route common = append(common, middleware.GetHead) diff --git a/services/auth/middleware.go b/services/auth/middleware.go new file mode 100644 index 0000000000..cccaab2998 --- /dev/null +++ b/services/auth/middleware.go @@ -0,0 +1,60 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package auth + +import ( + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/web/middleware" +) + +// Auth is a middleware to authenticate a web user +func Auth(authMethod Method) func(*context.Context) { + return func(ctx *context.Context) { + if err := authShared(ctx, authMethod); err != nil { + log.Error("Failed to verify user: %v", err) + ctx.Error(http.StatusUnauthorized, "Verify") + return + } + if ctx.Doer == nil { + // ensure the session uid is deleted + _ = ctx.Session.Delete("uid") + } + } +} + +// APIAuth is a middleware to authenticate an api user +func APIAuth(authMethod Method) func(*context.APIContext) { + return func(ctx *context.APIContext) { + if err := authShared(ctx.Context, authMethod); err != nil { + ctx.Error(http.StatusUnauthorized, "APIAuth", err) + } + } +} + +func authShared(ctx *context.Context, authMethod Method) error { + var err error + ctx.Doer, err = authMethod.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) + if err != nil { + return err + } + if ctx.Doer != nil { + if ctx.Locale.Language() != ctx.Doer.Language { + ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) + } + ctx.IsBasicAuth = ctx.Data["AuthedMethod"].(string) == BasicMethodName + ctx.IsSigned = true + ctx.Data["IsSigned"] = ctx.IsSigned + ctx.Data["SignedUser"] = ctx.Doer + ctx.Data["SignedUserID"] = ctx.Doer.ID + ctx.Data["SignedUserName"] = ctx.Doer.Name + ctx.Data["IsAdmin"] = ctx.Doer.IsAdmin + } else { + ctx.Data["SignedUserID"] = int64(0) + ctx.Data["SignedUserName"] = "" + } + return nil +} diff --git a/services/auth/source/ldap/source_authenticate.go b/services/auth/source/ldap/source_authenticate.go index 321cf5540d..fba8da7934 100644 --- a/services/auth/source/ldap/source_authenticate.go +++ b/services/auth/source/ldap/source_authenticate.go @@ -10,9 +10,10 @@ import ( asymkey_model "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/organization" user_model "code.gitea.io/gitea/models/user" + auth_module "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/util" + source_service "code.gitea.io/gitea/services/auth/source" "code.gitea.io/gitea/services/mailer" user_service "code.gitea.io/gitea/services/user" ) @@ -64,61 +65,66 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str } if user != nil { - if source.GroupsEnabled && (source.GroupTeamMap != "" || source.GroupTeamMapRemoval) { - orgCache := make(map[string]*organization.Organization) - teamCache := make(map[string]*organization.Team) - source.SyncLdapGroupsToTeams(user, sr.LdapTeamAdd, sr.LdapTeamRemove, orgCache, teamCache) - } if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(user, source.authSource, sr.SSHPublicKey) { - return user, asymkey_model.RewriteAllPublicKeys() + if err := asymkey_model.RewriteAllPublicKeys(); err != nil { + return user, err + } + } + } else { + // Fallback. + if len(sr.Username) == 0 { + sr.Username = userName + } + + if len(sr.Mail) == 0 { + sr.Mail = fmt.Sprintf("%s@localhost", sr.Username) + } + + user = &user_model.User{ + LowerName: strings.ToLower(sr.Username), + Name: sr.Username, + FullName: composeFullName(sr.Name, sr.Surname, sr.Username), + Email: sr.Mail, + LoginType: source.authSource.Type, + LoginSource: source.authSource.ID, + LoginName: userName, + IsAdmin: sr.IsAdmin, + } + overwriteDefault := &user_model.CreateUserOverwriteOptions{ + IsRestricted: util.OptionalBoolOf(sr.IsRestricted), + IsActive: util.OptionalBoolTrue, + } + + err := user_model.CreateUser(user, overwriteDefault) + if err != nil { + return user, err + } + + mailer.SendRegisterNotifyMail(user) + + if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(user, source.authSource, sr.SSHPublicKey) { + if err := asymkey_model.RewriteAllPublicKeys(); err != nil { + return user, err + } + } + if len(source.AttributeAvatar) > 0 { + if err := user_service.UploadAvatar(user, sr.Avatar); err != nil { + return user, err + } } - return user, nil } - // Fallback. - if len(sr.Username) == 0 { - sr.Username = userName - } - - if len(sr.Mail) == 0 { - sr.Mail = fmt.Sprintf("%s@localhost", sr.Username) - } - - user = &user_model.User{ - LowerName: strings.ToLower(sr.Username), - Name: sr.Username, - FullName: composeFullName(sr.Name, sr.Surname, sr.Username), - Email: sr.Mail, - LoginType: source.authSource.Type, - LoginSource: source.authSource.ID, - LoginName: userName, - IsAdmin: sr.IsAdmin, - } - overwriteDefault := &user_model.CreateUserOverwriteOptions{ - IsRestricted: util.OptionalBoolOf(sr.IsRestricted), - IsActive: util.OptionalBoolTrue, - } - - err := user_model.CreateUser(user, overwriteDefault) - if err != nil { - return user, err - } - - mailer.SendRegisterNotifyMail(user) - - if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(user, source.authSource, sr.SSHPublicKey) { - err = asymkey_model.RewriteAllPublicKeys() - } - if err == nil && len(source.AttributeAvatar) > 0 { - _ = user_service.UploadAvatar(user, sr.Avatar) - } if source.GroupsEnabled && (source.GroupTeamMap != "" || source.GroupTeamMapRemoval) { - orgCache := make(map[string]*organization.Organization) - teamCache := make(map[string]*organization.Team) - source.SyncLdapGroupsToTeams(user, sr.LdapTeamAdd, sr.LdapTeamRemove, orgCache, teamCache) + groupTeamMapping, err := auth_module.UnmarshalGroupTeamMapping(source.GroupTeamMap) + if err != nil { + return user, err + } + if err := source_service.SyncGroupsToTeams(db.DefaultContext, user, sr.Groups, groupTeamMapping, source.GroupTeamMapRemoval); err != nil { + return user, err + } } - return user, err + return user, nil } // IsSkipLocalTwoFA returns if this source should skip local 2fa for password authentication diff --git a/services/auth/source/ldap/source_group_sync.go b/services/auth/source/ldap/source_group_sync.go deleted file mode 100644 index 95a6084922..0000000000 --- a/services/auth/source/ldap/source_group_sync.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package ldap - -import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/organization" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" -) - -// SyncLdapGroupsToTeams maps LDAP groups to organization and team memberships -func (source *Source) SyncLdapGroupsToTeams(user *user_model.User, ldapTeamAdd, ldapTeamRemove map[string][]string, orgCache map[string]*organization.Organization, teamCache map[string]*organization.Team) { - var err error - if source.GroupsEnabled && source.GroupTeamMapRemoval { - // when the user is not a member of configs LDAP group, remove mapped organizations/teams memberships - removeMappedMemberships(user, ldapTeamRemove, orgCache, teamCache) - } - for orgName, teamNames := range ldapTeamAdd { - org, ok := orgCache[orgName] - if !ok { - org, err = organization.GetOrgByName(orgName) - if err != nil { - // organization must be created before LDAP group sync - log.Warn("LDAP group sync: Could not find organisation %s: %v", orgName, err) - continue - } - orgCache[orgName] = org - } - - for _, teamName := range teamNames { - team, ok := teamCache[orgName+teamName] - if !ok { - team, err = org.GetTeam(teamName) - if err != nil { - // team must be created before LDAP group sync - log.Warn("LDAP group sync: Could not find team %s: %v", teamName, err) - continue - } - teamCache[orgName+teamName] = team - } - if isMember, err := organization.IsTeamMember(db.DefaultContext, org.ID, team.ID, user.ID); !isMember && err == nil { - log.Trace("LDAP group sync: adding user [%s] to team [%s]", user.Name, org.Name) - } else { - continue - } - err := models.AddTeamMember(team, user.ID) - if err != nil { - log.Error("LDAP group sync: Could not add user to team: %v", err) - } - } - } -} - -// remove membership to organizations/teams if user is not member of corresponding LDAP group -// e.g. lets assume user is member of LDAP group "x", but LDAP group team map contains LDAP groups "x" and "y" -// then users membership gets removed for all organizations/teams mapped by LDAP group "y" -func removeMappedMemberships(user *user_model.User, ldapTeamRemove map[string][]string, orgCache map[string]*organization.Organization, teamCache map[string]*organization.Team) { - var err error - for orgName, teamNames := range ldapTeamRemove { - org, ok := orgCache[orgName] - if !ok { - org, err = organization.GetOrgByName(orgName) - if err != nil { - // organization must be created before LDAP group sync - log.Warn("LDAP group sync: Could not find organisation %s: %v", orgName, err) - continue - } - orgCache[orgName] = org - } - for _, teamName := range teamNames { - team, ok := teamCache[orgName+teamName] - if !ok { - team, err = org.GetTeam(teamName) - if err != nil { - // team must must be created before LDAP group sync - log.Warn("LDAP group sync: Could not find team %s: %v", teamName, err) - continue - } - } - if isMember, err := organization.IsTeamMember(db.DefaultContext, org.ID, team.ID, user.ID); isMember && err == nil { - log.Trace("LDAP group sync: removing user [%s] from team [%s]", user.Name, org.Name) - } else { - continue - } - err = models.RemoveTeamMember(team, user.ID) - if err != nil { - log.Error("LDAP group sync: Could not remove user from team: %v", err) - } - } - } -} diff --git a/services/auth/source/ldap/source_search.go b/services/auth/source/ldap/source_search.go index 16f13029f9..5a2d25b0c4 100644 --- a/services/auth/source/ldap/source_search.go +++ b/services/auth/source/ldap/source_search.go @@ -11,26 +11,24 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" "github.com/go-ldap/ldap/v3" ) // SearchResult : user data type SearchResult struct { - Username string // Username - Name string // Name - Surname string // Surname - Mail string // E-mail address - SSHPublicKey []string // SSH Public Key - IsAdmin bool // if user is administrator - IsRestricted bool // if user is restricted - LowerName string // LowerName - Avatar []byte - LdapTeamAdd map[string][]string // organizations teams to add - LdapTeamRemove map[string][]string // organizations teams to remove + Username string // Username + Name string // Name + Surname string // Surname + Mail string // E-mail address + SSHPublicKey []string // SSH Public Key + IsAdmin bool // if user is administrator + IsRestricted bool // if user is restricted + LowerName string // LowerName + Avatar []byte + Groups container.Set[string] } func (source *Source) sanitizedUserQuery(username string) (string, bool) { @@ -196,9 +194,8 @@ func checkRestricted(l *ldap.Conn, ls *Source, userDN string) bool { } // List all group memberships of a user -func (source *Source) listLdapGroupMemberships(l *ldap.Conn, uid string, applyGroupFilter bool) []string { - var ldapGroups []string - var searchFilter string +func (source *Source) listLdapGroupMemberships(l *ldap.Conn, uid string, applyGroupFilter bool) container.Set[string] { + ldapGroups := make(container.Set[string]) groupFilter, ok := source.sanitizedGroupFilter(source.GroupFilter) if !ok { @@ -210,12 +207,12 @@ func (source *Source) listLdapGroupMemberships(l *ldap.Conn, uid string, applyGr return ldapGroups } + var searchFilter string if applyGroupFilter { searchFilter = fmt.Sprintf("(&(%s)(%s=%s))", groupFilter, source.GroupMemberUID, ldap.EscapeFilter(uid)) } else { searchFilter = fmt.Sprintf("(%s=%s)", source.GroupMemberUID, ldap.EscapeFilter(uid)) } - result, err := l.Search(ldap.NewSearchRequest( groupDN, ldap.ScopeWholeSubtree, @@ -237,44 +234,12 @@ func (source *Source) listLdapGroupMemberships(l *ldap.Conn, uid string, applyGr log.Error("LDAP search was successful, but found no DN!") continue } - ldapGroups = append(ldapGroups, entry.DN) + ldapGroups.Add(entry.DN) } return ldapGroups } -// parse LDAP groups and return map of ldap groups to organizations teams -func (source *Source) mapLdapGroupsToTeams() map[string]map[string][]string { - ldapGroupsToTeams := make(map[string]map[string][]string) - err := json.Unmarshal([]byte(source.GroupTeamMap), &ldapGroupsToTeams) - if err != nil { - log.Error("Failed to unmarshall LDAP teams map: %v", err) - return ldapGroupsToTeams - } - return ldapGroupsToTeams -} - -// getMappedMemberships : returns the organizations and teams to modify the users membership -func (source *Source) getMappedMemberships(usersLdapGroups []string, uid string) (map[string][]string, map[string][]string) { - // unmarshall LDAP group team map from configs - ldapGroupsToTeams := source.mapLdapGroupsToTeams() - membershipsToAdd := map[string][]string{} - membershipsToRemove := map[string][]string{} - for group, memberships := range ldapGroupsToTeams { - isUserInGroup := util.SliceContainsString(usersLdapGroups, group) - if isUserInGroup { - for org, teams := range memberships { - membershipsToAdd[org] = teams - } - } else if !isUserInGroup { - for org, teams := range memberships { - membershipsToRemove[org] = teams - } - } - } - return membershipsToAdd, membershipsToRemove -} - func (source *Source) getUserAttributeListedInGroup(entry *ldap.Entry) string { if strings.ToLower(source.UserUID) == "dn" { return entry.DN @@ -399,23 +364,6 @@ func (source *Source) SearchEntry(name, passwd string, directBind bool) *SearchR surname := sr.Entries[0].GetAttributeValue(source.AttributeSurname) mail := sr.Entries[0].GetAttributeValue(source.AttributeMail) - teamsToAdd := make(map[string][]string) - teamsToRemove := make(map[string][]string) - - // Check group membership - if source.GroupsEnabled { - userAttributeListedInGroup := source.getUserAttributeListedInGroup(sr.Entries[0]) - usersLdapGroups := source.listLdapGroupMemberships(l, userAttributeListedInGroup, true) - - if source.GroupFilter != "" && len(usersLdapGroups) == 0 { - return nil - } - - if source.GroupTeamMap != "" || source.GroupTeamMapRemoval { - teamsToAdd, teamsToRemove = source.getMappedMemberships(usersLdapGroups, userAttributeListedInGroup) - } - } - if isAttributeSSHPublicKeySet { sshPublicKey = sr.Entries[0].GetAttributeValues(source.AttributeSSHPublicKey) } @@ -431,6 +379,17 @@ func (source *Source) SearchEntry(name, passwd string, directBind bool) *SearchR Avatar = sr.Entries[0].GetRawAttributeValue(source.AttributeAvatar) } + // Check group membership + var usersLdapGroups container.Set[string] + if source.GroupsEnabled { + userAttributeListedInGroup := source.getUserAttributeListedInGroup(sr.Entries[0]) + usersLdapGroups = source.listLdapGroupMemberships(l, userAttributeListedInGroup, true) + + if source.GroupFilter != "" && len(usersLdapGroups) == 0 { + return nil + } + } + if !directBind && source.AttributesInBind { // binds user (checking password) after looking-up attributes in BindDN context err = bindUser(l, userDN, passwd) @@ -440,17 +399,16 @@ func (source *Source) SearchEntry(name, passwd string, directBind bool) *SearchR } return &SearchResult{ - LowerName: strings.ToLower(username), - Username: username, - Name: firstname, - Surname: surname, - Mail: mail, - SSHPublicKey: sshPublicKey, - IsAdmin: isAdmin, - IsRestricted: isRestricted, - Avatar: Avatar, - LdapTeamAdd: teamsToAdd, - LdapTeamRemove: teamsToRemove, + LowerName: strings.ToLower(username), + Username: username, + Name: firstname, + Surname: surname, + Mail: mail, + SSHPublicKey: sshPublicKey, + IsAdmin: isAdmin, + IsRestricted: isRestricted, + Avatar: Avatar, + Groups: usersLdapGroups, } } @@ -512,33 +470,29 @@ func (source *Source) SearchEntries() ([]*SearchResult, error) { result := make([]*SearchResult, 0, len(sr.Entries)) for _, v := range sr.Entries { - teamsToAdd := make(map[string][]string) - teamsToRemove := make(map[string][]string) - + var usersLdapGroups container.Set[string] if source.GroupsEnabled { userAttributeListedInGroup := source.getUserAttributeListedInGroup(v) if source.GroupFilter != "" { - usersLdapGroups := source.listLdapGroupMemberships(l, userAttributeListedInGroup, true) + usersLdapGroups = source.listLdapGroupMemberships(l, userAttributeListedInGroup, true) if len(usersLdapGroups) == 0 { continue } } if source.GroupTeamMap != "" || source.GroupTeamMapRemoval { - usersLdapGroups := source.listLdapGroupMemberships(l, userAttributeListedInGroup, false) - teamsToAdd, teamsToRemove = source.getMappedMemberships(usersLdapGroups, userAttributeListedInGroup) + usersLdapGroups = source.listLdapGroupMemberships(l, userAttributeListedInGroup, false) } } user := &SearchResult{ - Username: v.GetAttributeValue(source.AttributeUsername), - Name: v.GetAttributeValue(source.AttributeName), - Surname: v.GetAttributeValue(source.AttributeSurname), - Mail: v.GetAttributeValue(source.AttributeMail), - IsAdmin: checkAdmin(l, source, v.DN), - LdapTeamAdd: teamsToAdd, - LdapTeamRemove: teamsToRemove, + Username: v.GetAttributeValue(source.AttributeUsername), + Name: v.GetAttributeValue(source.AttributeName), + Surname: v.GetAttributeValue(source.AttributeSurname), + Mail: v.GetAttributeValue(source.AttributeMail), + IsAdmin: checkAdmin(l, source, v.DN), + Groups: usersLdapGroups, } if !user.IsAdmin { diff --git a/services/auth/source/ldap/source_sync.go b/services/auth/source/ldap/source_sync.go index 73e8309aca..4571ff6540 100644 --- a/services/auth/source/ldap/source_sync.go +++ b/services/auth/source/ldap/source_sync.go @@ -13,8 +13,10 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/organization" user_model "code.gitea.io/gitea/models/user" + auth_module "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" + source_service "code.gitea.io/gitea/services/auth/source" user_service "code.gitea.io/gitea/services/user" ) @@ -65,6 +67,11 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { orgCache := make(map[string]*organization.Organization) teamCache := make(map[string]*organization.Team) + groupTeamMapping, err := auth_module.UnmarshalGroupTeamMapping(source.GroupTeamMap) + if err != nil { + return err + } + for _, su := range sr { select { case <-ctx.Done(): @@ -173,7 +180,9 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } // Synchronize LDAP groups with organization and team memberships if source.GroupsEnabled && (source.GroupTeamMap != "" || source.GroupTeamMapRemoval) { - source.SyncLdapGroupsToTeams(usr, su.LdapTeamAdd, su.LdapTeamRemove, orgCache, teamCache) + if err := source_service.SyncGroupsToTeamsCached(ctx, usr, su.Groups, groupTeamMapping, source.GroupTeamMapRemoval, orgCache, teamCache); err != nil { + log.Error("SyncGroupsToTeamsCached: %v", err) + } } } diff --git a/services/auth/source/oauth2/source.go b/services/auth/source/oauth2/source.go index 0abebc04ec..675005e55a 100644 --- a/services/auth/source/oauth2/source.go +++ b/services/auth/source/oauth2/source.go @@ -8,13 +8,6 @@ import ( "code.gitea.io/gitea/modules/json" ) -// ________ _____ __ .__ ________ -// \_____ \ / _ \ __ ___/ |_| |__ \_____ \ -// / | \ / /_\ \| | \ __\ | \ / ____/ -// / | \/ | \ | /| | | Y \/ \ -// \_______ /\____|__ /____/ |__| |___| /\_______ \ -// \/ \/ \/ \/ - // Source holds configuration for the OAuth2 login source. type Source struct { Provider string @@ -24,13 +17,15 @@ type Source struct { CustomURLMapping *CustomURLMapping IconURL string - Scopes []string - RequiredClaimName string - RequiredClaimValue string - GroupClaimName string - AdminGroup string - RestrictedGroup string - SkipLocalTwoFA bool `json:",omitempty"` + Scopes []string + RequiredClaimName string + RequiredClaimValue string + GroupClaimName string + AdminGroup string + GroupTeamMap string + GroupTeamMapRemoval bool + RestrictedGroup string + SkipLocalTwoFA bool `json:",omitempty"` // reference to the authSource authSource *auth.Source diff --git a/services/auth/source/source_group_sync.go b/services/auth/source/source_group_sync.go new file mode 100644 index 0000000000..20b6095345 --- /dev/null +++ b/services/auth/source/source_group_sync.go @@ -0,0 +1,116 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package source + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/organization" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/log" +) + +type syncType int + +const ( + syncAdd syncType = iota + syncRemove +) + +// SyncGroupsToTeams maps authentication source groups to organization and team memberships +func SyncGroupsToTeams(ctx context.Context, user *user_model.User, sourceUserGroups container.Set[string], sourceGroupTeamMapping map[string]map[string][]string, performRemoval bool) error { + orgCache := make(map[string]*organization.Organization) + teamCache := make(map[string]*organization.Team) + return SyncGroupsToTeamsCached(ctx, user, sourceUserGroups, sourceGroupTeamMapping, performRemoval, orgCache, teamCache) +} + +// SyncGroupsToTeamsCached maps authentication source groups to organization and team memberships +func SyncGroupsToTeamsCached(ctx context.Context, user *user_model.User, sourceUserGroups container.Set[string], sourceGroupTeamMapping map[string]map[string][]string, performRemoval bool, orgCache map[string]*organization.Organization, teamCache map[string]*organization.Team) error { + membershipsToAdd, membershipsToRemove := resolveMappedMemberships(sourceUserGroups, sourceGroupTeamMapping) + + if performRemoval { + if err := syncGroupsToTeamsCached(ctx, user, membershipsToRemove, syncRemove, orgCache, teamCache); err != nil { + return fmt.Errorf("could not sync[remove] user groups: %w", err) + } + } + + if err := syncGroupsToTeamsCached(ctx, user, membershipsToAdd, syncAdd, orgCache, teamCache); err != nil { + return fmt.Errorf("could not sync[add] user groups: %w", err) + } + + return nil +} + +func resolveMappedMemberships(sourceUserGroups container.Set[string], sourceGroupTeamMapping map[string]map[string][]string) (map[string][]string, map[string][]string) { + membershipsToAdd := map[string][]string{} + membershipsToRemove := map[string][]string{} + for group, memberships := range sourceGroupTeamMapping { + isUserInGroup := sourceUserGroups.Contains(group) + if isUserInGroup { + for org, teams := range memberships { + membershipsToAdd[org] = teams + } + } else { + for org, teams := range memberships { + membershipsToRemove[org] = teams + } + } + } + return membershipsToAdd, membershipsToRemove +} + +func syncGroupsToTeamsCached(ctx context.Context, user *user_model.User, orgTeamMap map[string][]string, action syncType, orgCache map[string]*organization.Organization, teamCache map[string]*organization.Team) error { + for orgName, teamNames := range orgTeamMap { + var err error + org, ok := orgCache[orgName] + if !ok { + org, err = organization.GetOrgByName(ctx, orgName) + if err != nil { + if organization.IsErrOrgNotExist(err) { + // organization must be created before group sync + log.Warn("group sync: Could not find organisation %s: %v", orgName, err) + continue + } + return err + } + orgCache[orgName] = org + } + for _, teamName := range teamNames { + team, ok := teamCache[orgName+teamName] + if !ok { + team, err = org.GetTeam(ctx, teamName) + if err != nil { + if organization.IsErrTeamNotExist(err) { + // team must be created before group sync + log.Warn("group sync: Could not find team %s: %v", teamName, err) + continue + } + return err + } + teamCache[orgName+teamName] = team + } + + isMember, err := organization.IsTeamMember(ctx, org.ID, team.ID, user.ID) + if err != nil { + return err + } + + if action == syncAdd && !isMember { + if err := models.AddTeamMember(team, user.ID); err != nil { + log.Error("group sync: Could not add user to team: %v", err) + return err + } + } else if action == syncRemove && isMember { + if err := models.RemoveTeamMember(team, user.ID); err != nil { + log.Error("group sync: Could not remove user from team: %v", err) + return err + } + } + } + } + return nil +} diff --git a/services/forms/auth_form.go b/services/forms/auth_form.go index 0cede07f95..5625aa1e2e 100644 --- a/services/forms/auth_form.go +++ b/services/forms/auth_form.go @@ -72,13 +72,15 @@ type AuthenticationForm struct { Oauth2GroupClaimName string Oauth2AdminGroup string Oauth2RestrictedGroup string + Oauth2GroupTeamMap string `binding:"ValidGroupTeamMap"` + Oauth2GroupTeamMapRemoval bool SkipLocalTwoFA bool SSPIAutoCreateUsers bool SSPIAutoActivateUsers bool SSPIStripDomainNames bool SSPISeparatorReplacement string `binding:"AlphaDashDot;MaxSize(5)"` SSPIDefaultLanguage string - GroupTeamMap string + GroupTeamMap string `binding:"ValidGroupTeamMap"` GroupTeamMapRemoval bool } diff --git a/templates/admin/auth/edit.tmpl b/templates/admin/auth/edit.tmpl index 91e4b1df52..a3c94a6cc2 100644 --- a/templates/admin/auth/edit.tmpl +++ b/templates/admin/auth/edit.tmpl @@ -361,6 +361,14 @@
      +
      + + +
      +
      + + +
      {{end}} diff --git a/templates/admin/auth/source/ldap.tmpl b/templates/admin/auth/source/ldap.tmpl index b44eb799b9..8d199854ae 100644 --- a/templates/admin/auth/source/ldap.tmpl +++ b/templates/admin/auth/source/ldap.tmpl @@ -52,7 +52,7 @@
      - +

      {{.locale.Tr "admin.auths.restricted_filter_helper"}}

      diff --git a/templates/admin/auth/source/oauth.tmpl b/templates/admin/auth/source/oauth.tmpl index 166373a324..85c77343a5 100644 --- a/templates/admin/auth/source/oauth.tmpl +++ b/templates/admin/auth/source/oauth.tmpl @@ -98,4 +98,12 @@
      +
      + + +
      +
      + + +
    diff --git a/tests/integration/auth_ldap_test.go b/tests/integration/auth_ldap_test.go index 9dfa4c73d8..883c9d80a3 100644 --- a/tests/integration/auth_ldap_test.go +++ b/tests/integration/auth_ldap_test.go @@ -112,23 +112,14 @@ func getLDAPServerPort() string { return port } -func addAuthSourceLDAP(t *testing.T, sshKeyAttribute, groupFilter string, groupMapParams ...string) { - groupTeamMapRemoval := "off" - groupTeamMap := "" - if len(groupMapParams) == 2 { - groupTeamMapRemoval = groupMapParams[0] - groupTeamMap = groupMapParams[1] - } - +func buildAuthSourceLDAPPayload(csrf, sshKeyAttribute, groupFilter, groupTeamMap, groupTeamMapRemoval string) map[string]string { // Modify user filter to test group filter explicitly userFilter := "(&(objectClass=inetOrgPerson)(memberOf=cn=git,ou=people,dc=planetexpress,dc=com)(uid=%s))" if groupFilter != "" { userFilter = "(&(objectClass=inetOrgPerson)(uid=%s))" } - session := loginUser(t, "user1") - csrf := GetCSRF(t, session, "/admin/auths/new") - req := NewRequestWithValues(t, "POST", "/admin/auths/new", map[string]string{ + return map[string]string{ "_csrf": csrf, "type": "2", "name": "ldap", @@ -154,7 +145,19 @@ func addAuthSourceLDAP(t *testing.T, sshKeyAttribute, groupFilter string, groupM "group_team_map": groupTeamMap, "group_team_map_removal": groupTeamMapRemoval, "user_uid": "DN", - }) + } +} + +func addAuthSourceLDAP(t *testing.T, sshKeyAttribute, groupFilter string, groupMapParams ...string) { + groupTeamMapRemoval := "off" + groupTeamMap := "" + if len(groupMapParams) == 2 { + groupTeamMapRemoval = groupMapParams[0] + groupTeamMap = groupMapParams[1] + } + session := loginUser(t, "user1") + csrf := GetCSRF(t, session, "/admin/auths/new") + req := NewRequestWithValues(t, "POST", "/admin/auths/new", buildAuthSourceLDAPPayload(csrf, sshKeyAttribute, groupFilter, groupTeamMap, groupTeamMapRemoval)) session.MakeRequest(t, req, http.StatusSeeOther) } @@ -202,26 +205,7 @@ func TestLDAPAuthChange(t *testing.T) { binddn, _ := doc.Find(`input[name="bind_dn"]`).Attr("value") assert.Equal(t, binddn, "uid=gitea,ou=service,dc=planetexpress,dc=com") - req = NewRequestWithValues(t, "POST", href, map[string]string{ - "_csrf": csrf, - "type": "2", - "name": "ldap", - "host": getLDAPServerHost(), - "port": "389", - "bind_dn": "uid=gitea,ou=service,dc=planetexpress,dc=com", - "bind_password": "password", - "user_base": "ou=people,dc=planetexpress,dc=com", - "filter": "(&(objectClass=inetOrgPerson)(memberOf=cn=git,ou=people,dc=planetexpress,dc=com)(uid=%s))", - "admin_filter": "(memberOf=cn=admin_staff,ou=people,dc=planetexpress,dc=com)", - "restricted_filter": "(uid=leela)", - "attribute_username": "uid", - "attribute_name": "givenName", - "attribute_surname": "sn", - "attribute_mail": "mail", - "attribute_ssh_public_key": "", - "is_sync_enabled": "on", - "is_active": "on", - }) + req = NewRequestWithValues(t, "POST", href, buildAuthSourceLDAPPayload(csrf, "", "", "", "off")) session.MakeRequest(t, req, http.StatusSeeOther) req = NewRequest(t, "GET", href) @@ -395,7 +379,7 @@ func TestLDAPGroupTeamSyncAddMember(t *testing.T) { } defer tests.PrepareTestEnv(t)() addAuthSourceLDAP(t, "", "", "on", `{"cn=ship_crew,ou=people,dc=planetexpress,dc=com":{"org26": ["team11"]},"cn=admin_staff,ou=people,dc=planetexpress,dc=com": {"non-existent": ["non-existent"]}}`) - org, err := organization.GetOrgByName("org26") + org, err := organization.GetOrgByName(db.DefaultContext, "org26") assert.NoError(t, err) team, err := organization.GetTeam(db.DefaultContext, org.ID, "team11") assert.NoError(t, err) @@ -440,7 +424,7 @@ func TestLDAPGroupTeamSyncRemoveMember(t *testing.T) { } defer tests.PrepareTestEnv(t)() addAuthSourceLDAP(t, "", "", "on", `{"cn=dispatch,ou=people,dc=planetexpress,dc=com": {"org26": ["team11"]}}`) - org, err := organization.GetOrgByName("org26") + org, err := organization.GetOrgByName(db.DefaultContext, "org26") assert.NoError(t, err) team, err := organization.GetTeam(db.DefaultContext, org.ID, "team11") assert.NoError(t, err) @@ -468,24 +452,15 @@ func TestLDAPGroupTeamSyncRemoveMember(t *testing.T) { assert.False(t, isMember, "User membership should have been removed from team") } -// Login should work even if Team Group Map contains a broken JSON -func TestBrokenLDAPMapUserSignin(t *testing.T) { +func TestLDAPPreventInvalidGroupTeamMap(t *testing.T) { if skipLDAPTests() { t.Skip() return } defer tests.PrepareTestEnv(t)() - addAuthSourceLDAP(t, "", "", "on", `{"NOT_A_VALID_JSON"["MISSING_DOUBLE_POINT"]}`) - u := gitLDAPUsers[0] - - session := loginUserWithPassword(t, u.UserName, u.Password) - req := NewRequest(t, "GET", "/user/settings") - resp := session.MakeRequest(t, req, http.StatusOK) - - htmlDoc := NewHTMLParser(t, resp.Body) - - assert.Equal(t, u.UserName, htmlDoc.GetInputValueByName("name")) - assert.Equal(t, u.FullName, htmlDoc.GetInputValueByName("full_name")) - assert.Equal(t, u.Email, htmlDoc.Find(`label[for="email"]`).Siblings().First().Text()) + session := loginUser(t, "user1") + csrf := GetCSRF(t, session, "/admin/auths/new") + req := NewRequestWithValues(t, "POST", "/admin/auths/new", buildAuthSourceLDAPPayload(csrf, "", "", `{"NOT_A_VALID_JSON"["MISSING_DOUBLE_POINT"]}`, "off")) + session.MakeRequest(t, req, http.StatusOK) // StatusOK = failed, StatusSeeOther = ok } From 7d3c4c3e8a7b3bf505e3115cbf05b8bb7029a7e8 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 8 Feb 2023 15:55:57 +0800 Subject: [PATCH 19/33] Fix rerun button of Actions (#22798) When clicking the return button, the page should be refreshed. However, the browser may cancel the previous fetch request, and it fails to rerun the job. It's easier to reproduce the bug in Safari or Firefox than Chrome for some reason. image image --- web_src/js/components/RepoActionView.vue | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/web_src/js/components/RepoActionView.vue b/web_src/js/components/RepoActionView.vue index 4c7408d1b3..703fe59d81 100644 --- a/web_src/js/components/RepoActionView.vue +++ b/web_src/js/components/RepoActionView.vue @@ -20,7 +20,8 @@ {{ job.name }} - @@ -162,12 +163,14 @@ const sfc = { } }, // rerun a job - rerunJob(idx) { - this.fetch(`${this.run.link}/jobs/${idx}/rerun`); + async rerunJob(idx) { + const jobLink = `${this.run.link}/jobs/${idx}`; + await this.fetchPost(`${jobLink}/rerun`); + window.location.href = jobLink; }, // cancel a run cancelRun() { - this.fetch(`${this.run.link}/cancel`); + this.fetchPost(`${this.run.link}/cancel`); }, createLogLine(line) { @@ -205,7 +208,7 @@ const sfc = { // for example: make cursor=null means the first time to fetch logs, cursor=eof means no more logs, etc return {step: idx, cursor: it.cursor, expanded: it.expanded}; }); - const resp = await this.fetch( + const resp = await this.fetchPost( `${this.actionsURL}/runs/${this.runIndex}/jobs/${this.jobIndex}`, JSON.stringify({logCursors}), ); @@ -245,7 +248,7 @@ const sfc = { } }, - fetch(url, body) { + fetchPost(url, body) { return fetch(url, { method: 'POST', headers: { From 5ae07d4c2f7bd0f58a15292fb93f2e0af45ab351 Mon Sep 17 00:00:00 2001 From: Michal Date: Wed, 8 Feb 2023 18:54:01 +0100 Subject: [PATCH 20/33] include build info in Prometheus metrics (#22819) Related to: https://github.com/go-gitea/gitea/issues/18061 This PR adds build info to the Prometheus metrics. This includes: - goarch: https://pkg.go.dev/runtime#GOARCH - goos: https://pkg.go.dev/runtime#pkg-constants - goversion: https://pkg.go.dev/runtime#Version - gitea version: just exposes the existing code.gitea.io/gitea/modules/setting.AppVer It's a similar approach to what some other Golang projects are doing, e.g. Prometheus: https://github.com/prometheus/common/blob/main/version/info.go example /metrics response from Prometheus: ``` # HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build. # TYPE prometheus_build_info gauge prometheus_build_info{branch="HEAD",goarch="amd64",goos="linux",goversion="go1.19.4",revision="c0d8a56c69014279464c0e15d8bfb0e153af0dab",version="2.41.0"} 1 ``` /metrics response from gitea with this PR: ``` # HELP gitea_build_info Build information # TYPE gitea_build_info gauge gitea_build_info{goarch="amd64",goos="linux",goversion="go1.20",version="2c6cc0b8c"} 1 ``` Signed-off-by: Michal Wasilewski Signed-off-by: Michal Wasilewski --- modules/metrics/collector.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 17f8dd133f..94699c161c 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -4,7 +4,10 @@ package metrics import ( + "runtime" + activities_model "code.gitea.io/gitea/models/activities" + "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" ) @@ -17,6 +20,7 @@ type Collector struct { Accesses *prometheus.Desc Actions *prometheus.Desc Attachments *prometheus.Desc + BuildInfo *prometheus.Desc Comments *prometheus.Desc Follows *prometheus.Desc HookTasks *prometheus.Desc @@ -62,6 +66,16 @@ func NewCollector() Collector { "Number of Attachments", nil, nil, ), + BuildInfo: prometheus.NewDesc( + namespace+"build_info", + "Build information", + []string{ + "goarch", + "goos", + "goversion", + "version", + }, nil, + ), Comments: prometheus.NewDesc( namespace+"comments", "Number of Comments", @@ -195,6 +209,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { ch <- c.Accesses ch <- c.Actions ch <- c.Attachments + ch <- c.BuildInfo ch <- c.Comments ch <- c.Follows ch <- c.HookTasks @@ -241,6 +256,15 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(stats.Counter.Attachment), ) + ch <- prometheus.MustNewConstMetric( + c.BuildInfo, + prometheus.GaugeValue, + 1, + runtime.GOARCH, + runtime.GOOS, + runtime.Version(), + setting.AppVer, + ) ch <- prometheus.MustNewConstMetric( c.Comments, prometheus.GaugeValue, From 4dd7d61ac8c8bff81a0f9e376032f45eb12f51a9 Mon Sep 17 00:00:00 2001 From: John Olheiser Date: Wed, 8 Feb 2023 20:47:52 -0600 Subject: [PATCH 21/33] Load issue before accessing index in merge message (#22822) Fixes #22821 Signed-off-by: jolheiser --- services/pull/merge.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/pull/merge.go b/services/pull/merge.go index edd5b601da..a3d69df8df 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -98,6 +98,9 @@ func GetDefaultMergeMessage(ctx context.Context, baseGitRepo *git.Repository, pr } for _, ref := range refs { if ref.RefAction == references.XRefActionCloses { + if err := ref.LoadIssue(ctx); err != nil { + return "", "", err + } closeIssueIndexes = append(closeIssueIndexes, fmt.Sprintf("%s %s%d", closeWord, issueReference, ref.Issue.Index)) } } From 87261f3fb95da92d74d72d057fc0f1e9819f16a7 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 9 Feb 2023 04:51:02 +0100 Subject: [PATCH 22/33] Fix blame view missing lines (#22826) Creating a new buffered reader for every part of the blame can miss lines, as it will read and buffer bytes that the next buffered reader will not get. Co-authored-by: Lunny Xiao --- modules/git/blame.go | 31 +++++++++++++++++-------------- modules/git/blame_test.go | 2 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/modules/git/blame.go b/modules/git/blame.go index 3b6e4c95db..ec88accb10 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -20,11 +20,12 @@ type BlamePart struct { // BlameReader returns part of file blame one by one type BlameReader struct { - cmd *Command - output io.WriteCloser - reader io.ReadCloser - done chan error - lastSha *string + cmd *Command + output io.WriteCloser + reader io.ReadCloser + bufferedReader *bufio.Reader + done chan error + lastSha *string } var shaLineRegex = regexp.MustCompile("^([a-z0-9]{40})") @@ -33,8 +34,6 @@ var shaLineRegex = regexp.MustCompile("^([a-z0-9]{40})") func (r *BlameReader) NextPart() (*BlamePart, error) { var blamePart *BlamePart - reader := bufio.NewReader(r.reader) - if r.lastSha != nil { blamePart = &BlamePart{*r.lastSha, make([]string, 0)} } @@ -44,7 +43,7 @@ func (r *BlameReader) NextPart() (*BlamePart, error) { var err error for err != io.EOF { - line, isPrefix, err = reader.ReadLine() + line, isPrefix, err = r.bufferedReader.ReadLine() if err != nil && err != io.EOF { return blamePart, err } @@ -66,7 +65,7 @@ func (r *BlameReader) NextPart() (*BlamePart, error) { r.lastSha = &sha1 // need to munch to end of line... for isPrefix { - _, isPrefix, err = reader.ReadLine() + _, isPrefix, err = r.bufferedReader.ReadLine() if err != nil && err != io.EOF { return blamePart, err } @@ -81,7 +80,7 @@ func (r *BlameReader) NextPart() (*BlamePart, error) { // need to munch to end of line... for isPrefix { - _, isPrefix, err = reader.ReadLine() + _, isPrefix, err = r.bufferedReader.ReadLine() if err != nil && err != io.EOF { return blamePart, err } @@ -96,6 +95,7 @@ func (r *BlameReader) NextPart() (*BlamePart, error) { // Close BlameReader - don't run NextPart after invoking that func (r *BlameReader) Close() error { err := <-r.done + r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() return err @@ -126,10 +126,13 @@ func CreateBlameReader(ctx context.Context, repoPath, commitID, file string) (*B done <- err }(cmd, repoPath, stdout, done) + bufferedReader := bufio.NewReader(reader) + return &BlameReader{ - cmd: cmd, - output: stdout, - reader: reader, - done: done, + cmd: cmd, + output: stdout, + reader: reader, + bufferedReader: bufferedReader, + done: done, }, nil } diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index a2c8fe8e75..1c0cd5c4aa 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -28,7 +28,7 @@ func TestReadingBlameOutput(t *testing.T) { }, { "f32b0a9dfd09a60f616f29158f772cedd89942d2", - []string{}, + []string{"", "Do not make any changes to this repo it is used for unit testing"}, }, } From 90cf07a2c8dc2917d0c02648f0a69e0229cfce21 Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 9 Feb 2023 06:42:18 +0100 Subject: [PATCH 23/33] Improve notification and stopwatch styles (#22169) - Add dot-style indicators to notification and time tracker - Slightly reduce whitespace between right-aligned icons - Move notification icon to right on mobile - Switch menu icon to SVG Screenshot 2022-12-19 at 19 40 32 Screenshot 2022-12-19 at 19 41 04 --------- Co-authored-by: Lunny Xiao --- templates/base/head_navbar.tmpl | 48 ++++++++++++++++----------------- web_src/less/_base.less | 28 +++++++++++++++++++ web_src/less/helpers.less | 1 + 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 90a54608e6..1c995e73e7 100644 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -3,22 +3,24 @@ {{if .IsSigned}} {{if .NotificationUnreadCount}}{{$notificationUnreadCount = call .NotificationUnreadCount}}{{end}} {{end}} -
    +
    - {{if .IsSigned}} - - - {{svg "octicon-bell"}} - - {{$notificationUnreadCount}} +
    + {{if .IsSigned}} + + + {{svg "octicon-bell"}} + + {{$notificationUnreadCount}} + - - - {{end}} -
    @@ -78,12 +80,10 @@ {{else if .IsSigned}}
    {{if EnableTimetracking}} - - - - {{svg "octicon-stopwatch"}} - {{svg "octicon-dot-fill"}} - + + + {{svg "octicon-stopwatch"}} + {{.locale.Tr "active_stopwatch"}} @@ -118,16 +118,16 @@
    {{end}} - - - {{svg "octicon-bell"}} - + + + {{svg "octicon-bell"}} + {{$notificationUnreadCount}} -
    +
    {{svg "octicon-plus"}} {{.locale.Tr "create_new"}} @@ -150,7 +150,7 @@
    -
    +
    {{avatar .SignedUser 24 "tiny"}} {{.locale.Tr "user_profile_and_more"}} diff --git a/web_src/less/_base.less b/web_src/less/_base.less index e58bf53f5d..6f35a49ad8 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -1364,6 +1364,7 @@ a.ui.card:hover, visibility: hidden; } +.text.primary { color: var(--color-primary) !important; } .text.red { color: var(--color-red) !important; } .text.orange { color: var(--color-orange) !important; } .text.yellow { color: var(--color-yellow) !important; } @@ -2434,6 +2435,33 @@ a.ui.basic.label:hover { margin-top: 1rem; } +.header-stopwatch-dot { + position: absolute; + left: 8px; + top: -8px; + width: 13px; + height: 13px; + background: var(--color-primary); + border: 2px solid var(--color-header-bar); + border-radius: 100%; +} + +.notification_count { + position: absolute; + left: 5px; + top: -8px; + min-width: 1.5em; + text-align: center; + background: var(--color-primary); + border: 2px solid var(--color-header-bar); + color: var(--color-header-bar); + padding: 2px; + border-radius: 1em; + font-size: 10px; + font-weight: 700; + line-height: .7; +} + table th[data-sortt-asc], table th[data-sortt-desc] { &:hover { diff --git a/web_src/less/helpers.less b/web_src/less/helpers.less index 46284811ee..20670a3786 100644 --- a/web_src/less/helpers.less +++ b/web_src/less/helpers.less @@ -21,6 +21,7 @@ /* below class names match Tailwind CSS */ .pointer-events-none { pointer-events: none !important; } +.relative { position: relative !important; } .mono { font-family: var(--fonts-monospace) !important; From 7ae10cb7f1a520d46c2c92d60cf5181c3d7b6850 Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Thu, 9 Feb 2023 15:40:34 +0900 Subject: [PATCH 24/33] change org_type.go to visible_type.go and fix the notes (#22752) It seems `VisibleType` is only designed for org at first. But it is also used by user's visibility now. So I think `org_type.go` can be changed to `visible_type.go`. --- modules/structs/{org_type.go => visible_type.go} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename modules/structs/{org_type.go => visible_type.go} (87%) diff --git a/modules/structs/org_type.go b/modules/structs/visible_type.go similarity index 87% rename from modules/structs/org_type.go rename to modules/structs/visible_type.go index 69f323fa99..b5ff353b87 100644 --- a/modules/structs/org_type.go +++ b/modules/structs/visible_type.go @@ -3,7 +3,7 @@ package structs -// VisibleType defines the visibility (Organization only) +// VisibleType defines the visibility of user and org type VisibleType int const ( @@ -13,11 +13,11 @@ const ( // VisibleTypeLimited Visible for every connected user VisibleTypeLimited - // VisibleTypePrivate Visible only for organization's members + // VisibleTypePrivate Visible only for self or admin user VisibleTypePrivate ) -// VisibilityModes is a map of org Visibility types +// VisibilityModes is a map of Visibility types var VisibilityModes = map[string]VisibleType{ "public": VisibleTypePublic, "limited": VisibleTypeLimited, From 29aea3642f5de5f1a8d4264f8360ddb5d072a861 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 9 Feb 2023 17:29:13 +0800 Subject: [PATCH 25/33] Make clone URL use current page's host (#22808) Follow #21986 Even if the ROOT_URL is incorrect, the clone URL on the UI should be correct. --------- Co-authored-by: Lunny Xiao --- templates/repo/clone_script.tmpl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/templates/repo/clone_script.tmpl b/templates/repo/clone_script.tmpl index afd90040fb..88a67d8235 100644 --- a/templates/repo/clone_script.tmpl +++ b/templates/repo/clone_script.tmpl @@ -17,7 +17,14 @@ const btn = isSSH ? sshBtn : httpsBtn; if (!btn) return; - const link = btn.getAttribute('data-link'); + let link = btn.getAttribute('data-link'); + if (link.startsWith('http://') || link.startsWith('https://')) { + // use current protocol/host as the clone link + const url = new URL(link); + url.protocol = window.location.protocol; + url.host = window.location.host; + link = url.toString(); + } for (const el of document.getElementsByClassName('js-clone-url')) { el[el.nodeName === 'INPUT' ? 'value' : 'textContent'] = link; } From e253888a0e68d03da0616930427d7162eefa6aaf Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 9 Feb 2023 20:51:36 +0800 Subject: [PATCH 26/33] Fix isAllowed of escapeStreamer (#22814) The use of `sort.Search` is wrong: The slice should be sorted, and `return >= 0` doen't mean it exists, see the [manual](https://pkg.go.dev/sort#Search). Could be fixed like this if we really need it: ```diff diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go index 823b63513..fcf1ffbc1 100644 --- a/modules/charset/escape_stream.go +++ b/modules/charset/escape_stream.go @@ -20,6 +20,9 @@ import ( var defaultWordRegexp = regexp.MustCompile(`(-?\d*\.\d\w*)|([^\` + "`" + `\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s\x00-\x1f]+)`) func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...rune) HTMLStreamer { + sort.Slice(allowed, func(i, j int) bool { + return allowed[i] < allowed[j] + }) return &escapeStreamer{ escaped: &EscapeStatus{}, PassthroughHTMLStreamer: *NewPassthroughStreamer(next), @@ -284,14 +287,8 @@ func (e *escapeStreamer) runeTypes(runes ...rune) (types []runeType, confusables } func (e *escapeStreamer) isAllowed(r rune) bool { - if len(e.allowed) == 0 { - return false - } - if len(e.allowed) == 1 { - return e.allowed[0] == r - } - - return sort.Search(len(e.allowed), func(i int) bool { + i := sort.Search(len(e.allowed), func(i int) bool { return e.allowed[i] >= r - }) >= 0 + }) + return i < len(e.allowed) && e.allowed[i] == r } ``` But I don't think so, a map is better to do it. --- modules/charset/escape_stream.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go index 823b635137..1b956bf4ab 100644 --- a/modules/charset/escape_stream.go +++ b/modules/charset/escape_stream.go @@ -6,7 +6,6 @@ package charset import ( "fmt" "regexp" - "sort" "strings" "unicode" "unicode/utf8" @@ -20,12 +19,16 @@ import ( var defaultWordRegexp = regexp.MustCompile(`(-?\d*\.\d\w*)|([^\` + "`" + `\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s\x00-\x1f]+)`) func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...rune) HTMLStreamer { + allowedM := make(map[rune]bool, len(allowed)) + for _, v := range allowed { + allowedM[v] = true + } return &escapeStreamer{ escaped: &EscapeStatus{}, PassthroughHTMLStreamer: *NewPassthroughStreamer(next), locale: locale, ambiguousTables: AmbiguousTablesForLocale(locale), - allowed: allowed, + allowed: allowedM, } } @@ -34,7 +37,7 @@ type escapeStreamer struct { escaped *EscapeStatus locale translation.Locale ambiguousTables []*AmbiguousTable - allowed []rune + allowed map[rune]bool } func (e *escapeStreamer) EscapeStatus() *EscapeStatus { @@ -256,7 +259,7 @@ func (e *escapeStreamer) runeTypes(runes ...rune) (types []runeType, confusables runeCounts.numBrokenRunes++ case r == ' ' || r == '\t' || r == '\n': runeCounts.numBasicRunes++ - case e.isAllowed(r): + case e.allowed[r]: if r > 0x7e || r < 0x20 { types[i] = nonBasicASCIIRuneType runeCounts.numNonConfusingNonBasicRunes++ @@ -282,16 +285,3 @@ func (e *escapeStreamer) runeTypes(runes ...rune) (types []runeType, confusables } return types, confusables, runeCounts } - -func (e *escapeStreamer) isAllowed(r rune) bool { - if len(e.allowed) == 0 { - return false - } - if len(e.allowed) == 1 { - return e.allowed[0] == r - } - - return sort.Search(len(e.allowed), func(i int) bool { - return e.allowed[i] >= r - }) >= 0 -} From cef8f50286ce1bc04c2e5989998d5a0774870de7 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 10 Feb 2023 00:14:45 +0800 Subject: [PATCH 27/33] Improve AppUrl/ROOT_URL checking (#22836) After some PRs: * #21986 * #22795 * #22808 * #22831 * #22839 Users won't be affected by the ROOT_URL problem in most cases. Close #19345 This PR improves AppUrl/ROOT_URL checking, only check it on the admin page, and the message is also updated. Feel free to suggest about more English-native messages. ![image](https://user-images.githubusercontent.com/2114189/217811809-7d44ddb7-2c4a-46d0-a5db-8ae6ee65f8c3.png) --- web_src/js/features/admin/common.js | 7 ++++++- web_src/js/features/common-global.js | 7 ++----- web_src/js/index.js | 2 -- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web_src/js/features/admin/common.js b/web_src/js/features/admin/common.js index 2438fcf62b..d0b3e461d5 100644 --- a/web_src/js/features/admin/common.js +++ b/web_src/js/features/admin/common.js @@ -1,12 +1,17 @@ import $ from 'jquery'; +import {checkAppUrl} from '../common-global.js'; const {csrfToken} = window.config; export function initAdminCommon() { - if ($('.admin').length === 0) { + if ($('.page-content.admin').length === 0) { return; } + // check whether appUrl(ROOT_URL) is correct, if not, show an error message + // only admin pages need this check because most templates are using relative URLs now + checkAppUrl(); + // New user if ($('.admin.new.user').length > 0 || $('.admin.edit.user').length > 0) { $('#login_type').on('change', function () { diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index 4677eeac0c..e655feec0b 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -381,9 +381,6 @@ export function checkAppUrl() { if (curUrl.startsWith(appUrl) || `${curUrl}/` === appUrl) { return; } - if (document.querySelector('.page-content.install')) { - return; // no need to show the message on the installation page - } - showGlobalErrorMessage(`Your ROOT_URL in app.ini is ${appUrl} but you are visiting ${curUrl} -You should set ROOT_URL correctly, otherwise the web may not work correctly.`); + showGlobalErrorMessage(`Your ROOT_URL in app.ini is "${appUrl}", it's unlikely matching the site you are visiting. +Mismatched ROOT_URL config causes wrong URL links for web UI/mail content/webhook notification.`); } diff --git a/web_src/js/index.js b/web_src/js/index.js index 74d80776b5..611c09d2b8 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -48,7 +48,6 @@ import { initCommitStatuses, } from './features/repo-commit.js'; import { - checkAppUrl, initFootLanguageMenu, initGlobalButtonClickOnEnter, initGlobalButtons, @@ -199,5 +198,4 @@ $(document).ready(() => { initUserAuthWebAuthnRegister(); initUserSettings(); initViewedCheckboxListenerFor(); - checkAppUrl(); }); From 0c190e396df7025683b56f0e6f64356c907a8512 Mon Sep 17 00:00:00 2001 From: John Olheiser Date: Thu, 9 Feb 2023 10:15:07 -0600 Subject: [PATCH 28/33] Fix unmatched div in project filter (#22832) (Note that the below screenshots aren't the same repo, the former is try and the latter is local) Before ![div-before](https://user-images.githubusercontent.com/42128690/217723899-a15da77f-a196-4b23-a157-e7f1e1979610.png) After ![div-after](https://user-images.githubusercontent.com/42128690/217723878-e54235bc-a7d7-425e-bd0d-47d1814f18ba.png) Signed-off-by: jolheiser Co-authored-by: techknowlogick --- templates/repo/issue/list.tmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index cf2a2a6bba..4b7de641f1 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -276,7 +276,6 @@ {{.Title}}
    {{end}} -
    {{end}}
    From 24a9caa2f33db2664510e79f184892c8cb266b62 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 10 Feb 2023 00:31:30 +0800 Subject: [PATCH 29/33] Fix more HTMLURL in templates (#22831) I haven't tested `runs_list.tmpl` but I think it could be right. After this PR, besides the `` in html head, the only explicit HTMLURL usage is in `pull_merge_instruction.tmpl`, which doesn't affect users too much and it's difficult to fix at the moment. There are still many usages of `AppUrl` in the templates (eg: the package help manual), they are similar problems as the HTMLURL in pull_merge_instruction, and they might be fixed together in the future. Diff without space: https://github.com/go-gitea/gitea/pull/22831/files?diff=unified&w=1 --- models/issues/issue_xref.go | 14 +- modules/templates/helper.go | 4 + templates/projects/view.tmpl | 2 +- templates/repo/actions/runs_list.tmpl | 2 +- .../repo/issue/view_content/comments.tmpl | 4 +- templates/repo/issue/view_content/pull.tmpl | 139 +++++++++--------- .../view_content/pull_merge_instruction.tmpl | 1 + 7 files changed, 85 insertions(+), 81 deletions(-) diff --git a/models/issues/issue_xref.go b/models/issues/issue_xref.go index 21ee24210f..a1086f9e81 100644 --- a/models/issues/issue_xref.go +++ b/models/issues/issue_xref.go @@ -277,26 +277,26 @@ func CommentTypeIsRef(t CommentType) bool { return t == CommentTypeCommentRef || t == CommentTypePullRef || t == CommentTypeIssueRef } -// RefCommentHTMLURL returns the HTML URL for the comment that created this reference -func (c *Comment) RefCommentHTMLURL() string { +// RefCommentLink returns the relative URL for the comment that created this reference +func (c *Comment) RefCommentLink() string { // Edge case for when the reference is inside the title or the description of the referring issue if c.RefCommentID == 0 { - return c.RefIssueHTMLURL() + return c.RefIssueLink() } if err := c.LoadRefComment(); err != nil { // Silently dropping errors :unamused: log.Error("LoadRefComment(%d): %v", c.RefCommentID, err) return "" } - return c.RefComment.HTMLURL() + return c.RefComment.Link() } -// RefIssueHTMLURL returns the HTML URL of the issue where this reference was created -func (c *Comment) RefIssueHTMLURL() string { +// RefIssueLink returns the relative URL of the issue where this reference was created +func (c *Comment) RefIssueLink() string { if err := c.LoadRefIssue(); err != nil { // Silently dropping errors :unamused: log.Error("LoadRefIssue(%d): %v", c.RefCommentID, err) return "" } - return c.RefIssue.HTMLURL() + return c.RefIssue.Link() } // RefIssueTitle returns the title of the issue where this reference was created diff --git a/modules/templates/helper.go b/modules/templates/helper.go index a390d94592..7afc3aa59b 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -72,6 +72,10 @@ func NewFuncMap() []template.FuncMap { return setting.StaticURLPrefix + "/assets" }, "AppUrl": func() string { + // The usage of AppUrl should be avoided as much as possible, + // because the AppURL(ROOT_URL) may not match user's visiting site and the ROOT_URL in app.ini may be incorrect. + // And it's difficult for Gitea to guess absolute URL correctly with zero configuration, + // because Gitea doesn't know whether the scheme is HTTP or HTTPS unless the reverse proxy could tell Gitea. return setting.AppURL }, "AppVer": func() string { diff --git a/templates/projects/view.tmpl b/templates/projects/view.tmpl index 42eb578074..72131e30bb 100644 --- a/templates/projects/view.tmpl +++ b/templates/projects/view.tmpl @@ -238,7 +238,7 @@ {{end}}
    {{range .Assignees}} - {{avatar . 28 "mini mr-3"}} + {{avatar . 28 "mini mr-3"}} {{end}}
    diff --git a/templates/repo/actions/runs_list.tmpl b/templates/repo/actions/runs_list.tmpl index d3eb12430f..2a85222943 100644 --- a/templates/repo/actions/runs_list.tmpl +++ b/templates/repo/actions/runs_list.tmpl @@ -6,7 +6,7 @@
    - + {{.Title}} diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 39fbc638ce..37aa82345f 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -151,12 +151,12 @@ {{if eq .RefAction 3}}{{end}} {{template "shared/user/authorlink" .Poster}} - {{$.locale.Tr $refTr (.EventTag|Escape) $createdStr (.RefCommentHTMLURL|Escape) $refFrom | Safe}} + {{$.locale.Tr $refTr (.EventTag|Escape) $createdStr (.RefCommentLink|Escape) $refFrom | Safe}} {{if eq .RefAction 3}}{{end}}
    - {{.RefIssueTitle}} {{.RefIssueIdent}} + {{.RefIssueTitle}} {{.RefIssueIdent}}
    {{else if eq .Type 4}} diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index c68a150832..f32c278b62 100644 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -303,79 +303,78 @@ {{$hasPendingPullRequestMergeTip = $.locale.Tr "repo.pulls.auto_merge_has_pending_schedule" .PendingPullRequestMerge.Doer.Name $createdPRMergeStr}} {{end}}
    -
    diff --git a/templates/repo/issue/view_content/pull_merge_instruction.tmpl b/templates/repo/issue/view_content/pull_merge_instruction.tmpl index 816f25cbcf..21bb3d8e79 100644 --- a/templates/repo/issue/view_content/pull_merge_instruction.tmpl +++ b/templates/repo/issue/view_content/pull_merge_instruction.tmpl @@ -5,6 +5,7 @@
    {{if eq $.Issue.PullRequest.Flow 0}}
    git checkout -b {{if ne $.Issue.PullRequest.HeadRepo.ID $.Issue.PullRequest.BaseRepo.ID}}{{$.Issue.PullRequest.HeadRepo.OwnerName}}-{{end}}{{$.Issue.PullRequest.HeadBranch}} {{$.Issue.PullRequest.BaseBranch}}
    + {{/* the only legacy HTMLURL used in template, which doesn't affect users too much and is very diffcult to fix, it should be fixed together with other AppUrl usages*/}}
    git pull {{if ne $.Issue.PullRequest.HeadRepo.ID $.Issue.PullRequest.BaseRepo.ID}}{{$.Issue.PullRequest.HeadRepo.HTMLURL}}{{else}}origin{{end}} {{$.Issue.PullRequest.HeadBranch}}
    {{else}}
    git fetch origin {{$.Issue.PullRequest.GetGitRefName}}:{{$.Issue.PullRequest.HeadBranch}}
    From 137fcc989bbde6dcddbbf53650e174c119d38068 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 9 Feb 2023 17:39:31 +0100 Subject: [PATCH 30/33] Fix inconsistent Filter Project name in issue list (#22827) Use Project instead of Filter Project like the other filter menus. --- templates/repo/issue/list.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 4b7de641f1..c04b074221 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -77,13 +77,13 @@
    - {{.locale.Tr "repo.issues.filter_projects"}} + {{.locale.Tr "repo.issues.filter_project"}} {{svg "octicon-triangle-down" 14 "dropdown icon"}}
    {{svg "octicon-search" 16}} - +
    {{.locale.Tr "repo.issues.filter_project_all"}} {{.locale.Tr "repo.issues.filter_project_none"}} From bdd2c9d2eff76fed2cc200377723f519e8af64d0 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 9 Feb 2023 18:08:42 +0100 Subject: [PATCH 31/33] Fix update by rebase being wrongly disabled by protected base branch (#22825) The branch this is force pushing to is the head branch in the head repo, so it should be checking if that is protected, not the base. --- services/pull/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/pull/update.go b/services/pull/update.go index ede89bcdff..b9525cf0c9 100644 --- a/services/pull/update.go +++ b/services/pull/update.go @@ -106,7 +106,7 @@ func IsUserAllowedToUpdate(ctx context.Context, pull *issues_model.PullRequest, BaseBranch: pull.HeadBranch, } - pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pull.BaseRepoID, pull.BaseBranch) + pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pr.BaseRepoID, pr.BaseBranch) if err != nil { return false, false, err } From affdd40296960a08a4223330ccbd1fb88c96ea1a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 10 Feb 2023 01:11:16 +0800 Subject: [PATCH 32/33] Make issue title edit buttons focusable and fix incorrect ajax requests (#22807) Replace #19922 , which is stale since my last review: https://github.com/go-gitea/gitea/pull/19922#pullrequestreview-1003546506 and https://github.com/go-gitea/gitea/pull/19922#issuecomment-1153181546 Close #19769 Changes: 1. Use `
    {{end}}

    {{RenderIssueTitle $.Context .Issue.Title $.RepoLink $.Repository.ComposeMetas | RenderCodeBlock}} #{{.Issue.Index}} -

    {{if and (or .HasIssuesOrPullsWritePermission .IsIssuePoster) (not .Repository.IsArchived)}}
    - - + +
    {{end}}
    diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index e655feec0b..f2edf31249 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -60,6 +60,7 @@ export function initGlobalButtonClickOnEnter() { $(document).on('keypress', '.ui.button', (e) => { if (e.keyCode === 13 || e.keyCode === 32) { // enter key or space bar $(e.target).trigger('click'); + e.preventDefault(); } }); } diff --git a/web_src/js/features/repo-issue.js b/web_src/js/features/repo-issue.js index 56d294e81a..f562584c11 100644 --- a/web_src/js/features/repo-issue.js +++ b/web_src/js/features/repo-issue.js @@ -605,6 +605,7 @@ export function initRepoIssueTitleEdit() { const targetBranch = $('#pull-target-branch').data('branch'); const $branchTarget = $('#branch_target'); if (targetBranch === $branchTarget.text()) { + window.location.reload(); return false; } $.post(update_url, { @@ -617,19 +618,22 @@ export function initRepoIssueTitleEdit() { }); }; - const pullrequest_target_update_url = $(this).data('target-update-url'); + const pullrequest_target_update_url = $(this).attr('data-target-update-url'); if ($editInput.val().length === 0 || $editInput.val() === $issueTitle.text()) { $editInput.val($issueTitle.text()); pullrequest_targetbranch_change(pullrequest_target_update_url); } else { - $.post($(this).data('update-url'), { + $.post($(this).attr('data-update-url'), { _csrf: csrfToken, title: $editInput.val() }, (data) => { $editInput.val(data.title); $issueTitle.text(data.title); - pullrequest_targetbranch_change(pullrequest_target_update_url); - window.location.reload(); + if (pullrequest_target_update_url) { + pullrequest_targetbranch_change(pullrequest_target_update_url); // it will reload the window + } else { + window.location.reload(); + } }); } return false; diff --git a/web_src/less/_base.less b/web_src/less/_base.less index 6f35a49ad8..4a22b8af4b 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -2316,6 +2316,13 @@ a.ui.label:hover { .ui.basic.secondary.buttons .button:active, .ui.basic.secondary.button:active { color: var(--color-secondary-dark-8) !important; + border-color: var(--color-secondary-dark-1) !important; +} + +.ui.basic.secondary.button:focus, +.ui.basic.secondary.buttons .button:focus { + color: var(--color-secondary-dark-8) !important; + border-color: var(--color-secondary-dark-3) !important; } .ui.tertiary.button { From 1cb8d14bf71e0b8637c9eaa10808b4fd05139f45 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sat, 11 Feb 2023 01:39:50 +0100 Subject: [PATCH 33/33] Use proxy for pull mirror (#22771) - Use the proxy (if one is specified) for pull mirrors syncs. - Pulled the code from https://github.com/go-gitea/gitea/blob/c2774d9e80d9a436d9c2044960369c4db227e3a0/modules/git/repo.go#L164-L170 Downstream issue: https://codeberg.org/forgejo/forgejo/issues/302 --------- Co-authored-by: Lauris BH --- modules/git/repo.go | 6 ++---- modules/proxy/proxy.go | 14 ++++++++++++++ services/mirror/mirror_pull.go | 4 ++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/modules/git/repo.go b/modules/git/repo.go index e77a3a6ad8..233f7f20cf 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -163,10 +163,8 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op envs := os.Environ() u, err := url.Parse(from) - if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) { - if proxy.Match(u.Host) { - envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL())) - } + if err == nil { + envs = proxy.EnvWithProxy(u) } stderr := new(bytes.Buffer) diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go index f0cd366c12..1a6bdad7fb 100644 --- a/modules/proxy/proxy.go +++ b/modules/proxy/proxy.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "strings" "sync" "code.gitea.io/gitea/modules/log" @@ -82,3 +83,16 @@ func Proxy() func(req *http.Request) (*url.URL, error) { return http.ProxyFromEnvironment(req) } } + +// EnvWithProxy returns os.Environ(), with a https_proxy env, if the given url +// needs to be proxied. +func EnvWithProxy(u *url.URL) []string { + envs := os.Environ() + if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { + if Match(u.Host) { + envs = append(envs, "https_proxy="+GetProxyURL()) + } + } + + return envs +} diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 7dee90352e..126d2bf354 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/proxy" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -215,6 +216,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo return nil, false } + envs := proxy.EnvWithProxy(remoteURL.URL) + stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} if err := cmd. @@ -222,6 +225,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo Run(&git.RunOpts{ Timeout: timeout, Dir: repoPath, + Env: envs, Stdout: &stdoutBuilder, Stderr: &stderrBuilder, }); err != nil {