forgejo/services/webhook/deliver.go

333 lines
10 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package webhook
import (
"context"
"crypto/hmac"
"crypto/sha1"
[GITEA] Drop sha256-simd in favor of stdlib - In Go 1.21 the crypto/sha256 [got a massive improvement](https://go.dev/doc/go1.21#crypto/sha256) by utilizing the SHA instructions for AMD64 CPUs, which sha256-simd already was doing. The performance is now on par and I think it's preferable to use the standard library rather than a package when possible. ``` cpu: AMD Ryzen 5 3600X 6-Core Processor │ simd.txt │ go.txt │ │ sec/op │ sec/op vs base │ Hash/8Bytes-12 63.25n ± 1% 73.38n ± 1% +16.02% (p=0.002 n=6) Hash/64Bytes-12 98.73n ± 1% 105.30n ± 1% +6.65% (p=0.002 n=6) Hash/1K-12 567.2n ± 1% 572.8n ± 1% +0.99% (p=0.002 n=6) Hash/8K-12 4.062µ ± 1% 4.062µ ± 1% ~ (p=0.396 n=6) Hash/1M-12 512.1µ ± 0% 510.6µ ± 1% ~ (p=0.485 n=6) Hash/5M-12 2.556m ± 1% 2.564m ± 0% ~ (p=0.093 n=6) Hash/10M-12 5.112m ± 0% 5.127m ± 0% ~ (p=0.093 n=6) geomean 13.82µ 14.27µ +3.28% │ simd.txt │ go.txt │ │ B/s │ B/s vs base │ Hash/8Bytes-12 120.6Mi ± 1% 104.0Mi ± 1% -13.81% (p=0.002 n=6) Hash/64Bytes-12 618.2Mi ± 1% 579.8Mi ± 1% -6.22% (p=0.002 n=6) Hash/1K-12 1.682Gi ± 1% 1.665Gi ± 1% -0.98% (p=0.002 n=6) Hash/8K-12 1.878Gi ± 1% 1.878Gi ± 1% ~ (p=0.310 n=6) Hash/1M-12 1.907Gi ± 0% 1.913Gi ± 1% ~ (p=0.485 n=6) Hash/5M-12 1.911Gi ± 1% 1.904Gi ± 0% ~ (p=0.093 n=6) Hash/10M-12 1.910Gi ± 0% 1.905Gi ± 0% ~ (p=0.093 n=6) geomean 1.066Gi 1.032Gi -3.18% ``` (cherry picked from commit abd94ff5b59c86e793fd9bf12187ea6cfd1f3fa1) (cherry picked from commit 15e81637abf70576a564cf9eecaa9640228afb5b) Conflicts: go.mod https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit 325d92917f655c999b81b08832ee623d6b669f0f) Conflicts: modules/context/context_cookie.go https://codeberg.org/forgejo/forgejo/pulls/1617 (cherry picked from commit 358819e8959886faa171ac16541097500d0a703e) (cherry picked from commit 362fd7aae17832fa922fa017794bc564ca43060d) (cherry picked from commit 4f64ee294ee05c93042b6ec68f0a179ec249dab9) (cherry picked from commit 4bde77f7b13c5f961c141c01b6da1f9eda5ec387) (cherry picked from commit 1311e30a811675eb623692349e4e808a85aabef6) (cherry picked from commit 57b69e334c2973118488b9b5dbdc8a2c88135756) (cherry picked from commit 52dc892fadecf39e89c3c351edc9efb42522257b) (cherry picked from commit 77f54f4187869c6eabcc837742fd3f908093a76c) (cherry picked from commit 0d0392f3a510ce3683bb649dee1e65b45dd91354) Conflicts: go.mod https://codeberg.org/forgejo/forgejo/pulls/2034 (cherry picked from commit 92798364e8fe3188a2100b54f3adea943f8309e9) (cherry picked from commit 43d218127752aa9251c4c3ef71b9c060f109dffc) (cherry picked from commit 45c88b86a35729fc0b2dc6b72bc33caf9f69265f) (cherry picked from commit a1cd6f4e3a7956773cbc0aef8abb80d17b62eb49) (cherry picked from commit 01191dc2adf8c57ae448be37e73158005a8ff74d) (cherry picked from commit 151e07f37e2854ad633f1352fb0ce3cd06f4b2ae)
2023-09-29 22:45:31 +00:00
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
webhook_module "code.gitea.io/gitea/modules/webhook"
"github.com/gobwas/glob"
)
// Deliver deliver hook task
func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
w, err := webhook_model.GetWebhookByID(ctx, t.HookID)
if err != nil {
return err
}
defer func() {
err := recover()
if err == nil {
return
}
// There was a panic whilst delivering a hook...
log.Error("PANIC whilst trying to deliver webhook task[%d] to webhook %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
}()
t.IsDelivered = true
var req *http.Request
switch w.HTTPMethod {
case "":
log.Info("HTTP Method for webhook %s empty, setting to POST as default", w.URL)
fallthrough
case http.MethodPost:
switch w.ContentType {
case webhook_model.ContentTypeJSON:
req, err = http.NewRequest("POST", w.URL, strings.NewReader(t.PayloadContent))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
case webhook_model.ContentTypeForm:
forms := url.Values{
"payload": []string{t.PayloadContent},
}
req, err = http.NewRequest("POST", w.URL, strings.NewReader(forms.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
case http.MethodGet:
u, err := url.Parse(w.URL)
if err != nil {
return fmt.Errorf("unable to deliver webhook task[%d] as cannot parse webhook url %s: %w", t.ID, w.URL, err)
}
vals := u.Query()
vals["payload"] = []string{t.PayloadContent}
u.RawQuery = vals.Encode()
req, err = http.NewRequest("GET", u.String(), nil)
if err != nil {
return fmt.Errorf("unable to deliver webhook task[%d] as unable to create HTTP request for webhook url %s: %w", t.ID, w.URL, err)
}
case http.MethodPut:
switch w.Type {
case webhook_module.MATRIX:
Add Webhook authorization header (#20926) _This is a different approach to #20267, I took the liberty of adapting some parts, see below_ ## Context In some cases, a weebhook endpoint requires some kind of authentication. The usual way is by sending a static `Authorization` header, with a given token. For instance: - Matrix expects a `Bearer <token>` (already implemented, by storing the header cleartext in the metadata - which is buggy on retry #19872) - TeamCity #18667 - Gitea instances #20267 - SourceHut https://man.sr.ht/graphql.md#authentication-strategies (this is my actual personal need :) ## Proposed solution Add a dedicated encrypt column to the webhook table (instead of storing it as meta as proposed in #20267), so that it gets available for all present and future hook types (especially the custom ones #19307). This would also solve the buggy matrix retry #19872. As a first step, I would recommend focusing on the backend logic and improve the frontend at a later stage. For now the UI is a simple `Authorization` field (which could be later customized with `Bearer` and `Basic` switches): ![2022-08-23-142911](https://user-images.githubusercontent.com/3864879/186162483-5b721504-eef5-4932-812e-eb96a68494cc.png) The header name is hard-coded, since I couldn't fine any usecase justifying otherwise. ## Questions - What do you think of this approach? @justusbunsi @Gusted @silverwind - ~~How are the migrations generated? Do I have to manually create a new file, or is there a command for that?~~ - ~~I started adding it to the API: should I complete it or should I drop it? (I don't know how much the API is actually used)~~ ## Done as well: - add a migration for the existing matrix webhooks and remove the `Authorization` logic there _Closes #19872_ Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: Gusted <williamzijl7@hotmail.com> Co-authored-by: delvh <dev.lh@web.de>
2022-11-03 18:23:20 +00:00
txnID, err := getMatrixTxnID([]byte(t.PayloadContent))
if err != nil {
return err
}
url := fmt.Sprintf("%s/%s", w.URL, url.PathEscape(txnID))
req, err = http.NewRequest("PUT", url, strings.NewReader(t.PayloadContent))
if err != nil {
return fmt.Errorf("unable to deliver webhook task[%d] as cannot create matrix request for webhook url %s: %w", t.ID, w.URL, err)
}
default:
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}
default:
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}
var signatureSHA1 string
var signatureSHA256 string
if len(w.Secret) > 0 {
sig1 := hmac.New(sha1.New, []byte(w.Secret))
sig256 := hmac.New(sha256.New, []byte(w.Secret))
_, err = io.MultiWriter(sig1, sig256).Write([]byte(t.PayloadContent))
if err != nil {
log.Error("prepareWebhooks.sigWrite: %v", err)
}
signatureSHA1 = hex.EncodeToString(sig1.Sum(nil))
signatureSHA256 = hex.EncodeToString(sig256.Sum(nil))
}
event := t.EventType.Event()
eventType := string(t.EventType)
[BRANDING] add X-Forgejo-* headers (cherry picked from commit 0a3388f93f53f53cce79ebcc194d67979cbe61cc) (cherry picked from commit 7eba0a440a3138443d0b7461560f9f6f1c46b256) (cherry picked from commit eb9646c7ef0b1a9e73bf22806acbc2caeaee7ff3) (cherry picked from commit f1972578f5cb72ad9e1ef1ded9aa324b1d669864) Conflicts: (cherry picked from commit 7f96222fb4b4a773193a11d4ba76f449013c30ef) (cherry picked from commit e3c7c9fe7b5809cdb75f1eb629c82b27986f7e98) (cherry picked from commit 84fdead90242e68493efb96a7fee1e3b5fd417e9) (cherry picked from commit 85148e11961b86269b7cda30294a4cdc817787d0) (cherry picked from commit c0086bd70d39d9a1075fa624cdbfdf5ef26f7a2c) (cherry picked from commit d1e31ef31817e5e811f6bea7aee5582383d53b12) (cherry picked from commit 681d3ed5c43d45f802dc13c58c5b821da938374d) (cherry picked from commit 76a3001f5bc5a064054514c179ba14008952e914) (cherry picked from commit a55a9567d36f6b60557df3df3a0801c0fccec963) (cherry picked from commit aa7adc167d4e4ee7879b88fe799f2afca2fd5531) (cherry picked from commit d5354cb52c26ab0090388d019efe61770e7ec452) (cherry picked from commit 472c48999699420d15aaa7c61a42db8136221616) (cherry picked from commit dc816d065b715f2bfdc7fa68fa2d254f3d481b35) (cherry picked from commit 4795f9ea85eac21c70db1159914a7b7b035b7374) (cherry picked from commit ddd4ae53433d4adab823aa974c0f1c39d0394414) (cherry picked from commit 0e95f2a36b1c0f480f94fef85d341792c7a7b759) (cherry picked from commit 47a41a4e7b3c5d3cf247dafef17a64415069f171) (cherry picked from commit d4416fcd3ee0f1d74c2b03fbe40c15b83f80b56d) (cherry picked from commit f279d153b61677e2a766859c7a7688b0327e7c9a) (cherry picked from commit 959f908ffdc9dd2fb07aefdf7cb85ba34f9071a3) (cherry picked from commit 82df95310176a083abb9eaa51de9e9754d4f72c6) (cherry picked from commit 8f8c9fd9e3deca0af0f087fc44fcd271e8440a7b) (cherry picked from commit d4a0db7706f30a394192c6023072ad5185a7d600) (cherry picked from commit 44594d623942128072bd86a20b5b90603517005e) (cherry picked from commit 62b1de579ebfe326340b218cc9a980b705b8e6a2) (cherry picked from commit 507abee353f120247276fe9165f09976421f926d) (cherry picked from commit 8c36ac42c7091661b2f3fec70fc88a74043628fc) (cherry picked from commit 72f74067f4c781e1c4936f0da2653a35a6f1c82b) (cherry picked from commit 8e5a9e187b50a6eb8523c6dda4ce1a23c08a0b5a) (cherry picked from commit cff8f43c7e4ab86a468ea49f7b945412d9911151) (cherry picked from commit 493cc93e6d4b738a5b81d1a0db831bf9f430e9de) (cherry picked from commit c94352b4d460e631c223b45516e42765357bf67d) (cherry picked from commit 3f622c51a70200bf67d209d0cec78db2f25b65db) (cherry picked from commit 84c62ad447a94ebd5d1906e41c29efd3a0816c70) (cherry picked from commit ba186c8ee40e95cd4f40b6205d673dfe20528825) (cherry picked from commit 4238ef291dffb68b350aa2f410e934d52cb0a1eb) (cherry picked from commit 3ef1bd61b9a45271e69c1529875e84b379073318) (cherry picked from commit f304086bb693d3a0a58871ba8cb1354ba290695c) (cherry picked from commit 64a2edabcb7bc9e3961ea45f83cf6488e4cd99f2) (cherry picked from commit 6accea6ba7f2b0f98b6bef5b1483ee39ad68edba) (cherry picked from commit 2296e93a42939af90dee718bc8ff691009238c47) (cherry picked from commit 2bf0be3b1bff62923dd04e1a02956ced1eda97af) (cherry picked from commit fb4652b197ec64280689e6d065492f916f7e26f3) (cherry picked from commit 7d462cf353ef87007025bbd76aebef429be10562) (cherry picked from commit f5319660ddecea208163b5bcd37444322e571b51)
2023-01-14 07:16:30 +00:00
req.Header.Add("X-Forgejo-Delivery", t.UUID)
req.Header.Add("X-Forgejo-Event", event)
req.Header.Add("X-Forgejo-Event-Type", eventType)
req.Header.Add("X-Forgejo-Signature", signatureSHA256)
req.Header.Add("X-Gitea-Delivery", t.UUID)
req.Header.Add("X-Gitea-Event", event)
req.Header.Add("X-Gitea-Event-Type", eventType)
req.Header.Add("X-Gitea-Signature", signatureSHA256)
req.Header.Add("X-Gogs-Delivery", t.UUID)
req.Header.Add("X-Gogs-Event", event)
req.Header.Add("X-Gogs-Event-Type", eventType)
req.Header.Add("X-Gogs-Signature", signatureSHA256)
req.Header.Add("X-Hub-Signature", "sha1="+signatureSHA1)
req.Header.Add("X-Hub-Signature-256", "sha256="+signatureSHA256)
req.Header["X-GitHub-Delivery"] = []string{t.UUID}
req.Header["X-GitHub-Event"] = []string{event}
req.Header["X-GitHub-Event-Type"] = []string{eventType}
Add Webhook authorization header (#20926) _This is a different approach to #20267, I took the liberty of adapting some parts, see below_ ## Context In some cases, a weebhook endpoint requires some kind of authentication. The usual way is by sending a static `Authorization` header, with a given token. For instance: - Matrix expects a `Bearer <token>` (already implemented, by storing the header cleartext in the metadata - which is buggy on retry #19872) - TeamCity #18667 - Gitea instances #20267 - SourceHut https://man.sr.ht/graphql.md#authentication-strategies (this is my actual personal need :) ## Proposed solution Add a dedicated encrypt column to the webhook table (instead of storing it as meta as proposed in #20267), so that it gets available for all present and future hook types (especially the custom ones #19307). This would also solve the buggy matrix retry #19872. As a first step, I would recommend focusing on the backend logic and improve the frontend at a later stage. For now the UI is a simple `Authorization` field (which could be later customized with `Bearer` and `Basic` switches): ![2022-08-23-142911](https://user-images.githubusercontent.com/3864879/186162483-5b721504-eef5-4932-812e-eb96a68494cc.png) The header name is hard-coded, since I couldn't fine any usecase justifying otherwise. ## Questions - What do you think of this approach? @justusbunsi @Gusted @silverwind - ~~How are the migrations generated? Do I have to manually create a new file, or is there a command for that?~~ - ~~I started adding it to the API: should I complete it or should I drop it? (I don't know how much the API is actually used)~~ ## Done as well: - add a migration for the existing matrix webhooks and remove the `Authorization` logic there _Closes #19872_ Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: Gusted <williamzijl7@hotmail.com> Co-authored-by: delvh <dev.lh@web.de>
2022-11-03 18:23:20 +00:00
// Add Authorization Header
authorization, err := w.HeaderAuthorization()
if err != nil {
log.Error("Webhook could not get Authorization header [%d]: %v", w.ID, err)
return err
}
if authorization != "" {
req.Header["Authorization"] = []string{authorization}
}
// Record delivery information.
t.RequestInfo = &webhook_model.HookRequest{
URL: req.URL.String(),
HTTPMethod: req.Method,
Headers: map[string]string{},
}
for k, vals := range req.Header {
t.RequestInfo.Headers[k] = strings.Join(vals, ",")
}
t.ResponseInfo = &webhook_model.HookResponse{
Headers: map[string]string{},
}
// OK We're now ready to attempt to deliver the task - we must double check that it
// has not been delivered in the meantime
updated, err := webhook_model.MarkTaskDelivered(ctx, t)
if err != nil {
log.Error("MarkTaskDelivered[%d]: %v", t.ID, err)
return fmt.Errorf("unable to mark task[%d] delivered in the db: %w", t.ID, err)
}
if !updated {
// This webhook task has already been attempted to be delivered or is in the process of being delivered
log.Trace("Webhook Task[%d] already delivered", t.ID)
return nil
}
// All code from this point will update the hook task
defer func() {
t.Delivered = timeutil.TimeStampNanoNow()
if t.IsSucceed {
log.Trace("Hook delivered: %s", t.UUID)
} else if !w.IsActive {
log.Trace("Hook delivery skipped as webhook is inactive: %s", t.UUID)
} else {
log.Trace("Hook delivery failed: %s", t.UUID)
}
if err := webhook_model.UpdateHookTask(ctx, t); err != nil {
log.Error("UpdateHookTask [%d]: %v", t.ID, err)
}
// Update webhook last delivery status.
if t.IsSucceed {
w.LastStatus = webhook_module.HookStatusSucceed
} else {
w.LastStatus = webhook_module.HookStatusFail
}
if err = webhook_model.UpdateWebhookLastStatus(ctx, w); err != nil {
log.Error("UpdateWebhookLastStatus: %v", err)
return
}
}()
if setting.DisableWebhooks {
return fmt.Errorf("webhook task skipped (webhooks disabled): [%d]", t.ID)
}
if !w.IsActive {
log.Trace("Webhook %s in Webhook Task[%d] is not active", w.URL, t.ID)
return nil
}
resp, err := webhookHTTPClient.Do(req.WithContext(ctx))
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
return fmt.Errorf("unable to deliver webhook task[%d] in %s due to error in http client: %w", t.ID, w.URL, err)
}
defer resp.Body.Close()
// Status code is 20x can be seen as succeed.
t.IsSucceed = resp.StatusCode/100 == 2
t.ResponseInfo.Status = resp.StatusCode
for k, vals := range resp.Header {
t.ResponseInfo.Headers[k] = strings.Join(vals, ",")
}
p, err := io.ReadAll(resp.Body)
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)
return fmt.Errorf("unable to deliver webhook task[%d] in %s as unable to read response body: %w", t.ID, w.URL, err)
}
t.ResponseInfo.Body = string(p)
return nil
}
var (
webhookHTTPClient *http.Client
once sync.Once
hostMatchers []glob.Glob
)
func webhookProxy(allowList *hostmatcher.HostMatchList) func(req *http.Request) (*url.URL, error) {
if setting.Webhook.ProxyURL == "" {
return proxy.Proxy()
}
once.Do(func() {
for _, h := range setting.Webhook.ProxyHosts {
if g, err := glob.Compile(h); err == nil {
hostMatchers = append(hostMatchers, g)
} else {
log.Error("glob.Compile %s failed: %v", h, err)
}
}
})
return func(req *http.Request) (*url.URL, error) {
for _, v := range hostMatchers {
if v.Match(req.URL.Host) {
if !allowList.MatchHostName(req.URL.Host) {
return nil, fmt.Errorf("webhook can only call allowed HTTP servers (check your %s setting), deny '%s'", allowList.SettingKeyHint, req.URL.Host)
}
return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req)
}
}
return http.ProxyFromEnvironment(req)
}
}
// Init starts the hooks delivery thread
func Init() error {
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
allowedHostListValue := setting.Webhook.AllowedHostList
if allowedHostListValue == "" {
allowedHostListValue = hostmatcher.MatchBuiltinExternal
}
allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", allowedHostListValue)
webhookHTTPClient = &http.Client{
Timeout: timeout,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify},
Proxy: webhookProxy(allowedHostMatcher),
DialContext: hostmatcher.NewDialContextWithProxy("webhook", allowedHostMatcher, nil, setting.Webhook.ProxyURLFixed),
},
}
hookQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "webhook_sender", handler)
if hookQueue == nil {
return fmt.Errorf("unable to create webhook_sender queue")
}
go graceful.GetManager().RunWithCancel(hookQueue)
go graceful.GetManager().RunWithShutdownContext(populateWebhookSendingQueue)
return nil
}
func populateWebhookSendingQueue(ctx context.Context) {
ctx, _, finished := process.GetManager().AddContext(ctx, "Webhook: Populate sending queue")
defer finished()
lowerID := int64(0)
for {
taskIDs, err := webhook_model.FindUndeliveredHookTaskIDs(ctx, lowerID)
if err != nil {
log.Error("Unable to populate webhook queue as FindUndeliveredHookTaskIDs failed: %v", err)
return
}
if len(taskIDs) == 0 {
return
}
lowerID = taskIDs[len(taskIDs)-1]
for _, taskID := range taskIDs {
select {
case <-ctx.Done():
log.Warn("Shutdown before Webhook Sending queue finishing being populated")
return
default:
}
if err := enqueueHookTask(taskID); err != nil {
log.Error("Unable to push HookTask[%d] to the Webhook Sending queue: %v", taskID, err)
}
}
}
}