Mock HTTP requests in GitLab migration test

This introduces a new utility which can be added to other tests
making HTTP calls to a live service, to cache the responses of this
service in the repository.
This commit is contained in:
Antonin Delpeuch 2023-11-28 13:40:50 +01:00
parent 102db68d58
commit 52053b1389
3 changed files with 145 additions and 14 deletions

View file

@ -121,6 +121,8 @@ package "code.gitea.io/gitea/models/unittest"
func LoadFixtures func LoadFixtures
func Copy func Copy
func CopyDir func CopyDir
func NewMockWebServer
func NormalizedFullPath
func FixturesDir func FixturesDir
func fatalTestError func fatalTestError
func InitSettings func InitSettings

View file

@ -0,0 +1,131 @@
// Copyright 2017 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package unittest
import (
"bufio"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"code.gitea.io/gitea/modules/log"
"github.com/stretchr/testify/assert"
)
// Mocks HTTP responses of a third-party service (such as GitHub, GitLab…)
// This has two modes:
// - live mode: the requests made to the mock HTTP server are transmitted to the live
// service, and responses are saved as test data files
// - test mode: the responses to requests to the mock HTTP server are read from the
// test data files
func NewMockWebServer(t *testing.T, liveServerBaseURL, testDataDir string, liveMode bool) *httptest.Server {
mockServerBaseURL := ""
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := NormalizedFullPath(r.URL)
log.Info("Mock HTTP Server: got request for path %s", r.URL.Path)
// TODO check request method (support POST?)
fixturePath := fmt.Sprintf("%s/%s", testDataDir, strings.ReplaceAll(path, "/", "_"))
if liveMode {
liveURL := fmt.Sprintf("%s%s", liveServerBaseURL, path)
request, err := http.NewRequest(r.Method, liveURL, nil)
if err != nil {
assert.Fail(t, "constructing an HTTP request to %s failed", liveURL, err)
}
for headerName, headerValues := range r.Header {
// do not pass on the encoding: let the Transport of the HTTP client handle that for us
if strings.ToLower(headerName) != "accept-encoding" {
for _, headerValue := range headerValues {
request.Header.Add(headerName, headerValue)
}
}
}
response, err := http.DefaultClient.Do(request)
if err != nil {
assert.Fail(t, "HTTP request to %s failed: %s", liveURL, err)
}
fixture, err := os.Create(fixturePath)
if err != nil {
assert.Fail(t, fmt.Sprintf("failed to open the fixture file %s for writing", fixturePath), err)
}
defer fixture.Close()
fixtureWriter := bufio.NewWriter(fixture)
for headerName, headerValues := range response.Header {
for _, headerValue := range headerValues {
if strings.ToLower(headerName) != "host" {
_, err := fixtureWriter.WriteString(fmt.Sprintf("%s: %s\n", headerName, headerValue))
if err != nil {
assert.Fail(t, "writing the header of the HTTP response to the fixture file failed", err)
}
}
}
}
if _, err := fixtureWriter.WriteString("\n"); err != nil {
assert.Fail(t, "writing the header of the HTTP response to the fixture file failed")
}
fixtureWriter.Flush()
reader := response.Body
content, err := io.ReadAll(reader)
if err != nil {
assert.Fail(t, "reading the response of the HTTP request to %s failed: %s", liveURL, err)
}
log.Info("Mock HTTP Server: writing response to %s", fixturePath)
if _, err := fixture.Write(content); err != nil {
assert.Fail(t, "writing the body of the HTTP response to the fixture file failed", err)
}
if err := fixture.Sync(); err != nil {
assert.Fail(t, "writing the body of the HTTP response to the fixture file failed", err)
}
}
fixture, err := os.ReadFile(fixturePath)
if err != nil {
assert.Fail(t, "missing mock HTTP response: "+fixturePath)
return
}
w.WriteHeader(http.StatusOK)
// parse back the fixture file into a series of HTTP headers followed by response body
lines := strings.Split(string(fixture), "\n")
for idx, line := range lines {
colonIndex := strings.Index(line, ": ")
if colonIndex != -1 {
w.Header().Set(line[0:colonIndex], line[colonIndex+2:])
} else {
// we reached the end of the headers (empty line), so what follows is the body
responseBody := strings.Join(lines[idx+1:], "\n")
// replace any mention of the live HTTP service by the mocked host
responseBody = strings.ReplaceAll(responseBody, liveServerBaseURL, mockServerBaseURL)
if _, err := w.Write([]byte(responseBody)); err != nil {
assert.Fail(t, "writing the body of the HTTP response failed", err)
}
break
}
}
}))
mockServerBaseURL = server.URL
return server
}
func NormalizedFullPath(url *url.URL) string {
// TODO normalize path (remove trailing slash?)
// TODO normalize RawQuery (order query parameters?)
if len(url.Query()) == 0 {
return url.EscapedPath()
}
return fmt.Sprintf("%s?%s", url.EscapedPath(), url.RawQuery)
}

View file

@ -13,6 +13,7 @@ import (
"testing" "testing"
"time" "time"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/json"
base "code.gitea.io/gitea/modules/migration" base "code.gitea.io/gitea/modules/migration"
@ -21,18 +22,15 @@ import (
) )
func TestGitlabDownloadRepo(t *testing.T) { func TestGitlabDownloadRepo(t *testing.T) {
// Skip tests if Gitlab token is not found // If a GitLab access token is provided, this test will make HTTP requests to the live gitlab.com instance.
// When doing so, the responses from gitlab.com will be saved as test data files.
// If no access token is available, those cached responses will be used instead.
gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN") gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN")
if gitlabPersonalAccessToken == "" { fixturePath := "./testdata/gitlab/full_download"
t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment") server := unittest.NewMockWebServer(t, "https://gitlab.com", fixturePath, true)
} defer server.Close()
resp, err := http.Get("https://gitlab.com/gitea/test_repo") downloader, err := NewGitlabDownloader(context.Background(), server.URL, "gitea/test_repo", "", "", gitlabPersonalAccessToken)
if err != nil || resp.StatusCode != http.StatusOK {
t.Skipf("Can't access test repo, skipping %s", t.Name())
}
downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
if err != nil { if err != nil {
t.Fatalf("NewGitlabDownloader is nil: %v", err) t.Fatalf("NewGitlabDownloader is nil: %v", err)
} }
@ -43,8 +41,8 @@ func TestGitlabDownloadRepo(t *testing.T) {
Name: "test_repo", Name: "test_repo",
Owner: "", Owner: "",
Description: "Test repository for testing migration from gitlab to gitea", Description: "Test repository for testing migration from gitlab to gitea",
CloneURL: "https://gitlab.com/gitea/test_repo.git", CloneURL: server.URL + "/gitea/test_repo.git",
OriginalURL: "https://gitlab.com/gitea/test_repo", OriginalURL: server.URL + "/gitea/test_repo",
DefaultBranch: "master", DefaultBranch: "master",
}, repo) }, repo)
@ -281,10 +279,10 @@ func TestGitlabDownloadRepo(t *testing.T) {
UserName: "real6543", UserName: "real6543",
Content: "tada", Content: "tada",
}}, }},
PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch", PatchURL: server.URL + "/gitea/test_repo/-/merge_requests/2.patch",
Head: base.PullRequestBranch{ Head: base.PullRequestBranch{
Ref: "feat/test", Ref: "feat/test",
CloneURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2", CloneURL: server.URL + "/gitea/test_repo/-/merge_requests/2",
SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23", SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23",
RepoName: "test_repo", RepoName: "test_repo",
OwnerName: "lafriks", OwnerName: "lafriks",