From 52053b138948bd74c7eb88c0796c2e18f4247f3c Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 28 Nov 2023 13:40:50 +0100 Subject: [PATCH] Mock HTTP requests in GitLab migration test This introduces a new utility which can be added to other tests making HTTP calls to a live service, to cache the responses of this service in the repository. --- .deadcode-out | 2 + models/unittest/mock_http.go | 131 +++++++++++++++++++++++++++++ services/migrations/gitlab_test.go | 26 +++--- 3 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 models/unittest/mock_http.go diff --git a/.deadcode-out b/.deadcode-out index 5a4aa44a0f..78d4f02758 100644 --- a/.deadcode-out +++ b/.deadcode-out @@ -121,6 +121,8 @@ package "code.gitea.io/gitea/models/unittest" func LoadFixtures func Copy func CopyDir + func NewMockWebServer + func NormalizedFullPath func FixturesDir func fatalTestError func InitSettings diff --git a/models/unittest/mock_http.go b/models/unittest/mock_http.go new file mode 100644 index 0000000000..a76917e241 --- /dev/null +++ b/models/unittest/mock_http.go @@ -0,0 +1,131 @@ +// Copyright 2017 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package unittest + +import ( + "bufio" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "code.gitea.io/gitea/modules/log" + + "github.com/stretchr/testify/assert" +) + +// Mocks HTTP responses of a third-party service (such as GitHub, GitLab…) +// This has two modes: +// - live mode: the requests made to the mock HTTP server are transmitted to the live +// service, and responses are saved as test data files +// - test mode: the responses to requests to the mock HTTP server are read from the +// test data files +func NewMockWebServer(t *testing.T, liveServerBaseURL, testDataDir string, liveMode bool) *httptest.Server { + mockServerBaseURL := "" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := NormalizedFullPath(r.URL) + log.Info("Mock HTTP Server: got request for path %s", r.URL.Path) + // TODO check request method (support POST?) + fixturePath := fmt.Sprintf("%s/%s", testDataDir, strings.ReplaceAll(path, "/", "_")) + if liveMode { + liveURL := fmt.Sprintf("%s%s", liveServerBaseURL, path) + + request, err := http.NewRequest(r.Method, liveURL, nil) + if err != nil { + assert.Fail(t, "constructing an HTTP request to %s failed", liveURL, err) + } + for headerName, headerValues := range r.Header { + // do not pass on the encoding: let the Transport of the HTTP client handle that for us + if strings.ToLower(headerName) != "accept-encoding" { + for _, headerValue := range headerValues { + request.Header.Add(headerName, headerValue) + } + } + } + + response, err := http.DefaultClient.Do(request) + if err != nil { + assert.Fail(t, "HTTP request to %s failed: %s", liveURL, err) + } + + fixture, err := os.Create(fixturePath) + if err != nil { + assert.Fail(t, fmt.Sprintf("failed to open the fixture file %s for writing", fixturePath), err) + } + defer fixture.Close() + fixtureWriter := bufio.NewWriter(fixture) + + for headerName, headerValues := range response.Header { + for _, headerValue := range headerValues { + if strings.ToLower(headerName) != "host" { + _, err := fixtureWriter.WriteString(fmt.Sprintf("%s: %s\n", headerName, headerValue)) + if err != nil { + assert.Fail(t, "writing the header of the HTTP response to the fixture file failed", err) + } + } + } + } + if _, err := fixtureWriter.WriteString("\n"); err != nil { + assert.Fail(t, "writing the header of the HTTP response to the fixture file failed") + } + fixtureWriter.Flush() + + reader := response.Body + content, err := io.ReadAll(reader) + if err != nil { + assert.Fail(t, "reading the response of the HTTP request to %s failed: %s", liveURL, err) + } + log.Info("Mock HTTP Server: writing response to %s", fixturePath) + if _, err := fixture.Write(content); err != nil { + assert.Fail(t, "writing the body of the HTTP response to the fixture file failed", err) + } + + if err := fixture.Sync(); err != nil { + assert.Fail(t, "writing the body of the HTTP response to the fixture file failed", err) + } + } + + fixture, err := os.ReadFile(fixturePath) + if err != nil { + assert.Fail(t, "missing mock HTTP response: "+fixturePath) + return + } + + w.WriteHeader(http.StatusOK) + + // parse back the fixture file into a series of HTTP headers followed by response body + lines := strings.Split(string(fixture), "\n") + for idx, line := range lines { + colonIndex := strings.Index(line, ": ") + if colonIndex != -1 { + w.Header().Set(line[0:colonIndex], line[colonIndex+2:]) + } else { + // we reached the end of the headers (empty line), so what follows is the body + responseBody := strings.Join(lines[idx+1:], "\n") + // replace any mention of the live HTTP service by the mocked host + responseBody = strings.ReplaceAll(responseBody, liveServerBaseURL, mockServerBaseURL) + if _, err := w.Write([]byte(responseBody)); err != nil { + assert.Fail(t, "writing the body of the HTTP response failed", err) + } + break + } + } + })) + mockServerBaseURL = server.URL + return server +} + +func NormalizedFullPath(url *url.URL) string { + // TODO normalize path (remove trailing slash?) + // TODO normalize RawQuery (order query parameters?) + if len(url.Query()) == 0 { + return url.EscapedPath() + } + return fmt.Sprintf("%s?%s", url.EscapedPath(), url.RawQuery) +} diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 697066c8bf..6eb1c6f125 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/json" base "code.gitea.io/gitea/modules/migration" @@ -21,18 +22,15 @@ import ( ) func TestGitlabDownloadRepo(t *testing.T) { - // Skip tests if Gitlab token is not found + // If a GitLab access token is provided, this test will make HTTP requests to the live gitlab.com instance. + // When doing so, the responses from gitlab.com will be saved as test data files. + // If no access token is available, those cached responses will be used instead. gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN") - if gitlabPersonalAccessToken == "" { - t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment") - } + fixturePath := "./testdata/gitlab/full_download" + server := unittest.NewMockWebServer(t, "https://gitlab.com", fixturePath, true) + defer server.Close() - resp, err := http.Get("https://gitlab.com/gitea/test_repo") - if err != nil || resp.StatusCode != http.StatusOK { - t.Skipf("Can't access test repo, skipping %s", t.Name()) - } - - downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) + downloader, err := NewGitlabDownloader(context.Background(), server.URL, "gitea/test_repo", "", "", gitlabPersonalAccessToken) if err != nil { t.Fatalf("NewGitlabDownloader is nil: %v", err) } @@ -43,8 +41,8 @@ func TestGitlabDownloadRepo(t *testing.T) { Name: "test_repo", Owner: "", Description: "Test repository for testing migration from gitlab to gitea", - CloneURL: "https://gitlab.com/gitea/test_repo.git", - OriginalURL: "https://gitlab.com/gitea/test_repo", + CloneURL: server.URL + "/gitea/test_repo.git", + OriginalURL: server.URL + "/gitea/test_repo", DefaultBranch: "master", }, repo) @@ -281,10 +279,10 @@ func TestGitlabDownloadRepo(t *testing.T) { UserName: "real6543", Content: "tada", }}, - PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch", + PatchURL: server.URL + "/gitea/test_repo/-/merge_requests/2.patch", Head: base.PullRequestBranch{ Ref: "feat/test", - CloneURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2", + CloneURL: server.URL + "/gitea/test_repo/-/merge_requests/2", SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23", RepoName: "test_repo", OwnerName: "lafriks",