test: reuse Gitlab repos (#1923)

Author: Camila-B

e2e/nomostest/gitproviders/bitbucket.go

@@ -16,6 +16,7 @@ package gitproviders
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -35,7 +36,8 @@ const (
 	bitbucketProject = "CSCI"
 
 	// PrivateSSHKey is secret name of the private SSH key stored in the Cloud Secret Manager.
-	PrivateSSHKey = "config-sync-ci-ssh-private-key"
+	PrivateSSHKey           = "config-sync-ci-ssh-private-key"
+	bitbucketRequestTimeout = 10 * time.Second
 )
 
 // BitbucketClient is the client that calls the Bitbucket REST APIs.
@@ -59,9 +61,7 @@ func newBitbucketClient(repoSuffix string, logger *testlogger.TestLogger) (*Bitb
 		logger:     logger,
 		workspace:  *e2e.BitbucketWorkspace,
 		repoSuffix: repoSuffix,
-		httpClient: &http.Client{
-			Timeout: 10 * time.Second,
-		},
+		httpClient: &http.Client{},
 	}
 
 	var err error
@@ -112,7 +112,9 @@ func (b *BitbucketClient) CreateRepository(name string) (string, error) {
 	}
 
 	// Check if repository already exists
-	resp, err := b.sendRequest(http.MethodGet, repoURL, accessToken, nil)
+	getRequestCtx, getRequestCancel := context.WithTimeout(context.Background(), bitbucketRequestTimeout)
+	defer getRequestCancel()
+	resp, err := b.sendRequest(getRequestCtx, http.MethodGet, repoURL, accessToken, nil)
 	if err != nil {
 		return "", err
 	}
@@ -135,13 +137,15 @@ func (b *BitbucketClient) CreateRepository(name string) (string, error) {
 	}
 
 	// Create new Bitbucket repository
-	resp, err = b.sendRequest(http.MethodPost, repoURL, accessToken, bytes.NewReader(jsonPayload))
+	postRequestCtx, postRequestCancel := context.WithTimeout(context.Background(), bitbucketRequestTimeout)
+	resp, err = b.sendRequest(postRequestCtx, http.MethodPost, repoURL, accessToken, bytes.NewReader(jsonPayload))
 
 	defer func() {
 		if closeErr := resp.Body.Close(); closeErr != nil {
 			// Log the error as just printing it might be missed.
 			b.logger.Infof("failed to close response body: %v\n", closeErr)
 		}
+		postRequestCancel()
 	}()
 
 	if err != nil {
@@ -159,8 +163,8 @@ func (b *BitbucketClient) CreateRepository(name string) (string, error) {
 }
 
 // sendRequest sends an HTTP request to the Bitbucket API.
-func (b *BitbucketClient) sendRequest(method, url, accessToken string, body io.Reader) (*http.Response, error) {
-	req, err := http.NewRequest(method, url, body)
+func (b *BitbucketClient) sendRequest(ctx context.Context, method, url, accessToken string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
 	if err != nil {
 		return nil, fmt.Errorf("error creating request: %w", err)
 	}

e2e/nomostest/gitproviders/git-provider.go

@@ -58,7 +58,8 @@ func NewGitProvider(t testing.NTB, provider, clusterName string, logger *testlog
 		}
 		return client
 	case e2e.GitLab:
-		client, err := newGitlabClient()
+		repoSuffix := *e2e.GCPProject + "/" + clusterName
+		client, err := newGitlabClient(repoSuffix, logger)
 		if err != nil {
 			t.Fatal(err)
 		}

e2e/nomostest/gitproviders/gitlab.go

@@ -15,32 +15,40 @@
 package gitproviders
 
 import (
+	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"os/exec"
-	"strings"
+	"io"
+	"net/http"
 	"time"
 
 	"github.com/GoogleContainerTools/config-sync/e2e"
-	"github.com/google/uuid"
-	"go.uber.org/multierr"
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/gitproviders/util"
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/testlogger"
 )
 
 const (
-	projectNameMaxLength = 256
 	groupID              = 15698791
 	groupName            = "configsync"
+	gitlabRequestTimeout = 10 * time.Second
 )
 
 // GitlabClient is the client that will call Gitlab REST APIs.
 type GitlabClient struct {
 	privateToken string
+	logger       *testlogger.TestLogger
+	// repoSuffix is used to avoid overlap
+	repoSuffix string
+	httpClient *http.Client
 }
 
 // newGitlabClient instantiates a new GitlabClient.
-func newGitlabClient() (*GitlabClient, error) {
-	client := &GitlabClient{}
+func newGitlabClient(repoSuffix string, logger *testlogger.TestLogger) (*GitlabClient, error) {
+	client := &GitlabClient{
+		logger:     logger,
+		repoSuffix: repoSuffix,
+		httpClient: &http.Client{},
+	}
 
 	var err error
 
@@ -50,6 +58,10 @@ func newGitlabClient() (*GitlabClient, error) {
 	return client, nil
 }
 
+func (g *GitlabClient) fullName(name string) string {
+	return util.SanitizeGitlabRepoName(g.repoSuffix, name)
+}
+
 // Type returns the git provider type
 func (g *GitlabClient) Type() string {
 	return e2e.GitLab
@@ -68,190 +80,92 @@ func (g *GitlabClient) SyncURL(name string) string {
 // CreateRepository calls the POST API to create a project/repository on Gitlab.
 // The remote repo name is unique with a prefix of the local name.
 func (g *GitlabClient) CreateRepository(name string) (string, error) {
-	u, err := uuid.NewRandom()
+	fullName := g.fullName(name)
+	repoURL := fmt.Sprintf("https://gitlab.com/api/v4/projects?search=%s", fullName)
+
+	// Check if the repository already exists
+	getRequestCtx, getRequestCancel := context.WithTimeout(context.Background(), gitlabRequestTimeout)
+	defer getRequestCancel()
+	resp, err := g.sendRequest(getRequestCtx, http.MethodGet, repoURL, nil)
 	if err != nil {
-		return "", fmt.Errorf("failed to generate a new UUID: %w", err)
+		return "", err
 	}
 
-	repoName := name + "-" + u.String()
-	// Gitlab create projects API doesn't allow '/' character
-	// so all instances are replaced with '-'
-	repoName = strings.ReplaceAll(repoName, "/", "-")
-	if len(repoName) > projectNameMaxLength {
-		repoName = repoName[:projectNameMaxLength]
-	}
+	if resp.StatusCode == http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return "", fmt.Errorf("error reading response body: %w", err)
+		}
 
-	// Projects created under the `configsync` group (namespaceId: 15698791) has
-	// no protected branch.
-	out, err := exec.Command("curl", "-s", "--request", "POST",
-		fmt.Sprintf("https://gitlab.com/api/v4/projects?name=%s&namespace_id=%d&initialize_with_readme=true", repoName, groupID),
-		"--header", fmt.Sprintf("PRIVATE-TOKEN: %s", g.privateToken)).CombinedOutput()
+		var output []map[string]interface{}
+		if err := json.Unmarshal(body, &output); err != nil {
+			return "", fmt.Errorf("unmarshalling project search response: %w", err)
+		}
 
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", string(out), err)
-	}
-	if !strings.Contains(string(out), fmt.Sprintf("\"name\":\"%s\"", repoName)) {
-		return "", errors.New(string(out))
-	}
+		// the assumption is that our project name is unique, so we'll get exactly 1 result
+		if len(output) > 0 {
+			return fullName, nil
 
-	return repoName, nil
-}
+		}
+	} else {
+		return "", fmt.Errorf("failed to check if repository exists: status %d", resp.StatusCode)
+	}
 
-// GetProjectID is a helper function for DeleteRepositories
-// since Gitlab API only deletes by id
-func GetProjectID(g *GitlabClient, name string) (string, error) {
-	out, err := exec.Command("curl", "-s", "--request", "GET",
-		fmt.Sprintf("https://gitlab.com/api/v4/projects?search=%s", name),
-		"--header", fmt.Sprintf("PRIVATE-TOKEN: %s", g.privateToken)).CombinedOutput()
+	// Creates repository
+	repoURL = fmt.Sprintf("https://gitlab.com/api/v4/projects?name=%s&namespace_id=%d&initialize_with_readme=true", fullName, groupID)
+	postRequestCtx, postRequestCancel := context.WithTimeout(context.Background(), gitlabRequestTimeout)
+	resp, err = g.sendRequest(postRequestCtx, http.MethodPost, repoURL, nil)
 
+	defer func() {
+		if closeErr := resp.Body.Close(); closeErr != nil {
+			g.logger.Infof("failed to close response body: %v\n", closeErr)
+		}
+		postRequestCancel()
+	}()
 	if err != nil {
-		return "", fmt.Errorf("Failure retrieving id for project %s: %w", name, err)
+		return "", err
 	}
 
-	var response []interface{}
-
-	err = json.Unmarshal(out, &response)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", fmt.Errorf("%s: %w", string(out), err)
+		return "", fmt.Errorf("error reading response body: %w", err)
 	}
 
-	var float float64
-	var ok bool
-
-	// the assumption is that our project name is unique, so we'll get exactly 1 result
-	if len(response) < 1 {
-		return "", fmt.Errorf("Project with name %s: %w", name, err)
-	}
-	if len(response) > 1 {
-		return "", fmt.Errorf("Project with name %s is not unique: %w", name, err)
+	if resp.StatusCode != http.StatusCreated {
+		return "", fmt.Errorf("failed to create repository: status %d: %s", resp.StatusCode, string(body))
 	}
-	m := response[0].(map[string]interface{})
-	if x, found := m["id"]; found {
-		if float, ok = x.(float64); !ok {
-			return "", fmt.Errorf("Project id in the respose isn't a float: %w", err)
-		}
-	} else {
-		return "", fmt.Errorf("Project id wasn't found in the response: %w", err)
-	}
-	id := fmt.Sprintf("%.0f", float)
 
-	return id, nil
+	return fullName, nil
 }
 
-// DeleteRepositories calls the DELETE API to delete the list of project name in Gitlab.
-func (g *GitlabClient) DeleteRepositories(names ...string) error {
-	var errs error
-
-	for _, name := range names {
-		id, err := GetProjectID(g, name)
-		if err != nil {
-			errs = multierr.Append(errs, fmt.Errorf("invalid repo name: %w", err))
-		} else {
-			out, err := exec.Command("curl", "-s", "--request", "DELETE",
-				fmt.Sprintf("https://gitlab.com/api/v4/projects/%s", id),
-				"--header", fmt.Sprintf("PRIVATE-TOKEN: %s", g.privateToken)).CombinedOutput()
-
-			if err != nil {
-				errs = multierr.Append(errs, fmt.Errorf("%s: %w", string(out), err))
-			}
-
-			response := string(out)
-			// Check for successful deletion (202 Accepted)
-			if strings.Contains(response, "\"message\":\"202 Accepted\"") {
-				continue
-			}
-			// Check if project is already marked for deletion (this is also a success case)
-			if strings.Contains(response, "Project has already been marked for deletion") {
-				continue
-			}
-			// Any other response is treated as an error
-			return errors.New(response)
-		}
-	}
-	return errs
+// DeleteRepositories is a no-op because Gitlab repo names are determined by the
+// test cluster name and RSync namespace and name, so they can be reset and reused
+// across test runs
+func (g *GitlabClient) DeleteRepositories(_ ...string) error {
+	g.logger.Info("[Gitlab] Skip deletion of repos")
+	return nil
 }
 
-// DeleteObsoleteRepos deletes all projects that has been inactive more than 24 hours
+// DeleteObsoleteRepos is a no-op because Gitlab repo names are determined by the
+// test cluster name and RSync namespace and name, so it can be reused if it
+// failed to be deleted after the test.
 func (g *GitlabClient) DeleteObsoleteRepos() error {
-	repos, _ := g.GetObsoleteRepos()
-
-	err := g.DeleteRepoByID(repos...)
-	return err
+	return nil
 }
 
-// DeleteRepoByID calls the DELETE API to delete the list of project id in Gitlab.
-func (g *GitlabClient) DeleteRepoByID(ids ...string) error {
-	var errs error
-
-	for _, id := range ids {
-		out, err := exec.Command("curl", "-s", "--request", "DELETE",
-			fmt.Sprintf("https://gitlab.com/api/v4/projects/%s", id),
-			"--header", fmt.Sprintf("PRIVATE-TOKEN: %s", g.privateToken)).CombinedOutput()
-
-		if err != nil {
-			errs = multierr.Append(errs, fmt.Errorf("%s: %w", string(out), err))
-		}
-
-		response := string(out)
-		// Check for successful deletion (202 Accepted)
-		if strings.Contains(response, "\"message\":\"202 Accepted\"") {
-			continue
-		}
-		// Check if project is already marked for deletion (this is also a success case)
-		if strings.Contains(response, "Project has already been marked for deletion") {
-			continue
-		}
-		// Any other response is treated as an error
-		return fmt.Errorf("unexpected response in DeleteRepoByID: %s", response)
+// sendRequest sends an HTTP request to the Gitlab API.
+func (g *GitlabClient) sendRequest(ctx context.Context, method, url string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return nil, fmt.Errorf("error creating request: %w", err)
 	}
-	return errs
-}
-
-// GetObsoleteRepos is a helper function to get all project ids that has been inactive more than 24 hours
-func (g *GitlabClient) GetObsoleteRepos() ([]string, error) {
-	var result []string
-	pageNum := 1
-	cutOffDate := time.Now().AddDate(0, 0, -1)
-	formattedDate := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
-		cutOffDate.Year(), cutOffDate.Month(), cutOffDate.Day(),
-		cutOffDate.Hour(), cutOffDate.Minute(), cutOffDate.Second())
+	req.Header.Add("PRIVATE-TOKEN", g.privateToken)
+	req.Header.Set("Content-Type", "application/json")
 
-	for {
-		out, err := exec.Command("curl", "-s", "--request", "GET",
-			fmt.Sprintf("https://gitlab.com/api/v4/projects?last_activity_before=%s&owned=yes&simple=yes&page=%d", formattedDate, pageNum),
-			"--header", fmt.Sprintf("PRIVATE-TOKEN: %s", g.privateToken)).CombinedOutput()
-
-		if err != nil {
-			return result, fmt.Errorf("Failure retrieving obsolete repos: %w", err)
-		}
-
-		if len(out) <= 2 {
-			break
-		}
-
-		pageNum++
-		var response []interface{}
-
-		err = json.Unmarshal(out, &response)
-		if err != nil {
-			return nil, fmt.Errorf("%s: %w", string(out), err)
-		}
-
-		for i := range response {
-			m := response[i].(map[string]interface{})
-			if flt, found := m["id"]; found {
-				var id float64
-				var ok bool
-				if id, ok = flt.(float64); !ok {
-					return result, fmt.Errorf("Project id in the response isn't a float: %w", err)
-				}
-				result = append(result, fmt.Sprintf("%.0f", id))
-
-			} else {
-				return result, fmt.Errorf("Project id wasn't found in the response: %w", err)
-			}
-		}
+	resp, err := g.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error sending request: %w", err)
 	}
 
-	return result, nil
+	return resp, nil
 }