Merge branch 'main' into feature/oss-209

Author: kashifkhan0771

main.go

@@ -463,6 +463,7 @@ func run(state overseer.State) {
 
 	// OSS Default simplified gitlab enumeration
 	feature.UseSimplifiedGitlabEnumeration.Store(true)
+	feature.GitlabProjectsPerPage.Store(100)
 
 	// OSS Default using github graphql api for issues, pr's and comments
 	feature.UseGithubGraphqlAPI.Store(true)

pkg/analyzer/analyzers/launchdarkly/requests.go

@@ -38,7 +38,7 @@ var (
 		webhooksKey:     "/v2/webhooks",
 		/*
 			TODO:
-			release piplelines: https://launchdarkly.com/docs/api/release-pipelines-beta/get-all-release-pipelines (Beta)
+			release pipelines: https://launchdarkly.com/docs/api/release-pipelines-beta/get-all-release-pipelines (Beta)
 			insight deployments: https://launchdarkly.com/docs/api/insights-deployments-beta/get-deployments (Beta)
 			delivery configuration: https://launchdarkly.com/docs/api/integration-delivery-configurations-beta/get-integration-delivery-configuration-by-environment (Beta)
 			metrics: https://launchdarkly.com/docs/api/metrics-beta/get-metric-groups (Beta)

pkg/detectors/kontent/kontent.go

@@ -25,7 +25,7 @@ var (
 	apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kontent"}) + common.BuildRegexJWT("30,34", "200,400", "40,43"))
 	envIDPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"kontent", "env"}) + common.UUIDPattern)
 
-	// API return this error when the environment does not exist or the api key does not have the persmission to access that environment
+	// API return this error when the environment does not exist or the api key does not have the permission to access that environment
 	envErr = "The specified API key does not provide the permissions required to access the environment"
 )
 

pkg/detectors/satismeterwritekey/satismeterwritekey_integration_test.go

@@ -69,7 +69,7 @@ func TestSatismeterWritekey_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a satismeterwritekey project %s satismeter writekey %s and satismeter token %s in here but not vaild", inactiveProjectID, inactiveWriteKey, inactiveToken)), // the secret would satisfy the regex but not pass validation,
+				data:   []byte(fmt.Sprintf("You can find a satismeterwritekey project %s satismeter writekey %s and satismeter token %s in here but not valid", inactiveProjectID, inactiveWriteKey, inactiveToken)), // the secret would satisfy the regex but not pass validation,
 				verify: true,
 			},
 			want: []detectors.Result{

pkg/engine/gitlab.go

@@ -63,6 +63,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (source
 	}
 
 	connection.NoCleanup = c.NoCleanup
+
 	connection.PrintLegacyJson = c.PrintLegacyJSON
 
 	var conn anypb.Any

pkg/feature/feature.go

@@ -10,6 +10,7 @@ var (
 	UserAgentSuffix                AtomicString
 	UseSimplifiedGitlabEnumeration atomic.Bool
 	UseGitMirror                   atomic.Bool
+	GitlabProjectsPerPage          atomic.Int64
 	UseGithubGraphqlAPI            atomic.Bool // use github graphql api to fetch issues, pr's and comments
 )
 

pkg/sources/github/github.go

@@ -206,6 +206,11 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
 	return false
 }
 
+// wantRepo returns true if the repository should be included based on include/exclude patterns
+func (c *filteredRepoCache) wantRepo(s string) bool {
+	return !c.ignoreRepo(s) && c.includeRepo(s)
+}
+
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	err := git.CmdCheck()
@@ -434,14 +439,29 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
 	// Double make sure that all enumerated repositories in the
 	// filteredRepoCache have an entry in the repoInfoCache.
 	for _, repo := range s.filteredRepoCache.Values() {
-		ctx := context.WithValue(ctx, "repo", repo)
+		// Extract the repository name from the URL for filtering
+		repoName := repo
+		if strings.Contains(repo, "/") {
+			// Try to extract org/repo name from URL
+			if strings.Contains(repo, "github.com") {
+				parts := strings.Split(repo, "/")
+				if len(parts) >= 2 {
+					repoName = parts[len(parts)-2] + "/" + strings.TrimSuffix(parts[len(parts)-1], ".git")
+				}
+			}
+		}
 
-		repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
-		if err != nil {
-			ctx.Logger().Error(err, "error caching repo info")
-			_ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+		// Final filter check - only include repositories that pass the filter
+		if s.filteredRepoCache.wantRepo(repoName) {
+			ctx = context.WithValue(ctx, "repo", repo)
+
+			repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
+			if err != nil {
+				ctx.Logger().Error(err, "error caching repo info")
+				_ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+			}
+			s.repos = append(s.repos, repo)
 		}
-		s.repos = append(s.repos, repo)
 	}
 	githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))
 	ctx.Logger().Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))

pkg/sources/github/github_test.go

@@ -423,6 +423,41 @@ func TestNormalizeRepos(t *testing.T) {
 	}
 }
 
+func TestNormalizeRepo(t *testing.T) {
+	// Test that normalizeRepo correctly identifies URLs with protocols
+	source := &Source{}
+
+	// Test case 1: HTTP URL
+	result, err := source.normalizeRepo("https://github.com/org/repo.git")
+	assert.NoError(t, err)
+	assert.Contains(t, result, "github.com/org/repo")
+
+	// Test case 2: HTTP URL without .git
+	result, err = source.normalizeRepo("http://github.com/org/repo")
+	assert.NoError(t, err)
+	assert.Contains(t, result, "github.com/org/repo")
+
+	// Test case 3: Git protocol URL
+	result, err = source.normalizeRepo("git://github.com/org/repo.git")
+	assert.NoError(t, err)
+	assert.Contains(t, result, "github.com/org/repo")
+
+	// Test case 4: SSH URL
+	result, err = source.normalizeRepo("ssh://[email protected]/org/repo.git")
+	assert.NoError(t, err)
+	assert.Contains(t, result, "github.com/org/repo")
+
+	// Test case 5: Org/repo format (should convert to full URL)
+	result, err = source.normalizeRepo("org/repo")
+	assert.NoError(t, err)
+	assert.Contains(t, result, "github.com/org/repo")
+
+	// Test case 6: Invalid format (no protocol, no slash)
+	_, err = source.normalizeRepo("invalid")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "no repositories found")
+}
+
 func TestHandleRateLimit(t *testing.T) {
 	s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
 	ctx := context.Background()
@@ -493,7 +528,7 @@ func TestEnumerateWithToken(t *testing.T) {
 		JSON(map[string]string{"login": "super-secret-user"})
 
 	gock.New("https://api.github.com").
-		Get("/users/super-secret-user/repos").
+		Get("/user/repos").
 		MatchParam("per_page", "100").
 		Reply(200).
 		JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}})
@@ -574,7 +609,7 @@ func TestEnumerate(t *testing.T) {
 
 	//
 	gock.New("https://api.github.com").
-		Get("/users/super-secret-user/repos").
+		Get("/user/repos").
 		Reply(200).
 		JSON(`[{"name": "super-secret-repo", "full_name": "super-secret-user/super-secret-repo", "owner": {"login": "super-secret-user"}, "clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "has_wiki": false, "size": 1}]`)
 
@@ -978,6 +1013,39 @@ func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) {
 	}
 }
 
+func TestRepositoryFiltering(t *testing.T) {
+	// Test that the filteredRepoCache correctly filters repositories
+	source := &Source{}
+
+	// Test case 1: No filters specified (should include everything)
+	cache1 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{})
+	assert.True(t, cache1.wantRepo("org/repo1"))
+	assert.True(t, cache1.wantRepo("org/repo2"))
+	assert.True(t, cache1.wantRepo("org/repo3"))
+
+	// Test case 2: Include filter specified (should only include matching repos)
+	cache2 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1", "org/repo2"}, []string{})
+	assert.True(t, cache2.wantRepo("org/repo1"))
+	assert.True(t, cache2.wantRepo("org/repo2"))
+	assert.False(t, cache2.wantRepo("org/repo3"))
+
+	// Test case 3: Exclude filter specified (should exclude matching repos)
+	cache3 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{"org/repo1"})
+	assert.False(t, cache3.wantRepo("org/repo1"))
+	assert.True(t, cache3.wantRepo("org/repo2"))
+	assert.True(t, cache3.wantRepo("org/repo3"))
+
+	// Test case 4: Both include and exclude filters (exclude takes precedence)
+	cache4 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1"}, []string{"org/repo1"})
+	assert.False(t, cache4.wantRepo("org/repo1"))
+
+	// Test case 5: Wildcard patterns
+	cache5 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/*"}, []string{})
+	assert.True(t, cache5.wantRepo("org/repo1"))
+	assert.True(t, cache5.wantRepo("org/repo2"))
+	assert.False(t, cache5.wantRepo("other/repo1"))
+}
+
 func noopReporter() sources.UnitReporter {
 	return sources.VisitorReporter{
 		VisitUnit: func(context.Context, sources.SourceUnit) error {

pkg/sources/github/repo.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"regexp"
 	"strings"
 	"sync"
 
@@ -273,6 +274,12 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
 			}
 
 			repoName, repoURL := r.GetFullName(), r.GetCloneURL()
+
+			// Check if we should process this repository based on the filter
+			if !s.filteredRepoCache.wantRepo(repoName) {
+				continue
+			}
+
 			s.totalRepoSize += r.GetSize()
 			s.filteredRepoCache.Set(repoName, repoURL)
 			s.cacheRepoInfo(r)
@@ -349,10 +356,17 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool {
 
 // normalizeRepo normalizes a GitHub repository URL or name to its canonical form.
 func (s *Source) normalizeRepo(repo string) (string, error) {
-	// if the string contains a '/', assume it's a GitHub repository.
-	if strings.ContainsRune(repo, '/') {
+
+	// If it's a full URL (has protocol), normalize it
+	if regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
+
 		return giturl.NormalizeGithubRepo(repo)
 	}
+	// If it's a repository name (contains / but not http), convert to full URL first
+	if strings.Contains(repo, "/") && !regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
+		fullURL := "https://github.com/" + repo
+		return giturl.NormalizeGithubRepo(fullURL)
+	}
 
 	return "", fmt.Errorf("no repositories found for %s", repo)
 }

pkg/sources/gitlab/gitlab.go

@@ -66,9 +66,12 @@ type Source struct {
 
 	useAuthInUrl bool
 
-	clonePath       string
-	noCleanup       bool
+	clonePath string
+	noCleanup bool
+
 	printLegacyJSON bool
+
+	projectsPerPage int
 }
 
 // WithCustomContentWriter sets the useCustomContentWriter flag on the source.
@@ -171,6 +174,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	s.clonePath = conn.GetClonePath()
 	s.noCleanup = conn.GetNoCleanup()
 	s.printLegacyJSON = conn.GetPrintLegacyJson()
+	s.projectsPerPage = int(feature.GitlabProjectsPerPage.Load())
+
+	if s.projectsPerPage > 100 {
+		return fmt.Errorf("invalid config: maximum allowed projects per page for gitlab is 100")
+	}
 
 	// configuration uses the inverse logic of the `useAuthInUrl` flag.
 	s.useAuthInUrl = !conn.RemoveAuthInUrl
@@ -546,10 +554,10 @@ func (s *Source) getAllProjectRepos(
 	}
 
 	const (
-		orderBy         = "id" // TODO: Use keyset pagination (https://docs.gitlab.com/ee/api/rest/index.html#keyset-based-pagination)
-		paginationLimit = 100  // Default is 20, max is 100.
+		orderBy = "id"
 	)
-	listOpts := gitlab.ListOptions{PerPage: paginationLimit}
+	// Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20
+	listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage}
 
 	projectQueryOptions := &gitlab.ListProjectsOptions{OrderBy: gitlab.Ptr(orderBy), ListOptions: listOpts}
 	for {
@@ -654,14 +662,13 @@ func (s *Source) getAllProjectReposV2(
 ) error {
 	gitlabReposEnumerated.WithLabelValues(s.name).Set(0)
 
-	const paginationLimit = 100 // default is 20, max is 100.
-
 	// example: https://gitlab.com/gitlab-org/api/client-go/-/blob/main/examples/pagination.go#L55
 	listOpts := gitlab.ListOptions{
 		OrderBy:    "id",
 		Pagination: "keyset", // https://docs.gitlab.com/api/rest/#keyset-based-pagination
-		PerPage:    paginationLimit,
-		Sort:       "asc",
+		// Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20
+		PerPage: s.projectsPerPage,
+		Sort:    "asc",
 	}
 
 	projectQueryOptions := &gitlab.ListProjectsOptions{
@@ -756,11 +763,10 @@ func (s *Source) getAllProjectReposInGroups(
 
 	var projectsWithNamespace []string
 	const (
-		orderBy         = "id"
-		paginationLimit = 100
+		orderBy = "id"
 	)
 
-	listOpts := gitlab.ListOptions{PerPage: paginationLimit}
+	listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage}
 	projectOpts := &gitlab.ListGroupProjectsOptions{
 		ListOptions:      listOpts,
 		OrderBy:          gitlab.Ptr(orderBy),