Implemented Graphql requests for Github PR's, Issues and comments scanning

main.go

@@ -467,6 +467,9 @@ func run(state overseer.State) {
 	feature.UseSimplifiedGitlabEnumeration.Store(true)
 	feature.GitlabProjectsPerPage.Store(100)
 
+	// OSS Default using github graphql api for issues, pr's and comments
+	feature.UseGithubGraphQLAPI.Store(false)
+
 	conf := &config.Config{}
 	if *configFilename != "" {
 		var err error

pkg/feature/feature.go

@@ -11,6 +11,7 @@ var (
 	UseSimplifiedGitlabEnumeration atomic.Bool
 	UseGitMirror                   atomic.Bool
 	GitlabProjectsPerPage          atomic.Int64
+	UseGithubGraphQLAPI            atomic.Bool // use github graphql api to fetch issues, pr's and comments
 )
 
 type AtomicString struct {

pkg/sources/github/github.go

@@ -27,6 +27,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -1105,8 +1106,14 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep
 	if s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists {
 		return s.processGistComments(ctx, urlString, urlParts, repoInfo, reporter, cutoffTime)
 	} else if s.includeIssueComments || s.includePRComments {
-		return s.processRepoComments(ctx, repoInfo, reporter, cutoffTime)
+		// if we need to use graphql api for repo issues, prs and comments
+		if feature.UseGithubGraphQLAPI.Load() {
+			return s.processRepoIssueandPRsWithCommentsGraphql(ctx, repoInfo, reporter, cutoffTime)
+		}
+
+		return s.processIssueandPRsWithCommentsREST(ctx, repoInfo, reporter, cutoffTime)
 	}
+
 	return nil
 }
 
@@ -1273,7 +1280,10 @@ var (
 	state = "all"
 )
 
-func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+func (s *Source) processIssueandPRsWithCommentsREST(
+	ctx context.Context, repoInfo repoInfo,
+	reporter sources.ChunkReporter, cutoffTime *time.Time,
+) error {
 	if s.includeIssueComments {
 		ctx.Logger().V(2).Info("Scanning issues")
 		if err := s.processIssues(ctx, repoInfo, reporter); err != nil {
@@ -1297,6 +1307,31 @@ func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, rep
 	return nil
 }
 
+func (s *Source) processRepoIssueandPRsWithCommentsGraphql(
+	ctx context.Context, repoInfo repoInfo,
+	reporter sources.ChunkReporter, cutoffTime *time.Time,
+) error {
+	if s.includeIssueComments {
+		ctx.Logger().V(2).Info("Scanning issues")
+		if err := s.processIssuesWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {
+			return err
+		}
+	}
+
+	if s.includePRComments {
+		ctx.Logger().V(2).Info("Scanning pull requests")
+		if err := s.processPRWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {
+			return err
+		}
+
+		if err := s.processReviewThreads(ctx, repoInfo, reporter, cutoffTime); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error {
 	bodyTextsOpts := &github.IssueListByRepoOptions{
 		Sort:      sortType,

pkg/sources/github/github_integration_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -968,6 +969,64 @@ func TestSource_Validate(t *testing.T) {
 	}
 }
 
+func TestSource_ScanCommentsWithGraphql(t *testing.T) {
+	feature.UseGithubGraphQLAPI.Store(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	source := &sourcespb.GitHub{
+		Repositories:               []string{"https://github.com/trufflesecurity/driftwood.git"},
+		IncludeIssueComments:       true,
+		IncludePullRequestComments: true,
+		Credential:                 &sourcespb.GitHub_Unauthenticated{},
+	}
+
+	wantChunk := sources.Chunk{
+		SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+		SourceName: "test source",
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Github{
+				Github: &source_metadatapb.Github{
+					Link:      "https://github.com/trufflesecurity/driftwood.git/issues/1",
+					Username:  "truffle-sandbox",
+					Timestamp: "2023-06-22 23:33:46 +0000 UTC",
+				},
+			},
+		},
+		Verify: false,
+	}
+
+	s := Source{}
+
+	conn, err := anypb.New(source)
+	assert.NoError(t, err)
+
+	err = s.Init(ctx, "test-source", 0, 0, false, conn, 4)
+	assert.NoError(t, err)
+
+	chunksCh := make(chan *sources.Chunk, 1)
+	go func() {
+		// Close the channel
+		defer close(chunksCh)
+		err = s.Chunks(ctx, chunksCh)
+		assert.NoError(t, err)
+	}()
+
+	i := 0
+	for gotChunk := range chunksCh {
+		// Skip chunks that are not comments.
+		if gotChunk.SourceMetadata.GetGithub().GetCommit() != "" {
+			continue
+		}
+		i++
+		githubCommentCheckFunc(gotChunk, &wantChunk, i, t, "test-source")
+	}
+
+	// Confirm all comments were processed.
+	assert.Equal(t, i, 5)
+}
+
 type countChunkReporter struct {
 	chunkCount int
 	errCount   int