diff --git a/main.go b/main.go index d212fa7da4d6..992c5b0cbdb9 100644 --- a/main.go +++ b/main.go @@ -467,6 +467,9 @@ func run(state overseer.State) { feature.UseSimplifiedGitlabEnumeration.Store(true) feature.GitlabProjectsPerPage.Store(100) + // OSS Default using github graphql api for issues, pr's and comments + feature.UseGithubGraphQLAPI.Store(false) + conf := &config.Config{} if *configFilename != "" { var err error diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 3c030a631e58..a3642b417e8f 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -11,6 +11,7 @@ var ( UseSimplifiedGitlabEnumeration atomic.Bool UseGitMirror atomic.Bool GitlabProjectsPerPage atomic.Int64 + UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments ) type AtomicString struct { diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index b9699a065d2b..b11740b82fc4 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -27,6 +27,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" @@ -1105,8 +1106,14 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep if s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists { return s.processGistComments(ctx, urlString, urlParts, repoInfo, reporter, cutoffTime) } else if s.includeIssueComments || s.includePRComments { - return s.processRepoComments(ctx, repoInfo, reporter, cutoffTime) + // if we need to use graphql api for repo issues, prs and comments + if feature.UseGithubGraphQLAPI.Load() { + return s.processRepoIssueandPRsWithCommentsGraphql(ctx, repoInfo, reporter, cutoffTime) + } + + return s.processIssueandPRsWithCommentsREST(ctx, repoInfo, reporter, cutoffTime) } + return nil } @@ -1273,7 +1280,10 @@ var ( state = "all" ) -func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processIssueandPRsWithCommentsREST( + ctx context.Context, repoInfo repoInfo, + reporter sources.ChunkReporter, cutoffTime *time.Time, +) error { if s.includeIssueComments { ctx.Logger().V(2).Info("Scanning issues") if err := s.processIssues(ctx, repoInfo, reporter); err != nil { @@ -1297,6 +1307,31 @@ func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, rep return nil } +func (s *Source) processRepoIssueandPRsWithCommentsGraphql( + ctx context.Context, repoInfo repoInfo, + reporter sources.ChunkReporter, cutoffTime *time.Time, +) error { + if s.includeIssueComments { + ctx.Logger().V(2).Info("Scanning issues") + if err := s.processIssuesWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + return err + } + } + + if s.includePRComments { + ctx.Logger().V(2).Info("Scanning pull requests") + if err := s.processPRWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + return err + } + + if err := s.processReviewThreads(ctx, repoInfo, reporter, cutoffTime); err != nil { + return err + } + } + + return nil +} + func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error { bodyTextsOpts := &github.IssueListByRepoOptions{ Sort: sortType, diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 42af8f4e2a98..6e5ce443b9c6 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" @@ -968,6 +969,64 @@ func TestSource_Validate(t *testing.T) { } } +func TestSource_ScanCommentsWithGraphql(t *testing.T) { + feature.UseGithubGraphQLAPI.Store(true) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + source := &sourcespb.GitHub{ + Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, + IncludeIssueComments: true, + IncludePullRequestComments: true, + Credential: &sourcespb.GitHub_Unauthenticated{}, + } + + wantChunk := sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: "https://github.com/trufflesecurity/driftwood.git/issues/1", + Username: "truffle-sandbox", + Timestamp: "2023-06-22 23:33:46 +0000 UTC", + }, + }, + }, + Verify: false, + } + + s := Source{} + + conn, err := anypb.New(source) + assert.NoError(t, err) + + err = s.Init(ctx, "test-source", 0, 0, false, conn, 4) + assert.NoError(t, err) + + chunksCh := make(chan *sources.Chunk, 1) + go func() { + // Close the channel + defer close(chunksCh) + err = s.Chunks(ctx, chunksCh) + assert.NoError(t, err) + }() + + i := 0 + for gotChunk := range chunksCh { + // Skip chunks that are not comments. + if gotChunk.SourceMetadata.GetGithub().GetCommit() != "" { + continue + } + i++ + githubCommentCheckFunc(gotChunk, &wantChunk, i, t, "test-source") + } + + // Confirm all comments were processed. + assert.Equal(t, i, 5) +} + type countChunkReporter struct { chunkCount int errCount int diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.go new file mode 100644 index 000000000000..e7e2f08cf09d --- /dev/null +++ b/pkg/sources/github/graphql.go @@ -0,0 +1,495 @@ +package github + +import ( + "cmp" + "fmt" + "math/rand/v2" + "strings" + "time" + + "github.com/shurcooL/githubv4" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// processIssuesWithComments process github repo issues with comments using graphql API +func (s *Source) processIssuesWithComments( + ctx context.Context, repoInfo repoInfo, + reporter sources.ChunkReporter, cutoffTime *time.Time, +) error { + vars := map[string]any{ + owner: githubv4.String(repoInfo.owner), + repository: githubv4.String(repoInfo.name), + issuesPerPage: githubv4.Int(defaultPagination), + issuesPagination: (*githubv4.String)(nil), + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: (*githubv4.String)(nil), + } + + var totalIssues int + + // loop will continue as long as there are issues in the repository + for { + var query issuesWithComments + err := s.connector.GraphQLClient().Query(ctx, &query, vars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("error fetching issues: %w", err) + } + + totalIssues += len(query.GetIssues()) + + ctx.Logger().V(5).Info("Scanning Issues", + "total_issues", len(query.GetIssues())) + + if err := s.chunkGraphqlIssues(ctx, repoInfo, query.Repository.Issues.Nodes, reporter); err != nil { + return err + } + + // process each issue comments + for _, issue := range query.GetIssues() { + ctx.Logger().V(5).Info("Scanning Issue Comments", + "issue_id", issue.Number, + "total_comments", len(issue.GetIssueComments()), + ) + + if err := s.chunkComments(ctx, repoInfo, issue.GetIssueComments(), reporter, cutoffTime); err != nil { + return err + } + + // if issue has more than 100 comments, we need to send another request for that specific issue to pull more comments + for issue.Comments.PageInfo.HasNextPage { + commentVars := map[string]any{ + owner: githubv4.String(repoInfo.owner), + repository: githubv4.String(repoInfo.name), + issueNumber: githubv4.Int(issue.Number), + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: issue.Comments.PageInfo.EndCursor, + } + + // request this issue more comments + var commentsQuery singleIssueComments + err := s.connector.GraphQLClient().Query(ctx, &commentsQuery, commentVars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentsQuery.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("error fetching issue: %w", err) + } + + ctx.Logger().V(5).Info("Scanning additional issue comments", + "issue_id", issue.Number, + "total_comments", len(commentsQuery.GetIssueComments())) + + if err := s.chunkComments(ctx, repoInfo, commentsQuery.GetIssueComments(), reporter, cutoffTime); err != nil { + return err + } + + // update page info for loop + issue.Comments.PageInfo = commentsQuery.Repository.Issue.Comments.PageInfo + } + } + + // paginate issues + if !query.Repository.Issues.PageInfo.HasNextPage { + ctx.Logger().V(4).Info("Scanned all repository issues with comments", "total_issues_scanned", totalIssues) + break + } + + // update issues pagination to go to next page + vars[issuesPagination] = githubv4.NewString(query.Repository.Issues.PageInfo.EndCursor) + } + + return nil +} + +// processPRWithComments process github repo pull requests with inline comments +func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { + vars := map[string]any{ + owner: githubv4.String(repoInfo.owner), + repository: githubv4.String(repoInfo.name), + pullRequestPerPage: githubv4.Int(defaultPagination), + pullRequestPagination: (*githubv4.String)(nil), + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: (*githubv4.String)(nil), + } + + var totalPRs int + + // continue loop as long as there are pull requests remaining + for { + var query pullRequestWithComments + err := s.connector.GraphQLClient().Query(ctx, &query, vars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("error fetching pull requests with comments: %w", err) + } + + totalPRs += len(query.GetPullRequests()) + + ctx.Logger().V(5).Info("Scanning pull requests", + "total_pull_requests", len(query.GetPullRequests())) + + if err := s.chunkGraphqlPullRequests(ctx, repoInfo, query.GetPullRequests(), reporter); err != nil { + return err + } + + // process each pr comments + for _, pr := range query.GetPullRequests() { + ctx.Logger().V(5).Info("Scanning pull request comments", + "pull_request_no", pr.Number, + "total_comments", len(pr.Comments.Nodes)) + + if err := s.chunkComments(ctx, repoInfo, pr.GetPRComments(), reporter, cutoffTime); err != nil { + return err + } + + // if a pull request has more than 100 comments - some interns pull request might endup here :) + for pr.Comments.PageInfo.HasNextPage { + // request this pull request more comments + var commentQuery singlePRComments + singlePRVars := map[string]any{ + owner: githubv4.String(repoInfo.owner), + repository: githubv4.String(repoInfo.name), + pullRequestNumber: pr.Number, + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: pr.Comments.PageInfo.EndCursor, + } + + err := s.connector.GraphQLClient().Query(ctx, &commentQuery, singlePRVars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentQuery.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("error fetching pull request with comments: %w", err) + } + + ctx.Logger().V(5).Info("Scanning additional comments", + "pull_request_no", pr.Number, + "total_comments", len(commentQuery.GetPRComments())) + + if err := s.chunkComments(ctx, repoInfo, commentQuery.GetPRComments(), reporter, cutoffTime); err != nil { + return err + } + + // update pr.Comments.PageInfo so loop condition reflects new state + pr.Comments.PageInfo = commentQuery.Repository.PullRequest.Comments.PageInfo + } + } + + // move to next page of PRs + if !query.Repository.PullRequests.PageInfo.HasNextPage { + ctx.Logger().V(4).Info("Scanned all repository pull requests with comments", "total_pullrequests_scanned", totalPRs) + break + } + + // update pull request pagination to go to next page + vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor) + } + + return nil +} + +// processReviewThreads process github repo pull request review threads +func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { + vars := map[string]any{ + owner: githubv4.String(repoInfo.owner), + repository: githubv4.String(repoInfo.name), + pullRequestPerPage: githubv4.Int(defaultPagination), + pullRequestPagination: (*githubv4.String)(nil), + threadPerPage: githubv4.Int(defaultPagination), + threadPagination: (*githubv4.String)(nil), + } + + var threadIDs = make([]string, 0) + + // continue as long as pull requests have threads + for { + var query prWithReviewThreadIDs + err := s.connector.GraphQLClient().Query(ctx, &query, vars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("error fetching pr thread reviews: %w", err) + } + + // collect thread ids + for _, pr := range query.GetMinimalPullRequests() { + prThreadIDs := pr.ReviewThreads.GetThreadIDs() + threadIDs = append(threadIDs, prThreadIDs...) + } + + if !query.Repository.PullRequests.PageInfo.HasNextPage { + ctx.Logger().V(4).Info("Pulled all repository PR's threads IDs") + break + } + + // update pull request pagination to go to next page + vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor) + } + + ctx.Logger().V(4).Info("Pulled all thread IDs", "total_threads", len(threadIDs)) + + // if we got more than 0 threads unfortunately :( than we need to pull their comments in batches + if len(threadIDs) > 0 { + // process in batches of max 100 + for _, batch := range chunkIDs(threadIDs, 100) { + ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch)) + // fetch comments for the batch of threads + if err := s.fetchThreadComments(ctx, batch, repoInfo, reporter, cutoffTime); err != nil { + return fmt.Errorf("error fetching thread review comments: %w", err) + } + } + } + + return nil +} + +// fetchThreadComments process github repo pull request threads and their comments +func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { + // Process in batches of 100 + var query multiReviewThreadComments + vars := map[string]any{ + "ids": threadIDs, + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: (*githubv4.String)(nil), + } + + if err := s.connector.GraphQLClient().Query(ctx, &query, vars); err != nil { + return fmt.Errorf("multi-thread query failed: %w", err) + } + + // process each thread in batch + for _, thread := range query.GetThreads() { + if err := s.chunkComments(ctx, repoInfo, thread.GetThreadComments(), reporter, cutoffTime); err != nil { + return err + } + + // if a thread has more than 100 comments :) + for thread.Comments.PageInfo.HasNextPage { + // request this thread more comments + var query singleReviewThreadComments + reviewThreadVars := map[string]any{ + threadID: thread.ID, + commentsPerPage: githubv4.Int(defaultPagination), + commentsPagination: (*githubv4.String)(nil), + } + + err := s.connector.GraphQLClient().Query(ctx, &query, reviewThreadVars) + if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { + continue + } + + if err != nil { + return fmt.Errorf("single-thread query failed: %w", err) + } + + node := query.Node + if err := s.chunkComments(ctx, repoInfo, node.Comments.Nodes, reporter, cutoffTime); err != nil { + return err + } + + if !node.Comments.PageInfo.HasNextPage { + break + } + + // update thread comments pagination + reviewThreadVars[commentsPagination] = &node.Comments.PageInfo.EndCursor + } + } + + return nil +} + +// chunkIDs splits a slice of IDs into multiple chunks of at most `size` elements. +// For example, if you have 250 IDs and size=100, this will return +// +// [][]string{ +// {id0 ... id99}, // first 100 +// {id100 ... id199},// next 100 +// {id200 ... id249} // last 50 +// } +func chunkIDs(ids []string, size int) [][]string { + var chunks [][]string + for size < len(ids) { + ids, chunks = ids[size:], append(chunks, ids[0:size:size]) + } + return append(chunks, ids) +} + +// handleRateLimitWithChunkReporter is a wrapper around handleRateLimit that includes chunk reporting +func (s *Source) handleGraphqlRateLimitWithChunkReporter( + ctx context.Context, reporter sources.ChunkReporter, + rl *rateLimit, errIn error, +) bool { + return s.handleGraphQLRateLimit(ctx, rl, errIn, &chunkErrorReporter{reporter: reporter}) +} + +// handleGraphQLRateLimit inspects the rateLimit info returned in GraphQL queries. +func (s *Source) handleGraphQLRateLimit(ctx context.Context, rl *rateLimit, errIn error, reporters ...errorReporter) bool { + // check global resume time first (in case another request already set it) + rateLimitMu.RLock() + resumeTime := rateLimitResumeTime + rateLimitMu.RUnlock() + + // if resume time is not empty and is after current time, than put the request to sleep till that. + if !resumeTime.IsZero() && time.Now().Before(resumeTime) { + retryAfter := time.Until(resumeTime) + time.Sleep(retryAfter) + return true + } + + var retryAfter time.Duration + if errIn != nil && strings.Contains(errIn.Error(), "rate limit exceeded") { + now := time.Now() + + rateLimitMu.Lock() + rateLimitResumeTime = now.Add(1 * time.Minute) + retryAfter = time.Until(rateLimitResumeTime) + ctx.Logger().Info("GraphQL RATE_LIMITED error (fallback)", + "retry_after", retryAfter.String()) + rateLimitMu.Unlock() + } else if rl != nil { + // if rate limit remaining is more than 3 continue using graphql api + if rl.Remaining > 3 { + return false + } + + // === only reach here if error is nil and rate limit remaining is less than 3 (safety) + now := time.Now() + retryAfter = time.Until(rl.ResetAt) + // never negative and enforce a sane minimum backoff (avoid thrashing with 1s/2s retries) + if cmp.Less(retryAfter, 5*time.Second) { + retryAfter = 5 * time.Second + } + + jitter := time.Duration(rand.IntN(10)+1) * time.Second + retryAfter += jitter + + // update global resume time + rateLimitMu.Lock() + rateLimitResumeTime = now.Add(retryAfter) + ctx.Logger().Info("exceeded GraphQL rate limit", + "retry_after", retryAfter.String(), + "resume_time", rateLimitResumeTime.Format(time.RFC3339)) + rateLimitMu.Unlock() + + for _, reporter := range reporters { + _ = reporter.Err(ctx, fmt.Errorf("exceeded GraphQL rate limit")) + } + } + + githubNumRateLimitEncountered.WithLabelValues(s.name).Inc() + time.Sleep(retryAfter) + githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds()) + + return true +} + +func (s *Source) chunkGraphqlIssues(ctx context.Context, repoInfo repoInfo, issues []issue, reporter sources.ChunkReporter) error { + for _, issue := range issues { + // Create chunk and send it to the channel. + chunk := sources.Chunk{ + SourceName: s.name, + SourceID: s.SourceID(), + JobID: s.JobID(), + SourceType: s.Type(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: sanitizer.UTF8(issue.URL), + Username: sanitizer.UTF8(issue.Author.Login), + Repository: sanitizer.UTF8(repoInfo.fullName), + Timestamp: sanitizer.UTF8(issue.CreatedAt.String()), + Visibility: repoInfo.visibility, + }, + }, + }, + Data: []byte(sanitizer.UTF8(issue.Title + "\n" + issue.Body)), + Verify: s.verify, + } + + if err := reporter.ChunkOk(ctx, chunk); err != nil { + return err + } + } + return nil +} + +func (s *Source) chunkComments(ctx context.Context, repoInfo repoInfo, comments []comment, reporter sources.ChunkReporter, cutoffTime *time.Time) error { + for _, comment := range comments { + // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted + if cutoffTime != nil && comment.UpdatedAt.Before(*cutoffTime) { + continue + } + + // Create chunk and send it to the channel. + chunk := sources.Chunk{ + SourceName: s.name, + SourceID: s.SourceID(), + JobID: s.JobID(), + SourceType: s.Type(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: sanitizer.UTF8(comment.URL), + Username: sanitizer.UTF8(comment.Author.Login), + Repository: sanitizer.UTF8(repoInfo.fullName), + Timestamp: sanitizer.UTF8(comment.CreatedAt.String()), + Visibility: repoInfo.visibility, + }, + }, + }, + Data: []byte(sanitizer.UTF8(comment.Body)), + Verify: s.verify, + } + + if err := reporter.ChunkOk(ctx, chunk); err != nil { + return err + } + } + return nil +} + +func (s *Source) chunkGraphqlPullRequests(ctx context.Context, repoInfo repoInfo, prs []pullRequest, reporter sources.ChunkReporter) error { + for _, pr := range prs { + // Create chunk and send it to the channel. + chunk := sources.Chunk{ + SourceName: s.name, + SourceID: s.SourceID(), + SourceType: s.Type(), + JobID: s.JobID(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: sanitizer.UTF8(pr.URL), + Username: sanitizer.UTF8(pr.Author.Login), + Repository: sanitizer.UTF8(repoInfo.fullName), + Timestamp: sanitizer.UTF8(pr.CreatedAt.String()), + Visibility: repoInfo.visibility, + }, + }, + }, + Data: []byte(sanitizer.UTF8(pr.Title + "\n" + pr.Body)), + Verify: s.verify, + } + + if err := reporter.ChunkOk(ctx, chunk); err != nil { + return err + } + } + return nil +} diff --git a/pkg/sources/github/queries.go b/pkg/sources/github/queries.go new file mode 100644 index 000000000000..dacaa7ad4afa --- /dev/null +++ b/pkg/sources/github/queries.go @@ -0,0 +1,248 @@ +package github + +import ( + "time" + + "github.com/shurcooL/githubv4" +) + +const ( + // query variable keys + owner = "owner" + repository = "repo" + + pullRequestNumber = "number" + pullRequestPerPage = "first" + pullRequestPagination = "after" + + commentsPerPage = "commentsFirst" + commentsPagination = "commentsAfter" + + threadID = "threadID" + threadPerPage = "threadsFirst" + threadPagination = "threadsAfter" + + issueNumber = "number" + issuesPerPage = "issuesFirst" + issuesPagination = "issuesAfter" +) + +// === Pull Requests with Comments === + +// pullRequestWithComments represent a repository pull request nodes +type pullRequestWithComments struct { + Repository struct { + PullRequests pullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"` + } `graphql:"repository(owner: $owner, name: $repo)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// pullRequestNodes represents a paginated list of pull requests +type pullRequestNodes struct { + Nodes []pullRequest + PageInfo pageInfo +} + +// pullRequest represents a single pull request with comment nodes +type pullRequest struct { + Title string + Number int + URL string + Author author + CreatedAt time.Time + Body string + Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"` +} + +// commentNodes represents a paginated list of comments +type commentNodes struct { + Nodes []comment + PageInfo pageInfo +} + +// comment represents a single comment +type comment struct { + Body string + CreatedAt time.Time + UpdatedAt time.Time + Author author + URL string +} + +// singlePRComments represents a single PR comments +type singlePRComments struct { + Repository struct { + PullRequest struct { + Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"` + } `graphql:"pullRequest(number: $number)"` + } `graphql:"repository(owner: $owner, name: $repo)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// GetPullRequests return list of pull requests +func (p pullRequestWithComments) GetPullRequests() []pullRequest { + return p.Repository.PullRequests.Nodes +} + +// GetPRComments return list of comments of the PR +func (p pullRequest) GetPRComments() []comment { + return p.Comments.Nodes +} + +// GetPRComments return list of comments of the PR +func (p singlePRComments) GetPRComments() []comment { + return p.Repository.PullRequest.Comments.Nodes +} + +// === Pull Request Review Threads IDs === + +// prWithReviewThreadIDs represents repository pull requests with review threads IDs +type prWithReviewThreadIDs struct { + Repository struct { + PullRequests minimalPullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"` + } `graphql:"repository(owner: $owner, name: $repo)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// minimalPullRequestNodes represents a paginated list of pull request +type minimalPullRequestNodes struct { + Nodes []minimalPullRequest + PageInfo pageInfo +} + +// minimalPullRequest represents a pull request with no and review threads +type minimalPullRequest struct { + Number int + ReviewThreads reviewThreadIDNodes `graphql:"reviewThreads(first: $threadsFirst, after: $threadsAfter)"` +} + +// reviewThreadIDNodes represents a paginated list of pr review threads +type reviewThreadIDNodes struct { + Nodes []reviewThreadID + PageInfo pageInfo +} + +// reviewThreadID represents a single review thread with ID +type reviewThreadID struct { + ID string +} + +func (p prWithReviewThreadIDs) GetMinimalPullRequests() []minimalPullRequest { + return p.Repository.PullRequests.Nodes +} + +func (r reviewThreadIDNodes) GetThreadIDs() []string { + var ids = make([]string, 0) + + for _, id := range r.Nodes { + ids = append(ids, id.ID) + } + + return ids +} + +// === Review Threads with comments === + +// multiReviewThreadComments fetches multiple review threads (by IDs) and the first page of their comments. +type multiReviewThreadComments struct { + Nodes []struct { + PullRequestReviewThread reviewThreadComments `graphql:"... on PullRequestReviewThread"` + } `graphql:"nodes(ids: $ids)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// reviewThreadComments represents a single thread and its comments. +type reviewThreadComments struct { + ID string + Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"` +} + +// singleReviewThreadComments fetches one review thread with comment pagination +type singleReviewThreadComments struct { + Node struct { + ID string + Comments commentNodes `graphql:"... on PullRequestReviewThread"` + } `graphql:"node(id: $id)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// GetThreads returns all review thread nodes +func (r multiReviewThreadComments) GetThreads() []reviewThreadComments { + threads := make([]reviewThreadComments, 0) + for _, n := range r.Nodes { + if n.PullRequestReviewThread.ID != "" { + threads = append(threads, n.PullRequestReviewThread) + } + } + + return threads +} + +func (r reviewThreadComments) GetThreadComments() []comment { + return r.Comments.Nodes +} + +// === Issues with comments === + +// issuesWithComments represents a repository issues with comments +type issuesWithComments struct { + Repository struct { + Issues issueNodes `graphql:"issues(first: $issuesFirst, after: $issuesAfter, orderBy: {field: UPDATED_AT, direction: DESC})"` + } `graphql:"repository(owner: $owner, name: $repo)"` + RateLimit rateLimit `graphql:"rateLimit"` +} + +// issueNodes represents a paginated list of issues +type issueNodes struct { + Nodes []issue + PageInfo pageInfo +} + +// issue represents a single PR issue +type issue struct { + Number int + Title string + Body string + URL string + Author author + CreatedAt time.Time + Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"` +} + +// singleIssueComments represents a single PR issue with comments +type singleIssueComments struct { + Repository struct { + Issue struct { + Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"` + } `graphql:"issue(number: $number)"` + } `graphql:"repository(owner: $owner, name: $repo)"` + RateLimit rateLimit +} + +func (i issuesWithComments) GetIssues() []issue { + return i.Repository.Issues.Nodes +} + +func (i issue) GetIssueComments() []comment { + return i.Comments.Nodes +} + +func (i singleIssueComments) GetIssueComments() []comment { + return i.Repository.Issue.Comments.Nodes +} + +// === Others === + +type author struct { + Login string `graphql:"login"` +} + +type pageInfo struct { + HasNextPage bool `graphql:"hasNextPage"` + EndCursor githubv4.String `graphql:"endCursor"` +} + +type rateLimit struct { + Remaining int `graphql:"remaining"` + ResetAt time.Time `graphql:"resetAt"` +}