Merge pull request #100 from GSA-TTS/develop

Promote Develop to Master

Author: Cybonto

.flake8

@@ -1,13 +1,14 @@
 [flake8]
 max-line-length = 120
-extend-ignore = E203, W503, E501, F401, F541, E226, ANN101, ANN204, B009, E712
+extend-ignore = E203, W503, E501, F401, F541, E226, ANN101, ANN204, B009, E712, D402
 exclude =
     .git,
     __pycache__,
     .venv,
     venv,
     .env,
     env,
+    test_env,
     build,
     dist,
     .eggs,
@@ -20,7 +21,9 @@ exclude =
     violentutf_logs,
     security,
     .github,
-    scripts
+    scripts,
+    tools/agent_orchestrator,
+    agent_orchestrator
 per-file-ignores =
     __init__.py:F401
     tests/*:F401,F811,ANN101,ANN401,D202,ANN201,B007,B023,F841,E722,B001,D102,D107,ANN001,ANN002,ANN003,ANN202,C901,D401
@@ -35,7 +38,38 @@ per-file-ignores =
     app/api/endpoints/reports.py:C901
     app/api/endpoints/security_scans.py:C901
     app/api/endpoints/vulnerability_findings.py:C901
+    app/api/endpoints/vulnerability_taxonomies.py:C901
+    app/celery/tasks.py:C901
+    app/core/abac_permissions.py:C901
+    app/core/audit.py:C901
+    app/core/authority.py:C901
+    app/core/cache.py:C901
     app/core/decorators/circuit_breaker.py:C901
+    app/core/decorators/request_signing.py:C901
+    app/core/decorators/sanitization.py:C901
+    app/core/decorators/sql_injection.py:C901
+    app/core/external_services.py:C901
+    app/core/field_sanitization.py:C901
+    app/core/input_validation.py:C901
+    app/core/permissions.py:C901
+    app/core/request_signing.py:C901
+    app/core/sql_injection_prevention.py:C901
+    app/db/init_mfa_policies.py:C901
+    app/middleware/authentication.py:C901
+    app/middleware/csrf.py:C901
+    app/middleware/oauth.py:C901,ANN201,ANN202
+    app/middleware/rate_limiting.py:C901
+    app/middleware/request_signing.py:C901
+    app/models/role.py:C901
+    app/models/scan.py:C901
+    app/services/api_key_service.py:C901
+    app/services/architectural_report_generator.py:C901
+    app/services/audit_service.py:C901
+    app/services/mfa_policy_service.py:C901
+    app/services/rbac_service.py:C901
+    app/utils/performance_tracker.py:ANN401,D107,D401,E704
+    fix_test_patterns.py:C901
+    tools/pre_audit/*:C901,ANN201,ANN202
 max-complexity = 10
 statistics = True
 count = True

.github/scripts/analyze-bandit-report.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Analyze Bandit security report and determine if check should fail."""
+
+import json
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    report_file = Path("bandit-report.json")
+
+    if not report_file.exists():
+        print("❌ Bandit report not found")
+        sys.exit(1)
+
+    with open(report_file) as f:
+        data = json.load(f)
+
+    metrics = data.get("metrics", {})
+    total = metrics.get("_totals", {})
+    results = data.get("results", [])
+
+    # Display metrics
+    print("=== Bandit Security Scan Results ===")
+    print(f"Total issues found: {len(results)}")
+    print(f"Severity breakdown:")
+    print(f"  HIGH: {total.get('SEVERITY.HIGH', 0)}")
+    print(f"  MEDIUM: {total.get('SEVERITY.MEDIUM', 0)}")
+    print(f"  LOW: {total.get('SEVERITY.LOW', 0)}")
+    print(f"Confidence breakdown:")
+    print(f"  HIGH: {total.get('CONFIDENCE.HIGH', 0)}")
+    print(f"  MEDIUM: {total.get('CONFIDENCE.MEDIUM', 0)}")
+    print(f"  LOW: {total.get('CONFIDENCE.LOW', 0)}")
+
+    # Check for high severity issues
+    high_severity = total.get("SEVERITY.HIGH", 0)
+
+    if high_severity > 0:
+        print(f"\n❌ Found {high_severity} HIGH severity issues:")
+        for result in results:
+            if result.get("issue_severity", "").upper() == "HIGH":
+                print(f"  - {result.get('filename')}:{result.get('line_number')}: {result.get('issue_text')}")
+        print("\nFailing check due to HIGH severity security issues")
+        sys.exit(1)
+    else:
+        print("\n✅ No HIGH severity issues found - check passed")
+        if len(results) > 0:
+            print(f"ℹ️  {len(results)} lower severity issues found - review artifact for details")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/analyze-safety-report.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Analyze Safety vulnerability report and determine if check should fail."""
+
+import json
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    report_file = Path("safety-report.json")
+
+    if not report_file.exists():
+        print("ℹ️  Safety report not found - skipping vulnerability check")
+        sys.exit(0)
+
+    with open(report_file) as f:
+        data = json.load(f)
+
+    vulnerabilities = data.get("vulnerabilities", [])
+
+    print("=== Safety Dependency Check Results ===")
+
+    if not vulnerabilities:
+        print("✅ No known vulnerabilities found")
+        sys.exit(0)
+
+    # Count by severity (Safety uses different format than we expected)
+    # Adjust based on actual Safety output format
+    critical = 0
+    high = 0
+    medium = 0
+    low = 0
+
+    for vuln in vulnerabilities:
+        # Safety might use different field names, adjust as needed
+        severity = str(vuln.get("severity", vuln.get("vulnerability", ""))).lower()
+        if "critical" in severity:
+            critical += 1
+        elif "high" in severity:
+            high += 1
+        elif "medium" in severity:
+            medium += 1
+        else:
+            low += 1
+
+    print(f"Total vulnerabilities found: {len(vulnerabilities)}")
+    if critical + high + medium + low == 0:
+        # If we couldn't categorize, just show total
+        print(f"  Unable to categorize by severity")
+    else:
+        print(f"  CRITICAL: {critical}")
+        print(f"  HIGH: {high}")
+        print(f"  MEDIUM: {medium}")
+        print(f"  LOW: {low}")
+
+    # Show first few vulnerabilities for visibility
+    print("\nVulnerabilities found:")
+    for vuln in vulnerabilities[:5]:  # Show first 5
+        pkg = vuln.get("package", vuln.get("package_name", "Unknown"))
+        desc = vuln.get("vulnerability", vuln.get("advisory", "No description"))
+        print(f"  - {pkg}: {desc[:100]}...")
+
+    if len(vulnerabilities) > 5:
+        print(f"  ... and {len(vulnerabilities) - 5} more")
+
+    # Only fail on CRITICAL vulnerabilities
+    if critical > 0:
+        print(f"\n❌ Found {critical} CRITICAL vulnerabilities - failing check")
+        sys.exit(1)
+    else:
+        print("\n✅ No CRITICAL vulnerabilities found - check passed")
+        if len(vulnerabilities) > 0:
+            print(f"ℹ️  Review the uploaded artifact for full vulnerability details")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/audit-organization-repos.py

@@ -95,7 +95,14 @@ def matches(self, content: str) -> List[Tuple[int, str]]:
 class RepoViolation:
     """Represents a violation found in a repository."""
 
-    def __init__(self, repo_name: str, file_path: str, line_num: int, line_content: str, pattern: DangerousPattern):
+    def __init__(
+        self,
+        repo_name: str,
+        file_path: str,
+        line_num: int,
+        line_content: str,
+        pattern: DangerousPattern,
+    ):
         self.repo_name = repo_name
         self.file_path = file_path
         self.line_num = line_num
@@ -167,7 +174,12 @@ def scan_repository(self, repo: Any) -> List[RepoViolation]:
             # Check individual files
             individual_files = []
             for content in contents:
-                if content.name in ["Makefile", ".travis.yml", "azure-pipelines.yml", "Jenkinsfile"]:
+                if content.name in [
+                    "Makefile",
+                    ".travis.yml",
+                    "azure-pipelines.yml",
+                    "Jenkinsfile",
+                ]:
                     individual_files.append(content.path)
 
             # Scan all identified files
@@ -242,7 +254,12 @@ def generate_report(self, output_file: str = "audit-report.json") -> Dict[str, A
         """Generate a comprehensive audit report."""
 
         # Organize violations by severity and repository
-        by_severity: Dict[str, List[RepoViolation]] = {"CRITICAL": [], "HIGH": [], "MEDIUM": [], "LOW": []}
+        by_severity: Dict[str, List[RepoViolation]] = {
+            "CRITICAL": [],
+            "HIGH": [],
+            "MEDIUM": [],
+            "LOW": [],
+        }
         by_repo: Dict[str, List[RepoViolation]] = {}
 
         for violation in self.violations:
@@ -272,13 +289,17 @@ def generate_report(self, output_file: str = "audit-report.json") -> Dict[str, A
 
         # Top violating repositories
         top_violators = sorted(
-            [(repo, len(violations)) for repo, violations in by_repo.items()], key=lambda x: x[1], reverse=True
+            [(repo, len(violations)) for repo, violations in by_repo.items()],
+            key=lambda x: x[1],
+            reverse=True,
         )[:10]
 
         report = {
             "audit_metadata": stats,
             "summary": {
-                "total_repositories_scanned": len(set(v.repo_name for v in self.violations)) if self.violations else 0,
+                "total_repositories_scanned": (
+                    len(set(v.repo_name for v in self.violations)) if self.violations else 0
+                ),
                 "violations_by_severity": {k: len(v) for k, v in by_severity.items()},
                 "top_violating_repositories": [{"repo": repo, "violations": count} for repo, count in top_violators],
             },

.github/scripts/ban-test-masking.py

@@ -114,7 +114,12 @@ def should_check_file(file_path: Path) -> bool:
         return True
 
     # Check CI configuration files
-    ci_files = [".travis.yml", ".circleci/config.yml", "azure-pipelines.yml", "Jenkinsfile"]
+    ci_files = [
+        ".travis.yml",
+        ".circleci/config.yml",
+        "azure-pipelines.yml",
+        "Jenkinsfile",
+    ]
     if any(ci_file in path_str for ci_file in ci_files):
         return True
 

.github/scripts/test-workflow-execution.py

@@ -146,7 +146,10 @@ def _test_embedded_code_execution(self, command: str, step_id: str) -> bool:
 
                 # Test actual execution in safe mode
                 exec_result = subprocess.run(  # nosec B603, B607 - Safe testing with controlled input
-                    ["bash", f.name], capture_output=True, text=True, timeout=30  # Prevent hanging
+                    ["bash", f.name],
+                    capture_output=True,
+                    text=True,
+                    timeout=30,  # Prevent hanging
                 )
 
                 os.unlink(f.name)

.github/workflows/architectural-compliance.yml

@@ -33,7 +33,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
         cache: 'pip'
 
     - name: Cache Analysis Results
@@ -56,13 +56,13 @@ jobs:
       id: scope
       run: |
         if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-          echo "mode=incremental" >> $GITHUB_OUTPUT
+          echo "mode=ci" >> $GITHUB_OUTPUT
           echo "base_ref=origin/${{ github.base_ref }}" >> $GITHUB_OUTPUT
         elif [[ "${{ inputs.full_analysis }}" == "true" ]]; then
           echo "mode=full" >> $GITHUB_OUTPUT
           echo "base_ref=HEAD~1" >> $GITHUB_OUTPUT
         else
-          echo "mode=incremental" >> $GITHUB_OUTPUT
+          echo "mode=ci" >> $GITHUB_OUTPUT
           echo "base_ref=HEAD~1" >> $GITHUB_OUTPUT
         fi
 
@@ -194,6 +194,11 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
 
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+
     steps:
     - name: Add PR Label
       uses: actions/github-script@v7