From 1d620e8efa20a5f6a3925b89b981fe560dd49dbd Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Sun, 26 Oct 2025 15:09:27 +0000 Subject: [PATCH 01/47] [PRMP-585] Create ReviewProcessor lambda logic --- lambdas/enums/review_status.py | 9 + lambdas/handlers/review_processor_handler.py | 74 +++ lambdas/models/document_review.py | 49 ++ lambdas/models/sqs/review_message_body.py | 14 + lambdas/services/review_processor_service.py | 188 ++++++++ .../handlers/test_review_processor_handler.py | 324 +++++++++++++ .../services/test_review_processor_service.py | 453 ++++++++++++++++++ 7 files changed, 1111 insertions(+) create mode 100644 lambdas/enums/review_status.py create mode 100644 lambdas/handlers/review_processor_handler.py create mode 100644 lambdas/models/document_review.py create mode 100644 lambdas/models/sqs/review_message_body.py create mode 100644 lambdas/services/review_processor_service.py create mode 100644 lambdas/tests/unit/handlers/test_review_processor_handler.py create mode 100644 lambdas/tests/unit/services/test_review_processor_service.py diff --git a/lambdas/enums/review_status.py b/lambdas/enums/review_status.py new file mode 100644 index 000000000..64b51ce26 --- /dev/null +++ b/lambdas/enums/review_status.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class ReviewStatus(StrEnum): + """Status values for document review records.""" + + PENDING_REVIEW = "PENDING_REVIEW" + APPROVED = "APPROVED" + REJECTED = "REJECTED" diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py new file mode 100644 index 000000000..62edb6a93 --- /dev/null +++ b/lambdas/handlers/review_processor_handler.py @@ -0,0 +1,74 @@ +import json +from lambdas.models.sqs.review_message_body import ReviewMessageBody +# from services.review_processor_service import ReviewProcessorService // TODO +from lambdas.services.review_processor_service import ReviewProcessorService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.decorators.validate_sqs_message_event import validate_sqs_event +from utils.lambda_response import ApiGatewayResponse + +logger = LoggingService(__name__) + + +@set_request_context_for_logging +@override_error_check +@ensure_environment_variables( + names=[ + "DOCUMENT_REVIEW_DYNAMODB_NAME", + "STAGING_STORE_BUCKET_NAME", + "PENDING_REVIEW_BUCKET_NAME", + ] +) +@handle_lambda_exceptions +@validate_sqs_event +def lambda_handler(event, context): + """ + This handler consumes SQS messages from the document review queue, creates DynamoDB + records in the DocumentReview table, and moves files from the staging bucket + to the pending review bucket. + + Args: + event: Lambda event containing SQS Event + _context: Lambda context + + Returns: + ApiGatewayResponse with processing status + """ + logger.info("Starting review processor Lambda") + + sqs_messages = event.get("Records", []) + review_service = ReviewProcessorService() + + processed_count = 0 + failed_count = 0 + + for sqs_message in sqs_messages: + try: + sqs_message_body = json.loads(sqs_message["body"]) + review_message = ReviewMessageBody(**sqs_message_body) + + message = ReviewMessageBody.model_validate(review_message) + + review_service.process_review_message(message) + processed_count += 1 + except Exception as e: + logger.error( + f"Failed to process review message: {str(e)}", + {"Result": "Review processing failed"}, + ) + failed_count += 1 + + raise + + logger.info( + f"Review processor completed: {processed_count} processed, {failed_count} failed" + ) + + return ApiGatewayResponse( + status_code=200, + body=f"Processed {processed_count} messages", + methods="GET", + ).create_api_gateway_response() diff --git a/lambdas/models/document_review.py b/lambdas/models/document_review.py new file mode 100644 index 000000000..9b3eee106 --- /dev/null +++ b/lambdas/models/document_review.py @@ -0,0 +1,49 @@ +from typing import Optional +import uuid + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_pascal + +from lambdas.enums.review_status import ReviewStatus +from lambdas.enums.snomed_codes import SnomedCodes +from lambdas.models.document_reference import DocumentReferenceMetadataFields + + +class DocumentReviewFileDetails(BaseModel): + model_config = ConfigDict( + validate_by_alias=True, + validate_by_name=True, + alias_generator=to_pascal, + ) + + file_name: str + file_location: str + + +class DocumentsUploadReview(BaseModel): + model_config = ConfigDict( + validate_by_alias=True, + validate_by_name=True, + alias_generator=to_pascal, + use_enum_values=True, + ) + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + alias=str(DocumentReferenceMetadataFields.ID.value) + ) # id differse to nogas version + author: str + custodian: str + review_status: ReviewStatus = Field(default=ReviewStatus.PENDING_REVIEW) + review_reason: str + review_date: int | None = Field(default=None) + reviewer: str | None = Field(default=None) + upload_date: int + files: list[DocumentReviewFileDetails] = Field(default=[]) # differs to nogas version + nhs_number: str + ttl: Optional[int] = Field( + alias=str(DocumentReferenceMetadataFields.TTL.value), default=None + ) + document_reference_id: str | None = Field(default=None) + document_snomed_code_type: str = Field( + default=SnomedCodes.LLOYD_GEORGE.value.code + ) diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py new file mode 100644 index 000000000..c0cae7781 --- /dev/null +++ b/lambdas/models/sqs/review_message_body.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + + +class ReviewMessageBody(BaseModel): + """Model for SQS message body from the document review queue.""" + + file_name: str + file_path: str + """Location in the staging bucket""" + nhs_number: str + failure_reason: str + upload_date: str + uploader_ods: str + current_gp: str diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py new file mode 100644 index 000000000..8f5ab36d5 --- /dev/null +++ b/lambdas/services/review_processor_service.py @@ -0,0 +1,188 @@ +import os +from datetime import datetime, timezone + +from enums.review_status import ReviewStatus +from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview +from models.sqs.review_message_body import ReviewMessageBody +from services.base.dynamo_service import DynamoDBService +from services.base.s3_service import S3Service +from utils.audit_logging_setup import LoggingService +from utils.exceptions import S3FileNotFoundException +from utils.request_context import request_context + +logger = LoggingService(__name__) + + +class ReviewProcessorService: + """ + Service for processing single SQS messages from the document review queue. + """ + + def __init__(self): + """Initialize the review processor service with required AWS services.""" + self.dynamo_service = DynamoDBService() + self.s3_service = S3Service() + + self.review_table_name = os.environ["DOCUMENT_REVIEW_DYNAMODB_NAME"] + self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] + self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] + + def process_review_message(self, review_message: ReviewMessageBody) -> None: + """ + Process a single SQS message from the review queue. + + Args: + sqs_message: SQS message record containing file and failure information + + Raises: + InvalidMessageException: If message format is invalid or required fields missing + S3FileNotFoundException: If file doesn't exist in staging bucket + ClientError: For AWS service errors (DynamoDB, S3) + """ + logger.info("Processing review queue message") + + request_context.patient_nhs_no = review_message.nhs_number + + logger.info(f"Processing review for NHS: {review_message.nhs_number}, File: {review_message.file_name}") + + self._verify_file_exists_in_staging(review_message.file_path) + document_upload_review = self._create_review_record(review_message) + + new_file_key = self._move_file_to_review_bucket(review_message, document_upload_review.id) + self._update_review_record_with_file_location(document_upload_review.id, new_file_key) + + logger.info( + f"Successfully processed review for {review_message.nhs_number}", + {"Result": "Review record created and file moved"}, + ) + + def _verify_file_exists_in_staging(self, file_path: str) -> None: + """ + Verify the file exists in the staging bucket. + + Args: + file_path: S3 key of the file in staging bucket + + Raises: + S3FileNotFoundException: If file does not exist in staging bucket + """ + try: + file_exists = self.s3_service.file_exist_on_s3(s3_bucket_name=self.staging_bucket_name, file_key=file_path) + + if not file_exists: + raise S3FileNotFoundException(f"File not found in staging bucket: {file_path}") + + logger.info(f"Verified file exists in staging: {file_path}") + + except S3FileNotFoundException: + raise + except Exception as e: + logger.error(f"Error checking file in staging bucket: {str(e)}") + raise + + def _create_review_record(self, message_data: ReviewMessageBody) -> DocumentsUploadReview: + """ + Create a new review record in DynamoDB. + + Args: + message_data: Validated review queue message data + + Returns: + Created DocumentsUploadReview object + + Raises: + ClientError: If DynamoDB create operation fails + """ + try: + files = [DocumentReviewFileDetails( + file_name=message_data.file_name, + file_location=message_data.file_path + )] + + document_review = DocumentsUploadReview( + nhs_number=message_data.nhs_number, + upload_date=int(datetime.fromisoformat(message_data.upload_date).replace(tzinfo=timezone.utc).timestamp()), + review_status=ReviewStatus.PENDING_REVIEW, + review_reason=message_data.failure_reason, + author=message_data.uploader_ods, + custodian=message_data.current_gp, + files=files, + ) + + self.dynamo_service.create_item( + table_name=self.review_table_name, + item=document_review.model_dump(by_alias=True, exclude_none=True), + ) + + logger.info( + f"Created review record in DynamoDB with ID: {document_review.id}", + {"Result": "DynamoDB record created"}, + ) + + return document_review + + except Exception as e: + logger.error(f"Failed to create DynamoDB record: {str(e)}") + raise + + def _move_file_to_review_bucket(self, message_data: ReviewMessageBody, review_record_id: str) -> str: + """ + Move file from staging to review bucket. + + Args: + message_data: Review queue message data + review_record_id: ID of the review record (used in destination path) + + Returns: + New file key in review bucket + + Raises: + ClientError: If S3 copy or delete operations fail + """ + try: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{message_data.file_name}" + + logger.info(f"Copying file from ({message_data.file_path}) in staging to review bucket: {new_file_key}") + + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=message_data.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + ) + + logger.info("File successfully copied to review bucket") + logger.info(f"Deleting file from staging bucket: {message_data.file_path}") + + self._delete_from_staging(message_data.file_path) + logger.info(f"Successfully moved file to: {new_file_key}") + + return new_file_key + + except Exception as e: + logger.error(f"Failed to move file: {str(e)}") + raise + + def _delete_from_staging(self, file_key: str) -> None: + try: + self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) + + logger.info(f"Deleted file from staging bucket: {file_key}") + + except Exception as e: + logger.error(f"Error deleting file from staging: {str(e)}") + raise + + def _update_review_record_with_file_location(self, review_record_id: str, review_bucket_path: str) -> None: + try: + self.dynamo_service.update_item( + table_name=self.review_table_name, + key_pair={"ID": review_record_id}, + updated_fields={"ReviewBucketPath": review_bucket_path}, + ) + + logger.info(f"Updated review record {review_record_id} with file location: {review_bucket_path}") + + except Exception as e: + logger.error(f"Failed to update review record with file location: {str(e)}") + logger.warning("Review record created but file location not updated in DynamoDB") diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py new file mode 100644 index 000000000..e646eccb4 --- /dev/null +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -0,0 +1,324 @@ +import json + +import pytest +from handlers.review_processor_handler import lambda_handler +from models.sqs.review_message_body import ReviewMessageBody +from utils.lambda_response import ApiGatewayResponse + + +@pytest.fixture +def mock_review_service(mocker): + """Mock the ReviewProcessorService.""" + mocked_class = mocker.patch( + "handlers.review_processor_handler.ReviewProcessorService" + ) + mocked_instance = mocked_class.return_value + return mocked_instance + + + + + +@pytest.fixture +def sample_review_message_body(): + """Create a sample review message body.""" + return ReviewMessageBody( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + +@pytest.fixture +def sample_sqs_message(sample_review_message_body): + """Create a sample SQS message.""" + return { + "body": sample_review_message_body.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-1", + } + + +@pytest.fixture +def sample_sqs_event(sample_sqs_message): + """Create a sample SQS event with one message.""" + return {"Records": [sample_sqs_message]} + + +@pytest.fixture +def sample_sqs_event_multiple_messages(sample_review_message_body): + """Create a sample SQS event with multiple messages.""" + message_1 = ReviewMessageBody( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + message_2 = ReviewMessageBody( + file_name="document_2.pdf", + file_path="staging/9000000010/document_2.pdf", + nhs_number="9000000010", + failure_reason="Invalid file format", + upload_date="2024-01-15T10:35:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + message_3 = ReviewMessageBody( + file_name="document_3.pdf", + file_path="staging/9000000011/document_3.pdf", + nhs_number="9000000011", + failure_reason="Missing metadata", + upload_date="2024-01-15T10:40:00Z", + uploader_ods="Y67890", + current_gp="Y67890", + ) + + return { + "Records": [ + { + "body": message_1.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-1", + }, + { + "body": message_2.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-2", + }, + { + "body": message_3.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-3", + }, + ] + } + + +@pytest.fixture +def empty_sqs_event(): + """Create an empty SQS event.""" + return {"Records": []} + + +@pytest.fixture +def set_review_env(monkeypatch): + """Set up environment variables required for the handler.""" + monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") + monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") + monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") + + +def test_lambda_handler_processes_single_message_successfully( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler successfully processes a single SQS message.""" + expected_response = ApiGatewayResponse( + status_code=200, + body="Processed 1 messages", + methods="GET", + ).create_api_gateway_response() + + actual_response = lambda_handler(sample_sqs_event, context) + + assert actual_response == expected_response + mock_review_service.process_review_message.assert_called_once() + + +def test_lambda_handler_processes_multiple_messages_successfully( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler successfully processes multiple SQS messages.""" + expected_response = ApiGatewayResponse( + status_code=200, + body="Processed 3 messages", + methods="GET", + ).create_api_gateway_response() + + actual_response = lambda_handler(sample_sqs_event_multiple_messages, context) + + assert actual_response == expected_response + assert mock_review_service.process_review_message.call_count == 3 + + +def test_lambda_handler_calls_service_with_correct_message( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler calls service with the correctly parsed message.""" + lambda_handler(sample_sqs_event, context) + + # Verify the service was called once + mock_review_service.process_review_message.assert_called_once() + + # Get the actual call argument + call_args = mock_review_service.process_review_message.call_args[0][0] + + # Verify it's a ReviewMessageBody with correct data (check type name since isinstance fails with mock) + assert type(call_args).__name__ == "ReviewMessageBody" + assert call_args.file_name == "test_document.pdf" + assert call_args.nhs_number == "9000000009" + assert call_args.file_path == "staging/9000000009/test_document.pdf" + + +def test_lambda_handler_handles_empty_records_list( + set_review_env, context, empty_sqs_event, mock_review_service +): + """Test handler handles empty records list via @validate_sqs_event decorator.""" + # The @validate_sqs_event decorator returns 400 for empty records + actual_response = lambda_handler(empty_sqs_event, context) + + assert actual_response["statusCode"] == 400 + assert "SQS_4001" in actual_response["body"] # Error code for failed SQS parsing + mock_review_service.process_review_message.assert_not_called() + + +def test_lambda_handler_handles_service_error( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler catches exception when service fails (via @handle_lambda_exceptions decorator).""" + mock_review_service.process_review_message.side_effect = Exception( + "Service processing failed" + ) + + # The @handle_lambda_exceptions decorator catches exceptions + response = lambda_handler(sample_sqs_event, context) + + # Should return 500 error response + assert response["statusCode"] == 500 + assert "UE_500" in response["body"] # Unhandled exception error code + + +def test_lambda_handler_handles_invalid_message_format( + set_review_env, context, mock_review_service +): + """Test handler handles validation errors for invalid message format.""" + invalid_event = { + "Records": [ + { + "body": json.dumps( + { + "file_name": "test.pdf", + # Missing required fields + } + ), + "eventSource": "aws:sqs", + } + ] + } + + # The @handle_lambda_exceptions decorator catches the ValidationError + response = lambda_handler(invalid_event, context) + + assert response["statusCode"] == 500 + assert "UE_500" in response["body"] # Unhandled exception error code + + +def test_lambda_handler_handles_partial_failure( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler processes messages and handles first failure.""" + # First message succeeds, second fails + mock_review_service.process_review_message.side_effect = [ + None, # First message succeeds + Exception("Processing failed on second message"), # Second fails + ] + + # The @handle_lambda_exceptions decorator catches the exception + response = lambda_handler(sample_sqs_event_multiple_messages, context) + + # Should return error response + assert response["statusCode"] == 500 + + # Verify service was called twice before failing + assert mock_review_service.process_review_message.call_count == 2 + + +def test_lambda_handler_parses_json_body_correctly( + set_review_env, + context, + mock_review_service, +): + """Test handler correctly parses JSON from message body.""" + event = { + "Records": [ + { + "body": json.dumps( + { + "file_name": "test.pdf", + "file_path": "staging/test.pdf", + "nhs_number": "9000000009", + "failure_reason": "Test failure", + "upload_date": "2024-01-15T10:30:00Z", + "uploader_ods": "Y12345", + "current_gp": "Y12345", + } + ), + "eventSource": "aws:sqs", + } + ] + } + + lambda_handler(event, context) + + # Verify service was called with parsed ReviewMessageBody + mock_review_service.process_review_message.assert_called_once() + call_args = mock_review_service.process_review_message.call_args[0][0] + assert type(call_args).__name__ == "ReviewMessageBody" + assert call_args.file_name == "test.pdf" + + +def test_lambda_handler_logs_correct_counts( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler response contains correct processed count.""" + response = lambda_handler(sample_sqs_event_multiple_messages, context) + + # Extract response body + response_body = response["body"] + + assert "Processed 3 messages" in response_body + + +def test_lambda_handler_tracks_failed_count( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler tracks failed message count.""" + mock_review_service.process_review_message.side_effect = Exception("Test error") + + # The @handle_lambda_exceptions decorator catches the exception + response = lambda_handler(sample_sqs_event, context) + + # Should return error + assert response["statusCode"] == 500 + + # Verify service was still called + mock_review_service.process_review_message.assert_called_once() diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py new file mode 100644 index 000000000..68a942365 --- /dev/null +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -0,0 +1,453 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest +from botocore.exceptions import ClientError +from enums.review_status import ReviewStatus +from models.document_review import DocumentsUploadReview +from models.sqs.review_message_body import ReviewMessageBody +from services.review_processor_service import ReviewProcessorService +from utils.exceptions import S3FileNotFoundException + + +@pytest.fixture +def mock_dynamo_service(mocker): + """Mock the DynamoDBService.""" + return mocker.patch("services.review_processor_service.DynamoDBService") + + +@pytest.fixture +def mock_s3_service(mocker): + """Mock the S3Service.""" + return mocker.patch("services.review_processor_service.S3Service") + + +@pytest.fixture +def set_review_env(monkeypatch): + """Set up environment variables required for the service.""" + monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") + monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") + monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") + + +@pytest.fixture +def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): + """Create a ReviewProcessorService instance with mocked dependencies.""" + service = ReviewProcessorService() + return service + + +@pytest.fixture +def sample_review_message(): + """Create a sample review message.""" + return ReviewMessageBody( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + +# Test service initialization + + +def test_service_initializes_with_correct_environment_variables( + set_review_env, mock_dynamo_service, mock_s3_service +): + """Test service initializes correctly with environment variables.""" + service = ReviewProcessorService() + + assert service.review_table_name == "test_review_table" + assert service.staging_bucket_name == "test_staging_bucket" + assert service.review_bucket_name == "test_review_bucket" + mock_dynamo_service.assert_called_once() + mock_s3_service.assert_called_once() + + +# Tests for process_review_message method + + +def test_process_review_message_success( + service_under_test, sample_review_message, mocker +): + """Test successful processing of a review message.""" + # Mock all the internal methods + mock_verify = mocker.patch.object( + service_under_test, "_verify_file_exists_in_staging" + ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_move = mocker.patch.object( + service_under_test, "_move_file_to_review_bucket" + ) + mock_update = mocker.patch.object( + service_under_test, "_update_review_record_with_file_location" + ) + + # Configure return values + mock_review = MagicMock() + mock_review.id = "test-review-id" + mock_create.return_value = mock_review + mock_move.return_value = "9000000009/test-review-id/test_document.pdf" + + # Execute + service_under_test.process_review_message(sample_review_message) + + # Verify calls + mock_verify.assert_called_once_with(sample_review_message.file_path) + mock_create.assert_called_once_with(sample_review_message) + mock_move.assert_called_once_with(sample_review_message, "test-review-id") + mock_update.assert_called_once_with( + "test-review-id", "9000000009/test-review-id/test_document.pdf" + ) + + +def test_process_review_message_file_not_found( + service_under_test, sample_review_message, mocker +): + """Test processing fails when file doesn't exist in staging.""" + mocker.patch.object( + service_under_test, + "_verify_file_exists_in_staging", + side_effect=S3FileNotFoundException("File not found"), + ) + + with pytest.raises(S3FileNotFoundException, match="File not found"): + service_under_test.process_review_message(sample_review_message) + + +def test_process_review_message_dynamo_error( + service_under_test, sample_review_message, mocker +): + """Test processing fails when DynamoDB create fails.""" + mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + mocker.patch.object( + service_under_test, + "_create_review_record", + side_effect=ClientError( + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", + ), + ) + + with pytest.raises(ClientError): + service_under_test.process_review_message(sample_review_message) + + +def test_process_review_message_s3_copy_error( + service_under_test, sample_review_message, mocker +): + """Test processing fails when S3 copy operation fails.""" + mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + mock_review = MagicMock() + mock_review.id = "test-review-id" + mocker.patch.object( + service_under_test, "_create_review_record", return_value=mock_review + ) + mocker.patch.object( + service_under_test, + "_move_file_to_review_bucket", + side_effect=ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, + "CopyObject", + ), + ) + + with pytest.raises(ClientError): + service_under_test.process_review_message(sample_review_message) + + +# Tests for _verify_file_exists_in_staging method + + +def test_verify_file_exists_success(service_under_test): + """Test successful file verification.""" + service_under_test.s3_service.file_exist_on_s3.return_value = True + + # Should not raise exception + service_under_test._verify_file_exists_in_staging("staging/test.pdf") + + service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( + s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + ) + + +def test_verify_file_does_not_exist(service_under_test): + """Test file verification fails when file doesn't exist.""" + service_under_test.s3_service.file_exist_on_s3.return_value = False + + with pytest.raises( + S3FileNotFoundException, match="File not found in staging bucket" + ): + service_under_test._verify_file_exists_in_staging("staging/missing.pdf") + + +def test_verify_file_s3_error(service_under_test): + """Test file verification handles S3 errors.""" + service_under_test.s3_service.file_exist_on_s3.side_effect = ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "HeadObject", + ) + + with pytest.raises(ClientError): + service_under_test._verify_file_exists_in_staging("staging/test.pdf") + + +# Tests for _create_review_record method + + +def test_create_review_record_success( + service_under_test, sample_review_message, mocker +): + """Test successful creation of review record.""" + # Mock the DynamoDB create_item to succeed + service_under_test.dynamo_service.create_item.return_value = None + + result = service_under_test._create_review_record(sample_review_message) + + # Verify the result + assert isinstance(result, DocumentsUploadReview) + assert result.nhs_number == "9000000009" + assert result.review_status == ReviewStatus.PENDING_REVIEW + assert result.review_reason == "Failed virus scan" + assert result.author == "Y12345" + assert result.custodian == "Y12345" + assert len(result.files) == 1 + assert result.files[0].file_name == "test_document.pdf" + assert result.files[0].file_location == "staging/9000000009/test_document.pdf" + + # Verify upload_date is converted to timestamp + expected_timestamp = int( + datetime.fromisoformat("2024-01-15T10:30:00Z") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + assert result.upload_date == expected_timestamp + + # Verify DynamoDB was called + service_under_test.dynamo_service.create_item.assert_called_once() + call_args = service_under_test.dynamo_service.create_item.call_args + assert call_args[1]["table_name"] == "test_review_table" + + +def test_create_review_record_with_different_data(service_under_test, mocker): + """Test creation with different message data.""" + message = ReviewMessageBody( + file_name="another_doc.pdf", + file_path="staging/9000000010/another_doc.pdf", + nhs_number="9000000010", + failure_reason="Missing metadata", + upload_date="2024-02-20T14:45:30Z", + uploader_ods="Y67890", + current_gp="Y67890", + ) + + service_under_test.dynamo_service.create_item.return_value = None + + result = service_under_test._create_review_record(message) + + assert result.nhs_number == "9000000010" + assert result.review_reason == "Missing metadata" + assert result.author == "Y67890" + assert result.custodian == "Y67890" + + +def test_create_review_record_dynamo_error( + service_under_test, sample_review_message +): + """Test create record handles DynamoDB errors.""" + service_under_test.dynamo_service.create_item.side_effect = ClientError( + {"Error": {"Code": "ValidationException", "Message": "Invalid item"}}, + "PutItem", + ) + + with pytest.raises(ClientError): + service_under_test._create_review_record(sample_review_message) + + +# Tests for _move_file_to_review_bucket method + + +def test_move_file_success(service_under_test, sample_review_message, mocker): + """Test successful file move from staging to review bucket.""" + mocker.patch.object(service_under_test, "_delete_from_staging") + + new_file_key = service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id-123" + ) + + # Verify new file key format + expected_key = "9000000009/test-review-id-123/test_document.pdf" + assert new_file_key == expected_key + + # Verify S3 copy was called + service_under_test.s3_service.copy_across_bucket.assert_called_once_with( + source_bucket="test_staging_bucket", + source_file_key="staging/9000000009/test_document.pdf", + dest_bucket="test_review_bucket", + dest_file_key=expected_key, + ) + + # Verify delete was called + service_under_test._delete_from_staging.assert_called_once_with( + "staging/9000000009/test_document.pdf" + ) + + +def test_move_file_copy_error(service_under_test, sample_review_message, mocker): + """Test file move handles S3 copy errors.""" + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, + "CopyObject", + ) + + with pytest.raises(ClientError): + service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id" + ) + + +def test_move_file_delete_error(service_under_test, sample_review_message, mocker): + """Test file move handles delete errors.""" + mocker.patch.object( + service_under_test, + "_delete_from_staging", + side_effect=ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "DeleteObject", + ), + ) + + with pytest.raises(ClientError): + service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id" + ) + + +# Tests for _delete_from_staging method + + +def test_delete_from_staging_success(service_under_test): + """Test successful deletion from staging bucket.""" + service_under_test._delete_from_staging("staging/test.pdf") + + service_under_test.s3_service.delete_object.assert_called_once_with( + s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + ) + + +def test_delete_from_staging_error(service_under_test): + """Test delete from staging handles S3 errors.""" + service_under_test.s3_service.delete_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Key does not exist"}}, + "DeleteObject", + ) + + with pytest.raises(ClientError): + service_under_test._delete_from_staging("staging/test.pdf") + + +# Tests for _update_review_record_with_file_location method + + +def test_update_review_record_success(service_under_test): + """Test successful update of review record with file location.""" + service_under_test._update_review_record_with_file_location( + "test-review-id", "9000000009/test-review-id/test.pdf" + ) + + service_under_test.dynamo_service.update_item.assert_called_once_with( + table_name="test_review_table", + key_pair={"ID": "test-review-id"}, + updated_fields={"ReviewBucketPath": "9000000009/test-review-id/test.pdf"}, + ) + + +def test_update_review_record_dynamo_error(service_under_test, mocker): + """Test update review record handles DynamoDB errors gracefully.""" + service_under_test.dynamo_service.update_item.side_effect = ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "Failed"}}, + "UpdateItem", + ) + + # Mock logger to verify warning is logged + mock_logger = mocker.patch("services.review_processor_service.logger") + + # Should not raise - method logs error and warning instead + service_under_test._update_review_record_with_file_location( + "test-review-id", "path/to/file.pdf" + ) + + # Verify error and warning were logged + assert mock_logger.error.called + assert mock_logger.warning.called + + +# Integration scenario tests + + +def test_full_workflow_with_valid_message( + service_under_test, sample_review_message, mocker +): + """Test complete workflow from message to final update.""" + # Set up mocks + service_under_test.s3_service.file_exist_on_s3.return_value = True + service_under_test.dynamo_service.create_item.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.s3_service.delete_object.return_value = None + service_under_test.dynamo_service.update_item.return_value = None + + # Execute full workflow + service_under_test.process_review_message(sample_review_message) + + # Verify all steps were executed + service_under_test.s3_service.file_exist_on_s3.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() + service_under_test.s3_service.copy_across_bucket.assert_called_once() + service_under_test.s3_service.delete_object.assert_called_once() + service_under_test.dynamo_service.update_item.assert_called_once() + + +def test_workflow_stops_at_verification_failure( + service_under_test, sample_review_message +): + """Test workflow stops if file verification fails.""" + service_under_test.s3_service.file_exist_on_s3.return_value = False + + with pytest.raises(S3FileNotFoundException): + service_under_test.process_review_message(sample_review_message) + + # Verify subsequent operations were not called + service_under_test.dynamo_service.create_item.assert_not_called() + service_under_test.s3_service.copy_across_bucket.assert_not_called() + + +def test_workflow_handles_multiple_different_patients(service_under_test, mocker): + """Test processing messages for different patients.""" + service_under_test.s3_service.file_exist_on_s3.return_value = True + service_under_test.dynamo_service.create_item.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.s3_service.delete_object.return_value = None + service_under_test.dynamo_service.update_item.return_value = None + + messages = [ + ReviewMessageBody( + file_name=f"doc_{i}.pdf", + file_path=f"staging/900000000{i}/doc_{i}.pdf", + nhs_number=f"900000000{i}", + failure_reason="Test failure", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + for i in range(1, 4) + ] + + for message in messages: + service_under_test.process_review_message(message) + + # Verify service was called for each message + assert service_under_test.dynamo_service.create_item.call_count == 3 + assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 0f851820a71e421f615c372d3ac56f8cb629eadc Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Sun, 26 Oct 2025 15:35:27 +0000 Subject: [PATCH 02/47] [PRMP-585] minor fix --- .../handlers/test_review_processor_handler.py | 29 ++++--------------- .../services/test_review_processor_service.py | 20 ------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index e646eccb4..86b9b2548 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -164,13 +164,10 @@ def test_lambda_handler_calls_service_with_correct_message( """Test handler calls service with the correctly parsed message.""" lambda_handler(sample_sqs_event, context) - # Verify the service was called once mock_review_service.process_review_message.assert_called_once() - # Get the actual call argument call_args = mock_review_service.process_review_message.call_args[0][0] - # Verify it's a ReviewMessageBody with correct data (check type name since isinstance fails with mock) assert type(call_args).__name__ == "ReviewMessageBody" assert call_args.file_name == "test_document.pdf" assert call_args.nhs_number == "9000000009" @@ -181,11 +178,10 @@ def test_lambda_handler_handles_empty_records_list( set_review_env, context, empty_sqs_event, mock_review_service ): """Test handler handles empty records list via @validate_sqs_event decorator.""" - # The @validate_sqs_event decorator returns 400 for empty records actual_response = lambda_handler(empty_sqs_event, context) assert actual_response["statusCode"] == 400 - assert "SQS_4001" in actual_response["body"] # Error code for failed SQS parsing + assert "SQS_4001" in actual_response["body"] mock_review_service.process_review_message.assert_not_called() @@ -200,12 +196,10 @@ def test_lambda_handler_handles_service_error( "Service processing failed" ) - # The @handle_lambda_exceptions decorator catches exceptions response = lambda_handler(sample_sqs_event, context) - # Should return 500 error response assert response["statusCode"] == 500 - assert "UE_500" in response["body"] # Unhandled exception error code + assert "UE_500" in response["body"] def test_lambda_handler_handles_invalid_message_format( @@ -218,7 +212,6 @@ def test_lambda_handler_handles_invalid_message_format( "body": json.dumps( { "file_name": "test.pdf", - # Missing required fields } ), "eventSource": "aws:sqs", @@ -226,11 +219,10 @@ def test_lambda_handler_handles_invalid_message_format( ] } - # The @handle_lambda_exceptions decorator catches the ValidationError response = lambda_handler(invalid_event, context) assert response["statusCode"] == 500 - assert "UE_500" in response["body"] # Unhandled exception error code + assert "UE_500" in response["body"] def test_lambda_handler_handles_partial_failure( @@ -240,19 +232,14 @@ def test_lambda_handler_handles_partial_failure( mock_review_service, ): """Test handler processes messages and handles first failure.""" - # First message succeeds, second fails mock_review_service.process_review_message.side_effect = [ - None, # First message succeeds - Exception("Processing failed on second message"), # Second fails + None, + Exception("Processing failed on second message"), ] - # The @handle_lambda_exceptions decorator catches the exception response = lambda_handler(sample_sqs_event_multiple_messages, context) - # Should return error response assert response["statusCode"] == 500 - - # Verify service was called twice before failing assert mock_review_service.process_review_message.call_count == 2 @@ -283,7 +270,6 @@ def test_lambda_handler_parses_json_body_correctly( lambda_handler(event, context) - # Verify service was called with parsed ReviewMessageBody mock_review_service.process_review_message.assert_called_once() call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" @@ -299,7 +285,6 @@ def test_lambda_handler_logs_correct_counts( """Test handler response contains correct processed count.""" response = lambda_handler(sample_sqs_event_multiple_messages, context) - # Extract response body response_body = response["body"] assert "Processed 3 messages" in response_body @@ -314,11 +299,7 @@ def test_lambda_handler_tracks_failed_count( """Test handler tracks failed message count.""" mock_review_service.process_review_message.side_effect = Exception("Test error") - # The @handle_lambda_exceptions decorator catches the exception response = lambda_handler(sample_sqs_event, context) - # Should return error assert response["statusCode"] == 500 - - # Verify service was still called mock_review_service.process_review_message.assert_called_once() diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 68a942365..8907011b9 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -74,7 +74,6 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - # Mock all the internal methods mock_verify = mocker.patch.object( service_under_test, "_verify_file_exists_in_staging" ) @@ -86,16 +85,13 @@ def test_process_review_message_success( service_under_test, "_update_review_record_with_file_location" ) - # Configure return values mock_review = MagicMock() mock_review.id = "test-review-id" mock_create.return_value = mock_review mock_move.return_value = "9000000009/test-review-id/test_document.pdf" - # Execute service_under_test.process_review_message(sample_review_message) - # Verify calls mock_verify.assert_called_once_with(sample_review_message.file_path) mock_create.assert_called_once_with(sample_review_message) mock_move.assert_called_once_with(sample_review_message, "test-review-id") @@ -166,7 +162,6 @@ def test_verify_file_exists_success(service_under_test): """Test successful file verification.""" service_under_test.s3_service.file_exist_on_s3.return_value = True - # Should not raise exception service_under_test._verify_file_exists_in_staging("staging/test.pdf") service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( @@ -202,12 +197,10 @@ def test_create_review_record_success( service_under_test, sample_review_message, mocker ): """Test successful creation of review record.""" - # Mock the DynamoDB create_item to succeed service_under_test.dynamo_service.create_item.return_value = None result = service_under_test._create_review_record(sample_review_message) - # Verify the result assert isinstance(result, DocumentsUploadReview) assert result.nhs_number == "9000000009" assert result.review_status == ReviewStatus.PENDING_REVIEW @@ -218,7 +211,6 @@ def test_create_review_record_success( assert result.files[0].file_name == "test_document.pdf" assert result.files[0].file_location == "staging/9000000009/test_document.pdf" - # Verify upload_date is converted to timestamp expected_timestamp = int( datetime.fromisoformat("2024-01-15T10:30:00Z") .replace(tzinfo=timezone.utc) @@ -226,7 +218,6 @@ def test_create_review_record_success( ) assert result.upload_date == expected_timestamp - # Verify DynamoDB was called service_under_test.dynamo_service.create_item.assert_called_once() call_args = service_under_test.dynamo_service.create_item.call_args assert call_args[1]["table_name"] == "test_review_table" @@ -278,11 +269,9 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): sample_review_message, "test-review-id-123" ) - # Verify new file key format expected_key = "9000000009/test-review-id-123/test_document.pdf" assert new_file_key == expected_key - # Verify S3 copy was called service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bucket", source_file_key="staging/9000000009/test_document.pdf", @@ -290,7 +279,6 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): dest_file_key=expected_key, ) - # Verify delete was called service_under_test._delete_from_staging.assert_called_once_with( "staging/9000000009/test_document.pdf" ) @@ -372,15 +360,12 @@ def test_update_review_record_dynamo_error(service_under_test, mocker): "UpdateItem", ) - # Mock logger to verify warning is logged mock_logger = mocker.patch("services.review_processor_service.logger") - # Should not raise - method logs error and warning instead service_under_test._update_review_record_with_file_location( "test-review-id", "path/to/file.pdf" ) - # Verify error and warning were logged assert mock_logger.error.called assert mock_logger.warning.called @@ -392,17 +377,14 @@ def test_full_workflow_with_valid_message( service_under_test, sample_review_message, mocker ): """Test complete workflow from message to final update.""" - # Set up mocks service_under_test.s3_service.file_exist_on_s3.return_value = True service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.dynamo_service.update_item.return_value = None - # Execute full workflow service_under_test.process_review_message(sample_review_message) - # Verify all steps were executed service_under_test.s3_service.file_exist_on_s3.assert_called_once() service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() @@ -419,7 +401,6 @@ def test_workflow_stops_at_verification_failure( with pytest.raises(S3FileNotFoundException): service_under_test.process_review_message(sample_review_message) - # Verify subsequent operations were not called service_under_test.dynamo_service.create_item.assert_not_called() service_under_test.s3_service.copy_across_bucket.assert_not_called() @@ -448,6 +429,5 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker for message in messages: service_under_test.process_review_message(message) - # Verify service was called for each message assert service_under_test.dynamo_service.create_item.call_count == 3 assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 225bd3ffafb5bae42c692c22e0a15a1702842223 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 27 Oct 2025 14:44:30 +0000 Subject: [PATCH 03/47] [PRMP-585] minor fixes --- lambdas/handlers/review_processor_handler.py | 7 +------ lambdas/services/review_processor_service.py | 1 + .../tests/unit/handlers/test_review_processor_handler.py | 3 --- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index 62edb6a93..eb3194601 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,6 +1,5 @@ import json from lambdas.models.sqs.review_message_body import ReviewMessageBody -# from services.review_processor_service import ReviewProcessorService // TODO from lambdas.services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -48,9 +47,7 @@ def lambda_handler(event, context): for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) - review_message = ReviewMessageBody(**sqs_message_body) - - message = ReviewMessageBody.model_validate(review_message) + message = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) processed_count += 1 @@ -61,8 +58,6 @@ def lambda_handler(event, context): ) failed_count += 1 - raise - logger.info( f"Review processor completed: {processed_count} processed, {failed_count} failed" ) diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 8f5ab36d5..456d595bf 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -186,3 +186,4 @@ def _update_review_record_with_file_location(self, review_record_id: str, review except Exception as e: logger.error(f"Failed to update review record with file location: {str(e)}") logger.warning("Review record created but file location not updated in DynamoDB") + raise diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index 86b9b2548..cf9b8d98f 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -16,9 +16,6 @@ def mock_review_service(mocker): return mocked_instance - - - @pytest.fixture def sample_review_message_body(): """Create a sample review message body.""" From 335bc2dabe40f736d200458916eee504451e542c Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Tue, 28 Oct 2025 11:10:40 +0000 Subject: [PATCH 04/47] [PRMP-585] added support for multiple files in sqs message --- lambdas/handlers/review_processor_handler.py | 24 +- lambdas/models/sqs/review_message_body.py | 9 +- lambdas/services/review_processor_service.py | 128 +++---- .../handlers/test_review_processor_handler.py | 125 ++++--- .../services/test_review_processor_service.py | 325 +++++++++++------- 5 files changed, 338 insertions(+), 273 deletions(-) diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index eb3194601..f92dc8eab 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,19 +1,17 @@ import json + +from pydantic import ValidationError from lambdas.models.sqs.review_message_body import ReviewMessageBody from lambdas.services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables -from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging -from utils.decorators.validate_sqs_message_event import validate_sqs_event -from utils.lambda_response import ApiGatewayResponse logger = LoggingService(__name__) @set_request_context_for_logging -@override_error_check @ensure_environment_variables( names=[ "DOCUMENT_REVIEW_DYNAMODB_NAME", @@ -21,8 +19,7 @@ "PENDING_REVIEW_BUCKET_NAME", ] ) -@handle_lambda_exceptions -@validate_sqs_event +@override_error_check def lambda_handler(event, context): """ This handler consumes SQS messages from the document review queue, creates DynamoDB @@ -34,7 +31,7 @@ def lambda_handler(event, context): _context: Lambda context Returns: - ApiGatewayResponse with processing status + None """ logger.info("Starting review processor Lambda") @@ -47,23 +44,22 @@ def lambda_handler(event, context): for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) - message = ReviewMessageBody.model_validate(sqs_message_body) + message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) processed_count += 1 + except ValidationError as error: + logger.error("Malformed review message") + logger.error(error) + except Exception as e: logger.error( f"Failed to process review message: {str(e)}", {"Result": "Review processing failed"}, ) failed_count += 1 + logger.info("Continuing to next message.") logger.info( f"Review processor completed: {processed_count} processed, {failed_count} failed" ) - - return ApiGatewayResponse( - status_code=200, - body=f"Processed {processed_count} messages", - methods="GET", - ).create_api_gateway_response() diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index c0cae7781..a34c84214 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -1,12 +1,17 @@ from pydantic import BaseModel -class ReviewMessageBody(BaseModel): - """Model for SQS message body from the document review queue.""" +class ReviewMessageFile(BaseModel): + """Model for individual file in SQS message body from the document review queue.""" file_name: str file_path: str """Location in the staging bucket""" + +class ReviewMessageBody(BaseModel): + """Model for SQS message body from the document review queue.""" + + files: list[ReviewMessageFile] nhs_number: str failure_reason: str upload_date: str diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 456d595bf..0e9416ac8 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -1,5 +1,6 @@ import os from datetime import datetime, timezone +import uuid from enums.review_status import ReviewStatus from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview @@ -43,13 +44,17 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: request_context.patient_nhs_no = review_message.nhs_number - logger.info(f"Processing review for NHS: {review_message.nhs_number}, File: {review_message.file_name}") + logger.info(f"Processing review for NHS: {review_message.nhs_number} with {len(review_message.files)} files") - self._verify_file_exists_in_staging(review_message.file_path) - document_upload_review = self._create_review_record(review_message) + for file in review_message.files: + logger.info(f"Processing review file: {file.file_name}") + self._verify_file_exists_in_staging(file.file_path) - new_file_key = self._move_file_to_review_bucket(review_message, document_upload_review.id) - self._update_review_record_with_file_location(document_upload_review.id, new_file_key) + review_id = uuid.uuid4().hex + files = self._move_files_to_review_bucket(review_message, review_id) + document_upload_review = self._build_review_record(review_message, review_id, files) + + self._create_review_record(document_upload_review) logger.info( f"Successfully processed review for {review_message.nhs_number}", @@ -80,89 +85,64 @@ def _verify_file_exists_in_staging(self, file_path: str) -> None: logger.error(f"Error checking file in staging bucket: {str(e)}") raise - def _create_review_record(self, message_data: ReviewMessageBody) -> DocumentsUploadReview: - """ - Create a new review record in DynamoDB. - - Args: - message_data: Validated review queue message data - - Returns: - Created DocumentsUploadReview object - - Raises: - ClientError: If DynamoDB create operation fails - """ - try: - files = [DocumentReviewFileDetails( - file_name=message_data.file_name, - file_location=message_data.file_path - )] - - document_review = DocumentsUploadReview( - nhs_number=message_data.nhs_number, - upload_date=int(datetime.fromisoformat(message_data.upload_date).replace(tzinfo=timezone.utc).timestamp()), - review_status=ReviewStatus.PENDING_REVIEW, - review_reason=message_data.failure_reason, - author=message_data.uploader_ods, - custodian=message_data.current_gp, - files=files, - ) - - self.dynamo_service.create_item( - table_name=self.review_table_name, - item=document_review.model_dump(by_alias=True, exclude_none=True), - ) - - logger.info( - f"Created review record in DynamoDB with ID: {document_review.id}", - {"Result": "DynamoDB record created"}, - ) - - return document_review - - except Exception as e: - logger.error(f"Failed to create DynamoDB record: {str(e)}") - raise + def _build_review_record( + self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] + ) -> DocumentsUploadReview: + return DocumentsUploadReview( + id=review_id, + nhs_number=message_data.nhs_number, + review_status=ReviewStatus.PENDING_REVIEW, + review_reason=message_data.failure_reason, + author=message_data.uploader_ods, + custodian=message_data.current_gp, + files=files, + upload_date=int(datetime.now(tz=timezone.utc).timestamp()) + ) - def _move_file_to_review_bucket(self, message_data: ReviewMessageBody, review_record_id: str) -> str: + def _move_files_to_review_bucket( + self, message_data: ReviewMessageBody, review_record_id: str + ) -> list[DocumentReviewFileDetails]: """ Move file from staging to review bucket. Args: message_data: Review queue message data - review_record_id: ID of the review record (used in destination path) + review_record_id: ID of the review record being created Returns: - New file key in review bucket - - Raises: - ClientError: If S3 copy or delete operations fail + List of DocumentReviewFileDetails objects for the moved files """ + moved_files = [] try: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{message_data.file_name}" + for file in message_data.files: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" - logger.info(f"Copying file from ({message_data.file_path}) in staging to review bucket: {new_file_key}") + logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=message_data.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - ) + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + ) - logger.info("File successfully copied to review bucket") - logger.info(f"Deleting file from staging bucket: {message_data.file_path}") + logger.info("File successfully copied to review bucket") + logger.info(f"Deleting file from staging bucket: {file.file_path}") - self._delete_from_staging(message_data.file_path) - logger.info(f"Successfully moved file to: {new_file_key}") + self._delete_from_staging(file.file_path) + logger.info(f"Successfully moved file to: {new_file_key}") - return new_file_key + moved_files.append(DocumentReviewFileDetails( + file_name=file.file_name, + file_location=new_file_key + )) except Exception as e: logger.error(f"Failed to move file: {str(e)}") raise + return moved_files + def _delete_from_staging(self, file_key: str) -> None: try: self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) @@ -173,17 +153,15 @@ def _delete_from_staging(self, file_key: str) -> None: logger.error(f"Error deleting file from staging: {str(e)}") raise - def _update_review_record_with_file_location(self, review_record_id: str, review_bucket_path: str) -> None: + def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: - self.dynamo_service.update_item( + self.dynamo_service.create_item( table_name=self.review_table_name, - key_pair={"ID": review_record_id}, - updated_fields={"ReviewBucketPath": review_bucket_path}, + item=review_record ) - logger.info(f"Updated review record {review_record_id} with file location: {review_bucket_path}") + logger.info(f"Created review record {review_record.id}") except Exception as e: - logger.error(f"Failed to update review record with file location: {str(e)}") - logger.warning("Review record created but file location not updated in DynamoDB") + logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") raise diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index cf9b8d98f..85be717a4 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -2,8 +2,7 @@ import pytest from handlers.review_processor_handler import lambda_handler -from models.sqs.review_message_body import ReviewMessageBody -from utils.lambda_response import ApiGatewayResponse +from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile @pytest.fixture @@ -20,8 +19,12 @@ def mock_review_service(mocker): def sample_review_message_body(): """Create a sample review message body.""" return ReviewMessageBody( - file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf", + files=[ + ReviewMessageFile( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -50,8 +53,12 @@ def sample_sqs_event(sample_sqs_message): def sample_sqs_event_multiple_messages(sample_review_message_body): """Create a sample SQS event with multiple messages.""" message_1 = ReviewMessageBody( - file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf", + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -60,8 +67,12 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_2 = ReviewMessageBody( - file_name="document_2.pdf", - file_path="staging/9000000010/document_2.pdf", + files=[ + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000010/document_2.pdf" + ) + ], nhs_number="9000000010", failure_reason="Invalid file format", upload_date="2024-01-15T10:35:00Z", @@ -70,8 +81,12 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_3 = ReviewMessageBody( - file_name="document_3.pdf", - file_path="staging/9000000011/document_3.pdf", + files=[ + ReviewMessageFile( + file_name="document_3.pdf", + file_path="staging/9000000011/document_3.pdf" + ) + ], nhs_number="9000000011", failure_reason="Missing metadata", upload_date="2024-01-15T10:40:00Z", @@ -121,15 +136,8 @@ def test_lambda_handler_processes_single_message_successfully( mock_review_service, ): """Test handler successfully processes a single SQS message.""" - expected_response = ApiGatewayResponse( - status_code=200, - body="Processed 1 messages", - methods="GET", - ).create_api_gateway_response() - - actual_response = lambda_handler(sample_sqs_event, context) + lambda_handler(sample_sqs_event, context) - assert actual_response == expected_response mock_review_service.process_review_message.assert_called_once() @@ -140,15 +148,8 @@ def test_lambda_handler_processes_multiple_messages_successfully( mock_review_service, ): """Test handler successfully processes multiple SQS messages.""" - expected_response = ApiGatewayResponse( - status_code=200, - body="Processed 3 messages", - methods="GET", - ).create_api_gateway_response() + lambda_handler(sample_sqs_event_multiple_messages, context) - actual_response = lambda_handler(sample_sqs_event_multiple_messages, context) - - assert actual_response == expected_response assert mock_review_service.process_review_message.call_count == 3 @@ -166,19 +167,18 @@ def test_lambda_handler_calls_service_with_correct_message( call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" - assert call_args.file_name == "test_document.pdf" + assert len(call_args.files) == 1 + assert call_args.files[0].file_name == "test_document.pdf" assert call_args.nhs_number == "9000000009" - assert call_args.file_path == "staging/9000000009/test_document.pdf" + assert call_args.files[0].file_path == "staging/9000000009/test_document.pdf" def test_lambda_handler_handles_empty_records_list( set_review_env, context, empty_sqs_event, mock_review_service ): - """Test handler handles empty records list via @validate_sqs_event decorator.""" - actual_response = lambda_handler(empty_sqs_event, context) + """Test handler handles empty records list gracefully.""" + lambda_handler(empty_sqs_event, context) - assert actual_response["statusCode"] == 400 - assert "SQS_4001" in actual_response["body"] mock_review_service.process_review_message.assert_not_called() @@ -188,15 +188,15 @@ def test_lambda_handler_handles_service_error( sample_sqs_event, mock_review_service, ): - """Test handler catches exception when service fails (via @handle_lambda_exceptions decorator).""" + """Test handler logs error but doesn't raise when service fails.""" mock_review_service.process_review_message.side_effect = Exception( "Service processing failed" ) - response = lambda_handler(sample_sqs_event, context) + result = lambda_handler(sample_sqs_event, context) - assert response["statusCode"] == 500 - assert "UE_500" in response["body"] + assert result is None + mock_review_service.process_review_message.assert_called_once() def test_lambda_handler_handles_invalid_message_format( @@ -208,7 +208,7 @@ def test_lambda_handler_handles_invalid_message_format( { "body": json.dumps( { - "file_name": "test.pdf", + "files": [{"file_name": "test.pdf"}], } ), "eventSource": "aws:sqs", @@ -216,10 +216,10 @@ def test_lambda_handler_handles_invalid_message_format( ] } - response = lambda_handler(invalid_event, context) + lambda_handler(invalid_event, context) - assert response["statusCode"] == 500 - assert "UE_500" in response["body"] + # Service should not be called if message validation fails + mock_review_service.process_review_message.assert_not_called() def test_lambda_handler_handles_partial_failure( @@ -228,16 +228,16 @@ def test_lambda_handler_handles_partial_failure( sample_sqs_event_multiple_messages, mock_review_service, ): - """Test handler processes messages and handles first failure.""" + """Test handler processes all messages even when one fails.""" mock_review_service.process_review_message.side_effect = [ None, Exception("Processing failed on second message"), + None, ] - response = lambda_handler(sample_sqs_event_multiple_messages, context) - - assert response["statusCode"] == 500 - assert mock_review_service.process_review_message.call_count == 2 + lambda_handler(sample_sqs_event_multiple_messages, context) + + assert mock_review_service.process_review_message.call_count == 3 def test_lambda_handler_parses_json_body_correctly( @@ -251,8 +251,12 @@ def test_lambda_handler_parses_json_body_correctly( { "body": json.dumps( { - "file_name": "test.pdf", - "file_path": "staging/test.pdf", + "files": [ + { + "file_name": "test.pdf", + "file_path": "staging/test.pdf" + } + ], "nhs_number": "9000000009", "failure_reason": "Test failure", "upload_date": "2024-01-15T10:30:00Z", @@ -270,7 +274,8 @@ def test_lambda_handler_parses_json_body_correctly( mock_review_service.process_review_message.assert_called_once() call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" - assert call_args.file_name == "test.pdf" + assert len(call_args.files) == 1 + assert call_args.files[0].file_name == "test.pdf" def test_lambda_handler_logs_correct_counts( @@ -278,13 +283,16 @@ def test_lambda_handler_logs_correct_counts( context, sample_sqs_event_multiple_messages, mock_review_service, + mocker, ): - """Test handler response contains correct processed count.""" - response = lambda_handler(sample_sqs_event_multiple_messages, context) - - response_body = response["body"] + """Test handler logs correct processed count.""" + mock_logger = mocker.patch("handlers.review_processor_handler.logger") + + lambda_handler(sample_sqs_event_multiple_messages, context) - assert "Processed 3 messages" in response_body + mock_logger.info.assert_any_call( + "Review processor completed: 3 processed, 0 failed" + ) def test_lambda_handler_tracks_failed_count( @@ -292,11 +300,16 @@ def test_lambda_handler_tracks_failed_count( context, sample_sqs_event, mock_review_service, + mocker, ): - """Test handler tracks failed message count.""" + """Test handler tracks failed message count in logs.""" mock_review_service.process_review_message.side_effect = Exception("Test error") + mock_logger = mocker.patch("handlers.review_processor_handler.logger") - response = lambda_handler(sample_sqs_event, context) + lambda_handler(sample_sqs_event, context) - assert response["statusCode"] == 500 mock_review_service.process_review_message.assert_called_once() + mock_logger.error.assert_called() + mock_logger.info.assert_any_call( + "Review processor completed: 0 processed, 1 failed" + ) diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 8907011b9..9c19b3f60 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -1,11 +1,8 @@ -from datetime import datetime, timezone -from unittest.mock import MagicMock - import pytest from botocore.exceptions import ClientError from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview -from models.sqs.review_message_body import ReviewMessageBody +from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.review_processor_service import ReviewProcessorService from utils.exceptions import S3FileNotFoundException @@ -41,8 +38,12 @@ def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): def sample_review_message(): """Create a sample review message.""" return ReviewMessageBody( - file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf", + files=[ + ReviewMessageFile( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -77,27 +78,63 @@ def test_process_review_message_success( mock_verify = mocker.patch.object( service_under_test, "_verify_file_exists_in_staging" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") mock_move = mocker.patch.object( - service_under_test, "_move_file_to_review_bucket" - ) - mock_update = mocker.patch.object( - service_under_test, "_update_review_record_with_file_location" + service_under_test, "_move_files_to_review_bucket" ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") - mock_review = MagicMock() - mock_review.id = "test-review-id" - mock_create.return_value = mock_review - mock_move.return_value = "9000000009/test-review-id/test_document.pdf" + mock_move.return_value = [ + {"file_name": "test_document.pdf", "file_location": "9000000009/test-review-id/test_document.pdf"} + ] service_under_test.process_review_message(sample_review_message) - mock_verify.assert_called_once_with(sample_review_message.file_path) - mock_create.assert_called_once_with(sample_review_message) - mock_move.assert_called_once_with(sample_review_message, "test-review-id") - mock_update.assert_called_once_with( - "test-review-id", "9000000009/test-review-id/test_document.pdf" + mock_verify.assert_called_once_with(sample_review_message.files[0].file_path) + mock_move.assert_called_once() + mock_create.assert_called_once() + + +def test_process_review_message_multiple_files( + service_under_test, mocker +): + """Test successful processing of a review message with multiple files.""" + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + mock_verify = mocker.patch.object( + service_under_test, "_verify_file_exists_in_staging" ) + mock_move = mocker.patch.object( + service_under_test, "_move_files_to_review_bucket" + ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") + + mock_move.return_value = [ + {"file_name": "document_1.pdf", "file_location": "9000000009/test-review-id/document_1.pdf"}, + {"file_name": "document_2.pdf", "file_location": "9000000009/test-review-id/document_2.pdf"} + ] + + service_under_test.process_review_message(message) + + assert mock_verify.call_count == 2 + mock_move.assert_called_once() + mock_create.assert_called_once() + def test_process_review_message_file_not_found( @@ -114,17 +151,17 @@ def test_process_review_message_file_not_found( service_under_test.process_review_message(sample_review_message) -def test_process_review_message_dynamo_error( +def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when DynamoDB create fails.""" + """Test processing fails when S3 copy operation fails.""" mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") mocker.patch.object( service_under_test, - "_create_review_record", + "_move_files_to_review_bucket", side_effect=ClientError( - {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, - "PutItem", + {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, + "CopyObject", ), ) @@ -132,22 +169,18 @@ def test_process_review_message_dynamo_error( service_under_test.process_review_message(sample_review_message) -def test_process_review_message_s3_copy_error( +def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when S3 copy operation fails.""" + """Test processing fails when DynamoDB create fails.""" mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") - mock_review = MagicMock() - mock_review.id = "test-review-id" - mocker.patch.object( - service_under_test, "_create_review_record", return_value=mock_review - ) + mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) mocker.patch.object( service_under_test, - "_move_file_to_review_bucket", + "_create_review_record", side_effect=ClientError( - {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, - "CopyObject", + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", ), ) @@ -155,6 +188,7 @@ def test_process_review_message_s3_copy_error( service_under_test.process_review_message(sample_review_message) + # Tests for _verify_file_exists_in_staging method @@ -190,18 +224,26 @@ def test_verify_file_s3_error(service_under_test): service_under_test._verify_file_exists_in_staging("staging/test.pdf") -# Tests for _create_review_record method +# Tests for _build_review_record and _create_review_record methods -def test_create_review_record_success( - service_under_test, sample_review_message, mocker -): - """Test successful creation of review record.""" - service_under_test.dynamo_service.create_item.return_value = None - - result = service_under_test._create_review_record(sample_review_message) +def test_build_review_record_success(service_under_test, sample_review_message): + """Test successful building of review record.""" + from models.document_review import DocumentReviewFileDetails + + files = [ + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-review-id/test_document.pdf" + ) + ] + + result = service_under_test._build_review_record( + sample_review_message, "test-review-id", files + ) assert isinstance(result, DocumentsUploadReview) + assert result.id == "test-review-id" assert result.nhs_number == "9000000009" assert result.review_status == ReviewStatus.PENDING_REVIEW assert result.review_reason == "Failed virus scan" @@ -209,68 +251,96 @@ def test_create_review_record_success( assert result.custodian == "Y12345" assert len(result.files) == 1 assert result.files[0].file_name == "test_document.pdf" - assert result.files[0].file_location == "staging/9000000009/test_document.pdf" + assert result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + - expected_timestamp = int( - datetime.fromisoformat("2024-01-15T10:30:00Z") - .replace(tzinfo=timezone.utc) - .timestamp() +def test_build_review_record_with_multiple_files(service_under_test): + """Test building review record with multiple files.""" + from models.document_review import DocumentReviewFileDetails + + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", ) - assert result.upload_date == expected_timestamp + + files = [ + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-review-id/document_1.pdf" + ), + DocumentReviewFileDetails( + file_name="document_2.pdf", + file_location="9000000009/test-review-id/document_2.pdf" + ) + ] + + result = service_under_test._build_review_record(message, "test-review-id", files) - service_under_test.dynamo_service.create_item.assert_called_once() - call_args = service_under_test.dynamo_service.create_item.call_args - assert call_args[1]["table_name"] == "test_review_table" + assert len(result.files) == 2 + assert result.files[0].file_name == "document_1.pdf" + assert result.files[1].file_name == "document_2.pdf" -def test_create_review_record_with_different_data(service_under_test, mocker): - """Test creation with different message data.""" - message = ReviewMessageBody( - file_name="another_doc.pdf", - file_path="staging/9000000010/another_doc.pdf", - nhs_number="9000000010", - failure_reason="Missing metadata", - upload_date="2024-02-20T14:45:30Z", - uploader_ods="Y67890", - current_gp="Y67890", +def test_create_review_record_success(service_under_test, sample_review_message): + """Test successful creation of review record in DynamoDB.""" + from models.document_review import DocumentReviewFileDetails + + review_record = DocumentsUploadReview( + id="test-review-id", + nhs_number="9000000009", + review_status=ReviewStatus.PENDING_REVIEW, + review_reason="Failed virus scan", + author="Y12345", + custodian="Y12345", + files=[ + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-review-id/test_document.pdf" + ) + ], + upload_date=1705319400 ) service_under_test.dynamo_service.create_item.return_value = None - result = service_under_test._create_review_record(message) - - assert result.nhs_number == "9000000010" - assert result.review_reason == "Missing metadata" - assert result.author == "Y67890" - assert result.custodian == "Y67890" + service_under_test._create_review_record(review_record) + service_under_test.dynamo_service.create_item.assert_called_once() + call_args = service_under_test.dynamo_service.create_item.call_args + assert call_args[1]["table_name"] == "test_review_table" + assert call_args[1]["item"] == review_record -def test_create_review_record_dynamo_error( - service_under_test, sample_review_message -): - """Test create record handles DynamoDB errors.""" - service_under_test.dynamo_service.create_item.side_effect = ClientError( - {"Error": {"Code": "ValidationException", "Message": "Invalid item"}}, - "PutItem", - ) - - with pytest.raises(ClientError): - service_under_test._create_review_record(sample_review_message) -# Tests for _move_file_to_review_bucket method +# Tests for _move_files_to_review_bucket method -def test_move_file_success(service_under_test, sample_review_message, mocker): +def test_move_files_success(service_under_test, sample_review_message, mocker): """Test successful file move from staging to review bucket.""" mocker.patch.object(service_under_test, "_delete_from_staging") - new_file_key = service_under_test._move_file_to_review_bucket( + files = service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id-123" ) expected_key = "9000000009/test-review-id-123/test_document.pdf" - assert new_file_key == expected_key + + assert len(files) == 1 + assert files[0].file_name == "test_document.pdf" + assert files[0].file_location == expected_key service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bucket", @@ -284,7 +354,41 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): ) -def test_move_file_copy_error(service_under_test, sample_review_message, mocker): +def test_move_multiple_files_success(service_under_test, mocker): + """Test successful move of multiple files.""" + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + mocker.patch.object(service_under_test, "_delete_from_staging") + + files = service_under_test._move_files_to_review_bucket(message, "test-review-id") + + assert len(files) == 2 + assert files[0].file_name == "document_1.pdf" + assert files[0].file_location == "9000000009/test-review-id/document_1.pdf" + assert files[1].file_name == "document_2.pdf" + assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" + + assert service_under_test.s3_service.copy_across_bucket.call_count == 2 + assert service_under_test._delete_from_staging.call_count == 2 + + +def test_move_files_copy_error(service_under_test, sample_review_message, mocker): """Test file move handles S3 copy errors.""" service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, @@ -292,12 +396,12 @@ def test_move_file_copy_error(service_under_test, sample_review_message, mocker) ) with pytest.raises(ClientError): - service_under_test._move_file_to_review_bucket( + service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) -def test_move_file_delete_error(service_under_test, sample_review_message, mocker): +def test_move_files_delete_error(service_under_test, sample_review_message, mocker): """Test file move handles delete errors.""" mocker.patch.object( service_under_test, @@ -309,11 +413,12 @@ def test_move_file_delete_error(service_under_test, sample_review_message, mocke ) with pytest.raises(ClientError): - service_under_test._move_file_to_review_bucket( + service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) + # Tests for _delete_from_staging method @@ -337,51 +442,17 @@ def test_delete_from_staging_error(service_under_test): service_under_test._delete_from_staging("staging/test.pdf") -# Tests for _update_review_record_with_file_location method - - -def test_update_review_record_success(service_under_test): - """Test successful update of review record with file location.""" - service_under_test._update_review_record_with_file_location( - "test-review-id", "9000000009/test-review-id/test.pdf" - ) - - service_under_test.dynamo_service.update_item.assert_called_once_with( - table_name="test_review_table", - key_pair={"ID": "test-review-id"}, - updated_fields={"ReviewBucketPath": "9000000009/test-review-id/test.pdf"}, - ) - - -def test_update_review_record_dynamo_error(service_under_test, mocker): - """Test update review record handles DynamoDB errors gracefully.""" - service_under_test.dynamo_service.update_item.side_effect = ClientError( - {"Error": {"Code": "ConditionalCheckFailedException", "Message": "Failed"}}, - "UpdateItem", - ) - - mock_logger = mocker.patch("services.review_processor_service.logger") - - service_under_test._update_review_record_with_file_location( - "test-review-id", "path/to/file.pdf" - ) - - assert mock_logger.error.called - assert mock_logger.warning.called - - # Integration scenario tests def test_full_workflow_with_valid_message( service_under_test, sample_review_message, mocker ): - """Test complete workflow from message to final update.""" + """Test complete workflow from message to final record creation.""" service_under_test.s3_service.file_exist_on_s3.return_value = True service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None - service_under_test.dynamo_service.update_item.return_value = None service_under_test.process_review_message(sample_review_message) @@ -389,7 +460,6 @@ def test_full_workflow_with_valid_message( service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() - service_under_test.dynamo_service.update_item.assert_called_once() def test_workflow_stops_at_verification_failure( @@ -411,12 +481,15 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None - service_under_test.dynamo_service.update_item.return_value = None messages = [ ReviewMessageBody( - file_name=f"doc_{i}.pdf", - file_path=f"staging/900000000{i}/doc_{i}.pdf", + files=[ + ReviewMessageFile( + file_name=f"doc_{i}.pdf", + file_path=f"staging/900000000{i}/doc_{i}.pdf" + ) + ], nhs_number=f"900000000{i}", failure_reason="Test failure", upload_date="2024-01-15T10:30:00Z", From e118f3bfa1ca156a3b993dedf3b9a0bc8574266d Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Tue, 28 Oct 2025 16:18:02 +0000 Subject: [PATCH 05/47] [PRMP-585] code review comment improvements --- .../base-lambdas-reusable-deploy-all.yml | 14 +++ lambdas/handlers/review_processor_handler.py | 21 ++--- lambdas/models/document_review.py | 10 +- lambdas/models/sqs/review_message_body.py | 4 +- lambdas/services/review_processor_service.py | 19 ++-- .../handlers/test_review_processor_handler.py | 94 ------------------- .../services/test_review_processor_service.py | 12 +-- lambdas/utils/exceptions.py | 18 ++++ 8 files changed, 66 insertions(+), 126 deletions(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index ee41d2cd3..d0e4bef45 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -668,3 +668,17 @@ jobs: lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_review_processor_lambda: + name: Deploy Review Processor Lambda + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: review_processor_handler + lambda_aws_name: ReviewProcessorLambda + lambda_layer_names: "core_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index f92dc8eab..7dfc229b6 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,8 +1,8 @@ import json from pydantic import ValidationError -from lambdas.models.sqs.review_message_body import ReviewMessageBody -from lambdas.services.review_processor_service import ReviewProcessorService +from models.sqs.review_message_body import ReviewMessageBody +from services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.override_error_check import override_error_check @@ -38,28 +38,23 @@ def lambda_handler(event, context): sqs_messages = event.get("Records", []) review_service = ReviewProcessorService() - processed_count = 0 - failed_count = 0 - for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) - processed_count += 1 + except ValidationError as error: logger.error("Malformed review message") logger.error(error) + raise error - except Exception as e: + except Exception as error: logger.error( - f"Failed to process review message: {str(e)}", + f"Failed to process review message: {str(error)}", {"Result": "Review processing failed"}, ) - failed_count += 1 + raise error + logger.info("Continuing to next message.") - - logger.info( - f"Review processor completed: {processed_count} processed, {failed_count} failed" - ) diff --git a/lambdas/models/document_review.py b/lambdas/models/document_review.py index 9b3eee106..76af50606 100644 --- a/lambdas/models/document_review.py +++ b/lambdas/models/document_review.py @@ -4,9 +4,9 @@ from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_pascal -from lambdas.enums.review_status import ReviewStatus -from lambdas.enums.snomed_codes import SnomedCodes -from lambdas.models.document_reference import DocumentReferenceMetadataFields +from enums.review_status import ReviewStatus +from enums.snomed_codes import SnomedCodes +from models.document_reference import DocumentReferenceMetadataFields class DocumentReviewFileDetails(BaseModel): @@ -30,7 +30,7 @@ class DocumentsUploadReview(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), alias=str(DocumentReferenceMetadataFields.ID.value) - ) # id differse to nogas version + ) author: str custodian: str review_status: ReviewStatus = Field(default=ReviewStatus.PENDING_REVIEW) @@ -38,7 +38,7 @@ class DocumentsUploadReview(BaseModel): review_date: int | None = Field(default=None) reviewer: str | None = Field(default=None) upload_date: int - files: list[DocumentReviewFileDetails] = Field(default=[]) # differs to nogas version + files: list[DocumentReviewFileDetails] nhs_number: str ttl: Optional[int] = Field( alias=str(DocumentReferenceMetadataFields.TTL.value), default=None diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index a34c84214..48000b5d2 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -1,11 +1,11 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field class ReviewMessageFile(BaseModel): """Model for individual file in SQS message body from the document review queue.""" file_name: str - file_path: str + file_path: str = Field(description="Location in the staging bucket") """Location in the staging bucket""" class ReviewMessageBody(BaseModel): diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 0e9416ac8..1a9b9eb1b 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -8,7 +8,7 @@ from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import S3FileNotFoundException +from utils.exceptions import ReviewProcessCreateRecordException, ReviewProcessDeleteException, ReviewProcessMovingException, ReviewProcessVerifyingException, S3FileNotFoundException from utils.request_context import request_context logger = LoggingService(__name__) @@ -79,11 +79,16 @@ def _verify_file_exists_in_staging(self, file_path: str) -> None: logger.info(f"Verified file exists in staging: {file_path}") - except S3FileNotFoundException: + except S3FileNotFoundException as e: + logger.info(e) + logger.info( + f"File not found in staging bucket {self.staging_bucket_name} for file_path {file_path}" + ) + raise except Exception as e: logger.error(f"Error checking file in staging bucket: {str(e)}") - raise + raise ReviewProcessVerifyingException(f"Error checking file in staging bucket: {str(e)}") def _build_review_record( self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] @@ -139,7 +144,7 @@ def _move_files_to_review_bucket( except Exception as e: logger.error(f"Failed to move file: {str(e)}") - raise + raise ReviewProcessMovingException(f"Failed to move file: {str(e)}") return moved_files @@ -151,7 +156,7 @@ def _delete_from_staging(self, file_key: str) -> None: except Exception as e: logger.error(f"Error deleting file from staging: {str(e)}") - raise + raise ReviewProcessDeleteException(f"Error deleting file from staging: {str(e)}") def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: @@ -164,4 +169,6 @@ def _create_review_record(self, review_record: DocumentsUploadReview) -> None: except Exception as e: logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") - raise + raise ReviewProcessCreateRecordException( + f"Failed to create review record with id: {review_record.id} -- {str(e)}" + ) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index 85be717a4..b3794863a 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -182,64 +182,6 @@ def test_lambda_handler_handles_empty_records_list( mock_review_service.process_review_message.assert_not_called() -def test_lambda_handler_handles_service_error( - set_review_env, - context, - sample_sqs_event, - mock_review_service, -): - """Test handler logs error but doesn't raise when service fails.""" - mock_review_service.process_review_message.side_effect = Exception( - "Service processing failed" - ) - - result = lambda_handler(sample_sqs_event, context) - - assert result is None - mock_review_service.process_review_message.assert_called_once() - - -def test_lambda_handler_handles_invalid_message_format( - set_review_env, context, mock_review_service -): - """Test handler handles validation errors for invalid message format.""" - invalid_event = { - "Records": [ - { - "body": json.dumps( - { - "files": [{"file_name": "test.pdf"}], - } - ), - "eventSource": "aws:sqs", - } - ] - } - - lambda_handler(invalid_event, context) - - # Service should not be called if message validation fails - mock_review_service.process_review_message.assert_not_called() - - -def test_lambda_handler_handles_partial_failure( - set_review_env, - context, - sample_sqs_event_multiple_messages, - mock_review_service, -): - """Test handler processes all messages even when one fails.""" - mock_review_service.process_review_message.side_effect = [ - None, - Exception("Processing failed on second message"), - None, - ] - - lambda_handler(sample_sqs_event_multiple_messages, context) - - assert mock_review_service.process_review_message.call_count == 3 - - def test_lambda_handler_parses_json_body_correctly( set_review_env, context, @@ -277,39 +219,3 @@ def test_lambda_handler_parses_json_body_correctly( assert len(call_args.files) == 1 assert call_args.files[0].file_name == "test.pdf" - -def test_lambda_handler_logs_correct_counts( - set_review_env, - context, - sample_sqs_event_multiple_messages, - mock_review_service, - mocker, -): - """Test handler logs correct processed count.""" - mock_logger = mocker.patch("handlers.review_processor_handler.logger") - - lambda_handler(sample_sqs_event_multiple_messages, context) - - mock_logger.info.assert_any_call( - "Review processor completed: 3 processed, 0 failed" - ) - - -def test_lambda_handler_tracks_failed_count( - set_review_env, - context, - sample_sqs_event, - mock_review_service, - mocker, -): - """Test handler tracks failed message count in logs.""" - mock_review_service.process_review_message.side_effect = Exception("Test error") - mock_logger = mocker.patch("handlers.review_processor_handler.logger") - - lambda_handler(sample_sqs_event, context) - - mock_review_service.process_review_message.assert_called_once() - mock_logger.error.assert_called() - mock_logger.info.assert_any_call( - "Review processor completed: 0 processed, 1 failed" - ) diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 9c19b3f60..a1501e335 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -3,8 +3,8 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.review_processor_service import ReviewProcessorService -from utils.exceptions import S3FileNotFoundException +from services.review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException @pytest.fixture @@ -220,7 +220,7 @@ def test_verify_file_s3_error(service_under_test): "HeadObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessVerifyingException): service_under_test._verify_file_exists_in_staging("staging/test.pdf") @@ -395,7 +395,7 @@ def test_move_files_copy_error(service_under_test, sample_review_message, mocker "CopyObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessMovingException): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) @@ -412,7 +412,7 @@ def test_move_files_delete_error(service_under_test, sample_review_message, mock ), ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessMovingException): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) @@ -438,7 +438,7 @@ def test_delete_from_staging_error(service_under_test): "DeleteObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessDeleteException): service_under_test._delete_from_staging("staging/test.pdf") diff --git a/lambdas/utils/exceptions.py b/lambdas/utils/exceptions.py index 40eba98ed..bc4516a02 100644 --- a/lambdas/utils/exceptions.py +++ b/lambdas/utils/exceptions.py @@ -159,8 +159,26 @@ class InvalidFileNameException(Exception): class MetadataPreprocessingException(Exception): pass + class FhirDocumentReferenceException(Exception): pass + class TransactionConflictException(Exception): pass + + +class ReviewProcessVerifyingException(Exception): + pass + + +class ReviewProcessMovingException(Exception): + pass + + +class ReviewProcessDeleteException(Exception): + pass + + +class ReviewProcessCreateRecordException(Exception): + pass From 67332740e2e931b2ba3b57852e60eca6bc056976 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Wed, 29 Oct 2025 11:19:11 +0000 Subject: [PATCH 06/47] [PRMP-585] Code review comment changes --- lambdas/services/review_processor_service.py | 2 +- lambdas/tests/unit/conftest.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 1a9b9eb1b..fd3360239 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -50,7 +50,7 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: logger.info(f"Processing review file: {file.file_name}") self._verify_file_exists_in_staging(file.file_path) - review_id = uuid.uuid4().hex + review_id = str(uuid.uuid4) files = self._move_files_to_review_bucket(review_message, review_id) document_upload_review = self._build_review_record(review_message, review_id, files) diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index 6d710159d..940317950 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -61,6 +61,8 @@ MOCK_APPCONFIG_CONFIGURATION_ENV_NAME = "APPCONFIG_CONFIGURATION" MOCK_STATISTICS_TABLE_NAME = "STATISTICS_TABLE" MOCK_STATISTICAL_REPORTS_BUCKET_ENV_NAME = "STATISTICAL_REPORTS_BUCKET" +MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME = "DOCUMENT_REVIEW_DYNAMODB_NAME" +MOCK_PENDING_REVIEW_BUCKET_NAME = "PENDING_REVIEW_BUCKET_NAME" MOCK_ARF_TABLE_NAME = "test_arf_dynamoDB_table" MOCK_PDM_TABLE_NAME = "test_pdm_dynamoDB_table" @@ -77,6 +79,8 @@ MOCK_LG_INVALID_SQS_QUEUE = "INVALID_SQS_QUEUE_URL" MOCK_STATISTICS_TABLE = "test_statistics_table" MOCK_STATISTICS_REPORT_BUCKET_NAME = "test_statistics_report_bucket" +MOCK_PENDING_REVIEW_BUCKET = "test_review_bucket" +MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME = "test_review_table" TEST_NHS_NUMBER = "9000000009" TEST_UUID = "1234-4567-8912-HSDF-TEST" @@ -225,6 +229,8 @@ def set_env(monkeypatch): monkeypatch.setenv("SLACK_BOT_TOKEN", MOCK_SLACK_BOT_TOKEN) monkeypatch.setenv("SLACK_CHANNEL_ID", MOCK_ALERTING_SLACK_CHANNEL_ID) monkeypatch.setenv("ITOC_TESTING_ODS_CODES", MOCK_ITOC_ODS_CODES) + monkeypatch.setenv(MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME, MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME) + monkeypatch.setenv(MOCK_PENDING_REVIEW_BUCKET_NAME, MOCK_PENDING_REVIEW_BUCKET) EXPECTED_PARSED_PATIENT_BASE_CASE = PatientDetails( From c32d4a8d2104cca4873c65c7a5c936ddaee58ce3 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Wed, 29 Oct 2025 16:16:48 +0000 Subject: [PATCH 07/47] [PRMP-585] required changes to match terraform --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 4 ++-- ...cessor_handler.py => document_review_processor_handler.py} | 2 +- ...cessor_service.py => document_review_processor_service.py} | 0 lambdas/tests/unit/handlers/test_review_processor_handler.py | 2 +- lambdas/tests/unit/services/test_review_processor_service.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename lambdas/handlers/{review_processor_handler.py => document_review_processor_handler.py} (96%) rename lambdas/services/{review_processor_service.py => document_review_processor_service.py} (100%) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index d0e4bef45..23b0e6d7a 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -677,8 +677,8 @@ jobs: python_version: ${{ inputs.python_version }} build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} - lambda_handler_name: review_processor_handler - lambda_aws_name: ReviewProcessorLambda + lambda_handler_name: document_review_processor_handler + lambda_aws_name: DocumentReviewProcessor lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py similarity index 96% rename from lambdas/handlers/review_processor_handler.py rename to lambdas/handlers/document_review_processor_handler.py index 7dfc229b6..44f0a01cd 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -2,7 +2,7 @@ from pydantic import ValidationError from models.sqs.review_message_body import ReviewMessageBody -from services.review_processor_service import ReviewProcessorService +from services.document_review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.override_error_check import override_error_check diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/document_review_processor_service.py similarity index 100% rename from lambdas/services/review_processor_service.py rename to lambdas/services/document_review_processor_service.py diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index b3794863a..4718fa678 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -1,7 +1,7 @@ import json import pytest -from handlers.review_processor_handler import lambda_handler +from handlers.document_review_processor_handler import lambda_handler from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index a1501e335..586813882 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -3,7 +3,7 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from services.document_review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException From 6a60022afcdd607c4deda76e6069680fc74de30a Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Thu, 30 Oct 2025 12:15:27 +0000 Subject: [PATCH 08/47] [PRPM-585] minor fixes --- .../document_review_processor_service.py | 8 ++++- ...test_document_review_processor_handler.py} | 2 +- ...test_document_review_processor_service.py} | 30 ++++++++----------- 3 files changed, 21 insertions(+), 19 deletions(-) rename lambdas/tests/unit/handlers/{test_review_processor_handler.py => test_document_review_processor_handler.py} (98%) rename lambdas/tests/unit/services/{test_review_processor_service.py => test_document_review_processor_service.py} (94%) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fd3360239..fc5dfd12e 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -8,7 +8,13 @@ from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import ReviewProcessCreateRecordException, ReviewProcessDeleteException, ReviewProcessMovingException, ReviewProcessVerifyingException, S3FileNotFoundException +from utils.exceptions import ( + ReviewProcessCreateRecordException, + ReviewProcessDeleteException, + ReviewProcessMovingException, + ReviewProcessVerifyingException, + S3FileNotFoundException +) from utils.request_context import request_context logger = LoggingService(__name__) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py similarity index 98% rename from lambdas/tests/unit/handlers/test_review_processor_handler.py rename to lambdas/tests/unit/handlers/test_document_review_processor_handler.py index 4718fa678..b652f0306 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py @@ -9,7 +9,7 @@ def mock_review_service(mocker): """Mock the ReviewProcessorService.""" mocked_class = mocker.patch( - "handlers.review_processor_handler.ReviewProcessorService" + "handlers.document_review_processor_handler.ReviewProcessorService" ) mocked_instance = mocked_class.return_value return mocked_instance diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py similarity index 94% rename from lambdas/tests/unit/services/test_review_processor_service.py rename to lambdas/tests/unit/services/test_document_review_processor_service.py index 586813882..38e177151 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -3,32 +3,28 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.document_review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from services.document_review_processor_service import ( + ReviewProcessMovingException, + ReviewProcessVerifyingException, + ReviewProcessorService, +) from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException @pytest.fixture def mock_dynamo_service(mocker): """Mock the DynamoDBService.""" - return mocker.patch("services.review_processor_service.DynamoDBService") + return mocker.patch("services.document_review_processor_service.DynamoDBService") @pytest.fixture def mock_s3_service(mocker): """Mock the S3Service.""" - return mocker.patch("services.review_processor_service.S3Service") + return mocker.patch("services.document_review_processor_service.S3Service") @pytest.fixture -def set_review_env(monkeypatch): - """Set up environment variables required for the service.""" - monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") - monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") - monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") - - -@pytest.fixture -def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): +def service_under_test(set_env, mock_dynamo_service, mock_s3_service): """Create a ReviewProcessorService instance with mocked dependencies.""" service = ReviewProcessorService() return service @@ -56,13 +52,13 @@ def sample_review_message(): def test_service_initializes_with_correct_environment_variables( - set_review_env, mock_dynamo_service, mock_s3_service + set_env, mock_dynamo_service, mock_s3_service ): """Test service initializes correctly with environment variables.""" service = ReviewProcessorService() assert service.review_table_name == "test_review_table" - assert service.staging_bucket_name == "test_staging_bucket" + assert service.staging_bucket_name == "test_staging_bulk_store" assert service.review_bucket_name == "test_review_bucket" mock_dynamo_service.assert_called_once() mock_s3_service.assert_called_once() @@ -199,7 +195,7 @@ def test_verify_file_exists_success(service_under_test): service_under_test._verify_file_exists_in_staging("staging/test.pdf") service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( - s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" ) @@ -343,7 +339,7 @@ def test_move_files_success(service_under_test, sample_review_message, mocker): assert files[0].file_location == expected_key service_under_test.s3_service.copy_across_bucket.assert_called_once_with( - source_bucket="test_staging_bucket", + source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", dest_file_key=expected_key, @@ -427,7 +423,7 @@ def test_delete_from_staging_success(service_under_test): service_under_test._delete_from_staging("staging/test.pdf") service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" ) From c505e2f01e9c1a6283feb815ccc25e0e304e5a2b Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Thu, 30 Oct 2025 15:13:53 +0000 Subject: [PATCH 09/47] [PRMP-585] exclude none --- lambdas/services/document_review_processor_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fc5dfd12e..665b456e5 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -168,7 +168,7 @@ def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: self.dynamo_service.create_item( table_name=self.review_table_name, - item=review_record + item=review_record.model_dump(by_alias=True, exclude_none=True) ) logger.info(f"Created review record {review_record.id}") From cc6cba22a9ff8753470833efd1b56d3d38b9e94f Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 14:51:54 +0000 Subject: [PATCH 10/47] [PRMP-585] enhance message checks --- .../document_review_processor_handler.py | 12 +- lambdas/models/sqs/review_message_body.py | 2 +- .../layers/requirements_core_lambda_layer.txt | 3 +- lambdas/services/base/dynamo_service.py | 32 ++- lambdas/services/base/s3_service.py | 19 +- .../document_review_processor_service.py | 147 ++++-------- .../test_document_review_processor_handler.py | 41 +++- .../unit/services/base/test_dynamo_service.py | 80 +++++++ .../unit/services/base/test_s3_service.py | 19 ++ .../test_document_review_processor_service.py | 213 ++++++------------ 10 files changed, 297 insertions(+), 271 deletions(-) diff --git a/lambdas/handlers/document_review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py index 44f0a01cd..84cf6b275 100644 --- a/lambdas/handlers/document_review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -28,7 +28,7 @@ def lambda_handler(event, context): Args: event: Lambda event containing SQS Event - _context: Lambda context + context: Lambda context Returns: None @@ -39,12 +39,16 @@ def lambda_handler(event, context): review_service = ReviewProcessorService() for sqs_message in sqs_messages: + message: ReviewMessageBody | None = None try: sqs_message_body = json.loads(sqs_message["body"]) - message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) + message = ReviewMessageBody.model_validate(sqs_message_body) - review_service.process_review_message(message) - + if isinstance(message, ReviewMessageBody): + review_service.process_review_message(message) + else: + raise ValidationError("Invalid review message format") + except ValidationError as error: logger.error("Malformed review message") logger.error(error) diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index 48000b5d2..e213168f2 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -10,7 +10,7 @@ class ReviewMessageFile(BaseModel): class ReviewMessageBody(BaseModel): """Model for SQS message body from the document review queue.""" - + upload_id: str files: list[ReviewMessageFile] nhs_number: str failure_reason: str diff --git a/lambdas/requirements/layers/requirements_core_lambda_layer.txt b/lambdas/requirements/layers/requirements_core_lambda_layer.txt index 499b03712..da90257d9 100644 --- a/lambdas/requirements/layers/requirements_core_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_core_lambda_layer.txt @@ -17,4 +17,5 @@ responses==0.23.1 six==1.16.0 types-PyYAML==6.0.12.11 regex==2023.12.25 -pikepdf==8.4.0 \ No newline at end of file +pikepdf==8.4.0 +types-boto3[dynamodb,s3]==1.40.64 \ No newline at end of file diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 3daffe33d..9ad9aeada 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,6 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError +from types_boto3_dynamodb.type_defs import PutItemOutputTableTypeDef from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -11,6 +12,7 @@ create_update_expression, ) from utils.exceptions import DynamoServiceException +from types_boto3_dynamodb import DynamoDBServiceResource logger = LoggingService(__name__) @@ -26,7 +28,7 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb = boto3.resource("dynamodb", region_name="eu-west-2") + self.dynamodb: DynamoDBServiceResource = boto3.resource("dynamodb", region_name="eu-west-2") self.initialised = True def get_table(self, table_name: str): @@ -88,6 +90,34 @@ def create_item(self, table_name, item): str(e), {"Result": f"Unable to write item to table: {table_name}"} ) raise e + + def put_item(self, table_name, item, key_name, check_exists: bool = True) -> PutItemOutputTableTypeDef: + """ + Put an item into the specified DynamoDB table with a condition on the existence of the key. + Args: + table_name: Name of the DynamoDB table + item: The item to be inserted (as a dictionary) + key_name: The name of the key field + check_exists: If True, the item will only be inserted if the key does not already exist. + If False, the item will only be inserted if the key already exists. + Returns: + Response from the DynamoDB put_item operation + Raises: + ClientError: For AWS service errors (DynamoDB) + """ + try: + table = self.get_table(table_name) + logger.info(f"Writing item to table: {table_name}") + return table.put_item(Item=item, Expected={ + key_name: { + 'Exists': check_exists + } + }) + except ClientError as e: + logger.error( + str(e), {"Result": f"Unable to write item to table: {table_name}"} + ) + raise e def update_item( self, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 5c6a690cb..d94afc5de 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -5,6 +5,7 @@ import boto3 from botocore.client import Config as BotoConfig +from types_boto3_s3 import S3Client from botocore.exceptions import ClientError from services.base.iam_service import IAMService from utils.audit_logging_setup import LoggingService @@ -32,7 +33,7 @@ def __init__(self, custom_aws_role=None): max_pool_connections=20, ) self.presigned_url_expiry = 1800 - self.client = boto3.client("s3", config=self.config) + self.client: S3Client = boto3.client("s3", config=self.config) self.initialised = True self.custom_client = None self.custom_aws_role = custom_aws_role @@ -130,6 +131,22 @@ def delete_object(self, s3_bucket_name: str, file_key: str, version_id: str | No return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key, VersionId=version_id) + def copy_across_bucket_if_none_match( + self, + source_bucket: str, + source_file_key: str, + dest_bucket: str, + dest_file_key: str, + if_none_match: str, + ): + return self.client.copy_object( + Bucket=dest_bucket, + Key=dest_file_key, + CopySource={"Bucket": source_bucket, "Key": source_file_key}, + IfNoneMatch=if_none_match, + StorageClass="INTELLIGENT_TIERING", + ) + def create_object_tag( self, s3_bucket_name: str, file_key: str, tag_key: str, tag_value: str ): diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 665b456e5..fe26f498e 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -1,20 +1,13 @@ import os from datetime import datetime, timezone -import uuid from enums.review_status import ReviewStatus +from models.document_reference import DocumentReferenceMetadataFields from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import ( - ReviewProcessCreateRecordException, - ReviewProcessDeleteException, - ReviewProcessMovingException, - ReviewProcessVerifyingException, - S3FileNotFoundException -) from utils.request_context import request_context logger = LoggingService(__name__) @@ -34,6 +27,7 @@ def __init__(self): self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] + def process_review_message(self, review_message: ReviewMessageBody) -> None: """ Process a single SQS message from the review queue. @@ -46,58 +40,25 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: S3FileNotFoundException: If file doesn't exist in staging bucket ClientError: For AWS service errors (DynamoDB, S3) """ + logger.info("Processing review queue message") request_context.patient_nhs_no = review_message.nhs_number - logger.info(f"Processing review for NHS: {review_message.nhs_number} with {len(review_message.files)} files") - - for file in review_message.files: - logger.info(f"Processing review file: {file.file_name}") - self._verify_file_exists_in_staging(file.file_path) - - review_id = str(uuid.uuid4) - files = self._move_files_to_review_bucket(review_message, review_id) - document_upload_review = self._build_review_record(review_message, review_id, files) - - self._create_review_record(document_upload_review) - - logger.info( - f"Successfully processed review for {review_message.nhs_number}", - {"Result": "Review record created and file moved"}, + review_id = review_message.upload_id + review_files = self._move_files_to_review_bucket(review_message, review_id) + document_upload_review = self._build_review_record(review_message, review_id, review_files) + self.dynamo_service.put_item( + table_name=self.review_table_name, + item=document_upload_review.model_dump(by_alias=True, exclude_none=True), + key_name=DocumentReferenceMetadataFields.ID.value ) - def _verify_file_exists_in_staging(self, file_path: str) -> None: - """ - Verify the file exists in the staging bucket. - - Args: - file_path: S3 key of the file in staging bucket - - Raises: - S3FileNotFoundException: If file does not exist in staging bucket - """ - try: - file_exists = self.s3_service.file_exist_on_s3(s3_bucket_name=self.staging_bucket_name, file_key=file_path) - - if not file_exists: - raise S3FileNotFoundException(f"File not found in staging bucket: {file_path}") - - logger.info(f"Verified file exists in staging: {file_path}") - - except S3FileNotFoundException as e: - logger.info(e) - logger.info( - f"File not found in staging bucket {self.staging_bucket_name} for file_path {file_path}" - ) - - raise - except Exception as e: - logger.error(f"Error checking file in staging bucket: {str(e)}") - raise ReviewProcessVerifyingException(f"Error checking file in staging bucket: {str(e)}") + logger.info(f"Created review record {document_upload_review.id}") + self._delete_files_from_staging(review_message) def _build_review_record( - self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] + self, message_data: ReviewMessageBody, review_id: str, review_files: list[DocumentReviewFileDetails] ) -> DocumentsUploadReview: return DocumentsUploadReview( id=review_id, @@ -106,7 +67,7 @@ def _build_review_record( review_reason=message_data.failure_reason, author=message_data.uploader_ods, custodian=message_data.current_gp, - files=files, + files=review_files, upload_date=int(datetime.now(tz=timezone.utc).timestamp()) ) @@ -121,60 +82,40 @@ def _move_files_to_review_bucket( review_record_id: ID of the review record being created Returns: - List of DocumentReviewFileDetails objects for the moved files + List of DocumentReviewFileDetails with new file locations in review bucket """ - moved_files = [] - try: - for file in message_data.files: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" - - logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") - - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=file.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - ) - - logger.info("File successfully copied to review bucket") - logger.info(f"Deleting file from staging bucket: {file.file_path}") - - self._delete_from_staging(file.file_path) - logger.info(f"Successfully moved file to: {new_file_key}") + new_file_keys: list[DocumentReviewFileDetails] = [] + for file in message_data.files: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + + logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") + + self.s3_service.copy_across_bucket_if_none_match( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + if_none_match="*", + ) - moved_files.append(DocumentReviewFileDetails( + new_file_keys.append( + DocumentReviewFileDetails( file_name=file.file_name, - file_location=new_file_key - )) - - except Exception as e: - logger.error(f"Failed to move file: {str(e)}") - raise ReviewProcessMovingException(f"Failed to move file: {str(e)}") - - return moved_files - - def _delete_from_staging(self, file_key: str) -> None: - try: - self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) - - logger.info(f"Deleted file from staging bucket: {file_key}") + file_location=new_file_key, + ) + ) - except Exception as e: - logger.error(f"Error deleting file from staging: {str(e)}") - raise ReviewProcessDeleteException(f"Error deleting file from staging: {str(e)}") + logger.info("File successfully copied to review bucket") + logger.info(f"Successfully moved file to: {new_file_key}") + return new_file_keys - def _create_review_record(self, review_record: DocumentsUploadReview) -> None: - try: - self.dynamo_service.create_item( - table_name=self.review_table_name, - item=review_record.model_dump(by_alias=True, exclude_none=True) - ) + def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: + for file in message_data.files: + try: + logger.info(f"Deleting file from staging bucket: {file.file_path}") + self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file.file_path) + except Exception as e: + logger.error(f"Error deleting files from staging: {str(e)}") + # Continue processing as files - logger.info(f"Created review record {review_record.id}") - except Exception as e: - logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") - raise ReviewProcessCreateRecordException( - f"Failed to create review record with id: {review_record.id} -- {str(e)}" - ) diff --git a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py index b652f0306..53521de91 100644 --- a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py @@ -19,6 +19,7 @@ def mock_review_service(mocker): def sample_review_message_body(): """Create a sample review message body.""" return ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="test_document.pdf", @@ -53,6 +54,7 @@ def sample_sqs_event(sample_sqs_message): def sample_sqs_event_multiple_messages(sample_review_message_body): """Create a sample SQS event with multiple messages.""" message_1 = ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -67,6 +69,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_2 = ReviewMessageBody( + upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_2.pdf", @@ -81,6 +84,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_3 = ReviewMessageBody( + upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_3.pdf", @@ -121,16 +125,9 @@ def empty_sqs_event(): return {"Records": []} -@pytest.fixture -def set_review_env(monkeypatch): - """Set up environment variables required for the handler.""" - monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") - monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") - monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") - def test_lambda_handler_processes_single_message_successfully( - set_review_env, + set_env, context, sample_sqs_event, mock_review_service, @@ -142,7 +139,7 @@ def test_lambda_handler_processes_single_message_successfully( def test_lambda_handler_processes_multiple_messages_successfully( - set_review_env, + set_env, context, sample_sqs_event_multiple_messages, mock_review_service, @@ -154,7 +151,7 @@ def test_lambda_handler_processes_multiple_messages_successfully( def test_lambda_handler_calls_service_with_correct_message( - set_review_env, + set_env, context, sample_sqs_event, mock_review_service, @@ -174,7 +171,7 @@ def test_lambda_handler_calls_service_with_correct_message( def test_lambda_handler_handles_empty_records_list( - set_review_env, context, empty_sqs_event, mock_review_service + set_env, context, empty_sqs_event, mock_review_service ): """Test handler handles empty records list gracefully.""" lambda_handler(empty_sqs_event, context) @@ -183,7 +180,7 @@ def test_lambda_handler_handles_empty_records_list( def test_lambda_handler_parses_json_body_correctly( - set_review_env, + set_env, context, mock_review_service, ): @@ -193,6 +190,7 @@ def test_lambda_handler_parses_json_body_correctly( { "body": json.dumps( { + "upload_id": "test-upload-id-123", "files": [ { "file_name": "test.pdf", @@ -219,3 +217,22 @@ def test_lambda_handler_parses_json_body_correctly( assert len(call_args.files) == 1 assert call_args.files[0].file_name == "test.pdf" + +def test_lambda_handler_calls_process_review_message_on_service( + set_env, + context, + sample_sqs_event, + sample_review_message_body, + mock_review_service, +): + """Test that handler calls process_review_message method on ReviewProcessorService.""" + lambda_handler(sample_sqs_event, context) + + mock_review_service.process_review_message.assert_called_once() + called_message = mock_review_service.process_review_message.call_args[0][0] + + assert isinstance(called_message, ReviewMessageBody) + assert called_message.upload_id == sample_review_message_body.upload_id + assert called_message.nhs_number == sample_review_message_body.nhs_number + assert called_message.failure_reason == sample_review_message_body.failure_reason + diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index ce9594ad6..d7c6cc9b7 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -292,6 +292,86 @@ def test_create_item_raise_client_error(mock_service, mock_table): assert MOCK_CLIENT_ERROR == actual_response.value +def test_put_item_with_check_exists_true(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=True) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': True + } + } + ) + + +def test_put_item_with_check_exists_false(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=False) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': False + } + } + ) + + +def test_put_item_default_check_exists(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + # Test that default value is True + mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': True + } + } + ) + + +def test_put_item_returns_response(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + expected_response = { + 'ResponseMetadata': { + 'HTTPStatusCode': 200 + } + } + + mock_table.return_value.put_item.return_value = expected_response + + actual_response = mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + assert actual_response == expected_response + + +def test_put_item_raises_client_error(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_table.return_value.put_item.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as actual_response: + mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + assert MOCK_CLIENT_ERROR == actual_response.value + + def test_delete_item_is_called_with_correct_parameters(mock_service, mock_table): mock_service.delete_item(MOCK_TABLE_NAME, {"NhsNumber": TEST_NHS_NUMBER}) diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index 3d3d8b4ed..ef68ddaff 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -139,6 +139,25 @@ def test_copy_across_bucket(mock_service, mock_client): ) +def test_copy_across_bucket_if_none_match(mock_service, mock_client): + test_etag = '"abc123def456"' + + mock_service.copy_across_bucket_if_none_match( + source_bucket="bucket_to_copy_from", + source_file_key=TEST_FILE_KEY, + dest_bucket="bucket_to_copy_to", + dest_file_key=f"{TEST_NHS_NUMBER}/{TEST_UUID}", + if_none_match=test_etag, + ) + + mock_client.copy_object.assert_called_once_with( + Bucket="bucket_to_copy_to", + Key=f"{TEST_NHS_NUMBER}/{TEST_UUID}", + CopySource={"Bucket": "bucket_to_copy_from", "Key": TEST_FILE_KEY}, + IfNoneMatch=test_etag, + StorageClass="INTELLIGENT_TIERING", + ) + def test_delete_object(mock_service, mock_client): mock_service.delete_object(s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 38e177151..7a323bb87 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -4,11 +4,10 @@ from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.document_review_processor_service import ( - ReviewProcessMovingException, - ReviewProcessVerifyingException, ReviewProcessorService, ) -from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException +from utils.exceptions import S3FileNotFoundException +from models.document_review import DocumentReviewFileDetails @pytest.fixture @@ -34,6 +33,7 @@ def service_under_test(set_env, mock_dynamo_service, mock_s3_service): def sample_review_message(): """Create a sample review message.""" return ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="test_document.pdf", @@ -71,23 +71,25 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - mock_verify = mocker.patch.object( - service_under_test, "_verify_file_exists_in_staging" - ) mock_move = mocker.patch.object( service_under_test, "_move_files_to_review_bucket" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_delete = mocker.patch.object( + service_under_test, "_delete_files_from_staging" + ) mock_move.return_value = [ - {"file_name": "test_document.pdf", "file_location": "9000000009/test-review-id/test_document.pdf"} + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-upload-id-123/test_document.pdf" + ) ] service_under_test.process_review_message(sample_review_message) - mock_verify.assert_called_once_with(sample_review_message.files[0].file_path) mock_move.assert_called_once() - mock_create.assert_called_once() + service_under_test.dynamo_service.put_item.assert_called_once() + mock_delete.assert_called_once_with(sample_review_message) def test_process_review_message_multiple_files( @@ -95,6 +97,7 @@ def test_process_review_message_multiple_files( ): """Test successful processing of a review message with multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -112,46 +115,36 @@ def test_process_review_message_multiple_files( current_gp="Y12345", ) - mock_verify = mocker.patch.object( - service_under_test, "_verify_file_exists_in_staging" - ) mock_move = mocker.patch.object( service_under_test, "_move_files_to_review_bucket" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_delete = mocker.patch.object( + service_under_test, "_delete_files_from_staging" + ) mock_move.return_value = [ - {"file_name": "document_1.pdf", "file_location": "9000000009/test-review-id/document_1.pdf"}, - {"file_name": "document_2.pdf", "file_location": "9000000009/test-review-id/document_2.pdf"} + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-upload-id-456/document_1.pdf" + ), + DocumentReviewFileDetails( + file_name="document_2.pdf", + file_location="9000000009/test-upload-id-456/document_2.pdf" + ) ] service_under_test.process_review_message(message) - assert mock_verify.call_count == 2 mock_move.assert_called_once() - mock_create.assert_called_once() - - - -def test_process_review_message_file_not_found( - service_under_test, sample_review_message, mocker -): - """Test processing fails when file doesn't exist in staging.""" - mocker.patch.object( - service_under_test, - "_verify_file_exists_in_staging", - side_effect=S3FileNotFoundException("File not found"), - ) + service_under_test.dynamo_service.put_item.assert_called_once() + mock_delete.assert_called_once_with(message) - with pytest.raises(S3FileNotFoundException, match="File not found"): - service_under_test.process_review_message(sample_review_message) def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): """Test processing fails when S3 copy operation fails.""" - mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") mocker.patch.object( service_under_test, "_move_files_to_review_bucket", @@ -168,23 +161,17 @@ def test_process_review_message_s3_copy_error( def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when DynamoDB create fails.""" - mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + """Test processing fails when DynamoDB put fails.""" mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) - mocker.patch.object( - service_under_test, - "_create_review_record", - side_effect=ClientError( - {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, - "PutItem", - ), + service_under_test.dynamo_service.put_item.side_effect = ClientError( + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", ) with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) - # Tests for _verify_file_exists_in_staging method @@ -224,9 +211,7 @@ def test_verify_file_s3_error(service_under_test): def test_build_review_record_success(service_under_test, sample_review_message): - """Test successful building of review record.""" - from models.document_review import DocumentReviewFileDetails - + """Test successful building of review record.""" files = [ DocumentReviewFileDetails( file_name="test_document.pdf", @@ -251,10 +236,9 @@ def test_build_review_record_success(service_under_test, sample_review_message): def test_build_review_record_with_multiple_files(service_under_test): - """Test building review record with multiple files.""" - from models.document_review import DocumentReviewFileDetails - + """Test building review record with multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -290,44 +274,11 @@ def test_build_review_record_with_multiple_files(service_under_test): assert result.files[1].file_name == "document_2.pdf" -def test_create_review_record_success(service_under_test, sample_review_message): - """Test successful creation of review record in DynamoDB.""" - from models.document_review import DocumentReviewFileDetails - - review_record = DocumentsUploadReview( - id="test-review-id", - nhs_number="9000000009", - review_status=ReviewStatus.PENDING_REVIEW, - review_reason="Failed virus scan", - author="Y12345", - custodian="Y12345", - files=[ - DocumentReviewFileDetails( - file_name="test_document.pdf", - file_location="9000000009/test-review-id/test_document.pdf" - ) - ], - upload_date=1705319400 - ) - - service_under_test.dynamo_service.create_item.return_value = None - - service_under_test._create_review_record(review_record) - - service_under_test.dynamo_service.create_item.assert_called_once() - call_args = service_under_test.dynamo_service.create_item.call_args - assert call_args[1]["table_name"] == "test_review_table" - assert call_args[1]["item"] == review_record - - - # Tests for _move_files_to_review_bucket method -def test_move_files_success(service_under_test, sample_review_message, mocker): +def test_move_files_success(service_under_test, sample_review_message): """Test successful file move from staging to review bucket.""" - mocker.patch.object(service_under_test, "_delete_from_staging") - files = service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id-123" ) @@ -338,21 +289,19 @@ def test_move_files_success(service_under_test, sample_review_message, mocker): assert files[0].file_name == "test_document.pdf" assert files[0].file_location == expected_key - service_under_test.s3_service.copy_across_bucket.assert_called_once_with( + service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once_with( source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", dest_file_key=expected_key, - ) - - service_under_test._delete_from_staging.assert_called_once_with( - "staging/9000000009/test_document.pdf" + if_none_match="*", ) -def test_move_multiple_files_success(service_under_test, mocker): +def test_move_multiple_files_success(service_under_test): """Test successful move of multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-999", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -369,8 +318,6 @@ def test_move_multiple_files_success(service_under_test, mocker): uploader_ods="Y12345", current_gp="Y12345", ) - - mocker.patch.object(service_under_test, "_delete_from_staging") files = service_under_test._move_files_to_review_bucket(message, "test-review-id") @@ -380,106 +327,75 @@ def test_move_multiple_files_success(service_under_test, mocker): assert files[1].file_name == "document_2.pdf" assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" - assert service_under_test.s3_service.copy_across_bucket.call_count == 2 - assert service_under_test._delete_from_staging.call_count == 2 + assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 2 -def test_move_files_copy_error(service_under_test, sample_review_message, mocker): +def test_move_files_copy_error(service_under_test, sample_review_message): """Test file move handles S3 copy errors.""" - service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + service_under_test.s3_service.copy_across_bucket_if_none_match.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, "CopyObject", ) - with pytest.raises(ReviewProcessMovingException): - service_under_test._move_files_to_review_bucket( - sample_review_message, "test-review-id" - ) - - -def test_move_files_delete_error(service_under_test, sample_review_message, mocker): - """Test file move handles delete errors.""" - mocker.patch.object( - service_under_test, - "_delete_from_staging", - side_effect=ClientError( - {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, - "DeleteObject", - ), - ) - - with pytest.raises(ReviewProcessMovingException): + with pytest.raises(ClientError): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) -# Tests for _delete_from_staging method +# Tests for _delete_files_from_staging method -def test_delete_from_staging_success(service_under_test): +def test_delete_from_staging_success(service_under_test, sample_review_message): """Test successful deletion from staging bucket.""" - service_under_test._delete_from_staging("staging/test.pdf") + service_under_test._delete_files_from_staging(sample_review_message) service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/9000000009/test_document.pdf" ) -def test_delete_from_staging_error(service_under_test): - """Test delete from staging handles S3 errors.""" +def test_delete_from_staging_handles_errors(service_under_test, sample_review_message): + """Test deletion from staging handles errors gracefully.""" service_under_test.s3_service.delete_object.side_effect = ClientError( - {"Error": {"Code": "NoSuchKey", "Message": "Key does not exist"}}, + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, "DeleteObject", ) - with pytest.raises(ReviewProcessDeleteException): - service_under_test._delete_from_staging("staging/test.pdf") + # Should not raise exception - errors are caught and logged + service_under_test._delete_files_from_staging(sample_review_message) + + service_under_test.s3_service.delete_object.assert_called_once() # Integration scenario tests def test_full_workflow_with_valid_message( - service_under_test, sample_review_message, mocker + service_under_test, sample_review_message ): """Test complete workflow from message to final record creation.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - service_under_test.dynamo_service.create_item.return_value = None - service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.dynamo_service.put_item.return_value = None + service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) - service_under_test.s3_service.file_exist_on_s3.assert_called_once() - service_under_test.dynamo_service.create_item.assert_called_once() - service_under_test.s3_service.copy_across_bucket.assert_called_once() + service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() -def test_workflow_stops_at_verification_failure( - service_under_test, sample_review_message -): - """Test workflow stops if file verification fails.""" - service_under_test.s3_service.file_exist_on_s3.return_value = False - - with pytest.raises(S3FileNotFoundException): - service_under_test.process_review_message(sample_review_message) - - service_under_test.dynamo_service.create_item.assert_not_called() - service_under_test.s3_service.copy_across_bucket.assert_not_called() - - -def test_workflow_handles_multiple_different_patients(service_under_test, mocker): +def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - service_under_test.dynamo_service.create_item.return_value = None - service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.dynamo_service.put_item.return_value = None + service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None service_under_test.s3_service.delete_object.return_value = None messages = [ ReviewMessageBody( + upload_id=f"test-upload-id-{i}", files=[ ReviewMessageFile( file_name=f"doc_{i}.pdf", @@ -498,5 +414,6 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker for message in messages: service_under_test.process_review_message(message) - assert service_under_test.dynamo_service.create_item.call_count == 3 - assert service_under_test.s3_service.copy_across_bucket.call_count == 3 + assert service_under_test.dynamo_service.put_item.call_count == 3 + assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 3 + From 0a60c7112a2bbe2a42b17302b9df45fa8420845c Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 14:56:14 +0000 Subject: [PATCH 11/47] fix test --- .../test_document_review_processor_service.py | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 7a323bb87..65425ec5d 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -171,42 +171,6 @@ def test_process_review_message_dynamo_error( with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) - -# Tests for _verify_file_exists_in_staging method - - -def test_verify_file_exists_success(service_under_test): - """Test successful file verification.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - - service_under_test._verify_file_exists_in_staging("staging/test.pdf") - - service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" - ) - - -def test_verify_file_does_not_exist(service_under_test): - """Test file verification fails when file doesn't exist.""" - service_under_test.s3_service.file_exist_on_s3.return_value = False - - with pytest.raises( - S3FileNotFoundException, match="File not found in staging bucket" - ): - service_under_test._verify_file_exists_in_staging("staging/missing.pdf") - - -def test_verify_file_s3_error(service_under_test): - """Test file verification handles S3 errors.""" - service_under_test.s3_service.file_exist_on_s3.side_effect = ClientError( - {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, - "HeadObject", - ) - - with pytest.raises(ReviewProcessVerifyingException): - service_under_test._verify_file_exists_in_staging("staging/test.pdf") - - # Tests for _build_review_record and _create_review_record methods From 58111a804c85659c707ff64f90d5a903dba819c1 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 15:52:58 +0000 Subject: [PATCH 12/47] [PRMP-585] remove unsed import --- .../unit/services/test_document_review_processor_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 65425ec5d..c36f13f48 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -6,7 +6,6 @@ from services.document_review_processor_service import ( ReviewProcessorService, ) -from utils.exceptions import S3FileNotFoundException from models.document_review import DocumentReviewFileDetails From eb9c4d48bf2010a17ba8e59e41adf2c8c7357135 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 16:51:52 +0000 Subject: [PATCH 13/47] Code comment changes --- .../document_review_processor_handler.py | 17 +-- lambdas/services/base/s3_service.py | 27 ++--- .../document_review_processor_service.py | 30 +++-- .../unit/services/base/test_s3_service.py | 31 +++-- .../test_document_review_processor_service.py | 114 ++++++++---------- 5 files changed, 105 insertions(+), 114 deletions(-) diff --git a/lambdas/handlers/document_review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py index 84cf6b275..43a33d055 100644 --- a/lambdas/handlers/document_review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -1,7 +1,5 @@ -import json - -from pydantic import ValidationError from models.sqs.review_message_body import ReviewMessageBody +from pydantic import ValidationError from services.document_review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -39,16 +37,11 @@ def lambda_handler(event, context): review_service = ReviewProcessorService() for sqs_message in sqs_messages: - message: ReviewMessageBody | None = None try: - sqs_message_body = json.loads(sqs_message["body"]) - message = ReviewMessageBody.model_validate(sqs_message_body) + message = ReviewMessageBody.model_validate_json(sqs_message["body"]) + + review_service.process_review_message(message) - if isinstance(message, ReviewMessageBody): - review_service.process_review_message(message) - else: - raise ValidationError("Invalid review message format") - except ValidationError as error: logger.error("Malformed review message") logger.error(error) @@ -60,5 +53,5 @@ def lambda_handler(event, context): {"Result": "Review processing failed"}, ) raise error - + logger.info("Continuing to next message.") diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index d94afc5de..e45bca015 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -5,9 +5,9 @@ import boto3 from botocore.client import Config as BotoConfig -from types_boto3_s3 import S3Client from botocore.exceptions import ClientError from services.base.iam_service import IAMService +from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException @@ -117,7 +117,16 @@ def copy_across_bucket( source_file_key: str, dest_bucket: str, dest_file_key: str, + if_none_match: str | None = None, ): + if if_none_match is not None: + return self.client.copy_object( + Bucket=dest_bucket, + Key=dest_file_key, + CopySource={"Bucket": source_bucket, "Key": source_file_key}, + IfNoneMatch=if_none_match, + StorageClass="INTELLIGENT_TIERING", + ) return self.client.copy_object( Bucket=dest_bucket, Key=dest_file_key, @@ -131,22 +140,6 @@ def delete_object(self, s3_bucket_name: str, file_key: str, version_id: str | No return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key, VersionId=version_id) - def copy_across_bucket_if_none_match( - self, - source_bucket: str, - source_file_key: str, - dest_bucket: str, - dest_file_key: str, - if_none_match: str, - ): - return self.client.copy_object( - Bucket=dest_bucket, - Key=dest_file_key, - CopySource={"Bucket": source_bucket, "Key": source_file_key}, - IfNoneMatch=if_none_match, - StorageClass="INTELLIGENT_TIERING", - ) - def create_object_tag( self, s3_bucket_name: str, file_key: str, tag_key: str, tag_value: str ): diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fe26f498e..8dfa0ad5b 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -27,7 +27,6 @@ def __init__(self): self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] - def process_review_message(self, review_message: ReviewMessageBody) -> None: """ Process a single SQS message from the review queue. @@ -47,18 +46,23 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: review_id = review_message.upload_id review_files = self._move_files_to_review_bucket(review_message, review_id) - document_upload_review = self._build_review_record(review_message, review_id, review_files) + document_upload_review = self._build_review_record( + review_message, review_id, review_files + ) self.dynamo_service.put_item( table_name=self.review_table_name, item=document_upload_review.model_dump(by_alias=True, exclude_none=True), - key_name=DocumentReferenceMetadataFields.ID.value + key_name=DocumentReferenceMetadataFields.ID.value, ) logger.info(f"Created review record {document_upload_review.id}") self._delete_files_from_staging(review_message) def _build_review_record( - self, message_data: ReviewMessageBody, review_id: str, review_files: list[DocumentReviewFileDetails] + self, + message_data: ReviewMessageBody, + review_id: str, + review_files: list[DocumentReviewFileDetails], ) -> DocumentsUploadReview: return DocumentsUploadReview( id=review_id, @@ -68,7 +72,7 @@ def _build_review_record( author=message_data.uploader_ods, custodian=message_data.current_gp, files=review_files, - upload_date=int(datetime.now(tz=timezone.utc).timestamp()) + upload_date=int(datetime.now(tz=timezone.utc).timestamp()), ) def _move_files_to_review_bucket( @@ -86,11 +90,15 @@ def _move_files_to_review_bucket( """ new_file_keys: list[DocumentReviewFileDetails] = [] for file in message_data.files: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + new_file_key = ( + f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + ) - logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") + logger.info( + f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}" + ) - self.s3_service.copy_across_bucket_if_none_match( + self.s3_service.copy_across_bucket( source_bucket=self.staging_bucket_name, source_file_key=file.file_path, dest_bucket=self.review_bucket_name, @@ -113,9 +121,9 @@ def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: for file in message_data.files: try: logger.info(f"Deleting file from staging bucket: {file.file_path}") - self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file.file_path) + self.s3_service.delete_object( + s3_bucket_name=self.staging_bucket_name, file_key=file.file_path + ) except Exception as e: logger.error(f"Error deleting files from staging: {str(e)}") # Continue processing as files - - diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index ef68ddaff..ead525f00 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -141,8 +141,8 @@ def test_copy_across_bucket(mock_service, mock_client): def test_copy_across_bucket_if_none_match(mock_service, mock_client): test_etag = '"abc123def456"' - - mock_service.copy_across_bucket_if_none_match( + + mock_service.copy_across_bucket( source_bucket="bucket_to_copy_from", source_file_key=TEST_FILE_KEY, dest_bucket="bucket_to_copy_to", @@ -158,6 +158,7 @@ def test_copy_across_bucket_if_none_match(mock_service, mock_client): StorageClass="INTELLIGENT_TIERING", ) + def test_delete_object(mock_service, mock_client): mock_service.delete_object(s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME) @@ -530,30 +531,38 @@ def test_get_head_object_returns_metadata(mock_service, mock_client): result = mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) assert result == mock_response - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) -def test_get_head_object_raises_client_error_when_object_not_found(mock_service, mock_client): +def test_get_head_object_raises_client_error_when_object_not_found( + mock_service, mock_client +): mock_error = ClientError( - {"Error": {"Code": "404", "Message": "Not Found"}}, - "HeadObject" + {"Error": {"Code": "404", "Message": "Not Found"}}, "HeadObject" ) mock_client.head_object.side_effect = mock_error with pytest.raises(ClientError): mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) -def test_get_head_object_raises_client_error_on_access_denied(mock_service, mock_client): +def test_get_head_object_raises_client_error_on_access_denied( + mock_service, mock_client +): mock_error = ClientError( - {"Error": {"Code": "403", "Message": "Forbidden"}}, - "HeadObject" + {"Error": {"Code": "403", "Message": "Forbidden"}}, "HeadObject" ) mock_client.head_object.side_effect = mock_error with pytest.raises(ClientError): mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index c36f13f48..d8b146678 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -1,12 +1,9 @@ import pytest from botocore.exceptions import ClientError from enums.review_status import ReviewStatus -from models.document_review import DocumentsUploadReview +from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.document_review_processor_service import ( - ReviewProcessorService, -) -from models.document_review import DocumentReviewFileDetails +from services.document_review_processor_service import ReviewProcessorService @pytest.fixture @@ -36,7 +33,7 @@ def sample_review_message(): files=[ ReviewMessageFile( file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf" + file_path="staging/9000000009/test_document.pdf", ) ], nhs_number="9000000009", @@ -70,17 +67,13 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - mock_move = mocker.patch.object( - service_under_test, "_move_files_to_review_bucket" - ) - mock_delete = mocker.patch.object( - service_under_test, "_delete_files_from_staging" - ) + mock_move = mocker.patch.object(service_under_test, "_move_files_to_review_bucket") + mock_delete = mocker.patch.object(service_under_test, "_delete_files_from_staging") mock_move.return_value = [ DocumentReviewFileDetails( file_name="test_document.pdf", - file_location="9000000009/test-upload-id-123/test_document.pdf" + file_location="9000000009/test-upload-id-123/test_document.pdf", ) ] @@ -91,21 +84,19 @@ def test_process_review_message_success( mock_delete.assert_called_once_with(sample_review_message) -def test_process_review_message_multiple_files( - service_under_test, mocker -): +def test_process_review_message_multiple_files(service_under_test, mocker): """Test successful processing of a review message with multiple files.""" message = ReviewMessageBody( upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -113,23 +104,19 @@ def test_process_review_message_multiple_files( uploader_ods="Y12345", current_gp="Y12345", ) - - mock_move = mocker.patch.object( - service_under_test, "_move_files_to_review_bucket" - ) - mock_delete = mocker.patch.object( - service_under_test, "_delete_files_from_staging" - ) + + mock_move = mocker.patch.object(service_under_test, "_move_files_to_review_bucket") + mock_delete = mocker.patch.object(service_under_test, "_delete_files_from_staging") mock_move.return_value = [ DocumentReviewFileDetails( file_name="document_1.pdf", - file_location="9000000009/test-upload-id-456/document_1.pdf" + file_location="9000000009/test-upload-id-456/document_1.pdf", ), DocumentReviewFileDetails( file_name="document_2.pdf", - file_location="9000000009/test-upload-id-456/document_2.pdf" - ) + file_location="9000000009/test-upload-id-456/document_2.pdf", + ), ] service_under_test.process_review_message(message) @@ -139,7 +126,6 @@ def test_process_review_message_multiple_files( mock_delete.assert_called_once_with(message) - def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): @@ -161,7 +147,9 @@ def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): """Test processing fails when DynamoDB put fails.""" - mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) + mocker.patch.object( + service_under_test, "_move_files_to_review_bucket", return_value=[] + ) service_under_test.dynamo_service.put_item.side_effect = ClientError( {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, "PutItem", @@ -170,18 +158,19 @@ def test_process_review_message_dynamo_error( with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) + # Tests for _build_review_record and _create_review_record methods def test_build_review_record_success(service_under_test, sample_review_message): - """Test successful building of review record.""" + """Test successful building of review record.""" files = [ DocumentReviewFileDetails( file_name="test_document.pdf", - file_location="9000000009/test-review-id/test_document.pdf" + file_location="9000000009/test-review-id/test_document.pdf", ) ] - + result = service_under_test._build_review_record( sample_review_message, "test-review-id", files ) @@ -195,22 +184,24 @@ def test_build_review_record_success(service_under_test, sample_review_message): assert result.custodian == "Y12345" assert len(result.files) == 1 assert result.files[0].file_name == "test_document.pdf" - assert result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + assert ( + result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + ) def test_build_review_record_with_multiple_files(service_under_test): - """Test building review record with multiple files.""" + """Test building review record with multiple files.""" message = ReviewMessageBody( upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -218,18 +209,18 @@ def test_build_review_record_with_multiple_files(service_under_test): uploader_ods="Y12345", current_gp="Y12345", ) - + files = [ DocumentReviewFileDetails( file_name="document_1.pdf", - file_location="9000000009/test-review-id/document_1.pdf" + file_location="9000000009/test-review-id/document_1.pdf", ), DocumentReviewFileDetails( file_name="document_2.pdf", - file_location="9000000009/test-review-id/document_2.pdf" - ) + file_location="9000000009/test-review-id/document_2.pdf", + ), ] - + result = service_under_test._build_review_record(message, "test-review-id", files) assert len(result.files) == 2 @@ -247,12 +238,12 @@ def test_move_files_success(service_under_test, sample_review_message): ) expected_key = "9000000009/test-review-id-123/test_document.pdf" - + assert len(files) == 1 assert files[0].file_name == "test_document.pdf" assert files[0].file_location == expected_key - service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once_with( + service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", @@ -268,12 +259,12 @@ def test_move_multiple_files_success(service_under_test): files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -289,13 +280,13 @@ def test_move_multiple_files_success(service_under_test): assert files[0].file_location == "9000000009/test-review-id/document_1.pdf" assert files[1].file_name == "document_2.pdf" assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" - - assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 2 + + assert service_under_test.s3_service.copy_across_bucket.call_count == 2 def test_move_files_copy_error(service_under_test, sample_review_message): """Test file move handles S3 copy errors.""" - service_under_test.s3_service.copy_across_bucket_if_none_match.side_effect = ClientError( + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, "CopyObject", ) @@ -306,7 +297,6 @@ def test_move_files_copy_error(service_under_test, sample_review_message): ) - # Tests for _delete_files_from_staging method @@ -315,7 +305,8 @@ def test_delete_from_staging_success(service_under_test, sample_review_message): service_under_test._delete_files_from_staging(sample_review_message) service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/9000000009/test_document.pdf" + s3_bucket_name="test_staging_bulk_store", + file_key="staging/9000000009/test_document.pdf", ) @@ -335,25 +326,23 @@ def test_delete_from_staging_handles_errors(service_under_test, sample_review_me # Integration scenario tests -def test_full_workflow_with_valid_message( - service_under_test, sample_review_message -): +def test_full_workflow_with_valid_message(service_under_test, sample_review_message): """Test complete workflow from message to final record creation.""" service_under_test.dynamo_service.put_item.return_value = None - service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) service_under_test.dynamo_service.put_item.assert_called_once() - service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once() + service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" service_under_test.dynamo_service.put_item.return_value = None - service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None messages = [ @@ -362,7 +351,7 @@ def test_workflow_handles_multiple_different_patients(service_under_test): files=[ ReviewMessageFile( file_name=f"doc_{i}.pdf", - file_path=f"staging/900000000{i}/doc_{i}.pdf" + file_path=f"staging/900000000{i}/doc_{i}.pdf", ) ], nhs_number=f"900000000{i}", @@ -378,5 +367,4 @@ def test_workflow_handles_multiple_different_patients(service_under_test): service_under_test.process_review_message(message) assert service_under_test.dynamo_service.put_item.call_count == 3 - assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 3 - + assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From dc81e95dc54570d83c60743deec06909f062ee8c Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 17:46:09 +0000 Subject: [PATCH 14/47] code comment changes --- lambdas/services/base/dynamo_service.py | 33 ++-- .../unit/services/base/test_dynamo_service.py | 148 +++++++----------- 2 files changed, 67 insertions(+), 114 deletions(-) diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 9ad9aeada..f9ec26885 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -from types_boto3_dynamodb.type_defs import PutItemOutputTableTypeDef +from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -12,7 +12,6 @@ create_update_expression, ) from utils.exceptions import DynamoServiceException -from types_boto3_dynamodb import DynamoDBServiceResource logger = LoggingService(__name__) @@ -28,7 +27,9 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb: DynamoDBServiceResource = boto3.resource("dynamodb", region_name="eu-west-2") + self.dynamodb: DynamoDBServiceResource = boto3.resource( + "dynamodb", region_name="eu-west-2" + ) self.initialised = True def get_table(self, table_name: str): @@ -80,26 +81,13 @@ def query_table( logger.error(str(e), {"Result": f"Unable to query table: {table_name}"}) raise e - def create_item(self, table_name, item): - try: - table = self.get_table(table_name) - logger.info(f"Writing item to table: {table_name}") - table.put_item(Item=item) - except ClientError as e: - logger.error( - str(e), {"Result": f"Unable to write item to table: {table_name}"} - ) - raise e - - def put_item(self, table_name, item, key_name, check_exists: bool = True) -> PutItemOutputTableTypeDef: + def create_item(self, table_name, item, key_name: str | None = None): """ Put an item into the specified DynamoDB table with a condition on the existence of the key. Args: table_name: Name of the DynamoDB table item: The item to be inserted (as a dictionary) - key_name: The name of the key field - check_exists: If True, the item will only be inserted if the key does not already exist. - If False, the item will only be inserted if the key already exists. + key_name: The name of the key field to check existance for conditional put Returns: Response from the DynamoDB put_item operation Raises: @@ -108,11 +96,10 @@ def put_item(self, table_name, item, key_name, check_exists: bool = True) -> Put try: table = self.get_table(table_name) logger.info(f"Writing item to table: {table_name}") - return table.put_item(Item=item, Expected={ - key_name: { - 'Exists': check_exists - } - }) + if key_name: + return table.put_item(Item=item, Expected={key_name: {"Exists": True}}) + else: + return table.put_item(Item=item) except ClientError as e: logger.error( str(e), {"Result": f"Unable to write item to table: {table_name}"} diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index d7c6cc9b7..ad1443740 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -292,82 +292,25 @@ def test_create_item_raise_client_error(mock_service, mock_table): assert MOCK_CLIENT_ERROR == actual_response.value -def test_put_item_with_check_exists_true(mock_service, mock_table): +def test_create_item_with_key_name(mock_service, mock_table): item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} key_name = "NhsNumber" - - mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=True) + mock_service.create_item(MOCK_TABLE_NAME, item, key_name) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': True - } - } - ) - - -def test_put_item_with_check_exists_false(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - - mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=False) - - mock_table.assert_called_with(MOCK_TABLE_NAME) - mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': False - } - } - ) - - -def test_put_item_default_check_exists(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - - # Test that default value is True - mock_service.put_item(MOCK_TABLE_NAME, item, key_name) - - mock_table.assert_called_with(MOCK_TABLE_NAME) - mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': True - } - } + Item=item, Expected={key_name: {"Exists": True}} ) -def test_put_item_returns_response(mock_service, mock_table): +def test_create_item_raises_client_error(mock_service, mock_table): item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} key_name = "NhsNumber" - expected_response = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200 - } - } - - mock_table.return_value.put_item.return_value = expected_response - - actual_response = mock_service.put_item(MOCK_TABLE_NAME, item, key_name) - - assert actual_response == expected_response - -def test_put_item_raises_client_error(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - mock_table.return_value.put_item.side_effect = MOCK_CLIENT_ERROR with pytest.raises(ClientError) as actual_response: - mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + mock_service.create_item(MOCK_TABLE_NAME, item, key_name) assert MOCK_CLIENT_ERROR == actual_response.value @@ -728,7 +671,7 @@ def test_update_item_with_condition_expression(mock_service, mock_table): update_key = {"ID": "9000000009"} condition_expression = "attribute_exists(FileName)" expression_attribute_values = {":expected_val": "expected_value"} - + expected_update_expression = "SET #FileName_attr = :FileName_val" expected_expr_attr_names = {"#FileName_attr": "FileName"} expected_expr_attr_values = { @@ -739,7 +682,9 @@ def test_update_item_with_condition_expression(mock_service, mock_table): mock_service.update_item( table_name=MOCK_TABLE_NAME, key_pair={"ID": TEST_NHS_NUMBER}, - updated_fields={DocumentReferenceMetadataFields.FILE_NAME.value: "test-filename"}, + updated_fields={ + DocumentReferenceMetadataFields.FILE_NAME.value: "test-filename" + }, condition_expression=condition_expression, expression_attribute_values=expression_attribute_values, ) @@ -761,14 +706,16 @@ def test_batch_writing_is_called_with_correct_parameters(mock_service, mock_tabl {"ID": "id2", "Name": "Item 2"}, {"ID": "id3", "Name": "Item 3"}, ] - - mock_batch_writer = mock_table.return_value.batch_writer.return_value.__enter__.return_value - + + mock_batch_writer = ( + mock_table.return_value.batch_writer.return_value.__enter__.return_value + ) + mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.batch_writer.assert_called_once() - + assert mock_batch_writer.put_item.call_count == 3 for item in items_to_write: mock_batch_writer.put_item.assert_any_call(Item=item) @@ -777,8 +724,10 @@ def test_batch_writing_is_called_with_correct_parameters(mock_service, mock_tabl def test_batch_writing_client_error_raises_exception(mock_service, mock_table): items_to_write = [{"ID": "id1", "Name": "Item 1"}] expected_response = MOCK_CLIENT_ERROR - - mock_table.return_value.batch_writer.return_value.__enter__.side_effect = MOCK_CLIENT_ERROR + + mock_table.return_value.batch_writer.return_value.__enter__.side_effect = ( + MOCK_CLIENT_ERROR + ) with pytest.raises(ClientError) as actual_response: mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) @@ -788,9 +737,11 @@ def test_batch_writing_client_error_raises_exception(mock_service, mock_table): def test_batch_writing_with_empty_list(mock_service, mock_table): items_to_write = [] - - mock_batch_writer = mock_table.return_value.batch_writer.return_value.__enter__.return_value - + + mock_batch_writer = ( + mock_table.return_value.batch_writer.return_value.__enter__.return_value + ) + mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) mock_table.assert_called_with(MOCK_TABLE_NAME) @@ -816,7 +767,7 @@ def test_transact_write_items_success(mock_service, mock_dynamo_service): } }, ] - + mock_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} mock_dynamo_service.meta.client.transact_write_items.return_value = mock_response @@ -837,9 +788,12 @@ def test_transact_write_items_transaction_cancelled(mock_service, mock_dynamo_se } } ] - + error_response = { - "Error": {"Code": "TransactionCanceledException", "Message": "Transaction cancelled"}, + "Error": { + "Code": "TransactionCanceledException", + "Message": "Transaction cancelled", + }, "CancellationReasons": [{"Code": "ConditionalCheckFailed"}], } mock_dynamo_service.meta.client.transact_write_items.side_effect = ClientError( @@ -861,7 +815,7 @@ def test_transact_write_items_generic_client_error(mock_service, mock_dynamo_ser } } ] - + mock_dynamo_service.meta.client.transact_write_items.side_effect = MOCK_CLIENT_ERROR with pytest.raises(ClientError) as exc_info: @@ -882,19 +836,29 @@ def test_build_update_transaction_item_single_condition(mock_service): assert "Update" in result update_item = result["Update"] - + assert update_item["TableName"] == table_name assert update_item["Key"] == document_key - assert "SET #FileName_attr = :FileName_val, #Deleted_attr = :Deleted_val" == update_item["UpdateExpression"] - assert update_item["ConditionExpression"] == "#DocStatus_attr = :DocStatus_condition_val" - + assert ( + "SET #FileName_attr = :FileName_val, #Deleted_attr = :Deleted_val" + == update_item["UpdateExpression"] + ) + assert ( + update_item["ConditionExpression"] + == "#DocStatus_attr = :DocStatus_condition_val" + ) + assert update_item["ExpressionAttributeNames"]["#FileName_attr"] == "FileName" assert update_item["ExpressionAttributeNames"]["#Deleted_attr"] == "Deleted" assert update_item["ExpressionAttributeNames"]["#DocStatus_attr"] == "DocStatus" - - assert update_item["ExpressionAttributeValues"][":FileName_val"] == "new_filename.pdf" + + assert ( + update_item["ExpressionAttributeValues"][":FileName_val"] == "new_filename.pdf" + ) assert update_item["ExpressionAttributeValues"][":Deleted_val"] == "" - assert update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + assert ( + update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + ) def test_build_update_transaction_item_multiple_conditions(mock_service): @@ -909,26 +873,28 @@ def test_build_update_transaction_item_multiple_conditions(mock_service): assert "Update" in result update_item = result["Update"] - + assert update_item["TableName"] == table_name assert update_item["Key"] == document_key - + # Check that all conditions are present (order might vary) condition_expr = update_item["ConditionExpression"] assert "#DocStatus_attr = :DocStatus_condition_val" in condition_expr assert "#Version_attr = :Version_condition_val" in condition_expr assert "#Uploaded_attr = :Uploaded_condition_val" in condition_expr assert condition_expr.count(" AND ") == 2 - + # Check all attribute names are present assert update_item["ExpressionAttributeNames"]["#FileName_attr"] == "FileName" assert update_item["ExpressionAttributeNames"]["#DocStatus_attr"] == "DocStatus" assert update_item["ExpressionAttributeNames"]["#Version_attr"] == "Version" assert update_item["ExpressionAttributeNames"]["#Uploaded_attr"] == "Uploaded" - + # Check all attribute values are present assert update_item["ExpressionAttributeValues"][":FileName_val"] == "updated.pdf" - assert update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + assert ( + update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + ) assert update_item["ExpressionAttributeValues"][":Version_condition_val"] == 1 assert update_item["ExpressionAttributeValues"][":Uploaded_condition_val"] is True @@ -945,8 +911,8 @@ def test_build_update_transaction_item_empty_condition_fields(mock_service): assert "Update" in result update_item = result["Update"] - + # With empty condition_fields, condition expression should be empty string assert update_item["ConditionExpression"] == "" assert update_item["TableName"] == table_name - assert update_item["Key"] == document_key \ No newline at end of file + assert update_item["Key"] == document_key From 7e1585e21fc453161077166153db65044ccfcd5f Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 17:53:38 +0000 Subject: [PATCH 15/47] missed tests --- .../services/document_review_processor_service.py | 2 +- .../test_document_review_processor_service.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 8dfa0ad5b..91a785453 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -49,7 +49,7 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: document_upload_review = self._build_review_record( review_message, review_id, review_files ) - self.dynamo_service.put_item( + self.dynamo_service.create_item( table_name=self.review_table_name, item=document_upload_review.model_dump(by_alias=True, exclude_none=True), key_name=DocumentReferenceMetadataFields.ID.value, diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index d8b146678..c78fc8a2e 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -80,7 +80,7 @@ def test_process_review_message_success( service_under_test.process_review_message(sample_review_message) mock_move.assert_called_once() - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() mock_delete.assert_called_once_with(sample_review_message) @@ -122,7 +122,7 @@ def test_process_review_message_multiple_files(service_under_test, mocker): service_under_test.process_review_message(message) mock_move.assert_called_once() - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() mock_delete.assert_called_once_with(message) @@ -150,7 +150,7 @@ def test_process_review_message_dynamo_error( mocker.patch.object( service_under_test, "_move_files_to_review_bucket", return_value=[] ) - service_under_test.dynamo_service.put_item.side_effect = ClientError( + service_under_test.dynamo_service.create_item.side_effect = ClientError( {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, "PutItem", ) @@ -328,20 +328,20 @@ def test_delete_from_staging_handles_errors(service_under_test, sample_review_me def test_full_workflow_with_valid_message(service_under_test, sample_review_message): """Test complete workflow from message to final record creation.""" - service_under_test.dynamo_service.put_item.return_value = None + service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" - service_under_test.dynamo_service.put_item.return_value = None + service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None @@ -366,5 +366,5 @@ def test_workflow_handles_multiple_different_patients(service_under_test): for message in messages: service_under_test.process_review_message(message) - assert service_under_test.dynamo_service.put_item.call_count == 3 + assert service_under_test.dynamo_service.create_item.call_count == 3 assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 763aca5f3376c7c5dc265e5ebb4f700ea09eb5d1 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:08:52 +0000 Subject: [PATCH 16/47] [PRMP-585] bump boto3 version --- .../requirements/layers/requirements_core_lambda_layer.txt | 4 ++-- lambdas/services/base/dynamo_service.py | 4 ++-- lambdas/services/base/s3_service.py | 2 +- lambdas/tests/unit/services/base/test_dynamo_service.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lambdas/requirements/layers/requirements_core_lambda_layer.txt b/lambdas/requirements/layers/requirements_core_lambda_layer.txt index da90257d9..6728aaf74 100644 --- a/lambdas/requirements/layers/requirements_core_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_core_lambda_layer.txt @@ -1,7 +1,7 @@ PyJWT==2.8.0 PyYAML==6.0.1 -boto3==1.34.128 -botocore==1.34.128 +boto3==1.40.71 +botocore==1.40.71 charset-normalizer==3.2.0 cryptography==44.0.1 idna==3.7 diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index f9ec26885..055628f31 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -from types_boto3_dynamodb import DynamoDBServiceResource +# from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -97,7 +97,7 @@ def create_item(self, table_name, item, key_name: str | None = None): table = self.get_table(table_name) logger.info(f"Writing item to table: {table_name}") if key_name: - return table.put_item(Item=item, Expected={key_name: {"Exists": True}}) + return table.put_item(Item=item, ConditionExpression=f"attribute_not_exists({key_name})") else: return table.put_item(Item=item) except ClientError as e: diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index e45bca015..53250e2c8 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -7,7 +7,7 @@ from botocore.client import Config as BotoConfig from botocore.exceptions import ClientError from services.base.iam_service import IAMService -from types_boto3_s3 import S3Client +# from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index ad1443740..55541386e 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -299,7 +299,7 @@ def test_create_item_with_key_name(mock_service, mock_table): mock_service.create_item(MOCK_TABLE_NAME, item, key_name) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.put_item.assert_called_once_with( - Item=item, Expected={key_name: {"Exists": True}} + Item=item, ConditionExpression=f"attribute_not_exists({key_name})" ) From a934f81d96b42115dca936c75cc543e6a42e4aa3 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:32:27 +0000 Subject: [PATCH 17/47] [PRMP-585] add types --- lambdas/services/base/dynamo_service.py | 2 +- lambdas/services/base/s3_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 055628f31..5f68ca68f 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -# from types_boto3_dynamodb import DynamoDBServiceResource +from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 53250e2c8..e45bca015 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -7,7 +7,7 @@ from botocore.client import Config as BotoConfig from botocore.exceptions import ClientError from services.base.iam_service import IAMService -# from types_boto3_s3 import S3Client +from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException From b42582474f526909c14cf499ee1d8c794e805032 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:06:00 +0000 Subject: [PATCH 18/47] [PRMP-585] review processor handles if object already exists in review bucket --- .../document_review_processor_service.py | 25 +++++++++++++------ .../test_document_review_processor_service.py | 10 ++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 91a785453..0fd8d653d 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -1,6 +1,8 @@ import os from datetime import datetime, timezone +from botocore.exceptions import ClientError + from enums.review_status import ReviewStatus from models.document_reference import DocumentReferenceMetadataFields from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview @@ -89,6 +91,7 @@ def _move_files_to_review_bucket( List of DocumentReviewFileDetails with new file locations in review bucket """ new_file_keys: list[DocumentReviewFileDetails] = [] + for file in message_data.files: new_file_key = ( f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" @@ -97,14 +100,21 @@ def _move_files_to_review_bucket( logger.info( f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}" ) + try: - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=file.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - if_none_match="*", - ) + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + if_none_match="*", + ) + + except ClientError as e: + if e.response["Error"]["Code"] == "PreconditionFailed": + pass + else: + raise e new_file_keys.append( DocumentReviewFileDetails( @@ -115,6 +125,7 @@ def _move_files_to_review_bucket( logger.info("File successfully copied to review bucket") logger.info(f"Successfully moved file to: {new_file_key}") + return new_file_keys def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index c78fc8a2e..11fadf869 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -297,6 +297,16 @@ def test_move_files_copy_error(service_under_test, sample_review_message): ) +def test_move_files_to_review_bucket_continues_file_already_exists_in_review_bucket(service_under_test, sample_review_message): + + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + {"Error": {"Code": "PreconditionFailed", "Message": "At least one of the pre-conditions you specified did not hold"}}, + "CopyObject", + ) + + service_under_test.process_review_message(sample_review_message) + service_under_test.dynamo_service.create_item.assert_called() + # Tests for _delete_files_from_staging method From be312f8a21ba3576f99a93b9086dd6e79636e811 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Sun, 26 Oct 2025 15:09:27 +0000 Subject: [PATCH 19/47] [PRMP-585] Create ReviewProcessor lambda logic --- lambdas/enums/review_status.py | 9 + lambdas/handlers/review_processor_handler.py | 74 +++ lambdas/models/document_review.py | 49 ++ lambdas/models/sqs/review_message_body.py | 14 + lambdas/services/review_processor_service.py | 188 ++++++++ .../handlers/test_review_processor_handler.py | 324 +++++++++++++ .../services/test_review_processor_service.py | 453 ++++++++++++++++++ 7 files changed, 1111 insertions(+) create mode 100644 lambdas/enums/review_status.py create mode 100644 lambdas/handlers/review_processor_handler.py create mode 100644 lambdas/models/document_review.py create mode 100644 lambdas/models/sqs/review_message_body.py create mode 100644 lambdas/services/review_processor_service.py create mode 100644 lambdas/tests/unit/handlers/test_review_processor_handler.py create mode 100644 lambdas/tests/unit/services/test_review_processor_service.py diff --git a/lambdas/enums/review_status.py b/lambdas/enums/review_status.py new file mode 100644 index 000000000..64b51ce26 --- /dev/null +++ b/lambdas/enums/review_status.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class ReviewStatus(StrEnum): + """Status values for document review records.""" + + PENDING_REVIEW = "PENDING_REVIEW" + APPROVED = "APPROVED" + REJECTED = "REJECTED" diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py new file mode 100644 index 000000000..62edb6a93 --- /dev/null +++ b/lambdas/handlers/review_processor_handler.py @@ -0,0 +1,74 @@ +import json +from lambdas.models.sqs.review_message_body import ReviewMessageBody +# from services.review_processor_service import ReviewProcessorService // TODO +from lambdas.services.review_processor_service import ReviewProcessorService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.decorators.validate_sqs_message_event import validate_sqs_event +from utils.lambda_response import ApiGatewayResponse + +logger = LoggingService(__name__) + + +@set_request_context_for_logging +@override_error_check +@ensure_environment_variables( + names=[ + "DOCUMENT_REVIEW_DYNAMODB_NAME", + "STAGING_STORE_BUCKET_NAME", + "PENDING_REVIEW_BUCKET_NAME", + ] +) +@handle_lambda_exceptions +@validate_sqs_event +def lambda_handler(event, context): + """ + This handler consumes SQS messages from the document review queue, creates DynamoDB + records in the DocumentReview table, and moves files from the staging bucket + to the pending review bucket. + + Args: + event: Lambda event containing SQS Event + _context: Lambda context + + Returns: + ApiGatewayResponse with processing status + """ + logger.info("Starting review processor Lambda") + + sqs_messages = event.get("Records", []) + review_service = ReviewProcessorService() + + processed_count = 0 + failed_count = 0 + + for sqs_message in sqs_messages: + try: + sqs_message_body = json.loads(sqs_message["body"]) + review_message = ReviewMessageBody(**sqs_message_body) + + message = ReviewMessageBody.model_validate(review_message) + + review_service.process_review_message(message) + processed_count += 1 + except Exception as e: + logger.error( + f"Failed to process review message: {str(e)}", + {"Result": "Review processing failed"}, + ) + failed_count += 1 + + raise + + logger.info( + f"Review processor completed: {processed_count} processed, {failed_count} failed" + ) + + return ApiGatewayResponse( + status_code=200, + body=f"Processed {processed_count} messages", + methods="GET", + ).create_api_gateway_response() diff --git a/lambdas/models/document_review.py b/lambdas/models/document_review.py new file mode 100644 index 000000000..9b3eee106 --- /dev/null +++ b/lambdas/models/document_review.py @@ -0,0 +1,49 @@ +from typing import Optional +import uuid + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_pascal + +from lambdas.enums.review_status import ReviewStatus +from lambdas.enums.snomed_codes import SnomedCodes +from lambdas.models.document_reference import DocumentReferenceMetadataFields + + +class DocumentReviewFileDetails(BaseModel): + model_config = ConfigDict( + validate_by_alias=True, + validate_by_name=True, + alias_generator=to_pascal, + ) + + file_name: str + file_location: str + + +class DocumentsUploadReview(BaseModel): + model_config = ConfigDict( + validate_by_alias=True, + validate_by_name=True, + alias_generator=to_pascal, + use_enum_values=True, + ) + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + alias=str(DocumentReferenceMetadataFields.ID.value) + ) # id differse to nogas version + author: str + custodian: str + review_status: ReviewStatus = Field(default=ReviewStatus.PENDING_REVIEW) + review_reason: str + review_date: int | None = Field(default=None) + reviewer: str | None = Field(default=None) + upload_date: int + files: list[DocumentReviewFileDetails] = Field(default=[]) # differs to nogas version + nhs_number: str + ttl: Optional[int] = Field( + alias=str(DocumentReferenceMetadataFields.TTL.value), default=None + ) + document_reference_id: str | None = Field(default=None) + document_snomed_code_type: str = Field( + default=SnomedCodes.LLOYD_GEORGE.value.code + ) diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py new file mode 100644 index 000000000..c0cae7781 --- /dev/null +++ b/lambdas/models/sqs/review_message_body.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + + +class ReviewMessageBody(BaseModel): + """Model for SQS message body from the document review queue.""" + + file_name: str + file_path: str + """Location in the staging bucket""" + nhs_number: str + failure_reason: str + upload_date: str + uploader_ods: str + current_gp: str diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py new file mode 100644 index 000000000..8f5ab36d5 --- /dev/null +++ b/lambdas/services/review_processor_service.py @@ -0,0 +1,188 @@ +import os +from datetime import datetime, timezone + +from enums.review_status import ReviewStatus +from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview +from models.sqs.review_message_body import ReviewMessageBody +from services.base.dynamo_service import DynamoDBService +from services.base.s3_service import S3Service +from utils.audit_logging_setup import LoggingService +from utils.exceptions import S3FileNotFoundException +from utils.request_context import request_context + +logger = LoggingService(__name__) + + +class ReviewProcessorService: + """ + Service for processing single SQS messages from the document review queue. + """ + + def __init__(self): + """Initialize the review processor service with required AWS services.""" + self.dynamo_service = DynamoDBService() + self.s3_service = S3Service() + + self.review_table_name = os.environ["DOCUMENT_REVIEW_DYNAMODB_NAME"] + self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] + self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] + + def process_review_message(self, review_message: ReviewMessageBody) -> None: + """ + Process a single SQS message from the review queue. + + Args: + sqs_message: SQS message record containing file and failure information + + Raises: + InvalidMessageException: If message format is invalid or required fields missing + S3FileNotFoundException: If file doesn't exist in staging bucket + ClientError: For AWS service errors (DynamoDB, S3) + """ + logger.info("Processing review queue message") + + request_context.patient_nhs_no = review_message.nhs_number + + logger.info(f"Processing review for NHS: {review_message.nhs_number}, File: {review_message.file_name}") + + self._verify_file_exists_in_staging(review_message.file_path) + document_upload_review = self._create_review_record(review_message) + + new_file_key = self._move_file_to_review_bucket(review_message, document_upload_review.id) + self._update_review_record_with_file_location(document_upload_review.id, new_file_key) + + logger.info( + f"Successfully processed review for {review_message.nhs_number}", + {"Result": "Review record created and file moved"}, + ) + + def _verify_file_exists_in_staging(self, file_path: str) -> None: + """ + Verify the file exists in the staging bucket. + + Args: + file_path: S3 key of the file in staging bucket + + Raises: + S3FileNotFoundException: If file does not exist in staging bucket + """ + try: + file_exists = self.s3_service.file_exist_on_s3(s3_bucket_name=self.staging_bucket_name, file_key=file_path) + + if not file_exists: + raise S3FileNotFoundException(f"File not found in staging bucket: {file_path}") + + logger.info(f"Verified file exists in staging: {file_path}") + + except S3FileNotFoundException: + raise + except Exception as e: + logger.error(f"Error checking file in staging bucket: {str(e)}") + raise + + def _create_review_record(self, message_data: ReviewMessageBody) -> DocumentsUploadReview: + """ + Create a new review record in DynamoDB. + + Args: + message_data: Validated review queue message data + + Returns: + Created DocumentsUploadReview object + + Raises: + ClientError: If DynamoDB create operation fails + """ + try: + files = [DocumentReviewFileDetails( + file_name=message_data.file_name, + file_location=message_data.file_path + )] + + document_review = DocumentsUploadReview( + nhs_number=message_data.nhs_number, + upload_date=int(datetime.fromisoformat(message_data.upload_date).replace(tzinfo=timezone.utc).timestamp()), + review_status=ReviewStatus.PENDING_REVIEW, + review_reason=message_data.failure_reason, + author=message_data.uploader_ods, + custodian=message_data.current_gp, + files=files, + ) + + self.dynamo_service.create_item( + table_name=self.review_table_name, + item=document_review.model_dump(by_alias=True, exclude_none=True), + ) + + logger.info( + f"Created review record in DynamoDB with ID: {document_review.id}", + {"Result": "DynamoDB record created"}, + ) + + return document_review + + except Exception as e: + logger.error(f"Failed to create DynamoDB record: {str(e)}") + raise + + def _move_file_to_review_bucket(self, message_data: ReviewMessageBody, review_record_id: str) -> str: + """ + Move file from staging to review bucket. + + Args: + message_data: Review queue message data + review_record_id: ID of the review record (used in destination path) + + Returns: + New file key in review bucket + + Raises: + ClientError: If S3 copy or delete operations fail + """ + try: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{message_data.file_name}" + + logger.info(f"Copying file from ({message_data.file_path}) in staging to review bucket: {new_file_key}") + + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=message_data.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + ) + + logger.info("File successfully copied to review bucket") + logger.info(f"Deleting file from staging bucket: {message_data.file_path}") + + self._delete_from_staging(message_data.file_path) + logger.info(f"Successfully moved file to: {new_file_key}") + + return new_file_key + + except Exception as e: + logger.error(f"Failed to move file: {str(e)}") + raise + + def _delete_from_staging(self, file_key: str) -> None: + try: + self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) + + logger.info(f"Deleted file from staging bucket: {file_key}") + + except Exception as e: + logger.error(f"Error deleting file from staging: {str(e)}") + raise + + def _update_review_record_with_file_location(self, review_record_id: str, review_bucket_path: str) -> None: + try: + self.dynamo_service.update_item( + table_name=self.review_table_name, + key_pair={"ID": review_record_id}, + updated_fields={"ReviewBucketPath": review_bucket_path}, + ) + + logger.info(f"Updated review record {review_record_id} with file location: {review_bucket_path}") + + except Exception as e: + logger.error(f"Failed to update review record with file location: {str(e)}") + logger.warning("Review record created but file location not updated in DynamoDB") diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py new file mode 100644 index 000000000..e646eccb4 --- /dev/null +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -0,0 +1,324 @@ +import json + +import pytest +from handlers.review_processor_handler import lambda_handler +from models.sqs.review_message_body import ReviewMessageBody +from utils.lambda_response import ApiGatewayResponse + + +@pytest.fixture +def mock_review_service(mocker): + """Mock the ReviewProcessorService.""" + mocked_class = mocker.patch( + "handlers.review_processor_handler.ReviewProcessorService" + ) + mocked_instance = mocked_class.return_value + return mocked_instance + + + + + +@pytest.fixture +def sample_review_message_body(): + """Create a sample review message body.""" + return ReviewMessageBody( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + +@pytest.fixture +def sample_sqs_message(sample_review_message_body): + """Create a sample SQS message.""" + return { + "body": sample_review_message_body.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-1", + } + + +@pytest.fixture +def sample_sqs_event(sample_sqs_message): + """Create a sample SQS event with one message.""" + return {"Records": [sample_sqs_message]} + + +@pytest.fixture +def sample_sqs_event_multiple_messages(sample_review_message_body): + """Create a sample SQS event with multiple messages.""" + message_1 = ReviewMessageBody( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + message_2 = ReviewMessageBody( + file_name="document_2.pdf", + file_path="staging/9000000010/document_2.pdf", + nhs_number="9000000010", + failure_reason="Invalid file format", + upload_date="2024-01-15T10:35:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + message_3 = ReviewMessageBody( + file_name="document_3.pdf", + file_path="staging/9000000011/document_3.pdf", + nhs_number="9000000011", + failure_reason="Missing metadata", + upload_date="2024-01-15T10:40:00Z", + uploader_ods="Y67890", + current_gp="Y67890", + ) + + return { + "Records": [ + { + "body": message_1.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-1", + }, + { + "body": message_2.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-2", + }, + { + "body": message_3.model_dump_json(), + "eventSource": "aws:sqs", + "messageId": "test-message-id-3", + }, + ] + } + + +@pytest.fixture +def empty_sqs_event(): + """Create an empty SQS event.""" + return {"Records": []} + + +@pytest.fixture +def set_review_env(monkeypatch): + """Set up environment variables required for the handler.""" + monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") + monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") + monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") + + +def test_lambda_handler_processes_single_message_successfully( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler successfully processes a single SQS message.""" + expected_response = ApiGatewayResponse( + status_code=200, + body="Processed 1 messages", + methods="GET", + ).create_api_gateway_response() + + actual_response = lambda_handler(sample_sqs_event, context) + + assert actual_response == expected_response + mock_review_service.process_review_message.assert_called_once() + + +def test_lambda_handler_processes_multiple_messages_successfully( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler successfully processes multiple SQS messages.""" + expected_response = ApiGatewayResponse( + status_code=200, + body="Processed 3 messages", + methods="GET", + ).create_api_gateway_response() + + actual_response = lambda_handler(sample_sqs_event_multiple_messages, context) + + assert actual_response == expected_response + assert mock_review_service.process_review_message.call_count == 3 + + +def test_lambda_handler_calls_service_with_correct_message( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler calls service with the correctly parsed message.""" + lambda_handler(sample_sqs_event, context) + + # Verify the service was called once + mock_review_service.process_review_message.assert_called_once() + + # Get the actual call argument + call_args = mock_review_service.process_review_message.call_args[0][0] + + # Verify it's a ReviewMessageBody with correct data (check type name since isinstance fails with mock) + assert type(call_args).__name__ == "ReviewMessageBody" + assert call_args.file_name == "test_document.pdf" + assert call_args.nhs_number == "9000000009" + assert call_args.file_path == "staging/9000000009/test_document.pdf" + + +def test_lambda_handler_handles_empty_records_list( + set_review_env, context, empty_sqs_event, mock_review_service +): + """Test handler handles empty records list via @validate_sqs_event decorator.""" + # The @validate_sqs_event decorator returns 400 for empty records + actual_response = lambda_handler(empty_sqs_event, context) + + assert actual_response["statusCode"] == 400 + assert "SQS_4001" in actual_response["body"] # Error code for failed SQS parsing + mock_review_service.process_review_message.assert_not_called() + + +def test_lambda_handler_handles_service_error( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler catches exception when service fails (via @handle_lambda_exceptions decorator).""" + mock_review_service.process_review_message.side_effect = Exception( + "Service processing failed" + ) + + # The @handle_lambda_exceptions decorator catches exceptions + response = lambda_handler(sample_sqs_event, context) + + # Should return 500 error response + assert response["statusCode"] == 500 + assert "UE_500" in response["body"] # Unhandled exception error code + + +def test_lambda_handler_handles_invalid_message_format( + set_review_env, context, mock_review_service +): + """Test handler handles validation errors for invalid message format.""" + invalid_event = { + "Records": [ + { + "body": json.dumps( + { + "file_name": "test.pdf", + # Missing required fields + } + ), + "eventSource": "aws:sqs", + } + ] + } + + # The @handle_lambda_exceptions decorator catches the ValidationError + response = lambda_handler(invalid_event, context) + + assert response["statusCode"] == 500 + assert "UE_500" in response["body"] # Unhandled exception error code + + +def test_lambda_handler_handles_partial_failure( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler processes messages and handles first failure.""" + # First message succeeds, second fails + mock_review_service.process_review_message.side_effect = [ + None, # First message succeeds + Exception("Processing failed on second message"), # Second fails + ] + + # The @handle_lambda_exceptions decorator catches the exception + response = lambda_handler(sample_sqs_event_multiple_messages, context) + + # Should return error response + assert response["statusCode"] == 500 + + # Verify service was called twice before failing + assert mock_review_service.process_review_message.call_count == 2 + + +def test_lambda_handler_parses_json_body_correctly( + set_review_env, + context, + mock_review_service, +): + """Test handler correctly parses JSON from message body.""" + event = { + "Records": [ + { + "body": json.dumps( + { + "file_name": "test.pdf", + "file_path": "staging/test.pdf", + "nhs_number": "9000000009", + "failure_reason": "Test failure", + "upload_date": "2024-01-15T10:30:00Z", + "uploader_ods": "Y12345", + "current_gp": "Y12345", + } + ), + "eventSource": "aws:sqs", + } + ] + } + + lambda_handler(event, context) + + # Verify service was called with parsed ReviewMessageBody + mock_review_service.process_review_message.assert_called_once() + call_args = mock_review_service.process_review_message.call_args[0][0] + assert type(call_args).__name__ == "ReviewMessageBody" + assert call_args.file_name == "test.pdf" + + +def test_lambda_handler_logs_correct_counts( + set_review_env, + context, + sample_sqs_event_multiple_messages, + mock_review_service, +): + """Test handler response contains correct processed count.""" + response = lambda_handler(sample_sqs_event_multiple_messages, context) + + # Extract response body + response_body = response["body"] + + assert "Processed 3 messages" in response_body + + +def test_lambda_handler_tracks_failed_count( + set_review_env, + context, + sample_sqs_event, + mock_review_service, +): + """Test handler tracks failed message count.""" + mock_review_service.process_review_message.side_effect = Exception("Test error") + + # The @handle_lambda_exceptions decorator catches the exception + response = lambda_handler(sample_sqs_event, context) + + # Should return error + assert response["statusCode"] == 500 + + # Verify service was still called + mock_review_service.process_review_message.assert_called_once() diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py new file mode 100644 index 000000000..68a942365 --- /dev/null +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -0,0 +1,453 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest +from botocore.exceptions import ClientError +from enums.review_status import ReviewStatus +from models.document_review import DocumentsUploadReview +from models.sqs.review_message_body import ReviewMessageBody +from services.review_processor_service import ReviewProcessorService +from utils.exceptions import S3FileNotFoundException + + +@pytest.fixture +def mock_dynamo_service(mocker): + """Mock the DynamoDBService.""" + return mocker.patch("services.review_processor_service.DynamoDBService") + + +@pytest.fixture +def mock_s3_service(mocker): + """Mock the S3Service.""" + return mocker.patch("services.review_processor_service.S3Service") + + +@pytest.fixture +def set_review_env(monkeypatch): + """Set up environment variables required for the service.""" + monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") + monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") + monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") + + +@pytest.fixture +def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): + """Create a ReviewProcessorService instance with mocked dependencies.""" + service = ReviewProcessorService() + return service + + +@pytest.fixture +def sample_review_message(): + """Create a sample review message.""" + return ReviewMessageBody( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf", + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + +# Test service initialization + + +def test_service_initializes_with_correct_environment_variables( + set_review_env, mock_dynamo_service, mock_s3_service +): + """Test service initializes correctly with environment variables.""" + service = ReviewProcessorService() + + assert service.review_table_name == "test_review_table" + assert service.staging_bucket_name == "test_staging_bucket" + assert service.review_bucket_name == "test_review_bucket" + mock_dynamo_service.assert_called_once() + mock_s3_service.assert_called_once() + + +# Tests for process_review_message method + + +def test_process_review_message_success( + service_under_test, sample_review_message, mocker +): + """Test successful processing of a review message.""" + # Mock all the internal methods + mock_verify = mocker.patch.object( + service_under_test, "_verify_file_exists_in_staging" + ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_move = mocker.patch.object( + service_under_test, "_move_file_to_review_bucket" + ) + mock_update = mocker.patch.object( + service_under_test, "_update_review_record_with_file_location" + ) + + # Configure return values + mock_review = MagicMock() + mock_review.id = "test-review-id" + mock_create.return_value = mock_review + mock_move.return_value = "9000000009/test-review-id/test_document.pdf" + + # Execute + service_under_test.process_review_message(sample_review_message) + + # Verify calls + mock_verify.assert_called_once_with(sample_review_message.file_path) + mock_create.assert_called_once_with(sample_review_message) + mock_move.assert_called_once_with(sample_review_message, "test-review-id") + mock_update.assert_called_once_with( + "test-review-id", "9000000009/test-review-id/test_document.pdf" + ) + + +def test_process_review_message_file_not_found( + service_under_test, sample_review_message, mocker +): + """Test processing fails when file doesn't exist in staging.""" + mocker.patch.object( + service_under_test, + "_verify_file_exists_in_staging", + side_effect=S3FileNotFoundException("File not found"), + ) + + with pytest.raises(S3FileNotFoundException, match="File not found"): + service_under_test.process_review_message(sample_review_message) + + +def test_process_review_message_dynamo_error( + service_under_test, sample_review_message, mocker +): + """Test processing fails when DynamoDB create fails.""" + mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + mocker.patch.object( + service_under_test, + "_create_review_record", + side_effect=ClientError( + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", + ), + ) + + with pytest.raises(ClientError): + service_under_test.process_review_message(sample_review_message) + + +def test_process_review_message_s3_copy_error( + service_under_test, sample_review_message, mocker +): + """Test processing fails when S3 copy operation fails.""" + mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + mock_review = MagicMock() + mock_review.id = "test-review-id" + mocker.patch.object( + service_under_test, "_create_review_record", return_value=mock_review + ) + mocker.patch.object( + service_under_test, + "_move_file_to_review_bucket", + side_effect=ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, + "CopyObject", + ), + ) + + with pytest.raises(ClientError): + service_under_test.process_review_message(sample_review_message) + + +# Tests for _verify_file_exists_in_staging method + + +def test_verify_file_exists_success(service_under_test): + """Test successful file verification.""" + service_under_test.s3_service.file_exist_on_s3.return_value = True + + # Should not raise exception + service_under_test._verify_file_exists_in_staging("staging/test.pdf") + + service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( + s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + ) + + +def test_verify_file_does_not_exist(service_under_test): + """Test file verification fails when file doesn't exist.""" + service_under_test.s3_service.file_exist_on_s3.return_value = False + + with pytest.raises( + S3FileNotFoundException, match="File not found in staging bucket" + ): + service_under_test._verify_file_exists_in_staging("staging/missing.pdf") + + +def test_verify_file_s3_error(service_under_test): + """Test file verification handles S3 errors.""" + service_under_test.s3_service.file_exist_on_s3.side_effect = ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "HeadObject", + ) + + with pytest.raises(ClientError): + service_under_test._verify_file_exists_in_staging("staging/test.pdf") + + +# Tests for _create_review_record method + + +def test_create_review_record_success( + service_under_test, sample_review_message, mocker +): + """Test successful creation of review record.""" + # Mock the DynamoDB create_item to succeed + service_under_test.dynamo_service.create_item.return_value = None + + result = service_under_test._create_review_record(sample_review_message) + + # Verify the result + assert isinstance(result, DocumentsUploadReview) + assert result.nhs_number == "9000000009" + assert result.review_status == ReviewStatus.PENDING_REVIEW + assert result.review_reason == "Failed virus scan" + assert result.author == "Y12345" + assert result.custodian == "Y12345" + assert len(result.files) == 1 + assert result.files[0].file_name == "test_document.pdf" + assert result.files[0].file_location == "staging/9000000009/test_document.pdf" + + # Verify upload_date is converted to timestamp + expected_timestamp = int( + datetime.fromisoformat("2024-01-15T10:30:00Z") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + assert result.upload_date == expected_timestamp + + # Verify DynamoDB was called + service_under_test.dynamo_service.create_item.assert_called_once() + call_args = service_under_test.dynamo_service.create_item.call_args + assert call_args[1]["table_name"] == "test_review_table" + + +def test_create_review_record_with_different_data(service_under_test, mocker): + """Test creation with different message data.""" + message = ReviewMessageBody( + file_name="another_doc.pdf", + file_path="staging/9000000010/another_doc.pdf", + nhs_number="9000000010", + failure_reason="Missing metadata", + upload_date="2024-02-20T14:45:30Z", + uploader_ods="Y67890", + current_gp="Y67890", + ) + + service_under_test.dynamo_service.create_item.return_value = None + + result = service_under_test._create_review_record(message) + + assert result.nhs_number == "9000000010" + assert result.review_reason == "Missing metadata" + assert result.author == "Y67890" + assert result.custodian == "Y67890" + + +def test_create_review_record_dynamo_error( + service_under_test, sample_review_message +): + """Test create record handles DynamoDB errors.""" + service_under_test.dynamo_service.create_item.side_effect = ClientError( + {"Error": {"Code": "ValidationException", "Message": "Invalid item"}}, + "PutItem", + ) + + with pytest.raises(ClientError): + service_under_test._create_review_record(sample_review_message) + + +# Tests for _move_file_to_review_bucket method + + +def test_move_file_success(service_under_test, sample_review_message, mocker): + """Test successful file move from staging to review bucket.""" + mocker.patch.object(service_under_test, "_delete_from_staging") + + new_file_key = service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id-123" + ) + + # Verify new file key format + expected_key = "9000000009/test-review-id-123/test_document.pdf" + assert new_file_key == expected_key + + # Verify S3 copy was called + service_under_test.s3_service.copy_across_bucket.assert_called_once_with( + source_bucket="test_staging_bucket", + source_file_key="staging/9000000009/test_document.pdf", + dest_bucket="test_review_bucket", + dest_file_key=expected_key, + ) + + # Verify delete was called + service_under_test._delete_from_staging.assert_called_once_with( + "staging/9000000009/test_document.pdf" + ) + + +def test_move_file_copy_error(service_under_test, sample_review_message, mocker): + """Test file move handles S3 copy errors.""" + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, + "CopyObject", + ) + + with pytest.raises(ClientError): + service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id" + ) + + +def test_move_file_delete_error(service_under_test, sample_review_message, mocker): + """Test file move handles delete errors.""" + mocker.patch.object( + service_under_test, + "_delete_from_staging", + side_effect=ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "DeleteObject", + ), + ) + + with pytest.raises(ClientError): + service_under_test._move_file_to_review_bucket( + sample_review_message, "test-review-id" + ) + + +# Tests for _delete_from_staging method + + +def test_delete_from_staging_success(service_under_test): + """Test successful deletion from staging bucket.""" + service_under_test._delete_from_staging("staging/test.pdf") + + service_under_test.s3_service.delete_object.assert_called_once_with( + s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + ) + + +def test_delete_from_staging_error(service_under_test): + """Test delete from staging handles S3 errors.""" + service_under_test.s3_service.delete_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Key does not exist"}}, + "DeleteObject", + ) + + with pytest.raises(ClientError): + service_under_test._delete_from_staging("staging/test.pdf") + + +# Tests for _update_review_record_with_file_location method + + +def test_update_review_record_success(service_under_test): + """Test successful update of review record with file location.""" + service_under_test._update_review_record_with_file_location( + "test-review-id", "9000000009/test-review-id/test.pdf" + ) + + service_under_test.dynamo_service.update_item.assert_called_once_with( + table_name="test_review_table", + key_pair={"ID": "test-review-id"}, + updated_fields={"ReviewBucketPath": "9000000009/test-review-id/test.pdf"}, + ) + + +def test_update_review_record_dynamo_error(service_under_test, mocker): + """Test update review record handles DynamoDB errors gracefully.""" + service_under_test.dynamo_service.update_item.side_effect = ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "Failed"}}, + "UpdateItem", + ) + + # Mock logger to verify warning is logged + mock_logger = mocker.patch("services.review_processor_service.logger") + + # Should not raise - method logs error and warning instead + service_under_test._update_review_record_with_file_location( + "test-review-id", "path/to/file.pdf" + ) + + # Verify error and warning were logged + assert mock_logger.error.called + assert mock_logger.warning.called + + +# Integration scenario tests + + +def test_full_workflow_with_valid_message( + service_under_test, sample_review_message, mocker +): + """Test complete workflow from message to final update.""" + # Set up mocks + service_under_test.s3_service.file_exist_on_s3.return_value = True + service_under_test.dynamo_service.create_item.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.s3_service.delete_object.return_value = None + service_under_test.dynamo_service.update_item.return_value = None + + # Execute full workflow + service_under_test.process_review_message(sample_review_message) + + # Verify all steps were executed + service_under_test.s3_service.file_exist_on_s3.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() + service_under_test.s3_service.copy_across_bucket.assert_called_once() + service_under_test.s3_service.delete_object.assert_called_once() + service_under_test.dynamo_service.update_item.assert_called_once() + + +def test_workflow_stops_at_verification_failure( + service_under_test, sample_review_message +): + """Test workflow stops if file verification fails.""" + service_under_test.s3_service.file_exist_on_s3.return_value = False + + with pytest.raises(S3FileNotFoundException): + service_under_test.process_review_message(sample_review_message) + + # Verify subsequent operations were not called + service_under_test.dynamo_service.create_item.assert_not_called() + service_under_test.s3_service.copy_across_bucket.assert_not_called() + + +def test_workflow_handles_multiple_different_patients(service_under_test, mocker): + """Test processing messages for different patients.""" + service_under_test.s3_service.file_exist_on_s3.return_value = True + service_under_test.dynamo_service.create_item.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.s3_service.delete_object.return_value = None + service_under_test.dynamo_service.update_item.return_value = None + + messages = [ + ReviewMessageBody( + file_name=f"doc_{i}.pdf", + file_path=f"staging/900000000{i}/doc_{i}.pdf", + nhs_number=f"900000000{i}", + failure_reason="Test failure", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + for i in range(1, 4) + ] + + for message in messages: + service_under_test.process_review_message(message) + + # Verify service was called for each message + assert service_under_test.dynamo_service.create_item.call_count == 3 + assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 027c96cf0d50e41e2d24660a0f3097ef2fff868d Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Sun, 26 Oct 2025 15:35:27 +0000 Subject: [PATCH 20/47] [PRMP-585] minor fix --- .../handlers/test_review_processor_handler.py | 29 ++++--------------- .../services/test_review_processor_service.py | 20 ------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index e646eccb4..86b9b2548 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -164,13 +164,10 @@ def test_lambda_handler_calls_service_with_correct_message( """Test handler calls service with the correctly parsed message.""" lambda_handler(sample_sqs_event, context) - # Verify the service was called once mock_review_service.process_review_message.assert_called_once() - # Get the actual call argument call_args = mock_review_service.process_review_message.call_args[0][0] - # Verify it's a ReviewMessageBody with correct data (check type name since isinstance fails with mock) assert type(call_args).__name__ == "ReviewMessageBody" assert call_args.file_name == "test_document.pdf" assert call_args.nhs_number == "9000000009" @@ -181,11 +178,10 @@ def test_lambda_handler_handles_empty_records_list( set_review_env, context, empty_sqs_event, mock_review_service ): """Test handler handles empty records list via @validate_sqs_event decorator.""" - # The @validate_sqs_event decorator returns 400 for empty records actual_response = lambda_handler(empty_sqs_event, context) assert actual_response["statusCode"] == 400 - assert "SQS_4001" in actual_response["body"] # Error code for failed SQS parsing + assert "SQS_4001" in actual_response["body"] mock_review_service.process_review_message.assert_not_called() @@ -200,12 +196,10 @@ def test_lambda_handler_handles_service_error( "Service processing failed" ) - # The @handle_lambda_exceptions decorator catches exceptions response = lambda_handler(sample_sqs_event, context) - # Should return 500 error response assert response["statusCode"] == 500 - assert "UE_500" in response["body"] # Unhandled exception error code + assert "UE_500" in response["body"] def test_lambda_handler_handles_invalid_message_format( @@ -218,7 +212,6 @@ def test_lambda_handler_handles_invalid_message_format( "body": json.dumps( { "file_name": "test.pdf", - # Missing required fields } ), "eventSource": "aws:sqs", @@ -226,11 +219,10 @@ def test_lambda_handler_handles_invalid_message_format( ] } - # The @handle_lambda_exceptions decorator catches the ValidationError response = lambda_handler(invalid_event, context) assert response["statusCode"] == 500 - assert "UE_500" in response["body"] # Unhandled exception error code + assert "UE_500" in response["body"] def test_lambda_handler_handles_partial_failure( @@ -240,19 +232,14 @@ def test_lambda_handler_handles_partial_failure( mock_review_service, ): """Test handler processes messages and handles first failure.""" - # First message succeeds, second fails mock_review_service.process_review_message.side_effect = [ - None, # First message succeeds - Exception("Processing failed on second message"), # Second fails + None, + Exception("Processing failed on second message"), ] - # The @handle_lambda_exceptions decorator catches the exception response = lambda_handler(sample_sqs_event_multiple_messages, context) - # Should return error response assert response["statusCode"] == 500 - - # Verify service was called twice before failing assert mock_review_service.process_review_message.call_count == 2 @@ -283,7 +270,6 @@ def test_lambda_handler_parses_json_body_correctly( lambda_handler(event, context) - # Verify service was called with parsed ReviewMessageBody mock_review_service.process_review_message.assert_called_once() call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" @@ -299,7 +285,6 @@ def test_lambda_handler_logs_correct_counts( """Test handler response contains correct processed count.""" response = lambda_handler(sample_sqs_event_multiple_messages, context) - # Extract response body response_body = response["body"] assert "Processed 3 messages" in response_body @@ -314,11 +299,7 @@ def test_lambda_handler_tracks_failed_count( """Test handler tracks failed message count.""" mock_review_service.process_review_message.side_effect = Exception("Test error") - # The @handle_lambda_exceptions decorator catches the exception response = lambda_handler(sample_sqs_event, context) - # Should return error assert response["statusCode"] == 500 - - # Verify service was still called mock_review_service.process_review_message.assert_called_once() diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 68a942365..8907011b9 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -74,7 +74,6 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - # Mock all the internal methods mock_verify = mocker.patch.object( service_under_test, "_verify_file_exists_in_staging" ) @@ -86,16 +85,13 @@ def test_process_review_message_success( service_under_test, "_update_review_record_with_file_location" ) - # Configure return values mock_review = MagicMock() mock_review.id = "test-review-id" mock_create.return_value = mock_review mock_move.return_value = "9000000009/test-review-id/test_document.pdf" - # Execute service_under_test.process_review_message(sample_review_message) - # Verify calls mock_verify.assert_called_once_with(sample_review_message.file_path) mock_create.assert_called_once_with(sample_review_message) mock_move.assert_called_once_with(sample_review_message, "test-review-id") @@ -166,7 +162,6 @@ def test_verify_file_exists_success(service_under_test): """Test successful file verification.""" service_under_test.s3_service.file_exist_on_s3.return_value = True - # Should not raise exception service_under_test._verify_file_exists_in_staging("staging/test.pdf") service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( @@ -202,12 +197,10 @@ def test_create_review_record_success( service_under_test, sample_review_message, mocker ): """Test successful creation of review record.""" - # Mock the DynamoDB create_item to succeed service_under_test.dynamo_service.create_item.return_value = None result = service_under_test._create_review_record(sample_review_message) - # Verify the result assert isinstance(result, DocumentsUploadReview) assert result.nhs_number == "9000000009" assert result.review_status == ReviewStatus.PENDING_REVIEW @@ -218,7 +211,6 @@ def test_create_review_record_success( assert result.files[0].file_name == "test_document.pdf" assert result.files[0].file_location == "staging/9000000009/test_document.pdf" - # Verify upload_date is converted to timestamp expected_timestamp = int( datetime.fromisoformat("2024-01-15T10:30:00Z") .replace(tzinfo=timezone.utc) @@ -226,7 +218,6 @@ def test_create_review_record_success( ) assert result.upload_date == expected_timestamp - # Verify DynamoDB was called service_under_test.dynamo_service.create_item.assert_called_once() call_args = service_under_test.dynamo_service.create_item.call_args assert call_args[1]["table_name"] == "test_review_table" @@ -278,11 +269,9 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): sample_review_message, "test-review-id-123" ) - # Verify new file key format expected_key = "9000000009/test-review-id-123/test_document.pdf" assert new_file_key == expected_key - # Verify S3 copy was called service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bucket", source_file_key="staging/9000000009/test_document.pdf", @@ -290,7 +279,6 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): dest_file_key=expected_key, ) - # Verify delete was called service_under_test._delete_from_staging.assert_called_once_with( "staging/9000000009/test_document.pdf" ) @@ -372,15 +360,12 @@ def test_update_review_record_dynamo_error(service_under_test, mocker): "UpdateItem", ) - # Mock logger to verify warning is logged mock_logger = mocker.patch("services.review_processor_service.logger") - # Should not raise - method logs error and warning instead service_under_test._update_review_record_with_file_location( "test-review-id", "path/to/file.pdf" ) - # Verify error and warning were logged assert mock_logger.error.called assert mock_logger.warning.called @@ -392,17 +377,14 @@ def test_full_workflow_with_valid_message( service_under_test, sample_review_message, mocker ): """Test complete workflow from message to final update.""" - # Set up mocks service_under_test.s3_service.file_exist_on_s3.return_value = True service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.dynamo_service.update_item.return_value = None - # Execute full workflow service_under_test.process_review_message(sample_review_message) - # Verify all steps were executed service_under_test.s3_service.file_exist_on_s3.assert_called_once() service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() @@ -419,7 +401,6 @@ def test_workflow_stops_at_verification_failure( with pytest.raises(S3FileNotFoundException): service_under_test.process_review_message(sample_review_message) - # Verify subsequent operations were not called service_under_test.dynamo_service.create_item.assert_not_called() service_under_test.s3_service.copy_across_bucket.assert_not_called() @@ -448,6 +429,5 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker for message in messages: service_under_test.process_review_message(message) - # Verify service was called for each message assert service_under_test.dynamo_service.create_item.call_count == 3 assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 6971e63ea564b3e45ee0b7bec3617ffd387a3d04 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 27 Oct 2025 14:44:30 +0000 Subject: [PATCH 21/47] [PRMP-585] minor fixes --- lambdas/handlers/review_processor_handler.py | 7 +------ lambdas/services/review_processor_service.py | 1 + .../tests/unit/handlers/test_review_processor_handler.py | 3 --- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index 62edb6a93..eb3194601 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,6 +1,5 @@ import json from lambdas.models.sqs.review_message_body import ReviewMessageBody -# from services.review_processor_service import ReviewProcessorService // TODO from lambdas.services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -48,9 +47,7 @@ def lambda_handler(event, context): for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) - review_message = ReviewMessageBody(**sqs_message_body) - - message = ReviewMessageBody.model_validate(review_message) + message = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) processed_count += 1 @@ -61,8 +58,6 @@ def lambda_handler(event, context): ) failed_count += 1 - raise - logger.info( f"Review processor completed: {processed_count} processed, {failed_count} failed" ) diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 8f5ab36d5..456d595bf 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -186,3 +186,4 @@ def _update_review_record_with_file_location(self, review_record_id: str, review except Exception as e: logger.error(f"Failed to update review record with file location: {str(e)}") logger.warning("Review record created but file location not updated in DynamoDB") + raise diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index 86b9b2548..cf9b8d98f 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -16,9 +16,6 @@ def mock_review_service(mocker): return mocked_instance - - - @pytest.fixture def sample_review_message_body(): """Create a sample review message body.""" From dfb0b43459ecd4cc3719fc4018fd7f3bfcb4179a Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Tue, 28 Oct 2025 11:10:40 +0000 Subject: [PATCH 22/47] [PRMP-585] added support for multiple files in sqs message --- lambdas/handlers/review_processor_handler.py | 24 +- lambdas/models/sqs/review_message_body.py | 9 +- lambdas/services/review_processor_service.py | 128 +++---- .../handlers/test_review_processor_handler.py | 125 ++++--- .../services/test_review_processor_service.py | 325 +++++++++++------- 5 files changed, 338 insertions(+), 273 deletions(-) diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index eb3194601..f92dc8eab 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,19 +1,17 @@ import json + +from pydantic import ValidationError from lambdas.models.sqs.review_message_body import ReviewMessageBody from lambdas.services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables -from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging -from utils.decorators.validate_sqs_message_event import validate_sqs_event -from utils.lambda_response import ApiGatewayResponse logger = LoggingService(__name__) @set_request_context_for_logging -@override_error_check @ensure_environment_variables( names=[ "DOCUMENT_REVIEW_DYNAMODB_NAME", @@ -21,8 +19,7 @@ "PENDING_REVIEW_BUCKET_NAME", ] ) -@handle_lambda_exceptions -@validate_sqs_event +@override_error_check def lambda_handler(event, context): """ This handler consumes SQS messages from the document review queue, creates DynamoDB @@ -34,7 +31,7 @@ def lambda_handler(event, context): _context: Lambda context Returns: - ApiGatewayResponse with processing status + None """ logger.info("Starting review processor Lambda") @@ -47,23 +44,22 @@ def lambda_handler(event, context): for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) - message = ReviewMessageBody.model_validate(sqs_message_body) + message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) processed_count += 1 + except ValidationError as error: + logger.error("Malformed review message") + logger.error(error) + except Exception as e: logger.error( f"Failed to process review message: {str(e)}", {"Result": "Review processing failed"}, ) failed_count += 1 + logger.info("Continuing to next message.") logger.info( f"Review processor completed: {processed_count} processed, {failed_count} failed" ) - - return ApiGatewayResponse( - status_code=200, - body=f"Processed {processed_count} messages", - methods="GET", - ).create_api_gateway_response() diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index c0cae7781..a34c84214 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -1,12 +1,17 @@ from pydantic import BaseModel -class ReviewMessageBody(BaseModel): - """Model for SQS message body from the document review queue.""" +class ReviewMessageFile(BaseModel): + """Model for individual file in SQS message body from the document review queue.""" file_name: str file_path: str """Location in the staging bucket""" + +class ReviewMessageBody(BaseModel): + """Model for SQS message body from the document review queue.""" + + files: list[ReviewMessageFile] nhs_number: str failure_reason: str upload_date: str diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 456d595bf..0e9416ac8 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -1,5 +1,6 @@ import os from datetime import datetime, timezone +import uuid from enums.review_status import ReviewStatus from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview @@ -43,13 +44,17 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: request_context.patient_nhs_no = review_message.nhs_number - logger.info(f"Processing review for NHS: {review_message.nhs_number}, File: {review_message.file_name}") + logger.info(f"Processing review for NHS: {review_message.nhs_number} with {len(review_message.files)} files") - self._verify_file_exists_in_staging(review_message.file_path) - document_upload_review = self._create_review_record(review_message) + for file in review_message.files: + logger.info(f"Processing review file: {file.file_name}") + self._verify_file_exists_in_staging(file.file_path) - new_file_key = self._move_file_to_review_bucket(review_message, document_upload_review.id) - self._update_review_record_with_file_location(document_upload_review.id, new_file_key) + review_id = uuid.uuid4().hex + files = self._move_files_to_review_bucket(review_message, review_id) + document_upload_review = self._build_review_record(review_message, review_id, files) + + self._create_review_record(document_upload_review) logger.info( f"Successfully processed review for {review_message.nhs_number}", @@ -80,89 +85,64 @@ def _verify_file_exists_in_staging(self, file_path: str) -> None: logger.error(f"Error checking file in staging bucket: {str(e)}") raise - def _create_review_record(self, message_data: ReviewMessageBody) -> DocumentsUploadReview: - """ - Create a new review record in DynamoDB. - - Args: - message_data: Validated review queue message data - - Returns: - Created DocumentsUploadReview object - - Raises: - ClientError: If DynamoDB create operation fails - """ - try: - files = [DocumentReviewFileDetails( - file_name=message_data.file_name, - file_location=message_data.file_path - )] - - document_review = DocumentsUploadReview( - nhs_number=message_data.nhs_number, - upload_date=int(datetime.fromisoformat(message_data.upload_date).replace(tzinfo=timezone.utc).timestamp()), - review_status=ReviewStatus.PENDING_REVIEW, - review_reason=message_data.failure_reason, - author=message_data.uploader_ods, - custodian=message_data.current_gp, - files=files, - ) - - self.dynamo_service.create_item( - table_name=self.review_table_name, - item=document_review.model_dump(by_alias=True, exclude_none=True), - ) - - logger.info( - f"Created review record in DynamoDB with ID: {document_review.id}", - {"Result": "DynamoDB record created"}, - ) - - return document_review - - except Exception as e: - logger.error(f"Failed to create DynamoDB record: {str(e)}") - raise + def _build_review_record( + self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] + ) -> DocumentsUploadReview: + return DocumentsUploadReview( + id=review_id, + nhs_number=message_data.nhs_number, + review_status=ReviewStatus.PENDING_REVIEW, + review_reason=message_data.failure_reason, + author=message_data.uploader_ods, + custodian=message_data.current_gp, + files=files, + upload_date=int(datetime.now(tz=timezone.utc).timestamp()) + ) - def _move_file_to_review_bucket(self, message_data: ReviewMessageBody, review_record_id: str) -> str: + def _move_files_to_review_bucket( + self, message_data: ReviewMessageBody, review_record_id: str + ) -> list[DocumentReviewFileDetails]: """ Move file from staging to review bucket. Args: message_data: Review queue message data - review_record_id: ID of the review record (used in destination path) + review_record_id: ID of the review record being created Returns: - New file key in review bucket - - Raises: - ClientError: If S3 copy or delete operations fail + List of DocumentReviewFileDetails objects for the moved files """ + moved_files = [] try: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{message_data.file_name}" + for file in message_data.files: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" - logger.info(f"Copying file from ({message_data.file_path}) in staging to review bucket: {new_file_key}") + logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=message_data.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - ) + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + ) - logger.info("File successfully copied to review bucket") - logger.info(f"Deleting file from staging bucket: {message_data.file_path}") + logger.info("File successfully copied to review bucket") + logger.info(f"Deleting file from staging bucket: {file.file_path}") - self._delete_from_staging(message_data.file_path) - logger.info(f"Successfully moved file to: {new_file_key}") + self._delete_from_staging(file.file_path) + logger.info(f"Successfully moved file to: {new_file_key}") - return new_file_key + moved_files.append(DocumentReviewFileDetails( + file_name=file.file_name, + file_location=new_file_key + )) except Exception as e: logger.error(f"Failed to move file: {str(e)}") raise + return moved_files + def _delete_from_staging(self, file_key: str) -> None: try: self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) @@ -173,17 +153,15 @@ def _delete_from_staging(self, file_key: str) -> None: logger.error(f"Error deleting file from staging: {str(e)}") raise - def _update_review_record_with_file_location(self, review_record_id: str, review_bucket_path: str) -> None: + def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: - self.dynamo_service.update_item( + self.dynamo_service.create_item( table_name=self.review_table_name, - key_pair={"ID": review_record_id}, - updated_fields={"ReviewBucketPath": review_bucket_path}, + item=review_record ) - logger.info(f"Updated review record {review_record_id} with file location: {review_bucket_path}") + logger.info(f"Created review record {review_record.id}") except Exception as e: - logger.error(f"Failed to update review record with file location: {str(e)}") - logger.warning("Review record created but file location not updated in DynamoDB") + logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") raise diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index cf9b8d98f..85be717a4 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -2,8 +2,7 @@ import pytest from handlers.review_processor_handler import lambda_handler -from models.sqs.review_message_body import ReviewMessageBody -from utils.lambda_response import ApiGatewayResponse +from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile @pytest.fixture @@ -20,8 +19,12 @@ def mock_review_service(mocker): def sample_review_message_body(): """Create a sample review message body.""" return ReviewMessageBody( - file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf", + files=[ + ReviewMessageFile( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -50,8 +53,12 @@ def sample_sqs_event(sample_sqs_message): def sample_sqs_event_multiple_messages(sample_review_message_body): """Create a sample SQS event with multiple messages.""" message_1 = ReviewMessageBody( - file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf", + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -60,8 +67,12 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_2 = ReviewMessageBody( - file_name="document_2.pdf", - file_path="staging/9000000010/document_2.pdf", + files=[ + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000010/document_2.pdf" + ) + ], nhs_number="9000000010", failure_reason="Invalid file format", upload_date="2024-01-15T10:35:00Z", @@ -70,8 +81,12 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_3 = ReviewMessageBody( - file_name="document_3.pdf", - file_path="staging/9000000011/document_3.pdf", + files=[ + ReviewMessageFile( + file_name="document_3.pdf", + file_path="staging/9000000011/document_3.pdf" + ) + ], nhs_number="9000000011", failure_reason="Missing metadata", upload_date="2024-01-15T10:40:00Z", @@ -121,15 +136,8 @@ def test_lambda_handler_processes_single_message_successfully( mock_review_service, ): """Test handler successfully processes a single SQS message.""" - expected_response = ApiGatewayResponse( - status_code=200, - body="Processed 1 messages", - methods="GET", - ).create_api_gateway_response() - - actual_response = lambda_handler(sample_sqs_event, context) + lambda_handler(sample_sqs_event, context) - assert actual_response == expected_response mock_review_service.process_review_message.assert_called_once() @@ -140,15 +148,8 @@ def test_lambda_handler_processes_multiple_messages_successfully( mock_review_service, ): """Test handler successfully processes multiple SQS messages.""" - expected_response = ApiGatewayResponse( - status_code=200, - body="Processed 3 messages", - methods="GET", - ).create_api_gateway_response() + lambda_handler(sample_sqs_event_multiple_messages, context) - actual_response = lambda_handler(sample_sqs_event_multiple_messages, context) - - assert actual_response == expected_response assert mock_review_service.process_review_message.call_count == 3 @@ -166,19 +167,18 @@ def test_lambda_handler_calls_service_with_correct_message( call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" - assert call_args.file_name == "test_document.pdf" + assert len(call_args.files) == 1 + assert call_args.files[0].file_name == "test_document.pdf" assert call_args.nhs_number == "9000000009" - assert call_args.file_path == "staging/9000000009/test_document.pdf" + assert call_args.files[0].file_path == "staging/9000000009/test_document.pdf" def test_lambda_handler_handles_empty_records_list( set_review_env, context, empty_sqs_event, mock_review_service ): - """Test handler handles empty records list via @validate_sqs_event decorator.""" - actual_response = lambda_handler(empty_sqs_event, context) + """Test handler handles empty records list gracefully.""" + lambda_handler(empty_sqs_event, context) - assert actual_response["statusCode"] == 400 - assert "SQS_4001" in actual_response["body"] mock_review_service.process_review_message.assert_not_called() @@ -188,15 +188,15 @@ def test_lambda_handler_handles_service_error( sample_sqs_event, mock_review_service, ): - """Test handler catches exception when service fails (via @handle_lambda_exceptions decorator).""" + """Test handler logs error but doesn't raise when service fails.""" mock_review_service.process_review_message.side_effect = Exception( "Service processing failed" ) - response = lambda_handler(sample_sqs_event, context) + result = lambda_handler(sample_sqs_event, context) - assert response["statusCode"] == 500 - assert "UE_500" in response["body"] + assert result is None + mock_review_service.process_review_message.assert_called_once() def test_lambda_handler_handles_invalid_message_format( @@ -208,7 +208,7 @@ def test_lambda_handler_handles_invalid_message_format( { "body": json.dumps( { - "file_name": "test.pdf", + "files": [{"file_name": "test.pdf"}], } ), "eventSource": "aws:sqs", @@ -216,10 +216,10 @@ def test_lambda_handler_handles_invalid_message_format( ] } - response = lambda_handler(invalid_event, context) + lambda_handler(invalid_event, context) - assert response["statusCode"] == 500 - assert "UE_500" in response["body"] + # Service should not be called if message validation fails + mock_review_service.process_review_message.assert_not_called() def test_lambda_handler_handles_partial_failure( @@ -228,16 +228,16 @@ def test_lambda_handler_handles_partial_failure( sample_sqs_event_multiple_messages, mock_review_service, ): - """Test handler processes messages and handles first failure.""" + """Test handler processes all messages even when one fails.""" mock_review_service.process_review_message.side_effect = [ None, Exception("Processing failed on second message"), + None, ] - response = lambda_handler(sample_sqs_event_multiple_messages, context) - - assert response["statusCode"] == 500 - assert mock_review_service.process_review_message.call_count == 2 + lambda_handler(sample_sqs_event_multiple_messages, context) + + assert mock_review_service.process_review_message.call_count == 3 def test_lambda_handler_parses_json_body_correctly( @@ -251,8 +251,12 @@ def test_lambda_handler_parses_json_body_correctly( { "body": json.dumps( { - "file_name": "test.pdf", - "file_path": "staging/test.pdf", + "files": [ + { + "file_name": "test.pdf", + "file_path": "staging/test.pdf" + } + ], "nhs_number": "9000000009", "failure_reason": "Test failure", "upload_date": "2024-01-15T10:30:00Z", @@ -270,7 +274,8 @@ def test_lambda_handler_parses_json_body_correctly( mock_review_service.process_review_message.assert_called_once() call_args = mock_review_service.process_review_message.call_args[0][0] assert type(call_args).__name__ == "ReviewMessageBody" - assert call_args.file_name == "test.pdf" + assert len(call_args.files) == 1 + assert call_args.files[0].file_name == "test.pdf" def test_lambda_handler_logs_correct_counts( @@ -278,13 +283,16 @@ def test_lambda_handler_logs_correct_counts( context, sample_sqs_event_multiple_messages, mock_review_service, + mocker, ): - """Test handler response contains correct processed count.""" - response = lambda_handler(sample_sqs_event_multiple_messages, context) - - response_body = response["body"] + """Test handler logs correct processed count.""" + mock_logger = mocker.patch("handlers.review_processor_handler.logger") + + lambda_handler(sample_sqs_event_multiple_messages, context) - assert "Processed 3 messages" in response_body + mock_logger.info.assert_any_call( + "Review processor completed: 3 processed, 0 failed" + ) def test_lambda_handler_tracks_failed_count( @@ -292,11 +300,16 @@ def test_lambda_handler_tracks_failed_count( context, sample_sqs_event, mock_review_service, + mocker, ): - """Test handler tracks failed message count.""" + """Test handler tracks failed message count in logs.""" mock_review_service.process_review_message.side_effect = Exception("Test error") + mock_logger = mocker.patch("handlers.review_processor_handler.logger") - response = lambda_handler(sample_sqs_event, context) + lambda_handler(sample_sqs_event, context) - assert response["statusCode"] == 500 mock_review_service.process_review_message.assert_called_once() + mock_logger.error.assert_called() + mock_logger.info.assert_any_call( + "Review processor completed: 0 processed, 1 failed" + ) diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 8907011b9..9c19b3f60 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -1,11 +1,8 @@ -from datetime import datetime, timezone -from unittest.mock import MagicMock - import pytest from botocore.exceptions import ClientError from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview -from models.sqs.review_message_body import ReviewMessageBody +from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.review_processor_service import ReviewProcessorService from utils.exceptions import S3FileNotFoundException @@ -41,8 +38,12 @@ def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): def sample_review_message(): """Create a sample review message.""" return ReviewMessageBody( - file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf", + files=[ + ReviewMessageFile( + file_name="test_document.pdf", + file_path="staging/9000000009/test_document.pdf" + ) + ], nhs_number="9000000009", failure_reason="Failed virus scan", upload_date="2024-01-15T10:30:00Z", @@ -77,27 +78,63 @@ def test_process_review_message_success( mock_verify = mocker.patch.object( service_under_test, "_verify_file_exists_in_staging" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") mock_move = mocker.patch.object( - service_under_test, "_move_file_to_review_bucket" - ) - mock_update = mocker.patch.object( - service_under_test, "_update_review_record_with_file_location" + service_under_test, "_move_files_to_review_bucket" ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") - mock_review = MagicMock() - mock_review.id = "test-review-id" - mock_create.return_value = mock_review - mock_move.return_value = "9000000009/test-review-id/test_document.pdf" + mock_move.return_value = [ + {"file_name": "test_document.pdf", "file_location": "9000000009/test-review-id/test_document.pdf"} + ] service_under_test.process_review_message(sample_review_message) - mock_verify.assert_called_once_with(sample_review_message.file_path) - mock_create.assert_called_once_with(sample_review_message) - mock_move.assert_called_once_with(sample_review_message, "test-review-id") - mock_update.assert_called_once_with( - "test-review-id", "9000000009/test-review-id/test_document.pdf" + mock_verify.assert_called_once_with(sample_review_message.files[0].file_path) + mock_move.assert_called_once() + mock_create.assert_called_once() + + +def test_process_review_message_multiple_files( + service_under_test, mocker +): + """Test successful processing of a review message with multiple files.""" + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + mock_verify = mocker.patch.object( + service_under_test, "_verify_file_exists_in_staging" ) + mock_move = mocker.patch.object( + service_under_test, "_move_files_to_review_bucket" + ) + mock_create = mocker.patch.object(service_under_test, "_create_review_record") + + mock_move.return_value = [ + {"file_name": "document_1.pdf", "file_location": "9000000009/test-review-id/document_1.pdf"}, + {"file_name": "document_2.pdf", "file_location": "9000000009/test-review-id/document_2.pdf"} + ] + + service_under_test.process_review_message(message) + + assert mock_verify.call_count == 2 + mock_move.assert_called_once() + mock_create.assert_called_once() + def test_process_review_message_file_not_found( @@ -114,17 +151,17 @@ def test_process_review_message_file_not_found( service_under_test.process_review_message(sample_review_message) -def test_process_review_message_dynamo_error( +def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when DynamoDB create fails.""" + """Test processing fails when S3 copy operation fails.""" mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") mocker.patch.object( service_under_test, - "_create_review_record", + "_move_files_to_review_bucket", side_effect=ClientError( - {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, - "PutItem", + {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, + "CopyObject", ), ) @@ -132,22 +169,18 @@ def test_process_review_message_dynamo_error( service_under_test.process_review_message(sample_review_message) -def test_process_review_message_s3_copy_error( +def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when S3 copy operation fails.""" + """Test processing fails when DynamoDB create fails.""" mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") - mock_review = MagicMock() - mock_review.id = "test-review-id" - mocker.patch.object( - service_under_test, "_create_review_record", return_value=mock_review - ) + mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) mocker.patch.object( service_under_test, - "_move_file_to_review_bucket", + "_create_review_record", side_effect=ClientError( - {"Error": {"Code": "NoSuchKey", "Message": "Source file not found"}}, - "CopyObject", + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", ), ) @@ -155,6 +188,7 @@ def test_process_review_message_s3_copy_error( service_under_test.process_review_message(sample_review_message) + # Tests for _verify_file_exists_in_staging method @@ -190,18 +224,26 @@ def test_verify_file_s3_error(service_under_test): service_under_test._verify_file_exists_in_staging("staging/test.pdf") -# Tests for _create_review_record method +# Tests for _build_review_record and _create_review_record methods -def test_create_review_record_success( - service_under_test, sample_review_message, mocker -): - """Test successful creation of review record.""" - service_under_test.dynamo_service.create_item.return_value = None - - result = service_under_test._create_review_record(sample_review_message) +def test_build_review_record_success(service_under_test, sample_review_message): + """Test successful building of review record.""" + from models.document_review import DocumentReviewFileDetails + + files = [ + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-review-id/test_document.pdf" + ) + ] + + result = service_under_test._build_review_record( + sample_review_message, "test-review-id", files + ) assert isinstance(result, DocumentsUploadReview) + assert result.id == "test-review-id" assert result.nhs_number == "9000000009" assert result.review_status == ReviewStatus.PENDING_REVIEW assert result.review_reason == "Failed virus scan" @@ -209,68 +251,96 @@ def test_create_review_record_success( assert result.custodian == "Y12345" assert len(result.files) == 1 assert result.files[0].file_name == "test_document.pdf" - assert result.files[0].file_location == "staging/9000000009/test_document.pdf" + assert result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + - expected_timestamp = int( - datetime.fromisoformat("2024-01-15T10:30:00Z") - .replace(tzinfo=timezone.utc) - .timestamp() +def test_build_review_record_with_multiple_files(service_under_test): + """Test building review record with multiple files.""" + from models.document_review import DocumentReviewFileDetails + + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", ) - assert result.upload_date == expected_timestamp + + files = [ + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-review-id/document_1.pdf" + ), + DocumentReviewFileDetails( + file_name="document_2.pdf", + file_location="9000000009/test-review-id/document_2.pdf" + ) + ] + + result = service_under_test._build_review_record(message, "test-review-id", files) - service_under_test.dynamo_service.create_item.assert_called_once() - call_args = service_under_test.dynamo_service.create_item.call_args - assert call_args[1]["table_name"] == "test_review_table" + assert len(result.files) == 2 + assert result.files[0].file_name == "document_1.pdf" + assert result.files[1].file_name == "document_2.pdf" -def test_create_review_record_with_different_data(service_under_test, mocker): - """Test creation with different message data.""" - message = ReviewMessageBody( - file_name="another_doc.pdf", - file_path="staging/9000000010/another_doc.pdf", - nhs_number="9000000010", - failure_reason="Missing metadata", - upload_date="2024-02-20T14:45:30Z", - uploader_ods="Y67890", - current_gp="Y67890", +def test_create_review_record_success(service_under_test, sample_review_message): + """Test successful creation of review record in DynamoDB.""" + from models.document_review import DocumentReviewFileDetails + + review_record = DocumentsUploadReview( + id="test-review-id", + nhs_number="9000000009", + review_status=ReviewStatus.PENDING_REVIEW, + review_reason="Failed virus scan", + author="Y12345", + custodian="Y12345", + files=[ + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-review-id/test_document.pdf" + ) + ], + upload_date=1705319400 ) service_under_test.dynamo_service.create_item.return_value = None - result = service_under_test._create_review_record(message) - - assert result.nhs_number == "9000000010" - assert result.review_reason == "Missing metadata" - assert result.author == "Y67890" - assert result.custodian == "Y67890" + service_under_test._create_review_record(review_record) + service_under_test.dynamo_service.create_item.assert_called_once() + call_args = service_under_test.dynamo_service.create_item.call_args + assert call_args[1]["table_name"] == "test_review_table" + assert call_args[1]["item"] == review_record -def test_create_review_record_dynamo_error( - service_under_test, sample_review_message -): - """Test create record handles DynamoDB errors.""" - service_under_test.dynamo_service.create_item.side_effect = ClientError( - {"Error": {"Code": "ValidationException", "Message": "Invalid item"}}, - "PutItem", - ) - - with pytest.raises(ClientError): - service_under_test._create_review_record(sample_review_message) -# Tests for _move_file_to_review_bucket method +# Tests for _move_files_to_review_bucket method -def test_move_file_success(service_under_test, sample_review_message, mocker): +def test_move_files_success(service_under_test, sample_review_message, mocker): """Test successful file move from staging to review bucket.""" mocker.patch.object(service_under_test, "_delete_from_staging") - new_file_key = service_under_test._move_file_to_review_bucket( + files = service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id-123" ) expected_key = "9000000009/test-review-id-123/test_document.pdf" - assert new_file_key == expected_key + + assert len(files) == 1 + assert files[0].file_name == "test_document.pdf" + assert files[0].file_location == expected_key service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bucket", @@ -284,7 +354,41 @@ def test_move_file_success(service_under_test, sample_review_message, mocker): ) -def test_move_file_copy_error(service_under_test, sample_review_message, mocker): +def test_move_multiple_files_success(service_under_test, mocker): + """Test successful move of multiple files.""" + message = ReviewMessageBody( + files=[ + ReviewMessageFile( + file_name="document_1.pdf", + file_path="staging/9000000009/document_1.pdf" + ), + ReviewMessageFile( + file_name="document_2.pdf", + file_path="staging/9000000009/document_2.pdf" + ) + ], + nhs_number="9000000009", + failure_reason="Failed virus scan", + upload_date="2024-01-15T10:30:00Z", + uploader_ods="Y12345", + current_gp="Y12345", + ) + + mocker.patch.object(service_under_test, "_delete_from_staging") + + files = service_under_test._move_files_to_review_bucket(message, "test-review-id") + + assert len(files) == 2 + assert files[0].file_name == "document_1.pdf" + assert files[0].file_location == "9000000009/test-review-id/document_1.pdf" + assert files[1].file_name == "document_2.pdf" + assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" + + assert service_under_test.s3_service.copy_across_bucket.call_count == 2 + assert service_under_test._delete_from_staging.call_count == 2 + + +def test_move_files_copy_error(service_under_test, sample_review_message, mocker): """Test file move handles S3 copy errors.""" service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, @@ -292,12 +396,12 @@ def test_move_file_copy_error(service_under_test, sample_review_message, mocker) ) with pytest.raises(ClientError): - service_under_test._move_file_to_review_bucket( + service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) -def test_move_file_delete_error(service_under_test, sample_review_message, mocker): +def test_move_files_delete_error(service_under_test, sample_review_message, mocker): """Test file move handles delete errors.""" mocker.patch.object( service_under_test, @@ -309,11 +413,12 @@ def test_move_file_delete_error(service_under_test, sample_review_message, mocke ) with pytest.raises(ClientError): - service_under_test._move_file_to_review_bucket( + service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) + # Tests for _delete_from_staging method @@ -337,51 +442,17 @@ def test_delete_from_staging_error(service_under_test): service_under_test._delete_from_staging("staging/test.pdf") -# Tests for _update_review_record_with_file_location method - - -def test_update_review_record_success(service_under_test): - """Test successful update of review record with file location.""" - service_under_test._update_review_record_with_file_location( - "test-review-id", "9000000009/test-review-id/test.pdf" - ) - - service_under_test.dynamo_service.update_item.assert_called_once_with( - table_name="test_review_table", - key_pair={"ID": "test-review-id"}, - updated_fields={"ReviewBucketPath": "9000000009/test-review-id/test.pdf"}, - ) - - -def test_update_review_record_dynamo_error(service_under_test, mocker): - """Test update review record handles DynamoDB errors gracefully.""" - service_under_test.dynamo_service.update_item.side_effect = ClientError( - {"Error": {"Code": "ConditionalCheckFailedException", "Message": "Failed"}}, - "UpdateItem", - ) - - mock_logger = mocker.patch("services.review_processor_service.logger") - - service_under_test._update_review_record_with_file_location( - "test-review-id", "path/to/file.pdf" - ) - - assert mock_logger.error.called - assert mock_logger.warning.called - - # Integration scenario tests def test_full_workflow_with_valid_message( service_under_test, sample_review_message, mocker ): - """Test complete workflow from message to final update.""" + """Test complete workflow from message to final record creation.""" service_under_test.s3_service.file_exist_on_s3.return_value = True service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None - service_under_test.dynamo_service.update_item.return_value = None service_under_test.process_review_message(sample_review_message) @@ -389,7 +460,6 @@ def test_full_workflow_with_valid_message( service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() - service_under_test.dynamo_service.update_item.assert_called_once() def test_workflow_stops_at_verification_failure( @@ -411,12 +481,15 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None - service_under_test.dynamo_service.update_item.return_value = None messages = [ ReviewMessageBody( - file_name=f"doc_{i}.pdf", - file_path=f"staging/900000000{i}/doc_{i}.pdf", + files=[ + ReviewMessageFile( + file_name=f"doc_{i}.pdf", + file_path=f"staging/900000000{i}/doc_{i}.pdf" + ) + ], nhs_number=f"900000000{i}", failure_reason="Test failure", upload_date="2024-01-15T10:30:00Z", From f3b3cdc31b7c948f6a6775b33a31ce6e55a8215f Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Tue, 28 Oct 2025 16:18:02 +0000 Subject: [PATCH 23/47] [PRMP-585] code review comment improvements # Conflicts: # lambdas/utils/exceptions.py --- .../base-lambdas-reusable-deploy-all.yml | 14 +++ lambdas/handlers/review_processor_handler.py | 21 ++--- lambdas/models/document_review.py | 10 +- lambdas/models/sqs/review_message_body.py | 4 +- lambdas/services/review_processor_service.py | 19 ++-- .../handlers/test_review_processor_handler.py | 94 ------------------- .../services/test_review_processor_service.py | 12 +-- lambdas/utils/exceptions.py | 20 +++- 8 files changed, 67 insertions(+), 127 deletions(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index ee41d2cd3..d0e4bef45 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -668,3 +668,17 @@ jobs: lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_review_processor_lambda: + name: Deploy Review Processor Lambda + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: review_processor_handler + lambda_aws_name: ReviewProcessorLambda + lambda_layer_names: "core_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/review_processor_handler.py index f92dc8eab..7dfc229b6 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/review_processor_handler.py @@ -1,8 +1,8 @@ import json from pydantic import ValidationError -from lambdas.models.sqs.review_message_body import ReviewMessageBody -from lambdas.services.review_processor_service import ReviewProcessorService +from models.sqs.review_message_body import ReviewMessageBody +from services.review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.override_error_check import override_error_check @@ -38,28 +38,23 @@ def lambda_handler(event, context): sqs_messages = event.get("Records", []) review_service = ReviewProcessorService() - processed_count = 0 - failed_count = 0 - for sqs_message in sqs_messages: try: sqs_message_body = json.loads(sqs_message["body"]) message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) review_service.process_review_message(message) - processed_count += 1 + except ValidationError as error: logger.error("Malformed review message") logger.error(error) + raise error - except Exception as e: + except Exception as error: logger.error( - f"Failed to process review message: {str(e)}", + f"Failed to process review message: {str(error)}", {"Result": "Review processing failed"}, ) - failed_count += 1 + raise error + logger.info("Continuing to next message.") - - logger.info( - f"Review processor completed: {processed_count} processed, {failed_count} failed" - ) diff --git a/lambdas/models/document_review.py b/lambdas/models/document_review.py index 9b3eee106..76af50606 100644 --- a/lambdas/models/document_review.py +++ b/lambdas/models/document_review.py @@ -4,9 +4,9 @@ from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_pascal -from lambdas.enums.review_status import ReviewStatus -from lambdas.enums.snomed_codes import SnomedCodes -from lambdas.models.document_reference import DocumentReferenceMetadataFields +from enums.review_status import ReviewStatus +from enums.snomed_codes import SnomedCodes +from models.document_reference import DocumentReferenceMetadataFields class DocumentReviewFileDetails(BaseModel): @@ -30,7 +30,7 @@ class DocumentsUploadReview(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), alias=str(DocumentReferenceMetadataFields.ID.value) - ) # id differse to nogas version + ) author: str custodian: str review_status: ReviewStatus = Field(default=ReviewStatus.PENDING_REVIEW) @@ -38,7 +38,7 @@ class DocumentsUploadReview(BaseModel): review_date: int | None = Field(default=None) reviewer: str | None = Field(default=None) upload_date: int - files: list[DocumentReviewFileDetails] = Field(default=[]) # differs to nogas version + files: list[DocumentReviewFileDetails] nhs_number: str ttl: Optional[int] = Field( alias=str(DocumentReferenceMetadataFields.TTL.value), default=None diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index a34c84214..48000b5d2 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -1,11 +1,11 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field class ReviewMessageFile(BaseModel): """Model for individual file in SQS message body from the document review queue.""" file_name: str - file_path: str + file_path: str = Field(description="Location in the staging bucket") """Location in the staging bucket""" class ReviewMessageBody(BaseModel): diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 0e9416ac8..1a9b9eb1b 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -8,7 +8,7 @@ from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import S3FileNotFoundException +from utils.exceptions import ReviewProcessCreateRecordException, ReviewProcessDeleteException, ReviewProcessMovingException, ReviewProcessVerifyingException, S3FileNotFoundException from utils.request_context import request_context logger = LoggingService(__name__) @@ -79,11 +79,16 @@ def _verify_file_exists_in_staging(self, file_path: str) -> None: logger.info(f"Verified file exists in staging: {file_path}") - except S3FileNotFoundException: + except S3FileNotFoundException as e: + logger.info(e) + logger.info( + f"File not found in staging bucket {self.staging_bucket_name} for file_path {file_path}" + ) + raise except Exception as e: logger.error(f"Error checking file in staging bucket: {str(e)}") - raise + raise ReviewProcessVerifyingException(f"Error checking file in staging bucket: {str(e)}") def _build_review_record( self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] @@ -139,7 +144,7 @@ def _move_files_to_review_bucket( except Exception as e: logger.error(f"Failed to move file: {str(e)}") - raise + raise ReviewProcessMovingException(f"Failed to move file: {str(e)}") return moved_files @@ -151,7 +156,7 @@ def _delete_from_staging(self, file_key: str) -> None: except Exception as e: logger.error(f"Error deleting file from staging: {str(e)}") - raise + raise ReviewProcessDeleteException(f"Error deleting file from staging: {str(e)}") def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: @@ -164,4 +169,6 @@ def _create_review_record(self, review_record: DocumentsUploadReview) -> None: except Exception as e: logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") - raise + raise ReviewProcessCreateRecordException( + f"Failed to create review record with id: {review_record.id} -- {str(e)}" + ) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index 85be717a4..b3794863a 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -182,64 +182,6 @@ def test_lambda_handler_handles_empty_records_list( mock_review_service.process_review_message.assert_not_called() -def test_lambda_handler_handles_service_error( - set_review_env, - context, - sample_sqs_event, - mock_review_service, -): - """Test handler logs error but doesn't raise when service fails.""" - mock_review_service.process_review_message.side_effect = Exception( - "Service processing failed" - ) - - result = lambda_handler(sample_sqs_event, context) - - assert result is None - mock_review_service.process_review_message.assert_called_once() - - -def test_lambda_handler_handles_invalid_message_format( - set_review_env, context, mock_review_service -): - """Test handler handles validation errors for invalid message format.""" - invalid_event = { - "Records": [ - { - "body": json.dumps( - { - "files": [{"file_name": "test.pdf"}], - } - ), - "eventSource": "aws:sqs", - } - ] - } - - lambda_handler(invalid_event, context) - - # Service should not be called if message validation fails - mock_review_service.process_review_message.assert_not_called() - - -def test_lambda_handler_handles_partial_failure( - set_review_env, - context, - sample_sqs_event_multiple_messages, - mock_review_service, -): - """Test handler processes all messages even when one fails.""" - mock_review_service.process_review_message.side_effect = [ - None, - Exception("Processing failed on second message"), - None, - ] - - lambda_handler(sample_sqs_event_multiple_messages, context) - - assert mock_review_service.process_review_message.call_count == 3 - - def test_lambda_handler_parses_json_body_correctly( set_review_env, context, @@ -277,39 +219,3 @@ def test_lambda_handler_parses_json_body_correctly( assert len(call_args.files) == 1 assert call_args.files[0].file_name == "test.pdf" - -def test_lambda_handler_logs_correct_counts( - set_review_env, - context, - sample_sqs_event_multiple_messages, - mock_review_service, - mocker, -): - """Test handler logs correct processed count.""" - mock_logger = mocker.patch("handlers.review_processor_handler.logger") - - lambda_handler(sample_sqs_event_multiple_messages, context) - - mock_logger.info.assert_any_call( - "Review processor completed: 3 processed, 0 failed" - ) - - -def test_lambda_handler_tracks_failed_count( - set_review_env, - context, - sample_sqs_event, - mock_review_service, - mocker, -): - """Test handler tracks failed message count in logs.""" - mock_review_service.process_review_message.side_effect = Exception("Test error") - mock_logger = mocker.patch("handlers.review_processor_handler.logger") - - lambda_handler(sample_sqs_event, context) - - mock_review_service.process_review_message.assert_called_once() - mock_logger.error.assert_called() - mock_logger.info.assert_any_call( - "Review processor completed: 0 processed, 1 failed" - ) diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index 9c19b3f60..a1501e335 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -3,8 +3,8 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.review_processor_service import ReviewProcessorService -from utils.exceptions import S3FileNotFoundException +from services.review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException @pytest.fixture @@ -220,7 +220,7 @@ def test_verify_file_s3_error(service_under_test): "HeadObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessVerifyingException): service_under_test._verify_file_exists_in_staging("staging/test.pdf") @@ -395,7 +395,7 @@ def test_move_files_copy_error(service_under_test, sample_review_message, mocker "CopyObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessMovingException): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) @@ -412,7 +412,7 @@ def test_move_files_delete_error(service_under_test, sample_review_message, mock ), ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessMovingException): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) @@ -438,7 +438,7 @@ def test_delete_from_staging_error(service_under_test): "DeleteObject", ) - with pytest.raises(ClientError): + with pytest.raises(ReviewProcessDeleteException): service_under_test._delete_from_staging("staging/test.pdf") diff --git a/lambdas/utils/exceptions.py b/lambdas/utils/exceptions.py index 7f6d3171b..75341ffdc 100644 --- a/lambdas/utils/exceptions.py +++ b/lambdas/utils/exceptions.py @@ -159,9 +159,11 @@ class InvalidFileNameException(Exception): class MetadataPreprocessingException(Exception): pass + class FhirDocumentReferenceException(Exception): pass + class TransactionConflictException(Exception): pass @@ -172,7 +174,7 @@ def __init__(self, message: str, item_id: str): self.item_id = item_id def to_dict(self): - return {"itemId": self.item_id, "message": self.message} + return {"itemId": self.item_id, "message": self.message} class MigrationRetryableException(Exception): def __init__(self, message: str, segment_id: str): @@ -182,3 +184,19 @@ def __init__(self, message: str, segment_id: str): def to_dict(self): return {"segmentId": self.segment_id, "message": self.message} + + +class ReviewProcessVerifyingException(Exception): + pass + + +class ReviewProcessMovingException(Exception): + pass + + +class ReviewProcessDeleteException(Exception): + pass + + +class ReviewProcessCreateRecordException(Exception): + pass From 714ebfa9c31a1bb2b5e24af55a445e445c466fa8 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Wed, 29 Oct 2025 11:19:11 +0000 Subject: [PATCH 24/47] [PRMP-585] Code review comment changes --- lambdas/services/review_processor_service.py | 2 +- lambdas/tests/unit/conftest.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/review_processor_service.py index 1a9b9eb1b..fd3360239 100644 --- a/lambdas/services/review_processor_service.py +++ b/lambdas/services/review_processor_service.py @@ -50,7 +50,7 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: logger.info(f"Processing review file: {file.file_name}") self._verify_file_exists_in_staging(file.file_path) - review_id = uuid.uuid4().hex + review_id = str(uuid.uuid4) files = self._move_files_to_review_bucket(review_message, review_id) document_upload_review = self._build_review_record(review_message, review_id, files) diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index 7eaf123bd..3f5fb643d 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -61,6 +61,8 @@ MOCK_APPCONFIG_CONFIGURATION_ENV_NAME = "APPCONFIG_CONFIGURATION" MOCK_STATISTICS_TABLE_NAME = "STATISTICS_TABLE" MOCK_STATISTICAL_REPORTS_BUCKET_ENV_NAME = "STATISTICAL_REPORTS_BUCKET" +MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME = "DOCUMENT_REVIEW_DYNAMODB_NAME" +MOCK_PENDING_REVIEW_BUCKET_NAME = "PENDING_REVIEW_BUCKET_NAME" MOCK_ARF_TABLE_NAME = "test_arf_dynamoDB_table" MOCK_PDM_TABLE_NAME = "test_pdm_dynamoDB_table" @@ -77,6 +79,8 @@ MOCK_LG_INVALID_SQS_QUEUE = "INVALID_SQS_QUEUE_URL" MOCK_STATISTICS_TABLE = "test_statistics_table" MOCK_STATISTICS_REPORT_BUCKET_NAME = "test_statistics_report_bucket" +MOCK_PENDING_REVIEW_BUCKET = "test_review_bucket" +MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME = "test_review_table" TEST_NHS_NUMBER = "9000000009" TEST_UUID = "1234-4567-8912-HSDF-TEST" @@ -227,6 +231,8 @@ def set_env(monkeypatch): monkeypatch.setenv("ITOC_TESTING_ODS_CODES", MOCK_ITOC_ODS_CODES) monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", MOCK_STAGING_STORE_BUCKET) monkeypatch.setenv("METADATA_SQS_QUEUE_URL", MOCK_LG_METADATA_SQS_QUEUE) + monkeypatch.setenv(MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME, MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME) + monkeypatch.setenv(MOCK_PENDING_REVIEW_BUCKET_NAME, MOCK_PENDING_REVIEW_BUCKET) EXPECTED_PARSED_PATIENT_BASE_CASE = PatientDetails( From 97ab571f80e1c94a9372fc8894717699e8558541 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Wed, 29 Oct 2025 16:16:48 +0000 Subject: [PATCH 25/47] [PRMP-585] required changes to match terraform --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 4 ++-- ...cessor_handler.py => document_review_processor_handler.py} | 2 +- ...cessor_service.py => document_review_processor_service.py} | 0 lambdas/tests/unit/handlers/test_review_processor_handler.py | 2 +- lambdas/tests/unit/services/test_review_processor_service.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename lambdas/handlers/{review_processor_handler.py => document_review_processor_handler.py} (96%) rename lambdas/services/{review_processor_service.py => document_review_processor_service.py} (100%) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index d0e4bef45..23b0e6d7a 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -677,8 +677,8 @@ jobs: python_version: ${{ inputs.python_version }} build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} - lambda_handler_name: review_processor_handler - lambda_aws_name: ReviewProcessorLambda + lambda_handler_name: document_review_processor_handler + lambda_aws_name: DocumentReviewProcessor lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py similarity index 96% rename from lambdas/handlers/review_processor_handler.py rename to lambdas/handlers/document_review_processor_handler.py index 7dfc229b6..44f0a01cd 100644 --- a/lambdas/handlers/review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -2,7 +2,7 @@ from pydantic import ValidationError from models.sqs.review_message_body import ReviewMessageBody -from services.review_processor_service import ReviewProcessorService +from services.document_review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.override_error_check import override_error_check diff --git a/lambdas/services/review_processor_service.py b/lambdas/services/document_review_processor_service.py similarity index 100% rename from lambdas/services/review_processor_service.py rename to lambdas/services/document_review_processor_service.py diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_review_processor_handler.py index b3794863a..4718fa678 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_review_processor_handler.py @@ -1,7 +1,7 @@ import json import pytest -from handlers.review_processor_handler import lambda_handler +from handlers.document_review_processor_handler import lambda_handler from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_review_processor_service.py index a1501e335..586813882 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_review_processor_service.py @@ -3,7 +3,7 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from services.document_review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException From 058aa47f313a6cb25c9eaa2416c5de895e84dff4 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Thu, 30 Oct 2025 12:15:27 +0000 Subject: [PATCH 26/47] [PRPM-585] minor fixes --- .../document_review_processor_service.py | 8 ++++- ...test_document_review_processor_handler.py} | 2 +- ...test_document_review_processor_service.py} | 30 ++++++++----------- 3 files changed, 21 insertions(+), 19 deletions(-) rename lambdas/tests/unit/handlers/{test_review_processor_handler.py => test_document_review_processor_handler.py} (98%) rename lambdas/tests/unit/services/{test_review_processor_service.py => test_document_review_processor_service.py} (94%) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fd3360239..fc5dfd12e 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -8,7 +8,13 @@ from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import ReviewProcessCreateRecordException, ReviewProcessDeleteException, ReviewProcessMovingException, ReviewProcessVerifyingException, S3FileNotFoundException +from utils.exceptions import ( + ReviewProcessCreateRecordException, + ReviewProcessDeleteException, + ReviewProcessMovingException, + ReviewProcessVerifyingException, + S3FileNotFoundException +) from utils.request_context import request_context logger = LoggingService(__name__) diff --git a/lambdas/tests/unit/handlers/test_review_processor_handler.py b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py similarity index 98% rename from lambdas/tests/unit/handlers/test_review_processor_handler.py rename to lambdas/tests/unit/handlers/test_document_review_processor_handler.py index 4718fa678..b652f0306 100644 --- a/lambdas/tests/unit/handlers/test_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py @@ -9,7 +9,7 @@ def mock_review_service(mocker): """Mock the ReviewProcessorService.""" mocked_class = mocker.patch( - "handlers.review_processor_handler.ReviewProcessorService" + "handlers.document_review_processor_handler.ReviewProcessorService" ) mocked_instance = mocked_class.return_value return mocked_instance diff --git a/lambdas/tests/unit/services/test_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py similarity index 94% rename from lambdas/tests/unit/services/test_review_processor_service.py rename to lambdas/tests/unit/services/test_document_review_processor_service.py index 586813882..38e177151 100644 --- a/lambdas/tests/unit/services/test_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -3,32 +3,28 @@ from enums.review_status import ReviewStatus from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.document_review_processor_service import ReviewProcessMovingException, ReviewProcessVerifyingException, ReviewProcessorService +from services.document_review_processor_service import ( + ReviewProcessMovingException, + ReviewProcessVerifyingException, + ReviewProcessorService, +) from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException @pytest.fixture def mock_dynamo_service(mocker): """Mock the DynamoDBService.""" - return mocker.patch("services.review_processor_service.DynamoDBService") + return mocker.patch("services.document_review_processor_service.DynamoDBService") @pytest.fixture def mock_s3_service(mocker): """Mock the S3Service.""" - return mocker.patch("services.review_processor_service.S3Service") + return mocker.patch("services.document_review_processor_service.S3Service") @pytest.fixture -def set_review_env(monkeypatch): - """Set up environment variables required for the service.""" - monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") - monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") - monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") - - -@pytest.fixture -def service_under_test(set_review_env, mock_dynamo_service, mock_s3_service): +def service_under_test(set_env, mock_dynamo_service, mock_s3_service): """Create a ReviewProcessorService instance with mocked dependencies.""" service = ReviewProcessorService() return service @@ -56,13 +52,13 @@ def sample_review_message(): def test_service_initializes_with_correct_environment_variables( - set_review_env, mock_dynamo_service, mock_s3_service + set_env, mock_dynamo_service, mock_s3_service ): """Test service initializes correctly with environment variables.""" service = ReviewProcessorService() assert service.review_table_name == "test_review_table" - assert service.staging_bucket_name == "test_staging_bucket" + assert service.staging_bucket_name == "test_staging_bulk_store" assert service.review_bucket_name == "test_review_bucket" mock_dynamo_service.assert_called_once() mock_s3_service.assert_called_once() @@ -199,7 +195,7 @@ def test_verify_file_exists_success(service_under_test): service_under_test._verify_file_exists_in_staging("staging/test.pdf") service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( - s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" ) @@ -343,7 +339,7 @@ def test_move_files_success(service_under_test, sample_review_message, mocker): assert files[0].file_location == expected_key service_under_test.s3_service.copy_across_bucket.assert_called_once_with( - source_bucket="test_staging_bucket", + source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", dest_file_key=expected_key, @@ -427,7 +423,7 @@ def test_delete_from_staging_success(service_under_test): service_under_test._delete_from_staging("staging/test.pdf") service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bucket", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" ) From 1557d6218a6a086b7ab19a4fa702764bb541675e Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Thu, 30 Oct 2025 15:13:53 +0000 Subject: [PATCH 27/47] [PRMP-585] exclude none --- lambdas/services/document_review_processor_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fc5dfd12e..665b456e5 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -168,7 +168,7 @@ def _create_review_record(self, review_record: DocumentsUploadReview) -> None: try: self.dynamo_service.create_item( table_name=self.review_table_name, - item=review_record + item=review_record.model_dump(by_alias=True, exclude_none=True) ) logger.info(f"Created review record {review_record.id}") From 4510521f0c9af2153b3483ce128281d7570ee781 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 14:51:54 +0000 Subject: [PATCH 28/47] [PRMP-585] enhance message checks --- .../document_review_processor_handler.py | 12 +- lambdas/models/sqs/review_message_body.py | 2 +- .../layers/requirements_core_lambda_layer.txt | 3 +- lambdas/services/base/dynamo_service.py | 32 ++- lambdas/services/base/s3_service.py | 19 +- .../document_review_processor_service.py | 147 ++++-------- .../test_document_review_processor_handler.py | 41 +++- .../unit/services/base/test_dynamo_service.py | 80 +++++++ .../unit/services/base/test_s3_service.py | 19 ++ .../test_document_review_processor_service.py | 213 ++++++------------ 10 files changed, 297 insertions(+), 271 deletions(-) diff --git a/lambdas/handlers/document_review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py index 44f0a01cd..84cf6b275 100644 --- a/lambdas/handlers/document_review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -28,7 +28,7 @@ def lambda_handler(event, context): Args: event: Lambda event containing SQS Event - _context: Lambda context + context: Lambda context Returns: None @@ -39,12 +39,16 @@ def lambda_handler(event, context): review_service = ReviewProcessorService() for sqs_message in sqs_messages: + message: ReviewMessageBody | None = None try: sqs_message_body = json.loads(sqs_message["body"]) - message: ReviewMessageBody = ReviewMessageBody.model_validate(sqs_message_body) + message = ReviewMessageBody.model_validate(sqs_message_body) - review_service.process_review_message(message) - + if isinstance(message, ReviewMessageBody): + review_service.process_review_message(message) + else: + raise ValidationError("Invalid review message format") + except ValidationError as error: logger.error("Malformed review message") logger.error(error) diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index 48000b5d2..e213168f2 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -10,7 +10,7 @@ class ReviewMessageFile(BaseModel): class ReviewMessageBody(BaseModel): """Model for SQS message body from the document review queue.""" - + upload_id: str files: list[ReviewMessageFile] nhs_number: str failure_reason: str diff --git a/lambdas/requirements/layers/requirements_core_lambda_layer.txt b/lambdas/requirements/layers/requirements_core_lambda_layer.txt index 499b03712..da90257d9 100644 --- a/lambdas/requirements/layers/requirements_core_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_core_lambda_layer.txt @@ -17,4 +17,5 @@ responses==0.23.1 six==1.16.0 types-PyYAML==6.0.12.11 regex==2023.12.25 -pikepdf==8.4.0 \ No newline at end of file +pikepdf==8.4.0 +types-boto3[dynamodb,s3]==1.40.64 \ No newline at end of file diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 3daffe33d..9ad9aeada 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,6 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError +from types_boto3_dynamodb.type_defs import PutItemOutputTableTypeDef from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -11,6 +12,7 @@ create_update_expression, ) from utils.exceptions import DynamoServiceException +from types_boto3_dynamodb import DynamoDBServiceResource logger = LoggingService(__name__) @@ -26,7 +28,7 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb = boto3.resource("dynamodb", region_name="eu-west-2") + self.dynamodb: DynamoDBServiceResource = boto3.resource("dynamodb", region_name="eu-west-2") self.initialised = True def get_table(self, table_name: str): @@ -88,6 +90,34 @@ def create_item(self, table_name, item): str(e), {"Result": f"Unable to write item to table: {table_name}"} ) raise e + + def put_item(self, table_name, item, key_name, check_exists: bool = True) -> PutItemOutputTableTypeDef: + """ + Put an item into the specified DynamoDB table with a condition on the existence of the key. + Args: + table_name: Name of the DynamoDB table + item: The item to be inserted (as a dictionary) + key_name: The name of the key field + check_exists: If True, the item will only be inserted if the key does not already exist. + If False, the item will only be inserted if the key already exists. + Returns: + Response from the DynamoDB put_item operation + Raises: + ClientError: For AWS service errors (DynamoDB) + """ + try: + table = self.get_table(table_name) + logger.info(f"Writing item to table: {table_name}") + return table.put_item(Item=item, Expected={ + key_name: { + 'Exists': check_exists + } + }) + except ClientError as e: + logger.error( + str(e), {"Result": f"Unable to write item to table: {table_name}"} + ) + raise e def update_item( self, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 5c6a690cb..d94afc5de 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -5,6 +5,7 @@ import boto3 from botocore.client import Config as BotoConfig +from types_boto3_s3 import S3Client from botocore.exceptions import ClientError from services.base.iam_service import IAMService from utils.audit_logging_setup import LoggingService @@ -32,7 +33,7 @@ def __init__(self, custom_aws_role=None): max_pool_connections=20, ) self.presigned_url_expiry = 1800 - self.client = boto3.client("s3", config=self.config) + self.client: S3Client = boto3.client("s3", config=self.config) self.initialised = True self.custom_client = None self.custom_aws_role = custom_aws_role @@ -130,6 +131,22 @@ def delete_object(self, s3_bucket_name: str, file_key: str, version_id: str | No return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key, VersionId=version_id) + def copy_across_bucket_if_none_match( + self, + source_bucket: str, + source_file_key: str, + dest_bucket: str, + dest_file_key: str, + if_none_match: str, + ): + return self.client.copy_object( + Bucket=dest_bucket, + Key=dest_file_key, + CopySource={"Bucket": source_bucket, "Key": source_file_key}, + IfNoneMatch=if_none_match, + StorageClass="INTELLIGENT_TIERING", + ) + def create_object_tag( self, s3_bucket_name: str, file_key: str, tag_key: str, tag_value: str ): diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 665b456e5..fe26f498e 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -1,20 +1,13 @@ import os from datetime import datetime, timezone -import uuid from enums.review_status import ReviewStatus +from models.document_reference import DocumentReferenceMetadataFields from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service from utils.audit_logging_setup import LoggingService -from utils.exceptions import ( - ReviewProcessCreateRecordException, - ReviewProcessDeleteException, - ReviewProcessMovingException, - ReviewProcessVerifyingException, - S3FileNotFoundException -) from utils.request_context import request_context logger = LoggingService(__name__) @@ -34,6 +27,7 @@ def __init__(self): self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] + def process_review_message(self, review_message: ReviewMessageBody) -> None: """ Process a single SQS message from the review queue. @@ -46,58 +40,25 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: S3FileNotFoundException: If file doesn't exist in staging bucket ClientError: For AWS service errors (DynamoDB, S3) """ + logger.info("Processing review queue message") request_context.patient_nhs_no = review_message.nhs_number - logger.info(f"Processing review for NHS: {review_message.nhs_number} with {len(review_message.files)} files") - - for file in review_message.files: - logger.info(f"Processing review file: {file.file_name}") - self._verify_file_exists_in_staging(file.file_path) - - review_id = str(uuid.uuid4) - files = self._move_files_to_review_bucket(review_message, review_id) - document_upload_review = self._build_review_record(review_message, review_id, files) - - self._create_review_record(document_upload_review) - - logger.info( - f"Successfully processed review for {review_message.nhs_number}", - {"Result": "Review record created and file moved"}, + review_id = review_message.upload_id + review_files = self._move_files_to_review_bucket(review_message, review_id) + document_upload_review = self._build_review_record(review_message, review_id, review_files) + self.dynamo_service.put_item( + table_name=self.review_table_name, + item=document_upload_review.model_dump(by_alias=True, exclude_none=True), + key_name=DocumentReferenceMetadataFields.ID.value ) - def _verify_file_exists_in_staging(self, file_path: str) -> None: - """ - Verify the file exists in the staging bucket. - - Args: - file_path: S3 key of the file in staging bucket - - Raises: - S3FileNotFoundException: If file does not exist in staging bucket - """ - try: - file_exists = self.s3_service.file_exist_on_s3(s3_bucket_name=self.staging_bucket_name, file_key=file_path) - - if not file_exists: - raise S3FileNotFoundException(f"File not found in staging bucket: {file_path}") - - logger.info(f"Verified file exists in staging: {file_path}") - - except S3FileNotFoundException as e: - logger.info(e) - logger.info( - f"File not found in staging bucket {self.staging_bucket_name} for file_path {file_path}" - ) - - raise - except Exception as e: - logger.error(f"Error checking file in staging bucket: {str(e)}") - raise ReviewProcessVerifyingException(f"Error checking file in staging bucket: {str(e)}") + logger.info(f"Created review record {document_upload_review.id}") + self._delete_files_from_staging(review_message) def _build_review_record( - self, message_data: ReviewMessageBody, review_id: str, files: list[DocumentReviewFileDetails] + self, message_data: ReviewMessageBody, review_id: str, review_files: list[DocumentReviewFileDetails] ) -> DocumentsUploadReview: return DocumentsUploadReview( id=review_id, @@ -106,7 +67,7 @@ def _build_review_record( review_reason=message_data.failure_reason, author=message_data.uploader_ods, custodian=message_data.current_gp, - files=files, + files=review_files, upload_date=int(datetime.now(tz=timezone.utc).timestamp()) ) @@ -121,60 +82,40 @@ def _move_files_to_review_bucket( review_record_id: ID of the review record being created Returns: - List of DocumentReviewFileDetails objects for the moved files + List of DocumentReviewFileDetails with new file locations in review bucket """ - moved_files = [] - try: - for file in message_data.files: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" - - logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") - - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=file.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - ) - - logger.info("File successfully copied to review bucket") - logger.info(f"Deleting file from staging bucket: {file.file_path}") - - self._delete_from_staging(file.file_path) - logger.info(f"Successfully moved file to: {new_file_key}") + new_file_keys: list[DocumentReviewFileDetails] = [] + for file in message_data.files: + new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + + logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") + + self.s3_service.copy_across_bucket_if_none_match( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + if_none_match="*", + ) - moved_files.append(DocumentReviewFileDetails( + new_file_keys.append( + DocumentReviewFileDetails( file_name=file.file_name, - file_location=new_file_key - )) - - except Exception as e: - logger.error(f"Failed to move file: {str(e)}") - raise ReviewProcessMovingException(f"Failed to move file: {str(e)}") - - return moved_files - - def _delete_from_staging(self, file_key: str) -> None: - try: - self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file_key) - - logger.info(f"Deleted file from staging bucket: {file_key}") + file_location=new_file_key, + ) + ) - except Exception as e: - logger.error(f"Error deleting file from staging: {str(e)}") - raise ReviewProcessDeleteException(f"Error deleting file from staging: {str(e)}") + logger.info("File successfully copied to review bucket") + logger.info(f"Successfully moved file to: {new_file_key}") + return new_file_keys - def _create_review_record(self, review_record: DocumentsUploadReview) -> None: - try: - self.dynamo_service.create_item( - table_name=self.review_table_name, - item=review_record.model_dump(by_alias=True, exclude_none=True) - ) + def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: + for file in message_data.files: + try: + logger.info(f"Deleting file from staging bucket: {file.file_path}") + self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file.file_path) + except Exception as e: + logger.error(f"Error deleting files from staging: {str(e)}") + # Continue processing as files - logger.info(f"Created review record {review_record.id}") - except Exception as e: - logger.error(f"Failed to create review record with id: {review_record.id} -- {str(e)}") - raise ReviewProcessCreateRecordException( - f"Failed to create review record with id: {review_record.id} -- {str(e)}" - ) diff --git a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py index b652f0306..53521de91 100644 --- a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py @@ -19,6 +19,7 @@ def mock_review_service(mocker): def sample_review_message_body(): """Create a sample review message body.""" return ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="test_document.pdf", @@ -53,6 +54,7 @@ def sample_sqs_event(sample_sqs_message): def sample_sqs_event_multiple_messages(sample_review_message_body): """Create a sample SQS event with multiple messages.""" message_1 = ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -67,6 +69,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_2 = ReviewMessageBody( + upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_2.pdf", @@ -81,6 +84,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): ) message_3 = ReviewMessageBody( + upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_3.pdf", @@ -121,16 +125,9 @@ def empty_sqs_event(): return {"Records": []} -@pytest.fixture -def set_review_env(monkeypatch): - """Set up environment variables required for the handler.""" - monkeypatch.setenv("DOCUMENT_REVIEW_DYNAMODB_NAME", "test_review_table") - monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", "test_staging_bucket") - monkeypatch.setenv("PENDING_REVIEW_BUCKET_NAME", "test_review_bucket") - def test_lambda_handler_processes_single_message_successfully( - set_review_env, + set_env, context, sample_sqs_event, mock_review_service, @@ -142,7 +139,7 @@ def test_lambda_handler_processes_single_message_successfully( def test_lambda_handler_processes_multiple_messages_successfully( - set_review_env, + set_env, context, sample_sqs_event_multiple_messages, mock_review_service, @@ -154,7 +151,7 @@ def test_lambda_handler_processes_multiple_messages_successfully( def test_lambda_handler_calls_service_with_correct_message( - set_review_env, + set_env, context, sample_sqs_event, mock_review_service, @@ -174,7 +171,7 @@ def test_lambda_handler_calls_service_with_correct_message( def test_lambda_handler_handles_empty_records_list( - set_review_env, context, empty_sqs_event, mock_review_service + set_env, context, empty_sqs_event, mock_review_service ): """Test handler handles empty records list gracefully.""" lambda_handler(empty_sqs_event, context) @@ -183,7 +180,7 @@ def test_lambda_handler_handles_empty_records_list( def test_lambda_handler_parses_json_body_correctly( - set_review_env, + set_env, context, mock_review_service, ): @@ -193,6 +190,7 @@ def test_lambda_handler_parses_json_body_correctly( { "body": json.dumps( { + "upload_id": "test-upload-id-123", "files": [ { "file_name": "test.pdf", @@ -219,3 +217,22 @@ def test_lambda_handler_parses_json_body_correctly( assert len(call_args.files) == 1 assert call_args.files[0].file_name == "test.pdf" + +def test_lambda_handler_calls_process_review_message_on_service( + set_env, + context, + sample_sqs_event, + sample_review_message_body, + mock_review_service, +): + """Test that handler calls process_review_message method on ReviewProcessorService.""" + lambda_handler(sample_sqs_event, context) + + mock_review_service.process_review_message.assert_called_once() + called_message = mock_review_service.process_review_message.call_args[0][0] + + assert isinstance(called_message, ReviewMessageBody) + assert called_message.upload_id == sample_review_message_body.upload_id + assert called_message.nhs_number == sample_review_message_body.nhs_number + assert called_message.failure_reason == sample_review_message_body.failure_reason + diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index ce9594ad6..d7c6cc9b7 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -292,6 +292,86 @@ def test_create_item_raise_client_error(mock_service, mock_table): assert MOCK_CLIENT_ERROR == actual_response.value +def test_put_item_with_check_exists_true(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=True) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': True + } + } + ) + + +def test_put_item_with_check_exists_false(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=False) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': False + } + } + ) + + +def test_put_item_default_check_exists(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + # Test that default value is True + mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.put_item.assert_called_once_with( + Item=item, + Expected={ + key_name: { + 'Exists': True + } + } + ) + + +def test_put_item_returns_response(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + expected_response = { + 'ResponseMetadata': { + 'HTTPStatusCode': 200 + } + } + + mock_table.return_value.put_item.return_value = expected_response + + actual_response = mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + assert actual_response == expected_response + + +def test_put_item_raises_client_error(mock_service, mock_table): + item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} + key_name = "NhsNumber" + + mock_table.return_value.put_item.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as actual_response: + mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + + assert MOCK_CLIENT_ERROR == actual_response.value + + def test_delete_item_is_called_with_correct_parameters(mock_service, mock_table): mock_service.delete_item(MOCK_TABLE_NAME, {"NhsNumber": TEST_NHS_NUMBER}) diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index 3d3d8b4ed..ef68ddaff 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -139,6 +139,25 @@ def test_copy_across_bucket(mock_service, mock_client): ) +def test_copy_across_bucket_if_none_match(mock_service, mock_client): + test_etag = '"abc123def456"' + + mock_service.copy_across_bucket_if_none_match( + source_bucket="bucket_to_copy_from", + source_file_key=TEST_FILE_KEY, + dest_bucket="bucket_to_copy_to", + dest_file_key=f"{TEST_NHS_NUMBER}/{TEST_UUID}", + if_none_match=test_etag, + ) + + mock_client.copy_object.assert_called_once_with( + Bucket="bucket_to_copy_to", + Key=f"{TEST_NHS_NUMBER}/{TEST_UUID}", + CopySource={"Bucket": "bucket_to_copy_from", "Key": TEST_FILE_KEY}, + IfNoneMatch=test_etag, + StorageClass="INTELLIGENT_TIERING", + ) + def test_delete_object(mock_service, mock_client): mock_service.delete_object(s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 38e177151..7a323bb87 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -4,11 +4,10 @@ from models.document_review import DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.document_review_processor_service import ( - ReviewProcessMovingException, - ReviewProcessVerifyingException, ReviewProcessorService, ) -from utils.exceptions import ReviewProcessDeleteException, S3FileNotFoundException +from utils.exceptions import S3FileNotFoundException +from models.document_review import DocumentReviewFileDetails @pytest.fixture @@ -34,6 +33,7 @@ def service_under_test(set_env, mock_dynamo_service, mock_s3_service): def sample_review_message(): """Create a sample review message.""" return ReviewMessageBody( + upload_id="test-upload-id-123", files=[ ReviewMessageFile( file_name="test_document.pdf", @@ -71,23 +71,25 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - mock_verify = mocker.patch.object( - service_under_test, "_verify_file_exists_in_staging" - ) mock_move = mocker.patch.object( service_under_test, "_move_files_to_review_bucket" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_delete = mocker.patch.object( + service_under_test, "_delete_files_from_staging" + ) mock_move.return_value = [ - {"file_name": "test_document.pdf", "file_location": "9000000009/test-review-id/test_document.pdf"} + DocumentReviewFileDetails( + file_name="test_document.pdf", + file_location="9000000009/test-upload-id-123/test_document.pdf" + ) ] service_under_test.process_review_message(sample_review_message) - mock_verify.assert_called_once_with(sample_review_message.files[0].file_path) mock_move.assert_called_once() - mock_create.assert_called_once() + service_under_test.dynamo_service.put_item.assert_called_once() + mock_delete.assert_called_once_with(sample_review_message) def test_process_review_message_multiple_files( @@ -95,6 +97,7 @@ def test_process_review_message_multiple_files( ): """Test successful processing of a review message with multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -112,46 +115,36 @@ def test_process_review_message_multiple_files( current_gp="Y12345", ) - mock_verify = mocker.patch.object( - service_under_test, "_verify_file_exists_in_staging" - ) mock_move = mocker.patch.object( service_under_test, "_move_files_to_review_bucket" ) - mock_create = mocker.patch.object(service_under_test, "_create_review_record") + mock_delete = mocker.patch.object( + service_under_test, "_delete_files_from_staging" + ) mock_move.return_value = [ - {"file_name": "document_1.pdf", "file_location": "9000000009/test-review-id/document_1.pdf"}, - {"file_name": "document_2.pdf", "file_location": "9000000009/test-review-id/document_2.pdf"} + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-upload-id-456/document_1.pdf" + ), + DocumentReviewFileDetails( + file_name="document_2.pdf", + file_location="9000000009/test-upload-id-456/document_2.pdf" + ) ] service_under_test.process_review_message(message) - assert mock_verify.call_count == 2 mock_move.assert_called_once() - mock_create.assert_called_once() - - - -def test_process_review_message_file_not_found( - service_under_test, sample_review_message, mocker -): - """Test processing fails when file doesn't exist in staging.""" - mocker.patch.object( - service_under_test, - "_verify_file_exists_in_staging", - side_effect=S3FileNotFoundException("File not found"), - ) + service_under_test.dynamo_service.put_item.assert_called_once() + mock_delete.assert_called_once_with(message) - with pytest.raises(S3FileNotFoundException, match="File not found"): - service_under_test.process_review_message(sample_review_message) def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): """Test processing fails when S3 copy operation fails.""" - mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") mocker.patch.object( service_under_test, "_move_files_to_review_bucket", @@ -168,23 +161,17 @@ def test_process_review_message_s3_copy_error( def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): - """Test processing fails when DynamoDB create fails.""" - mocker.patch.object(service_under_test, "_verify_file_exists_in_staging") + """Test processing fails when DynamoDB put fails.""" mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) - mocker.patch.object( - service_under_test, - "_create_review_record", - side_effect=ClientError( - {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, - "PutItem", - ), + service_under_test.dynamo_service.put_item.side_effect = ClientError( + {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, + "PutItem", ) with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) - # Tests for _verify_file_exists_in_staging method @@ -224,9 +211,7 @@ def test_verify_file_s3_error(service_under_test): def test_build_review_record_success(service_under_test, sample_review_message): - """Test successful building of review record.""" - from models.document_review import DocumentReviewFileDetails - + """Test successful building of review record.""" files = [ DocumentReviewFileDetails( file_name="test_document.pdf", @@ -251,10 +236,9 @@ def test_build_review_record_success(service_under_test, sample_review_message): def test_build_review_record_with_multiple_files(service_under_test): - """Test building review record with multiple files.""" - from models.document_review import DocumentReviewFileDetails - + """Test building review record with multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -290,44 +274,11 @@ def test_build_review_record_with_multiple_files(service_under_test): assert result.files[1].file_name == "document_2.pdf" -def test_create_review_record_success(service_under_test, sample_review_message): - """Test successful creation of review record in DynamoDB.""" - from models.document_review import DocumentReviewFileDetails - - review_record = DocumentsUploadReview( - id="test-review-id", - nhs_number="9000000009", - review_status=ReviewStatus.PENDING_REVIEW, - review_reason="Failed virus scan", - author="Y12345", - custodian="Y12345", - files=[ - DocumentReviewFileDetails( - file_name="test_document.pdf", - file_location="9000000009/test-review-id/test_document.pdf" - ) - ], - upload_date=1705319400 - ) - - service_under_test.dynamo_service.create_item.return_value = None - - service_under_test._create_review_record(review_record) - - service_under_test.dynamo_service.create_item.assert_called_once() - call_args = service_under_test.dynamo_service.create_item.call_args - assert call_args[1]["table_name"] == "test_review_table" - assert call_args[1]["item"] == review_record - - - # Tests for _move_files_to_review_bucket method -def test_move_files_success(service_under_test, sample_review_message, mocker): +def test_move_files_success(service_under_test, sample_review_message): """Test successful file move from staging to review bucket.""" - mocker.patch.object(service_under_test, "_delete_from_staging") - files = service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id-123" ) @@ -338,21 +289,19 @@ def test_move_files_success(service_under_test, sample_review_message, mocker): assert files[0].file_name == "test_document.pdf" assert files[0].file_location == expected_key - service_under_test.s3_service.copy_across_bucket.assert_called_once_with( + service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once_with( source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", dest_file_key=expected_key, - ) - - service_under_test._delete_from_staging.assert_called_once_with( - "staging/9000000009/test_document.pdf" + if_none_match="*", ) -def test_move_multiple_files_success(service_under_test, mocker): +def test_move_multiple_files_success(service_under_test): """Test successful move of multiple files.""" message = ReviewMessageBody( + upload_id="test-upload-id-999", files=[ ReviewMessageFile( file_name="document_1.pdf", @@ -369,8 +318,6 @@ def test_move_multiple_files_success(service_under_test, mocker): uploader_ods="Y12345", current_gp="Y12345", ) - - mocker.patch.object(service_under_test, "_delete_from_staging") files = service_under_test._move_files_to_review_bucket(message, "test-review-id") @@ -380,106 +327,75 @@ def test_move_multiple_files_success(service_under_test, mocker): assert files[1].file_name == "document_2.pdf" assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" - assert service_under_test.s3_service.copy_across_bucket.call_count == 2 - assert service_under_test._delete_from_staging.call_count == 2 + assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 2 -def test_move_files_copy_error(service_under_test, sample_review_message, mocker): +def test_move_files_copy_error(service_under_test, sample_review_message): """Test file move handles S3 copy errors.""" - service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + service_under_test.s3_service.copy_across_bucket_if_none_match.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, "CopyObject", ) - with pytest.raises(ReviewProcessMovingException): - service_under_test._move_files_to_review_bucket( - sample_review_message, "test-review-id" - ) - - -def test_move_files_delete_error(service_under_test, sample_review_message, mocker): - """Test file move handles delete errors.""" - mocker.patch.object( - service_under_test, - "_delete_from_staging", - side_effect=ClientError( - {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, - "DeleteObject", - ), - ) - - with pytest.raises(ReviewProcessMovingException): + with pytest.raises(ClientError): service_under_test._move_files_to_review_bucket( sample_review_message, "test-review-id" ) -# Tests for _delete_from_staging method +# Tests for _delete_files_from_staging method -def test_delete_from_staging_success(service_under_test): +def test_delete_from_staging_success(service_under_test, sample_review_message): """Test successful deletion from staging bucket.""" - service_under_test._delete_from_staging("staging/test.pdf") + service_under_test._delete_files_from_staging(sample_review_message) service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" + s3_bucket_name="test_staging_bulk_store", file_key="staging/9000000009/test_document.pdf" ) -def test_delete_from_staging_error(service_under_test): - """Test delete from staging handles S3 errors.""" +def test_delete_from_staging_handles_errors(service_under_test, sample_review_message): + """Test deletion from staging handles errors gracefully.""" service_under_test.s3_service.delete_object.side_effect = ClientError( - {"Error": {"Code": "NoSuchKey", "Message": "Key does not exist"}}, + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, "DeleteObject", ) - with pytest.raises(ReviewProcessDeleteException): - service_under_test._delete_from_staging("staging/test.pdf") + # Should not raise exception - errors are caught and logged + service_under_test._delete_files_from_staging(sample_review_message) + + service_under_test.s3_service.delete_object.assert_called_once() # Integration scenario tests def test_full_workflow_with_valid_message( - service_under_test, sample_review_message, mocker + service_under_test, sample_review_message ): """Test complete workflow from message to final record creation.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - service_under_test.dynamo_service.create_item.return_value = None - service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.dynamo_service.put_item.return_value = None + service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) - service_under_test.s3_service.file_exist_on_s3.assert_called_once() - service_under_test.dynamo_service.create_item.assert_called_once() - service_under_test.s3_service.copy_across_bucket.assert_called_once() + service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() -def test_workflow_stops_at_verification_failure( - service_under_test, sample_review_message -): - """Test workflow stops if file verification fails.""" - service_under_test.s3_service.file_exist_on_s3.return_value = False - - with pytest.raises(S3FileNotFoundException): - service_under_test.process_review_message(sample_review_message) - - service_under_test.dynamo_service.create_item.assert_not_called() - service_under_test.s3_service.copy_across_bucket.assert_not_called() - - -def test_workflow_handles_multiple_different_patients(service_under_test, mocker): +def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - service_under_test.dynamo_service.create_item.return_value = None - service_under_test.s3_service.copy_across_bucket.return_value = None + service_under_test.dynamo_service.put_item.return_value = None + service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None service_under_test.s3_service.delete_object.return_value = None messages = [ ReviewMessageBody( + upload_id=f"test-upload-id-{i}", files=[ ReviewMessageFile( file_name=f"doc_{i}.pdf", @@ -498,5 +414,6 @@ def test_workflow_handles_multiple_different_patients(service_under_test, mocker for message in messages: service_under_test.process_review_message(message) - assert service_under_test.dynamo_service.create_item.call_count == 3 - assert service_under_test.s3_service.copy_across_bucket.call_count == 3 + assert service_under_test.dynamo_service.put_item.call_count == 3 + assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 3 + From 300ab46cb958ed142e5faa46c4958720e1052177 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 14:56:14 +0000 Subject: [PATCH 29/47] fix test --- .../test_document_review_processor_service.py | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 7a323bb87..65425ec5d 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -171,42 +171,6 @@ def test_process_review_message_dynamo_error( with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) - -# Tests for _verify_file_exists_in_staging method - - -def test_verify_file_exists_success(service_under_test): - """Test successful file verification.""" - service_under_test.s3_service.file_exist_on_s3.return_value = True - - service_under_test._verify_file_exists_in_staging("staging/test.pdf") - - service_under_test.s3_service.file_exist_on_s3.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/test.pdf" - ) - - -def test_verify_file_does_not_exist(service_under_test): - """Test file verification fails when file doesn't exist.""" - service_under_test.s3_service.file_exist_on_s3.return_value = False - - with pytest.raises( - S3FileNotFoundException, match="File not found in staging bucket" - ): - service_under_test._verify_file_exists_in_staging("staging/missing.pdf") - - -def test_verify_file_s3_error(service_under_test): - """Test file verification handles S3 errors.""" - service_under_test.s3_service.file_exist_on_s3.side_effect = ClientError( - {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, - "HeadObject", - ) - - with pytest.raises(ReviewProcessVerifyingException): - service_under_test._verify_file_exists_in_staging("staging/test.pdf") - - # Tests for _build_review_record and _create_review_record methods From 15daf3fceda96c3865d7424925472af41dfb8ff2 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 3 Nov 2025 15:52:58 +0000 Subject: [PATCH 30/47] [PRMP-585] remove unsed import --- .../unit/services/test_document_review_processor_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 65425ec5d..c36f13f48 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -6,7 +6,6 @@ from services.document_review_processor_service import ( ReviewProcessorService, ) -from utils.exceptions import S3FileNotFoundException from models.document_review import DocumentReviewFileDetails From ab75868f26fb631ce5073d8b272023a754e68a62 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 16:51:52 +0000 Subject: [PATCH 31/47] Code comment changes --- .../document_review_processor_handler.py | 17 +-- lambdas/services/base/s3_service.py | 27 ++--- .../document_review_processor_service.py | 30 +++-- .../unit/services/base/test_s3_service.py | 31 +++-- .../test_document_review_processor_service.py | 114 ++++++++---------- 5 files changed, 105 insertions(+), 114 deletions(-) diff --git a/lambdas/handlers/document_review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py index 84cf6b275..43a33d055 100644 --- a/lambdas/handlers/document_review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -1,7 +1,5 @@ -import json - -from pydantic import ValidationError from models.sqs.review_message_body import ReviewMessageBody +from pydantic import ValidationError from services.document_review_processor_service import ReviewProcessorService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -39,16 +37,11 @@ def lambda_handler(event, context): review_service = ReviewProcessorService() for sqs_message in sqs_messages: - message: ReviewMessageBody | None = None try: - sqs_message_body = json.loads(sqs_message["body"]) - message = ReviewMessageBody.model_validate(sqs_message_body) + message = ReviewMessageBody.model_validate_json(sqs_message["body"]) + + review_service.process_review_message(message) - if isinstance(message, ReviewMessageBody): - review_service.process_review_message(message) - else: - raise ValidationError("Invalid review message format") - except ValidationError as error: logger.error("Malformed review message") logger.error(error) @@ -60,5 +53,5 @@ def lambda_handler(event, context): {"Result": "Review processing failed"}, ) raise error - + logger.info("Continuing to next message.") diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index d94afc5de..e45bca015 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -5,9 +5,9 @@ import boto3 from botocore.client import Config as BotoConfig -from types_boto3_s3 import S3Client from botocore.exceptions import ClientError from services.base.iam_service import IAMService +from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException @@ -117,7 +117,16 @@ def copy_across_bucket( source_file_key: str, dest_bucket: str, dest_file_key: str, + if_none_match: str | None = None, ): + if if_none_match is not None: + return self.client.copy_object( + Bucket=dest_bucket, + Key=dest_file_key, + CopySource={"Bucket": source_bucket, "Key": source_file_key}, + IfNoneMatch=if_none_match, + StorageClass="INTELLIGENT_TIERING", + ) return self.client.copy_object( Bucket=dest_bucket, Key=dest_file_key, @@ -131,22 +140,6 @@ def delete_object(self, s3_bucket_name: str, file_key: str, version_id: str | No return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key, VersionId=version_id) - def copy_across_bucket_if_none_match( - self, - source_bucket: str, - source_file_key: str, - dest_bucket: str, - dest_file_key: str, - if_none_match: str, - ): - return self.client.copy_object( - Bucket=dest_bucket, - Key=dest_file_key, - CopySource={"Bucket": source_bucket, "Key": source_file_key}, - IfNoneMatch=if_none_match, - StorageClass="INTELLIGENT_TIERING", - ) - def create_object_tag( self, s3_bucket_name: str, file_key: str, tag_key: str, tag_value: str ): diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index fe26f498e..8dfa0ad5b 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -27,7 +27,6 @@ def __init__(self): self.staging_bucket_name = os.environ["STAGING_STORE_BUCKET_NAME"] self.review_bucket_name = os.environ["PENDING_REVIEW_BUCKET_NAME"] - def process_review_message(self, review_message: ReviewMessageBody) -> None: """ Process a single SQS message from the review queue. @@ -47,18 +46,23 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: review_id = review_message.upload_id review_files = self._move_files_to_review_bucket(review_message, review_id) - document_upload_review = self._build_review_record(review_message, review_id, review_files) + document_upload_review = self._build_review_record( + review_message, review_id, review_files + ) self.dynamo_service.put_item( table_name=self.review_table_name, item=document_upload_review.model_dump(by_alias=True, exclude_none=True), - key_name=DocumentReferenceMetadataFields.ID.value + key_name=DocumentReferenceMetadataFields.ID.value, ) logger.info(f"Created review record {document_upload_review.id}") self._delete_files_from_staging(review_message) def _build_review_record( - self, message_data: ReviewMessageBody, review_id: str, review_files: list[DocumentReviewFileDetails] + self, + message_data: ReviewMessageBody, + review_id: str, + review_files: list[DocumentReviewFileDetails], ) -> DocumentsUploadReview: return DocumentsUploadReview( id=review_id, @@ -68,7 +72,7 @@ def _build_review_record( author=message_data.uploader_ods, custodian=message_data.current_gp, files=review_files, - upload_date=int(datetime.now(tz=timezone.utc).timestamp()) + upload_date=int(datetime.now(tz=timezone.utc).timestamp()), ) def _move_files_to_review_bucket( @@ -86,11 +90,15 @@ def _move_files_to_review_bucket( """ new_file_keys: list[DocumentReviewFileDetails] = [] for file in message_data.files: - new_file_key = f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + new_file_key = ( + f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" + ) - logger.info(f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}") + logger.info( + f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}" + ) - self.s3_service.copy_across_bucket_if_none_match( + self.s3_service.copy_across_bucket( source_bucket=self.staging_bucket_name, source_file_key=file.file_path, dest_bucket=self.review_bucket_name, @@ -113,9 +121,9 @@ def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: for file in message_data.files: try: logger.info(f"Deleting file from staging bucket: {file.file_path}") - self.s3_service.delete_object(s3_bucket_name=self.staging_bucket_name, file_key=file.file_path) + self.s3_service.delete_object( + s3_bucket_name=self.staging_bucket_name, file_key=file.file_path + ) except Exception as e: logger.error(f"Error deleting files from staging: {str(e)}") # Continue processing as files - - diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index ef68ddaff..ead525f00 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -141,8 +141,8 @@ def test_copy_across_bucket(mock_service, mock_client): def test_copy_across_bucket_if_none_match(mock_service, mock_client): test_etag = '"abc123def456"' - - mock_service.copy_across_bucket_if_none_match( + + mock_service.copy_across_bucket( source_bucket="bucket_to_copy_from", source_file_key=TEST_FILE_KEY, dest_bucket="bucket_to_copy_to", @@ -158,6 +158,7 @@ def test_copy_across_bucket_if_none_match(mock_service, mock_client): StorageClass="INTELLIGENT_TIERING", ) + def test_delete_object(mock_service, mock_client): mock_service.delete_object(s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME) @@ -530,30 +531,38 @@ def test_get_head_object_returns_metadata(mock_service, mock_client): result = mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) assert result == mock_response - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) -def test_get_head_object_raises_client_error_when_object_not_found(mock_service, mock_client): +def test_get_head_object_raises_client_error_when_object_not_found( + mock_service, mock_client +): mock_error = ClientError( - {"Error": {"Code": "404", "Message": "Not Found"}}, - "HeadObject" + {"Error": {"Code": "404", "Message": "Not Found"}}, "HeadObject" ) mock_client.head_object.side_effect = mock_error with pytest.raises(ClientError): mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) -def test_get_head_object_raises_client_error_on_access_denied(mock_service, mock_client): +def test_get_head_object_raises_client_error_on_access_denied( + mock_service, mock_client +): mock_error = ClientError( - {"Error": {"Code": "403", "Message": "Forbidden"}}, - "HeadObject" + {"Error": {"Code": "403", "Message": "Forbidden"}}, "HeadObject" ) mock_client.head_object.side_effect = mock_error with pytest.raises(ClientError): mock_service.get_head_object(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) - mock_client.head_object.assert_called_once_with(Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY) + mock_client.head_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index c36f13f48..d8b146678 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -1,12 +1,9 @@ import pytest from botocore.exceptions import ClientError from enums.review_status import ReviewStatus -from models.document_review import DocumentsUploadReview +from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile -from services.document_review_processor_service import ( - ReviewProcessorService, -) -from models.document_review import DocumentReviewFileDetails +from services.document_review_processor_service import ReviewProcessorService @pytest.fixture @@ -36,7 +33,7 @@ def sample_review_message(): files=[ ReviewMessageFile( file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf" + file_path="staging/9000000009/test_document.pdf", ) ], nhs_number="9000000009", @@ -70,17 +67,13 @@ def test_process_review_message_success( service_under_test, sample_review_message, mocker ): """Test successful processing of a review message.""" - mock_move = mocker.patch.object( - service_under_test, "_move_files_to_review_bucket" - ) - mock_delete = mocker.patch.object( - service_under_test, "_delete_files_from_staging" - ) + mock_move = mocker.patch.object(service_under_test, "_move_files_to_review_bucket") + mock_delete = mocker.patch.object(service_under_test, "_delete_files_from_staging") mock_move.return_value = [ DocumentReviewFileDetails( file_name="test_document.pdf", - file_location="9000000009/test-upload-id-123/test_document.pdf" + file_location="9000000009/test-upload-id-123/test_document.pdf", ) ] @@ -91,21 +84,19 @@ def test_process_review_message_success( mock_delete.assert_called_once_with(sample_review_message) -def test_process_review_message_multiple_files( - service_under_test, mocker -): +def test_process_review_message_multiple_files(service_under_test, mocker): """Test successful processing of a review message with multiple files.""" message = ReviewMessageBody( upload_id="test-upload-id-456", files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -113,23 +104,19 @@ def test_process_review_message_multiple_files( uploader_ods="Y12345", current_gp="Y12345", ) - - mock_move = mocker.patch.object( - service_under_test, "_move_files_to_review_bucket" - ) - mock_delete = mocker.patch.object( - service_under_test, "_delete_files_from_staging" - ) + + mock_move = mocker.patch.object(service_under_test, "_move_files_to_review_bucket") + mock_delete = mocker.patch.object(service_under_test, "_delete_files_from_staging") mock_move.return_value = [ DocumentReviewFileDetails( file_name="document_1.pdf", - file_location="9000000009/test-upload-id-456/document_1.pdf" + file_location="9000000009/test-upload-id-456/document_1.pdf", ), DocumentReviewFileDetails( file_name="document_2.pdf", - file_location="9000000009/test-upload-id-456/document_2.pdf" - ) + file_location="9000000009/test-upload-id-456/document_2.pdf", + ), ] service_under_test.process_review_message(message) @@ -139,7 +126,6 @@ def test_process_review_message_multiple_files( mock_delete.assert_called_once_with(message) - def test_process_review_message_s3_copy_error( service_under_test, sample_review_message, mocker ): @@ -161,7 +147,9 @@ def test_process_review_message_dynamo_error( service_under_test, sample_review_message, mocker ): """Test processing fails when DynamoDB put fails.""" - mocker.patch.object(service_under_test, "_move_files_to_review_bucket", return_value=[]) + mocker.patch.object( + service_under_test, "_move_files_to_review_bucket", return_value=[] + ) service_under_test.dynamo_service.put_item.side_effect = ClientError( {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, "PutItem", @@ -170,18 +158,19 @@ def test_process_review_message_dynamo_error( with pytest.raises(ClientError): service_under_test.process_review_message(sample_review_message) + # Tests for _build_review_record and _create_review_record methods def test_build_review_record_success(service_under_test, sample_review_message): - """Test successful building of review record.""" + """Test successful building of review record.""" files = [ DocumentReviewFileDetails( file_name="test_document.pdf", - file_location="9000000009/test-review-id/test_document.pdf" + file_location="9000000009/test-review-id/test_document.pdf", ) ] - + result = service_under_test._build_review_record( sample_review_message, "test-review-id", files ) @@ -195,22 +184,24 @@ def test_build_review_record_success(service_under_test, sample_review_message): assert result.custodian == "Y12345" assert len(result.files) == 1 assert result.files[0].file_name == "test_document.pdf" - assert result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + assert ( + result.files[0].file_location == "9000000009/test-review-id/test_document.pdf" + ) def test_build_review_record_with_multiple_files(service_under_test): - """Test building review record with multiple files.""" + """Test building review record with multiple files.""" message = ReviewMessageBody( upload_id="test-upload-id-789", files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -218,18 +209,18 @@ def test_build_review_record_with_multiple_files(service_under_test): uploader_ods="Y12345", current_gp="Y12345", ) - + files = [ DocumentReviewFileDetails( file_name="document_1.pdf", - file_location="9000000009/test-review-id/document_1.pdf" + file_location="9000000009/test-review-id/document_1.pdf", ), DocumentReviewFileDetails( file_name="document_2.pdf", - file_location="9000000009/test-review-id/document_2.pdf" - ) + file_location="9000000009/test-review-id/document_2.pdf", + ), ] - + result = service_under_test._build_review_record(message, "test-review-id", files) assert len(result.files) == 2 @@ -247,12 +238,12 @@ def test_move_files_success(service_under_test, sample_review_message): ) expected_key = "9000000009/test-review-id-123/test_document.pdf" - + assert len(files) == 1 assert files[0].file_name == "test_document.pdf" assert files[0].file_location == expected_key - service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once_with( + service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", dest_bucket="test_review_bucket", @@ -268,12 +259,12 @@ def test_move_multiple_files_success(service_under_test): files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ), ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000009/document_2.pdf" - ) + file_path="staging/9000000009/document_2.pdf", + ), ], nhs_number="9000000009", failure_reason="Failed virus scan", @@ -289,13 +280,13 @@ def test_move_multiple_files_success(service_under_test): assert files[0].file_location == "9000000009/test-review-id/document_1.pdf" assert files[1].file_name == "document_2.pdf" assert files[1].file_location == "9000000009/test-review-id/document_2.pdf" - - assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 2 + + assert service_under_test.s3_service.copy_across_bucket.call_count == 2 def test_move_files_copy_error(service_under_test, sample_review_message): """Test file move handles S3 copy errors.""" - service_under_test.s3_service.copy_across_bucket_if_none_match.side_effect = ClientError( + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Source not found"}}, "CopyObject", ) @@ -306,7 +297,6 @@ def test_move_files_copy_error(service_under_test, sample_review_message): ) - # Tests for _delete_files_from_staging method @@ -315,7 +305,8 @@ def test_delete_from_staging_success(service_under_test, sample_review_message): service_under_test._delete_files_from_staging(sample_review_message) service_under_test.s3_service.delete_object.assert_called_once_with( - s3_bucket_name="test_staging_bulk_store", file_key="staging/9000000009/test_document.pdf" + s3_bucket_name="test_staging_bulk_store", + file_key="staging/9000000009/test_document.pdf", ) @@ -335,25 +326,23 @@ def test_delete_from_staging_handles_errors(service_under_test, sample_review_me # Integration scenario tests -def test_full_workflow_with_valid_message( - service_under_test, sample_review_message -): +def test_full_workflow_with_valid_message(service_under_test, sample_review_message): """Test complete workflow from message to final record creation.""" service_under_test.dynamo_service.put_item.return_value = None - service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) service_under_test.dynamo_service.put_item.assert_called_once() - service_under_test.s3_service.copy_across_bucket_if_none_match.assert_called_once() + service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" service_under_test.dynamo_service.put_item.return_value = None - service_under_test.s3_service.copy_across_bucket_if_none_match.return_value = None + service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None messages = [ @@ -362,7 +351,7 @@ def test_workflow_handles_multiple_different_patients(service_under_test): files=[ ReviewMessageFile( file_name=f"doc_{i}.pdf", - file_path=f"staging/900000000{i}/doc_{i}.pdf" + file_path=f"staging/900000000{i}/doc_{i}.pdf", ) ], nhs_number=f"900000000{i}", @@ -378,5 +367,4 @@ def test_workflow_handles_multiple_different_patients(service_under_test): service_under_test.process_review_message(message) assert service_under_test.dynamo_service.put_item.call_count == 3 - assert service_under_test.s3_service.copy_across_bucket_if_none_match.call_count == 3 - + assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 2dc220c48a2a561afb028a010ece83ed216ba138 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 17:46:09 +0000 Subject: [PATCH 32/47] code comment changes --- lambdas/services/base/dynamo_service.py | 33 ++-- .../unit/services/base/test_dynamo_service.py | 148 +++++++----------- 2 files changed, 67 insertions(+), 114 deletions(-) diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 9ad9aeada..f9ec26885 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -from types_boto3_dynamodb.type_defs import PutItemOutputTableTypeDef +from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -12,7 +12,6 @@ create_update_expression, ) from utils.exceptions import DynamoServiceException -from types_boto3_dynamodb import DynamoDBServiceResource logger = LoggingService(__name__) @@ -28,7 +27,9 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb: DynamoDBServiceResource = boto3.resource("dynamodb", region_name="eu-west-2") + self.dynamodb: DynamoDBServiceResource = boto3.resource( + "dynamodb", region_name="eu-west-2" + ) self.initialised = True def get_table(self, table_name: str): @@ -80,26 +81,13 @@ def query_table( logger.error(str(e), {"Result": f"Unable to query table: {table_name}"}) raise e - def create_item(self, table_name, item): - try: - table = self.get_table(table_name) - logger.info(f"Writing item to table: {table_name}") - table.put_item(Item=item) - except ClientError as e: - logger.error( - str(e), {"Result": f"Unable to write item to table: {table_name}"} - ) - raise e - - def put_item(self, table_name, item, key_name, check_exists: bool = True) -> PutItemOutputTableTypeDef: + def create_item(self, table_name, item, key_name: str | None = None): """ Put an item into the specified DynamoDB table with a condition on the existence of the key. Args: table_name: Name of the DynamoDB table item: The item to be inserted (as a dictionary) - key_name: The name of the key field - check_exists: If True, the item will only be inserted if the key does not already exist. - If False, the item will only be inserted if the key already exists. + key_name: The name of the key field to check existance for conditional put Returns: Response from the DynamoDB put_item operation Raises: @@ -108,11 +96,10 @@ def put_item(self, table_name, item, key_name, check_exists: bool = True) -> Put try: table = self.get_table(table_name) logger.info(f"Writing item to table: {table_name}") - return table.put_item(Item=item, Expected={ - key_name: { - 'Exists': check_exists - } - }) + if key_name: + return table.put_item(Item=item, Expected={key_name: {"Exists": True}}) + else: + return table.put_item(Item=item) except ClientError as e: logger.error( str(e), {"Result": f"Unable to write item to table: {table_name}"} diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index d7c6cc9b7..ad1443740 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -292,82 +292,25 @@ def test_create_item_raise_client_error(mock_service, mock_table): assert MOCK_CLIENT_ERROR == actual_response.value -def test_put_item_with_check_exists_true(mock_service, mock_table): +def test_create_item_with_key_name(mock_service, mock_table): item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} key_name = "NhsNumber" - - mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=True) + mock_service.create_item(MOCK_TABLE_NAME, item, key_name) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': True - } - } - ) - - -def test_put_item_with_check_exists_false(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - - mock_service.put_item(MOCK_TABLE_NAME, item, key_name, check_exists=False) - - mock_table.assert_called_with(MOCK_TABLE_NAME) - mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': False - } - } - ) - - -def test_put_item_default_check_exists(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - - # Test that default value is True - mock_service.put_item(MOCK_TABLE_NAME, item, key_name) - - mock_table.assert_called_with(MOCK_TABLE_NAME) - mock_table.return_value.put_item.assert_called_once_with( - Item=item, - Expected={ - key_name: { - 'Exists': True - } - } + Item=item, Expected={key_name: {"Exists": True}} ) -def test_put_item_returns_response(mock_service, mock_table): +def test_create_item_raises_client_error(mock_service, mock_table): item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} key_name = "NhsNumber" - expected_response = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200 - } - } - - mock_table.return_value.put_item.return_value = expected_response - - actual_response = mock_service.put_item(MOCK_TABLE_NAME, item, key_name) - - assert actual_response == expected_response - -def test_put_item_raises_client_error(mock_service, mock_table): - item = {"NhsNumber": TEST_NHS_NUMBER, "Name": "Test Patient"} - key_name = "NhsNumber" - mock_table.return_value.put_item.side_effect = MOCK_CLIENT_ERROR with pytest.raises(ClientError) as actual_response: - mock_service.put_item(MOCK_TABLE_NAME, item, key_name) + mock_service.create_item(MOCK_TABLE_NAME, item, key_name) assert MOCK_CLIENT_ERROR == actual_response.value @@ -728,7 +671,7 @@ def test_update_item_with_condition_expression(mock_service, mock_table): update_key = {"ID": "9000000009"} condition_expression = "attribute_exists(FileName)" expression_attribute_values = {":expected_val": "expected_value"} - + expected_update_expression = "SET #FileName_attr = :FileName_val" expected_expr_attr_names = {"#FileName_attr": "FileName"} expected_expr_attr_values = { @@ -739,7 +682,9 @@ def test_update_item_with_condition_expression(mock_service, mock_table): mock_service.update_item( table_name=MOCK_TABLE_NAME, key_pair={"ID": TEST_NHS_NUMBER}, - updated_fields={DocumentReferenceMetadataFields.FILE_NAME.value: "test-filename"}, + updated_fields={ + DocumentReferenceMetadataFields.FILE_NAME.value: "test-filename" + }, condition_expression=condition_expression, expression_attribute_values=expression_attribute_values, ) @@ -761,14 +706,16 @@ def test_batch_writing_is_called_with_correct_parameters(mock_service, mock_tabl {"ID": "id2", "Name": "Item 2"}, {"ID": "id3", "Name": "Item 3"}, ] - - mock_batch_writer = mock_table.return_value.batch_writer.return_value.__enter__.return_value - + + mock_batch_writer = ( + mock_table.return_value.batch_writer.return_value.__enter__.return_value + ) + mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.batch_writer.assert_called_once() - + assert mock_batch_writer.put_item.call_count == 3 for item in items_to_write: mock_batch_writer.put_item.assert_any_call(Item=item) @@ -777,8 +724,10 @@ def test_batch_writing_is_called_with_correct_parameters(mock_service, mock_tabl def test_batch_writing_client_error_raises_exception(mock_service, mock_table): items_to_write = [{"ID": "id1", "Name": "Item 1"}] expected_response = MOCK_CLIENT_ERROR - - mock_table.return_value.batch_writer.return_value.__enter__.side_effect = MOCK_CLIENT_ERROR + + mock_table.return_value.batch_writer.return_value.__enter__.side_effect = ( + MOCK_CLIENT_ERROR + ) with pytest.raises(ClientError) as actual_response: mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) @@ -788,9 +737,11 @@ def test_batch_writing_client_error_raises_exception(mock_service, mock_table): def test_batch_writing_with_empty_list(mock_service, mock_table): items_to_write = [] - - mock_batch_writer = mock_table.return_value.batch_writer.return_value.__enter__.return_value - + + mock_batch_writer = ( + mock_table.return_value.batch_writer.return_value.__enter__.return_value + ) + mock_service.batch_writing(MOCK_TABLE_NAME, items_to_write) mock_table.assert_called_with(MOCK_TABLE_NAME) @@ -816,7 +767,7 @@ def test_transact_write_items_success(mock_service, mock_dynamo_service): } }, ] - + mock_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} mock_dynamo_service.meta.client.transact_write_items.return_value = mock_response @@ -837,9 +788,12 @@ def test_transact_write_items_transaction_cancelled(mock_service, mock_dynamo_se } } ] - + error_response = { - "Error": {"Code": "TransactionCanceledException", "Message": "Transaction cancelled"}, + "Error": { + "Code": "TransactionCanceledException", + "Message": "Transaction cancelled", + }, "CancellationReasons": [{"Code": "ConditionalCheckFailed"}], } mock_dynamo_service.meta.client.transact_write_items.side_effect = ClientError( @@ -861,7 +815,7 @@ def test_transact_write_items_generic_client_error(mock_service, mock_dynamo_ser } } ] - + mock_dynamo_service.meta.client.transact_write_items.side_effect = MOCK_CLIENT_ERROR with pytest.raises(ClientError) as exc_info: @@ -882,19 +836,29 @@ def test_build_update_transaction_item_single_condition(mock_service): assert "Update" in result update_item = result["Update"] - + assert update_item["TableName"] == table_name assert update_item["Key"] == document_key - assert "SET #FileName_attr = :FileName_val, #Deleted_attr = :Deleted_val" == update_item["UpdateExpression"] - assert update_item["ConditionExpression"] == "#DocStatus_attr = :DocStatus_condition_val" - + assert ( + "SET #FileName_attr = :FileName_val, #Deleted_attr = :Deleted_val" + == update_item["UpdateExpression"] + ) + assert ( + update_item["ConditionExpression"] + == "#DocStatus_attr = :DocStatus_condition_val" + ) + assert update_item["ExpressionAttributeNames"]["#FileName_attr"] == "FileName" assert update_item["ExpressionAttributeNames"]["#Deleted_attr"] == "Deleted" assert update_item["ExpressionAttributeNames"]["#DocStatus_attr"] == "DocStatus" - - assert update_item["ExpressionAttributeValues"][":FileName_val"] == "new_filename.pdf" + + assert ( + update_item["ExpressionAttributeValues"][":FileName_val"] == "new_filename.pdf" + ) assert update_item["ExpressionAttributeValues"][":Deleted_val"] == "" - assert update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + assert ( + update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + ) def test_build_update_transaction_item_multiple_conditions(mock_service): @@ -909,26 +873,28 @@ def test_build_update_transaction_item_multiple_conditions(mock_service): assert "Update" in result update_item = result["Update"] - + assert update_item["TableName"] == table_name assert update_item["Key"] == document_key - + # Check that all conditions are present (order might vary) condition_expr = update_item["ConditionExpression"] assert "#DocStatus_attr = :DocStatus_condition_val" in condition_expr assert "#Version_attr = :Version_condition_val" in condition_expr assert "#Uploaded_attr = :Uploaded_condition_val" in condition_expr assert condition_expr.count(" AND ") == 2 - + # Check all attribute names are present assert update_item["ExpressionAttributeNames"]["#FileName_attr"] == "FileName" assert update_item["ExpressionAttributeNames"]["#DocStatus_attr"] == "DocStatus" assert update_item["ExpressionAttributeNames"]["#Version_attr"] == "Version" assert update_item["ExpressionAttributeNames"]["#Uploaded_attr"] == "Uploaded" - + # Check all attribute values are present assert update_item["ExpressionAttributeValues"][":FileName_val"] == "updated.pdf" - assert update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + assert ( + update_item["ExpressionAttributeValues"][":DocStatus_condition_val"] == "final" + ) assert update_item["ExpressionAttributeValues"][":Version_condition_val"] == 1 assert update_item["ExpressionAttributeValues"][":Uploaded_condition_val"] is True @@ -945,8 +911,8 @@ def test_build_update_transaction_item_empty_condition_fields(mock_service): assert "Update" in result update_item = result["Update"] - + # With empty condition_fields, condition expression should be empty string assert update_item["ConditionExpression"] == "" assert update_item["TableName"] == table_name - assert update_item["Key"] == document_key \ No newline at end of file + assert update_item["Key"] == document_key From a94a6f3e2954e41f1278f42b5123b822daa21ac8 Mon Sep 17 00:00:00 2001 From: Lillie Dae Date: Mon, 10 Nov 2025 17:53:38 +0000 Subject: [PATCH 33/47] missed tests --- .../services/document_review_processor_service.py | 2 +- .../test_document_review_processor_service.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 8dfa0ad5b..91a785453 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -49,7 +49,7 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: document_upload_review = self._build_review_record( review_message, review_id, review_files ) - self.dynamo_service.put_item( + self.dynamo_service.create_item( table_name=self.review_table_name, item=document_upload_review.model_dump(by_alias=True, exclude_none=True), key_name=DocumentReferenceMetadataFields.ID.value, diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index d8b146678..c78fc8a2e 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -80,7 +80,7 @@ def test_process_review_message_success( service_under_test.process_review_message(sample_review_message) mock_move.assert_called_once() - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() mock_delete.assert_called_once_with(sample_review_message) @@ -122,7 +122,7 @@ def test_process_review_message_multiple_files(service_under_test, mocker): service_under_test.process_review_message(message) mock_move.assert_called_once() - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() mock_delete.assert_called_once_with(message) @@ -150,7 +150,7 @@ def test_process_review_message_dynamo_error( mocker.patch.object( service_under_test, "_move_files_to_review_bucket", return_value=[] ) - service_under_test.dynamo_service.put_item.side_effect = ClientError( + service_under_test.dynamo_service.create_item.side_effect = ClientError( {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, "PutItem", ) @@ -328,20 +328,20 @@ def test_delete_from_staging_handles_errors(service_under_test, sample_review_me def test_full_workflow_with_valid_message(service_under_test, sample_review_message): """Test complete workflow from message to final record creation.""" - service_under_test.dynamo_service.put_item.return_value = None + service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None service_under_test.process_review_message(sample_review_message) - service_under_test.dynamo_service.put_item.assert_called_once() + service_under_test.dynamo_service.create_item.assert_called_once() service_under_test.s3_service.copy_across_bucket.assert_called_once() service_under_test.s3_service.delete_object.assert_called_once() def test_workflow_handles_multiple_different_patients(service_under_test): """Test processing messages for different patients.""" - service_under_test.dynamo_service.put_item.return_value = None + service_under_test.dynamo_service.create_item.return_value = None service_under_test.s3_service.copy_across_bucket.return_value = None service_under_test.s3_service.delete_object.return_value = None @@ -366,5 +366,5 @@ def test_workflow_handles_multiple_different_patients(service_under_test): for message in messages: service_under_test.process_review_message(message) - assert service_under_test.dynamo_service.put_item.call_count == 3 + assert service_under_test.dynamo_service.create_item.call_count == 3 assert service_under_test.s3_service.copy_across_bucket.call_count == 3 From 6f58e8461a77239ccebac225dc0b478eb7ee50b4 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:08:52 +0000 Subject: [PATCH 34/47] [PRMP-585] bump boto3 version --- .../requirements/layers/requirements_core_lambda_layer.txt | 4 ++-- lambdas/services/base/dynamo_service.py | 4 ++-- lambdas/services/base/s3_service.py | 2 +- lambdas/tests/unit/services/base/test_dynamo_service.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lambdas/requirements/layers/requirements_core_lambda_layer.txt b/lambdas/requirements/layers/requirements_core_lambda_layer.txt index da90257d9..6728aaf74 100644 --- a/lambdas/requirements/layers/requirements_core_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_core_lambda_layer.txt @@ -1,7 +1,7 @@ PyJWT==2.8.0 PyYAML==6.0.1 -boto3==1.34.128 -botocore==1.34.128 +boto3==1.40.71 +botocore==1.40.71 charset-normalizer==3.2.0 cryptography==44.0.1 idna==3.7 diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index f9ec26885..055628f31 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -from types_boto3_dynamodb import DynamoDBServiceResource +# from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -97,7 +97,7 @@ def create_item(self, table_name, item, key_name: str | None = None): table = self.get_table(table_name) logger.info(f"Writing item to table: {table_name}") if key_name: - return table.put_item(Item=item, Expected={key_name: {"Exists": True}}) + return table.put_item(Item=item, ConditionExpression=f"attribute_not_exists({key_name})") else: return table.put_item(Item=item) except ClientError as e: diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index e45bca015..53250e2c8 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -7,7 +7,7 @@ from botocore.client import Config as BotoConfig from botocore.exceptions import ClientError from services.base.iam_service import IAMService -from types_boto3_s3 import S3Client +# from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index ad1443740..55541386e 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -299,7 +299,7 @@ def test_create_item_with_key_name(mock_service, mock_table): mock_service.create_item(MOCK_TABLE_NAME, item, key_name) mock_table.assert_called_with(MOCK_TABLE_NAME) mock_table.return_value.put_item.assert_called_once_with( - Item=item, Expected={key_name: {"Exists": True}} + Item=item, ConditionExpression=f"attribute_not_exists({key_name})" ) From 0247b4f1998c5abbc314dbacc6a57bc987da4b33 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:32:27 +0000 Subject: [PATCH 35/47] [PRMP-585] add types --- lambdas/services/base/dynamo_service.py | 2 +- lambdas/services/base/s3_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 055628f31..5f68ca68f 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,7 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -# from types_boto3_dynamodb import DynamoDBServiceResource +from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 53250e2c8..e45bca015 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -7,7 +7,7 @@ from botocore.client import Config as BotoConfig from botocore.exceptions import ClientError from services.base.iam_service import IAMService -# from types_boto3_s3 import S3Client +from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException From 64d54252383c06d3cff6fc42638b09aa284a045d Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:06:00 +0000 Subject: [PATCH 36/47] [PRMP-585] review processor handles if object already exists in review bucket --- .../document_review_processor_service.py | 25 +++++++++++++------ .../test_document_review_processor_service.py | 10 ++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 91a785453..0fd8d653d 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -1,6 +1,8 @@ import os from datetime import datetime, timezone +from botocore.exceptions import ClientError + from enums.review_status import ReviewStatus from models.document_reference import DocumentReferenceMetadataFields from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview @@ -89,6 +91,7 @@ def _move_files_to_review_bucket( List of DocumentReviewFileDetails with new file locations in review bucket """ new_file_keys: list[DocumentReviewFileDetails] = [] + for file in message_data.files: new_file_key = ( f"{message_data.nhs_number}/{review_record_id}/{file.file_name}" @@ -97,14 +100,21 @@ def _move_files_to_review_bucket( logger.info( f"Copying file from ({file.file_path}) in staging to review bucket: {new_file_key}" ) + try: - self.s3_service.copy_across_bucket( - source_bucket=self.staging_bucket_name, - source_file_key=file.file_path, - dest_bucket=self.review_bucket_name, - dest_file_key=new_file_key, - if_none_match="*", - ) + self.s3_service.copy_across_bucket( + source_bucket=self.staging_bucket_name, + source_file_key=file.file_path, + dest_bucket=self.review_bucket_name, + dest_file_key=new_file_key, + if_none_match="*", + ) + + except ClientError as e: + if e.response["Error"]["Code"] == "PreconditionFailed": + pass + else: + raise e new_file_keys.append( DocumentReviewFileDetails( @@ -115,6 +125,7 @@ def _move_files_to_review_bucket( logger.info("File successfully copied to review bucket") logger.info(f"Successfully moved file to: {new_file_key}") + return new_file_keys def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index c78fc8a2e..11fadf869 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -297,6 +297,16 @@ def test_move_files_copy_error(service_under_test, sample_review_message): ) +def test_move_files_to_review_bucket_continues_file_already_exists_in_review_bucket(service_under_test, sample_review_message): + + service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( + {"Error": {"Code": "PreconditionFailed", "Message": "At least one of the pre-conditions you specified did not hold"}}, + "CopyObject", + ) + + service_under_test.process_review_message(sample_review_message) + service_under_test.dynamo_service.create_item.assert_called() + # Tests for _delete_files_from_staging method From 6326457684eac84febcdb6748ed9c7f6b9b5debf Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Thu, 13 Nov 2025 11:21:00 +0000 Subject: [PATCH 37/47] [PRMP-585] add log message for file having already been copied to review bucket --- lambdas/services/document_review_processor_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 0fd8d653d..67566227c 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -109,9 +109,12 @@ def _move_files_to_review_bucket( dest_file_key=new_file_key, if_none_match="*", ) + logger.info("File successfully copied to review bucket") + logger.info(f"Successfully moved file to: {new_file_key}") except ClientError as e: if e.response["Error"]["Code"] == "PreconditionFailed": + logger.info("File already exists in the Review Bucket") pass else: raise e @@ -123,9 +126,6 @@ def _move_files_to_review_bucket( ) ) - logger.info("File successfully copied to review bucket") - logger.info(f"Successfully moved file to: {new_file_key}") - return new_file_keys def _delete_files_from_staging(self, message_data: ReviewMessageBody) -> None: From 2ba964546c253ee1c10cad87150e0f2098fc52a5 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:04:05 +0000 Subject: [PATCH 38/47] [PRMP-585] add dynamo conditional failure handling --- .../document_review_processor_service.py | 20 ++++++++++++----- .../test_document_review_processor_service.py | 22 ++++++++++++++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 67566227c..e65032101 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -51,13 +51,21 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: document_upload_review = self._build_review_record( review_message, review_id, review_files ) - self.dynamo_service.create_item( - table_name=self.review_table_name, - item=document_upload_review.model_dump(by_alias=True, exclude_none=True), - key_name=DocumentReferenceMetadataFields.ID.value, - ) + try: + self.dynamo_service.create_item( + table_name=self.review_table_name, + item=document_upload_review.model_dump(by_alias=True, exclude_none=True), + key_name=DocumentReferenceMetadataFields.ID.value, + ) + + logger.info(f"Created review record {document_upload_review.id}") + except ClientError as e: + if e.response["Error"]["Code"] == "ConditionalCheckFailedException": + logger.info("Entry already exists on Document Review table") + pass + else: + raise e - logger.info(f"Created review record {document_upload_review.id}") self._delete_files_from_staging(review_message) def _build_review_record( diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 11fadf869..2432a164b 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -143,7 +143,7 @@ def test_process_review_message_s3_copy_error( service_under_test.process_review_message(sample_review_message) -def test_process_review_message_dynamo_error( +def test_process_review_message_dynamo_error_not_precondition( service_under_test, sample_review_message, mocker ): """Test processing fails when DynamoDB put fails.""" @@ -159,6 +159,26 @@ def test_process_review_message_dynamo_error( service_under_test.process_review_message(sample_review_message) +def test_process_review_message_continues_dynamo_conditional_check_failure( + service_under_test, sample_review_message, mocker +): + + mocker.patch.object( + service_under_test, "_move_files_to_review_bucket", return_value=[] + ) + mocker.patch.object(service_under_test, "_delete_files_from_staging") + service_under_test.dynamo_service.create_item.side_effect = ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "DynamoDB error"}}, + "PutItem", + ) + + + service_under_test.process_review_message(sample_review_message) + + service_under_test._delete_files_from_staging.assert_called() + + + # Tests for _build_review_record and _create_review_record methods From fd39504b4e367d1c762de90ac2915658d1aa4929 Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Fri, 14 Nov 2025 10:01:12 +0000 Subject: [PATCH 39/47] [PRMP-585] remove unnecessary pass statements --- lambdas/services/document_review_processor_service.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index e65032101..9bf4c7075 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -62,7 +62,6 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: except ClientError as e: if e.response["Error"]["Code"] == "ConditionalCheckFailedException": logger.info("Entry already exists on Document Review table") - pass else: raise e @@ -123,7 +122,6 @@ def _move_files_to_review_bucket( except ClientError as e: if e.response["Error"]["Code"] == "PreconditionFailed": logger.info("File already exists in the Review Bucket") - pass else: raise e From 11739b3ef6bed6dbb54e211884f67926733dd8ee Mon Sep 17 00:00:00 2001 From: steph-torres-nhs <173282814+steph-torres-nhs@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:54:17 +0000 Subject: [PATCH 40/47] [PRMP-585] remove requirement --- lambdas/handlers/document_review_processor_handler.py | 2 ++ lambdas/requirements/layers/requirements_core_lambda_layer.txt | 3 +-- lambdas/services/base/dynamo_service.py | 3 +-- lambdas/services/base/s3_service.py | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/handlers/document_review_processor_handler.py b/lambdas/handlers/document_review_processor_handler.py index 43a33d055..a363ccf7d 100644 --- a/lambdas/handlers/document_review_processor_handler.py +++ b/lambdas/handlers/document_review_processor_handler.py @@ -5,6 +5,7 @@ from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.request_context import request_context logger = LoggingService(__name__) @@ -54,4 +55,5 @@ def lambda_handler(event, context): ) raise error + request_context.patient_nhs_no = "" logger.info("Continuing to next message.") diff --git a/lambdas/requirements/layers/requirements_core_lambda_layer.txt b/lambdas/requirements/layers/requirements_core_lambda_layer.txt index 6728aaf74..8bbe1c192 100644 --- a/lambdas/requirements/layers/requirements_core_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_core_lambda_layer.txt @@ -17,5 +17,4 @@ responses==0.23.1 six==1.16.0 types-PyYAML==6.0.12.11 regex==2023.12.25 -pikepdf==8.4.0 -types-boto3[dynamodb,s3]==1.40.64 \ No newline at end of file +pikepdf==8.4.0 \ No newline at end of file diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 5f68ca68f..b325ee562 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -4,7 +4,6 @@ import boto3 from boto3.dynamodb.conditions import Attr, ConditionBase, Key from botocore.exceptions import ClientError -from types_boto3_dynamodb import DynamoDBServiceResource from utils.audit_logging_setup import LoggingService from utils.dynamo_utils import ( create_expression_attribute_values, @@ -27,7 +26,7 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb: DynamoDBServiceResource = boto3.resource( + self.dynamodb = boto3.resource( "dynamodb", region_name="eu-west-2" ) self.initialised = True diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index e45bca015..b62ab59d1 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -7,7 +7,6 @@ from botocore.client import Config as BotoConfig from botocore.exceptions import ClientError from services.base.iam_service import IAMService -from types_boto3_s3 import S3Client from utils.audit_logging_setup import LoggingService from utils.exceptions import TagNotFoundException @@ -33,7 +32,7 @@ def __init__(self, custom_aws_role=None): max_pool_connections=20, ) self.presigned_url_expiry = 1800 - self.client: S3Client = boto3.client("s3", config=self.config) + self.client = boto3.client("s3", config=self.config) self.initialised = True self.custom_client = None self.custom_aws_role = custom_aws_role From dd125ab0f9ac0d3f1a9891fb3b7ccfe002070ff9 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:21:46 +0000 Subject: [PATCH 41/47] [PRMP-585] fix merge conflict --- lambdas/enums/review_status.py | 9 ------ lambdas/models/document_review.py | 8 ++--- lambdas/models/sqs/review_message_body.py | 2 ++ .../document_review_processor_service.py | 14 ++++---- .../test_document_review_processor_service.py | 32 +++++++++++++------ 5 files changed, 35 insertions(+), 30 deletions(-) delete mode 100644 lambdas/enums/review_status.py diff --git a/lambdas/enums/review_status.py b/lambdas/enums/review_status.py deleted file mode 100644 index 64b51ce26..000000000 --- a/lambdas/enums/review_status.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import StrEnum - - -class ReviewStatus(StrEnum): - """Status values for document review records.""" - - PENDING_REVIEW = "PENDING_REVIEW" - APPROVED = "APPROVED" - REJECTED = "REJECTED" diff --git a/lambdas/models/document_review.py b/lambdas/models/document_review.py index 7670716cf..780fd19c7 100644 --- a/lambdas/models/document_review.py +++ b/lambdas/models/document_review.py @@ -28,14 +28,14 @@ class DocumentUploadReviewReference(BaseModel): ) id: str = Field( default_factory=lambda: str(uuid.uuid4()), - alias=str(DocumentReferenceMetadataFields.ID.value) + alias=str(DocumentReferenceMetadataFields.ID.value), ) author: str custodian: str review_status: DocumentReviewStatus = Field( default=DocumentReviewStatus.PENDING_REVIEW ) - review_reason: str + review_reason: str review_date: int | None = Field(default=None) reviewer: str | None = Field(default=None) @@ -46,6 +46,4 @@ class DocumentUploadReviewReference(BaseModel): alias=str(DocumentReferenceMetadataFields.TTL.value), default=None ) document_reference_id: str | None = Field(default=None) - document_snomed_code_type: str = Field( - default=SnomedCodes.LLOYD_GEORGE.value.code - ) + document_snomed_code_type: str = Field(default=SnomedCodes.LLOYD_GEORGE.value.code) diff --git a/lambdas/models/sqs/review_message_body.py b/lambdas/models/sqs/review_message_body.py index e213168f2..ef24c791f 100644 --- a/lambdas/models/sqs/review_message_body.py +++ b/lambdas/models/sqs/review_message_body.py @@ -8,8 +8,10 @@ class ReviewMessageFile(BaseModel): file_path: str = Field(description="Location in the staging bucket") """Location in the staging bucket""" + class ReviewMessageBody(BaseModel): """Model for SQS message body from the document review queue.""" + upload_id: str files: list[ReviewMessageFile] nhs_number: str diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 9bf4c7075..98ecc5ee9 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -3,9 +3,9 @@ from botocore.exceptions import ClientError -from enums.review_status import ReviewStatus +from enums.document_review_status import DocumentReviewStatus from models.document_reference import DocumentReferenceMetadataFields -from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview +from models.document_review import DocumentReviewFileDetails, DocumentUploadReviewReference from models.sqs.review_message_body import ReviewMessageBody from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service @@ -54,7 +54,9 @@ def process_review_message(self, review_message: ReviewMessageBody) -> None: try: self.dynamo_service.create_item( table_name=self.review_table_name, - item=document_upload_review.model_dump(by_alias=True, exclude_none=True), + item=document_upload_review.model_dump( + by_alias=True, exclude_none=True + ), key_name=DocumentReferenceMetadataFields.ID.value, ) @@ -72,11 +74,11 @@ def _build_review_record( message_data: ReviewMessageBody, review_id: str, review_files: list[DocumentReviewFileDetails], - ) -> DocumentsUploadReview: - return DocumentsUploadReview( + ) -> DocumentUploadReviewReference: + return DocumentUploadReviewReference( id=review_id, nhs_number=message_data.nhs_number, - review_status=ReviewStatus.PENDING_REVIEW, + review_status=DocumentReviewStatus.PENDING_REVIEW, review_reason=message_data.failure_reason, author=message_data.uploader_ods, custodian=message_data.current_gp, diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 2432a164b..665b14c75 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -1,7 +1,8 @@ import pytest from botocore.exceptions import ClientError -from enums.review_status import ReviewStatus -from models.document_review import DocumentReviewFileDetails, DocumentsUploadReview + +from enums.document_review_status import DocumentReviewStatus +from models.document_review import DocumentReviewFileDetails, DocumentUploadReviewReference from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.document_review_processor_service import ReviewProcessorService @@ -168,17 +169,20 @@ def test_process_review_message_continues_dynamo_conditional_check_failure( ) mocker.patch.object(service_under_test, "_delete_files_from_staging") service_under_test.dynamo_service.create_item.side_effect = ClientError( - {"Error": {"Code": "ConditionalCheckFailedException", "Message": "DynamoDB error"}}, + { + "Error": { + "Code": "ConditionalCheckFailedException", + "Message": "DynamoDB error", + } + }, "PutItem", ) - service_under_test.process_review_message(sample_review_message) service_under_test._delete_files_from_staging.assert_called() - # Tests for _build_review_record and _create_review_record methods @@ -195,10 +199,10 @@ def test_build_review_record_success(service_under_test, sample_review_message): sample_review_message, "test-review-id", files ) - assert isinstance(result, DocumentsUploadReview) + assert isinstance(result, DocumentUploadReviewReference) assert result.id == "test-review-id" assert result.nhs_number == "9000000009" - assert result.review_status == ReviewStatus.PENDING_REVIEW + assert result.review_status == DocumentReviewStatus.PENDING_REVIEW assert result.review_reason == "Failed virus scan" assert result.author == "Y12345" assert result.custodian == "Y12345" @@ -317,16 +321,24 @@ def test_move_files_copy_error(service_under_test, sample_review_message): ) -def test_move_files_to_review_bucket_continues_file_already_exists_in_review_bucket(service_under_test, sample_review_message): +def test_move_files_to_review_bucket_continues_file_already_exists_in_review_bucket( + service_under_test, sample_review_message +): service_under_test.s3_service.copy_across_bucket.side_effect = ClientError( - {"Error": {"Code": "PreconditionFailed", "Message": "At least one of the pre-conditions you specified did not hold"}}, - "CopyObject", + { + "Error": { + "Code": "PreconditionFailed", + "Message": "At least one of the pre-conditions you specified did not hold", + } + }, + "CopyObject", ) service_under_test.process_review_message(sample_review_message) service_under_test.dynamo_service.create_item.assert_called() + # Tests for _delete_files_from_staging method From 7d565542d9a7d642f5429dee26b0b4d87e7b9e17 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:28:02 +0000 Subject: [PATCH 42/47] [PRMP-585] Update mock environment variables --- lambdas/tests/unit/conftest.py | 8 +++---- .../test_document_review_processor_handler.py | 21 +++++++------------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index 0775f9812..90afb62d4 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -62,7 +62,7 @@ MOCK_STATISTICS_TABLE_NAME = "STATISTICS_TABLE" MOCK_STATISTICAL_REPORTS_BUCKET_ENV_NAME = "STATISTICAL_REPORTS_BUCKET" MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME = "DOCUMENT_REVIEW_DYNAMODB_NAME" -MOCK_PENDING_REVIEW_BUCKET_NAME = "PENDING_REVIEW_BUCKET_NAME" +MOCK_PENDING_REVIEW_BUCKET_NAME = "DOCUMENT_REVIEW_S3_BUCKET_NAME" MOCK_ARF_TABLE_NAME = "test_arf_dynamoDB_table" MOCK_PDM_TABLE_NAME = "test_pdm_dynamoDB_table" @@ -140,8 +140,6 @@ MOCK_TEAMS_WEBHOOK = "test_teams_webhook" MOCK_SLACK_BOT_TOKEN = f"xoxb-{TEST_UUID}" MOCK_ALERTING_SLACK_CHANNEL_ID = "slack_channel_id" -MOCK_DOCUMENT_REVIEW_TABLE = "test_document_review" -MOCK_DOCUMENT_REVIEW_BUCKET = "test_document_review_bucket" @pytest.fixture def set_env(monkeypatch): @@ -230,7 +228,9 @@ def set_env(monkeypatch): monkeypatch.setenv("SLACK_BOT_TOKEN", MOCK_SLACK_BOT_TOKEN) monkeypatch.setenv("SLACK_CHANNEL_ID", MOCK_ALERTING_SLACK_CHANNEL_ID) monkeypatch.setenv("ITOC_TESTING_ODS_CODES", MOCK_ITOC_ODS_CODES) - monkeypatch.setenv(MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME, MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME) + monkeypatch.setenv( + MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME, MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME + ) monkeypatch.setenv(MOCK_PENDING_REVIEW_BUCKET_NAME, MOCK_PENDING_REVIEW_BUCKET) monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", MOCK_STAGING_STORE_BUCKET) monkeypatch.setenv("METADATA_SQS_QUEUE_URL", MOCK_LG_METADATA_SQS_QUEUE) diff --git a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py index 53521de91..11c6bbdea 100644 --- a/lambdas/tests/unit/handlers/test_document_review_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_document_review_processor_handler.py @@ -23,7 +23,7 @@ def sample_review_message_body(): files=[ ReviewMessageFile( file_name="test_document.pdf", - file_path="staging/9000000009/test_document.pdf" + file_path="staging/9000000009/test_document.pdf", ) ], nhs_number="9000000009", @@ -58,7 +58,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): files=[ ReviewMessageFile( file_name="document_1.pdf", - file_path="staging/9000000009/document_1.pdf" + file_path="staging/9000000009/document_1.pdf", ) ], nhs_number="9000000009", @@ -73,7 +73,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): files=[ ReviewMessageFile( file_name="document_2.pdf", - file_path="staging/9000000010/document_2.pdf" + file_path="staging/9000000010/document_2.pdf", ) ], nhs_number="9000000010", @@ -88,7 +88,7 @@ def sample_sqs_event_multiple_messages(sample_review_message_body): files=[ ReviewMessageFile( file_name="document_3.pdf", - file_path="staging/9000000011/document_3.pdf" + file_path="staging/9000000011/document_3.pdf", ) ], nhs_number="9000000011", @@ -125,7 +125,6 @@ def empty_sqs_event(): return {"Records": []} - def test_lambda_handler_processes_single_message_successfully( set_env, context, @@ -160,9 +159,9 @@ def test_lambda_handler_calls_service_with_correct_message( lambda_handler(sample_sqs_event, context) mock_review_service.process_review_message.assert_called_once() - + call_args = mock_review_service.process_review_message.call_args[0][0] - + assert type(call_args).__name__ == "ReviewMessageBody" assert len(call_args.files) == 1 assert call_args.files[0].file_name == "test_document.pdf" @@ -192,10 +191,7 @@ def test_lambda_handler_parses_json_body_correctly( { "upload_id": "test-upload-id-123", "files": [ - { - "file_name": "test.pdf", - "file_path": "staging/test.pdf" - } + {"file_name": "test.pdf", "file_path": "staging/test.pdf"} ], "nhs_number": "9000000009", "failure_reason": "Test failure", @@ -230,9 +226,8 @@ def test_lambda_handler_calls_process_review_message_on_service( mock_review_service.process_review_message.assert_called_once() called_message = mock_review_service.process_review_message.call_args[0][0] - + assert isinstance(called_message, ReviewMessageBody) assert called_message.upload_id == sample_review_message_body.upload_id assert called_message.nhs_number == sample_review_message_body.nhs_number assert called_message.failure_reason == sample_review_message_body.failure_reason - From 1841958d3c3fd0031faefeb66394d913792b9316 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:30:57 +0000 Subject: [PATCH 43/47] [PRMP-585] change to test_document_upload_review_service --- .../services/test_document_upload_review_service.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lambdas/tests/unit/services/test_document_upload_review_service.py b/lambdas/tests/unit/services/test_document_upload_review_service.py index e399a6ee3..73f5522d7 100644 --- a/lambdas/tests/unit/services/test_document_upload_review_service.py +++ b/lambdas/tests/unit/services/test_document_upload_review_service.py @@ -3,7 +3,11 @@ import pytest from models.document_review import DocumentUploadReviewReference from services.document_upload_review_service import DocumentUploadReviewService -from tests.unit.conftest import TEST_NHS_NUMBER, MOCK_DOCUMENT_REVIEW_TABLE, MOCK_DOCUMENT_REVIEW_BUCKET +from tests.unit.conftest import ( + MOCK_PENDING_REVIEW_BUCKET, + MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME, + TEST_NHS_NUMBER, +) TEST_ODS_CODE = "Y12345" NEW_ODS_CODE = "Z98765" @@ -39,7 +43,7 @@ def mock_document_review_references(): def test_table_name(mock_service): """Test that table_name property returns correct environment variable.""" - assert mock_service.table_name == MOCK_DOCUMENT_REVIEW_TABLE + assert mock_service.table_name == MOCK_DOCUMENT_REVIEW_DYNAMOODB_TABLE_NAME def test_model_class(mock_service): @@ -50,7 +54,7 @@ def test_model_class(mock_service): def test_s3_bucket(mock_service, monkeypatch): """Test that s3_bucket property returns the correct environment variable.""" - assert mock_service.s3_bucket == MOCK_DOCUMENT_REVIEW_BUCKET + assert mock_service.s3_bucket == MOCK_PENDING_REVIEW_BUCKET def test_update_document_review_custodian_updates_all_documents( From 2e8f673d3dda0c5647ea24f97ee21811a5fae183 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:37:58 +0000 Subject: [PATCH 44/47] Update environment variable for pending review bucket in document upload service --- lambdas/services/document_upload_review_service.py | 3 ++- lambdas/tests/unit/conftest.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lambdas/services/document_upload_review_service.py b/lambdas/services/document_upload_review_service.py index c4c4ee1d2..c6df051ec 100644 --- a/lambdas/services/document_upload_review_service.py +++ b/lambdas/services/document_upload_review_service.py @@ -9,10 +9,11 @@ class DocumentUploadReviewService(DocumentService): """Service for handling DocumentUploadReviewReference operations.""" + def __init__(self): super().__init__() self._table_name = os.environ.get("DOCUMENT_REVIEW_DYNAMODB_NAME") - self._s3_bucket = os.environ.get("DOCUMENT_REVIEW_S3_BUCKET_NAME") + self._s3_bucket = os.environ.get("PENDING_REVIEW_BUCKET_NAME") @property def table_name(self) -> str: diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index 90afb62d4..1df8bc85c 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -62,7 +62,7 @@ MOCK_STATISTICS_TABLE_NAME = "STATISTICS_TABLE" MOCK_STATISTICAL_REPORTS_BUCKET_ENV_NAME = "STATISTICAL_REPORTS_BUCKET" MOCK_DOCUMENT_REVIEW_DYNAMODB_NAME = "DOCUMENT_REVIEW_DYNAMODB_NAME" -MOCK_PENDING_REVIEW_BUCKET_NAME = "DOCUMENT_REVIEW_S3_BUCKET_NAME" +MOCK_PENDING_REVIEW_BUCKET_NAME = "PENDING_REVIEW_BUCKET_NAME" MOCK_ARF_TABLE_NAME = "test_arf_dynamoDB_table" MOCK_PDM_TABLE_NAME = "test_pdm_dynamoDB_table" From 354a3c0ce214356ab5c2e33e225f0a9914cffa46 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:03:23 +0000 Subject: [PATCH 45/47] format --- lambdas/services/base/dynamo_service.py | 8 ++++---- lambdas/services/document_review_processor_service.py | 6 ++++-- lambdas/services/document_upload_review_service.py | 2 ++ lambdas/tests/unit/conftest.py | 2 ++ .../services/test_document_review_processor_service.py | 6 ++++-- lambdas/utils/exceptions.py | 2 ++ 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index d63edf601..4266be82a 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -26,9 +26,7 @@ def __new__(cls): def __init__(self): if not self.initialised: - self.dynamodb = boto3.resource( - "dynamodb", region_name="eu-west-2" - ) + self.dynamodb = boto3.resource("dynamodb", region_name="eu-west-2") self.initialised = True def get_table(self, table_name: str): @@ -159,7 +157,9 @@ def create_item(self, table_name, item, key_name: str | None = None): table = self.get_table(table_name) logger.info(f"Writing item to table: {table_name}") if key_name: - return table.put_item(Item=item, ConditionExpression=f"attribute_not_exists({key_name})") + return table.put_item( + Item=item, ConditionExpression=f"attribute_not_exists({key_name})" + ) else: return table.put_item(Item=item) except ClientError as e: diff --git a/lambdas/services/document_review_processor_service.py b/lambdas/services/document_review_processor_service.py index 98ecc5ee9..8d825b3c3 100644 --- a/lambdas/services/document_review_processor_service.py +++ b/lambdas/services/document_review_processor_service.py @@ -2,10 +2,12 @@ from datetime import datetime, timezone from botocore.exceptions import ClientError - from enums.document_review_status import DocumentReviewStatus from models.document_reference import DocumentReferenceMetadataFields -from models.document_review import DocumentReviewFileDetails, DocumentUploadReviewReference +from models.document_review import ( + DocumentReviewFileDetails, + DocumentUploadReviewReference, +) from models.sqs.review_message_body import ReviewMessageBody from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service diff --git a/lambdas/services/document_upload_review_service.py b/lambdas/services/document_upload_review_service.py index 81a7987a0..719adf9b0 100644 --- a/lambdas/services/document_upload_review_service.py +++ b/lambdas/services/document_upload_review_service.py @@ -17,7 +17,9 @@ class DocumentUploadReviewService(DocumentService): """Service for handling DocumentUploadReviewReference operations.""" + DEFAULT_QUERY_LIMIT = 50 + def __init__(self): super().__init__() self._table_name = os.environ.get("DOCUMENT_REVIEW_DYNAMODB_NAME") diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index 62e7ebc95..50a48a676 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -140,6 +140,7 @@ MOCK_DOCUMENT_REVIEW_BUCKET = "test_document_review_bucket" MOCK_EDGE_REFERENCE_TABLE = "test_edge_reference_table" + @pytest.fixture def set_env(monkeypatch): monkeypatch.setenv("AWS_DEFAULT_REGION", REGION_NAME) @@ -233,6 +234,7 @@ def set_env(monkeypatch): monkeypatch.setenv("METADATA_SQS_QUEUE_URL", MOCK_LG_METADATA_SQS_QUEUE) monkeypatch.setenv("EDGE_REFERENCE_TABLE", MOCK_EDGE_REFERENCE_TABLE) + EXPECTED_PARSED_PATIENT_BASE_CASE = PatientDetails( givenName=["Jane"], familyName="Smith", diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 665b14c75..0fc304397 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -1,8 +1,10 @@ import pytest from botocore.exceptions import ClientError - from enums.document_review_status import DocumentReviewStatus -from models.document_review import DocumentReviewFileDetails, DocumentUploadReviewReference +from models.document_review import ( + DocumentReviewFileDetails, + DocumentUploadReviewReference, +) from models.sqs.review_message_body import ReviewMessageBody, ReviewMessageFile from services.document_review_processor_service import ReviewProcessorService diff --git a/lambdas/utils/exceptions.py b/lambdas/utils/exceptions.py index 75341ffdc..a67ae9b2b 100644 --- a/lambdas/utils/exceptions.py +++ b/lambdas/utils/exceptions.py @@ -167,6 +167,7 @@ class FhirDocumentReferenceException(Exception): class TransactionConflictException(Exception): pass + class MigrationUnrecoverableException(Exception): def __init__(self, message: str, item_id: str): super().__init__(message) @@ -176,6 +177,7 @@ def __init__(self, message: str, item_id: str): def to_dict(self): return {"itemId": self.item_id, "message": self.message} + class MigrationRetryableException(Exception): def __init__(self, message: str, segment_id: str): super().__init__(message) From dc746bfe1c22fea7cfe282c89fbfe0e62ff37132 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:06:35 +0000 Subject: [PATCH 46/47] Rename MOCK_EDGE_TABLE to MOCK_EDGE_REFERENCE_TABLE in test_get_document_review_service.py --- .../tests/unit/services/test_get_document_review_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/tests/unit/services/test_get_document_review_service.py b/lambdas/tests/unit/services/test_get_document_review_service.py index 53396790f..7d2bb22e3 100644 --- a/lambdas/tests/unit/services/test_get_document_review_service.py +++ b/lambdas/tests/unit/services/test_get_document_review_service.py @@ -12,7 +12,7 @@ from services.get_document_review_service import GetDocumentReviewService from tests.unit.conftest import ( MOCK_DOCUMENT_REVIEW_BUCKET, - MOCK_EDGE_TABLE, + MOCK_EDGE_REFERENCE_TABLE, TEST_NHS_NUMBER, ) from utils.exceptions import DynamoServiceException @@ -210,7 +210,7 @@ def test_create_cloudfront_presigned_url(mock_service, mock_uuid, mocker): call_args = ( mock_service.document_review_service.dynamo_service.create_item.call_args ) - assert call_args[0][0] == MOCK_EDGE_TABLE + assert call_args[0][0] == MOCK_EDGE_REFERENCE_TABLE assert call_args[0][1]["ID"] == f"review/{mock_uuid}" assert call_args[0][1]["presignedUrl"] == TEST_PRESIGNED_URL_1 assert "TTL" in call_args[0][1] From d74d3af466007964bec7501ca54bfc8831a860e5 Mon Sep 17 00:00:00 2001 From: NogaNHS <127490765+NogaNHS@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:05:26 +0000 Subject: [PATCH 47/47] format --- lambdas/services/base/s3_service.py | 10 ++++-- .../test_document_review_processor_service.py | 31 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index b62ab59d1..f98fdea22 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -133,11 +133,15 @@ def copy_across_bucket( StorageClass="INTELLIGENT_TIERING", ) - def delete_object(self, s3_bucket_name: str, file_key: str, version_id: str | None = None): + def delete_object( + self, s3_bucket_name: str, file_key: str, version_id: str | None = None + ): if version_id is None: return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key) - - return self.client.delete_object(Bucket=s3_bucket_name, Key=file_key, VersionId=version_id) + + return self.client.delete_object( + Bucket=s3_bucket_name, Key=file_key, VersionId=version_id + ) def create_object_tag( self, s3_bucket_name: str, file_key: str, tag_key: str, tag_value: str diff --git a/lambdas/tests/unit/services/test_document_review_processor_service.py b/lambdas/tests/unit/services/test_document_review_processor_service.py index 0fc304397..51fb3dac3 100644 --- a/lambdas/tests/unit/services/test_document_review_processor_service.py +++ b/lambdas/tests/unit/services/test_document_review_processor_service.py @@ -47,25 +47,18 @@ def sample_review_message(): ) -# Test service initialization - - def test_service_initializes_with_correct_environment_variables( set_env, mock_dynamo_service, mock_s3_service ): - """Test service initializes correctly with environment variables.""" service = ReviewProcessorService() - assert service.review_table_name == "test_review_table" + assert service.review_table_name == "test_document_review" assert service.staging_bucket_name == "test_staging_bulk_store" - assert service.review_bucket_name == "test_review_bucket" + assert service.review_bucket_name == "test_document_review_bucket" mock_dynamo_service.assert_called_once() mock_s3_service.assert_called_once() -# Tests for process_review_message method - - def test_process_review_message_success( service_under_test, sample_review_message, mocker ): @@ -151,7 +144,14 @@ def test_process_review_message_dynamo_error_not_precondition( ): """Test processing fails when DynamoDB put fails.""" mocker.patch.object( - service_under_test, "_move_files_to_review_bucket", return_value=[] + service_under_test, + "_move_files_to_review_bucket", + return_value=[ + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-upload-id-456/document_1.pdf", + ) + ], ) service_under_test.dynamo_service.create_item.side_effect = ClientError( {"Error": {"Code": "InternalServerError", "Message": "DynamoDB error"}}, @@ -167,7 +167,14 @@ def test_process_review_message_continues_dynamo_conditional_check_failure( ): mocker.patch.object( - service_under_test, "_move_files_to_review_bucket", return_value=[] + service_under_test, + "_move_files_to_review_bucket", + return_value=[ + DocumentReviewFileDetails( + file_name="document_1.pdf", + file_location="9000000009/test-upload-id-456/document_1.pdf", + ) + ], ) mocker.patch.object(service_under_test, "_delete_files_from_staging") service_under_test.dynamo_service.create_item.side_effect = ClientError( @@ -272,7 +279,7 @@ def test_move_files_success(service_under_test, sample_review_message): service_under_test.s3_service.copy_across_bucket.assert_called_once_with( source_bucket="test_staging_bulk_store", source_file_key="staging/9000000009/test_document.pdf", - dest_bucket="test_review_bucket", + dest_bucket="test_document_review_bucket", dest_file_key=expected_key, if_none_match="*", )