Skip to content

Commit fc9fae5

Browse files
MohammadIqbalAD-NHSMohammadIqbalAD-NHS
authored
[PRMP-191] Update Bulk upload lambda to use the standardised file name (#794)
1 parent 05b4fc0 commit fc9fae5

17 files changed

+182
-15
lines changed

‎lambdas/enums/metadata_report.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ class MetadataReport(StrEnum):
88
PdsOdsCode = "PdsOdsCode"
99
UploaderOdsCode = "UploaderOdsCode"
1010
FilePath = "FilePath"
11+
StoredFileName = "StoredFileName"
1112
Date = "Date"
1213
Timestamp = "Timestamp"
1314
ID = "ID"

‎lambdas/models/report/bulk_upload_report.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class BulkUploadReport(BaseModel):
2525
default_factory=lambda: date_string_yyyymmdd(datetime.now()),
2626
)
2727
file_path: str = Field(alias=MetadataReport.FilePath)
28+
stored_file_name: str = Field(alias=MetadataReport.StoredFileName, default=None)
2829
pds_ods_code: str = Field(alias=MetadataReport.PdsOdsCode)
2930
uploader_ods_code: str = Field(alias=MetadataReport.UploaderOdsCode)
3031
reason: Optional[str] = Field(default="", alias=MetadataReport.Reason)

‎lambdas/repositories/bulk_upload/bulk_upload_dynamo_repository.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def write_report_upload_to_dynamo(
4242
nhs_number=nhs_number,
4343
reason=reason,
4444
file_path=file.file_path,
45+
stored_file_name=file.stored_file_name,
4546
pds_ods_code=pds_ods_code,
4647
uploader_ods_code=file.gp_practice_code,
4748
)

‎lambdas/services/bulk_upload_metadata_service.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@
88

99
import pydantic
1010
from botocore.exceptions import ClientError
11-
1211
from models.staging_metadata import (
1312
NHS_NUMBER_FIELD_NAME,
1413
ODS_CODE,
14+
BulkUploadQueueMetadata,
1515
MetadataFile,
16-
StagingSqsMetadata, BulkUploadQueueMetadata,
16+
StagingSqsMetadata,
1717
)
1818
from services.base.s3_service import S3Service
1919
from services.base.sqs_service import SQSService
@@ -114,7 +114,8 @@ def csv_to_staging_sqs_metadata(csv_file_path: str) -> list[StagingSqsMetadata]:
114114
nhs_number=nhs_number,
115115
files=[
116116
BulkUploadQueueMetadata(
117-
**metadata_file.model_dump(), stored_file_name=metadata_file.file_path
117+
**metadata_file.model_dump(),
118+
stored_file_name=metadata_file.file_path,
118119
)
119120
for metadata_file in patients[nhs_number, ods_code]
120121
],

‎lambdas/services/bulk_upload_report_service.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,8 @@ def write_items_to_csv(items: list[BulkUploadReport], csv_file_path: str):
301301
for item in items:
302302
dict_writer_object.writerow(
303303
item.model_dump(
304-
exclude={str(MetadataReport.ID).lower()}, by_alias=True
304+
exclude={str(MetadataReport.ID).lower(), "stored_file_name"},
305+
by_alias=True,
305306
)
306307
)
307308

‎lambdas/services/bulk_upload_service.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
import pydantic
66
from botocore.exceptions import ClientError
7-
87
from enums.patient_ods_inactive_status import PatientOdsInactiveStatus
98
from enums.snomed_codes import SnomedCodes
109
from enums.upload_status import UploadStatus
@@ -128,7 +127,7 @@ def handle_sqs_message(self, message: dict):
128127
try:
129128
file_names = []
130129
for file_metadata in staging_metadata.files:
131-
file_names.append(os.path.basename(file_metadata.file_path))
130+
file_names.append(os.path.basename(file_metadata.stored_file_name))
132131
file_metadata.scan_date = validate_scan_date(file_metadata.scan_date)
133132
request_context.patient_nhs_no = staging_metadata.nhs_number
134133
validate_nhs_number(staging_metadata.nhs_number)
@@ -395,7 +394,7 @@ def convert_to_document_reference(
395394
current_gp_ods: str,
396395
) -> DocumentReference:
397396
s3_bucket_name = self.bulk_upload_s3_repository.lg_bucket_name
398-
file_name = os.path.basename(file_metadata.file_path)
397+
file_name = os.path.basename(file_metadata.stored_file_name)
399398
document_reference = DocumentReference(
400399
id=str(uuid.uuid4()),
401400
nhs_number=nhs_number,

‎lambdas/tests/unit/helpers/data/bulk_upload/dynamo_responses.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"Date": "2012-01-13",
1313
"NhsNumber": "9000000000",
1414
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
15+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
1516
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
1617
"UploadStatus": "complete",
1718
"ID": TEST_UUID,
@@ -22,6 +23,7 @@
2223
"Date": "2012-01-13",
2324
"NhsNumber": "9000000001",
2425
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
26+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
2527
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
2628
"Reason": "Patient is deceased - INFORMAL",
2729
"UploadStatus": "complete",
@@ -33,6 +35,7 @@
3335
"Date": "2012-01-13",
3436
"NhsNumber": "9000000002",
3537
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
38+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
3639
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
3740
"UploadStatus": "complete",
3841
"ID": TEST_UUID,
@@ -43,6 +46,7 @@
4346
"Date": "2012-01-13",
4447
"NhsNumber": "9000000003",
4548
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
49+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
4650
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
4751
"UploadStatus": "complete",
4852
"ID": TEST_UUID,
@@ -53,6 +57,7 @@
5357
"Date": "2012-01-13",
5458
"NhsNumber": "9000000003",
5559
"FilePath": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
60+
"StoredFileName": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
5661
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
5762
"Reason": "Could not find the given patient on PDS",
5863
"UploadStatus": "failed",
@@ -64,6 +69,7 @@
6469
"Date": "2012-01-13",
6570
"NhsNumber": "9000000004",
6671
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
72+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
6773
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
6874
"UploadStatus": "complete",
6975
"ID": TEST_UUID,
@@ -74,6 +80,7 @@
7480
"Date": "2012-01-13",
7581
"NhsNumber": "9000000004",
7682
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
83+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
7784
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
7885
"UploadStatus": "complete",
7986
"ID": TEST_UUID,
@@ -84,6 +91,7 @@
8491
"Date": "2012-01-13",
8592
"NhsNumber": "9000000005",
8693
"FilePath": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
94+
"StoredFileName": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
8795
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
8896
"Reason": "Could not find the given patient on PDS",
8997
"UploadStatus": "failed",
@@ -95,6 +103,7 @@
95103
"Date": "2012-01-13",
96104
"NhsNumber": "9000000006",
97105
"FilePath": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
106+
"StoredFileName": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
98107
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
99108
"Reason": "Could not find the given patient on PDS",
100109
"UploadStatus": "failed",
@@ -106,6 +115,7 @@
106115
"Date": "2012-01-13",
107116
"NhsNumber": "9000000006",
108117
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
118+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
109119
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
110120
"Reason": "Lloyd George file already exists",
111121
"UploadStatus": "failed",
@@ -117,6 +127,7 @@
117127
"Date": "2012-01-13",
118128
"NhsNumber": "9000000007",
119129
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
130+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
120131
"UploaderOdsCode": TEST_UPLOADER_ODS_1,
121132
"Reason": "Lloyd George file already exists",
122133
"UploadStatus": "failed",
@@ -131,6 +142,7 @@
131142
"Date": "2012-01-13",
132143
"NhsNumber": "9000000009",
133144
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
145+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
134146
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
135147
"UploadStatus": "complete",
136148
"ID": TEST_UUID,
@@ -141,6 +153,7 @@
141153
"Date": "2012-01-13",
142154
"NhsNumber": "9000000010",
143155
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
156+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
144157
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
145158
"Reason": "Patient is deceased - FORMAL",
146159
"UploadStatus": "complete",
@@ -152,6 +165,7 @@
152165
"Date": "2012-01-13",
153166
"NhsNumber": "9000000011",
154167
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
168+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
155169
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
156170
"UploadStatus": "complete",
157171
"ID": TEST_UUID,
@@ -162,6 +176,7 @@
162176
"Date": "2012-01-13",
163177
"NhsNumber": "9000000012",
164178
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
179+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
165180
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
166181
"UploadStatus": "complete",
167182
"ID": TEST_UUID,
@@ -172,6 +187,7 @@
172187
"Date": "2012-01-13",
173188
"NhsNumber": "9000000012",
174189
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
190+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
175191
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
176192
"UploadStatus": "complete",
177193
"ID": TEST_UUID,
@@ -182,6 +198,7 @@
182198
"Date": "2012-01-13",
183199
"NhsNumber": "9000000013",
184200
"FilePath": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
201+
"StoredFileName": "/0000000000/1of1_Lloyd_George_Record_[NAME]_[0000000000]_[DOB].pdf",
185202
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
186203
"UploadStatus": "complete",
187204
"ID": TEST_UUID,
@@ -192,6 +209,7 @@
192209
"Date": "2012-01-13",
193210
"NhsNumber": "9000000014",
194211
"FilePath": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
212+
"StoredFileName": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
195213
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
196214
"Reason": "Could not find the given patient on PDS",
197215
"UploadStatus": "failed",
@@ -203,6 +221,7 @@
203221
"Date": "2012-01-13",
204222
"NhsNumber": "9000000015",
205223
"FilePath": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
224+
"StoredFileName": "/0000000000/1of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
206225
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
207226
"Reason": "Could not find the given patient on PDS",
208227
"UploadStatus": "failed",
@@ -214,6 +233,7 @@
214233
"Date": "2012-01-13",
215234
"NhsNumber": "9000000015",
216235
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
236+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
217237
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
218238
"Reason": "Lloyd George file already exists",
219239
"UploadStatus": "failed",
@@ -225,6 +245,7 @@
225245
"Date": "2012-01-13",
226246
"NhsNumber": "9000000016",
227247
"FilePath": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
248+
"StoredFileName": "/0000000000/2of2_Lloyd_George_Record_[NAME_2]_[0000000000]_[DOB].pdf",
228249
"UploaderOdsCode": TEST_UPLOADER_ODS_2,
229250
"Reason": "Lloyd George file already exists",
230251
"UploadStatus": "failed",

‎lambdas/tests/unit/helpers/data/bulk_upload/metadata.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-I
22
/1234567890/1of2_Lloyd_George_Record_[Joe Bloggs]_[1234567890]_[25-12-2019].pdf,,Y12345,1234567890,LG,,03/09/2022,NEC,NEC,04/10/2023
33
/1234567890/2of2_Lloyd_George_Record_[Joe Bloggs]_[1234567890]_[25-12-2019].pdf,,Y12345,1234567890,LG,,03/09/2022,NEC,NEC,04/10/2023
44
1of1_Lloyd_George_Record_[Joe Bloggs_invalid]_[123456789]_[25-12-2019].txt,,Y12345,123456789,LG,,04/09/2022,NEC,NEC,04/10/2023
5-
1of1_Lloyd_George_Record_[Jane Smith]_[1234567892]_[25-12-2019].txt,,Y12345,,LG,,04/09/2022,NEC,NEC,04/10/2023
5+
1of1_Lloyd_George_Record_[Jane Smith]_[1234567892]_[25-12-2019].txt,,Y12345,,LG,,04/09/2022,NEC,NEC,04/10/2023

‎lambdas/tests/unit/helpers/data/bulk_upload/metadata_invalid_empty_gp_practice_code.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD
22
/1234567890/1of2_Lloyd_George_Record_[Joe Bloggs]_[1234567890]_[25-12-2019].pdf,,,1234567890,LG,,03/09/2022,NEC,NEC,04/10/2023
33
/1234567890/2of2_Lloyd_George_Record_[Joe Bloggs]_[1234567890]_[25-12-2019].pdf,,,1234567890,LG,,03/09/2022,NEC,NEC,04/10/2023
4-
1of1_Lloyd_George_Record_[Joe Bloggs_invalid]_[1234567891]_[25-12-2019].txt,,Y12345,1234567891,LG,,04/09/2022,NEC,NEC,04/10/2023
4+
1of1_Lloyd_George_Record_[Joe Bloggs_invalid]_[1234567891]_[25-12-2019].txt,,Y12345,1234567891,LG,,04/09/2022,NEC,NEC,04/10/2023

‎lambdas/tests/unit/helpers/data/bulk_upload/metadata_with_duplicates_different_ods.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-I
66
1of1_Lloyd_George_Record_[Joe Bloggs_invalid]_[123456789]_[25-12-2019].txt,,Y12345,123456789,LG,,04/09/2022,NEC,NEC,04/10/2023
77
1of1_Lloyd_George_Record_[Joe Bloggs_invalid]_[123456789]_[25-12-2019].txt,,Y6789,123456789,LG,,04/09/2022,NEC,NEC,04/10/2023
88
1of1_Lloyd_George_Record_[Jane Smith]_[1234567892]_[25-12-2019].txt,,Y12345,,LG,,04/09/2022,NEC,NEC,04/10/2023
9-
1of1_Lloyd_George_Record_[Jane Smith]_[1234567892]_[25-12-2019].txt,,Y6789,,LG,,04/09/2022,NEC,NEC,04/10/2023
9+
1of1_Lloyd_George_Record_[Jane Smith]_[1234567892]_[25-12-2019].txt,,Y6789,,LG,,04/09/2022,NEC,NEC,04/10/2023

0 commit comments

Comments
 (0)