From c9119c48dc7682565e80d6b710505c167a795319 Mon Sep 17 00:00:00 2001 From: Emma Date: Tue, 8 Apr 2025 13:46:07 -0400 Subject: [PATCH 01/10] Fix required in export and added item and fields format --- datacontract/export/odcs_v3_exporter.py | 2 +- datacontract/imports/odcs_v3_importer.py | 28 ++++++- .../odcs_v3/adventureworks.datacontract.yml | 74 +++++++++++++++++++ .../fixtures/odcs_v3/adventureworks.odcs.yaml | 50 +++++++++++++ 4 files changed, 151 insertions(+), 3 deletions(-) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index f523ee387..a76b82c3a 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -217,7 +217,7 @@ def to_property(field_name: str, field: Field) -> dict: if field.description is not None: property["description"] = field.description if field.required is not None: - property["isNullable"] = not field.required + property["required"] = not field.required if field.unique is not None: property["isUnique"] = field.unique if field.classification is not None: diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 56e9998c7..822341d2b 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -17,6 +17,7 @@ Quality, Retention, Server, + ServerRole, ServiceLevel, Terms, ) @@ -121,7 +122,11 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") server.roles = odcs_server.get("roles") - + server.roles = [ServerRole(name = role.get("role"), + description = role.get("description"), + model_config = role + ) for role in odcs_server.get("roles")] if odcs_server.get("roles") is not None else None + server.storageAccount = odcs_server.get("storageAccount") servers[server_name] = server return servers @@ -264,7 +269,7 @@ def import_fields( description=" ".join(description.splitlines()) if description is not None else None, type=mapped_type, title=odcs_property.get("businessName"), - required=not odcs_property.get("nullable") if odcs_property.get("nullable") is not None else False, + required= odcs_property.get("required") if odcs_property.get("required") is not None else False, primaryKey=odcs_property.get("primaryKey") if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None else False, @@ -275,8 +280,27 @@ def import_fields( else "", tags=odcs_property.get("tags") if odcs_property.get("tags") is not None else None, quality=odcs_property.get("quality") if odcs_property.get("quality") is not None else [], + fields=import_fields(odcs_property.get("properties"), custom_type_mappings, server_type) + if odcs_property.get("properties") is not None else {}, config=import_field_config(odcs_property, server_type), + format=odcs_property.get("format") if odcs_property.get("format") is not None else None, ) + #mapped_type is array + if field.type == "array" and odcs_property.get("items") is not None : + #nested array object + if odcs_property.get("items").get("logicalType") == "object": + field.items= Field(type="object", + fields=import_fields(odcs_property.get("items").get("properties"), custom_type_mappings, server_type)) + #array of simple type + elif odcs_property.get("items").get("logicalType") is not None: + field.items= Field(type = odcs_property.get("items").get("logicalType")) + + # enum from quality validValues as enum + if field.type is "string": + for q in field.quality: + if hasattr(q,"validValues"): + field.enum = q.validValues + result[property_name] = field else: logger.info( diff --git a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml index a55dec04f..9a8a6ed04 100644 --- a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml +++ b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml @@ -4863,3 +4863,77 @@ models: criticalDataElement: false partitioned: false physicalType: timestamp + StoreHolidayHours: + title: StoreHolidayHours + type: array + required: false + primaryKey: false + unique: false + classification: '' + items: + type: object + fields: + Date: + title: Date + type: date + required: false + primaryKey: false + classification: '' + examples: + - '2024-08-13' + config: + physicalType: string + Close: + title: Close + type: date + required: false + primaryKey: false + classification: '' + examples: + - 02:00 PM + config: + physicalType: string + Open: + title: Open + type: date + required: false + primaryKey: false + classification: '' + examples: + - 10:00 AM + config: + physicalType: string + config: + physicalType: array + extendedData: + title: extendedData + type: object + required: true + primaryKey: false + unique: false + classification: '' + fields: + pharmacyUUID: + title: pharmacyUUID + type: string + required: true + primaryKey: false + unique: true + classification: '' + examples: + - ec43dd63-c258-4506-8965-88a9e0c130ad + config: + physicalType: string + config: + physicalType: object + ArrayComments: + title: ArrayComments + type: array + required: false + primaryKey: false + unique: false + classification: '' + items: + type: string + config: + physicalType: array \ No newline at end of file diff --git a/tests/fixtures/odcs_v3/adventureworks.odcs.yaml b/tests/fixtures/odcs_v3/adventureworks.odcs.yaml index c5e2a42dc..b1fed2cbc 100644 --- a/tests/fixtures/odcs_v3/adventureworks.odcs.yaml +++ b/tests/fixtures/odcs_v3/adventureworks.odcs.yaml @@ -5320,4 +5320,54 @@ schema: criticalDataElement: false primaryKey: false required: false + - name: StoreHolidayHours + businessName: StoreHolidayHours + logicalType: array + physicalType: array + required: false + unique: false + items: + logicalType: object + properties: + - name: Date + businessName: Date + logicalType: date + physicalType: string + examples: + - "2024-08-13" + - name: Close + businessName: Close + logicalType: date + physicalType: string + examples: + - "02:00 PM" + - name: Open + businessName: Open + logicalType: date + physicalType: string + examples: + - "10:00 AM" + - name: extendedData + businessName: extendedData + logicalType: object + physicalType: object + required: true + unique: false + properties: + - name : pharmacyUUID + businessName: pharmacyUUID + logicalType: string + physicalType: string + required: true + unique: true + examples: + - "ec43dd63-c258-4506-8965-88a9e0c130ad" + - name: ArrayComments + businessName: ArrayComments + logicalType: array + physicalType: array + required: false + unique: false + items: + logicalType: string contractCreatedTs: "2023-09-28T20:24:49.331+00:00" From 26281e29324f641b567823fe37be86f146c27b4e Mon Sep 17 00:00:00 2001 From: Emma Date: Tue, 8 Apr 2025 13:49:34 -0400 Subject: [PATCH 02/10] Removed not from not field.required --- datacontract/export/odcs_v3_exporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/export/odcs_v3_exporter.py b/datacontract/export/odcs_v3_exporter.py index a76b82c3a..fc8ae56f7 100644 --- a/datacontract/export/odcs_v3_exporter.py +++ b/datacontract/export/odcs_v3_exporter.py @@ -217,7 +217,7 @@ def to_property(field_name: str, field: Field) -> dict: if field.description is not None: property["description"] = field.description if field.required is not None: - property["required"] = not field.required + property["required"] = field.required if field.unique is not None: property["isUnique"] = field.unique if field.classification is not None: From 79157678306c3cdbb7168bceaa26005758fd75b2 Mon Sep 17 00:00:00 2001 From: ezhao-mck Date: Tue, 8 Apr 2025 15:54:53 -0400 Subject: [PATCH 03/10] Update odcs_v3_importer.py --- datacontract/imports/odcs_v3_importer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index f88a992b0..1af0e42c5 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -133,10 +133,7 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.dataProductId = odcs_server.get("dataProductId") server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") - server.roles = [ServerRole(name = role.get("role"), - description = role.get("description"), - model_config = role - ) for role in odcs_server.get("roles")] if odcs_server.get("roles") is not None else None + server.roles = import_server_roles(odcs_server.get("roles")) server.storageAccount = odcs_server.get("storageAccount") servers[server_name] = server return servers @@ -280,7 +277,7 @@ def import_fields( description=" ".join(description.splitlines()) if description is not None else None, type=mapped_type, title=odcs_property.get("businessName"), - required= odcs_property.get("required") if odcs_property.get("required") is not None else None, + required=odcs_property.get("required") if odcs_property.get("required") is not None else None, primaryKey=odcs_property.get("primaryKey") if not has_composite_primary_key(odcs_properties) and odcs_property.get("primaryKey") is not None else False, From 76a01f6c4760274a269f3b891a000ccde95497b6 Mon Sep 17 00:00:00 2001 From: ezhao-mck Date: Wed, 9 Apr 2025 10:06:21 -0400 Subject: [PATCH 04/10] Update odcs_v3_importer.py Changed server.storageAccount --- datacontract/imports/odcs_v3_importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 1af0e42c5..7c7889fbe 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -4,6 +4,7 @@ from venv import logger import yaml +import re from datacontract.imports.importer import Importer from datacontract.lint.resources import read_resource @@ -134,7 +135,7 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") server.roles = import_server_roles(odcs_server.get("roles")) - server.storageAccount = odcs_server.get("storageAccount") + server.storageAccount = re.search(r"@([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None servers[server_name] = server return servers From 5877ea2ece61d356c0d9db48481820c21ade5e75 Mon Sep 17 00:00:00 2001 From: ezhao-mck Date: Wed, 9 Apr 2025 10:13:34 -0400 Subject: [PATCH 05/10] Updated regex requirements --- datacontract/imports/odcs_v3_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 7c7889fbe..68a3031db 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -135,7 +135,7 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") server.roles = import_server_roles(odcs_server.get("roles")) - server.storageAccount = re.search(r"@([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None + server.storageAccount = re.search(r"(?:@|abfss://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None servers[server_name] = server return servers From a32e1584171415127c36f9cac03b5ddb9fb56258 Mon Sep 17 00:00:00 2001 From: Emma Date: Wed, 9 Apr 2025 10:41:25 -0400 Subject: [PATCH 06/10] Updated regex to be more universal --- datacontract/imports/odcs_v3_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 68a3031db..45163260b 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -135,7 +135,7 @@ def import_servers(odcs_contract: Dict[str, Any]) -> Dict[str, Server] | None: server.outputPortId = odcs_server.get("outputPortId") server.driver = odcs_server.get("driver") server.roles = import_server_roles(odcs_server.get("roles")) - server.storageAccount = re.search(r"(?:@|abfss://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None + server.storageAccount = re.search(r"(?:@|://)([^.]+)\.",odcs_server.get("location"),re.IGNORECASE) if server.type == "azure" else None servers[server_name] = server return servers From 8b510423376eb2b08aeeb27bb6314489e3f2ca37 Mon Sep 17 00:00:00 2001 From: Emma Date: Thu, 10 Apr 2025 09:23:00 -0400 Subject: [PATCH 07/10] Fixed formatting --- datacontract/imports/odcs_v3_importer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 45163260b..0693129b5 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -1,10 +1,11 @@ import datetime import logging +import re from typing import Any, Dict, List from venv import logger import yaml -import re + from datacontract.imports.importer import Importer from datacontract.lint.resources import read_resource @@ -305,7 +306,7 @@ def import_fields( field.items= Field(type = odcs_property.get("items").get("logicalType")) # enum from quality validValues as enum - if field.type is "string": + if field.type == "string": for q in field.quality: if hasattr(q,"validValues"): field.enum = q.validValues From 07487b8ba5f5e8dce781c06ee94f1916ce9b429e Mon Sep 17 00:00:00 2001 From: Emma Date: Thu, 10 Apr 2025 09:32:47 -0400 Subject: [PATCH 08/10] Fixed more formatting --- datacontract/imports/odcs_v3_importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datacontract/imports/odcs_v3_importer.py b/datacontract/imports/odcs_v3_importer.py index 0693129b5..c64224327 100644 --- a/datacontract/imports/odcs_v3_importer.py +++ b/datacontract/imports/odcs_v3_importer.py @@ -6,7 +6,6 @@ import yaml - from datacontract.imports.importer import Importer from datacontract.lint.resources import read_resource from datacontract.model.data_contract_specification import ( From 9863cd24d3297d779c2d38ddd8ccfd46f28423e1 Mon Sep 17 00:00:00 2001 From: Damien Maresma Date: Thu, 17 Apr 2025 14:06:29 -0400 Subject: [PATCH 09/10] fix turn nullable to required in test --- tests/test_export_odcs_v3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_export_odcs_v3.py b/tests/test_export_odcs_v3.py index ee922aee1..c82d2a93c 100644 --- a/tests/test_export_odcs_v3.py +++ b/tests/test_export_odcs_v3.py @@ -46,7 +46,7 @@ def test_to_odcs(): maxLength: 10 pattern: ^B[0-9]+$ physicalType: varchar - nullable: false + required: true unique: true tags: - "order_id" @@ -65,7 +65,7 @@ def test_to_odcs(): minimum: 0 maximum: 1000000 physicalType: bigint - nullable: false + required: true description: The order_total field quality: - type: sql @@ -77,7 +77,7 @@ def test_to_odcs(): - name: order_status logicalType: string physicalType: text - nullable: false + required: true quality: - type: sql description: Row Count From a9294d07fa2de1bc100abf575c162b6b85addb48 Mon Sep 17 00:00:00 2001 From: Damien Maresma Date: Thu, 17 Apr 2025 18:13:58 -0400 Subject: [PATCH 10/10] fix PR issues --- tests/fixtures/odcs_v3/adventureworks.datacontract.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml index 1adb9e113..969d21a0b 100644 --- a/tests/fixtures/odcs_v3/adventureworks.datacontract.yml +++ b/tests/fixtures/odcs_v3/adventureworks.datacontract.yml @@ -4413,16 +4413,13 @@ models: required: false primaryKey: false unique: false - classification: '' items: type: object fields: Date: title: Date type: date - required: false primaryKey: false - classification: '' examples: - '2024-08-13' config: @@ -4430,9 +4427,7 @@ models: Close: title: Close type: date - required: false primaryKey: false - classification: '' examples: - 02:00 PM config: @@ -4440,9 +4435,7 @@ models: Open: title: Open type: date - required: false primaryKey: false - classification: '' examples: - 10:00 AM config: @@ -4455,7 +4448,6 @@ models: required: true primaryKey: false unique: false - classification: '' fields: pharmacyUUID: title: pharmacyUUID @@ -4463,7 +4455,6 @@ models: required: true primaryKey: false unique: true - classification: '' examples: - ec43dd63-c258-4506-8965-88a9e0c130ad config: @@ -4476,7 +4467,6 @@ models: required: false primaryKey: false unique: false - classification: '' items: type: string config: