Skip to content

Update CSV import to better handle complex models, add download template functionality #11918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vouidaskis wants to merge 14 commits into
base: dev/8.1.x
Choose a base branch
from
+443 −119
Open

Update CSV import to better handle complex models, add download template functionality #11918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fb7722d
Allow import of values formatted as date in Excel (#11879)
chiatt Mar 26, 2025
c523295
Add keyboard inputs for moving overlays (#11877)
samuelscandrett Mar 26, 2025
7a84dbd
Update version to 7.6.8
chiatt Mar 27, 2025
3481494
Avoid unidiomatic type checking in date transform_value_for_tile()
jacobtylerwalls Mar 31, 2025
0ab1948
Add instructions to update frontend (#11908)
chiatt Apr 1, 2025
035958c
csv upgrade to load complex cases
vouidaskis Mar 17, 2025
96d7cef
small change code configuration
vouidaskis Mar 20, 2025
ee5ba4f
small change configuration code
vouidaskis Mar 20, 2025
8d108ff
change libraries
vouidaskis Mar 20, 2025
99af626
take all cases resourceid
vouidaskis Mar 26, 2025
3ffe1b4
upgrade sorted
vouidaskis Mar 31, 2025
d0f8187
update release
vouidaskis Apr 3, 2025
d77d291
Change files followe dev/8.1.x
vouidaskis Apr 17, 2025
ec9b631
delete files
vouidaskis Apr 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions arches/app/datatypes/concept_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
from django.utils.translation import gettext as _
from arches.app.models import models
from arches.app.models import concept
from arches.app.models.models import Value
from django.core.cache import cache
from arches.app.models.system_settings import settings
from arches.app.datatypes.base import BaseDataType
import re
from arches.app.datatypes.datatypes import DataTypeFactory, get_value_from_jsonld
from arches.app.models.concept import (
get_preflabel_from_valueid,
Expand Down Expand Up @@ -50,10 +52,20 @@ def __init__(self, model=None):
def lookup_label(self, label, collectionid):
ret = label
collection_values = self.collection_lookup[collectionid]

for concept in collection_values:
if concept[1] in (label, label.strip()):
ret = concept[2]
return ret

try:
uuid.UUID(str(ret))
return ret
except:
try:
valueid = Value.objects.get(value=ret).valueid
return str(valueid)
except:
return ret

def lookup_labelid_from_label(self, value, config):
if "rdmCollection" in config:
Expand Down Expand Up @@ -430,9 +442,8 @@ def validate(

def transform_value_for_tile(self, value, **kwargs):
ret = []
for val in csv.reader([value], delimiter=",", quotechar='"'):
lines = [line for line in val]
for v in lines:
for v in re.split(r',\s*(?![^()]*\))', value):

try:
stripped = v.strip()
uuid.UUID(stripped)
Expand Down
43 changes: 30 additions & 13 deletions arches/app/datatypes/datatypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2235,7 +2235,14 @@ def get_search_terms(self, nodevalue, nodeid=None):

def transform_value_for_tile(self, value, **kwargs):
Copy link
Member

@whatisgalen whatisgalen Apr 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've actually been working on this in a branch off dev/7.6.x to enable more flexibility of acceptable value formats (e.g. a string pointing to a resourceinstance.legacyid, etc.)

def transform_value_for_tile(self, value, **kwargs):
        # kwargs config looks like this:
        # {
        #     "graphs": [
        #         {
        #             "name": "Person or Group",
        #             "graphid": "ccbd1537-ac5e-11e6-84a5-026d961c88e6",
        #             "relationshipConcept": "6f26aa04-52af-4b17-a656-674c547ade2a",
        #             "relationshipCollection": "00000000-0000-0000-0000-000000000005",
        #             "useOntologyRelationship": False,
        #             "inverseRelationshipConcept": "6f26aa04-52af-4b17-a656-674c547ade2a"
        #         }
        #     ],
        #     "searchDsl": "",
        #     "searchString": ""
        # }
        from arches.app.search.search_engine_factory import SearchEngineFactory

        relatable_graphs = kwargs.get("graphs", [])
        default_values_lookup = dict()
        for graph in relatable_graphs:
            if graph.get("useOntologyRelationship", False) or not graph.get(
                "relationshipConcept", None
            ):
                default_values_lookup[graph["graphid"]] = {
                    "ontologyProperty": "",
                    "inverseOntologyProperty": "",
                }
            else:
                default_values_lookup[graph["graphid"]] = {
                    "ontologyProperty": graph["relationshipConcept"],
                    "inverseOntologyProperty": graph["inverseRelationshipConcept"],
                }

        def build_resource_instance_object(hit):
            return {
                "resourceId": hit["_id"],
                "ontologyProperty": (
                    default_values_lookup[hit["_source"]["graph_id"]][
                        "ontologyProperty"
                    ]
                ),
                "inverseOntologyProperty": (
                    default_values_lookup[hit["_source"]["graph_id"]][
                        "inverseOntologyProperty"
                    ]
                ),
                "resourceXresourceId": str(uuid.uuid4()),
            }

        subtypes_dict = {
            "uuid": uuid.UUID,
            "dict": dict,
            "str": str,
            "int": int,
            "float": float,
        }

        if isinstance(value, str):
            for test_method in [uuid.UUID, json.loads, ast.literal_eval]:
                try:
                    converted_value = test_method(value)
                    break
                except:
                    converted_value = False

            if converted_value is False and value != "":
                converted_value = value.split(",")  # is a string, likely legacyid
                converted_value = [val.strip() for val in converted_value if val]
                try:
                    converted_value = [uuid.UUID(val) for val in converted_value]
                except:
                    pass
            elif converted_value is False:
                logger.warning("ResourceInstanceDataType: value is empty")
                return []
        else:
            converted_value = value

        value_type = None
        if not isinstance(converted_value, list):
            converted_value = [converted_value]
        for value_subtype_label, value_subtype_class in list(subtypes_dict.items()):
            if isinstance(converted_value[0], value_subtype_class):
                value_type = value_subtype_label
                break

        se = SearchEngineFactory().create()
        query = Query(se)
        query.include("graph_id")
        boolquery = Bool()
        transformed_value = []

        match value_type:
            case "uuid":
                results = query.search(
                    index=RESOURCES_INDEX, id=[str(val) for val in converted_value]
                )
                for hit in results["docs"]:
                    transformed_value.append(build_resource_instance_object(hit))

            case "dict":  # assume data correctly parsed via ast.literal
                for val in converted_value:
                    try:
                        uuid.UUID(val["resourceId"])
                    except:
                        continue
                    transformed_value.append(val)
            case _:  # default case (handles str/legacyid and any other types)
                if value_type != "str":
                    converted_value = [str(val) for val in converted_value]
                boolquery.must(
                    Terms(field="legacyid.keyword", terms=converted_value)
                )  # exact match on keyword
                query.add_query(boolquery)
                results = query.search(index=RESOURCES_INDEX)
                print(f"{len(results['hits']['hits'])} hits")
                for hit in results["hits"]["hits"]:
                    transformed_value.append(build_resource_instance_object(hit))

        if len(transformed_value) == 0:
            logger.warning(
                f"ResourceInstanceDataType: no resources found for {converted_value}"
            )
            return
        return transformed_value

try:
return json.loads(value)
new_values=json.loads(value)
if isinstance(new_values, list):

for new_value in new_values:
new_value['resourceXresourceId']=uuid.uuid4()
else:
new_values['resourceXresourceId']=uuid.uuid4()
return new_values
except ValueError:
# do this if json (invalid) is formatted with single quotes, re #6390
try:
Expand All @@ -2250,21 +2257,31 @@ def transform_value_for_tile(self, value, **kwargs):
def transform_export_values(self, value, *args, **kwargs):
return json.dumps(value)

def append_in_list_search_filters(self, value, node, query):
values_list = value.get("val", [])
if values_list:
field_name = f"tiles.data.{node.pk}"
for val in values_list:
match_q = Term(
field=f"tiles.data.{node.pk}.resourceId.keyword",
term=val,
)

match value["op"]:
case "" | "in_list_any":
query.should(match_q)
case "in_list_all":
query.must(match_q)
case "!" | "in_list_none":
query.must_not(match_q)
query.filter(Exists(field=field_name))

def append_search_filters(self, value, node, query, request):
try:
if value["op"] == "null" or value["op"] == "not_null":
self.append_null_search_filters(value, node, query, request)
elif value["val"] != "" and value["val"] != []:
# search_query = Match(field="tiles.data.%s.resourceId" % (str(node.pk)), type="phrase", query=value["val"])
search_query = Terms(
field="tiles.data.%s.resourceId.keyword" % (str(node.pk)),
terms=value["val"],
)
if "!" in value["op"]:
query.must_not(search_query)
query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
else:
query.must(search_query)
else:
self.append_in_list_search_filters(value, node, query)
except KeyError as e:
pass

Expand Down Expand Up @@ -2514,4 +2531,4 @@ def default_es_mapping(self):
"type": {"type": "keyword"},
}
}
return mapping
return mapping
Loading
Loading