Skip to content

Commit 0fadc7f

Browse files
authored
Merge pull request #91 from tharropoulos/dot-notation
feat(utils): implement nested field extraction
2 parents 6736da0 + f4507c3 commit 0fadc7f

File tree

7 files changed

+326
-29
lines changed

7 files changed

+326
-29
lines changed

functions/package-lock.json

Lines changed: 11 additions & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

functions/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"@babel/runtime": "^7.24.7",
1919
"firebase-admin": "^12.2.0",
2020
"firebase-functions": "^5.0.1",
21-
"flat": "^6.0.1",
21+
"lodash.get": "^4.4.2",
2222
"typesense": "^1.8.2"
2323
},
2424
"devDependencies": {

functions/src/utils.js

Lines changed: 106 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,27 +27,124 @@ const mapValue = (value) => {
2727
}
2828
};
2929

30+
/**
31+
* Sets a nested value in an object using a dot-notated path.
32+
* @param {Object} obj - The object to modify.
33+
* @param {string} path - The dot-notated path to the value.
34+
* @param {*} value - The value to set.
35+
* @return {Object} The modified object.
36+
*/
37+
function setNestedValue(obj, path, value) {
38+
const keys = path.split(".");
39+
keys.reduce((acc, key, index) => {
40+
if (index === keys.length - 1) {
41+
acc[key] = value;
42+
return acc;
43+
}
44+
if (acc[key] === undefined) {
45+
acc[key] = Number.isInteger(+keys[index + 1]) ? [] : {};
46+
}
47+
return acc[key];
48+
}, obj);
49+
return obj;
50+
}
51+
52+
/**
53+
* Gets a nested value from an object using a dot-notated path.
54+
* @param {Object} obj - The object to retrieve the value from.
55+
* @param {string} path - The dot-notated path to the value.
56+
* @return {*} The value at the specified path, or undefined if not found.
57+
*/
58+
function getNestedValue(obj, path) {
59+
const keys = path.split(".");
60+
return keys.reduce((current, key) => {
61+
if (current === undefined) return undefined;
62+
if (Array.isArray(current)) {
63+
return Number.isInteger(+key) ? current[+key] : current.map((item) => ({[key]: item[key]}));
64+
}
65+
return current[key];
66+
}, obj);
67+
}
68+
69+
/**
70+
* Merges an array of objects into a single array, combining objects at the same index.
71+
* @param {Array<Object[]>} arrays - An array of object arrays to merge.
72+
* @return {Object[]} A merged array of objects.
73+
*/
74+
function mergeArrays(arrays) {
75+
const maxLength = Math.max(...arrays.map((arr) => arr.length));
76+
return Array.from({length: maxLength}, (_, i) => Object.assign({}, ...arrays.map((arr) => arr[i] || {})));
77+
}
78+
79+
/**
80+
* Extracts a field from the data and adds it to the accumulator.
81+
* @param {Object} data - The source data object.
82+
* @param {Object} acc - The accumulator object.
83+
* @param {string} field - The field to extract.
84+
* @return {Object} The updated accumulator.
85+
*/
86+
function extractField(data, acc, field) {
87+
const value = getNestedValue(data, field);
88+
if (value === undefined) return acc;
89+
const [topLevelField] = field.split(".");
90+
const isArrayOfObjects = Array.isArray(value) && typeof value[0] === "object";
91+
if (isArrayOfObjects) {
92+
return {
93+
...acc,
94+
[topLevelField]: acc[topLevelField] ? mergeArrays([acc[topLevelField], value]) : value,
95+
};
96+
} else {
97+
return setNestedValue(acc, field, value);
98+
}
99+
}
100+
101+
/**
102+
* Flattens a nested object, converting nested properties to dot-notation.
103+
* @param {Object} obj - The object to flatten.
104+
* @param {string} [prefix=""] - The prefix to use for flattened keys.
105+
* @return {Object} A new flattened object.
106+
*/
107+
function flattenDocument(obj, prefix = "") {
108+
return Object.keys(obj).reduce((acc, key) => {
109+
const newKey = prefix ? `${prefix}.${key}` : key;
110+
const value = obj[key];
111+
// Handle primitive values (including null)
112+
if (typeof value !== "object" || value === null) {
113+
acc[newKey] = value;
114+
return acc;
115+
}
116+
// Handle arrays
117+
if (Array.isArray(value)) {
118+
if (value.length === 0 || typeof value[0] !== "object") {
119+
acc[newKey] = value;
120+
return acc;
121+
}
122+
Object.keys(value[0]).forEach((subKey) => {
123+
acc[`${newKey}.${subKey}`] = value.map((item) => item[subKey]).filter((v) => v !== undefined);
124+
});
125+
return acc;
126+
}
127+
// Handle nested objects
128+
return {...acc, ...flattenDocument(value, newKey)};
129+
}, {});
130+
}
131+
30132
/**
31133
* @param {DocumentSnapshot} firestoreDocumentSnapshot
32134
* @param {Array} fieldsToExtract
33135
* @return {Object} typesenseDocument
34136
*/
35137
exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, fieldsToExtract = config.firestoreCollectionFields) => {
36-
const flat = await import("flat");
37138
const data = firestoreDocumentSnapshot.data();
38139

39-
let entries = Object.entries(data);
40-
41-
if (fieldsToExtract.length) {
42-
entries = entries.filter(([key]) => fieldsToExtract.includes(key));
43-
}
140+
const extractedData = fieldsToExtract.length === 0 ? data : fieldsToExtract.reduce((acc, field) => extractField(data, acc, field), {});
44141

45-
// Build a document with just the fields requested by the user, and mapped from Firestore types to Typesense types
46-
const mappedDocument = Object.fromEntries(entries.map(([key, value]) => [key, mapValue(value)]));
142+
const mappedDocument = Object.fromEntries(Object.entries(extractedData).map(([key, value]) => [key, mapValue(value)]));
47143

48144
// using flat to flatten nested objects for older versions of Typesense that did not support nested fields
49145
// https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields
50-
const typesenseDocument = config.shouldFlattenNestedDocuments ? flat.flatten(mappedDocument, {safe: true}) : mappedDocument;
146+
const typesenseDocument = config.shouldFlattenNestedDocuments ? flattenDocument(mappedDocument) : mappedDocument;
147+
console.log("typesenseDocument", typesenseDocument);
51148

52149
typesenseDocument.id = firestoreDocumentSnapshot.id;
53150

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
"emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
66
"export": "firebase emulators:export emulator_data",
77
"test": "npm run test-part-1 && npm run test-part-2",
8-
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=test/indexOnWriteWithoutFlattening.spec.js'",
9-
"test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathPattern=test/indexOnWriteWithoutFlattening.spec.js'",
8+
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\"'",
9+
"test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
1010
"typesenseServer": "docker compose up",
1111
"lint:fix": "eslint . --fix",
1212
"lint": "eslint ."
File renamed without changes.

test/utils.spec.js renamed to test/utilsWithFlattening.spec.js

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,5 +117,87 @@ describe("Utils", () => {
117117
});
118118
});
119119
});
120+
describe("Nested fields extraction", () => {
121+
it("extracts nested fields using dot notation", async () => {
122+
const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
123+
const documentSnapshot = test.firestore.makeDocumentSnapshot(
124+
{
125+
user: {
126+
name: "John Doe",
127+
address: {
128+
city: "New York",
129+
country: "USA",
130+
},
131+
},
132+
tags: ["tag1", "tag2"],
133+
},
134+
"id",
135+
);
136+
const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["user.name", "user.address.city", "tags"]);
137+
expect(result).toEqual({
138+
id: "id",
139+
"user.name": "John Doe",
140+
"user.address.city": "New York",
141+
tags: ["tag1", "tag2"],
142+
});
143+
});
144+
145+
it("handles missing nested fields gracefully", async () => {
146+
const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
147+
const documentSnapshot = test.firestore.makeDocumentSnapshot(
148+
{
149+
user: {
150+
name: "John Doe",
151+
},
152+
},
153+
"id",
154+
);
155+
const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["user.name", "user.address.city"]);
156+
expect(result).toEqual({
157+
id: "id",
158+
"user.name": "John Doe",
159+
});
160+
});
161+
162+
it("extracts nested fields alongside top-level fields", async () => {
163+
const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
164+
const documentSnapshot = test.firestore.makeDocumentSnapshot(
165+
{
166+
title: "Main Title",
167+
user: {
168+
name: "John Doe",
169+
age: 30,
170+
},
171+
},
172+
"id",
173+
);
174+
const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["title", "user.name"]);
175+
expect(result).toEqual({
176+
id: "id",
177+
title: "Main Title",
178+
"user.name": "John Doe",
179+
});
180+
});
181+
182+
it("handles array indexing in dot notation", async () => {
183+
const typesenseDocumentFromSnapshot = (await import("../functions/src/utils.js")).typesenseDocumentFromSnapshot;
184+
const documentSnapshot = test.firestore.makeDocumentSnapshot(
185+
{
186+
comments: [
187+
{author: "Alice", text: "Great post!"},
188+
{author: "Bob", text: "Thanks for sharing.", likes: 5},
189+
],
190+
},
191+
"id",
192+
);
193+
const result = await typesenseDocumentFromSnapshot(documentSnapshot, ["comments.author", "comments.text", "comments.likes"]);
194+
expect(result).toEqual({
195+
id: "id",
196+
"comments.author": ["Alice", "Bob"],
197+
"comments.text": ["Great post!", "Thanks for sharing."],
198+
"comments.likes": [5],
199+
});
200+
});
201+
});
120202
});
121203
});

0 commit comments

Comments
 (0)