Fixing bug with single and double quotes inside values

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
-## [0.2.0]
+## [0.2.1]
 ### Bug
+- Addressing issues with single and double quote characters inside filtered values.
+
+## [0.2.0]
+### Added
 - Adding ability to compare with variables.
 - Adding condition priority.
 

package.json

@@ -1,6 +1,6 @@
 {
     "name": "js-array-filter",
-    "version": "0.2.0",
+    "version": "0.2.1",
     "description": "Apply filter to an array",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",

src/utils/filterExpression.ts

@@ -105,7 +105,10 @@ const parseLiteralValue = (rawValue: string, columnType: ItemTypeParsed, isMulti
         if (match === null) {
             throw new Error(`Invalid string value ${rawValue}`);
         }
-        return match[2];
+        const quote = match[1];
+        const value = match[2];
+        const escapedQuote = quote === '"' ? '\\\\"' : "\\\\'";
+        return value.replace(new RegExp(escapedQuote, "g"), quote);
     }
 
     const items = rawValue.trim().replace(/^\((.*)\)$/s, "$1");
@@ -118,7 +121,10 @@ const parseLiteralValue = (rawValue: string, columnType: ItemTypeParsed, isMulti
         if (match === null) {
             throw new Error(`Invalid string value ${item}`);
         }
-        return match[2];
+        const quote = match[1];
+        const value = match[2];
+        const escapedQuote = quote === '"' ? '\\\\"' : "\\\\'";
+        return value.replace(new RegExp(escapedQuote, "g"), quote);
     });
 };
 
@@ -199,17 +205,18 @@ const readConditionText = (state: ParserState): string => {
     let nestedDepth = 0;
     let inSingle = false;
     let inDouble = false;
+    let previousCharWasEscape = false;
 
     while (state.position < state.input.length) {
         const character = state.input[state.position];
 
-        if (character === '"' && !inSingle) {
+        if (character === '"' && !inSingle && !previousCharWasEscape) {
             inDouble = !inDouble;
             state.position += 1;
             continue;
         }
 
-        if (character === "'" && !inDouble) {
+        if (character === "'" && !inDouble && !previousCharWasEscape) {
             inSingle = !inSingle;
             state.position += 1;
             continue;
@@ -247,6 +254,11 @@ const readConditionText = (state: ParserState): string => {
             }
         }
 
+        if (character === "\\" && !previousCharWasEscape) {
+            previousCharWasEscape = true;
+        } else {
+            previousCharWasEscape = false;
+        }
         state.position += 1;
     }
 
@@ -439,12 +451,15 @@ export const conditionToString = (condition: FilterCondition): string => {
             if (value.length === 0) {
                 valueString = "()";
             } else if (typeof value[0] === "string") {
-                valueString = `("${value.join('", "')}")`;
+                const escapedValues = value.map((v) => (v as string).replace(/"/g, '\\"'));
+                valueString = `("${escapedValues.join('", "')}")`;
             } else {
                 valueString = `(${value.join(", ")})`;
             }
         } else if (typeof value === "string") {
-            valueString = `"${value}"`;
+            // Escape double quotes in the string value
+            const escapedValue = value.replace(/"/g, '\\"');
+            valueString = `"${escapedValue}"`;
         } else {
             valueString = String(value);
         }

src/utils/filterRegex.ts

@@ -1,7 +1,7 @@
 export const filterRegex = {
     variable: /\w+/,
     variableParse: /(\w+)/,
-    itemString: /["][^"]*?["]|['][^']*?[']|null/i,
+    itemString: /"(?:\\"|[^"])*"|'(?:\\'|[^'])*'|null/i,
     itemNumber: /[^'",][^,\s]*|null/i,
     itemBoolean: /True|False|null/i,
     item: / /,

test/filterToString.test.ts

@@ -413,4 +413,19 @@ describe("filterToString", () => {
             '(age > 50 or age in (20, 30, 40) or trt01p != trt01a) and (name in ("John", "Dave") or trt01p = "Placebo")',
         );
     });
+    it("should escape double quotes correctly", () => {
+        const filter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: 'John "Doe"' },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+        };
+        const columns = filter.conditions.map((condition) => ({
+            name: condition.variable,
+            dataType: typeof condition.value === "string" ? "string" : ("integer" as ItemTypeDatasetJson),
+        })) as ColumnMetadataDatasetJson[];
+        const expectedString = 'name = "John \\"Doe\\"" and age > 30';
+        expect(new Filter("dataset-json1.1", columns, filter).toString()).toBe(expectedString);
+    });
 });

test/stringToFilter.test.ts

@@ -326,4 +326,97 @@ describe("stringToFilter", () => {
 
         expect(new Filter("parsed", columns, filterString).toBasicFilter()).toEqual(expectedFilter);
     });
+    it("should create filter with a value having multiple double quotes", () => {
+        const filterString = 'name = "John \\"Doe\\"" and age > 30';
+        const expectedFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: 'John "Doe"' },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+        };
+
+        const filter = new Filter("parsed", columns, filterString).toBasicFilter();
+
+        expect(filter).toEqual(expectedFilter);
+    });
+    it("should create filter with a value having single double quote", () => {
+        const filterString = 'name = "John \\"Doe" and age > 30';
+        const expectedFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: 'John "Doe' },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+        };
+
+        const filter = new Filter("parsed", columns, filterString).toBasicFilter();
+
+        expect(filter).toEqual(expectedFilter);
+    });
+    it("should create filter with a value having multiple single quotes", () => {
+        const filterString = "name = 'John \\'Doe\\'' and age > 30";
+        const expectedFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: "John 'Doe'" },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+        };
+
+        const filter = new Filter("parsed", columns, filterString).toBasicFilter();
+
+        expect(filter).toEqual(expectedFilter);
+    });
+    it("should create filter with a value having one single quote", () => {
+        const filterString = "name = 'John \\'Doe' and age > 30";
+        const expectedFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: "John 'Doe" },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+        };
+
+        const filter = new Filter("parsed", columns, filterString).toBasicFilter();
+
+        expect(filter).toEqual(expectedFilter);
+    });
+    it("should pass conversion round trip for a value with escaped double quotes", () => {
+        const filterString = 'name = "John \\"Doe\\"" and age > 30';
+        const basicFilter = new Filter("parsed", columns, filterString).toBasicFilter();
+        const newFilter = new Filter("parsed", columns, basicFilter).toString();
+
+        expect(newFilter).toEqual(filterString);
+    });
+    it("should pass conversion round trip for a value with escaped single quotes", () => {
+        const filterString = "name = 'John \\'Doe' and age > 30";
+        const basicFilter = new Filter("parsed", columns, filterString).toBasicFilter();
+        const newFilter = new Filter("parsed", columns, basicFilter).toString();
+        const expectedFilterString = 'name = "John \'Doe" and age > 30';
+
+        expect(newFilter).toEqual(expectedFilterString);
+    });
+    it("should save options in basic filter", () => {
+        const sourceFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: "John" },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+            options: { caseInsensitive: true },
+        };
+        const basicFilter = new Filter("parsed", columns, sourceFilter).toBasicFilter();
+
+        const expectedFilter: BasicFilter = {
+            conditions: [
+                { variable: "name", operator: "eq", value: "John" },
+                { variable: "age", operator: "gt", value: 30 },
+            ],
+            connectors: ["and"],
+            options: { caseInsensitive: true },
+        };
+
+        expect(basicFilter).toEqual(expectedFilter);
+    });
 });

test/validateFilterString.test.ts

@@ -101,4 +101,12 @@ describe("validateFilterString", () => {
         const filterString = "name = age";
         expect(new Filter("parsed", columns, "").validateFilterString(filterString)).toBe(false);
     });
+    it("should validate a filter string with escaped single quotes", () => {
+        const filterString = "name = 'John \\'Doe'";
+        expect(new Filter("parsed", columns, "").validateFilterString(filterString)).toBe(true);
+    });
+    it("should validate a filter string with escaped double quotes", () => {
+        const filterString = 'name = "John \\"Doe\\""';
+        expect(new Filter("parsed", columns, "").validateFilterString(filterString)).toBe(true);
+    });
 });