Add include/exclude logic for database service

packages/core/components/AnnotationFilterForm/SearchBoxForm/SearchBoxForm.module.css

@@ -2,6 +2,10 @@
     margin-bottom: 6px;
 }
 
+.toggle-hidden {
+    display: none;
+}
+
 .toggle input:disabled + label {
     cursor: not-allowed;
     opacity: 0.5;

packages/core/components/AnnotationFilterForm/SearchBoxForm/index.tsx

@@ -19,6 +19,7 @@ interface SearchBoxFormProps {
     onSearch: (filterValue: string, type: FilterType) => void;
     fieldName: string;
     fuzzySearchEnabled?: boolean;
+    hideFuzzyToggle?: boolean;
     defaultValue: FileFilter | undefined;
 }
 
@@ -38,7 +39,9 @@ export default function SearchBoxForm(props: SearchBoxFormProps) {
             <h3 className={styles.title}>{props.title}</h3>
             <Toggle
                 label="Fuzzy search"
-                className={styles.toggle}
+                className={classNames(styles.toggle, {
+                    [styles.toggleHidden]: !!props?.hideFuzzyToggle,
+                })}
                 defaultChecked={isFuzzySearching}
                 onText="On"
                 inlineLabel

packages/core/components/AnnotationFilterForm/index.tsx

@@ -33,6 +33,7 @@ export default function AnnotationFilterForm(props: AnnotationFilterFormProps) {
     const dispatch = useDispatch();
     const allFilters = useSelector(selection.selectors.getFileFilters);
     const fuzzyFilters = useSelector(selection.selectors.getFuzzyFilters);
+    const canFuzzySearch = useSelector(selection.selectors.isQueryingAicsFms);
     const annotationService = useSelector(interaction.selectors.getAnnotationService);
     const [annotationValues, isLoading, errorMessage] = useAnnotationValues(
         props.annotation.name,
@@ -196,6 +197,7 @@ export default function AnnotationFilterForm(props: AnnotationFilterFormProps) {
                             fuzzySearchEnabled={fuzzySearchEnabled}
                             fieldName={props.annotation.displayName}
                             defaultValue={filtersForAnnotation?.[0]}
+                            hideFuzzyToggle={!canFuzzySearch}
                         />
                     );
                 }

packages/core/entity/FileFilter/index.ts

@@ -1,3 +1,5 @@
+import SQLBuilder from "../SQLBuilder";
+
 export interface FileFilterJson {
     name: string;
     value: any;
@@ -71,6 +73,22 @@ export default class FileFilter {
         return `${this.annotationName}=${this.annotationValue}`;
     }
 
+    /** Unlike with FileSort, we shouldn't construct a new SQLBuilder since these will
+     * be applied to a pre-existing SQLBuilder.
+     * Instead, generate the string that can be passed into the .where() clause.
+     */
+    public toSQLWhereString(): string {
+        switch (this.type) {
+            case FilterType.ANY:
+                return `"${this.annotationName}" IS NOT NULL`;
+            case FilterType.EXCLUDE:
+                return `"${this.annotationName}" IS NULL`;
+            case FilterType.FUZZY:
+            default:
+                return SQLBuilder.regexMatchValueInList(this.annotationName, this.annotationValue);
+        }
+    }
+
     public toJSON(): FileFilterJson {
         return {
             name: this.annotationName,

packages/core/entity/FileSet/index.ts

@@ -187,30 +187,21 @@ export default class FileSet {
      */
     public toQuerySQLBuilder(): SQLBuilder {
         // Map the filter values to the annotation names they filter
-        const filterValuesByAnnotation = this.filters.reduce(
+        const filtersGroupedByAnnotation = this.filters.reduce(
             (map, filter) => ({
                 ...map,
-                [filter.name]:
-                    filter.name in map ? [...map[filter.name], filter.value] : [filter.value],
+                [filter.name]: filter.name in map ? [...map[filter.name], filter] : [filter],
             }),
-            {} as { [name: string]: string[] }
+            {} as { [name: string]: FileFilter[] }
         );
 
         // Transform the map above into SQL comparison clauses
         const sqlBuilder = this.sort ? this.sort.toQuerySQLBuilder() : new SQLBuilder();
 
-        Object.entries(filterValuesByAnnotation).forEach(([annotation, filterValues]) => {
-            // If a filter value is `null` then we need to modify the way we approach filtering
-            // it in SQL
-            if (filterValues.length === 0) {
-                sqlBuilder.where(`"${annotation}" IS NOT NULL`);
-            } else {
-                sqlBuilder.where(
-                    filterValues
-                        .map((fv) => SQLBuilder.regexMatchValueInList(annotation, fv))
-                        .join(") OR (")
-                );
-            }
+        Object.entries(filtersGroupedByAnnotation).forEach(([_, appliedFilters]) => {
+            sqlBuilder.where(
+                appliedFilters.map((filter) => filter.toSQLWhereString()).join(") OR (")
+            );
         });
 
         return sqlBuilder;

packages/core/entity/FileSet/test/FileSet.test.ts

@@ -14,6 +14,7 @@ import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileD
 
 describe("FileSet", () => {
     const scientistEqualsJane = new FileFilter("scientist", "jane");
+    const scientistEqualsJohn = new FileFilter("scientist", "john");
     const matrigelIsHard = new FileFilter("matrigel_is_hardened", true);
     const dateCreatedDescending = new FileSort("date_created", SortOrder.DESC);
     const fuzzyFileName = new FuzzyFilter("file_name");
@@ -86,6 +87,50 @@ describe("FileSet", () => {
         });
     });
 
+    describe("toQuerySQLBuilder", () => {
+        const mockDatasource = "testSource";
+
+        it("builds SQL queries with include filters", () => {
+            const fileSet = new FileSet({ filters: [anyGene] });
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'WHERE ("gene" IS NOT NULL'
+            );
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).not.to.contain(
+                "IS NULL"
+            );
+        });
+
+        it("builds SQL queries with exclude filters", () => {
+            const fileSet = new FileSet({ filters: [noCellBatch] });
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'WHERE ("cell_batch" IS NULL'
+            );
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).not.to.contain(
+                "NOT NULL"
+            );
+        });
+
+        it("builds SQL queries with regular filters for different annotations", () => {
+            const fileSet = new FileSet({ filters: [scientistEqualsJane, matrigelIsHard] });
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'WHERE (REGEXP_MATCHES("scientist"'
+            );
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'AND (REGEXP_MATCHES("matrigel_is_hardened"'
+            );
+        });
+
+        it("builds SQL queries with regular filters for same annotation with different values", () => {
+            const fileSet = new FileSet({ filters: [scientistEqualsJane, scientistEqualsJohn] });
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'WHERE (REGEXP_MATCHES("scientist"'
+            );
+            expect(fileSet.toQuerySQLBuilder().from(mockDatasource).toString()).to.contain(
+                'OR (REGEXP_MATCHES("scientist"'
+            );
+        });
+    });
+
     describe("fetchFileRange", () => {
         const sandbox = createSandbox();
         const fileIds = ["abc123", "def456", "ghi789", "jkl012", "mno345"];

packages/core/entity/SQLBuilder/index.ts

@@ -91,7 +91,7 @@ export default class SQLBuilder {
 
     public toSQL(): string {
         if (!this.fromStatement) {
-            throw new Error("Unable to build SLQ without a FROM statement");
+            throw new Error("Unable to build SQL without a FROM statement");
         }
         return `
             ${this.isSummarizing ? "SUMMARIZE" : ""}

packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts

@@ -5,6 +5,7 @@ import DatabaseService from "../../DatabaseService";
 import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop";
 import Annotation from "../../../entity/Annotation";
 import FileFilter from "../../../entity/FileFilter";
+import IncludeFilter from "../../../entity/FileFilter/IncludeFilter";
 import SQLBuilder from "../../../entity/SQLBuilder";
 
 interface Config {
@@ -61,27 +62,32 @@ export default class DatabaseAnnotationService implements AnnotationService {
         const filtersByAnnotation = filters.reduce(
             (map, filter) => ({
                 ...map,
-                [filter.name]: map[filter.name]
-                    ? [...map[filter.name], filter.value]
-                    : [filter.value],
+                [filter.name]: map[filter.name] ? [...map[filter.name], filter] : [filter],
             }),
-            {} as { [name: string]: (string | null)[] }
+            {} as { [name: string]: FileFilter[] }
         );
 
         hierarchy
             // Map before filter because index is important to map to the path
             .forEach((annotation, index) => {
                 if (!filtersByAnnotation[annotation]) {
-                    filtersByAnnotation[annotation] = [index < path.length ? path[index] : null];
+                    filtersByAnnotation[annotation] = [
+                        index < path.length
+                            ? new FileFilter(annotation, path[index])
+                            : new IncludeFilter(annotation), // If no value provided in hierachy, equivalent to Include filter
+                    ];
                 }
             });
 
         return this.fetchFilteredValuesForAnnotation(hierarchy[path.length], filtersByAnnotation);
     }
 
+    // Given a particular annotation in the hierarchy list, apply filters to the files in that category
     private async fetchFilteredValuesForAnnotation(
         annotation: string,
-        filtersByAnnotation: { [name: string]: (string | null)[] } = {}
+        filtersByAnnotation: {
+            [name: string]: FileFilter[];
+        } = {}
     ): Promise<string[]> {
         if (!this.dataSourceNames.length) {
             return [];
@@ -92,16 +98,10 @@ export default class DatabaseAnnotationService implements AnnotationService {
             .from(this.dataSourceNames);
 
         Object.keys(filtersByAnnotation).forEach((annotationToFilter) => {
-            const annotationValues = filtersByAnnotation[annotationToFilter];
-            if (annotationValues[0] === null) {
-                sqlBuilder.where(`"${annotationToFilter}" IS NOT NULL`);
-            } else {
-                sqlBuilder.where(
-                    annotationValues
-                        .map((v) => SQLBuilder.regexMatchValueInList(annotationToFilter, v))
-                        .join(") OR (")
-                );
-            }
+            const appliedFilters = filtersByAnnotation[annotationToFilter];
+            sqlBuilder.where(
+                appliedFilters.map((filter) => filter.toSQLWhereString()).join(") OR (")
+            );
         });
 
         const rows = await this.databaseService.query(sqlBuilder.toSQL());

packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts

@@ -1,6 +1,6 @@
 import { expect } from "chai";
 
-import FileFilter from "../../../../entity/FileFilter";
+import FileFilter, { FilterType } from "../../../../entity/FileFilter";
 import DatabaseServiceNoop from "../../../DatabaseService/DatabaseServiceNoop";
 
 import DatabaseAnnotationService from "..";
@@ -28,9 +28,9 @@ describe("DatabaseAnnotationService", () => {
     });
 
     describe("fetchRootHierarchyValues", () => {
-        const annotationNames = ["Cell Line", "Is Split Scene", "Whatever"];
+        const annotationNames = ["Cell Line", "Is Split Scene", "Gene"];
         const annotations = annotationNames.map((name, index) => ({
-            foo: name + index,
+            mock_annotation: name + index,
             column_name: name,
             column_type: "VARCHAR",
         }));
@@ -40,27 +40,52 @@ describe("DatabaseAnnotationService", () => {
             }
         }
         const databaseService = new MockDatabaseService();
+        const mockDataSourceName = "mockDataSourceName";
+        const mockAnnotationName = "mock_annotation"; // snake case to match annotation properties in annotation map
 
+        // This test suite does not test the implementation or return values of fetchRootHierarchyValues
+        // It simply checks that a DatabaseService query is successfully issued
         it("issues a request for annotation values for the first level of the annotation hierarchy", async () => {
             const annotationService = new DatabaseAnnotationService({
-                dataSourceNames: ["d"],
+                dataSourceNames: [mockDataSourceName],
                 databaseService,
             });
-            const values = await annotationService.fetchRootHierarchyValues(["foo"], []);
-            expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Whatever2"]);
+            const values = await annotationService.fetchRootHierarchyValues(
+                [mockAnnotationName],
+                []
+            );
+            expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Gene2"]);
         });
 
         it("issues a request for annotation values for the first level of the annotation hierarchy with filters", async () => {
             const annotationService = new DatabaseAnnotationService({
-                dataSourceNames: ["e"],
+                dataSourceNames: [mockDataSourceName],
                 databaseService,
             });
-            const filter = new FileFilter("bar", "barValue");
-            const values = await annotationService.fetchRootHierarchyValues(["foo"], [filter]);
-            expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Whatever2"]);
+            const filter = new FileFilter("annotationName", "annotationValue");
+            const values = await annotationService.fetchRootHierarchyValues(
+                [mockAnnotationName],
+                [filter]
+            );
+            expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Gene2"]);
+        });
+
+        it("issues a request for annotation values for the first level of the annotation hierarchy with typed filters", async () => {
+            const annotationService = new DatabaseAnnotationService({
+                dataSourceNames: [mockDataSourceName],
+                databaseService,
+            });
+            const filter = new FileFilter("annotationName", "annotationValue", FilterType.ANY);
+            const values = await annotationService.fetchRootHierarchyValues(
+                [mockAnnotationName],
+                [filter]
+            );
+            expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Gene2"]);
         });
     });
 
+    // This test suite does not test the implementation or return values of fetchHierarchyValuesUnderPath
+    // It simply checks that a DatabaseService query is successfully issued
     describe("fetchHierarchyValuesUnderPath", () => {
         const annotations = ["A", "B", "Cc", "dD"].map((name, index) => ({
             foo: name + index,
@@ -89,8 +114,6 @@ describe("DatabaseAnnotationService", () => {
         });
 
         it("issues request for hierarchy values under a specific path within the hierarchy with filters", async () => {
-            const expectedValues = ["A0", "B1", "Cc2", "dD3"];
-
             const annotationService = new DatabaseAnnotationService({
                 dataSourceNames: ["mock1"],
                 databaseService,
@@ -101,7 +124,21 @@ describe("DatabaseAnnotationService", () => {
                 ["baz"],
                 [filter]
             );
-            expect(values).to.deep.equal(expectedValues);
+            expect(values).to.deep.equal(["A0", "B1", "Cc2", "dD3"]);
+        });
+
+        it("issues request for hierarchy values under a specific path within the hierarchy with typed filters", async () => {
+            const annotationService = new DatabaseAnnotationService({
+                dataSourceNames: ["mockDataSource"],
+                databaseService,
+            });
+            const filter = new FileFilter("bar", "barValue", FilterType.FUZZY);
+            const values = await annotationService.fetchHierarchyValuesUnderPath(
+                ["foo", "bar"],
+                ["baz"],
+                [filter]
+            );
+            expect(values).to.deep.equal(["A0", "B1", "Cc2", "dD3"]);
         });
     });