Add Whitespace Query Support to Quick File Open

What it Does

Fixes [eclipse-theia#8747](eclipse-theia#8747)

- Allows whitespaces to be included in a `quick file open` search query.
- Adds support for whitespaces in the scoring of file search results
- Adds tests for whitespace queries (considers fuzzy matching and search
  term order).

How to Test

1. `ctrl + p` to `quick file open`
2. Search for a file with a query that includes whitespaces (eg. `readme core`)
3. Observe that whitespaces do not affect the search results

Alternatively, run `@theia/file-search` tests.

Signed-off-by: seantan22 <sean.a.tan@ericsson.com>

packages/file-search/src/browser/quick-file-open.ts

@@ -22,7 +22,7 @@ import {
 } from '@theia/core/lib/browser';
 import { WorkspaceService } from '@theia/workspace/lib/browser/workspace-service';
 import URI from '@theia/core/lib/common/uri';
-import { FileSearchService } from '../common/file-search-service';
+import { FileSearchService, WHITESPACE_QUERY_SEPARATOR } from '../common/file-search-service';
 import { CancellationTokenSource } from '@theia/core/lib/common';
 import { LabelProvider } from '@theia/core/lib/browser/label-provider';
 import { Command } from '@theia/core/lib/common';
@@ -74,6 +74,15 @@ export class QuickFileOpenService implements QuickOpenModel, QuickOpenHandler {
      */
     protected currentLookFor: string = '';
 
+    /**
+     * The score constants when comparing file search results.
+     */
+    private static readonly Scores = {
+        max: 1000,  // represents the maximum score from fuzzy matching (Infinity).
+        exact: 500, // represents the score assigned to exact matching.
+        partial: 250 // represents the score assigned to partial matching.
+    };
+
     readonly prefix: string = '...';
 
     get description(): string {
@@ -254,9 +263,36 @@ export class QuickFileOpenService implements QuickOpenModel, QuickOpenHandler {
          * @returns the score.
          */
         function score(str: string): number {
-            const match = fuzzy.match(query, str);
+            // Adjust for whitespaces in the query.
+            const querySplit = query.split(WHITESPACE_QUERY_SEPARATOR);
+            const queryJoin = querySplit.join('');
+
+            // Check exact and partial exact matches.
+            let exactMatch = true;
+            let partialMatch = false;
+            querySplit.forEach(part => {
+                const partMatches = str.includes(part);
+                exactMatch = exactMatch && partMatches;
+                partialMatch = partialMatch || partMatches;
+            });
+
+            // Check fuzzy matches.
+            const fuzzyMatch = fuzzy.match(queryJoin, str);
+            let matchScore = 0;
             // eslint-disable-next-line no-null/no-null
-            return (match === null) ? 0 : match.score;
+            if (!!fuzzyMatch && matchScore !== null) {
+                matchScore = (fuzzyMatch.score === Infinity) ? QuickFileOpenService.Scores.max : fuzzyMatch.score;
+            }
+
+            // Prioritize exact matches, then partial exact matches, then fuzzy matches.
+            if (exactMatch) {
+                return matchScore + QuickFileOpenService.Scores.exact;
+            } else if (partialMatch) {
+                return matchScore + QuickFileOpenService.Scores.partial;
+            } else {
+                // eslint-disable-next-line no-null/no-null
+                return (fuzzyMatch === null) ? 0 : matchScore;
+            }
         }
 
         // Get the item's member values for comparison.

packages/file-search/src/common/file-search-service.ts

@@ -54,3 +54,5 @@ export namespace FileSearchService {
         defaultIgnorePatterns?: string[]
     }
 }
+
+export const WHITESPACE_QUERY_SEPARATOR = /\s+/;

packages/file-search/src/node/file-search-service-impl.spec.ts

@@ -195,4 +195,34 @@ describe('search-service', function (): void {
         });
     });
 
+    describe('search with whitespaces', () => {
+        const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources')).toString();
+
+        it('should support file searches with whitespaces', async () => {
+            const matches = await service.find('foo sub', { rootUris: [rootUri], fuzzyMatch: true, useGitIgnore: true, limit: 200 });
+
+            expect(matches).to.be.length(2);
+            expect(matches[0].endsWith('subdir1/sub-bar/foo.txt'));
+            expect(matches[1].endsWith('subdir1/sub2/foo.txt'));
+        });
+
+        it('should support fuzzy file searches with whitespaces', async () => {
+            const matchesExact = await service.find('foo sbd2', { rootUris: [rootUri], fuzzyMatch: false, useGitIgnore: true, limit: 200 });
+            const matchesFuzzy = await service.find('foo sbd2', { rootUris: [rootUri], fuzzyMatch: true, useGitIgnore: true, limit: 200 });
+
+            expect(matchesExact).to.be.length(0);
+            expect(matchesFuzzy).to.be.length(1);
+            expect(matchesFuzzy[0].endsWith('subdir1/sub2/foo.txt'));
+        });
+
+        it('should support file searches with whitespaces regardless of order', async () => {
+            const matchesA = await service.find('foo sub', { rootUris: [rootUri], fuzzyMatch: true, useGitIgnore: true, limit: 200 });
+            const matchesB = await service.find('sub foo', { rootUris: [rootUri], fuzzyMatch: true, useGitIgnore: true, limit: 200 });
+
+            expect(matchesA).to.not.be.empty;
+            expect(matchesB).to.not.be.empty;
+            expect(matchesA).to.deep.eq(matchesB);
+        });
+    });
+
 });

packages/file-search/src/node/file-search-service-impl.ts

@@ -23,7 +23,7 @@ import URI from '@theia/core/lib/common/uri';
 import { FileUri } from '@theia/core/lib/node/file-uri';
 import { CancellationTokenSource, CancellationToken, ILogger, isWindows } from '@theia/core';
 import { RawProcessFactory } from '@theia/process/lib/node';
-import { FileSearchService } from '../common/file-search-service';
+import { FileSearchService, WHITESPACE_QUERY_SEPARATOR } from '../common/file-search-service';
 import * as path from 'path';
 
 @injectable()
@@ -80,24 +80,38 @@ export class FileSearchServiceImpl implements FileSearchService {
             searchPattern = searchPattern.replace(/\//g, '\\');
         }
 
-        const stringPattern = searchPattern.toLocaleLowerCase();
+        const patterns = searchPattern.toLocaleLowerCase().trim().split(WHITESPACE_QUERY_SEPARATOR);
+
         await Promise.all(Object.keys(roots).map(async root => {
             try {
                 const rootUri = new URI(root);
                 const rootPath = FileUri.fsPath(rootUri);
                 const rootOptions = roots[root];
+
                 await this.doFind(rootUri, rootOptions, candidate => {
+
                     // Convert OS-native candidate path to a file URI string
                     const fileUri = FileUri.create(path.resolve(rootPath, candidate)).toString();
+
                     // Skip results that have already been matched.
                     if (exactMatches.has(fileUri) || fuzzyMatches.has(fileUri)) {
                         return;
                     }
-                    if (!searchPattern || searchPattern === '*' || candidate.toLocaleLowerCase().indexOf(stringPattern) !== -1) {
+
+                    // Determine if the candidate matches any of the patterns exactly or fuzzy
+                    const candidatePattern = candidate.toLocaleLowerCase();
+                    const patternExists = patterns.every(pattern => candidatePattern.indexOf(pattern) !== -1);
+                    if (patternExists) {
+                        exactMatches.add(fileUri);
+                    } else if (!searchPattern || searchPattern === '*') {
                         exactMatches.add(fileUri);
-                    } else if (opts.fuzzyMatch && fuzzy.test(searchPattern, candidate)) {
-                        fuzzyMatches.add(fileUri);
+                    } else {
+                        const fuzzyPatternExists = patterns.every(pattern => fuzzy.test(pattern, candidate));
+                        if (opts.fuzzyMatch && fuzzyPatternExists) {
+                            fuzzyMatches.add(fileUri);
+                        }
                     }
+
                     // Preemptively terminate the search when the list of exact matches reaches the limit.
                     if (exactMatches.size === opts.limit) {
                         cancellationSource.cancel();
@@ -107,6 +121,7 @@ export class FileSearchServiceImpl implements FileSearchService {
                 console.error('Failed to search:', root, e);
             }
         }));
+
         if (clientToken && clientToken.isCancellationRequested) {
             return [];
         }