From 2abfedc74c63069f124cf2c8065c396b7c3aa187 Mon Sep 17 00:00:00 2001 From: andreamah Date: Wed, 12 Apr 2023 15:08:52 -0700 Subject: [PATCH] show closed-notebook matches in search view --- .../search/browser/searchActionsNotebook.ts | 26 +++-- .../contrib/search/browser/searchModel.ts | 98 +++++++++++++++++-- .../search/browser/searchNotebookHelpers.ts | 63 +++++++++++- 3 files changed, 164 insertions(+), 23 deletions(-) diff --git a/src/vs/workbench/contrib/search/browser/searchActionsNotebook.ts b/src/vs/workbench/contrib/search/browser/searchActionsNotebook.ts index 697486b4ef18b..397296c5a0643 100644 --- a/src/vs/workbench/contrib/search/browser/searchActionsNotebook.ts +++ b/src/vs/workbench/contrib/search/browser/searchActionsNotebook.ts @@ -16,6 +16,8 @@ import { NotebookData } from 'vs/workbench/contrib/notebook/common/notebookCommo import { IFileQuery, ISearchService, QueryType } from 'vs/workbench/services/search/common/search'; import { CancellationToken } from 'vs/base/common/cancellation'; import { category } from 'vs/workbench/contrib/search/browser/searchActionsBase'; +import { URI } from 'vs/base/common/uri'; +import { CellSearchModel } from 'vs/workbench/contrib/search/browser/searchNotebookHelpers'; registerAction2(class NotebookDeserializeTest extends Action2 { @@ -41,10 +43,6 @@ registerAction2(class NotebookDeserializeTest extends Action2 { const currWorkspace = workspacesService.getWorkspace(); const uri = currWorkspace.folders[0].uri; - - // const queryBuilder = instantiationService.createInstance(QueryBuilder); - - // const query = queryBuilder.text(content, folderResources.map(folder => folder.uri)) const query: IFileQuery = { type: QueryType.File, filePattern: '**/*.ipynb', @@ -54,16 +52,16 @@ registerAction2(class NotebookDeserializeTest extends Action2 { query, CancellationToken.None ); - // glob(dir + '/**/*.ipynb', {}, async (err, files) => { logService.info('notebook deserialize START'); let processedFiles = 0; let processedBytes = 0; let processedCells = 0; + let matchCount = 0; const start = Date.now(); - // const pattern = "text"; + const pattern = 'start_index'; let i = 0; for (const fileMatch of searchComplete.results) { - if (i > 10) { + if (i > Number.MAX_SAFE_INTEGER) { break; } i++; @@ -86,9 +84,10 @@ registerAction2(class NotebookDeserializeTest extends Action2 { } - _data.cells.forEach(cell => { + _data.cells.forEach((cell, index) => { const input = cell.source; - logService.info(input); + const matches = this.getMatches(input, uri, index, pattern); + matchCount += matches.length; }); processedFiles += 1; @@ -100,7 +99,14 @@ registerAction2(class NotebookDeserializeTest extends Action2 { } } const end = Date.now(); + logService.info(`${matchCount} matches found`); logService.info(`notebook deserialize END | ${end - start}ms | ${((processedBytes / 1024) / 1024).toFixed(2)}MB | Number of Files: ${processedFiles} | Number of Cells: ${processedCells}`); - // }); + + } + + getMatches(source: string, uri: URI, cellIndex: number, target: string) { + const cellModel = new CellSearchModel(source, uri, cellIndex); + return cellModel.find(target); } }); + diff --git a/src/vs/workbench/contrib/search/browser/searchModel.ts b/src/vs/workbench/contrib/search/browser/searchModel.ts index 8961a7bb0479c..87cdfd34d5268 100644 --- a/src/vs/workbench/contrib/search/browser/searchModel.ts +++ b/src/vs/workbench/contrib/search/browser/searchModel.ts @@ -5,6 +5,7 @@ import * as arrays from 'vs/base/common/arrays'; import { RunOnceScheduler } from 'vs/base/common/async'; +import { streamToBuffer } from 'vs/base/common/buffer'; import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation'; import { compareFileExtensions, compareFileNames, comparePaths } from 'vs/base/common/comparers'; import { memoize } from 'vs/base/common/decorators'; @@ -36,11 +37,12 @@ import { FindMatchDecorationModel } from 'vs/workbench/contrib/notebook/browser/ import { CellEditState, CellFindMatchWithIndex, CellWebviewFindMatch, ICellViewModel } from 'vs/workbench/contrib/notebook/browser/notebookBrowser'; import { NotebookEditorWidget } from 'vs/workbench/contrib/notebook/browser/notebookEditorWidget'; import { INotebookEditorService } from 'vs/workbench/contrib/notebook/browser/services/notebookEditorService'; -import { NotebookCellsChangeType } from 'vs/workbench/contrib/notebook/common/notebookCommon'; +import { NotebookCellsChangeType, NotebookData } from 'vs/workbench/contrib/notebook/common/notebookCommon'; +import { INotebookService, SimpleNotebookProviderInfo } from 'vs/workbench/contrib/notebook/common/notebookService'; import { IReplaceService } from 'vs/workbench/contrib/search/browser/replace'; -import { ICellMatch, IFileMatchWithCells, contentMatchesToTextSearchMatches, isIFileMatchWithCells, webviewMatchesToTextSearchMatches } from 'vs/workbench/contrib/search/browser/searchNotebookHelpers'; +import { CellSearchModel, ICellMatch, IFileMatchWithCells, contentMatchesToTextSearchMatches, isIFileMatchWithCells, webviewMatchesToTextSearchMatches } from 'vs/workbench/contrib/search/browser/searchNotebookHelpers'; import { ReplacePattern } from 'vs/workbench/services/search/common/replace'; -import { IFileMatch, IPatternInfo, ISearchComplete, ISearchConfigurationProperties, ISearchProgressItem, ISearchRange, ISearchService, ITextQuery, ITextSearchContext, ITextSearchMatch, ITextSearchPreviewOptions, ITextSearchResult, ITextSearchStats, OneLineRange, QueryType, resultIsMatch, SearchCompletionExitCode, SearchSortOrder } from 'vs/workbench/services/search/common/search'; +import { IFileMatch, IFileQuery, IPatternInfo, ISearchComplete, ISearchConfigurationProperties, ISearchProgressItem, ISearchRange, ISearchService, ITextQuery, ITextSearchContext, ITextSearchMatch, ITextSearchPreviewOptions, ITextSearchResult, ITextSearchStats, OneLineRange, QueryType, resultIsMatch, SearchCompletionExitCode, SearchSortOrder } from 'vs/workbench/services/search/common/search'; import { addContextToEditorMatches, editorMatchesToTextSearchResults } from 'vs/workbench/services/search/common/searchHelpers'; export class Match { @@ -187,7 +189,7 @@ export class CellMatch { constructor( private readonly _parent: FileMatch, - private readonly _cell: ICellViewModel, + private readonly _cell: ICellViewModel | CellSearchModel, private readonly _cellIndex: number, ) { @@ -231,6 +233,9 @@ export class CellMatch { } public addContext(textSearchMatches: ITextSearchMatch[]) { + if (this.cell instanceof CellSearchModel) { + return; + } this.cell.resolveTextModel().then((textModel) => { const textResultsWithContext = addContextToEditorMatches(textSearchMatches, textModel, this.parent.parent().query!); const contexts = textResultsWithContext.filter((result => !resultIsMatch(result)) as ((a: any) => a is ITextSearchContext)); @@ -260,7 +265,7 @@ export class CellMatch { return this._cellIndex; } - get cell(): ICellViewModel { + get cell(): ICellViewModel | CellSearchModel { return this._cell; } @@ -812,7 +817,7 @@ export class FileMatch extends Disposable implements IFileMatch { } private async highlightCurrentFindMatchDecoration(match: MatchInNotebook): Promise { - if (!this._findMatchDecorationModel) { + if (!this._findMatchDecorationModel || match.cell instanceof CellSearchModel) { return null; } if (match.webviewIndex === undefined) { @@ -823,7 +828,7 @@ export class FileMatch extends Disposable implements IFileMatch { } private revealCellRange(match: MatchInNotebook, outputOffset: number | null) { - if (!this._notebookEditorWidget) { + if (!this._notebookEditorWidget || match.cell instanceof CellSearchModel) { return; } if (match.webviewIndex !== undefined) { @@ -1906,6 +1911,9 @@ export class SearchModel extends Disposable { @IInstantiationService private readonly instantiationService: IInstantiationService, @IUriIdentityService private readonly uriIdentityService: IUriIdentityService, @INotebookEditorService private readonly notebookEditorService: INotebookEditorService, + @INotebookService private readonly notebookService: INotebookService, + @IFileService private readonly fileService: IFileService, + @ILogService private readonly logService: ILogService, ) { super(); this._searchResult = this.instantiationService.createInstance(SearchResult, this); @@ -1946,6 +1954,74 @@ export class SearchModel extends Disposable { get searchResult(): SearchResult { return this._searchResult; } + private async getClosedNotebookResults(textQuery: ITextQuery, token: CancellationToken): Promise<{ results: ResourceMap; limitHit: boolean }> { + const results = new ResourceMap(uri => this.uriIdentityService.extUri.getComparisonKey(uri)); + const query: IFileQuery = { + type: QueryType.File, + filePattern: '**/*.ipynb', + folderQueries: textQuery.folderQueries + }; + + const searchComplete = await this.searchService.fileSearch( + query, + CancellationToken.None + ); + for (const fileMatch of searchComplete.results) { + const cellMatches: ICellMatch[] = []; + const uri = fileMatch.resource; + const content = await this.fileService.readFileStream(uri); + try { + const info = await this.notebookService.withNotebookDataProvider('jupyter-notebook'); + if (!(info instanceof SimpleNotebookProviderInfo)) { + throw new Error('CANNOT open file notebook with this provider'); + } + + let _data: NotebookData = { + metadata: {}, + cells: [] + }; + if (uri.scheme !== Schemas.vscodeInteractive) { + const bytes = await streamToBuffer(content.value); + _data = await info.serializer.dataToNotebook(bytes); + } + + + _data.cells.forEach((cell, index) => { + const source = cell.source; + const target = textQuery.contentPattern.pattern; + + const cellModel = new CellSearchModel(source, uri, index); + const matches = cellModel.find(target); + if (matches.length > 0) { + const cellMatch: ICellMatch = { + cell: cellModel, + index: index, + contentResults: contentMatchesToTextSearchMatches(matches, cellModel), + webviewResults: [] + }; + cellMatches.push(cellMatch); + } + }); + + if (cellMatches.length > 0) { + const fileMatch: IFileMatchWithCells = { + resource: uri, cellResults: cellMatches + }; + results.set(uri, fileMatch); + } + + } catch (e) { + this.logService.info('error: ' + e); + continue; + } + } + + return { + results: results, + limitHit: false + }; + } + private async getLocalNotebookResults(query: ITextQuery, token: CancellationToken): Promise<{ results: ResourceMap; limitHit: boolean }> { const localResults = new ResourceMap(uri => this.uriIdentityService.extUri.getComparisonKey(uri)); @@ -2006,16 +2082,18 @@ export class SearchModel extends Disposable { async notebookSearch(query: ITextQuery, token: CancellationToken, onProgress?: (result: ISearchProgressItem) => void): Promise<{ completeData: ISearchComplete; scannedFiles: ResourceSet }> { const localResults = await this.getLocalNotebookResults(query, token); + const closedResults = await this.getClosedNotebookResults(query, token); + if (onProgress) { - arrays.coalesce([...localResults.results.values()]).forEach(onProgress); + arrays.coalesce([...localResults.results.values(), ...closedResults.results.values()]).forEach(onProgress); } return { completeData: { messages: [], limitHit: localResults.limitHit, - results: arrays.coalesce([...localResults.results.values()]), + results: arrays.coalesce([...localResults.results.values(), ...closedResults.results.values()]), }, - scannedFiles: new ResourceSet([...localResults.results.keys()], uri => this.uriIdentityService.extUri.getComparisonKey(uri)) + scannedFiles: new ResourceSet([...localResults.results.keys(), ...closedResults.results.keys()], uri => this.uriIdentityService.extUri.getComparisonKey(uri)) }; } diff --git a/src/vs/workbench/contrib/search/browser/searchNotebookHelpers.ts b/src/vs/workbench/contrib/search/browser/searchNotebookHelpers.ts index cefeda6ce1d90..9072d04ad4962 100644 --- a/src/vs/workbench/contrib/search/browser/searchNotebookHelpers.ts +++ b/src/vs/workbench/contrib/search/browser/searchNotebookHelpers.ts @@ -3,17 +3,21 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { FindMatch } from 'vs/editor/common/model'; +import { DefaultEndOfLine, FindMatch, IReadonlyTextBuffer } from 'vs/editor/common/model'; import { CellWebviewFindMatch, ICellViewModel } from 'vs/workbench/contrib/notebook/browser/notebookBrowser'; import { IFileMatch, ITextSearchMatch, TextSearchMatch } from 'vs/workbench/services/search/common/search'; import { Range } from 'vs/editor/common/core/range'; +import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder'; +import { SearchParams } from 'vs/editor/common/model/textModelSearch'; +import { Disposable } from 'vs/base/common/lifecycle'; +import { URI } from 'vs/base/common/uri'; export interface IFileMatchWithCells extends IFileMatch { cellResults: ICellMatch[]; } export interface ICellMatch { - cell: ICellViewModel; + cell: ICellViewModel | CellSearchModel; index: number; contentResults: ITextSearchMatch[]; webviewResults: ITextSearchMatch[]; @@ -24,7 +28,7 @@ export function isIFileMatchWithCells(object: IFileMatch): object is IFileMatchW // to text search results -export function contentMatchesToTextSearchMatches(contentMatches: FindMatch[], cell: ICellViewModel): ITextSearchMatch[] { +export function contentMatchesToTextSearchMatches(contentMatches: FindMatch[], cell: ICellViewModel | CellSearchModel): ITextSearchMatch[] { let previousEndLine = -1; const contextGroupings: FindMatch[][] = []; let currentContextGrouping: FindMatch[] = []; @@ -73,4 +77,57 @@ export function webviewMatchesToTextSearchMatches(webviewMatches: CellWebviewFin ).filter((e): e is ITextSearchMatch => !!e); } +// experimental +export class CellSearchModel extends Disposable { + constructor(readonly _source: string, private _uri: URI, private _cellIndex: number) { + // need cell index + super(); + } + + get id() { + return `${this._uri.toString()}#${this._cellIndex}`; + } + + get uri() { + return this._uri; + } + + public getFullModelRange(): Range { + const lineCount = this.textBuffer.getLineCount(); + return new Range(1, 1, lineCount, this.getLineMaxColumn(lineCount)); + } + + public getLineMaxColumn(lineNumber: number): number { + if (lineNumber < 1 || lineNumber > this.textBuffer.getLineCount()) { + throw new Error('Illegal value for lineNumber'); + } + return this.textBuffer.getLineLength(lineNumber) + 1; + } + + private _textBuffer!: IReadonlyTextBuffer; + get textBuffer() { + if (this._textBuffer) { + return this._textBuffer; + } + + const builder = new PieceTreeTextBufferBuilder(); + builder.acceptChunk(this._source); + const bufferFactory = builder.finish(true); + const { textBuffer, disposable } = bufferFactory.create(DefaultEndOfLine.LF); + this._textBuffer = textBuffer; + this._register(disposable); + + return this._textBuffer; + } + + find(target: string): FindMatch[] { + const searchParams = new SearchParams(target, false, false, null); + const searchData = searchParams.parseSearchRequest(); + if (!searchData) { + return []; + } + const fullRange = this.getFullModelRange(); + return this.textBuffer.findMatchesLineByLine(fullRange, searchData, true, 5000); + } +}