From 767590151728baa77f2b3297a2b802c32121c8c2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 1 Dec 2017 10:18:47 -0800 Subject: [PATCH 01/49] Basic tokenizer --- .vscode/launch.json | 11 +- .vscode/tasks.json | 6 +- gulpfile.js | 89 +++++++++++--- package.json | 9 +- src/client/language/characterStream.ts | 134 +++++++++++++++++++++ src/client/language/definitions.ts | 81 +++++++++++++ src/client/language/textIterator.ts | 53 ++++++++ src/client/language/textRangeCollection.ts | 103 ++++++++++++++++ src/client/language/tokenizer.ts | 119 ++++++++++++++++++ src/client/providers/completionProvider.ts | 11 +- 10 files changed, 584 insertions(+), 32 deletions(-) create mode 100644 src/client/language/characterStream.ts create mode 100644 src/client/language/definitions.ts create mode 100644 src/client/language/textIterator.ts create mode 100644 src/client/language/textRangeCollection.ts create mode 100644 src/client/language/tokenizer.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 3fbf982ae0b3..a69c3396ff4e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,7 +15,7 @@ "outFiles": [ "${workspaceFolder}/out/**/*.js" ], - "preLaunchTask": "compile" + "preLaunchTask": "Compile" }, { "name": "Launch Extension as debugServer", // https://code.visualstudio.com/docs/extensions/example-debuggers @@ -30,7 +30,8 @@ "outFiles": [ "${workspaceFolder}/out/client/**/*.js" ], - "cwd": "${workspaceFolder}" + "cwd": "${workspaceFolder}", + "preLaunchTask": "Compile" }, { "name": "Launch Tests", @@ -47,7 +48,7 @@ "outFiles": [ "${workspaceFolder}/out/**/*.js" ], - "preLaunchTask": "compile" + "preLaunchTask": "Compile" }, { "name": "Launch Multiroot Tests", @@ -64,7 +65,7 @@ "outFiles": [ "${workspaceFolder}/out/**/*.js" ], - "preLaunchTask": "compile" + "preLaunchTask": "Compile" } ], "compounds": [ @@ -76,4 +77,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 155b94220ae6..ccf99a2c6f20 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,7 +13,7 @@ "script": "compile", "isBackground": true, "problemMatcher": [ - "$tsc", + "$tsc-watch", { "base": "$tslint5", "fileLocation": "relative" @@ -36,7 +36,7 @@ "panel": "shared" }, "problemMatcher": [ - "$tsc", + "$tsc-watch", { "base": "$tslint5", "fileLocation": "relative" @@ -72,4 +72,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js index 868a7281cc41..6c3f7819d003 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -22,7 +22,7 @@ const colors = require('colors/safe'); * named according to the checks performed on them. Each subset contains * the following one, as described in mathematical notation: * - * all ⊃ eol ⊇ indentation ⊃ copyright ⊃ typescript + * all ⊃ eol ⊇ indentation ⊃ typescript */ const all = [ @@ -115,12 +115,12 @@ const hygiene = (some, options) => { .toString('utf8') .split(/\r\n|\r|\n/) .forEach((line, i) => { - if (/^\s*$/.test(line)) { + if (/^\s*$/.test(line) || /^\S+.*$/.test(line)) { // Empty or whitespace lines are OK. } else if (/^(\s\s\s\s)+.*/.test(line)) { // Good indent. } else if (/^[\t]+.*/.test(line)) { - console.error(file.relative + '(' + (i + 1) + ',1): Bad whitespace indentation'); + console.error(file.relative + '(' + (i + 1) + ',1): Bad whitespace indentation (use 4 spaces instead of tabs or other)'); errorCount++; } }); @@ -137,7 +137,7 @@ const hygiene = (some, options) => { tsfmt: true }).then(result => { if (result.error) { - console.error(result.message); + console.error(result.message.trim()); errorCount++; } cb(null, file); @@ -147,14 +147,14 @@ const hygiene = (some, options) => { }); }); + const program = require('tslint').Linter.createProgram("./tsconfig.json"); + const linter = new tslint.Linter(options, program); const tsl = es.through(function (file) { const configuration = tslint.Configuration.findConfiguration(null, '.'); const options = { formatter: 'json' }; const contents = file.contents.toString('utf8'); - const program = require('tslint').Linter.createProgram("./tsconfig.json"); - const linter = new tslint.Linter(options, program); linter.lint(file.relative, contents, configuration.results); const result = linter.getResult(); if (result.failureCount > 0 || result.errorCount > 0) { @@ -206,22 +206,16 @@ const hygiene = (some, options) => { .pipe(filter(f => !f.stat.isDirectory())) .pipe(filter(eolFilter)) .pipe(options.skipEOL ? es.through() : eol) - .pipe(filter(indentationFilter)); - - if (!options.skipIndentationCheck) { - result = result - .pipe(indentation); - } + .pipe(filter(indentationFilter)) + .pipe(indentation); // Type script checks. let typescript = result - .pipe(filter(tslintFilter)); + .pipe(filter(tslintFilter)) + .pipe(formatting); - if (!options.skipFormatCheck) { - typescript = typescript - .pipe(formatting); - } - typescript = typescript.pipe(tsl) + typescript = typescript + .pipe(tsl) .pipe(tscFilesTracker) .pipe(tsc()); @@ -244,16 +238,32 @@ gulp.task('hygiene-staged', () => run({ mode: 'changes' })); gulp.task('hygiene-watch', ['hygiene-staged', 'hygiene-watch-runner']); gulp.task('hygiene-watch-runner', function () { + /** + * @type {Deferred} + */ + let runPromise; + return watch(all, { events: ['add', 'change'] }, function (event) { + // Damn bounce does not work, do our own checks. const start = new Date(); + if (runPromise && !runPromise.completed) { + console.log(`[${start.toLocaleTimeString()}] Already running`); + return; + } console.log(`[${start.toLocaleTimeString()}] Starting '${colors.cyan('hygiene-watch-runner')}'...`); + + runPromise = new Deferred(); // Skip indentation and formatting checks to speed up linting. - return run({ mode: 'watch', skipFormatCheck: true, skipIndentationCheck: true }) + run({ mode: 'watch', skipFormatCheck: true, skipIndentationCheck: true }) .then(() => { const end = new Date(); const time = (end.getTime() - start.getTime()) / 1000; console.log(`[${end.toLocaleTimeString()}] Finished '${colors.cyan('hygiene-watch-runner')}' after ${time} seconds`); - }); + runPromise.resolve(); + }) + .catch(runPromise.reject.bind); + + return runPromise.promise; }); }); @@ -402,7 +412,46 @@ function getModifiedFiles() { }); }); } + // this allows us to run hygiene as a git pre-commit hook. if (require.main === module) { run({ exitOnError: true, mode: 'staged' }); } + +class Deferred { + constructor(scope) { + this.scope = scope; + this._resolved = false; + this._rejected = false; + + this._promise = new Promise((resolve, reject) => { + this._resolve = resolve; + this._reject = reject; + }); + } + resolve(value) { + this._resolve.apply(this.scope ? this.scope : this, arguments); + this._resolved = true; + } + /** + * Rejects the promise + * @param {any} reason + * @memberof Deferred + */ + reject(reason) { + this._reject.apply(this.scope ? this.scope : this, arguments); + this._rejected = true; + } + get promise() { + return this._promise; + } + get resolved() { + return this._resolved === true; + } + get rejected() { + return this._rejected === true; + } + get completed() { + return this._rejected || this._resolved; + } +} \ No newline at end of file diff --git a/package.json b/package.json index 0155540cb629..17145aa47ed6 100644 --- a/package.json +++ b/package.json @@ -1529,7 +1529,7 @@ "fuzzy": "^0.1.3", "get-port": "^3.2.0", "iconv-lite": "^0.4.19", - "inversify": "^4.5.2", + "inversify": "^4.5.1", "line-by-line": "^0.1.5", "lodash": "^4.17.4", "minimatch": "^3.0.3", @@ -1583,5 +1583,10 @@ "typescript": "^2.5.2", "typescript-formatter": "^6.0.0", "vscode": "^1.1.5" + }, + "__metadata": { + "id": "f1f59ae4-9318-4f3c-a9b5-81b2eaa5f8a5", + "publisherDisplayName": "Microsoft", + "publisherId": "998b010b-e2af-44a5-a6cd-0b5fd3b9b6f8" } -} +} \ No newline at end of file diff --git a/src/client/language/characterStream.ts b/src/client/language/characterStream.ts new file mode 100644 index 000000000000..fbb65f903644 --- /dev/null +++ b/src/client/language/characterStream.ts @@ -0,0 +1,134 @@ +'use strict'; + +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { ICharacterStream, ITextIterator } from './definitions'; +import { TextIterator } from './textIterator'; + +export class CharacterStream implements ICharacterStream { + private text: ITextIterator; + private pos: number; + private curChar: number; + private endOfStream: boolean; + + constructor(text: string | ITextIterator) { + const iter = text as ITextIterator; + const s = text as string; + + this.text = iter !== null ? iter : new TextIterator(s); + this.pos = 0; + this.curChar = text.length > 0 ? text.charCodeAt(0) : 0; + this.endOfStream = text.length === 0; + } + + public getText(): string { + return this.text.getText(); + } + + public get position(): number { + return this.pos; + } + + public set position(value: number) { + this.pos = value; + this.checkBounds(); + } + + public get currentChar(): number { + return this.curChar; + } + + public get nextChar(): number { + return this.position + 1 < this.text.length ? this.text.charCodeAt(this.position + 1) : 0; + } + + public get prevChar(): number { + return this.position - 1 >= 0 ? this.text.charCodeAt(this.position - 1) : 0; + } + + public isEndOfStream(): boolean { + return this.endOfStream; + } + + public lookAhead(offset: number): number { + const pos = this.position + offset; + return pos < 0 || pos >= this.text.length ? 0 : this.text.charCodeAt(pos); + } + + public advance(offset: number) { + this.position += offset; + } + + public moveNext(): boolean { + if (this.pos < this.text.length - 1) { + // Most common case, no need to check bounds extensively + this.pos += 1; + this.curChar = this.text.charCodeAt(this.pos); + return true; + } + this.advance(1); + return !this.endOfStream; + } + + public isAtWhiteSpace(): boolean { + return this.curChar <= Char.Space || this.curChar === 0x200B; + } + + public isAtLineBreak(): boolean { + return this.curChar === Char.CarriageReturn || this.curChar === Char.DataLineEscape; + } + + public skipLineBreak(): void { + if (this.curChar === Char.CarriageReturn) { + this.moveNext(); + if (this.currentChar === Char.LineFeed) { + this.moveNext(); + } + } else if (this.curChar === Char.LineFeed) { + this.moveNext(); + } + } + + public skipWhitespace(): void { + while (!this.endOfStream && this.isAtWhiteSpace()) { + this.moveNext(); + } + } + + public skipToEol(): void { + while (!this.endOfStream && !this.isAtLineBreak()) { + this.moveNext(); + } + } + + public skipToWhitespace(): void { + while (!this.endOfStream && !this.isAtWhiteSpace()) { + this.moveNext(); + } + } + + public isAtString(): boolean { + return this.curChar === 0x22 || this.curChar === 0x27; + } + + public charCodeAt(index: number): number { + return this.text.charCodeAt(index); + } + + public get length(): number { + return this.text.length; + } + + private checkBounds(): void { + if (this.pos < 0) { + this.pos = 0; + } + + this.endOfStream = this.pos >= this.text.length; + if (this.endOfStream) { + this.pos = this.text.length; + } + + this.curChar = this.endOfStream ? 0 : this.text.charCodeAt(this.pos); + } +} diff --git a/src/client/language/definitions.ts b/src/client/language/definitions.ts new file mode 100644 index 000000000000..a4f7a22b4da7 --- /dev/null +++ b/src/client/language/definitions.ts @@ -0,0 +1,81 @@ +'use strict'; + +export interface ITextRange { + readonly start: number; + readonly end: number; + readonly length: number; + contains(position: number): boolean; +} + +export class TextRange implements ITextRange { + public readonly start: number; + public readonly length: number; + + constructor(start: number, length: number) { + if (start < 0) { + throw new Error('start must be non-negative'); + } + if (length < 0) { + throw new Error('length must be non-negative'); + } + this.start = start; + this.length = length; + } + + public static fromBounds(start: number, end: number) { + return new TextRange(start, end - start); + } + + public get end(): number { + return this.start + this.length; + } + + public contains(position: number): boolean { + return position >= this.start && position < this.end; + } +} + +export interface ITextRangeCollection extends ITextRange { + count: number; + getItemAt(index: number): T; + getItemAtPosition(position: number): number; + getItemContaining(position: number): number; +} + +export interface ITextIterator { + readonly length: number; + charCodeAt(index: number): number; + getText(): string; +} + +export interface ICharacterStream extends ITextIterator { + position: number; + readonly currentChar: number; + readonly nextChar: number; + readonly prevChar: number; + getText(): string; + isEndOfStream(): boolean; + lookAhead(offset: number): number; + advance(offset: number); + moveNext(): boolean; + isAtWhiteSpace(): boolean; + isAtLineBreak(): boolean; + isAtString(): boolean; + skipLineBreak(): void; + skipWhitespace(): void; + skipToEol(): void; + skipToWhitespace(): void; +} + +export enum TokenType { + String +} + +export interface IToken extends ITextRange { + readonly type: TokenType; +} + +export interface ITokenizer { + Tokenize(text: string): ITextRangeCollection; + Tokenize(text: string, start: number, length: number): ITextRangeCollection; +} diff --git a/src/client/language/textIterator.ts b/src/client/language/textIterator.ts new file mode 100644 index 000000000000..8af0e1caefda --- /dev/null +++ b/src/client/language/textIterator.ts @@ -0,0 +1,53 @@ +'use strict'; + +import { Position, Range, TextDocument } from 'vscode'; +import { ITextIterator } from './definitions'; + +export class TextIterator implements ITextIterator { + private text: string; + + constructor(text: string) { + this.text = text; + } + + public charCodeAt(index: number): number { + if (index >= 0 && index < this.length) { + return this.text.charCodeAt[index]; + } + return 0; + } + + public get length(): number { + return this.text.length; + } + + public getText(): string { + return this.text; + } +} + +export class DocumentTextIterator implements ITextIterator { + public readonly length: number; + + private document: TextDocument; + + constructor(document: TextDocument) { + this.document = document; + + const lastIndex = this.document.lineCount - 1; + const lastLine = this.document.lineAt(lastIndex); + const end = new Position(lastIndex, lastLine.range.end.character); + this.length = this.document.offsetAt(end); + } + + public charCodeAt(index: number): number { + const position = this.document.positionAt(index); + return this.document + .getText(new Range(position, position.translate(0, 1))) + .charCodeAt(position.character); + } + + public getText(): string { + return this.document.getText(); + } +} diff --git a/src/client/language/textRangeCollection.ts b/src/client/language/textRangeCollection.ts new file mode 100644 index 000000000000..5448445c3092 --- /dev/null +++ b/src/client/language/textRangeCollection.ts @@ -0,0 +1,103 @@ +'use strict'; + +import { ITextRange, ITextRangeCollection } from './definitions'; + +export class TextRangeCollection implements ITextRangeCollection { + private items: T[]; + + constructor(items: T[]) { + this.items = items; + } + + public get start(): number { + return this.items.length > 0 ? this.items[0].start : 0; + } + + public get end(): number { + return this.items.length > 0 ? this.items[this.items.length - 1].end : 0; + } + + public get length(): number { + return this.end - this.start; + } + + public get count(): number { + return this.items.length; + } + + public contains(position: number) { + return position >= this.start && position < this.end; + } + + public getItemAt(index: number): T { + if (index < 0 || index >= this.items.length) { + throw new Error('index is out of range'); + } + return this.items[index] as T; + } + + public getItemAtPosition(position: number): number { + if (this.count === 0) { + return -1; + } + if (position < this.start) { + return -1; + } + if (position >= this.end) { + return -1; + } + + let min = 0; + let max = this.count - 1; + + while (min <= max) { + const mid = min + (max - min) / 2; + const item = this.items[mid]; + + if (item.start === position) { + return mid; + } + + if (position < item.start) { + max = mid - 1; + } else { + min = mid + 1; + } + } + return -1; + } + + public getItemContaining(position: number): number { + if (this.count === 0) { + return -1; + } + if (position < this.start) { + return -1; + } + if (position > this.end) { + return -1; + } + + let min = 0; + let max = this.count - 1; + + while (min <= max) { + const mid = min + (max - min) / 2; + const item = this[mid]; + + if (item.Contains(position)) { + return mid; + } + if (mid < this.count - 1 && item.end <= position && position < this.items[mid + 1].start) { + return -1; + } + + if (position < item.Start) { + max = mid - 1; + } else { + min = mid + 1; + } + } + return -1; + } +} diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts new file mode 100644 index 000000000000..25af381dfc26 --- /dev/null +++ b/src/client/language/tokenizer.ts @@ -0,0 +1,119 @@ +'use strict'; + +import Char from 'typescript-char'; +import { CharacterStream } from './characterStream'; +import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './definitions'; +import { TextRangeCollection } from './textRangeCollection'; + +enum QuoteType { + None, + Single, + Double, + TripleSingle, + TripleDouble +} + +class Token extends TextRange implements IToken { + public readonly type: TokenType; + + constructor(type: TokenType, start: number, length: number) { + super(start, length); + this.type = type; + } +} + +export class Tokenizer implements ITokenizer { + private cs: ICharacterStream; + private tokens: IToken[] = []; + + public Tokenize(text: string): ITextRangeCollection; + public Tokenize(text: string, start: number, length: number): ITextRangeCollection; + + public Tokenize(text: string, start?: number, length?: number): ITextRangeCollection { + if (start === undefined) { + start = 0; + } else if (start < 0 || start >= text.length) { + throw new Error('Invalid range start'); + } + + if (length === undefined) { + length = text.length; + } else if (length < 0 || start + length >= text.length) { + throw new Error('Invalid range length'); + } + + this.cs = new CharacterStream(text); + this.cs.position = start; + + const end = start + length; + while (!this.cs.isEndOfStream()) { + this.AddNextToken(); + if (this.cs.position >= end) { + break; + } + } + return new TextRangeCollection(this.tokens); + } + + private AddNextToken(): void { + this.cs.skipWhitespace(); + if (this.cs.isEndOfStream()) { + return; + } + + if (!this.handleCharacter()) { + this.cs.moveNext(); + } + } + + private handleCharacter(): boolean { + const quoteType = this.getQuoteType(); + if (quoteType !== QuoteType.None) { + this.handleString(quoteType); + return true; + } + return false; + } + + private getQuoteType(): QuoteType { + if (this.cs.currentChar === Char.SingleQuote) { + return this.cs.nextChar === Char.SingleQuote && this.cs.lookAhead(2) === Char.SingleQuote + ? QuoteType.TripleSingle + : QuoteType.Single; + } + if (this.cs.currentChar === Char.DoubleQuote) { + return this.cs.nextChar === Char.DoubleQuote && this.cs.lookAhead(2) === Char.DoubleQuote + ? QuoteType.TripleDouble + : QuoteType.Double; + } + return QuoteType.None; + } + + private handleString(quoteType: QuoteType): void { + const start = this.cs.position; + if (quoteType === QuoteType.Single || quoteType === QuoteType.Double) { + this.cs.moveNext(); + this.skipToSingleEndQuote(quoteType === QuoteType.Single + ? Char.SingleQuote + : Char.DoubleQuote); + } else { + this.cs.advance(3); + this.skipToTripleEndQuote(quoteType === QuoteType.TripleSingle + ? Char.SingleQuote + : Char.DoubleQuote); + } + this.tokens.push(new Token(TokenType.String, start, this.cs.position - start)); + } + + private skipToSingleEndQuote(quote: number): void { + while (!this.cs.isEndOfStream() && this.cs.currentChar !== quote) { + this.cs.moveNext(); + } + } + + private skipToTripleEndQuote(quote: number): void { + while (!this.cs.isEndOfStream() && (this.cs.currentChar !== quote || this.cs.nextChar !== quote || this.cs.lookAhead(2) !== quote)) { + this.cs.moveNext(); + } + } +} diff --git a/src/client/providers/completionProvider.ts b/src/client/providers/completionProvider.ts index b453d19a7f19..020fb71daffc 100644 --- a/src/client/providers/completionProvider.ts +++ b/src/client/providers/completionProvider.ts @@ -1,8 +1,9 @@ 'use strict'; import * as vscode from 'vscode'; -import { ProviderResult, SnippetString, Uri } from 'vscode'; +import { Position, ProviderResult, SnippetString, Uri } from 'vscode'; import { PythonSettings } from '../common/configSettings'; +import { Tokenizer } from '../language/tokenizer'; import { JediFactory } from '../languageServices/jediProxyFactory'; import { captureTelemetry } from '../telemetry'; import { COMPLETION } from '../telemetry/constants'; @@ -47,7 +48,7 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid return Promise.resolve([]); } // If starts with a """ (possible doc string), then return - if (lineText.trim().startsWith('"""')) { + if (this.isPositionInsideString(document, position)) { return Promise.resolve([]); } const type = proxy.CommandType.Completions; @@ -66,4 +67,10 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid return PythonCompletionItemProvider.parseData(data, document.uri); }); } + + private isPositionInsideString(document: vscode.TextDocument, position: vscode.Position): boolean { + const text = document.getText(new vscode.Range(new Position(0, 0), position)); + const t = new Tokenizer(); + return t.Tokenize(text).getItemContaining(document.offsetAt(position)) >= 0; + } } From eb4266980d1308fbae7a0017a9da673b788a9e45 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 1 Dec 2017 12:19:00 -0800 Subject: [PATCH 02/49] Fixed property names --- src/client/language/textRangeCollection.ts | 6 +++--- src/client/language/tokenizer.ts | 3 +++ src/client/providers/completionProvider.ts | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/language/textRangeCollection.ts b/src/client/language/textRangeCollection.ts index 5448445c3092..0464dc945382 100644 --- a/src/client/language/textRangeCollection.ts +++ b/src/client/language/textRangeCollection.ts @@ -83,16 +83,16 @@ export class TextRangeCollection implements ITextRangeColl while (min <= max) { const mid = min + (max - min) / 2; - const item = this[mid]; + const item = this.items[mid]; - if (item.Contains(position)) { + if (item.contains(position)) { return mid; } if (mid < this.count - 1 && item.end <= position && position < this.items[mid + 1].start) { return -1; } - if (position < item.Start) { + if (position < item.start) { max = mid - 1; } else { min = mid + 1; diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 25af381dfc26..5cb0d4e3e474 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -1,5 +1,6 @@ 'use strict'; +// tslint:disable-next-line:import-name import Char from 'typescript-char'; import { CharacterStream } from './characterStream'; import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './definitions'; @@ -109,11 +110,13 @@ export class Tokenizer implements ITokenizer { while (!this.cs.isEndOfStream() && this.cs.currentChar !== quote) { this.cs.moveNext(); } + this.cs.moveNext(); } private skipToTripleEndQuote(quote: number): void { while (!this.cs.isEndOfStream() && (this.cs.currentChar !== quote || this.cs.nextChar !== quote || this.cs.lookAhead(2) !== quote)) { this.cs.moveNext(); } + this.cs.advance(3); } } diff --git a/src/client/providers/completionProvider.ts b/src/client/providers/completionProvider.ts index 020fb71daffc..b097b1633a9e 100644 --- a/src/client/providers/completionProvider.ts +++ b/src/client/providers/completionProvider.ts @@ -69,7 +69,8 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid } private isPositionInsideString(document: vscode.TextDocument, position: vscode.Position): boolean { - const text = document.getText(new vscode.Range(new Position(0, 0), position)); + const tokenizeTo = position.translate(1, 0); + const text = document.getText(new vscode.Range(new Position(0, 0), tokenizeTo)); const t = new Tokenizer(); return t.Tokenize(text).getItemContaining(document.offsetAt(position)) >= 0; } From 275697428d0b725083b2054b24e9aaa08b0db7b1 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 1 Dec 2017 15:10:20 -0800 Subject: [PATCH 03/49] Tests, round I --- src/client/language/characterStream.ts | 63 ++++++------- src/client/language/definitions.ts | 2 + src/client/language/textIterator.ts | 4 +- src/test/.vscode/settings.json | 3 +- src/test/index.ts | 3 +- src/test/language/characterStream.test.ts | 108 ++++++++++++++++++++++ src/test/language/textIterator.test.ts | 24 +++++ src/test/language/textRange.test.ts | 52 +++++++++++ 8 files changed, 222 insertions(+), 37 deletions(-) create mode 100644 src/test/language/characterStream.test.ts create mode 100644 src/test/language/textIterator.test.ts create mode 100644 src/test/language/textRange.test.ts diff --git a/src/client/language/characterStream.ts b/src/client/language/characterStream.ts index fbb65f903644..e90ef7f2e6b7 100644 --- a/src/client/language/characterStream.ts +++ b/src/client/language/characterStream.ts @@ -7,18 +7,15 @@ import { TextIterator } from './textIterator'; export class CharacterStream implements ICharacterStream { private text: ITextIterator; - private pos: number; - private curChar: number; - private endOfStream: boolean; + private _position: number; + private _currentChar: number; + private _isEndOfStream: boolean; constructor(text: string | ITextIterator) { - const iter = text as ITextIterator; - const s = text as string; - - this.text = iter !== null ? iter : new TextIterator(s); - this.pos = 0; - this.curChar = text.length > 0 ? text.charCodeAt(0) : 0; - this.endOfStream = text.length === 0; + this.text = typeof text === 'string' ? new TextIterator(text) : text; + this._position = 0; + this._currentChar = text.length > 0 ? text.charCodeAt(0) : 0; + this._isEndOfStream = text.length === 0; } public getText(): string { @@ -26,16 +23,16 @@ export class CharacterStream implements ICharacterStream { } public get position(): number { - return this.pos; + return this._position; } public set position(value: number) { - this.pos = value; + this._position = value; this.checkBounds(); } public get currentChar(): number { - return this.curChar; + return this._currentChar; } public get nextChar(): number { @@ -47,11 +44,11 @@ export class CharacterStream implements ICharacterStream { } public isEndOfStream(): boolean { - return this.endOfStream; + return this._isEndOfStream; } public lookAhead(offset: number): number { - const pos = this.position + offset; + const pos = this._position + offset; return pos < 0 || pos >= this.text.length ? 0 : this.text.charCodeAt(pos); } @@ -60,55 +57,55 @@ export class CharacterStream implements ICharacterStream { } public moveNext(): boolean { - if (this.pos < this.text.length - 1) { + if (this._position < this.text.length - 1) { // Most common case, no need to check bounds extensively - this.pos += 1; - this.curChar = this.text.charCodeAt(this.pos); + this._position += 1; + this._currentChar = this.text.charCodeAt(this._position); return true; } this.advance(1); - return !this.endOfStream; + return !this.isEndOfStream(); } public isAtWhiteSpace(): boolean { - return this.curChar <= Char.Space || this.curChar === 0x200B; + return this.currentChar <= Char.Space || this.currentChar === 0x200B; // Unicode whitespace } public isAtLineBreak(): boolean { - return this.curChar === Char.CarriageReturn || this.curChar === Char.DataLineEscape; + return this.currentChar === Char.CarriageReturn || this.currentChar === Char.LineFeed; } public skipLineBreak(): void { - if (this.curChar === Char.CarriageReturn) { + if (this._currentChar === Char.CarriageReturn) { this.moveNext(); if (this.currentChar === Char.LineFeed) { this.moveNext(); } - } else if (this.curChar === Char.LineFeed) { + } else if (this._currentChar === Char.LineFeed) { this.moveNext(); } } public skipWhitespace(): void { - while (!this.endOfStream && this.isAtWhiteSpace()) { + while (!this.isEndOfStream() && this.isAtWhiteSpace()) { this.moveNext(); } } public skipToEol(): void { - while (!this.endOfStream && !this.isAtLineBreak()) { + while (!this.isEndOfStream() && !this.isAtLineBreak()) { this.moveNext(); } } public skipToWhitespace(): void { - while (!this.endOfStream && !this.isAtWhiteSpace()) { + while (!this.isEndOfStream() && !this.isAtWhiteSpace()) { this.moveNext(); } } public isAtString(): boolean { - return this.curChar === 0x22 || this.curChar === 0x27; + return this.currentChar === Char.SingleQuote || this.currentChar === Char.DoubleQuote; } public charCodeAt(index: number): number { @@ -120,15 +117,15 @@ export class CharacterStream implements ICharacterStream { } private checkBounds(): void { - if (this.pos < 0) { - this.pos = 0; + if (this._position < 0) { + this._position = 0; } - this.endOfStream = this.pos >= this.text.length; - if (this.endOfStream) { - this.pos = this.text.length; + this._isEndOfStream = this._position >= this.text.length; + if (this._isEndOfStream) { + this._position = this.text.length; } - this.curChar = this.endOfStream ? 0 : this.text.charCodeAt(this.pos); + this._currentChar = this._isEndOfStream ? 0 : this.text.charCodeAt(this._position); } } diff --git a/src/client/language/definitions.ts b/src/client/language/definitions.ts index a4f7a22b4da7..3e965b0b91bf 100644 --- a/src/client/language/definitions.ts +++ b/src/client/language/definitions.ts @@ -8,6 +8,8 @@ export interface ITextRange { } export class TextRange implements ITextRange { + public static readonly empty = TextRange.fromBounds(0, 0); + public readonly start: number; public readonly length: number; diff --git a/src/client/language/textIterator.ts b/src/client/language/textIterator.ts index 8af0e1caefda..927078da3939 100644 --- a/src/client/language/textIterator.ts +++ b/src/client/language/textIterator.ts @@ -11,8 +11,8 @@ export class TextIterator implements ITextIterator { } public charCodeAt(index: number): number { - if (index >= 0 && index < this.length) { - return this.text.charCodeAt[index]; + if (index >= 0 && index < this.text.length) { + return this.text.charCodeAt(index); } return 0; } diff --git a/src/test/.vscode/settings.json b/src/test/.vscode/settings.json index 2218e2cecd87..12cde5b9dc53 100644 --- a/src/test/.vscode/settings.json +++ b/src/test/.vscode/settings.json @@ -21,5 +21,6 @@ "python.linting.pydocstyleEnabled": false, "python.linting.pylamaEnabled": false, "python.linting.mypyEnabled": false, - "python.formatting.provider": "yapf" + "python.formatting.provider": "yapf", + "python.pythonPath": "python" } \ No newline at end of file diff --git a/src/test/index.ts b/src/test/index.ts index 0202b4e2dc43..acce5db2392a 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -8,6 +8,7 @@ testRunner.configure({ ui: 'tdd', useColors: true, timeout: 25000, - retries: 3 + retries: 3, + grep: 'Language.CharacterStream' } as {}); module.exports = testRunner; diff --git a/src/test/language/characterStream.test.ts b/src/test/language/characterStream.test.ts new file mode 100644 index 000000000000..d1c003d6b2d9 --- /dev/null +++ b/src/test/language/characterStream.test.ts @@ -0,0 +1,108 @@ +import * as assert from 'assert'; +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { CharacterStream } from '../../client/language/characterStream'; +import { ICharacterStream, TextRange } from '../../client/language/definitions'; +import { TextIterator } from '../../client/language/textIterator'; + +// tslint:disable-next-line:max-func-body-length +suite('Language.CharacterStream', () => { + test('Iteration (string)', async () => { + const content = 'some text'; + const cs = new CharacterStream(content); + testIteration(cs, content); + }); + test('Iteration (iterator)', async () => { + const content = 'some text'; + const cs = new CharacterStream(new TextIterator(content)); + testIteration(cs, content); + }); + test('Positioning', async () => { + const content = 'some text'; + const cs = new CharacterStream(content); + assert.equal(cs.position, 0); + cs.advance(1); + assert.equal(cs.position, 1); + cs.advance(1); + assert.equal(cs.position, 2); + cs.advance(2); + assert.equal(cs.position, 4); + cs.advance(-3); + assert.equal(cs.position, 1); + cs.advance(-3); + assert.equal(cs.position, 0); + cs.advance(100); + assert.equal(cs.position, content.length); + }); + test('Characters', async () => { + const content = 'some \ttext "" \' \' \n text \r\n more text'; + const cs = new CharacterStream(content); + for (let i = 0; i < content.length; i += 1) { + assert.equal(cs.currentChar, content.charCodeAt(i)); + + assert.equal(cs.nextChar, i < content.length - 1 ? content.charCodeAt(i + 1) : 0); + assert.equal(cs.prevChar, i > 0 ? content.charCodeAt(i - 1) : 0); + + assert.equal(cs.lookAhead(2), i < content.length - 2 ? content.charCodeAt(i + 2) : 0); + assert.equal(cs.lookAhead(-2), i > 1 ? content.charCodeAt(i - 2) : 0); + + const ch = content.charCodeAt(i); + const isLineBreak = ch === Char.LineFeed || ch === Char.CarriageReturn; + assert.equal(cs.isAtWhiteSpace(), ch === Char.Tab || ch === Char.Space || isLineBreak); + assert.equal(cs.isAtLineBreak(), isLineBreak); + assert.equal(cs.isAtString(), ch === Char.SingleQuote || ch === Char.DoubleQuote); + + cs.moveNext(); + } + }); + test('Skip', async () => { + const content = 'some \ttext "" \' \' \n text \r\n more text'; + const cs = new CharacterStream(content); + + cs.skipWhitespace(); + assert.equal(cs.position, 0); + + cs.skipToWhitespace(); + assert.equal(cs.position, 4); + + cs.skipToWhitespace(); + assert.equal(cs.position, 4); + + cs.skipWhitespace(); + assert.equal(cs.position, 6); + + cs.skipLineBreak(); + assert.equal(cs.position, 6); + + cs.skipToEol(); + assert.equal(cs.position, 18); + + cs.skipLineBreak(); + assert.equal(cs.position, 19); + }); +}); + +function testIteration(cs: ICharacterStream, content: string) { + assert.equal(cs.position, 0); + assert.equal(cs.length, content.length); + assert.equal(cs.isEndOfStream(), false); + + for (let i = -2; i < content.length + 2; i += 1) { + const ch = cs.charCodeAt(i); + if (i < 0 || i >= content.length) { + assert.equal(ch, 0); + } else { + assert.equal(ch, content.charCodeAt(i)); + } + } + + for (let i = 0; i < content.length; i += 1) { + assert.equal(cs.isEndOfStream(), false); + assert.equal(cs.position, i); + assert.equal(cs.currentChar, content.charCodeAt(i)); + cs.moveNext(); + } + + assert.equal(cs.isEndOfStream(), true); + assert.equal(cs.position, content.length); +} diff --git a/src/test/language/textIterator.test.ts b/src/test/language/textIterator.test.ts new file mode 100644 index 000000000000..dddfe3478ec5 --- /dev/null +++ b/src/test/language/textIterator.test.ts @@ -0,0 +1,24 @@ +import * as assert from 'assert'; +import { TextIterator } from '../../client/language/textIterator'; + +// tslint:disable-next-line:max-func-body-length +suite('Language.TextIterator', () => { + test('Construction', async () => { + const content = 'some text'; + const ti = new TextIterator(content); + assert.equal(ti.length, content.length); + assert.equal(ti.getText(), content); + }); + test('Iteration', async () => { + const content = 'some text'; + const ti = new TextIterator(content); + for (let i = -2; i < content.length + 2; i += 1) { + const ch = ti.charCodeAt(i); + if (i < 0 || i >= content.length) { + assert.equal(ch, 0); + } else { + assert.equal(ch, content.charCodeAt(i)); + } + } + }); +}); diff --git a/src/test/language/textRange.test.ts b/src/test/language/textRange.test.ts new file mode 100644 index 000000000000..4e0da8feb06c --- /dev/null +++ b/src/test/language/textRange.test.ts @@ -0,0 +1,52 @@ +import * as assert from 'assert'; +import { TextRange } from '../../client/language/definitions'; + +// tslint:disable-next-line:max-func-body-length +suite('Language.TextRange', () => { + test('Empty static', async () => { + const e = TextRange.empty; + assert.equal(e.start, 0); + assert.equal(e.end, 0); + assert.equal(e.length, 0); + }); + test('Construction', async () => { + let r = new TextRange(10, 20); + assert.equal(r.start, 10); + assert.equal(r.end, 30); + assert.equal(r.length, 20); + r = new TextRange(10, 0); + assert.equal(r.start, 10); + assert.equal(r.end, 10); + assert.equal(r.length, 0); + }); + test('From bounds', async () => { + let r = TextRange.fromBounds(7, 9); + assert.equal(r.start, 7); + assert.equal(r.end, 9); + assert.equal(r.length, 2); + + r = TextRange.fromBounds(5, 5); + assert.equal(r.start, 5); + assert.equal(r.end, 5); + assert.equal(r.length, 0); + }); + test('Contains', async () => { + const r = TextRange.fromBounds(7, 9); + assert.equal(r.contains(-1), false); + assert.equal(r.contains(6), false); + assert.equal(r.contains(7), true); + assert.equal(r.contains(8), true); + assert.equal(r.contains(9), false); + assert.equal(r.contains(10), false); + }); + test('Exceptions', async () => { + assert.throws( + () => { const e = new TextRange(0, -1); }, + Error + ); + assert.throws( + () => { const e = TextRange.fromBounds(3, 1); }, + Error + ); + }); +}); From c2c1ced6cf64be60ca808c269291af191df3b3b2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 1 Dec 2017 16:07:28 -0800 Subject: [PATCH 04/49] Tests, round II --- src/client/language/definitions.ts | 3 +- src/client/language/tokenizer.ts | 13 +++++ src/client/providers/completionProvider.ts | 15 +++--- src/test/index.ts | 2 +- src/test/language/textRangeCollection.test.ts | 53 +++++++++++++++++++ src/test/language/tokenizer.test.ts | 39 ++++++++++++++ 6 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 src/test/language/textRangeCollection.test.ts create mode 100644 src/test/language/tokenizer.test.ts diff --git a/src/client/language/definitions.ts b/src/client/language/definitions.ts index 3e965b0b91bf..d001adccbd88 100644 --- a/src/client/language/definitions.ts +++ b/src/client/language/definitions.ts @@ -70,7 +70,8 @@ export interface ICharacterStream extends ITextIterator { } export enum TokenType { - String + String, + Comment } export interface IToken extends ITextRange { diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 5cb0d4e3e474..6d63f4168414 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -73,9 +73,22 @@ export class Tokenizer implements ITokenizer { this.handleString(quoteType); return true; } + switch (this.cs.currentChar) { + case Char.Hash: + this.handleComment(); + break; + default: + break; + } return false; } + private handleComment(): void { + const start = this.cs.position; + this.cs.skipToEol(); + this.tokens.push(new Token(TokenType.Comment, start, this.cs.position - start)); + } + private getQuoteType(): QuoteType { if (this.cs.currentChar === Char.SingleQuote) { return this.cs.nextChar === Char.SingleQuote && this.cs.lookAhead(2) === Char.SingleQuote diff --git a/src/client/providers/completionProvider.ts b/src/client/providers/completionProvider.ts index b097b1633a9e..10c930348e33 100644 --- a/src/client/providers/completionProvider.ts +++ b/src/client/providers/completionProvider.ts @@ -3,6 +3,7 @@ import * as vscode from 'vscode'; import { Position, ProviderResult, SnippetString, Uri } from 'vscode'; import { PythonSettings } from '../common/configSettings'; +import { TokenType } from '../language/definitions'; import { Tokenizer } from '../language/tokenizer'; import { JediFactory } from '../languageServices/jediProxyFactory'; import { captureTelemetry } from '../telemetry'; @@ -43,12 +44,8 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid if (lineText.match(/^\s*\/\//)) { return Promise.resolve([]); } - // If starts with a comment, then return - if (lineText.trim().startsWith('#')) { - return Promise.resolve([]); - } - // If starts with a """ (possible doc string), then return - if (this.isPositionInsideString(document, position)) { + // Suppress completion inside string and comments + if (this.isPositionInsideStringOrComment(document, position)) { return Promise.resolve([]); } const type = proxy.CommandType.Completions; @@ -68,10 +65,12 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid }); } - private isPositionInsideString(document: vscode.TextDocument, position: vscode.Position): boolean { + private isPositionInsideStringOrComment(document: vscode.TextDocument, position: vscode.Position): boolean { const tokenizeTo = position.translate(1, 0); const text = document.getText(new vscode.Range(new Position(0, 0), tokenizeTo)); const t = new Tokenizer(); - return t.Tokenize(text).getItemContaining(document.offsetAt(position)) >= 0; + const tokens = t.Tokenize(text); + const index = tokens.getItemContaining(document.offsetAt(position)); + return index >= 0 && (tokens[index].TokenType === TokenType.String || tokens[index].TokenType === TokenType.Comment); } } diff --git a/src/test/index.ts b/src/test/index.ts index acce5db2392a..6480c37439b6 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -9,6 +9,6 @@ testRunner.configure({ useColors: true, timeout: 25000, retries: 3, - grep: 'Language.CharacterStream' + grep: 'Language.Tokenizer' } as {}); module.exports = testRunner; diff --git a/src/test/language/textRangeCollection.test.ts b/src/test/language/textRangeCollection.test.ts new file mode 100644 index 000000000000..5b5dfad8c8b6 --- /dev/null +++ b/src/test/language/textRangeCollection.test.ts @@ -0,0 +1,53 @@ +import * as assert from 'assert'; +import { TextRange } from '../../client/language/definitions'; +import { TextRangeCollection } from '../../client/language/textRangeCollection'; + +// tslint:disable-next-line:max-func-body-length +suite('Language.TextRangeCollection', () => { + test('Empty', async () => { + const items: TextRange[] = []; + const c = new TextRangeCollection(items); + assert.equal(c.start, 0); + assert.equal(c.end, 0); + assert.equal(c.length, 0); + assert.equal(c.count, 0); + }); + test('Basic', async () => { + const items: TextRange[] = []; + items.push(new TextRange(2, 1)); + items.push(new TextRange(4, 2)); + const c = new TextRangeCollection(items); + assert.equal(c.start, 2); + assert.equal(c.end, 6); + assert.equal(c.length, 4); + assert.equal(c.count, 2); + + assert.equal(c.getItemAt(0).start, 2); + assert.equal(c.getItemAt(0).length, 1); + + assert.equal(c.getItemAt(1).start, 4); + assert.equal(c.getItemAt(1).length, 2); + }); + test('Contains position', async () => { + const items: TextRange[] = []; + items.push(new TextRange(2, 1)); + items.push(new TextRange(4, 2)); + const c = new TextRangeCollection(items); + const results = [-1, -1, 0, -1, 1, 1, -1]; + for (let i = 0; i < results.length; i += 1) { + const index = c.getItemContaining(i); + assert.equal(index, results[i]); + } + }); + test('Item at position', async () => { + const items: TextRange[] = []; + items.push(new TextRange(2, 1)); + items.push(new TextRange(4, 2)); + const c = new TextRangeCollection(items); + const results = [-1, -1, 0, -1, 1, 1, -1]; + for (let i = 0; i < results.length; i += 1) { + const index = c.getItemAtPosition(i); + assert.equal(index, results[i]); + } + }); +}); diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts new file mode 100644 index 000000000000..29874f75c26a --- /dev/null +++ b/src/test/language/tokenizer.test.ts @@ -0,0 +1,39 @@ +import * as assert from 'assert'; +import { TextRange, TokenType } from '../../client/language/definitions'; +import { TextRangeCollection } from '../../client/language/textRangeCollection'; +import { Tokenizer } from '../../client/language/tokenizer'; + +// tslint:disable-next-line:max-func-body-length +suite('Language.Tokenizer', () => { + test('Empty', async () => { + const t = new Tokenizer(); + const tokens = t.Tokenize(''); + assert.equal(tokens instanceof TextRangeCollection, true); + assert.equal(tokens.count, 0); + assert.equal(tokens.length, 0); + }); + test('Strings', async () => { + const t = new Tokenizer(); + const tokens = t.Tokenize(' "string" """line1\n#line2"""\t\'un#closed'); + assert.equal(tokens.count, 3); + + const ranges = [1, 8, 10, 18, 29, 10]; + for (let i = 0; i < ranges.length / 2; i += 2) { + assert.equal(tokens.getItemAt(i).start, ranges[i]); + assert.equal(tokens.getItemAt(i).length, ranges[i + 1]); + assert.equal(tokens.getItemAt(i).type, TokenType.String); + } + }); + test('Comments', async () => { + const t = new Tokenizer(); + const tokens = t.Tokenize(' #co"""mment1\n\t\n#comm\'ent2 '); + assert.equal(tokens.count, 2); + + const ranges = [1, 12, 15, 11]; + for (let i = 0; i < ranges.length / 2; i += 2) { + assert.equal(tokens.getItemAt(i).start, ranges[i]); + assert.equal(tokens.getItemAt(i).length, ranges[i + 1]); + assert.equal(tokens.getItemAt(i).type, TokenType.Comment); + } + }); +}); From 14864a580d0affdc9f8fda616fe20e2dbdbb2df9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Sun, 3 Dec 2017 16:42:46 -0800 Subject: [PATCH 05/49] tokenizer test --- src/test/language/tokenizer.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 29874f75c26a..1ce0165e4241 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -18,9 +18,9 @@ suite('Language.Tokenizer', () => { assert.equal(tokens.count, 3); const ranges = [1, 8, 10, 18, 29, 10]; - for (let i = 0; i < ranges.length / 2; i += 2) { - assert.equal(tokens.getItemAt(i).start, ranges[i]); - assert.equal(tokens.getItemAt(i).length, ranges[i + 1]); + for (let i = 0; i < tokens.count; i += 1) { + assert.equal(tokens.getItemAt(i).start, ranges[2 * i]); + assert.equal(tokens.getItemAt(i).length, ranges[2 * i + 1]); assert.equal(tokens.getItemAt(i).type, TokenType.String); } }); From 0ed51d64ee215356c7e5665c3b642c386e997f2d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Sun, 3 Dec 2017 16:46:19 -0800 Subject: [PATCH 06/49] Remove temorary change --- src/test/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/index.ts b/src/test/index.ts index 6480c37439b6..0202b4e2dc43 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -8,7 +8,6 @@ testRunner.configure({ ui: 'tdd', useColors: true, timeout: 25000, - retries: 3, - grep: 'Language.Tokenizer' + retries: 3 } as {}); module.exports = testRunner; From 51b544ca9df3515857f6d07411b7bf36bbb415d0 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Sun, 3 Dec 2017 17:08:35 -0800 Subject: [PATCH 07/49] Fix merge issue --- package.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/package.json b/package.json index 17145aa47ed6..e910867b6620 100644 --- a/package.json +++ b/package.json @@ -1529,7 +1529,7 @@ "fuzzy": "^0.1.3", "get-port": "^3.2.0", "iconv-lite": "^0.4.19", - "inversify": "^4.5.1", + "inversify": "^4.5.2", "line-by-line": "^0.1.5", "lodash": "^4.17.4", "minimatch": "^3.0.3", @@ -1583,10 +1583,5 @@ "typescript": "^2.5.2", "typescript-formatter": "^6.0.0", "vscode": "^1.1.5" - }, - "__metadata": { - "id": "f1f59ae4-9318-4f3c-a9b5-81b2eaa5f8a5", - "publisherDisplayName": "Microsoft", - "publisherId": "998b010b-e2af-44a5-a6cd-0b5fd3b9b6f8" } } \ No newline at end of file From 3cd11e649febd2cc750f9c8bd238a7f9e0a222d5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Sun, 3 Dec 2017 17:18:09 -0800 Subject: [PATCH 08/49] Merge conflict --- .vscode/settings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1884ba9a0255..de5f2d58fde1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,5 @@ "python.linting.enabled": false, "python.formatting.formatOnSave": false, "python.unitTest.promptToConfigure": false, - "python.workspaceSymbols.enabled": false, - "python.formatting.provider": "yapf" + "python.workspaceSymbols.enabled": false } From 82e0ad16d200cfdb2ba4be1270305a882015ac9b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Sun, 3 Dec 2017 17:25:50 -0800 Subject: [PATCH 09/49] Merge conflict --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index de5f2d58fde1..1884ba9a0255 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,6 @@ "python.linting.enabled": false, "python.formatting.formatOnSave": false, "python.unitTest.promptToConfigure": false, - "python.workspaceSymbols.enabled": false + "python.workspaceSymbols.enabled": false, + "python.formatting.provider": "yapf" } From 9295c1ab90d1a328351b771d35e5c99103d32ecb Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Dec 2017 11:23:18 -0800 Subject: [PATCH 10/49] Completion test --- package-lock.json | 5 +++ package.json | 3 +- src/test/autocomplete/base.test.ts | 53 +++++++++++++++++++---- src/test/pythonFiles/autocomp/suppress.py | 6 +++ 4 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 src/test/pythonFiles/autocomp/suppress.py diff --git a/package-lock.json b/package-lock.json index 10e608b1634d..50cf5f0c7820 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5457,6 +5457,11 @@ "integrity": "sha1-PFtv1/beCRQmkCfwPAlGdY92c6Q=", "dev": true }, + "typescript-char": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/typescript-char/-/typescript-char-0.0.0.tgz", + "integrity": "sha1-VY/tpzfHZaYQtzfu+7F3Xum8jas=" + }, "typescript-formatter": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/typescript-formatter/-/typescript-formatter-6.1.0.tgz", diff --git a/package.json b/package.json index e910867b6620..23dc208e4ccf 100644 --- a/package.json +++ b/package.json @@ -1539,6 +1539,7 @@ "semver": "^5.4.1", "tmp": "0.0.29", "tree-kill": "^1.1.0", + "typescript-char": "^0.0.0", "uint64be": "^1.0.1", "untildify": "^3.0.2", "vscode-debugadapter": "^1.0.1", @@ -1584,4 +1585,4 @@ "typescript-formatter": "^6.0.0", "vscode": "^1.1.5" } -} \ No newline at end of file +} diff --git a/src/test/autocomplete/base.test.ts b/src/test/autocomplete/base.test.ts index a5398b7beb43..59a7e51c14f4 100644 --- a/src/test/autocomplete/base.test.ts +++ b/src/test/autocomplete/base.test.ts @@ -1,19 +1,18 @@ // Note: This example test is leveraging the Mocha test framework. // Please refer to their documentation on https://mochajs.org/ for help. - // The module 'assert' provides assertion methods from node import * as assert from 'assert'; import { EOL } from 'os'; +import * as path from 'path'; // You can import and use all API from the 'vscode' module // as well as import your extension to test it import * as vscode from 'vscode'; -import * as path from 'path'; import * as settings from '../../client/common/configSettings'; -import { initialize, closeActiveWindows, initializeTest } from '../initialize'; -import { execPythonFile } from '../../client/common/utils'; import { PythonSettings } from '../../client/common/configSettings'; +import { execPythonFile } from '../../client/common/utils'; import { rootWorkspaceUri } from '../common'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'autocomp'); const fileOne = path.join(autoCompPath, 'one.py'); @@ -23,7 +22,9 @@ const fileLambda = path.join(autoCompPath, 'lamb.py'); const fileDecorator = path.join(autoCompPath, 'deco.py'); const fileEncoding = path.join(autoCompPath, 'four.py'); const fileEncodingUsed = path.join(autoCompPath, 'five.py'); +const fileSuppress = path.join(autoCompPath, 'suppress.py'); +// tslint:disable-next-line:max-func-body-length suite('Autocomplete', () => { let isPython3: Promise; suiteSetup(async () => { @@ -31,9 +32,9 @@ suite('Autocomplete', () => { const version = await execPythonFile(rootWorkspaceUri, PythonSettings.getInstance(rootWorkspaceUri).pythonPath, ['--version'], __dirname, true); isPython3 = Promise.resolve(version.indexOf('3.') >= 0); }); - setup(() => initializeTest()); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + setup(initializeTest); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); test('For "sys."', done => { let textEditor: vscode.TextEditor; @@ -115,7 +116,7 @@ suite('Autocomplete', () => { const position = new vscode.Position(10, 9); const list = await vscode.commands.executeCommand('vscode.executeCompletionItemProvider', textDocument.uri, position); assert.notEqual(list.items.filter(item => item.label === 'sleep').length, 0, 'sleep not found'); - assert.notEqual(list.items.filter(item => item.documentation.toString().startsWith("Delay execution for a given number of seconds. The argument may be")).length, 0, 'Documentation incorrect'); + assert.notEqual(list.items.filter(item => item.documentation.toString().startsWith('Delay execution for a given number of seconds. The argument may be')).length, 0, 'Documentation incorrect'); }); test('For custom class', done => { @@ -173,4 +174,40 @@ suite('Autocomplete', () => { assert.equal(list.items.filter(item => item.label === 'showMessage')[0].documentation, documentation, 'showMessage unicode documentation is incorrect'); }).then(done, done); }); + + // https://github.com/Microsoft/vscode-python/issues/110 + test('Suppress in strings/comments', done => { + let textEditor: vscode.TextEditor; + let textDocument: vscode.TextDocument; + const positions = [ + new vscode.Position(0, 1), // false + new vscode.Position(0, 9), // true + new vscode.Position(0, 12), // false + new vscode.Position(1, 1), // false + new vscode.Position(1, 3), // false + new vscode.Position(2, 7), // false + new vscode.Position(3, 0), // false + new vscode.Position(4, 2), // false + new vscode.Position(4, 8), // false + new vscode.Position(5, 4) // false + ]; + const expected = [ + false, true, false, false, false, false, false, false, false, false + ]; + vscode.workspace.openTextDocument(fileSuppress).then(document => { + textDocument = document; + return vscode.window.showTextDocument(textDocument); + }).then(editor => { + assert(vscode.window.activeTextEditor, 'No active editor'); + textEditor = editor; + for (let i = 0; i < positions.length; i += 1) { + vscode.commands.executeCommand('vscode.executeCompletionItemProvider', + textDocument.uri, positions[i]).then(list => { + const result = list.items.filter(item => item.label === 'abs').length; + assert.equal(result > 0, expected[i], + `Expected ${expected[i]} at position ${positions[i].line}:${positions[i].character} but got ${result}`); + }); + } + }).then(done, done); + }); }); diff --git a/src/test/pythonFiles/autocomp/suppress.py b/src/test/pythonFiles/autocomp/suppress.py new file mode 100644 index 000000000000..9f74959ef14b --- /dev/null +++ b/src/test/pythonFiles/autocomp/suppress.py @@ -0,0 +1,6 @@ +"string" #comment +""" +content +""" +#comment +'un#closed From 06eb1a56049bdbcb9db71c550d9cbd869a4fce63 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Dec 2017 11:26:07 -0800 Subject: [PATCH 11/49] Fix last line --- .vscode/launch.json | 2 +- .vscode/tasks.json | 2 +- gulpfile.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a69c3396ff4e..51590d047be8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -77,4 +77,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ccf99a2c6f20..8a17b7da905f 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -72,4 +72,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/gulpfile.js b/gulpfile.js index 6c3f7819d003..ecf4dd1d5bca 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -454,4 +454,4 @@ class Deferred { get completed() { return this._rejected || this._resolved; } -} \ No newline at end of file +} From e9db8e0de99936daef098000181c44db517c0d8c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Dec 2017 12:47:05 -0800 Subject: [PATCH 12/49] Fix javascript math --- src/client/language/textRangeCollection.ts | 4 ++-- src/test/index.ts | 3 ++- src/test/language/textRangeCollection.test.ts | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/client/language/textRangeCollection.ts b/src/client/language/textRangeCollection.ts index 0464dc945382..d09becea902f 100644 --- a/src/client/language/textRangeCollection.ts +++ b/src/client/language/textRangeCollection.ts @@ -51,7 +51,7 @@ export class TextRangeCollection implements ITextRangeColl let max = this.count - 1; while (min <= max) { - const mid = min + (max - min) / 2; + const mid = Math.floor(min + (max - min) / 2); const item = this.items[mid]; if (item.start === position) { @@ -82,7 +82,7 @@ export class TextRangeCollection implements ITextRangeColl let max = this.count - 1; while (min <= max) { - const mid = min + (max - min) / 2; + const mid = Math.floor(min + (max - min) / 2); const item = this.items[mid]; if (item.contains(position)) { diff --git a/src/test/index.ts b/src/test/index.ts index 0202b4e2dc43..9eb083463586 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -8,6 +8,7 @@ testRunner.configure({ ui: 'tdd', useColors: true, timeout: 25000, - retries: 3 + retries: 3, + grep: "Language.TextRangeCollection" } as {}); module.exports = testRunner; diff --git a/src/test/language/textRangeCollection.test.ts b/src/test/language/textRangeCollection.test.ts index 5b5dfad8c8b6..44666dd811ce 100644 --- a/src/test/language/textRangeCollection.test.ts +++ b/src/test/language/textRangeCollection.test.ts @@ -44,7 +44,7 @@ suite('Language.TextRangeCollection', () => { items.push(new TextRange(2, 1)); items.push(new TextRange(4, 2)); const c = new TextRangeCollection(items); - const results = [-1, -1, 0, -1, 1, 1, -1]; + const results = [-1, -1, 0, -1, 1, -1, -1]; for (let i = 0; i < results.length; i += 1) { const index = c.getItemAtPosition(i); assert.equal(index, results[i]); From d8ab041f3f8fb459a37bd3adc10de82fadfd3c7b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 5 Dec 2017 10:07:55 -0800 Subject: [PATCH 13/49] Make test await for results --- src/test/autocomplete/base.test.ts | 27 +++++++++------------------ src/test/index.ts | 3 +-- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/test/autocomplete/base.test.ts b/src/test/autocomplete/base.test.ts index 59a7e51c14f4..e4b8b854570d 100644 --- a/src/test/autocomplete/base.test.ts +++ b/src/test/autocomplete/base.test.ts @@ -176,9 +176,7 @@ suite('Autocomplete', () => { }); // https://github.com/Microsoft/vscode-python/issues/110 - test('Suppress in strings/comments', done => { - let textEditor: vscode.TextEditor; - let textDocument: vscode.TextDocument; + test('Suppress in strings/comments', async () => { const positions = [ new vscode.Position(0, 1), // false new vscode.Position(0, 9), // true @@ -194,20 +192,13 @@ suite('Autocomplete', () => { const expected = [ false, true, false, false, false, false, false, false, false, false ]; - vscode.workspace.openTextDocument(fileSuppress).then(document => { - textDocument = document; - return vscode.window.showTextDocument(textDocument); - }).then(editor => { - assert(vscode.window.activeTextEditor, 'No active editor'); - textEditor = editor; - for (let i = 0; i < positions.length; i += 1) { - vscode.commands.executeCommand('vscode.executeCompletionItemProvider', - textDocument.uri, positions[i]).then(list => { - const result = list.items.filter(item => item.label === 'abs').length; - assert.equal(result > 0, expected[i], - `Expected ${expected[i]} at position ${positions[i].line}:${positions[i].character} but got ${result}`); - }); - } - }).then(done, done); + const textDocument = await vscode.workspace.openTextDocument(fileSuppress); + await vscode.window.showTextDocument(textDocument); + for (let i = 0; i < positions.length; i += 1) { + const list = await vscode.commands.executeCommand('vscode.executeCompletionItemProvider', textDocument.uri, positions[i]); + const result = list.items.filter(item => item.label === 'abs').length; + assert.equal(result > 0, expected[i], + `Expected ${expected[i]} at position ${positions[i].line}:${positions[i].character} but got ${result}`); + } }); }); diff --git a/src/test/index.ts b/src/test/index.ts index 9eb083463586..0202b4e2dc43 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -8,7 +8,6 @@ testRunner.configure({ ui: 'tdd', useColors: true, timeout: 25000, - retries: 3, - grep: "Language.TextRangeCollection" + retries: 3 } as {}); module.exports = testRunner; From db75cd00e892d8721c92df14b7b77ca92d5c2a0a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 5 Dec 2017 10:22:37 -0800 Subject: [PATCH 14/49] Add license headers --- src/client/language/characterStream.ts | 2 ++ src/test/autocomplete/base.test.ts | 3 +++ src/test/language/characterStream.test.ts | 4 ++++ src/test/language/textIterator.test.ts | 4 ++++ src/test/language/textRange.test.ts | 4 ++++ src/test/language/textRangeCollection.test.ts | 4 ++++ src/test/language/tokenizer.test.ts | 4 ++++ 7 files changed, 25 insertions(+) diff --git a/src/client/language/characterStream.ts b/src/client/language/characterStream.ts index e90ef7f2e6b7..a4da08659a9d 100644 --- a/src/client/language/characterStream.ts +++ b/src/client/language/characterStream.ts @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. 'use strict'; // tslint:disable-next-line:import-name diff --git a/src/test/autocomplete/base.test.ts b/src/test/autocomplete/base.test.ts index e4b8b854570d..1873f86f0776 100644 --- a/src/test/autocomplete/base.test.ts +++ b/src/test/autocomplete/base.test.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + // Note: This example test is leveraging the Mocha test framework. // Please refer to their documentation on https://mochajs.org/ for help. diff --git a/src/test/language/characterStream.test.ts b/src/test/language/characterStream.test.ts index d1c003d6b2d9..165fdde51ae9 100644 --- a/src/test/language/characterStream.test.ts +++ b/src/test/language/characterStream.test.ts @@ -1,3 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + import * as assert from 'assert'; // tslint:disable-next-line:import-name import Char from 'typescript-char'; diff --git a/src/test/language/textIterator.test.ts b/src/test/language/textIterator.test.ts index dddfe3478ec5..34daa81534cd 100644 --- a/src/test/language/textIterator.test.ts +++ b/src/test/language/textIterator.test.ts @@ -1,3 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + import * as assert from 'assert'; import { TextIterator } from '../../client/language/textIterator'; diff --git a/src/test/language/textRange.test.ts b/src/test/language/textRange.test.ts index 4e0da8feb06c..fecf287032a0 100644 --- a/src/test/language/textRange.test.ts +++ b/src/test/language/textRange.test.ts @@ -1,3 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + import * as assert from 'assert'; import { TextRange } from '../../client/language/definitions'; diff --git a/src/test/language/textRangeCollection.test.ts b/src/test/language/textRangeCollection.test.ts index 44666dd811ce..5c56c3f139c7 100644 --- a/src/test/language/textRangeCollection.test.ts +++ b/src/test/language/textRangeCollection.test.ts @@ -1,3 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + import * as assert from 'assert'; import { TextRange } from '../../client/language/definitions'; import { TextRangeCollection } from '../../client/language/textRangeCollection'; diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 1ce0165e4241..139441aeea81 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -1,3 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + import * as assert from 'assert'; import { TextRange, TokenType } from '../../client/language/definitions'; import { TextRangeCollection } from '../../client/language/textRangeCollection'; From 9ab2c47a609629570fc387e3b3697cd60e8397e4 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 5 Dec 2017 10:35:58 -0800 Subject: [PATCH 15/49] Rename definitions to types --- src/client/language/characterStream.ts | 2 +- src/client/language/textIterator.ts | 2 +- src/client/language/textRangeCollection.ts | 2 +- src/client/language/tokenizer.ts | 2 +- src/client/language/{definitions.ts => types.ts} | 0 src/client/providers/completionProvider.ts | 2 +- src/test/language/characterStream.test.ts | 2 +- src/test/language/textRange.test.ts | 2 +- src/test/language/textRangeCollection.test.ts | 2 +- src/test/language/tokenizer.test.ts | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename src/client/language/{definitions.ts => types.ts} (100%) diff --git a/src/client/language/characterStream.ts b/src/client/language/characterStream.ts index a4da08659a9d..a95af7ede457 100644 --- a/src/client/language/characterStream.ts +++ b/src/client/language/characterStream.ts @@ -4,8 +4,8 @@ // tslint:disable-next-line:import-name import Char from 'typescript-char'; -import { ICharacterStream, ITextIterator } from './definitions'; import { TextIterator } from './textIterator'; +import { ICharacterStream, ITextIterator } from './types'; export class CharacterStream implements ICharacterStream { private text: ITextIterator; diff --git a/src/client/language/textIterator.ts b/src/client/language/textIterator.ts index 927078da3939..3984dfbe3458 100644 --- a/src/client/language/textIterator.ts +++ b/src/client/language/textIterator.ts @@ -1,7 +1,7 @@ 'use strict'; import { Position, Range, TextDocument } from 'vscode'; -import { ITextIterator } from './definitions'; +import { ITextIterator } from './types'; export class TextIterator implements ITextIterator { private text: string; diff --git a/src/client/language/textRangeCollection.ts b/src/client/language/textRangeCollection.ts index d09becea902f..47983f8fe0cb 100644 --- a/src/client/language/textRangeCollection.ts +++ b/src/client/language/textRangeCollection.ts @@ -1,6 +1,6 @@ 'use strict'; -import { ITextRange, ITextRangeCollection } from './definitions'; +import { ITextRange, ITextRangeCollection } from './types'; export class TextRangeCollection implements ITextRangeCollection { private items: T[]; diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 6d63f4168414..8b122da7c346 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -3,8 +3,8 @@ // tslint:disable-next-line:import-name import Char from 'typescript-char'; import { CharacterStream } from './characterStream'; -import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './definitions'; import { TextRangeCollection } from './textRangeCollection'; +import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './types'; enum QuoteType { None, diff --git a/src/client/language/definitions.ts b/src/client/language/types.ts similarity index 100% rename from src/client/language/definitions.ts rename to src/client/language/types.ts diff --git a/src/client/providers/completionProvider.ts b/src/client/providers/completionProvider.ts index 10c930348e33..b6d62c340dd7 100644 --- a/src/client/providers/completionProvider.ts +++ b/src/client/providers/completionProvider.ts @@ -3,8 +3,8 @@ import * as vscode from 'vscode'; import { Position, ProviderResult, SnippetString, Uri } from 'vscode'; import { PythonSettings } from '../common/configSettings'; -import { TokenType } from '../language/definitions'; import { Tokenizer } from '../language/tokenizer'; +import { TokenType } from '../language/types'; import { JediFactory } from '../languageServices/jediProxyFactory'; import { captureTelemetry } from '../telemetry'; import { COMPLETION } from '../telemetry/constants'; diff --git a/src/test/language/characterStream.test.ts b/src/test/language/characterStream.test.ts index 165fdde51ae9..63ea71f01746 100644 --- a/src/test/language/characterStream.test.ts +++ b/src/test/language/characterStream.test.ts @@ -6,8 +6,8 @@ import * as assert from 'assert'; // tslint:disable-next-line:import-name import Char from 'typescript-char'; import { CharacterStream } from '../../client/language/characterStream'; -import { ICharacterStream, TextRange } from '../../client/language/definitions'; import { TextIterator } from '../../client/language/textIterator'; +import { ICharacterStream, TextRange } from '../../client/language/types'; // tslint:disable-next-line:max-func-body-length suite('Language.CharacterStream', () => { diff --git a/src/test/language/textRange.test.ts b/src/test/language/textRange.test.ts index fecf287032a0..02cad753c16f 100644 --- a/src/test/language/textRange.test.ts +++ b/src/test/language/textRange.test.ts @@ -3,7 +3,7 @@ 'use strict'; import * as assert from 'assert'; -import { TextRange } from '../../client/language/definitions'; +import { TextRange } from '../../client/language/types'; // tslint:disable-next-line:max-func-body-length suite('Language.TextRange', () => { diff --git a/src/test/language/textRangeCollection.test.ts b/src/test/language/textRangeCollection.test.ts index 5c56c3f139c7..32522e63c778 100644 --- a/src/test/language/textRangeCollection.test.ts +++ b/src/test/language/textRangeCollection.test.ts @@ -3,8 +3,8 @@ 'use strict'; import * as assert from 'assert'; -import { TextRange } from '../../client/language/definitions'; import { TextRangeCollection } from '../../client/language/textRangeCollection'; +import { TextRange } from '../../client/language/types'; // tslint:disable-next-line:max-func-body-length suite('Language.TextRangeCollection', () => { diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 139441aeea81..7642b88acfaa 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -3,9 +3,9 @@ 'use strict'; import * as assert from 'assert'; -import { TextRange, TokenType } from '../../client/language/definitions'; import { TextRangeCollection } from '../../client/language/textRangeCollection'; import { Tokenizer } from '../../client/language/tokenizer'; +import { TextRange, TokenType } from '../../client/language/types'; // tslint:disable-next-line:max-func-body-length suite('Language.Tokenizer', () => { From d587485696c15d2a5dec35fdc9f317dfce3b9a39 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 5 Dec 2017 12:02:27 -0800 Subject: [PATCH 16/49] License headers --- src/client/language/textIterator.ts | 2 ++ src/client/language/textRangeCollection.ts | 2 ++ src/client/language/tokenizer.ts | 2 ++ src/client/language/types.ts | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/client/language/textIterator.ts b/src/client/language/textIterator.ts index 3984dfbe3458..d5eda4783e2c 100644 --- a/src/client/language/textIterator.ts +++ b/src/client/language/textIterator.ts @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. 'use strict'; import { Position, Range, TextDocument } from 'vscode'; diff --git a/src/client/language/textRangeCollection.ts b/src/client/language/textRangeCollection.ts index 47983f8fe0cb..8ce5a744c9a6 100644 --- a/src/client/language/textRangeCollection.ts +++ b/src/client/language/textRangeCollection.ts @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. 'use strict'; import { ITextRange, ITextRangeCollection } from './types'; diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 8b122da7c346..60d9fadc7e2e 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. 'use strict'; // tslint:disable-next-line:import-name diff --git a/src/client/language/types.ts b/src/client/language/types.ts index d001adccbd88..121ee682c085 100644 --- a/src/client/language/types.ts +++ b/src/client/language/types.ts @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. 'use strict'; export interface ITextRange { From 1ac4932c51f35f95aaf4067b40155d5b18ad3f49 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 11 Dec 2017 14:57:37 -0800 Subject: [PATCH 17/49] Fix typo in completion details (typo) --- src/client/providers/itemInfoSource.ts | 2 +- src/test/index.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 9dc906e23580..152bf10cc48f 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -117,7 +117,7 @@ export class ItemInfoSource { } const descriptionWithHighlightedCode = this.highlightCode(dnd[1]); - const tooltip = new vscode.MarkdownString(['y```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); + const tooltip = new vscode.MarkdownString(['```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(dnd[1]))); const key = signature + lines.join(''); diff --git a/src/test/index.ts b/src/test/index.ts index eebaa9b59c8d..4d3b12a351ca 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -12,8 +12,7 @@ const options: MochaSetupOptions & { retries: number } = { ui: 'tdd', useColors: true, timeout: 25000, - retries: 3, - grep: 'Autocomplete' + retries: 3 }; testRunner.configure(options); module.exports = testRunner; From 2aa5a6c32b0e1c8df853f65acb7ceb27bd519f78 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 11 Dec 2017 17:06:59 -0800 Subject: [PATCH 18/49] Fix hover test --- src/client/providers/itemInfoSource.ts | 2 +- src/test/definitions/hover.test.ts | 70 ++++++++++++++------------ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 152bf10cc48f..3cb471959bac 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -116,7 +116,7 @@ export class ItemInfoSource { lines.shift(); } - const descriptionWithHighlightedCode = this.highlightCode(dnd[1]); + const descriptionWithHighlightedCode = this.highlightCode(lines.join(EOL)); const tooltip = new vscode.MarkdownString(['```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(dnd[1]))); diff --git a/src/test/definitions/hover.test.ts b/src/test/definitions/hover.test.ts index 07f06b8843ca..adc7832b80fc 100644 --- a/src/test/definitions/hover.test.ts +++ b/src/test/definitions/hover.test.ts @@ -1,15 +1,14 @@ // Note: This example test is leveraging the Mocha test framework. // Please refer to their documentation on https://mochajs.org/ for help. - // The module 'assert' provides assertion methods from node import * as assert from 'assert'; import { EOL } from 'os'; // You can import and use all API from the 'vscode' module // as well as import your extension to test it -import * as vscode from 'vscode'; import * as path from 'path'; -import { initialize, closeActiveWindows, initializeTest } from '../initialize'; +import * as vscode from 'vscode'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; import { normalizeMarkedString } from '../textUtils'; const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'autocomp'); @@ -21,11 +20,12 @@ const fileEncodingUsed = path.join(autoCompPath, 'five.py'); const fileHover = path.join(autoCompPath, 'hoverTest.py'); const fileStringFormat = path.join(hoverPath, 'stringFormat.py'); +// tslint:disable-next-line:max-func-body-length suite('Hover Definition', () => { - suiteSetup(() => initialize()); - setup(() => initializeTest()); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteSetup(initialize); + setup(initializeTest); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); test('Method', done => { let textEditor: vscode.TextEditor; @@ -43,6 +43,7 @@ suite('Hover Definition', () => { assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '30,4', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '30,11', 'End position is incorrect'); assert.equal(def[0].contents.length, 1, 'Invalid content items'); + // tslint:disable-next-line:prefer-template const expectedContent = '```python' + EOL + 'def method1()' + EOL + '```' + EOL + 'This is method1'; assert.equal(normalizeMarkedString(def[0].contents[0]), expectedContent, 'function signature incorrect'); }).then(done, done); @@ -63,6 +64,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '1,9', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '1,12', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'def fun()' + EOL + '```' + EOL + 'This is fun', 'Invalid conents'); }).then(done, done); }); @@ -82,6 +84,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '25,4', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '25,7', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'def bar()' + EOL + '```' + EOL + '说明 - keep this line, it works' + EOL + 'delete following line, it works' + EOL + '如果存在需要等待审批或正在执行的任务，将不刷新页面', 'Invalid conents'); @@ -103,6 +106,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '1,5', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '1,16', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'def showMessage()' + EOL + '```' + EOL + @@ -158,19 +162,20 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '11,12', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '11,18', 'End position is incorrect'); - let documentation = "```python" + EOL + - "class Random(x=None)" + EOL + - "```" + EOL + - "Random number generator base class used by bound module functions." + EOL + - "" + EOL + - "Used to instantiate instances of Random to get generators that don't" + EOL + - "share state." + EOL + - "" + EOL + - "Class Random can also be subclassed if you want to use a different basic" + EOL + - "generator of your own devising: in that case, override the following" + EOL + EOL + - "`methods` random(), seed(), getstate(), and setstate()." + EOL + EOL + - "Optionally, implement a getrandbits() method so that randrange()" + EOL + - "can cover arbitrarily large ranges."; + // tslint:disable-next-line:prefer-template + const documentation = '```python' + EOL + + 'class Random(x=None)' + EOL + + '```' + EOL + + 'Random number generator base class used by bound module functions.' + EOL + + '' + EOL + + 'Used to instantiate instances of Random to get generators that don\'t' + EOL + + 'share state.' + EOL + + '' + EOL + + 'Class Random can also be subclassed if you want to use a different basic' + EOL + + 'generator of your own devising: in that case, override the following' + EOL + EOL + + '`methods` random(), seed(), getstate(), and setstate().' + EOL + EOL + + 'Optionally, implement a getrandbits() method so that randrange()' + EOL + + 'can cover arbitrarily large ranges.'; assert.equal(normalizeMarkedString(def[0].contents[0]), documentation, 'Invalid conents'); }).then(done, done); @@ -191,6 +196,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '12,5', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '12,12', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'def randint(a, b)' + EOL + '```' + EOL + @@ -213,6 +219,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '8,11', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '8,15', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'def acos(x)' + EOL + '```' + EOL + @@ -235,6 +242,7 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(`${def[0].range.start.line},${def[0].range.start.character}`, '14,9', 'Start position is incorrect'); assert.equal(`${def[0].range.end.line},${def[0].range.end.character}`, '14,15', 'End position is incorrect'); + // tslint:disable-next-line:prefer-template assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'class Thread(group=None, target=None, name=None, args=(), kwargs=None, verbose=None)' + EOL + '```' + EOL + @@ -262,14 +270,14 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(def[0].contents.length, 1, 'Only expected one result'); const contents = normalizeMarkedString(def[0].contents[0]); - if (contents.indexOf("```python") === -1) { - assert.fail(contents, "", "First line is incorrect", "compare"); + if (contents.indexOf('```python') === -1) { + assert.fail(contents, '', 'First line is incorrect', 'compare'); } - if (contents.indexOf("Random number generator base class used by bound module functions.") === -1) { - assert.fail(contents, "", "'Random number generator' message missing", "compare"); + if (contents.indexOf('Random number generator base class used by bound module functions.') === -1) { + assert.fail(contents, '', '\'Random number generator\' message missing', 'compare'); } - if (contents.indexOf("Class Random can also be subclassed if you want to use a different basic") === -1) { - assert.fail(contents, "", "'Class Random message' missing", "compare"); + if (contents.indexOf('Class Random can also be subclassed if you want to use a different basic') === -1) { + assert.fail(contents, '', '\'Class Random message\' missing', 'compare'); } }).then(done, done); }); @@ -282,12 +290,12 @@ suite('Hover Definition', () => { assert.equal(def.length, 1, 'Definition length is incorrect'); assert.equal(def[0].contents.length, 1, 'Only expected one result'); const contents = normalizeMarkedString(def[0].contents[0]); - if (contents.indexOf("def capitalize") === -1) { - assert.fail(contents, "", "'def capitalize' is missing", "compare"); + if (contents.indexOf('def capitalize') === -1) { + assert.fail(contents, '', '\'def capitalize\' is missing', 'compare'); } - if (contents.indexOf("Return a capitalized version of S") === -1 && - contents.indexOf("Return a copy of the string S with only its first character") === -1) { - assert.fail(contents, "", "'Return a capitalized version of S/Return a copy of the string S with only its first character' message missing", "compare"); + if (contents.indexOf('Return a capitalized version of S') === -1 && + contents.indexOf('Return a copy of the string S with only its first character') === -1) { + assert.fail(contents, '', '\'Return a capitalized version of S/Return a copy of the string S with only its first character\' message missing', 'compare'); } }); }); From 560d2af430b6188a78b086b51d4d2d669f685c61 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 12 Dec 2017 16:10:28 -0800 Subject: [PATCH 19/49] Russian translations --- package.nls.ru.json | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 package.nls.ru.json diff --git a/package.nls.ru.json b/package.nls.ru.json new file mode 100644 index 000000000000..4b082501ca78 --- /dev/null +++ b/package.nls.ru.json @@ -0,0 +1,50 @@ +{ + "python.command.python.sortImports.title": "Отсортировать Imports", + "python.command.python.startREPL.title": "Открыть REPL", + "python.command.python.buildWorkspaceSymbols.title": "Собрать символы рабочего пространства", + "python.command.python.runtests.title": "Запустить все тесты", + "python.command.python.debugtests.title": "Запустить все тесты под отладчиком", + "python.command.python.execInTerminal.title": "Выполнить файл в консоли", + "python.command.python.setInterpreter.title": "Выбрать интерпретатор", + "python.command.python.updateSparkLibrary.title": "Обновить библиотеки PySpark", + "python.command.python.refactorExtractVariable.title": "Создать переменную", + "python.command.python.refactorExtractMethod.title": "Создать метод", + "python.command.python.viewTestOutput.title": "Показать вывод теста", + "python.command.python.selectAndRunTestMethod.title": "Запусть тестовый метод...", + "python.command.python.selectAndDebugTestMethod.title": "Отладить тестовый метод...", + "python.command.python.selectAndRunTestFile.title": "Запустить тестовый файл...", + "python.command.python.runCurrentTestFile.title": "Запустить текущий тестовый файл", + "python.command.python.runFailedTests.title": "Запустить непрошедшие тесты", + "python.command.python.execSelectionInTerminal.title": "Выполнить выбранный текст или текущую строку в консоли", + "python.command.python.execSelectionInDjangoShell.title": "Выполнить выбранный текст или текущую строку в оболочке Django", + "python.command.jupyter.runSelectionLine.title": "Выполнить выбранный текст или текущую строку", + "python.command.jupyter.execCurrentCell.title": "Выполнить ячейку", + "python.command.jupyter.execCurrentCellAndAdvance.title": "Выполнить ячейку и перейти к следующей", + "python.command.jupyter.gotToPreviousCell.title": "Перейти к предыдущей ячейке", + "python.command.jupyter.gotToNextCell.title": "Перейти к следующей ячейке", + "python.command.python.goToPythonObject.title": "Перейти к объекту Python", + "python.snippet.launch.standard.label": "Python", + "python.snippet.launch.standard.description": "Отладить программу Python со стандартным выводом", + "python.snippet.launch.pyspark.label": "Python: PySpark", + "python.snippet.launch.pyspark.description": "Отладка PySpark", + "python.snippet.launch.module.label": "Python: Модуль", + "python.snippet.launch.module.description": "Отладка модуля", + "python.snippet.launch.terminal.label": "Python: Интегрированная консоль", + "python.snippet.launch.terminal.description": "Отладка программы Python в интегрированной консоли", + "python.snippet.launch.externalTerminal.label": "Python: Внешний терминал", + "python.snippet.launch.externalTerminal.description": "Отладка программы Python во внешней консоли", + "python.snippet.launch.django.label": "Python: Django", + "python.snippet.launch.django.description": "Отладка приложения Django", + "python.snippet.launch.flask.label": "Python: Flask (0.11.x или новее)", + "python.snippet.launch.flask.description": "Отладка приложения Flask", + "python.snippet.launch.flaskOld.label": "Python: Flask (0.10.x или старее)", + "python.snippet.launch.flaskOld.description": "Отладка приложения Flask (старый стиль)", + "python.snippet.launch.pyramid.label": "Python: Приложение Pyramid", + "python.snippet.launch.pyramid.description": "Отладка приложения Pyramid", + "python.snippet.launch.watson.label": "Python: Приложение Watson", + "python.snippet.launch.watson.description": "Отладка приложения Watson", + "python.snippet.launch.attach.label": "Python: Подключить отладчик", + "python.snippet.launch.attach.description": "Подключить отладчик для удаленной отладки", + "python.snippet.launch.scrapy.label": "Python: Scrapy", + "python.snippet.launch.scrapy.description": "Scrapy в интергрированной консоли" +} From 31aa087843c9bb84ca29c322110828909256ee15 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 10:32:54 -0800 Subject: [PATCH 20/49] Update to better translation --- package.nls.ru.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.nls.ru.json b/package.nls.ru.json index 4b082501ca78..1ab5894793da 100644 --- a/package.nls.ru.json +++ b/package.nls.ru.json @@ -7,8 +7,8 @@ "python.command.python.execInTerminal.title": "Выполнить файл в консоли", "python.command.python.setInterpreter.title": "Выбрать интерпретатор", "python.command.python.updateSparkLibrary.title": "Обновить библиотеки PySpark", - "python.command.python.refactorExtractVariable.title": "Создать переменную", - "python.command.python.refactorExtractMethod.title": "Создать метод", + "python.command.python.refactorExtractVariable.title": "Извлечь в переменную", + "python.command.python.refactorExtractMethod.title": "Извлечь в метод", "python.command.python.viewTestOutput.title": "Показать вывод теста", "python.command.python.selectAndRunTestMethod.title": "Запусть тестовый метод...", "python.command.python.selectAndDebugTestMethod.title": "Отладить тестовый метод...", From 593ae0558c05398ca818ad033f06a4d10587ec2c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 10:34:13 -0800 Subject: [PATCH 21/49] Fix typo --- package.nls.ru.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nls.ru.json b/package.nls.ru.json index 1ab5894793da..e0e22abbc06d 100644 --- a/package.nls.ru.json +++ b/package.nls.ru.json @@ -46,5 +46,5 @@ "python.snippet.launch.attach.label": "Python: Подключить отладчик", "python.snippet.launch.attach.description": "Подключить отладчик для удаленной отладки", "python.snippet.launch.scrapy.label": "Python: Scrapy", - "python.snippet.launch.scrapy.description": "Scrapy в интергрированной консоли" + "python.snippet.launch.scrapy.description": "Scrapy в интегрированной консоли" } From e6d69bb7a88eacae85b09a9ebab0026c707fb239 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 14:45:16 -0800 Subject: [PATCH 22/49] #70 How to get all parameter info when filling in a function param list --- src/client/providers/signatureProvider.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/client/providers/signatureProvider.ts b/src/client/providers/signatureProvider.ts index bf6480a4d3a2..af2ba64d9692 100644 --- a/src/client/providers/signatureProvider.ts +++ b/src/client/providers/signatureProvider.ts @@ -1,5 +1,6 @@ 'use strict'; +import { EOL } from 'os'; import * as vscode from 'vscode'; import { CancellationToken, Position, SignatureHelp, TextDocument } from 'vscode'; import { JediFactory } from '../languageServices/jediProxyFactory'; @@ -55,8 +56,13 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { signature.activeParameter = def.paramindex; // Don't display the documentation, as vs code doesn't format the docmentation. // i.e. line feeds are not respected, long content is stripped. + const docLines = def.docstring.splitLines(); + const label = docLines[0].trim(); + const documentation = docLines.length > 1 ? docLines.filter((line, index) => index > 0).join(EOL) : ''; + const sig = { - label: def.description, + label: label, + documentation: documentation, parameters: [] }; sig.parameters = def.params.map(arg => { @@ -65,7 +71,7 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { } return { documentation: arg.docstring.length > 0 ? arg.docstring : arg.description, - label: arg.description.length > 0 ? arg.description : arg.name + label: arg.name }; }); signature.signatures.push(sig); @@ -85,7 +91,7 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { source: document.getText() }; return this.jediFactory.getJediProxyHandler(document.uri).sendCommand(cmd, token).then(data => { - return PythonSignatureProvider.parseData(data); + return data ? PythonSignatureProvider.parseData(data) : undefined; }); } } From b5a23d3f24b5e55edff0838376a962c029d7a4cc Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 16:03:31 -0800 Subject: [PATCH 23/49] Fix #70 How to get all parameter info when filling in a function param list --- src/test/pythonFiles/signature/one.py | 6 ++ src/test/pythonFiles/signature/two.py | 1 + src/test/signature/signature.test.ts | 95 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 src/test/pythonFiles/signature/one.py create mode 100644 src/test/pythonFiles/signature/two.py create mode 100644 src/test/signature/signature.test.ts diff --git a/src/test/pythonFiles/signature/one.py b/src/test/pythonFiles/signature/one.py new file mode 100644 index 000000000000..baa4045489e7 --- /dev/null +++ b/src/test/pythonFiles/signature/one.py @@ -0,0 +1,6 @@ +class Person: + def __init__(self, name, age = 23): + self.name = name + self.age = age + +p1 = Person('Bob', ) diff --git a/src/test/pythonFiles/signature/two.py b/src/test/pythonFiles/signature/two.py new file mode 100644 index 000000000000..beaa970c7eb5 --- /dev/null +++ b/src/test/pythonFiles/signature/two.py @@ -0,0 +1 @@ +pow(c, 1, diff --git a/src/test/signature/signature.test.ts b/src/test/signature/signature.test.ts new file mode 100644 index 000000000000..d3578b62b8b3 --- /dev/null +++ b/src/test/signature/signature.test.ts @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + +import * as assert from 'assert'; +import { EOL } from 'os'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { PythonSettings } from '../../client/common/configSettings'; +import { execPythonFile } from '../../client/common/utils'; +import { rootWorkspaceUri } from '../common'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; + +const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'signature'); +const fileOne = path.join(autoCompPath, 'one.py'); +const fileTwo = path.join(autoCompPath, 'two.py'); + +class SignatureHelpResult { + constructor( + public line: number, + public index: number, + public signaturesCount: number, + public activeParameter: number, + public parameterName: string | null) { } +} + +// tslint:disable-next-line:max-func-body-length +suite('Signatures', () => { + suiteSetup(async () => { + await initialize(); + const version = await execPythonFile(rootWorkspaceUri, PythonSettings.getInstance(rootWorkspaceUri).pythonPath, ['--version'], __dirname, true); + }); + setup(initializeTest); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); + + test('For ctor', async () => { + const expected = [ + new SignatureHelpResult(5, 11, 0, 0, null), + new SignatureHelpResult(5, 12, 1, 0, 'name'), + new SignatureHelpResult(5, 13, 1, 0, 'name'), + new SignatureHelpResult(5, 14, 1, 0, 'name'), + new SignatureHelpResult(5, 15, 1, 0, 'name'), + new SignatureHelpResult(5, 16, 1, 0, 'name'), + new SignatureHelpResult(5, 17, 1, 0, 'name'), + new SignatureHelpResult(5, 18, 1, 1, 'age'), + new SignatureHelpResult(5, 19, 1, 1, 'age'), + new SignatureHelpResult(5, 20, 0, 0, null) + ]; + + const document = await openDocument(fileOne); + for (const e of expected) { + await checkSignature(e, document!.uri); + } + }); + + test('For intrinsic', async () => { + const expected = [ + new SignatureHelpResult(0, 0, 0, 0, null), + new SignatureHelpResult(0, 1, 0, 0, null), + new SignatureHelpResult(0, 2, 0, 0, null), + new SignatureHelpResult(0, 3, 0, 0, null), + new SignatureHelpResult(0, 4, 1, 0, 'x'), + new SignatureHelpResult(0, 5, 1, 0, 'x'), + new SignatureHelpResult(0, 6, 1, 1, 'y'), + new SignatureHelpResult(0, 7, 1, 1, 'y'), + new SignatureHelpResult(0, 8, 1, 1, 'y'), + new SignatureHelpResult(0, 9, 1, 2, 'z'), + new SignatureHelpResult(0, 10, 1, 2, 'z'), + new SignatureHelpResult(1, 0, 1, 2, 'z') + ]; + + const document = await openDocument(fileTwo); + for (const e of expected) { + await checkSignature(e, document!.uri); + } + }); +}); + +async function openDocument(documentPath: string): Promise { + const document = await vscode.workspace.openTextDocument(documentPath); + await vscode.window.showTextDocument(document!); + return document; +} + +async function checkSignature(expected: SignatureHelpResult, uri: vscode.Uri) { + const position = new vscode.Position(expected.line, expected.index); + const actual = await vscode.commands.executeCommand('vscode.executeSignatureHelpProvider', uri, position); + assert.equal(actual!.signatures.length, expected.signaturesCount); + if (expected.signaturesCount > 0) { + assert.equal(actual!.activeParameter, expected.activeParameter); + const parameter = actual!.signatures[0].parameters[expected.activeParameter]; + assert.equal(parameter.label, expected.parameterName); + } +} From cd200f7913a959c472ebfe1e194edc584f4f9249 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 16:05:04 -0800 Subject: [PATCH 24/49] Clean up --- src/test/signature/signature.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/signature/signature.test.ts b/src/test/signature/signature.test.ts index d3578b62b8b3..0295a49352f1 100644 --- a/src/test/signature/signature.test.ts +++ b/src/test/signature/signature.test.ts @@ -3,7 +3,6 @@ 'use strict'; import * as assert from 'assert'; -import { EOL } from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; import { PythonSettings } from '../../client/common/configSettings'; @@ -28,7 +27,6 @@ class SignatureHelpResult { suite('Signatures', () => { suiteSetup(async () => { await initialize(); - const version = await execPythonFile(rootWorkspaceUri, PythonSettings.getInstance(rootWorkspaceUri).pythonPath, ['--version'], __dirname, true); }); setup(initializeTest); suiteTeardown(closeActiveWindows); From 7c33228d2dd127f485ac7afde0c2776f9e293411 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 13 Dec 2017 16:05:48 -0800 Subject: [PATCH 25/49] Clean imports --- src/test/signature/signature.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/signature/signature.test.ts b/src/test/signature/signature.test.ts index 0295a49352f1..6cac87049e45 100644 --- a/src/test/signature/signature.test.ts +++ b/src/test/signature/signature.test.ts @@ -5,9 +5,6 @@ import * as assert from 'assert'; import * as path from 'path'; import * as vscode from 'vscode'; -import { PythonSettings } from '../../client/common/configSettings'; -import { execPythonFile } from '../../client/common/utils'; -import { rootWorkspaceUri } from '../common'; import { closeActiveWindows, initialize, initializeTest } from '../initialize'; const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'signature'); From c4a6b90d7d0e47c3dc3259a5001abd3d1fea4ab5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 14 Dec 2017 09:39:20 -0800 Subject: [PATCH 26/49] CR feedback --- src/client/providers/signatureProvider.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/providers/signatureProvider.ts b/src/client/providers/signatureProvider.ts index af2ba64d9692..d86afae0df7e 100644 --- a/src/client/providers/signatureProvider.ts +++ b/src/client/providers/signatureProvider.ts @@ -57,12 +57,12 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { // Don't display the documentation, as vs code doesn't format the docmentation. // i.e. line feeds are not respected, long content is stripped. const docLines = def.docstring.splitLines(); - const label = docLines[0].trim(); - const documentation = docLines.length > 1 ? docLines.filter((line, index) => index > 0).join(EOL) : ''; + const label = docLines.shift().trim(); + const documentation = docLines.join(EOL).trim(); const sig = { - label: label, - documentation: documentation, + label, + documentation, parameters: [] }; sig.parameters = def.params.map(arg => { @@ -91,7 +91,7 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { source: document.getText() }; return this.jediFactory.getJediProxyHandler(document.uri).sendCommand(cmd, token).then(data => { - return data ? PythonSignatureProvider.parseData(data) : undefined; + return data ? PythonSignatureProvider.parseData(data) : new SignatureHelp(); }); } } From f85b848a82c4e522831f060bd7b2c241ca8e5f0b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 14 Dec 2017 14:55:13 -0800 Subject: [PATCH 27/49] Trim whitespace for test stability --- src/client/providers/signatureProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/providers/signatureProvider.ts b/src/client/providers/signatureProvider.ts index d86afae0df7e..d6f24484cbb7 100644 --- a/src/client/providers/signatureProvider.ts +++ b/src/client/providers/signatureProvider.ts @@ -71,7 +71,7 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { } return { documentation: arg.docstring.length > 0 ? arg.docstring : arg.description, - label: arg.name + label: arg.name.trim() }; }); signature.signatures.push(sig); From 37c210ba5fa5f2cfe3cdace9e749ffbe1cb2365d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 14 Dec 2017 16:53:04 -0800 Subject: [PATCH 28/49] More tests --- pythonFiles/completion.py | 5 --- src/client/providers/signatureProvider.ts | 18 ++++++-- src/test/pythonFiles/signature/three.py | 1 + src/test/pythonFiles/signature/two.py | 2 +- src/test/signature/signature.test.ts | 53 ++++++++++++++++------- 5 files changed, 53 insertions(+), 26 deletions(-) create mode 100644 src/test/pythonFiles/signature/three.py diff --git a/pythonFiles/completion.py b/pythonFiles/completion.py index f072349d0999..ce798d246bab 100644 --- a/pythonFiles/completion.py +++ b/pythonFiles/completion.py @@ -6,7 +6,6 @@ import traceback import platform -WORD_RE = re.compile(r'\w') jediPreview = False class RedirectStdout(object): @@ -111,8 +110,6 @@ def _get_call_signatures(self, script): continue if param.name == 'self' and pos == 0: continue - if WORD_RE.match(param.name) is None: - continue try: name, value = param.description.split('=') except ValueError: @@ -155,8 +152,6 @@ def _get_call_signatures_with_args(self, script): continue if param.name == 'self' and pos == 0: continue - if WORD_RE.match(param.name) is None: - continue try: name, value = param.description.split('=') except ValueError: diff --git a/src/client/providers/signatureProvider.ts b/src/client/providers/signatureProvider.ts index d6f24484cbb7..43caa67cd9cf 100644 --- a/src/client/providers/signatureProvider.ts +++ b/src/client/providers/signatureProvider.ts @@ -54,11 +54,21 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { data.definitions.forEach(def => { signature.activeParameter = def.paramindex; - // Don't display the documentation, as vs code doesn't format the docmentation. + // Don't display the documentation, as vs code doesn't format the documentation. // i.e. line feeds are not respected, long content is stripped. - const docLines = def.docstring.splitLines(); - const label = docLines.shift().trim(); - const documentation = docLines.join(EOL).trim(); + + // Some functions do not come with parameter docs + let label: string; + let documentation: string; + + if (def.params && def.params.length > 0) { + const docLines = def.docstring.splitLines(); + label = docLines.shift().trim(); + documentation = docLines.join(EOL).trim(); + } else { + label = ''; + documentation = def.docstring; + } const sig = { label, diff --git a/src/test/pythonFiles/signature/three.py b/src/test/pythonFiles/signature/three.py new file mode 100644 index 000000000000..fe666b9ff4c8 --- /dev/null +++ b/src/test/pythonFiles/signature/three.py @@ -0,0 +1 @@ +print(a, b, z) diff --git a/src/test/pythonFiles/signature/two.py b/src/test/pythonFiles/signature/two.py index beaa970c7eb5..ae7a551707b8 100644 --- a/src/test/pythonFiles/signature/two.py +++ b/src/test/pythonFiles/signature/two.py @@ -1 +1 @@ -pow(c, 1, +range(c, 1, diff --git a/src/test/signature/signature.test.ts b/src/test/signature/signature.test.ts index 6cac87049e45..5fa42aee2ea0 100644 --- a/src/test/signature/signature.test.ts +++ b/src/test/signature/signature.test.ts @@ -10,6 +10,7 @@ import { closeActiveWindows, initialize, initializeTest } from '../initialize'; const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'signature'); const fileOne = path.join(autoCompPath, 'one.py'); const fileTwo = path.join(autoCompPath, 'two.py'); +const fileThree = path.join(autoCompPath, 'three.py'); class SignatureHelpResult { constructor( @@ -44,8 +45,8 @@ suite('Signatures', () => { ]; const document = await openDocument(fileOne); - for (const e of expected) { - await checkSignature(e, document!.uri); + for (let i = 0; i < expected.length; i += 1) { + await checkSignature(expected[i], document!.uri, i); } }); @@ -55,19 +56,39 @@ suite('Signatures', () => { new SignatureHelpResult(0, 1, 0, 0, null), new SignatureHelpResult(0, 2, 0, 0, null), new SignatureHelpResult(0, 3, 0, 0, null), - new SignatureHelpResult(0, 4, 1, 0, 'x'), - new SignatureHelpResult(0, 5, 1, 0, 'x'), - new SignatureHelpResult(0, 6, 1, 1, 'y'), - new SignatureHelpResult(0, 7, 1, 1, 'y'), - new SignatureHelpResult(0, 8, 1, 1, 'y'), - new SignatureHelpResult(0, 9, 1, 2, 'z'), - new SignatureHelpResult(0, 10, 1, 2, 'z'), - new SignatureHelpResult(1, 0, 1, 2, 'z') + new SignatureHelpResult(0, 4, 0, 0, null), + new SignatureHelpResult(0, 5, 0, 0, null), + new SignatureHelpResult(0, 6, 1, 0, 'start'), + new SignatureHelpResult(0, 7, 1, 0, 'start'), + new SignatureHelpResult(0, 8, 1, 1, 'stop'), + new SignatureHelpResult(0, 9, 1, 1, 'stop'), + new SignatureHelpResult(0, 10, 1, 1, 'stop'), + new SignatureHelpResult(0, 11, 1, 2, 'step'), + new SignatureHelpResult(1, 0, 1, 2, 'step') ]; const document = await openDocument(fileTwo); - for (const e of expected) { - await checkSignature(e, document!.uri); + for (let i = 0; i < expected.length; i += 1) { + await checkSignature(expected[i], document!.uri, i); + } + }); + + test('For ellipsis', async () => { + const expected = [ + new SignatureHelpResult(0, 4, 0, 0, null), + new SignatureHelpResult(0, 5, 0, 0, null), + new SignatureHelpResult(0, 6, 1, 0, 'value'), + new SignatureHelpResult(0, 7, 1, 0, 'value'), + new SignatureHelpResult(0, 8, 1, 1, '...'), + new SignatureHelpResult(0, 9, 1, 1, '...'), + new SignatureHelpResult(0, 10, 1, 1, '...'), + new SignatureHelpResult(0, 11, 1, 2, 'sep'), + new SignatureHelpResult(0, 12, 1, 2, 'sep') + ]; + + const document = await openDocument(fileThree); + for (let i = 0; i < expected.length; i += 1) { + await checkSignature(expected[i], document!.uri, i); } }); }); @@ -78,13 +99,13 @@ async function openDocument(documentPath: string): Promise('vscode.executeSignatureHelpProvider', uri, position); - assert.equal(actual!.signatures.length, expected.signaturesCount); + assert.equal(actual!.signatures.length, expected.signaturesCount, `Signature count does not match, case ${caseIndex}`); if (expected.signaturesCount > 0) { - assert.equal(actual!.activeParameter, expected.activeParameter); + assert.equal(actual!.activeParameter, expected.activeParameter, `Parameter index does not match, case ${caseIndex}`); const parameter = actual!.signatures[0].parameters[expected.activeParameter]; - assert.equal(parameter.label, expected.parameterName); + assert.equal(parameter.label, expected.parameterName, `Parameter name is incorrect, case ${caseIndex}`); } } From 61a56504912b81b163eb1de4547032f243d0920f Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 14 Dec 2017 17:02:09 -0800 Subject: [PATCH 29/49] Better handle no-parameters documentation --- src/client/providers/signatureProvider.ts | 26 +++++++++++++---------- src/test/index.ts | 3 ++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/client/providers/signatureProvider.ts b/src/client/providers/signatureProvider.ts index 43caa67cd9cf..12dad261c39b 100644 --- a/src/client/providers/signatureProvider.ts +++ b/src/client/providers/signatureProvider.ts @@ -60,13 +60,14 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { // Some functions do not come with parameter docs let label: string; let documentation: string; + const validParamInfo = def.params && def.params.length > 0 && def.docstring.startsWith(`${def.name}(`); - if (def.params && def.params.length > 0) { + if (validParamInfo) { const docLines = def.docstring.splitLines(); label = docLines.shift().trim(); documentation = docLines.join(EOL).trim(); } else { - label = ''; + label = def.description; documentation = def.docstring; } @@ -75,15 +76,18 @@ export class PythonSignatureProvider implements vscode.SignatureHelpProvider { documentation, parameters: [] }; - sig.parameters = def.params.map(arg => { - if (arg.docstring.length === 0) { - arg.docstring = extractParamDocString(arg.name, def.docstring); - } - return { - documentation: arg.docstring.length > 0 ? arg.docstring : arg.description, - label: arg.name.trim() - }; - }); + + if (validParamInfo) { + sig.parameters = def.params.map(arg => { + if (arg.docstring.length === 0) { + arg.docstring = extractParamDocString(arg.name, def.docstring); + } + return { + documentation: arg.docstring.length > 0 ? arg.docstring : arg.description, + label: arg.name.trim() + }; + }); + } signature.signatures.push(sig); }); return signature; diff --git a/src/test/index.ts b/src/test/index.ts index 4d3b12a351ca..3a5cdd7b0602 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -12,7 +12,8 @@ const options: MochaSetupOptions & { retries: number } = { ui: 'tdd', useColors: true, timeout: 25000, - retries: 3 + retries: 3, + grep: 'Signatures' }; testRunner.configure(options); module.exports = testRunner; From a10305e115dff47eaf5a00764709a004076341c7 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Dec 2017 09:40:02 -0800 Subject: [PATCH 30/49] Better handle ellipsis and Python3 --- src/client/common/utils.ts | 4 +- .../signature/{two.py => basicSig.py} | 1 + .../signature/{one.py => classCtor.py} | 0 src/test/pythonFiles/signature/ellipsis.py | 1 + src/test/pythonFiles/signature/noSigPy3.py | 1 + src/test/pythonFiles/signature/three.py | 1 - src/test/signature/signature.test.ts | 37 ++++++++++++++----- 7 files changed, 33 insertions(+), 12 deletions(-) rename src/test/pythonFiles/signature/{two.py => basicSig.py} (92%) rename src/test/pythonFiles/signature/{one.py => classCtor.py} (100%) create mode 100644 src/test/pythonFiles/signature/ellipsis.py create mode 100644 src/test/pythonFiles/signature/noSigPy3.py delete mode 100644 src/test/pythonFiles/signature/three.py diff --git a/src/client/common/utils.ts b/src/client/common/utils.ts index 34cefb118342..25e9a720ca01 100644 --- a/src/client/common/utils.ts +++ b/src/client/common/utils.ts @@ -340,8 +340,8 @@ export function getSubDirectories(rootDir: string): Promise { subDirs.push(fullPath); } } - catch (ex) { - } + // tslint:disable-next-line:no-empty + catch (ex) {} }); resolve(subDirs); }); diff --git a/src/test/pythonFiles/signature/two.py b/src/test/pythonFiles/signature/basicSig.py similarity index 92% rename from src/test/pythonFiles/signature/two.py rename to src/test/pythonFiles/signature/basicSig.py index ae7a551707b8..66ad4cbd0483 100644 --- a/src/test/pythonFiles/signature/two.py +++ b/src/test/pythonFiles/signature/basicSig.py @@ -1 +1,2 @@ range(c, 1, + diff --git a/src/test/pythonFiles/signature/one.py b/src/test/pythonFiles/signature/classCtor.py similarity index 100% rename from src/test/pythonFiles/signature/one.py rename to src/test/pythonFiles/signature/classCtor.py diff --git a/src/test/pythonFiles/signature/ellipsis.py b/src/test/pythonFiles/signature/ellipsis.py new file mode 100644 index 000000000000..c34faa6d231a --- /dev/null +++ b/src/test/pythonFiles/signature/ellipsis.py @@ -0,0 +1 @@ +print(a, b, c) diff --git a/src/test/pythonFiles/signature/noSigPy3.py b/src/test/pythonFiles/signature/noSigPy3.py new file mode 100644 index 000000000000..3d814698b7fe --- /dev/null +++ b/src/test/pythonFiles/signature/noSigPy3.py @@ -0,0 +1 @@ +pow() diff --git a/src/test/pythonFiles/signature/three.py b/src/test/pythonFiles/signature/three.py deleted file mode 100644 index fe666b9ff4c8..000000000000 --- a/src/test/pythonFiles/signature/three.py +++ /dev/null @@ -1 +0,0 @@ -print(a, b, z) diff --git a/src/test/signature/signature.test.ts b/src/test/signature/signature.test.ts index 5fa42aee2ea0..b42ecd115b16 100644 --- a/src/test/signature/signature.test.ts +++ b/src/test/signature/signature.test.ts @@ -5,12 +5,12 @@ import * as assert from 'assert'; import * as path from 'path'; import * as vscode from 'vscode'; +import { PythonSettings } from '../../client/common/configSettings'; +import { execPythonFile } from '../../client/common/utils'; +import { rootWorkspaceUri } from '../common'; import { closeActiveWindows, initialize, initializeTest } from '../initialize'; const autoCompPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'signature'); -const fileOne = path.join(autoCompPath, 'one.py'); -const fileTwo = path.join(autoCompPath, 'two.py'); -const fileThree = path.join(autoCompPath, 'three.py'); class SignatureHelpResult { constructor( @@ -23,8 +23,11 @@ class SignatureHelpResult { // tslint:disable-next-line:max-func-body-length suite('Signatures', () => { + let isPython3: Promise; suiteSetup(async () => { await initialize(); + const version = await execPythonFile(rootWorkspaceUri, PythonSettings.getInstance(rootWorkspaceUri).pythonPath, ['--version'], __dirname, true); + isPython3 = Promise.resolve(version.indexOf('3.') >= 0); }); setup(initializeTest); suiteTeardown(closeActiveWindows); @@ -44,7 +47,7 @@ suite('Signatures', () => { new SignatureHelpResult(5, 20, 0, 0, null) ]; - const document = await openDocument(fileOne); + const document = await openDocument(path.join(autoCompPath, 'classCtor.py')); for (let i = 0; i < expected.length; i += 1) { await checkSignature(expected[i], document!.uri, i); } @@ -67,15 +70,17 @@ suite('Signatures', () => { new SignatureHelpResult(1, 0, 1, 2, 'step') ]; - const document = await openDocument(fileTwo); + const document = await openDocument(path.join(autoCompPath, 'basicSig.py')); for (let i = 0; i < expected.length; i += 1) { await checkSignature(expected[i], document!.uri, i); } }); test('For ellipsis', async () => { + if (!await isPython3) { + return; + } const expected = [ - new SignatureHelpResult(0, 4, 0, 0, null), new SignatureHelpResult(0, 5, 0, 0, null), new SignatureHelpResult(0, 6, 1, 0, 'value'), new SignatureHelpResult(0, 7, 1, 0, 'value'), @@ -86,11 +91,23 @@ suite('Signatures', () => { new SignatureHelpResult(0, 12, 1, 2, 'sep') ]; - const document = await openDocument(fileThree); + const document = await openDocument(path.join(autoCompPath, 'ellipsis.py')); for (let i = 0; i < expected.length; i += 1) { await checkSignature(expected[i], document!.uri, i); } }); + + test('For pow', async () => { + let expected: SignatureHelpResult; + if (await isPython3) { + expected = new SignatureHelpResult(0, 4, 1, 0, null); + } else { + expected = new SignatureHelpResult(0, 4, 1, 0, 'x'); + } + + const document = await openDocument(path.join(autoCompPath, 'noSigPy3.py')); + await checkSignature(expected, document!.uri, 0); + }); }); async function openDocument(documentPath: string): Promise { @@ -105,7 +122,9 @@ async function checkSignature(expected: SignatureHelpResult, uri: vscode.Uri, ca assert.equal(actual!.signatures.length, expected.signaturesCount, `Signature count does not match, case ${caseIndex}`); if (expected.signaturesCount > 0) { assert.equal(actual!.activeParameter, expected.activeParameter, `Parameter index does not match, case ${caseIndex}`); - const parameter = actual!.signatures[0].parameters[expected.activeParameter]; - assert.equal(parameter.label, expected.parameterName, `Parameter name is incorrect, case ${caseIndex}`); + if (expected.parameterName) { + const parameter = actual!.signatures[0].parameters[expected.activeParameter]; + assert.equal(parameter.label, expected.parameterName, `Parameter name is incorrect, case ${caseIndex}`); + } } } From 9cb43e77e6e947f0d8ea951ddd19d6aa80b43fdc Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 9 Jan 2018 14:50:08 -0800 Subject: [PATCH 31/49] #385 Auto-Indentation doesn't work after comment --- src/client/extension.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/extension.ts b/src/client/extension.ts index 4dfb091988c4..710141b55f96 100644 --- a/src/client/extension.ts +++ b/src/client/extension.ts @@ -121,7 +121,7 @@ export async function activate(context: vscode.ExtensionContext) { vscode.languages.setLanguageConfiguration(PYTHON.language!, { onEnterRules: [ { - beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async).*?:\s*$/, + beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async).*$/, action: { indentAction: vscode.IndentAction.Indent } }, { From 5a9c3fd6a56dc94d47bb83328b7bb439ae7b9096 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 9 Jan 2018 15:41:46 -0800 Subject: [PATCH 32/49] #141 Auto indentation broken when return keyword involved --- src/client/extension.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/extension.ts b/src/client/extension.ts index 710141b55f96..b84ef1cbca0e 100644 --- a/src/client/extension.ts +++ b/src/client/extension.ts @@ -121,16 +121,17 @@ export async function activate(context: vscode.ExtensionContext) { vscode.languages.setLanguageConfiguration(PYTHON.language!, { onEnterRules: [ { - beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async).*$/, + beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*/, action: { indentAction: vscode.IndentAction.Indent } }, { - beforeText: /^ *#.*$/, + beforeText: /^\s*#.*/, afterText: /.+$/, action: { indentAction: vscode.IndentAction.None, appendText: '# ' } }, { - beforeText: /^\s+(continue|break|return)\b.*$/, + beforeText: /^\s+(continue|break|return)\b.*/, + afterText: /\s+$/, action: { indentAction: vscode.IndentAction.Outdent } } ] From 9800c4a7f5b3b169273450634d01c89784c7ba01 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 9 Jan 2018 15:43:33 -0800 Subject: [PATCH 33/49] Undo changes --- src/test/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/index.ts b/src/test/index.ts index 6f3209100654..234d1046c161 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -18,8 +18,7 @@ const options: MochaSetupOptions & { retries: number } = { ui: 'tdd', useColors: true, timeout: 25000, - retries: 3, - grep: 'Signatures' + retries: 3 }; testRunner.configure(options, { coverageConfig: '../coverconfig.json' }); module.exports = testRunner; From 30519c7d283d0ad00614324c2f28e8efd968c921 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 5 Feb 2018 11:58:17 -0800 Subject: [PATCH 34/49] #627 Docstrings for builtin methods are not parsed correctly --- src/client/providers/itemInfoSource.ts | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 3cb471959bac..427f36506902 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -118,7 +118,8 @@ export class ItemInfoSource { const descriptionWithHighlightedCode = this.highlightCode(lines.join(EOL)); const tooltip = new vscode.MarkdownString(['```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); - infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(dnd[1]))); + const documentation = this.escapeMarkdown(dnd[1]); + infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); // Sometimes we have duplicate documentation, one with a period at the end. @@ -137,7 +138,8 @@ export class ItemInfoSource { const lines = item.description.split(EOL); const dd = this.getDetailAndDescription(item, lines); - infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(dd[1]))); + const documentation = this.escapeMarkdown(dd[1]); + infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); // Sometimes we have duplicate documentation, one with a period at the end. @@ -226,4 +228,22 @@ export class ItemInfoSource { docstring = docstring.replace(/\r?\n[\+=]+\r?\n/g, s => s.replace(/\+/g, '|').replace(/=/g, '-')); return docstring.trim(); } + + private escapeMarkdown(text: string): string { + return text + .replace(/\\/g, '\\\\') + .replace(/\*/g, '\\*') + .replace(/\_/g, '\\_') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\#/g, '\\#') + .replace(/\+/g, '\\+') + .replace(/\-/g, '\\-') + .replace(/\./g, '\\.') + .replace(/\!/g, '\\!'); + } } From 96511cb1ba99e7a73161103d277138c45d9ed1cd Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 5 Feb 2018 15:49:30 -0800 Subject: [PATCH 35/49] reStructuredText converter --- .../common/markdown/restTextConverter.ts | 108 +++++++++++++++ src/client/providers/itemInfoSource.ts | 131 +++++------------- 2 files changed, 142 insertions(+), 97 deletions(-) create mode 100644 src/client/common/markdown/restTextConverter.ts diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts new file mode 100644 index 000000000000..fe4a71384aa1 --- /dev/null +++ b/src/client/common/markdown/restTextConverter.ts @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { EOL } from 'os'; +import { isWhiteSpace } from '../../language/characters'; + +export class RestTextConverter { + // tslint:disable-next-line:cyclomatic-complexity + public toMarkdown(docstring: string): string { + // This method uses several regexs to 'translate' reStructruredText + // (Python doc syntax) to Markdown syntax. + + // Determine if this is actually a reStructruredText + if (docstring.indexOf('::') < 0 && docstring.indexOf('..')) { + // If documentation contains markdown symbols such as ** (power of) in code, escape them. + return this.escapeMarkdown(docstring); + } + + const md: string[] = []; + let inCodeBlock = false; + + const lines = docstring.split(/\r?\n/); + for (let i = 0; i < lines.length; i += 1) { + let line = lines[i]; + + if (inCodeBlock) { + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { + md.push('```'); + inCodeBlock = false; + } + } + + if (line.startsWith('```')) { + md.push(line); + inCodeBlock = true; + continue; + } + + if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { + // Section title -> heading level 3 + md.push(`### ${line}`); + i += 1; + continue; + } + + if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { + md.push(`#### ${line}`); + i += 1; + continue; + } + + if (line.startsWith('..') && line.indexOf('::') >= 0) { + continue; + } + if (line.indexOf('generated/') >= 0) { + continue; + } + if (line.startsWith('===') || line.startsWith('---')) { + continue; + } + + if (line.endsWith('::')) { + // Literal blocks: begin with `::` + if (line.length > 2) { + md.push(line.substring(0, line.length - 1)); + } + md.push('```'); + inCodeBlock = true; + continue; + } + + line = line.replace(/``/g, '`'); + if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { + line = ` ${line} `; // Keep hard line breaks for the indented content + } + + if (md.length > 0 && (md[md.length - 1].length === 0 || md[md.length - 1] === '```') && line.length === 0) { + continue; // Avoid consequent empty lines + } + + md.push(line); + } + + if (inCodeBlock) { + md.push('```'); + } + return md.join(EOL).trim(); + } + + public escapeMarkdown(text: string): string { + // Not complete escape list so it does not interfere + // with subsequent code highlighting (see above). + return text + .replace(/\\/g, '\\\\') + .replace(/\*/g, '\\*') + .replace(/\_/g, '\\_') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\#/g, '\\#') + .replace(/\+/g, '\\+') + .replace(/\-/g, '\\-') + .replace(/\!/g, '\\!'); + } +} diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 427f36506902..f6d14f190cfd 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -4,6 +4,7 @@ import { EOL } from 'os'; import * as vscode from 'vscode'; +import { RestTextConverter } from '../common/markdown/restTextConverter'; import { JediFactory } from '../languageServices/jediProxyFactory'; import * as proxy from './jediProxy'; import { IHoverItem } from './jediProxy'; @@ -16,6 +17,7 @@ export class LanguageItemInfo { } export class ItemInfoSource { + private textConverter = new RestTextConverter(); constructor(private jediFactory: JediFactory) { } public async getItemInfoFromText(documentUri: vscode.Uri, fileName: string, range: vscode.Range, sourceText: string, token: vscode.CancellationToken) @@ -84,22 +86,8 @@ export class ItemInfoSource { const capturedInfo: string[] = []; data.items.forEach(item => { - let { signature } = item; - switch (item.kind) { - case vscode.SymbolKind.Constructor: - case vscode.SymbolKind.Function: - case vscode.SymbolKind.Method: { - signature = `def ${signature}`; - break; - } - case vscode.SymbolKind.Class: { - signature = `class ${signature}`; - break; - } - default: { - signature = typeof item.text === 'string' && item.text.length > 0 ? item.text : currentWord; - } - } + const signature = this.getSignature(item, currentWord); + let tooltip = new vscode.MarkdownString(); if (item.docstring) { let lines = item.docstring.split(/\r?\n/); const dnd = this.getDetailAndDescription(item, lines); @@ -116,9 +104,12 @@ export class ItemInfoSource { lines.shift(); } - const descriptionWithHighlightedCode = this.highlightCode(lines.join(EOL)); - const tooltip = new vscode.MarkdownString(['```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); - const documentation = this.escapeMarkdown(dnd[1]); + // Tooltip is only used in hover + tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + const description = this.textConverter.toMarkdown(lines.join(EOL)); + tooltip = tooltip.appendMarkdown(description); + + const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); @@ -132,13 +123,13 @@ export class ItemInfoSource { } if (item.description) { - const descriptionWithHighlightedCode = this.highlightCode(item.description); - // tslint:disable-next-line:prefer-template - const tooltip = new vscode.MarkdownString('```python' + `${EOL}${signature}${EOL}` + '```' + `${EOL}${descriptionWithHighlightedCode}`); + tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + const description = this.textConverter.toMarkdown(item.description); + tooltip.appendMarkdown(description); const lines = item.description.split(EOL); const dd = this.getDetailAndDescription(item, lines); - const documentation = this.escapeMarkdown(dd[1]); + const documentation = this.textConverter.escapeMarkdown(dd[1]); infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); @@ -159,7 +150,7 @@ export class ItemInfoSource { let detail: string; let description: string; - if (item.signature && item.signature.length > 0) { + if (item.signature && item.signature.length > 0 && lines.length > 0 && lines[0].indexOf(item.signature) >= 0) { detail = lines.length > 0 ? lines[0] : ''; description = lines.filter((line, index) => index > 0).join(EOL).trim(); } else { @@ -169,81 +160,27 @@ export class ItemInfoSource { return [detail, description]; } - private highlightCode(docstring: string): string { - /********** - * - * Magic. Do not touch. [What is the best comment in source code](https://stackoverflow.com/a/185106) - * - * This method uses several regexs to 'translate' reStructruedText syntax (Python doc syntax) to Markdown syntax. - * - * Let's just keep it unchanged unless a better solution becomes possible. - * - **********/ - // Add 2 line break before and after docstring (used to match a blank line) - docstring = EOL + EOL + docstring.trim() + EOL + EOL; - // Section title -> heading level 2 - docstring = docstring.replace(/(.+\r?\n)[-=]+\r?\n/g, `## $1${EOL}`); - // Directives: '.. directive::' -> '**directive**' - docstring = docstring.replace(/\.\. (.*)::/g, '**$1**'); - // Pattern of 'var : description' - const paramLinePattern = '[\\*\\w_]+ ?:[^:\r\n]+'; - // Add new line after and before param line - docstring = docstring.replace(new RegExp(`(${EOL + paramLinePattern})`, 'g'), `$1${EOL}`); - docstring = docstring.replace(new RegExp(`(${EOL + paramLinePattern + EOL})`, 'g'), `${EOL}$1`); - // 'var : description' -> '`var` description' - docstring = docstring.replace(/\r?\n([\*\w]+) ?: ?([^:\r\n]+\r?\n)/g, `${EOL}\`$1\` $2`); - // Doctest blocks: begin with `>>>` and end with blank line - // tslint:disable-next-line:prefer-template - docstring = docstring.replace(/(>>>[\w\W]+?\r?\n)\r?\n/g, `${'```python' + EOL}$1${'```' + EOL + EOL}`); - // Literal blocks: begin with `::` (literal blocks are indented or quoted; for simplicity, we end literal blocks with blank line) - // tslint:disable-next-line:prefer-template - docstring = docstring.replace(/(\r?\n[^\.]*)::\r?\n\r?\n([\w\W]+?\r?\n)\r?\n/g, `$1${EOL + '```' + EOL}$2${'```' + EOL + EOL}`); - // Remove indentation in Field lists and Literal blocks - let inCodeBlock = false; - let codeIndentation = 0; - const lines = docstring.split(/\r?\n/); - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i]; - if (line.startsWith('```')) { - inCodeBlock = !inCodeBlock; - if (inCodeBlock) { - const match = lines[i + 1].match(/^ */); - codeIndentation = match && match.length > 0 ? match[0].length : 0; - } - continue; + private getSignature(item: proxy.IHoverItem, currentWord: string): string { + let { signature } = item; + switch (item.kind) { + case vscode.SymbolKind.Constructor: + case vscode.SymbolKind.Function: + case vscode.SymbolKind.Method: { + signature = `def ${signature}`; + break; } - if (!inCodeBlock) { - lines[i] = line.replace(/^ {4,8}/, ''); - // Field lists: ':field:' -> '**field**' - lines[i] = lines[i].replace(/:(.+?):/g, '**$1** '); - } else { - if (codeIndentation !== 0) { - lines[i] = line.substring(codeIndentation); - } + case vscode.SymbolKind.Class: { + signature = `class ${signature}`; + break; + } + case vscode.SymbolKind.Module: { + signature = `module ${signature}`; + break; + } + default: { + signature = typeof item.text === 'string' && item.text.length > 0 ? item.text : currentWord; } } - docstring = lines.join(EOL); - // Grid Tables - docstring = docstring.replace(/\r?\n[\+-]+\r?\n/g, EOL); - docstring = docstring.replace(/\r?\n[\+=]+\r?\n/g, s => s.replace(/\+/g, '|').replace(/=/g, '-')); - return docstring.trim(); - } - - private escapeMarkdown(text: string): string { - return text - .replace(/\\/g, '\\\\') - .replace(/\*/g, '\\*') - .replace(/\_/g, '\\_') - .replace(/\{/g, '\\{') - .replace(/\}/g, '\\}') - .replace(/\[/g, '\\[') - .replace(/\]/g, '\\]') - .replace(/\(/g, '\\(') - .replace(/\)/g, '\\)') - .replace(/\#/g, '\\#') - .replace(/\+/g, '\\+') - .replace(/\-/g, '\\-') - .replace(/\./g, '\\.') - .replace(/\!/g, '\\!'); + return signature; } } From c8670b9083413530afb32ff6e229af88b122f73b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 5 Feb 2018 15:57:24 -0800 Subject: [PATCH 36/49] Fix: period is not an operator --- src/client/language/tokenizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index b84a11b62fd1..0a2160fc15c5 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -300,7 +300,7 @@ export class Tokenizer implements ITokenizer { break; default: - break; + return false; } this.tokens.push(new Token(TokenType.Operator, this.cs.position, length)); this.cs.advance(length); From 97f232f1eb433f78d9036951531b590f7b59304c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 5 Feb 2018 16:14:07 -0800 Subject: [PATCH 37/49] Minor fixes --- .../common/markdown/restTextConverter.ts | 28 ++++++++++++------- src/client/providers/itemInfoSource.ts | 12 ++++++-- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index fe4a71384aa1..eccd60e60edc 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -8,7 +8,9 @@ export class RestTextConverter { // tslint:disable-next-line:cyclomatic-complexity public toMarkdown(docstring: string): string { // This method uses several regexs to 'translate' reStructruredText - // (Python doc syntax) to Markdown syntax. + // https://en.wikipedia.org/wiki/ReStructuredText + // (Python doc syntax) to Markdown syntax. It only translates + // as much as needed to display tooltips in intellisense. // Determine if this is actually a reStructruredText if (docstring.indexOf('::') < 0 && docstring.indexOf('..')) { @@ -24,6 +26,9 @@ export class RestTextConverter { let line = lines[i]; if (inCodeBlock) { + // Pseudo-code block terminates by a line without leading + // whitespace. Pseudo-code blocks are used to preserve + // pre-formatted text. if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { md.push('```'); inCodeBlock = false; @@ -32,35 +37,37 @@ export class RestTextConverter { if (line.startsWith('```')) { md.push(line); - inCodeBlock = true; + inCodeBlock = !inCodeBlock; continue; } if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { // Section title -> heading level 3 md.push(`### ${line}`); - i += 1; + i += 1; // Eat line with === continue; } if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { + // Subsection title -> heading level 4 md.push(`#### ${line}`); - i += 1; + i += 1; // Eat line with --- continue; } if (line.startsWith('..') && line.indexOf('::') >= 0) { - continue; + continue; // Ignore assorted tags likes .. seealso:: } if (line.indexOf('generated/') >= 0) { - continue; + continue; // ignore generated content } if (line.startsWith('===') || line.startsWith('---')) { continue; } if (line.endsWith('::')) { - // Literal blocks: begin with `::` + // Literal blocks begin with `::`. Such as sequence like + // '... as shown below::' that is followed by a preformatted text. if (line.length > 2) { md.push(line.substring(0, line.length - 1)); } @@ -69,13 +76,14 @@ export class RestTextConverter { continue; } - line = line.replace(/``/g, '`'); + line = line.replace(/``/g, '`'); // Convert double backticks to single if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - line = ` ${line} `; // Keep hard line breaks for the indented content + // Keep hard line breaks for the pre-indented content + line = ` ${line} `; } if (md.length > 0 && (md[md.length - 1].length === 0 || md[md.length - 1] === '```') && line.length === 0) { - continue; // Avoid consequent empty lines + continue; // Avoid more than one empty line in a row } md.push(line); diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index f6d14f190cfd..db147dbfd431 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -105,7 +105,9 @@ export class ItemInfoSource { } // Tooltip is only used in hover - tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + if (signature.length > 0) { + tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + } const description = this.textConverter.toMarkdown(lines.join(EOL)); tooltip = tooltip.appendMarkdown(description); @@ -123,7 +125,9 @@ export class ItemInfoSource { } if (item.description) { - tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + if (signature.length > 0) { + tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + } const description = this.textConverter.toMarkdown(item.description); tooltip.appendMarkdown(description); @@ -174,7 +178,9 @@ export class ItemInfoSource { break; } case vscode.SymbolKind.Module: { - signature = `module ${signature}`; + if (signature.length > 0) { + signature = `module ${signature}`; + } break; } default: { From 768bffe5b56b8a24ac8c07c52b8eba819af4a61f Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 12:24:22 -0800 Subject: [PATCH 38/49] Restructure --- .../common/markdown/restTextConverter.ts | 218 ++++++++++++------ 1 file changed, 151 insertions(+), 67 deletions(-) diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index eccd60e60edc..bbb11347775a 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -5,112 +5,196 @@ import { EOL } from 'os'; import { isWhiteSpace } from '../../language/characters'; export class RestTextConverter { + private inPreBlock = false; + private inCodeBlock = false; + private md: string[] = []; + // tslint:disable-next-line:cyclomatic-complexity public toMarkdown(docstring: string): string { - // This method uses several regexs to 'translate' reStructruredText - // https://en.wikipedia.org/wiki/ReStructuredText - // (Python doc syntax) to Markdown syntax. It only translates - // as much as needed to display tooltips in intellisense. + // Translates reStructruredText (Python doc syntax) to markdown. + // It only translates as much as needed to display tooltips + // and documentation in the completion list. + // See https://en.wikipedia.org/wiki/ReStructuredText - // Determine if this is actually a reStructruredText + // Determine if this is actually a reStructruredText. if (docstring.indexOf('::') < 0 && docstring.indexOf('..')) { // If documentation contains markdown symbols such as ** (power of) in code, escape them. return this.escapeMarkdown(docstring); } + const result = this.transformLines(docstring); + + this.inPreBlock = this.inPreBlock = false; + this.md = []; - const md: string[] = []; - let inCodeBlock = false; + return result; + } + public escapeMarkdown(text: string): string { + // Not complete escape list so it does not interfere + // with subsequent code highlighting (see above). + return text + .replace(/\\/g, '\\\\') + .replace(/\*/g, '\\*') + .replace(/\_/g, '\\_') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/\#/g, '\\#') + .replace(/\+/g, '\\+') + .replace(/\-/g, '\\-') + .replace(/\!/g, '\\!'); + } + + private transformLines(docstring: string): string { const lines = docstring.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { let line = lines[i]; - if (inCodeBlock) { - // Pseudo-code block terminates by a line without leading - // whitespace. Pseudo-code blocks are used to preserve - // pre-formatted text. - if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { - md.push('```'); - inCodeBlock = false; - } - } - - if (line.startsWith('```')) { - md.push(line); - inCodeBlock = !inCodeBlock; + if (this.handleCodeBlock(line)) { continue; } - if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { - // Section title -> heading level 3 - md.push(`### ${line}`); - i += 1; // Eat line with === - continue; + if (this.inPreBlock) { + // Preformatted block terminates by a line without leading + // whitespace or any special line like ..ABC::. + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { + this.endPreformattedBlock(); + } } - if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { - // Subsection title -> heading level 4 - md.push(`#### ${line}`); - i += 1; // Eat line with --- + if (this.handleSectionHeader(lines, i)) { + i += 1; // Eat line with === or --- continue; } - if (line.startsWith('..') && line.indexOf('::') >= 0) { - continue; // Ignore assorted tags likes .. seealso:: - } if (line.indexOf('generated/') >= 0) { - continue; // ignore generated content + continue; // ignore generated content. } if (line.startsWith('===') || line.startsWith('---')) { - continue; + continue; // Eat standalone === or --- lines. } - if (line.endsWith('::')) { - // Literal blocks begin with `::`. Such as sequence like - // '... as shown below::' that is followed by a preformatted text. - if (line.length > 2) { - md.push(line.substring(0, line.length - 1)); - } - md.push('```'); - inCodeBlock = true; + if (this.handleDoubleColon(line)) { + continue; + } + if (line.startsWith('..') && line.indexOf('::') > 0) { + // Ignore lines likes .. sectionauthor:: John Doe. continue; } - line = line.replace(/``/g, '`'); // Convert double backticks to single + line = this.convertEmphasis(line); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - // Keep hard line breaks for the pre-indented content + // Keep hard line breaks for the pre-indented content. line = ` ${line} `; } - if (md.length > 0 && (md[md.length - 1].length === 0 || md[md.length - 1] === '```') && line.length === 0) { - continue; // Avoid more than one empty line in a row + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + continue; // Avoid more than one empty line in a row. } - md.push(line); + this.md.push(line); } - if (inCodeBlock) { - md.push('```'); + this.tryEndCodePreBlocks(); + return this.md.join(EOL).trim(); + } + + private handleCodeBlock(line: string): boolean { + if (!line.startsWith('```')) { + return false; + } + if (this.inCodeBlock) { + this.endCodeBlock(); + } else { + this.startCodeBlock(); } - return md.join(EOL).trim(); + return true; } - public escapeMarkdown(text: string): string { - // Not complete escape list so it does not interfere - // with subsequent code highlighting (see above). - return text - .replace(/\\/g, '\\\\') - .replace(/\*/g, '\\*') - .replace(/\_/g, '\\_') - .replace(/\{/g, '\\{') - .replace(/\}/g, '\\}') - .replace(/\[/g, '\\[') - .replace(/\]/g, '\\]') - .replace(/\(/g, '\\(') - .replace(/\)/g, '\\)') - .replace(/\#/g, '\\#') - .replace(/\+/g, '\\+') - .replace(/\-/g, '\\-') - .replace(/\!/g, '\\!'); + private handleSectionHeader(lines: string[], i: number): boolean { + const line = lines[i]; + if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { + // Section title -> heading level 3. + this.md.push(`### ${this.convertEmphasis(line)}`); + return true; + } + if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { + // Subsection title -> heading level 4. + this.md.push(`#### ${this.convertEmphasis(line)}`); + return true; + } + return false; + } + + private handleDoubleColon(line: string): boolean { + if (!line.endsWith('::')) { + return false; + } + // Literal blocks begin with `::`. Such as sequence like + // '... as shown below::' that is followed by a preformatted text. + if (line.length > 2 && !line.startsWith('..')) { + // Ignore lines likes .. autosummary:: John Doe. + // Trim trailing : so :: turns into :. + this.md.push(line.substring(0, line.length - 1)); + } + + this.startPreformattedBlock(); + return true; + } + + private tryEndCodePreBlocks(): void { + if (this.inCodeBlock) { + this.endCodeBlock(); + } + if (this.inPreBlock) { + this.endPreformattedBlock(); + } + } + + private startPreformattedBlock(): void { + // Remove previous empty line so we avoid double empties. + this.tryRemovePrecedingEmptyLine(); + // Lie about the language since we don't want preformatted text + // to be colorized as Python. HTML is more 'appropriate' as it does + // not colorize -- or + or keywords like 'from'. + this.md.push('```html'); + this.inPreBlock = true; + } + + private endPreformattedBlock(): void { + if (this.inPreBlock) { + this.md.push('```'); + this.inPreBlock = false; + } + } + + private startCodeBlock(): void { + // Remove previous empty line so we avoid double empties. + this.tryRemovePrecedingEmptyLine(); + this.md.push('```python'); + this.inCodeBlock = true; + } + + private endCodeBlock(): void { + if (this.inCodeBlock) { + this.md.push('```'); + this.inCodeBlock = false; + } + } + + private tryRemovePrecedingEmptyLine(): void { + if (this.md.length > 0 && this.md[this.md.length - 1].length === 0) { + this.md.pop(); + } + } + + private convertEmphasis(line: string): string { + return line.replace(/\:([\w\W]+)\:/g, '**$1**'); // Convert :word: to **word**. } } From 825f16b64d01b9342d7109da627968ccd034ca91 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 13:03:04 -0800 Subject: [PATCH 39/49] Tests --- .../common/markdown/restTextConverter.ts | 6 ++ src/test/markdown/restTextConverter.test.ts | 40 +++++++++++ .../markdown/scipy.spatial.distance.md | 58 +++++++++++++++ .../markdown/scipy.spatial.distance.pydoc | 71 +++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 src/test/markdown/restTextConverter.test.ts create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.distance.md create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index bbb11347775a..485801a35f49 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -48,11 +48,17 @@ export class RestTextConverter { .replace(/\!/g, '\\!'); } + // tslint:disable-next-line:cyclomatic-complexity private transformLines(docstring: string): string { const lines = docstring.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { let line = lines[i]; + // Avoid leading empty lines + if (this.md.length === 0 && line.length === 0) { + continue; + } + if (this.handleCodeBlock(line)) { continue; } diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts new file mode 100644 index 000000000000..52cc7ea7c497 --- /dev/null +++ b/src/test/markdown/restTextConverter.test.ts @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { expect } from 'chai'; +import * as fs from 'fs-extra'; +import * as path from 'path'; +import { RestTextConverter } from '../../client/common/markdown/restTextConverter'; + +const srcPythoFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'markdown'); + +function compareFiles(expectedContent: string, actualContent: string) { + const expectedLines = expectedContent.split(/\r?\n/); + const actualLines = actualContent.split(/\r?\n/); + + for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { + const e = expectedLines[i]; + const a = actualLines[i]; + expect(a, `Difference at line ${i}`).to.be.equal(e); + } + + expect(actualLines.length, + expectedLines.length > actualLines.length + ? 'Actual contains more lines than expected' + : 'Expected contains more lines than the actual' + ).to.be.equal(expectedLines.length); +} + +async function testConversion(fileName: string): Promise { + const cvt = new RestTextConverter(); + const file = path.join(srcPythoFilesPath, fileName); + const source = await fs.readFile(`${file}.pydoc`, 'utf8'); + const actual = cvt.toMarkdown(source); + const expected = await fs.readFile(`${file}.md`, 'utf8'); + compareFiles(expected, actual); +} + +// tslint:disable-next-line:max-func-body-length +suite('Hover - RestTextConverter', () => { + test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); +}); diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md new file mode 100644 index 000000000000..125b19f6cdeb --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -0,0 +1,58 @@ +### Distance computations (**mod**`scipy.spatial.distance`) + + +#### Function Reference + +Distance matrix computation from a collection of raw observation vectors +stored in a rectangular array. +```html + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays + +``` +Predicates for checking the validity of distance matrices, both +condensed and redundant. Also contained in this module are functions +for computing the number of observations in a distance matrix. +```html + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix + +``` +Distance functions between two numeric vectors `u` and `v`. Computing +distances over a large collection of vectors is inefficient for these +functions. Use `pdist` for this purpose. +```html + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. + +``` +Distance functions between two boolean vectors (representing sets) `u` and +`v`. As in the case of numerical vectors, `pdist` is more efficient for +computing the distances between all pairs. +```html + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. + +``` +**func**`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc b/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc new file mode 100644 index 000000000000..cfc9b7008b99 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc @@ -0,0 +1,71 @@ + +===================================================== +Distance computations (:mod:`scipy.spatial.distance`) +===================================================== + +.. sectionauthor:: Damian Eads + +Function Reference +------------------ + +Distance matrix computation from a collection of raw observation vectors +stored in a rectangular array. + +.. autosummary:: + :toctree: generated/ + + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays + +Predicates for checking the validity of distance matrices, both +condensed and redundant. Also contained in this module are functions +for computing the number of observations in a distance matrix. + +.. autosummary:: + :toctree: generated/ + + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix + +Distance functions between two numeric vectors ``u`` and ``v``. Computing +distances over a large collection of vectors is inefficient for these +functions. Use ``pdist`` for this purpose. + +.. autosummary:: + :toctree: generated/ + + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. + +Distance functions between two boolean vectors (representing sets) ``u`` and +``v``. As in the case of numerical vectors, ``pdist`` is more efficient for +computing the distances between all pairs. + +.. autosummary:: + :toctree: generated/ + + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. + +:func:`hamming` also operates over discrete numerical vectors. From eb36eefde01044e3c6cb88f66ceb935edd244d09 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 13:11:21 -0800 Subject: [PATCH 40/49] Tests --- src/test/markdown/restTextConverter.test.ts | 2 + src/test/pythonFiles/markdown/scipy.md | 48 +++++++++++ src/test/pythonFiles/markdown/scipy.pydoc | 53 ++++++++++++ .../pythonFiles/markdown/scipy.spatial.md | 70 +++++++++++++++ .../pythonFiles/markdown/scipy.spatial.pydoc | 86 +++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 src/test/pythonFiles/markdown/scipy.md create mode 100644 src/test/pythonFiles/markdown/scipy.pydoc create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.md create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.pydoc diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 52cc7ea7c497..ac649b912081 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -36,5 +36,7 @@ async function testConversion(fileName: string): Promise { // tslint:disable-next-line:max-func-body-length suite('Hover - RestTextConverter', () => { + test('scipy', async () => await testConversion('scipy')); + test('scipy.spatial', async () => await testConversion('scipy.spatial')); test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); }); diff --git a/src/test/pythonFiles/markdown/scipy.md b/src/test/pythonFiles/markdown/scipy.md new file mode 100644 index 000000000000..23721797aae3 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.md @@ -0,0 +1,48 @@ +### SciPy: A scientific computing package for Python + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +#### Contents +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +#### Subpackages +Using any of these subpackages requires an explicit import. For example, +`import scipy.cluster`. +```html + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- **Interface to the UMFPACK library** + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions + +``` +#### Utility tools +```html + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.pydoc b/src/test/pythonFiles/markdown/scipy.pydoc new file mode 100644 index 000000000000..293445fbea5b --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.pydoc @@ -0,0 +1,53 @@ +SciPy: A scientific computing package for Python +================================================ + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +Contents +-------- +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +Subpackages +----------- +Using any of these subpackages requires an explicit import. For example, +``import scipy.cluster``. + +:: + + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions + +Utility tools +------------- +:: + + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md new file mode 100644 index 000000000000..8b6c7d7d5c51 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -0,0 +1,70 @@ +### Spatial algorithms and data structures (**mod**`scipy.spatial`) + + +### Nearest-neighbor Queries +```html + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle + +``` +### Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +```html + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces + +``` +### Plotting Helpers +```html + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram + +``` +### Simplex representation +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme: +```html + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells + +``` +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: + + `tess.neighbors[i,j]` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. + +Convex hull facets also define a hyperplane equation: +```html + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 + +``` +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +#### Functions +```html + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.pydoc b/src/test/pythonFiles/markdown/scipy.spatial.pydoc new file mode 100644 index 000000000000..1613b94384b7 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.pydoc @@ -0,0 +1,86 @@ +============================================================= +Spatial algorithms and data structures (:mod:`scipy.spatial`) +============================================================= + +.. currentmodule:: scipy.spatial + +Nearest-neighbor Queries +======================== +.. autosummary:: + :toctree: generated/ + + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle + +Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +========================================================= + +.. autosummary:: + :toctree: generated/ + + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces + +Plotting Helpers +================ + +.. autosummary:: + :toctree: generated/ + + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram + +.. seealso:: :ref:`Tutorial ` + + +Simplex representation +====================== +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme:: + + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells + +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: + + ``tess.neighbors[i,j]`` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. + +Convex hull facets also define a hyperplane equation:: + + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 + +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes \ No newline at end of file From bab423966465f280325d64684cb7277964ab716c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 14:36:38 -0800 Subject: [PATCH 41/49] Code heuristics --- .../common/markdown/restTextConverter.ts | 42 +++++++++++++------ src/client/providers/itemInfoSource.ts | 3 +- src/test/markdown/restTextConverter.test.ts | 1 + src/test/pythonFiles/markdown/anydbm.md | 36 ++++++++++++++++ src/test/pythonFiles/markdown/anydbm.pydoc | 33 +++++++++++++++ 5 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 src/test/pythonFiles/markdown/anydbm.md create mode 100644 src/test/pythonFiles/markdown/anydbm.pydoc diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 485801a35f49..104230f273a1 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -10,19 +10,17 @@ export class RestTextConverter { private md: string[] = []; // tslint:disable-next-line:cyclomatic-complexity - public toMarkdown(docstring: string): string { + public toMarkdown(docstring: string, force?: boolean): string { // Translates reStructruredText (Python doc syntax) to markdown. // It only translates as much as needed to display tooltips // and documentation in the completion list. // See https://en.wikipedia.org/wiki/ReStructuredText - // Determine if this is actually a reStructruredText. - if (docstring.indexOf('::') < 0 && docstring.indexOf('..')) { - // If documentation contains markdown symbols such as ** (power of) in code, escape them. + if (!force && !this.shouldConvert(docstring)) { return this.escapeMarkdown(docstring); } - const result = this.transformLines(docstring); + const result = this.transformLines(docstring); this.inPreBlock = this.inPreBlock = false; this.md = []; @@ -33,7 +31,7 @@ export class RestTextConverter { // Not complete escape list so it does not interfere // with subsequent code highlighting (see above). return text - .replace(/\\/g, '\\\\') + .replace(/\#/g, '\\#') .replace(/\*/g, '\\*') .replace(/\_/g, '\\_') .replace(/\{/g, '\\{') @@ -42,10 +40,19 @@ export class RestTextConverter { .replace(/\]/g, '\\]') .replace(/\(/g, '\\(') .replace(/\)/g, '\\)') - .replace(/\#/g, '\\#') .replace(/\+/g, '\\+') - .replace(/\-/g, '\\-') - .replace(/\!/g, '\\!'); + .replace(/\-/g, '\\+'); + } + + private shouldConvert(docstring: string): boolean { + // heuristics + if (docstring.indexOf('::') >= 0 || docstring.indexOf('..') >= 0) { + return true; + } + if (docstring.indexOf('===') >= 0 || docstring.indexOf('---') >= 0) { + return true; + } + return false; } // tslint:disable-next-line:cyclomatic-complexity @@ -59,6 +66,13 @@ export class RestTextConverter { continue; } + if (!this.inPreBlock) { + // Anything indented is considered to be preformatted. + if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { + this.startPreformattedBlock(line); + } + } + if (this.handleCodeBlock(line)) { continue; } @@ -150,7 +164,7 @@ export class RestTextConverter { this.md.push(line.substring(0, line.length - 1)); } - this.startPreformattedBlock(); + this.startPreformattedBlock(line); return true; } @@ -163,13 +177,17 @@ export class RestTextConverter { } } - private startPreformattedBlock(): void { + private startPreformattedBlock(line: string): void { // Remove previous empty line so we avoid double empties. this.tryRemovePrecedingEmptyLine(); // Lie about the language since we don't want preformatted text // to be colorized as Python. HTML is more 'appropriate' as it does // not colorize -- or + or keywords like 'from'. - this.md.push('```html'); + if (line.indexOf('# ') >= 0) { + this.md.push('```python'); + } else { + this.md.push('```html'); + } this.inPreBlock = true; } diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index db147dbfd431..fc02d8097464 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -108,7 +108,8 @@ export class ItemInfoSource { if (signature.length > 0) { tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); } - const description = this.textConverter.toMarkdown(lines.join(EOL)); + + const description = this.textConverter.toMarkdown(lines.join(EOL), signature.length === 0); tooltip = tooltip.appendMarkdown(description); const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index ac649b912081..e40898e9dd6a 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -39,4 +39,5 @@ suite('Hover - RestTextConverter', () => { test('scipy', async () => await testConversion('scipy')); test('scipy.spatial', async () => await testConversion('scipy.spatial')); test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); + test('anydbm', async () => await testConversion('anydbm')); }); diff --git a/src/test/pythonFiles/markdown/anydbm.md b/src/test/pythonFiles/markdown/anydbm.md new file mode 100644 index 000000000000..a86897871374 --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.md @@ -0,0 +1,36 @@ +Generic interface to all dbm clones. + +Instead of +```html + import dbm + d = dbm.open(file, 'w', 0666) + +``` +use +```html + import anydbm + d = anydbm.open(file, 'w') + +``` +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): +```python + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) + +``` +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.pydoc b/src/test/pythonFiles/markdown/anydbm.pydoc new file mode 100644 index 000000000000..2d46b5881789 --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.pydoc @@ -0,0 +1,33 @@ +Generic interface to all dbm clones. + +Instead of + + import dbm + d = dbm.open(file, 'w', 0666) + +use + + import anydbm + d = anydbm.open(file, 'w') + +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): + + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) + +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file From 2a3020157a271ef7de71663d62260e4f898a06bf Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 15:05:03 -0800 Subject: [PATCH 42/49] Baselines --- src/test/language/tokenizer.test.ts | 7 +++++++ src/test/markdown/restTextConverter.test.ts | 2 +- src/test/pythonFiles/markdown/scipy.spatial.md | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 727ce969dd09..86deb9282249 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -76,4 +76,11 @@ suite('Language.Tokenizer', () => { assert.equal(tokens.getItemAt(i).type, TokenType.Comment); } }); + test('Unknown token', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('.'); + assert.equal(tokens.count, 1); + + assert.equal(tokens.getItemAt(0).type, TokenType.Unknown); + }); }); diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index e40898e9dd6a..7b2f9a97cdc8 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -29,7 +29,7 @@ async function testConversion(fileName: string): Promise { const cvt = new RestTextConverter(); const file = path.join(srcPythoFilesPath, fileName); const source = await fs.readFile(`${file}.pydoc`, 'utf8'); - const actual = cvt.toMarkdown(source); + const actual = cvt.toMarkdown(source, true); const expected = await fs.readFile(`${file}.md`, 'utf8'); compareFiles(expected, actual); } diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md index 8b6c7d7d5c51..3584e78f1bbc 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -42,11 +42,12 @@ tesselation (N-dim simplices), convex hull facets, and Voronoi ridges ``` For Delaunay triangulations and convex hulls, the neighborhood structure of the simplices satisfies the condition: - +```html `tess.neighbors[i,j]` is the neighboring simplex of the i-th simplex, opposite to the j-vertex. It is -1 in case of no neighbor. +``` Convex hull facets also define a hyperplane equation: ```html (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 From e430ef820ea5253447a06937f0f35f2b06d85c9e Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 15:42:16 -0800 Subject: [PATCH 43/49] HTML handling --- .../common/markdown/restTextConverter.ts | 30 +++- src/test/markdown/restTextConverter.test.ts | 1 + src/test/pythonFiles/markdown/aifc.md | 144 ++++++++++++++++++ src/test/pythonFiles/markdown/aifc.pydoc | 134 ++++++++++++++++ 4 files changed, 304 insertions(+), 5 deletions(-) create mode 100644 src/test/pythonFiles/markdown/aifc.md create mode 100644 src/test/pythonFiles/markdown/aifc.pydoc diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 104230f273a1..8e2bce47a411 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -2,6 +2,8 @@ // Licensed under the MIT License. import { EOL } from 'os'; +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; import { isWhiteSpace } from '../../language/characters'; export class RestTextConverter { @@ -45,11 +47,19 @@ export class RestTextConverter { } private shouldConvert(docstring: string): boolean { - // heuristics - if (docstring.indexOf('::') >= 0 || docstring.indexOf('..') >= 0) { - return true; + // Heuristics that determe if string should be converted + // to markdown or just escaped. + + // :: at the end of a string + const doubleColon = docstring.indexOf('::'); + if (doubleColon >= 0 && doubleColon < docstring.length - 2) { + const ch = docstring.charCodeAt(doubleColon + 2); + if (ch === Char.LineFeed || ch === Char.CarriageReturn) { + return true; + } } - if (docstring.indexOf('===') >= 0 || docstring.indexOf('---') >= 0) { + // Section headers or lists + if (docstring.indexOf('===') >= 0 || docstring.indexOf('---') >= 0 || docstring.indexOf('.. ') >= 0) { return true; } return false; @@ -118,13 +128,23 @@ export class RestTextConverter { continue; // Avoid more than one empty line in a row. } - this.md.push(line); + this.addLine(line); } this.tryEndCodePreBlocks(); return this.md.join(EOL).trim(); } + private addLine(line: string): void { + // Since we use HTML blocks as preformatted text + // make sure we drop angle brackets since otherwise + // they will render as tags and attributes + if (this.inPreBlock) { + line = line.replace(//g, ''); + } + this.md.push(line); + } + private handleCodeBlock(line: string): boolean { if (!line.startsWith('```')) { return false; diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 7b2f9a97cdc8..b3284f4727e7 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -40,4 +40,5 @@ suite('Hover - RestTextConverter', () => { test('scipy.spatial', async () => await testConversion('scipy.spatial')); test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); test('anydbm', async () => await testConversion('anydbm')); + test('aifc', async () => await testConversion('aifc')); }); diff --git a/src/test/pythonFiles/markdown/aifc.md b/src/test/pythonFiles/markdown/aifc.md new file mode 100644 index 000000000000..a2f120c6a3b2 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.md @@ -0,0 +1,144 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. +```html + +-----------------+ + | FORM | + +-----------------+ + | size | + +----+------------+ + | | AIFC | + | +------------+ + | | chunks | + | | . | + | | . | + | | . | + +----+------------+ + +``` +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. +```html + FVER + version number of AIFF-C defining document (AIFF-C only). + MARK + # of markers (2 bytes) + list of markers: + marker ID (2 bytes, must be 0) + position (4 bytes) + marker name ("pstring") + COMM + # of channels (2 bytes) + # of sound frames (4 bytes) + size of the samples (2 bytes) + sampling frequency (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + compression type (4 bytes) + human-readable version of compression type ("pstring") + SSND + offset (4 bytes, not used by this program) + blocksize (4 bytes, not used by this program) + sound data + +``` +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: +```html + f = aifc.open(file, 'r') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: +```html + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +``` +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: +```html + f = aifc.open(file, 'w') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: +```html + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +``` +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/aifc.pydoc b/src/test/pythonFiles/markdown/aifc.pydoc new file mode 100644 index 000000000000..a4cc346d5531 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.pydoc @@ -0,0 +1,134 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. + + +-----------------+ + | FORM | + +-----------------+ + | | + +----+------------+ + | | AIFC | + | +------------+ + | | | + | | . | + | | . | + | | . | + +----+------------+ + +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. + + FVER + (AIFF-C only). + MARK + <# of markers> (2 bytes) + list of markers: + (2 bytes, must be > 0) + (4 bytes) + ("pstring") + COMM + <# of channels> (2 bytes) + <# of sound frames> (4 bytes) + (2 bytes) + (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + (4 bytes) + ("pstring") + SSND + (4 bytes, not used by this program) + (4 bytes, not used by this program) + + +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: + f = aifc.open(file, 'r') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: + f = aifc.open(file, 'w') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file From 1afa841a7ba08c1041442b26494d0f7b523946e0 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 6 Feb 2018 16:03:22 -0800 Subject: [PATCH 44/49] Lists --- .../common/markdown/restTextConverter.ts | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 8e2bce47a411..447acef789ff 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -4,7 +4,7 @@ import { EOL } from 'os'; // tslint:disable-next-line:import-name import Char from 'typescript-char'; -import { isWhiteSpace } from '../../language/characters'; +import { isDecimal, isWhiteSpace } from '../../language/characters'; export class RestTextConverter { private inPreBlock = false; @@ -76,12 +76,7 @@ export class RestTextConverter { continue; } - if (!this.inPreBlock) { - // Anything indented is considered to be preformatted. - if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - this.startPreformattedBlock(line); - } - } + this.checkPreContent(lines, i); if (this.handleCodeBlock(line)) { continue; @@ -145,6 +140,43 @@ export class RestTextConverter { this.md.push(line); } + private checkPreContent(lines: string[], i: number): void { + if (this.inPreBlock) { + return; + } + // Indented is considered to be preformatted except + // when previous line is indented or begins list item. + const line = lines[i]; + if (line.length === 0 || !isWhiteSpace(line.charCodeAt(0))) { + return; + } + + let prevLine = i > 0 ? lines[i - 1] : undefined; + if (!prevLine) { + return; + } + if (prevLine.length === 0) { + this.startPreformattedBlock(line); + return; + } + if (isWhiteSpace(prevLine.charCodeAt(0))) { + return; + } + + prevLine = prevLine.trim(); + if (prevLine.length === 0) { + this.startPreformattedBlock(line); + return; + } + + const ch = prevLine.charCodeAt(0); + if (ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch)) { + return; + } + + this.startPreformattedBlock(line); + } + private handleCodeBlock(line: string): boolean { if (!line.startsWith('```')) { return false; From 6bffb0797bf7f20197253ce08671d1ef3ad11393 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 7 Feb 2018 13:19:52 -0800 Subject: [PATCH 45/49] State machine --- .../common/markdown/restTextConverter.ts | 236 +++++++++--------- src/test/markdown/restTextConverter.test.ts | 3 +- src/test/pythonFiles/markdown/aifc.md | 154 ++++++------ src/test/pythonFiles/markdown/anydbm.md | 29 +-- src/test/pythonFiles/markdown/astroid.md | 24 ++ src/test/pythonFiles/markdown/astroid.pydoc | 23 ++ src/test/pythonFiles/markdown/scipy.md | 61 +++-- .../markdown/scipy.spatial.distance.md | 62 +++-- .../pythonFiles/markdown/scipy.spatial.md | 64 +++-- 9 files changed, 346 insertions(+), 310 deletions(-) create mode 100644 src/test/pythonFiles/markdown/astroid.md create mode 100644 src/test/pythonFiles/markdown/astroid.pydoc diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 447acef789ff..da00da8e2fe3 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -6,9 +6,14 @@ import { EOL } from 'os'; import Char from 'typescript-char'; import { isDecimal, isWhiteSpace } from '../../language/characters'; +enum State { + Default, + Preformatted, + Code +} + export class RestTextConverter { - private inPreBlock = false; - private inCodeBlock = false; + private state: State = State.Default; private md: string[] = []; // tslint:disable-next-line:cyclomatic-complexity @@ -23,7 +28,7 @@ export class RestTextConverter { } const result = this.transformLines(docstring); - this.inPreBlock = this.inPreBlock = false; + this.state = State.Default; this.md = []; return result; @@ -65,128 +70,132 @@ export class RestTextConverter { return false; } - // tslint:disable-next-line:cyclomatic-complexity private transformLines(docstring: string): string { const lines = docstring.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { - let line = lines[i]; - + const line = lines[i]; // Avoid leading empty lines if (this.md.length === 0 && line.length === 0) { continue; } - this.checkPreContent(lines, i); - - if (this.handleCodeBlock(line)) { - continue; + switch (this.state) { + case State.Default: + i += this.inDefaultState(lines, i); + break; + case State.Preformatted: + i += this.inPreformattedState(lines, i); + break; + case State.Code: + this.inCodeState(line); + break; + default: + break; } + } - if (this.inPreBlock) { - // Preformatted block terminates by a line without leading - // whitespace or any special line like ..ABC::. - if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { - this.endPreformattedBlock(); - } - } + this.endCodeBlock(); + this.endPreformattedBlock(); - if (this.handleSectionHeader(lines, i)) { - i += 1; // Eat line with === or --- - continue; - } + return this.md.join(EOL).trim(); + } - if (line.indexOf('generated/') >= 0) { - continue; // ignore generated content. - } - if (line.startsWith('===') || line.startsWith('---')) { - continue; // Eat standalone === or --- lines. - } + private inDefaultState(lines: string[], i: number): number { + let line = lines[i]; + if (line.startsWith('```')) { + this.startCodeBlock(); + return 0; + } - if (this.handleDoubleColon(line)) { - continue; - } - if (line.startsWith('..') && line.indexOf('::') > 0) { - // Ignore lines likes .. sectionauthor:: John Doe. - continue; - } + if (line.startsWith('===') || line.startsWith('---')) { + return 0; // Eat standalone === or --- lines. + } + if (this.handleDoubleColon(line)) { + return 0; + } + if (this.isIgnorable(line)) { + return 0; + } - line = this.convertEmphasis(line); - line = line.replace(/``/g, '`'); // Convert double backticks to single. + if (this.handleSectionHeader(lines, i)) { + return 1; // Eat line with === or --- + } - if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - // Keep hard line breaks for the pre-indented content. - line = ` ${line} `; - } + const result = this.checkPreContent(lines, i); + if (this.state !== State.Default) { + return result; // Handle line in the new state + } - const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; - if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { - continue; // Avoid more than one empty line in a row. - } + line = this.convertEmphasis(line); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + this.md.push(line); + + return 0; + } - this.addLine(line); + private inPreformattedState(lines: string[], i: number): number { + let line = lines[i]; + if (this.isIgnorable(line)) { + return 0; + } + // Preformatted block terminates by a line without leading whitespace. + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + this.endPreformattedBlock(); + return -1; } - this.tryEndCodePreBlocks(); - return this.md.join(EOL).trim(); - } + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return 0; // Avoid more than one empty line in a row. + } - private addLine(line: string): void { // Since we use HTML blocks as preformatted text // make sure we drop angle brackets since otherwise // they will render as tags and attributes - if (this.inPreBlock) { - line = line.replace(//g, ''); - } - this.md.push(line); + line = line.replace(//g, ' '); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + // Keep hard line breaks for the preformatted content + this.md.push(`${line} `); + return 0; } - private checkPreContent(lines: string[], i: number): void { - if (this.inPreBlock) { - return; - } - // Indented is considered to be preformatted except - // when previous line is indented or begins list item. - const line = lines[i]; - if (line.length === 0 || !isWhiteSpace(line.charCodeAt(0))) { - return; + private inCodeState(line: string): void { + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return; // Avoid more than one empty line in a row. } - let prevLine = i > 0 ? lines[i - 1] : undefined; - if (!prevLine) { - return; - } - if (prevLine.length === 0) { - this.startPreformattedBlock(line); - return; - } - if (isWhiteSpace(prevLine.charCodeAt(0))) { - return; + if (line.startsWith('```')) { + this.endCodeBlock(); + } else { + this.md.push(line); } + } - prevLine = prevLine.trim(); - if (prevLine.length === 0) { - this.startPreformattedBlock(line); - return; + private isIgnorable(line: string): boolean { + if (line.indexOf('generated/') >= 0) { + return true; // Drop generated content. } - - const ch = prevLine.charCodeAt(0); - if (ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch)) { - return; + const trimmed = line.trim(); + if (trimmed.startsWith('..') && trimmed.indexOf('::') > 0) { + // Ignore lines likes .. sectionauthor:: John Doe. + return true; } - - this.startPreformattedBlock(line); + return false; } - private handleCodeBlock(line: string): boolean { - if (!line.startsWith('```')) { - return false; + private checkPreContent(lines: string[], i: number): number { + const line = lines[i]; + if (i === 0 || line.trim().length === 0) { + return 0; } - if (this.inCodeBlock) { - this.endCodeBlock(); - } else { - this.startCodeBlock(); + + if (!isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + return 0; // regular line, nothing to do here. } - return true; + // Indented content is considered to be preformatted. + this.startPreformattedBlock(); + return -1; } private handleSectionHeader(lines: string[], i: number): boolean { @@ -216,56 +225,45 @@ export class RestTextConverter { this.md.push(line.substring(0, line.length - 1)); } - this.startPreformattedBlock(line); + this.startPreformattedBlock(); return true; } - private tryEndCodePreBlocks(): void { - if (this.inCodeBlock) { - this.endCodeBlock(); - } - if (this.inPreBlock) { - this.endPreformattedBlock(); - } - } - - private startPreformattedBlock(line: string): void { + private startPreformattedBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); // Lie about the language since we don't want preformatted text // to be colorized as Python. HTML is more 'appropriate' as it does // not colorize -- or + or keywords like 'from'. - if (line.indexOf('# ') >= 0) { - this.md.push('```python'); - } else { - this.md.push('```html'); - } - this.inPreBlock = true; + this.md.push('```html'); + this.state = State.Preformatted; } private endPreformattedBlock(): void { - if (this.inPreBlock) { + if (this.state === State.Preformatted) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inPreBlock = false; + this.state = State.Default; } } private startCodeBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); this.md.push('```python'); - this.inCodeBlock = true; + this.state = State.Code; } private endCodeBlock(): void { - if (this.inCodeBlock) { + if (this.state === State.Code) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inCodeBlock = false; + this.state = State.Default; } } - private tryRemovePrecedingEmptyLine(): void { - if (this.md.length > 0 && this.md[this.md.length - 1].length === 0) { + private tryRemovePrecedingEmptyLines(): void { + while (this.md.length > 0 && this.md[this.md.length - 1].trim().length === 0) { this.md.pop(); } } @@ -273,4 +271,10 @@ export class RestTextConverter { private convertEmphasis(line: string): string { return line.replace(/\:([\w\W]+)\:/g, '**$1**'); // Convert :word: to **word**. } + + private isListItem(line: string): boolean { + const trimmed = line.trim(); + const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0; + return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch); + } } diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index b3284f4727e7..81b1ba5bbf12 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -15,7 +15,7 @@ function compareFiles(expectedContent: string, actualContent: string) { for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { const e = expectedLines[i]; const a = actualLines[i]; - expect(a, `Difference at line ${i}`).to.be.equal(e); + expect(e, `Difference at line ${i}`).to.be.equal(a); } expect(actualLines.length, @@ -41,4 +41,5 @@ suite('Hover - RestTextConverter', () => { test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); test('anydbm', async () => await testConversion('anydbm')); test('aifc', async () => await testConversion('aifc')); + test('astroid', async () => await testConversion('astroid')); }); diff --git a/src/test/pythonFiles/markdown/aifc.md b/src/test/pythonFiles/markdown/aifc.md index a2f120c6a3b2..fff22dece1e5 100644 --- a/src/test/pythonFiles/markdown/aifc.md +++ b/src/test/pythonFiles/markdown/aifc.md @@ -5,19 +5,18 @@ both for AIFF-C files and AIFF files. An AIFF-C file has the following structure. ```html - +-----------------+ - | FORM | - +-----------------+ - | size | - +----+------------+ - | | AIFC | - | +------------+ - | | chunks | - | | . | - | | . | - | | . | - +----+------------+ - + +-----------------+ + | FORM | + +-----------------+ + | size | + +----+------------+ + | | AIFC | + | +------------+ + | | chunks | + | | . | + | | . | + | | . | + +----+------------+ ``` An AIFF file has the string "AIFF" instead of "AIFC". @@ -27,28 +26,27 @@ the size of the 8 byte header. The following chunk types are recognized. ```html - FVER - version number of AIFF-C defining document (AIFF-C only). - MARK - # of markers (2 bytes) - list of markers: - marker ID (2 bytes, must be 0) - position (4 bytes) - marker name ("pstring") - COMM - # of channels (2 bytes) - # of sound frames (4 bytes) - size of the samples (2 bytes) - sampling frequency (10 bytes, IEEE 80-bit extended - floating point) - in AIFF-C files only: - compression type (4 bytes) - human-readable version of compression type ("pstring") - SSND - offset (4 bytes, not used by this program) - blocksize (4 bytes, not used by this program) - sound data - + FVER + version number of AIFF-C defining document (AIFF-C only). + MARK + # of markers (2 bytes) + list of markers: + marker ID (2 bytes, must be 0) + position (4 bytes) + marker name ("pstring") + COMM + # of channels (2 bytes) + # of sound frames (4 bytes) + size of the samples (2 bytes) + sampling frequency (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + compression type (4 bytes) + human-readable version of compression type ("pstring") + SSND + offset (4 bytes, not used by this program) + blocksize (4 bytes, not used by this program) + sound data ``` A pstring consists of 1 byte length, a string of characters, and 0 or 1 byte pad to make the total length even. @@ -57,7 +55,7 @@ Usage. Reading AIFF files: ```html - f = aifc.open(file, 'r') + f = aifc.open(file, 'r') ``` where file is either the name of a file or an open file pointer. The open file pointer must have methods read(), seek(), and close(). @@ -66,25 +64,25 @@ the seek() method is not necessary. This returns an instance of a class with the following public methods: ```html - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' for AIFF files) - getcompname() -- returns human-readable version of - compression type ('not compressed' for AIFF files) - getparams() -- returns a tuple consisting of all of the - above in the above order - getmarkers() -- get the list of marks in the audio file or None - if there are no marks - getmark(id) -- get mark with the specified id (raises an error - if the mark does not exist) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) ``` The position returned by tell(), the position given to setpos() and the position of marks are all compatible and have nothing to do with @@ -94,7 +92,7 @@ is destroyed. Writing AIFF files: ```html - f = aifc.open(file, 'w') + f = aifc.open(file, 'w') ``` where file is either the name of a file or an open file pointer. The open file pointer must have methods write(), tell(), seek(), and @@ -102,28 +100,28 @@ close(). This returns an instance of a class with the following public methods: ```html - aiff() -- create an AIFF file (AIFF-C default) - aifc() -- create an AIFF-C file - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple) - -- set all parameters at once - setmark(id, pos, name) - -- add specified mark to the list of marks - tell() -- return current position in output file (useful - in combination with setmark()) - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file ``` You should set the parameters before the first writeframesraw or writeframes. The total number of frames does not need to be set, diff --git a/src/test/pythonFiles/markdown/anydbm.md b/src/test/pythonFiles/markdown/anydbm.md index a86897871374..e5914dcbadde 100644 --- a/src/test/pythonFiles/markdown/anydbm.md +++ b/src/test/pythonFiles/markdown/anydbm.md @@ -2,15 +2,13 @@ Generic interface to all dbm clones. Instead of ```html - import dbm - d = dbm.open(file, 'w', 0666) - + import dbm + d = dbm.open(file, 'w', 0666) ``` use ```html - import anydbm - d = anydbm.open(file, 'w') - + import anydbm + d = anydbm.open(file, 'w') ``` The returned object is a dbhash, gdbm, dbm or dumbdbm object, dependent on the type of database being opened (determined by whichdb @@ -20,16 +18,15 @@ be determined by the availability of the modules (tested in the above order). It has the following interface (key and data are strings): -```python - d[key] = data # store data at key (may override data at - # existing key) - data = d[key] # retrieve data at key (raise KeyError if no - # such key) - del d[key] # delete data stored at key (raises KeyError - # if no such key) - flag = key in d # true if the key exists - list = d.keys() # return a list of all existing keys (slow!) - +```html + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) ``` Future versions may change the order in which implementations are tested for existence, and add interfaces to other dbm-like diff --git a/src/test/pythonFiles/markdown/astroid.md b/src/test/pythonFiles/markdown/astroid.md new file mode 100644 index 000000000000..d3c1bda813ee --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.md @@ -0,0 +1,24 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: +```html +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.pydoc b/src/test/pythonFiles/markdown/astroid.pydoc new file mode 100644 index 000000000000..84d58487ead5 --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.pydoc @@ -0,0 +1,23 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: + +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.md b/src/test/pythonFiles/markdown/scipy.md index 23721797aae3..d28c1e290abe 100644 --- a/src/test/pythonFiles/markdown/scipy.md +++ b/src/test/pythonFiles/markdown/scipy.md @@ -11,38 +11,37 @@ addition provides: Using any of these subpackages requires an explicit import. For example, `import scipy.cluster`. ```html - cluster --- Vector Quantization / Kmeans - fftpack --- Discrete Fourier Transform algorithms - integrate --- Integration routines - interpolate --- Interpolation Tools - io --- Data input and output - linalg --- Linear algebra routines - linalg.blas --- Wrappers to BLAS library - linalg.lapack --- Wrappers to LAPACK library - misc --- Various utilities that don't have - another home. - ndimage --- n-dimensional image package - odr --- Orthogonal Distance Regression - optimize --- Optimization Tools - signal --- Signal Processing Tools - sparse --- Sparse Matrices - sparse.linalg --- Sparse Linear Algebra - sparse.linalg.dsolve --- Linear Solvers - sparse.linalg.dsolve.umfpack --- **Interface to the UMFPACK library** - Conjugate Gradient Method (LOBPCG) - sparse.linalg.eigen --- Sparse Eigenvalue Solvers - sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned - Conjugate Gradient Method (LOBPCG) - spatial --- Spatial data structures and algorithms - special --- Special functions - stats --- Statistical Functions - + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions ``` #### Utility tools ```html - test --- Run scipy unittests - show_config --- Show scipy build configuration - show_numpy_config --- Show numpy build configuration - __version__ --- Scipy version string - __numpy_version__ --- Numpy version string + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string ``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md index 125b19f6cdeb..8e9dd996931d 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.distance.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -6,53 +6,49 @@ Distance matrix computation from a collection of raw observation vectors stored in a rectangular array. ```html - pdist -- pairwise distances between observation vectors. - cdist -- distances between two collections of observation vectors - squareform -- convert distance matrix to a condensed one and vice versa - directed_hausdorff -- directed Hausdorff distance between arrays - + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays ``` Predicates for checking the validity of distance matrices, both condensed and redundant. Also contained in this module are functions for computing the number of observations in a distance matrix. ```html - is_valid_dm -- checks for a valid distance matrix - is_valid_y -- checks for a valid condensed distance matrix - num_obs_dm -- # of observations in a distance matrix - num_obs_y -- # of observations in a condensed distance matrix - + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix ``` Distance functions between two numeric vectors `u` and `v`. Computing distances over a large collection of vectors is inefficient for these functions. Use `pdist` for this purpose. ```html - braycurtis -- the Bray-Curtis distance. - canberra -- the Canberra distance. - chebyshev -- the Chebyshev distance. - cityblock -- the Manhattan distance. - correlation -- the Correlation distance. - cosine -- the Cosine distance. - euclidean -- the Euclidean distance. - mahalanobis -- the Mahalanobis distance. - minkowski -- the Minkowski distance. - seuclidean -- the normalized Euclidean distance. - sqeuclidean -- the squared Euclidean distance. - wminkowski -- (deprecated) alias of `minkowski`. - + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. ``` Distance functions between two boolean vectors (representing sets) `u` and `v`. As in the case of numerical vectors, `pdist` is more efficient for computing the distances between all pairs. ```html - dice -- the Dice dissimilarity. - hamming -- the Hamming distance. - jaccard -- the Jaccard distance. - kulsinski -- the Kulsinski distance. - rogerstanimoto -- the Rogers-Tanimoto dissimilarity. - russellrao -- the Russell-Rao dissimilarity. - sokalmichener -- the Sokal-Michener dissimilarity. - sokalsneath -- the Sokal-Sneath dissimilarity. - yule -- the Yule dissimilarity. - + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. ``` **func**`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md index 3584e78f1bbc..ba9a8a615843 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -3,55 +3,49 @@ ### Nearest-neighbor Queries ```html - KDTree -- class for efficient nearest-neighbor queries - cKDTree -- class for efficient nearest-neighbor queries (faster impl.) - distance -- module containing many different distance measures - Rectangle - + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle ``` ### Delaunay Triangulation, Convex Hulls and Voronoi Diagrams ```html - Delaunay -- compute Delaunay triangulation of input points - ConvexHull -- compute a convex hull for input points - Voronoi -- compute a Voronoi diagram hull from input points - SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere - HalfspaceIntersection -- compute the intersection points of input halfspaces - + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces ``` ### Plotting Helpers ```html - delaunay_plot_2d -- plot 2-D triangulation - convex_hull_plot_2d -- plot 2-D convex hull - voronoi_plot_2d -- plot 2-D voronoi diagram - + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram ``` ### Simplex representation The simplices (triangles, tetrahedra, ...) appearing in the Delaunay tesselation (N-dim simplices), convex hull facets, and Voronoi ridges (N-1 dim simplices) are represented in the following scheme: ```html - tess = Delaunay(points) - hull = ConvexHull(points) - voro = Voronoi(points) - - # coordinates of the j-th vertex of the i-th simplex - tess.points[tess.simplices[i, j], :] # tesselation element - hull.points[hull.simplices[i, j], :] # convex hull facet - voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells - + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells ``` For Delaunay triangulations and convex hulls, the neighborhood structure of the simplices satisfies the condition: ```html - `tess.neighbors[i,j]` is the neighboring simplex of the i-th - simplex, opposite to the j-vertex. It is -1 in case of no - neighbor. - + `tess.neighbors[i,j]` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. ``` Convex hull facets also define a hyperplane equation: ```html - (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 - + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 ``` Similar hyperplane equations for the Delaunay triangulation correspond to the convex hull facets on the corresponding N+1 dimensional @@ -63,9 +57,9 @@ computations. #### Functions ```html - tsearch - distance_matrix - minkowski_distance - minkowski_distance_p - procrustes + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes ``` \ No newline at end of file From e436fde65a1d818338826edf7e4e0ec97686455d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 7 Feb 2018 13:42:45 -0800 Subject: [PATCH 46/49] Baselines --- .../common/markdown/restTextConverter.ts | 50 ++++--------------- src/client/providers/itemInfoSource.ts | 2 +- src/test/markdown/restTextConverter.test.ts | 2 +- src/test/pythonFiles/markdown/astroid.md | 2 +- .../markdown/scipy.spatial.distance.md | 4 +- .../pythonFiles/markdown/scipy.spatial.md | 2 +- 6 files changed, 16 insertions(+), 46 deletions(-) diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index da00da8e2fe3..e606fd46bfbc 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -17,16 +17,12 @@ export class RestTextConverter { private md: string[] = []; // tslint:disable-next-line:cyclomatic-complexity - public toMarkdown(docstring: string, force?: boolean): string { + public toMarkdown(docstring: string): string { // Translates reStructruredText (Python doc syntax) to markdown. // It only translates as much as needed to display tooltips // and documentation in the completion list. // See https://en.wikipedia.org/wiki/ReStructuredText - if (!force && !this.shouldConvert(docstring)) { - return this.escapeMarkdown(docstring); - } - const result = this.transformLines(docstring); this.state = State.Default; this.md = []; @@ -40,34 +36,7 @@ export class RestTextConverter { return text .replace(/\#/g, '\\#') .replace(/\*/g, '\\*') - .replace(/\_/g, '\\_') - .replace(/\{/g, '\\{') - .replace(/\}/g, '\\}') - .replace(/\[/g, '\\[') - .replace(/\]/g, '\\]') - .replace(/\(/g, '\\(') - .replace(/\)/g, '\\)') - .replace(/\+/g, '\\+') - .replace(/\-/g, '\\+'); - } - - private shouldConvert(docstring: string): boolean { - // Heuristics that determe if string should be converted - // to markdown or just escaped. - - // :: at the end of a string - const doubleColon = docstring.indexOf('::'); - if (doubleColon >= 0 && doubleColon < docstring.length - 2) { - const ch = docstring.charCodeAt(doubleColon + 2); - if (ch === Char.LineFeed || ch === Char.CarriageReturn) { - return true; - } - } - // Section headers or lists - if (docstring.indexOf('===') >= 0 || docstring.indexOf('---') >= 0 || docstring.indexOf('.. ') >= 0) { - return true; - } - return false; + .replace(/\_/g, '\\_'); } private transformLines(docstring: string): string { @@ -126,8 +95,9 @@ export class RestTextConverter { return result; // Handle line in the new state } - line = this.convertEmphasis(line); + line = this.cleanup(line); line = line.replace(/``/g, '`'); // Convert double backticks to single. + line = this.escapeMarkdown(line); this.md.push(line); return 0; @@ -202,12 +172,12 @@ export class RestTextConverter { const line = lines[i]; if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { // Section title -> heading level 3. - this.md.push(`### ${this.convertEmphasis(line)}`); + this.md.push(`### ${this.cleanup(line)}`); return true; } if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { // Subsection title -> heading level 4. - this.md.push(`#### ${this.convertEmphasis(line)}`); + this.md.push(`#### ${this.cleanup(line)}`); return true; } return false; @@ -268,13 +238,13 @@ export class RestTextConverter { } } - private convertEmphasis(line: string): string { - return line.replace(/\:([\w\W]+)\:/g, '**$1**'); // Convert :word: to **word**. - } - private isListItem(line: string): boolean { const trimmed = line.trim(); const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0; return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch); } + + private cleanup(line: string): string { + return line.replace(/:mod:/g, 'module:'); + } } diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index fc02d8097464..4159e9396f82 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -109,7 +109,7 @@ export class ItemInfoSource { tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); } - const description = this.textConverter.toMarkdown(lines.join(EOL), signature.length === 0); + const description = this.textConverter.toMarkdown(lines.join(EOL)); tooltip = tooltip.appendMarkdown(description); const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 81b1ba5bbf12..9b43d4d57657 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -29,7 +29,7 @@ async function testConversion(fileName: string): Promise { const cvt = new RestTextConverter(); const file = path.join(srcPythoFilesPath, fileName); const source = await fs.readFile(`${file}.pydoc`, 'utf8'); - const actual = cvt.toMarkdown(source, true); + const actual = cvt.toMarkdown(source); const expected = await fs.readFile(`${file}.md`, 'utf8'); compareFiles(expected, actual); } diff --git a/src/test/pythonFiles/markdown/astroid.md b/src/test/pythonFiles/markdown/astroid.md index d3c1bda813ee..b5ece21c1faf 100644 --- a/src/test/pythonFiles/markdown/astroid.md +++ b/src/test/pythonFiles/markdown/astroid.md @@ -5,7 +5,7 @@ python source code for projects such as pychecker, pyreverse, pylint... Well, actually the development of this library is essentially governed by pylint's needs. -It extends class defined in the python's _ast module with some +It extends class defined in the python's \_ast module with some additional methods and attributes. Instance attributes are added by a builder object, which can either generate extended ast (let's call them astroid ;) by visiting an existent ast tree or by inspecting living diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md index 8e9dd996931d..276acddef787 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.distance.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -1,4 +1,4 @@ -### Distance computations (**mod**`scipy.spatial.distance`) +### Distance computations (module:`scipy.spatial.distance`) #### Function Reference @@ -51,4 +51,4 @@ computing the distances between all pairs. sokalsneath -- the Sokal-Sneath dissimilarity. yule -- the Yule dissimilarity. ``` -**func**`hamming` also operates over discrete numerical vectors. \ No newline at end of file +:func:`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md index ba9a8a615843..2d5e891db625 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -1,4 +1,4 @@ -### Spatial algorithms and data structures (**mod**`scipy.spatial`) +### Spatial algorithms and data structures (module:`scipy.spatial`) ### Nearest-neighbor Queries From e52bcffbe29775cb7d95f65a75f97fda30a0f62e Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 7 Feb 2018 13:49:50 -0800 Subject: [PATCH 47/49] Squash --- CONTRIBUTING.md | 56 ++-- src/client/common/extensions.ts | 18 +- .../common/markdown/restTextConverter.ts | 252 ++++++++------- src/client/common/terminal/helper.ts | 2 +- src/client/providers/itemInfoSource.ts | 2 +- .../codeExecution/codeExecutionManager.ts | 5 +- .../codeExecution/djangoShellCodeExecution.ts | 2 +- .../codeExecution/terminalCodeExecution.ts | 4 +- src/client/unittests/main.ts | 2 +- src/test/common/extensions.test.ts | 42 +++ .../common/terminals/activation.bash.test.ts | 6 +- .../activation.commandPrompt.test.ts | 288 +++++++++--------- src/test/common/terminals/helper.test.ts | 4 +- .../interpreters/condaEnvFileService.test.ts | 11 +- src/test/markdown/restTextConverter.test.ts | 6 +- src/test/pythonFiles/markdown/aifc.md | 142 +++++++++ src/test/pythonFiles/markdown/aifc.pydoc | 134 ++++++++ src/test/pythonFiles/markdown/anydbm.md | 29 +- src/test/pythonFiles/markdown/astroid.md | 24 ++ src/test/pythonFiles/markdown/astroid.pydoc | 23 ++ src/test/pythonFiles/markdown/scipy.md | 61 ++-- .../markdown/scipy.spatial.distance.md | 66 ++-- .../pythonFiles/markdown/scipy.spatial.md | 66 ++-- .../extension.refactor.extract.var.test.ts | 4 +- .../djangoShellCodeExect.test.ts | 2 +- .../codeExecution/terminalCodeExec.test.ts | 5 +- src/test/unittests/debugger.test.ts | 29 +- .../unittests/stoppingDiscoverAndTest.test.ts | 21 +- 28 files changed, 863 insertions(+), 443 deletions(-) create mode 100644 src/test/common/extensions.test.ts create mode 100644 src/test/pythonFiles/markdown/aifc.md create mode 100644 src/test/pythonFiles/markdown/aifc.pydoc create mode 100644 src/test/pythonFiles/markdown/astroid.md create mode 100644 src/test/pythonFiles/markdown/astroid.pydoc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04abf6a0df58..2d5b4456385a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,7 +93,7 @@ for a release are tracked in a [milestone](https://github.com/Microsoft/vscode-python/milestones) (which is actively updated as plans change). -The overall schedule for a release is to code freeze for on the last +The overall schedule for a release is to feature freeze for on the last Monday of the month to coincide with Visual Studio Code's code freeze. We then aim to release later that week so the latest version of the extension is already live by the time Visual Studio Code launches @@ -104,16 +104,10 @@ between scheduled releases as necessary. All development is actively done in the `master` branch of the repository. It is what allows us to have an [insiders build](#insiders-build) which is expected to be stable at -all times. We do keep the previous release as a branch in case the +all times. We do keep the most recent release as a branch in case the need for a bugfix release arises. But once a new release is made we -convert the older branch into a tag and delete the branch as -Visual Studio Code's automatic updating makes keeping old versions -around unnecessary. - -Since we try to spend about 25% of our development time fixing bugs -and removing technical debt, the week of a release is mostly spent -focusing on that topic. That way we don't ignore the health of the -code base by accidentally focusing on new features exclusively. +delete the older release branch (all releases are appropriately +tagged, so history is lost). ### Issue triaging @@ -142,21 +136,17 @@ lexicographically sort from earliest stage to latest stage). The suffix term for each label then specifies what is currently blocking the issue from being closed. -* `1-` - * [`decision`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-decision): - The issue is a feature enhancement request and a decision has not - been made as to whether we would accept a pull request - implementing the enhancement - * [`more info`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-more%20info): - We need more information from the OP (original poster) - * [`verification`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-verification): - We need to verify that the issue can be replicated +* [`1-decision`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-decision): +The issue is a feature enhancement request and a decision has not +been made as to whether we would accept a pull request +implementing the enhancement +* [`1-more info`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-more%20info): +We need more information from the OP (original poster) +* [`1-verification`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-verification): + We need to verify that the issue is reproducible * [`2-PR`](https://github.com/Microsoft/vscode-python/labels/awaiting%202-PR): The issue is valid and is now awaiting a fix to be created and merged into the `master` branch -* [`4-validation`](https://github.com/Microsoft/vscode-python/labels/awaiting%204-validation): - A pull request has been merged and resolution of the issue should be - independently validated #### Closed issues @@ -168,22 +158,21 @@ it should have an appropriate `closed-` label. 1. Check that there is an issue corresponding to what the pull request is attempting to address - * If an issue exists, make sure it has reached the stage of being - labeled `awaiting 2-PR` + * If an issue exists, make sure it has reached the stage of + `awaiting 2-PR` * If no issue exists, open one and wait for it to reach the `awaiting 2-PR` stage before submitting the pull request -1. Open the pull request, mentioning the appropriate issue(s) in the +1. Create the pull request, mentioning the appropriate issue(s) in the pull request message body * The pull request is expected to have appropriate unit tests * The pull request must pass its CI run before merging will be considered - * Code coverage is expected to not worsen + * Code coverage is expected to (at minimum) not worsen 1. Make sure all status checks are green (e.g. CLA check, CI, etc.) 1. Address any review comments 1. [Maintainers only] Merge the pull request 1. [Maintainers only] Update affected issues to be: 1. Closed (with an appropriate `closed-` label) - 1. The stage is set to `awaiting 4-validation` 1. The issue(s) are attached to the current milestone 1. Register OSS usage 1. Email CELA about any 3rd-party usage changes @@ -194,11 +183,12 @@ Starting in 2018, the extension switched to [calendar versioning](http://calver.org/) since the extension auto-updates and thus there is no need to track its version number for backwards-compatibility. As such, the major version -is the current year, the minor version is the current month, and -the micro version is how many releases there have been that month in -the year (starting at 0). For example, the first release in July 2018 -would be `2018.7.0`, the second release that month would be -`2018.7.1`, etc. +is the current year, the minor version is the month when feature +freeze was reached, and the micro version is how many releases there +have been since that feature freeze (starting at 0). For example +the release made when we reach feature freeze in July 2018 +would be `2018.7.0`, and if a second release was necessary to fix a +critical bug it would be `2018.7.1`. ## Insiders Build @@ -214,7 +204,7 @@ file, please follow the instructions on [this page](https://code.visualstudio.com/docs/editor/extension-gallery#_install-from-a-vsix) to install the extension. -The insiders build of the extension ...: +The insiders build of the extension: * Will be replaced with new releases published onto the [VS Code Marketplace](https://marketplace.visualstudio.com/VSCode). diff --git a/src/client/common/extensions.ts b/src/client/common/extensions.ts index 629b809e8d06..71858e6e01da 100644 --- a/src/client/common/extensions.ts +++ b/src/client/common/extensions.ts @@ -20,6 +20,11 @@ interface String { * E.g. if an argument contains a space, then it will be enclosed within double quotes. */ toCommandArgument(): string; + /** + * Appropriately formats a a file path so it can be used as an argument for a command in a shell. + * E.g. if an argument contains a space, then it will be enclosed within double quotes. + */ + fileToCommandArgument(): string; } /** @@ -47,5 +52,16 @@ String.prototype.toCommandArgument = function (this: string): string { if (!this) { return this; } - return (this.indexOf(' ') > 0 && !this.startsWith('"') && !this.endsWith('"')) ? `"${this}"` : this.toString(); + return (this.indexOf(' ') >= 0 && !this.startsWith('"') && !this.endsWith('"')) ? `"${this}"` : this.toString(); +}; + +/** + * Appropriately formats a a file path so it can be used as an argument for a command in a shell. + * E.g. if an argument contains a space, then it will be enclosed within double quotes. + */ +String.prototype.fileToCommandArgument = function (this: string): string { + if (!this) { + return this; + } + return this.toCommandArgument().replace(/\\/g, '/'); }; diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 104230f273a1..e606fd46bfbc 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -2,26 +2,29 @@ // Licensed under the MIT License. import { EOL } from 'os'; -import { isWhiteSpace } from '../../language/characters'; +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { isDecimal, isWhiteSpace } from '../../language/characters'; + +enum State { + Default, + Preformatted, + Code +} export class RestTextConverter { - private inPreBlock = false; - private inCodeBlock = false; + private state: State = State.Default; private md: string[] = []; // tslint:disable-next-line:cyclomatic-complexity - public toMarkdown(docstring: string, force?: boolean): string { + public toMarkdown(docstring: string): string { // Translates reStructruredText (Python doc syntax) to markdown. // It only translates as much as needed to display tooltips // and documentation in the completion list. // See https://en.wikipedia.org/wiki/ReStructuredText - if (!force && !this.shouldConvert(docstring)) { - return this.escapeMarkdown(docstring); - } - const result = this.transformLines(docstring); - this.inPreBlock = this.inPreBlock = false; + this.state = State.Default; this.md = []; return result; @@ -33,120 +36,148 @@ export class RestTextConverter { return text .replace(/\#/g, '\\#') .replace(/\*/g, '\\*') - .replace(/\_/g, '\\_') - .replace(/\{/g, '\\{') - .replace(/\}/g, '\\}') - .replace(/\[/g, '\\[') - .replace(/\]/g, '\\]') - .replace(/\(/g, '\\(') - .replace(/\)/g, '\\)') - .replace(/\+/g, '\\+') - .replace(/\-/g, '\\+'); - } - - private shouldConvert(docstring: string): boolean { - // heuristics - if (docstring.indexOf('::') >= 0 || docstring.indexOf('..') >= 0) { - return true; - } - if (docstring.indexOf('===') >= 0 || docstring.indexOf('---') >= 0) { - return true; - } - return false; + .replace(/\_/g, '\\_'); } - // tslint:disable-next-line:cyclomatic-complexity private transformLines(docstring: string): string { const lines = docstring.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { - let line = lines[i]; - + const line = lines[i]; // Avoid leading empty lines if (this.md.length === 0 && line.length === 0) { continue; } - if (!this.inPreBlock) { - // Anything indented is considered to be preformatted. - if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - this.startPreformattedBlock(line); - } + switch (this.state) { + case State.Default: + i += this.inDefaultState(lines, i); + break; + case State.Preformatted: + i += this.inPreformattedState(lines, i); + break; + case State.Code: + this.inCodeState(line); + break; + default: + break; } + } - if (this.handleCodeBlock(line)) { - continue; - } + this.endCodeBlock(); + this.endPreformattedBlock(); - if (this.inPreBlock) { - // Preformatted block terminates by a line without leading - // whitespace or any special line like ..ABC::. - if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { - this.endPreformattedBlock(); - } - } + return this.md.join(EOL).trim(); + } - if (this.handleSectionHeader(lines, i)) { - i += 1; // Eat line with === or --- - continue; - } + private inDefaultState(lines: string[], i: number): number { + let line = lines[i]; + if (line.startsWith('```')) { + this.startCodeBlock(); + return 0; + } - if (line.indexOf('generated/') >= 0) { - continue; // ignore generated content. - } - if (line.startsWith('===') || line.startsWith('---')) { - continue; // Eat standalone === or --- lines. - } + if (line.startsWith('===') || line.startsWith('---')) { + return 0; // Eat standalone === or --- lines. + } + if (this.handleDoubleColon(line)) { + return 0; + } + if (this.isIgnorable(line)) { + return 0; + } - if (this.handleDoubleColon(line)) { - continue; - } - if (line.startsWith('..') && line.indexOf('::') > 0) { - // Ignore lines likes .. sectionauthor:: John Doe. - continue; - } + if (this.handleSectionHeader(lines, i)) { + return 1; // Eat line with === or --- + } - line = this.convertEmphasis(line); - line = line.replace(/``/g, '`'); // Convert double backticks to single. + const result = this.checkPreContent(lines, i); + if (this.state !== State.Default) { + return result; // Handle line in the new state + } - if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - // Keep hard line breaks for the pre-indented content. - line = ` ${line} `; - } + line = this.cleanup(line); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + line = this.escapeMarkdown(line); + this.md.push(line); - const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; - if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { - continue; // Avoid more than one empty line in a row. - } + return 0; + } - this.md.push(line); + private inPreformattedState(lines: string[], i: number): number { + let line = lines[i]; + if (this.isIgnorable(line)) { + return 0; + } + // Preformatted block terminates by a line without leading whitespace. + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + this.endPreformattedBlock(); + return -1; } - this.tryEndCodePreBlocks(); - return this.md.join(EOL).trim(); + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return 0; // Avoid more than one empty line in a row. + } + + // Since we use HTML blocks as preformatted text + // make sure we drop angle brackets since otherwise + // they will render as tags and attributes + line = line.replace(//g, ' '); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + // Keep hard line breaks for the preformatted content + this.md.push(`${line} `); + return 0; } - private handleCodeBlock(line: string): boolean { - if (!line.startsWith('```')) { - return false; + private inCodeState(line: string): void { + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return; // Avoid more than one empty line in a row. } - if (this.inCodeBlock) { + + if (line.startsWith('```')) { this.endCodeBlock(); } else { - this.startCodeBlock(); + this.md.push(line); } - return true; + } + + private isIgnorable(line: string): boolean { + if (line.indexOf('generated/') >= 0) { + return true; // Drop generated content. + } + const trimmed = line.trim(); + if (trimmed.startsWith('..') && trimmed.indexOf('::') > 0) { + // Ignore lines likes .. sectionauthor:: John Doe. + return true; + } + return false; + } + + private checkPreContent(lines: string[], i: number): number { + const line = lines[i]; + if (i === 0 || line.trim().length === 0) { + return 0; + } + + if (!isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + return 0; // regular line, nothing to do here. + } + // Indented content is considered to be preformatted. + this.startPreformattedBlock(); + return -1; } private handleSectionHeader(lines: string[], i: number): boolean { const line = lines[i]; if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { // Section title -> heading level 3. - this.md.push(`### ${this.convertEmphasis(line)}`); + this.md.push(`### ${this.cleanup(line)}`); return true; } if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { // Subsection title -> heading level 4. - this.md.push(`#### ${this.convertEmphasis(line)}`); + this.md.push(`#### ${this.cleanup(line)}`); return true; } return false; @@ -164,61 +195,56 @@ export class RestTextConverter { this.md.push(line.substring(0, line.length - 1)); } - this.startPreformattedBlock(line); + this.startPreformattedBlock(); return true; } - private tryEndCodePreBlocks(): void { - if (this.inCodeBlock) { - this.endCodeBlock(); - } - if (this.inPreBlock) { - this.endPreformattedBlock(); - } - } - - private startPreformattedBlock(line: string): void { + private startPreformattedBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); // Lie about the language since we don't want preformatted text // to be colorized as Python. HTML is more 'appropriate' as it does // not colorize -- or + or keywords like 'from'. - if (line.indexOf('# ') >= 0) { - this.md.push('```python'); - } else { - this.md.push('```html'); - } - this.inPreBlock = true; + this.md.push('```html'); + this.state = State.Preformatted; } private endPreformattedBlock(): void { - if (this.inPreBlock) { + if (this.state === State.Preformatted) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inPreBlock = false; + this.state = State.Default; } } private startCodeBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); this.md.push('```python'); - this.inCodeBlock = true; + this.state = State.Code; } private endCodeBlock(): void { - if (this.inCodeBlock) { + if (this.state === State.Code) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inCodeBlock = false; + this.state = State.Default; } } - private tryRemovePrecedingEmptyLine(): void { - if (this.md.length > 0 && this.md[this.md.length - 1].length === 0) { + private tryRemovePrecedingEmptyLines(): void { + while (this.md.length > 0 && this.md[this.md.length - 1].trim().length === 0) { this.md.pop(); } } - private convertEmphasis(line: string): string { - return line.replace(/\:([\w\W]+)\:/g, '**$1**'); // Convert :word: to **word**. + private isListItem(line: string): boolean { + const trimmed = line.trim(); + const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0; + return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch); + } + + private cleanup(line: string): string { + return line.replace(/:mod:/g, 'module:'); } } diff --git a/src/client/common/terminal/helper.ts b/src/client/common/terminal/helper.ts index 07c679b86f4f..bc2f9c78a237 100644 --- a/src/client/common/terminal/helper.ts +++ b/src/client/common/terminal/helper.ts @@ -68,7 +68,7 @@ export class TerminalHelper implements ITerminalHelper { public buildCommandForTerminal(terminalShellType: TerminalShellType, command: string, args: string[]) { const isPowershell = terminalShellType === TerminalShellType.powershell || terminalShellType === TerminalShellType.powershellCore; const commandPrefix = isPowershell ? '& ' : ''; - return `${commandPrefix}${command.toCommandArgument()} ${args.join(' ')}`.trim(); + return `${commandPrefix}${command.fileToCommandArgument()} ${args.join(' ')}`.trim(); } public async getEnvironmentActivationCommands(terminalShellType: TerminalShellType, resource?: Uri): Promise { const settings = this.serviceContainer.get(IConfigurationService).getSettings(resource); diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index fc02d8097464..4159e9396f82 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -109,7 +109,7 @@ export class ItemInfoSource { tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); } - const description = this.textConverter.toMarkdown(lines.join(EOL), signature.length === 0); + const description = this.textConverter.toMarkdown(lines.join(EOL)); tooltip = tooltip.appendMarkdown(description); const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list diff --git a/src/client/terminals/codeExecution/codeExecutionManager.ts b/src/client/terminals/codeExecution/codeExecutionManager.ts index fd967c56318f..5a6159a50aae 100644 --- a/src/client/terminals/codeExecution/codeExecutionManager.ts +++ b/src/client/terminals/codeExecution/codeExecutionManager.ts @@ -15,7 +15,7 @@ import { ICodeExecutionHelper, ICodeExecutionManager, ICodeExecutionService } fr @injectable() export class CodeExecutionManager implements ICodeExecutionManager { - constructor( @inject(ICommandManager) private commandManager: ICommandManager, + constructor(@inject(ICommandManager) private commandManager: ICommandManager, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IDisposableRegistry) private disposableRegistry: Disposable[], @inject(IServiceContainer) private serviceContainer: IServiceContainer) { @@ -28,8 +28,9 @@ export class CodeExecutionManager implements ICodeExecutionManager { this.disposableRegistry.push(this.commandManager.registerCommand(Commands.Exec_Selection_In_Django_Shell, this.executeSelectionInDjangoShell.bind(this))); } @captureTelemetry(EXECUTION_CODE, { scope: 'file' }, false) - private async executeFileInterTerminal(file: Uri) { + private async executeFileInterTerminal(file?: Uri) { const codeExecutionHelper = this.serviceContainer.get(ICodeExecutionHelper); + file = file instanceof Uri ? file : undefined; const fileToExecute = file ? file : await codeExecutionHelper.getFileToExecute(); if (!fileToExecute) { return; diff --git a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts index d188a2091547..5fbe2ef2d19f 100644 --- a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts +++ b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts @@ -40,7 +40,7 @@ export class DjangoShellCodeExecutionProvider extends TerminalCodeExecutionProvi const workspaceRoot = workspaceUri ? workspaceUri.uri.fsPath : defaultWorkspace; const managePyPath = workspaceRoot.length === 0 ? 'manage.py' : path.join(workspaceRoot, 'manage.py'); - args.push(managePyPath.toCommandArgument()); + args.push(managePyPath.fileToCommandArgument()); args.push('shell'); return { command, args }; } diff --git a/src/client/terminals/codeExecution/terminalCodeExecution.ts b/src/client/terminals/codeExecution/terminalCodeExecution.ts index a3aaedcc2584..4ed1d7da479c 100644 --- a/src/client/terminals/codeExecution/terminalCodeExecution.ts +++ b/src/client/terminals/codeExecution/terminalCodeExecution.ts @@ -34,7 +34,7 @@ export class TerminalCodeExecutionProvider implements ICodeExecutionService { const command = this.platformService.isWindows ? pythonSettings.pythonPath.replace(/\\/g, '/') : pythonSettings.pythonPath; const launchArgs = pythonSettings.terminal.launchArgs; - await this.getTerminalService(file).sendCommand(command, launchArgs.concat(file.fsPath.toCommandArgument())); + await this.getTerminalService(file).sendCommand(command, launchArgs.concat(file.fsPath.fileToCommandArgument())); } public async execute(code: string, resource?: Uri): Promise { @@ -47,7 +47,7 @@ export class TerminalCodeExecutionProvider implements ICodeExecutionService { } public async initializeRepl(resource?: Uri) { if (this.replActive && await this.replActive!) { - this._terminalService!.show(); + await this._terminalService!.show(); return; } this.replActive = new Promise(async resolve => { diff --git a/src/client/unittests/main.ts b/src/client/unittests/main.ts index c95777f78bcc..2e386edea0b0 100644 --- a/src/client/unittests/main.ts +++ b/src/client/unittests/main.ts @@ -158,7 +158,7 @@ async function selectAndRunTestMethod(cmdSource: CommandSource, resource: Uri, d return; } // tslint:disable-next-line:prefer-type-cast - await runTestsImpl(cmdSource, testManager.workspaceFolder, { testFunction: [selectedTestFn.testFunction] } as TestsToRun, debug); + await runTestsImpl(cmdSource, testManager.workspaceFolder, { testFunction: [selectedTestFn.testFunction] } as TestsToRun, false, debug); } async function selectAndRunTestFile(cmdSource: CommandSource) { const testManager = await getTestManager(true); diff --git a/src/test/common/extensions.test.ts b/src/test/common/extensions.test.ts new file mode 100644 index 000000000000..5724f3291274 --- /dev/null +++ b/src/test/common/extensions.test.ts @@ -0,0 +1,42 @@ +import { expect } from 'chai'; +import '../../client/common/extensions'; + +// Defines a Mocha test suite to group tests of similar kind together +suite('String Extensions', () => { + test('Should return empty string for empty arg', () => { + const argTotest = ''; + expect(argTotest.toCommandArgument()).to.be.equal(''); + }); + test('Should quote an empty space', () => { + const argTotest = ' '; + expect(argTotest.toCommandArgument()).to.be.equal('" "'); + }); + test('Should not quote command arguments without spaces', () => { + const argTotest = 'one.two.three'; + expect(argTotest.toCommandArgument()).to.be.equal(argTotest); + }); + test('Should quote command arguments with spaces', () => { + const argTotest = 'one two three'; + expect(argTotest.toCommandArgument()).to.be.equal(`"${argTotest}"`); + }); + test('Should return empty string for empty path', () => { + const fileToTest = ''; + expect(fileToTest.fileToCommandArgument()).to.be.equal(''); + }); + test('Should not quote file argument without spaces', () => { + const fileToTest = 'users/test/one'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(fileToTest); + }); + test('Should quote file argument with spaces', () => { + const fileToTest = 'one two three'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(`"${fileToTest}"`); + }); + test('Should replace all back slashes with forward slashes (irrespective of OS)', () => { + const fileToTest = 'c:\\users\\user\\conda\\scripts\\python.exe'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(fileToTest.replace(/\\/g, '/')); + }); + test('Should replace all back slashes with forward slashes (irrespective of OS) and quoted when file has spaces', () => { + const fileToTest = 'c:\\users\\user namne\\conda path\\scripts\\python.exe'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(`"${fileToTest.replace(/\\/g, '/')}"`); + }); +}); diff --git a/src/test/common/terminals/activation.bash.test.ts b/src/test/common/terminals/activation.bash.test.ts index ee7d2829ea46..c321528140ea 100644 --- a/src/test/common/terminals/activation.bash.test.ts +++ b/src/test/common/terminals/activation.bash.test.ts @@ -5,6 +5,7 @@ import { expect } from 'chai'; import * as path from 'path'; import * as TypeMoq from 'typemoq'; import { EnumEx } from '../../../client/common/enumUtils'; +import '../../../client/common/extensions'; import { IFileSystem } from '../../../client/common/platform/types'; import { Bash } from '../../../client/common/terminal/environmentActivationProviders/bash'; import { TerminalShellType } from '../../../client/common/terminal/types'; @@ -13,7 +14,7 @@ import { IServiceContainer } from '../../../client/ioc/types'; // tslint:disable-next-line:max-func-body-length suite('Terminal Environment Activation (bash)', () => { - ['usr/bin/python', 'usr/bin/env with spaces/env more/python'].forEach(pythonPath => { + ['usr/bin/python', 'usr/bin/env with spaces/env more/python', 'c:\\users\\windows paths\\conda\\python.exe'].forEach(pythonPath => { const hasSpaces = pythonPath.indexOf(' ') > 0; const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; suite(suiteTitle, () => { @@ -83,8 +84,7 @@ suite('Terminal Environment Activation (bash)', () => { // Ensure the path is quoted if it contains any spaces. // Ensure it contains the name of the environment as an argument to the script file. - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`source ${quotedScriptFile}`.trim()], 'Invalid command'); + expect(command).to.be.deep.equal([`source ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); } else { expect(command).to.be.equal(undefined, 'Command should be undefined'); } diff --git a/src/test/common/terminals/activation.commandPrompt.test.ts b/src/test/common/terminals/activation.commandPrompt.test.ts index 47b610ea0997..f6c7509ce00d 100644 --- a/src/test/common/terminals/activation.commandPrompt.test.ts +++ b/src/test/common/terminals/activation.commandPrompt.test.ts @@ -15,20 +15,64 @@ import { IConfigurationService, IPythonSettings } from '../../../client/common/t import { IServiceContainer } from '../../../client/ioc/types'; suite('Terminal Environment Activation (cmd/powershell)', () => { - ['c:/programfiles/python/python', 'c:/program files/python/python'].forEach(pythonPath => { - const hasSpaces = pythonPath.indexOf(' ') > 0; - const resource = Uri.file('a'); - - const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; - suite(suiteTitle, () => { - ['activate', 'activate.sh', 'activate.csh', 'activate.fish', 'activate.bat', 'activate.ps1'].forEach(scriptFileName => { - suite(`and script file is ${scriptFileName}`, () => { + ['c:/programfiles/python/python', 'c:/program files/python/python', + 'c:\\users\\windows paths\\conda\\python.exe'].forEach(pythonPath => { + const hasSpaces = pythonPath.indexOf(' ') > 0; + const resource = Uri.file('a'); + + const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; + suite(suiteTitle, () => { + ['activate', 'activate.sh', 'activate.csh', 'activate.fish', 'activate.bat', 'activate.ps1'].forEach(scriptFileName => { + suite(`and script file is ${scriptFileName}`, () => { + let serviceContainer: TypeMoq.IMock; + let fileSystem: TypeMoq.IMock; + setup(() => { + serviceContainer = TypeMoq.Mock.ofType(); + fileSystem = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + + const configService = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); + const settings = TypeMoq.Mock.ofType(); + settings.setup(s => s.pythonPath).returns(() => pythonPath); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); + }); + + EnumEx.getNamesAndValues(TerminalShellType).forEach(shellType => { + const isScriptFileSupported = ['activate.bat', 'activate.ps1'].indexOf(scriptFileName) >= 0; + const titleTitle = isScriptFileSupported ? `Ensure terminal type is supported (Shell: ${shellType.name})` : + `Ensure terminal type is not supported (Shell: ${shellType.name})`; + + test(titleTitle, async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); + + const supported = bash.isShellSupported(shellType.value); + switch (shellType.value) { + case TerminalShellType.commandPrompt: + case TerminalShellType.powershellCore: + case TerminalShellType.powershell: { + expect(supported).to.be.equal(true, `${shellType.name} shell not supported (it should be)`); + break; + } + default: { + expect(supported).to.be.equal(false, `${shellType.name} incorrectly supported (should not be)`); + } + } + }); + }); + }); + }); + + suite('and script file is activate.bat', () => { let serviceContainer: TypeMoq.IMock; let fileSystem: TypeMoq.IMock; + let platform: TypeMoq.IMock; setup(() => { serviceContainer = TypeMoq.Mock.ofType(); fileSystem = TypeMoq.Mock.ofType(); + platform = TypeMoq.Mock.ofType(); serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); const configService = TypeMoq.Mock.ofType(); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); @@ -37,173 +81,125 @@ suite('Terminal Environment Activation (cmd/powershell)', () => { configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); }); - EnumEx.getNamesAndValues(TerminalShellType).forEach(shellType => { - const isScriptFileSupported = ['activate.bat', 'activate.ps1'].indexOf(scriptFileName) >= 0; - const titleTitle = isScriptFileSupported ? `Ensure terminal type is supported (Shell: ${shellType.name})` : - `Ensure terminal type is not supported (Shell: ${shellType.name})`; - - test(titleTitle, async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); - - const supported = bash.isShellSupported(shellType.value); - switch (shellType.value) { - case TerminalShellType.commandPrompt: - case TerminalShellType.powershellCore: - case TerminalShellType.powershell: { - expect(supported).to.be.equal(true, `${shellType.name} shell not supported (it should be)`); - break; - } - default: { - expect(supported).to.be.equal(false, `${shellType.name} incorrectly supported (should not be)`); - } - } - }); - }); - }); - }); - - suite('and script file is activate.bat', () => { - let serviceContainer: TypeMoq.IMock; - let fileSystem: TypeMoq.IMock; - let platform: TypeMoq.IMock; - setup(() => { - serviceContainer = TypeMoq.Mock.ofType(); - fileSystem = TypeMoq.Mock.ofType(); - platform = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); - serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); - - const configService = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); - const settings = TypeMoq.Mock.ofType(); - settings.setup(s => s.pythonPath).returns(() => pythonPath); - configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); - }); + test('Ensure batch files are supported by command prompt', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - test('Ensure batch files are supported by command prompt', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const commands = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const commands = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); + // Ensure the script file is of the following form: + // source "" + // Ensure the path is quoted if it contains any spaces. + // Ensure it contains the name of the environment as an argument to the script file. - // Ensure the script file is of the following form: - // source "" - // Ensure the path is quoted if it contains any spaces. - // Ensure it contains the name of the environment as an argument to the script file. - - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(commands).to.be.deep.equal([`${quotedScriptFile}`.trim()], 'Invalid command'); - }); + expect(commands).to.be.deep.equal([pathToScriptFile.fileToCommandArgument()], 'Invalid command'); + }); - test('Ensure batch files are supported by powershell (on windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are supported by powershell (on windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - // Executing batch files from powershell requires going back to cmd, then into powershell + // Executing batch files from powershell requires going back to cmd, then into powershell - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - const activationCommand = `${quotedScriptFile}`.trim(); - const commands = [`& cmd /k "${activationCommand} & powershell"`]; - expect(command).to.be.deep.equal(commands, 'Invalid command'); - }); + const activationCommand = pathToScriptFile.fileToCommandArgument(); + const commands = [`& cmd /k "${activationCommand} & powershell"`]; + expect(command).to.be.deep.equal(commands, 'Invalid command'); + }); - test('Ensure batch files are supported by powershell core (on windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are supported by powershell core (on windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - // Executing batch files from powershell requires going back to cmd, then into powershell + // Executing batch files from powershell requires going back to cmd, then into powershell - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - const activationCommand = `${quotedScriptFile}`.trim(); - const commands = [`& cmd /k "${activationCommand} & pwsh"`]; - expect(command).to.be.deep.equal(commands, 'Invalid command'); - }); + const activationCommand = pathToScriptFile.fileToCommandArgument(); + const commands = [`& cmd /k "${activationCommand} & pwsh"`]; + expect(command).to.be.deep.equal(commands, 'Invalid command'); + }); - test('Ensure batch files are not supported by powershell (on non-windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are not supported by powershell (on non-windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => false); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => false); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - expect(command).to.be.equal(undefined, 'Invalid command'); - }); + expect(command).to.be.equal(undefined, 'Invalid command'); + }); - test('Ensure batch files are not supported by powershell core (on non-windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are not supported by powershell core (on non-windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => false); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => false); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - expect(command).to.be.equal(undefined, 'Invalid command'); + expect(command).to.be.equal(undefined, 'Invalid command'); + }); }); - }); - suite('and script file is activate.ps1', () => { - let serviceContainer: TypeMoq.IMock; - let fileSystem: TypeMoq.IMock; - let platform: TypeMoq.IMock; - setup(() => { - serviceContainer = TypeMoq.Mock.ofType(); - fileSystem = TypeMoq.Mock.ofType(); - platform = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); - serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); - - const configService = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); - const settings = TypeMoq.Mock.ofType(); - settings.setup(s => s.pythonPath).returns(() => pythonPath); - configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); - }); + suite('and script file is activate.ps1', () => { + let serviceContainer: TypeMoq.IMock; + let fileSystem: TypeMoq.IMock; + let platform: TypeMoq.IMock; + setup(() => { + serviceContainer = TypeMoq.Mock.ofType(); + fileSystem = TypeMoq.Mock.ofType(); + platform = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); + + const configService = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); + const settings = TypeMoq.Mock.ofType(); + settings.setup(s => s.pythonPath).returns(() => pythonPath); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); + }); - test('Ensure powershell files are not supported by command prompt', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are not supported by command prompt', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); - expect(command).to.be.deep.equal([], 'Invalid command (running powershell files are not supported on command prompt)'); - }); + expect(command).to.be.deep.equal([], 'Invalid command (running powershell files are not supported on command prompt)'); + }); - test('Ensure powershell files are supported by powershell', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are supported by powershell', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`& ${quotedScriptFile}`.trim()], 'Invalid command'); - }); + expect(command).to.be.deep.equal([`& ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); + }); - test('Ensure powershell files are supported by powershell core', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are supported by powershell core', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`& ${quotedScriptFile}`.trim()], 'Invalid command'); + expect(command).to.be.deep.equal([`& ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); + }); }); }); }); - }); }); diff --git a/src/test/common/terminals/helper.test.ts b/src/test/common/terminals/helper.test.ts index dea22cd4eef5..e692f480a6ca 100644 --- a/src/test/common/terminals/helper.test.ts +++ b/src/test/common/terminals/helper.test.ts @@ -102,7 +102,7 @@ suite('Terminal Service helpers', () => { const command = 'c:\\python 3.7.exe'; const args = ['1', '2']; const commandPrefix = (item.value === TerminalShellType.powershell || item.value === TerminalShellType.powershellCore) ? '& ' : ''; - const expectedTerminalCommand = `${commandPrefix}"${command}" 1 2`; + const expectedTerminalCommand = `${commandPrefix}${command.fileToCommandArgument()} 1 2`; const terminalCommand = helper.buildCommandForTerminal(item.value, command, args); expect(terminalCommand).to.equal(expectedTerminalCommand, `Incorrect command for Shell ${item.name}`); @@ -126,7 +126,7 @@ suite('Terminal Service helpers', () => { const command = 'c:\\python 3.7.exe'; const args = []; const commandPrefix = (item.value === TerminalShellType.powershell || item.value === TerminalShellType.powershellCore) ? '& ' : ''; - const expectedTerminalCommand = `${commandPrefix}"${command}"`; + const expectedTerminalCommand = `${commandPrefix}${command.fileToCommandArgument()}`; const terminalCommand = helper.buildCommandForTerminal(item.value, command, args); expect(terminalCommand).to.equal(expectedTerminalCommand, `Incorrect command for Shell ${item.name}`); diff --git a/src/test/interpreters/condaEnvFileService.test.ts b/src/test/interpreters/condaEnvFileService.test.ts index 85844e19c022..2207be82732a 100644 --- a/src/test/interpreters/condaEnvFileService.test.ts +++ b/src/test/interpreters/condaEnvFileService.test.ts @@ -104,15 +104,20 @@ suite('Interpreters from Conda Environments Text File', () => { const interpreterPaths = [ path.join(environmentsPath, 'conda', 'envs', 'numpy') ]; + const pythonPath = path.join(interpreterPaths[0], 'pythonPath'); condaService.setup(c => c.condaEnvironmentsFile).returns(() => environmentsFilePath); + condaService.setup(c => c.getInterpreterPath(TypeMoq.It.isAny())).returns(() => pythonPath); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pythonPath))).returns(() => Promise.resolve(true)); fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(environmentsFilePath))).returns(() => Promise.resolve(true)); fileSystem.setup(fs => fs.readFile(TypeMoq.It.isValue(environmentsFilePath))).returns(() => Promise.resolve(interpreterPaths.join(EOL))); - AnacondaCompanyNames.forEach(async companyDisplayName => { + for (const companyName of AnacondaCompanyNames) { + const versionWithCompanyName = `Mock Version :: ${companyName}`; + interpreterVersion.setup(c => c.getVersion(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve(versionWithCompanyName)); const interpreters = await condaFileProvider.getInterpreters(); assert.equal(interpreters.length, 1, 'Incorrect number of entries'); - assert.equal(interpreters[0].displayName, `${AnacondaDisplayName} Mock Version (numpy)`, 'Incorrect display name'); - }); + assert.equal(interpreters[0].displayName, `${AnacondaDisplayName} Mock Version`, 'Incorrect display name'); + } }); }); diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 7b2f9a97cdc8..9b43d4d57657 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -15,7 +15,7 @@ function compareFiles(expectedContent: string, actualContent: string) { for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { const e = expectedLines[i]; const a = actualLines[i]; - expect(a, `Difference at line ${i}`).to.be.equal(e); + expect(e, `Difference at line ${i}`).to.be.equal(a); } expect(actualLines.length, @@ -29,7 +29,7 @@ async function testConversion(fileName: string): Promise { const cvt = new RestTextConverter(); const file = path.join(srcPythoFilesPath, fileName); const source = await fs.readFile(`${file}.pydoc`, 'utf8'); - const actual = cvt.toMarkdown(source, true); + const actual = cvt.toMarkdown(source); const expected = await fs.readFile(`${file}.md`, 'utf8'); compareFiles(expected, actual); } @@ -40,4 +40,6 @@ suite('Hover - RestTextConverter', () => { test('scipy.spatial', async () => await testConversion('scipy.spatial')); test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); test('anydbm', async () => await testConversion('anydbm')); + test('aifc', async () => await testConversion('aifc')); + test('astroid', async () => await testConversion('astroid')); }); diff --git a/src/test/pythonFiles/markdown/aifc.md b/src/test/pythonFiles/markdown/aifc.md new file mode 100644 index 000000000000..fff22dece1e5 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.md @@ -0,0 +1,142 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. +```html + +-----------------+ + | FORM | + +-----------------+ + | size | + +----+------------+ + | | AIFC | + | +------------+ + | | chunks | + | | . | + | | . | + | | . | + +----+------------+ +``` +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. +```html + FVER + version number of AIFF-C defining document (AIFF-C only). + MARK + # of markers (2 bytes) + list of markers: + marker ID (2 bytes, must be 0) + position (4 bytes) + marker name ("pstring") + COMM + # of channels (2 bytes) + # of sound frames (4 bytes) + size of the samples (2 bytes) + sampling frequency (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + compression type (4 bytes) + human-readable version of compression type ("pstring") + SSND + offset (4 bytes, not used by this program) + blocksize (4 bytes, not used by this program) + sound data +``` +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: +```html + f = aifc.open(file, 'r') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: +```html + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +``` +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: +```html + f = aifc.open(file, 'w') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: +```html + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +``` +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/aifc.pydoc b/src/test/pythonFiles/markdown/aifc.pydoc new file mode 100644 index 000000000000..a4cc346d5531 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.pydoc @@ -0,0 +1,134 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. + + +-----------------+ + | FORM | + +-----------------+ + | | + +----+------------+ + | | AIFC | + | +------------+ + | | | + | | . | + | | . | + | | . | + +----+------------+ + +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. + + FVER + (AIFF-C only). + MARK + <# of markers> (2 bytes) + list of markers: + (2 bytes, must be > 0) + (4 bytes) + ("pstring") + COMM + <# of channels> (2 bytes) + <# of sound frames> (4 bytes) + (2 bytes) + (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + (4 bytes) + ("pstring") + SSND + (4 bytes, not used by this program) + (4 bytes, not used by this program) + + +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: + f = aifc.open(file, 'r') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: + f = aifc.open(file, 'w') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.md b/src/test/pythonFiles/markdown/anydbm.md index a86897871374..e5914dcbadde 100644 --- a/src/test/pythonFiles/markdown/anydbm.md +++ b/src/test/pythonFiles/markdown/anydbm.md @@ -2,15 +2,13 @@ Generic interface to all dbm clones. Instead of ```html - import dbm - d = dbm.open(file, 'w', 0666) - + import dbm + d = dbm.open(file, 'w', 0666) ``` use ```html - import anydbm - d = anydbm.open(file, 'w') - + import anydbm + d = anydbm.open(file, 'w') ``` The returned object is a dbhash, gdbm, dbm or dumbdbm object, dependent on the type of database being opened (determined by whichdb @@ -20,16 +18,15 @@ be determined by the availability of the modules (tested in the above order). It has the following interface (key and data are strings): -```python - d[key] = data # store data at key (may override data at - # existing key) - data = d[key] # retrieve data at key (raise KeyError if no - # such key) - del d[key] # delete data stored at key (raises KeyError - # if no such key) - flag = key in d # true if the key exists - list = d.keys() # return a list of all existing keys (slow!) - +```html + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) ``` Future versions may change the order in which implementations are tested for existence, and add interfaces to other dbm-like diff --git a/src/test/pythonFiles/markdown/astroid.md b/src/test/pythonFiles/markdown/astroid.md new file mode 100644 index 000000000000..b5ece21c1faf --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.md @@ -0,0 +1,24 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's \_ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: +```html +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.pydoc b/src/test/pythonFiles/markdown/astroid.pydoc new file mode 100644 index 000000000000..84d58487ead5 --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.pydoc @@ -0,0 +1,23 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: + +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.md b/src/test/pythonFiles/markdown/scipy.md index 23721797aae3..d28c1e290abe 100644 --- a/src/test/pythonFiles/markdown/scipy.md +++ b/src/test/pythonFiles/markdown/scipy.md @@ -11,38 +11,37 @@ addition provides: Using any of these subpackages requires an explicit import. For example, `import scipy.cluster`. ```html - cluster --- Vector Quantization / Kmeans - fftpack --- Discrete Fourier Transform algorithms - integrate --- Integration routines - interpolate --- Interpolation Tools - io --- Data input and output - linalg --- Linear algebra routines - linalg.blas --- Wrappers to BLAS library - linalg.lapack --- Wrappers to LAPACK library - misc --- Various utilities that don't have - another home. - ndimage --- n-dimensional image package - odr --- Orthogonal Distance Regression - optimize --- Optimization Tools - signal --- Signal Processing Tools - sparse --- Sparse Matrices - sparse.linalg --- Sparse Linear Algebra - sparse.linalg.dsolve --- Linear Solvers - sparse.linalg.dsolve.umfpack --- **Interface to the UMFPACK library** - Conjugate Gradient Method (LOBPCG) - sparse.linalg.eigen --- Sparse Eigenvalue Solvers - sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned - Conjugate Gradient Method (LOBPCG) - spatial --- Spatial data structures and algorithms - special --- Special functions - stats --- Statistical Functions - + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions ``` #### Utility tools ```html - test --- Run scipy unittests - show_config --- Show scipy build configuration - show_numpy_config --- Show numpy build configuration - __version__ --- Scipy version string - __numpy_version__ --- Numpy version string + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string ``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md index 125b19f6cdeb..276acddef787 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.distance.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -1,4 +1,4 @@ -### Distance computations (**mod**`scipy.spatial.distance`) +### Distance computations (module:`scipy.spatial.distance`) #### Function Reference @@ -6,53 +6,49 @@ Distance matrix computation from a collection of raw observation vectors stored in a rectangular array. ```html - pdist -- pairwise distances between observation vectors. - cdist -- distances between two collections of observation vectors - squareform -- convert distance matrix to a condensed one and vice versa - directed_hausdorff -- directed Hausdorff distance between arrays - + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays ``` Predicates for checking the validity of distance matrices, both condensed and redundant. Also contained in this module are functions for computing the number of observations in a distance matrix. ```html - is_valid_dm -- checks for a valid distance matrix - is_valid_y -- checks for a valid condensed distance matrix - num_obs_dm -- # of observations in a distance matrix - num_obs_y -- # of observations in a condensed distance matrix - + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix ``` Distance functions between two numeric vectors `u` and `v`. Computing distances over a large collection of vectors is inefficient for these functions. Use `pdist` for this purpose. ```html - braycurtis -- the Bray-Curtis distance. - canberra -- the Canberra distance. - chebyshev -- the Chebyshev distance. - cityblock -- the Manhattan distance. - correlation -- the Correlation distance. - cosine -- the Cosine distance. - euclidean -- the Euclidean distance. - mahalanobis -- the Mahalanobis distance. - minkowski -- the Minkowski distance. - seuclidean -- the normalized Euclidean distance. - sqeuclidean -- the squared Euclidean distance. - wminkowski -- (deprecated) alias of `minkowski`. - + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. ``` Distance functions between two boolean vectors (representing sets) `u` and `v`. As in the case of numerical vectors, `pdist` is more efficient for computing the distances between all pairs. ```html - dice -- the Dice dissimilarity. - hamming -- the Hamming distance. - jaccard -- the Jaccard distance. - kulsinski -- the Kulsinski distance. - rogerstanimoto -- the Rogers-Tanimoto dissimilarity. - russellrao -- the Russell-Rao dissimilarity. - sokalmichener -- the Sokal-Michener dissimilarity. - sokalsneath -- the Sokal-Sneath dissimilarity. - yule -- the Yule dissimilarity. - + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. ``` -**func**`hamming` also operates over discrete numerical vectors. \ No newline at end of file +:func:`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md index 3584e78f1bbc..2d5e891db625 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -1,57 +1,51 @@ -### Spatial algorithms and data structures (**mod**`scipy.spatial`) +### Spatial algorithms and data structures (module:`scipy.spatial`) ### Nearest-neighbor Queries ```html - KDTree -- class for efficient nearest-neighbor queries - cKDTree -- class for efficient nearest-neighbor queries (faster impl.) - distance -- module containing many different distance measures - Rectangle - + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle ``` ### Delaunay Triangulation, Convex Hulls and Voronoi Diagrams ```html - Delaunay -- compute Delaunay triangulation of input points - ConvexHull -- compute a convex hull for input points - Voronoi -- compute a Voronoi diagram hull from input points - SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere - HalfspaceIntersection -- compute the intersection points of input halfspaces - + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces ``` ### Plotting Helpers ```html - delaunay_plot_2d -- plot 2-D triangulation - convex_hull_plot_2d -- plot 2-D convex hull - voronoi_plot_2d -- plot 2-D voronoi diagram - + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram ``` ### Simplex representation The simplices (triangles, tetrahedra, ...) appearing in the Delaunay tesselation (N-dim simplices), convex hull facets, and Voronoi ridges (N-1 dim simplices) are represented in the following scheme: ```html - tess = Delaunay(points) - hull = ConvexHull(points) - voro = Voronoi(points) - - # coordinates of the j-th vertex of the i-th simplex - tess.points[tess.simplices[i, j], :] # tesselation element - hull.points[hull.simplices[i, j], :] # convex hull facet - voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells - + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells ``` For Delaunay triangulations and convex hulls, the neighborhood structure of the simplices satisfies the condition: ```html - `tess.neighbors[i,j]` is the neighboring simplex of the i-th - simplex, opposite to the j-vertex. It is -1 in case of no - neighbor. - + `tess.neighbors[i,j]` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. ``` Convex hull facets also define a hyperplane equation: ```html - (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 - + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 ``` Similar hyperplane equations for the Delaunay triangulation correspond to the convex hull facets on the corresponding N+1 dimensional @@ -63,9 +57,9 @@ computations. #### Functions ```html - tsearch - distance_matrix - minkowski_distance - minkowski_distance_p - procrustes + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes ``` \ No newline at end of file diff --git a/src/test/refactor/extension.refactor.extract.var.test.ts b/src/test/refactor/extension.refactor.extract.var.test.ts index 5ce6cc3f743c..d12283a74198 100644 --- a/src/test/refactor/extension.refactor.extract.var.test.ts +++ b/src/test/refactor/extension.refactor.extract.var.test.ts @@ -101,13 +101,13 @@ suite('Variable Extraction', () => { test('Extract Variable', async () => { const startPos = new vscode.Position(234, 29); const endPos = new vscode.Position(234, 38); - testingVariableExtraction(false, startPos, endPos); + await testingVariableExtraction(false, startPos, endPos); }); test('Extract Variable fails if whole string not selected', async () => { const startPos = new vscode.Position(234, 20); const endPos = new vscode.Position(234, 38); - testingVariableExtraction(true, startPos, endPos); + await testingVariableExtraction(true, startPos, endPos); }); function testingVariableExtractionEndToEnd(shouldError: boolean, startPos: Position, endPos: Position) { diff --git a/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts b/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts index 4778ffa55512..7b1dc4a55742 100644 --- a/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts +++ b/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts @@ -150,7 +150,7 @@ suite('Terminal - Django Shell Code Execution', () => { const workspaceFolder: WorkspaceFolder = { index: 0, name: 'blah', uri: workspaceUri }; workspace.setup(w => w.getWorkspaceFolder(TypeMoq.It.isAny())).returns(() => undefined); workspace.setup(w => w.workspaceFolders).returns(() => [workspaceFolder]); - const expectedTerminalArgs = terminalArgs.concat(path.join(workspaceUri.fsPath, 'manage.py'), 'shell'); + const expectedTerminalArgs = terminalArgs.concat(path.join(workspaceUri.fsPath, 'manage.py').fileToCommandArgument(), 'shell'); testReplCommandArguments(true, pythonPath, pythonPath, terminalArgs, expectedTerminalArgs, Uri.file('x')); }); diff --git a/src/test/terminals/codeExecution/terminalCodeExec.test.ts b/src/test/terminals/codeExecution/terminalCodeExec.test.ts index 90c92560ee20..0a907c1ce548 100644 --- a/src/test/terminals/codeExecution/terminalCodeExec.test.ts +++ b/src/test/terminals/codeExecution/terminalCodeExec.test.ts @@ -154,7 +154,8 @@ suite('Terminal Code Execution', () => { terminalSettings.setup(t => t.launchArgs).returns(() => []); await executor.executeFile(file); - terminalService.verify(async t => await t.sendText(TypeMoq.It.isValue(`cd "${path.dirname(file.fsPath)}"`)), TypeMoq.Times.once()); + const dir = `"${path.dirname(file.fsPath)}"`.fileToCommandArgument(); + terminalService.verify(async t => await t.sendText(TypeMoq.It.isValue(`cd ${dir}`)), TypeMoq.Times.once()); } test('Ensure we set current directory (and quote it when containing spaces) before executing file (non windows)', async () => { @@ -213,7 +214,7 @@ suite('Terminal Code Execution', () => { await executor.executeFile(file); const expectedPythonPath = isWindows ? pythonPath.replace(/\\/g, '/') : pythonPath; - const expectedArgs = terminalArgs.concat(file.fsPath.indexOf(' ') > 0 ? `"${file.fsPath}"` : file.fsPath); + const expectedArgs = terminalArgs.concat(file.fsPath.fileToCommandArgument()); terminalService.verify(async t => await t.sendCommand(TypeMoq.It.isValue(expectedPythonPath), TypeMoq.It.isValue(expectedArgs)), TypeMoq.Times.once()); } diff --git a/src/test/unittests/debugger.test.ts b/src/test/unittests/debugger.test.ts index 2daba9a848e1..c526961527a9 100644 --- a/src/test/unittests/debugger.test.ts +++ b/src/test/unittests/debugger.test.ts @@ -71,15 +71,30 @@ suite('Unit Tests - debugging', () => { assert.equal(tests.testFunctions.length, 2, 'Incorrect number of test functions'); assert.equal(tests.testSuites.length, 2, 'Incorrect number of test suites'); + const deferred = createDeferred(); const testFunction = [tests.testFunctions[0].testFunction]; - testManager.runTest(CommandSource.commandPalette, { testFunction }, false, true); - const launched = await mockDebugLauncher.launched; - assert.isTrue(launched, 'Debugger not launched'); + const runningPromise = testManager.runTest(CommandSource.commandPalette, { testFunction }, false, true); + + // This promise should never resolve nor reject. + runningPromise + .then(() => deferred.reject('Debugger stopped when it shouldn\'t have')) + .catch(error => deferred.reject(error)); + + mockDebugLauncher.launched + .then((launched) => { + if (launched) { + deferred.resolve(''); + } else { + deferred.reject('Debugger not launched'); + } + }) .catch(error => deferred.reject(error)); + + await deferred.promise; } test('Debugger should start (unittest)', async () => { await updateSetting('unitTest.unittestArgs', ['-s=./tests', '-p=test_*.py'], rootWorkspaceUri, configTarget); - await testStartingDebugger('unittest'); + await testStartingDebugger('unittest'); }); test('Debugger should start (pytest)', async () => { @@ -105,9 +120,10 @@ suite('Unit Tests - debugging', () => { const launched = await mockDebugLauncher.launched; assert.isTrue(launched, 'Debugger not launched'); - testManager.discoverTests(CommandSource.commandPalette, true, true, true); - + const discoveryPromise = testManager.discoverTests(CommandSource.commandPalette, true, true, true); await expect(runningPromise).to.be.rejectedWith(CANCELLATION_REASON, 'Incorrect reason for ending the debugger'); + ioc.dispose(); // will cancel test discovery + await expect(discoveryPromise).to.be.rejectedWith(CANCELLATION_REASON, 'Incorrect reason for ending the debugger'); } test('Debugger should stop when user invokes a test discovery (unittest)', async () => { @@ -151,6 +167,7 @@ suite('Unit Tests - debugging', () => { runningPromise .then(() => 'Debugger stopped when it shouldn\'t have') .catch(() => 'Debugger crashed when it shouldn\'t have') + // tslint:disable-next-line: no-floating-promises .then(error => { deferred.reject(error); }); diff --git a/src/test/unittests/stoppingDiscoverAndTest.test.ts b/src/test/unittests/stoppingDiscoverAndTest.test.ts index 3386ee2b6955..3b3558f12bd2 100644 --- a/src/test/unittests/stoppingDiscoverAndTest.test.ts +++ b/src/test/unittests/stoppingDiscoverAndTest.test.ts @@ -5,6 +5,7 @@ import { expect, use } from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; import * as path from 'path'; import { Uri } from 'vscode'; +import {createDeferred} from '../../client/common/helpers'; import { Product } from '../../client/common/types'; import { CANCELLATION_REASON, CommandSource, UNITTEST_PROVIDER } from '../../client/unittests/common/constants'; import { ITestDiscoveryService } from '../../client/unittests/common/types'; @@ -60,9 +61,23 @@ suite('Unit Tests Stopping Discovery and Runner', () => { const discoveryPromise = mockTestManager.discoverTests(CommandSource.auto); mockTestManager.discoveryDeferred.resolve(EmptyTests); - mockTestManager.runTest(CommandSource.ui); + const runningPromise = mockTestManager.runTest(CommandSource.ui); + const deferred = createDeferred(); - await expect(discoveryPromise).to.eventually.equal(EmptyTests); + // This promise should never resolve nor reject. + runningPromise + .then(() => Promise.reject('Debugger stopped when it shouldn\'t have')) + .catch(error => deferred.reject(error)); + + discoveryPromise.then(result => { + if (result === EmptyTests) { + deferred.resolve(''); + } else { + deferred.reject('tests not empty'); + } + }).catch(error => deferred.reject(error)); + + await deferred.promise; }); test('Discovering tests should stop running tests', async () => { @@ -75,7 +90,7 @@ suite('Unit Tests Stopping Discovery and Runner', () => { await new Promise(resolve => setTimeout(resolve, 1000)); // User manually discovering tests will kill the existing test runner. - mockTestManager.discoverTests(CommandSource.ui, true, false, true); + await mockTestManager.discoverTests(CommandSource.ui, true, false, true); await expect(runPromise).to.eventually.be.rejectedWith(CANCELLATION_REASON); }); }); From 3a0cfb1dca529aee3d3e3f474d00f10c74c5f592 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 7 Feb 2018 13:51:15 -0800 Subject: [PATCH 48/49] no message --- CONTRIBUTING.md | 56 ++-- src/client/common/extensions.ts | 18 +- .../common/markdown/restTextConverter.ts | 230 ++++++++------ src/client/common/terminal/helper.ts | 2 +- src/client/providers/itemInfoSource.ts | 1 + .../codeExecution/codeExecutionManager.ts | 5 +- .../codeExecution/djangoShellCodeExecution.ts | 2 +- .../codeExecution/terminalCodeExecution.ts | 4 +- src/client/unittests/main.ts | 2 +- src/test/common/extensions.test.ts | 42 +++ .../common/terminals/activation.bash.test.ts | 6 +- .../activation.commandPrompt.test.ts | 288 +++++++++--------- src/test/common/terminals/helper.test.ts | 4 +- .../interpreters/condaEnvFileService.test.ts | 11 +- src/test/language/tokenizer.test.ts | 7 + src/test/markdown/restTextConverter.test.ts | 7 +- src/test/pythonFiles/markdown/aifc.md | 142 +++++++++ src/test/pythonFiles/markdown/aifc.pydoc | 134 ++++++++ src/test/pythonFiles/markdown/anydbm.md | 33 ++ src/test/pythonFiles/markdown/anydbm.pydoc | 33 ++ src/test/pythonFiles/markdown/astroid.md | 24 ++ src/test/pythonFiles/markdown/astroid.pydoc | 23 ++ src/test/pythonFiles/markdown/scipy.md | 47 +++ src/test/pythonFiles/markdown/scipy.pydoc | 53 ++++ .../markdown/scipy.spatial.distance.md | 66 ++-- .../pythonFiles/markdown/scipy.spatial.md | 65 ++++ .../pythonFiles/markdown/scipy.spatial.pydoc | 86 ++++++ .../extension.refactor.extract.var.test.ts | 4 +- .../djangoShellCodeExect.test.ts | 2 +- .../codeExecution/terminalCodeExec.test.ts | 5 +- src/test/unittests/debugger.test.ts | 29 +- .../unittests/stoppingDiscoverAndTest.test.ts | 21 +- 32 files changed, 1114 insertions(+), 338 deletions(-) create mode 100644 src/test/common/extensions.test.ts create mode 100644 src/test/pythonFiles/markdown/aifc.md create mode 100644 src/test/pythonFiles/markdown/aifc.pydoc create mode 100644 src/test/pythonFiles/markdown/anydbm.md create mode 100644 src/test/pythonFiles/markdown/anydbm.pydoc create mode 100644 src/test/pythonFiles/markdown/astroid.md create mode 100644 src/test/pythonFiles/markdown/astroid.pydoc create mode 100644 src/test/pythonFiles/markdown/scipy.md create mode 100644 src/test/pythonFiles/markdown/scipy.pydoc create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.md create mode 100644 src/test/pythonFiles/markdown/scipy.spatial.pydoc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04abf6a0df58..2d5b4456385a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,7 +93,7 @@ for a release are tracked in a [milestone](https://github.com/Microsoft/vscode-python/milestones) (which is actively updated as plans change). -The overall schedule for a release is to code freeze for on the last +The overall schedule for a release is to feature freeze for on the last Monday of the month to coincide with Visual Studio Code's code freeze. We then aim to release later that week so the latest version of the extension is already live by the time Visual Studio Code launches @@ -104,16 +104,10 @@ between scheduled releases as necessary. All development is actively done in the `master` branch of the repository. It is what allows us to have an [insiders build](#insiders-build) which is expected to be stable at -all times. We do keep the previous release as a branch in case the +all times. We do keep the most recent release as a branch in case the need for a bugfix release arises. But once a new release is made we -convert the older branch into a tag and delete the branch as -Visual Studio Code's automatic updating makes keeping old versions -around unnecessary. - -Since we try to spend about 25% of our development time fixing bugs -and removing technical debt, the week of a release is mostly spent -focusing on that topic. That way we don't ignore the health of the -code base by accidentally focusing on new features exclusively. +delete the older release branch (all releases are appropriately +tagged, so history is lost). ### Issue triaging @@ -142,21 +136,17 @@ lexicographically sort from earliest stage to latest stage). The suffix term for each label then specifies what is currently blocking the issue from being closed. -* `1-` - * [`decision`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-decision): - The issue is a feature enhancement request and a decision has not - been made as to whether we would accept a pull request - implementing the enhancement - * [`more info`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-more%20info): - We need more information from the OP (original poster) - * [`verification`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-verification): - We need to verify that the issue can be replicated +* [`1-decision`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-decision): +The issue is a feature enhancement request and a decision has not +been made as to whether we would accept a pull request +implementing the enhancement +* [`1-more info`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-more%20info): +We need more information from the OP (original poster) +* [`1-verification`](https://github.com/Microsoft/vscode-python/labels/awaiting%201-verification): + We need to verify that the issue is reproducible * [`2-PR`](https://github.com/Microsoft/vscode-python/labels/awaiting%202-PR): The issue is valid and is now awaiting a fix to be created and merged into the `master` branch -* [`4-validation`](https://github.com/Microsoft/vscode-python/labels/awaiting%204-validation): - A pull request has been merged and resolution of the issue should be - independently validated #### Closed issues @@ -168,22 +158,21 @@ it should have an appropriate `closed-` label. 1. Check that there is an issue corresponding to what the pull request is attempting to address - * If an issue exists, make sure it has reached the stage of being - labeled `awaiting 2-PR` + * If an issue exists, make sure it has reached the stage of + `awaiting 2-PR` * If no issue exists, open one and wait for it to reach the `awaiting 2-PR` stage before submitting the pull request -1. Open the pull request, mentioning the appropriate issue(s) in the +1. Create the pull request, mentioning the appropriate issue(s) in the pull request message body * The pull request is expected to have appropriate unit tests * The pull request must pass its CI run before merging will be considered - * Code coverage is expected to not worsen + * Code coverage is expected to (at minimum) not worsen 1. Make sure all status checks are green (e.g. CLA check, CI, etc.) 1. Address any review comments 1. [Maintainers only] Merge the pull request 1. [Maintainers only] Update affected issues to be: 1. Closed (with an appropriate `closed-` label) - 1. The stage is set to `awaiting 4-validation` 1. The issue(s) are attached to the current milestone 1. Register OSS usage 1. Email CELA about any 3rd-party usage changes @@ -194,11 +183,12 @@ Starting in 2018, the extension switched to [calendar versioning](http://calver.org/) since the extension auto-updates and thus there is no need to track its version number for backwards-compatibility. As such, the major version -is the current year, the minor version is the current month, and -the micro version is how many releases there have been that month in -the year (starting at 0). For example, the first release in July 2018 -would be `2018.7.0`, the second release that month would be -`2018.7.1`, etc. +is the current year, the minor version is the month when feature +freeze was reached, and the micro version is how many releases there +have been since that feature freeze (starting at 0). For example +the release made when we reach feature freeze in July 2018 +would be `2018.7.0`, and if a second release was necessary to fix a +critical bug it would be `2018.7.1`. ## Insiders Build @@ -214,7 +204,7 @@ file, please follow the instructions on [this page](https://code.visualstudio.com/docs/editor/extension-gallery#_install-from-a-vsix) to install the extension. -The insiders build of the extension ...: +The insiders build of the extension: * Will be replaced with new releases published onto the [VS Code Marketplace](https://marketplace.visualstudio.com/VSCode). diff --git a/src/client/common/extensions.ts b/src/client/common/extensions.ts index 629b809e8d06..71858e6e01da 100644 --- a/src/client/common/extensions.ts +++ b/src/client/common/extensions.ts @@ -20,6 +20,11 @@ interface String { * E.g. if an argument contains a space, then it will be enclosed within double quotes. */ toCommandArgument(): string; + /** + * Appropriately formats a a file path so it can be used as an argument for a command in a shell. + * E.g. if an argument contains a space, then it will be enclosed within double quotes. + */ + fileToCommandArgument(): string; } /** @@ -47,5 +52,16 @@ String.prototype.toCommandArgument = function (this: string): string { if (!this) { return this; } - return (this.indexOf(' ') > 0 && !this.startsWith('"') && !this.endsWith('"')) ? `"${this}"` : this.toString(); + return (this.indexOf(' ') >= 0 && !this.startsWith('"') && !this.endsWith('"')) ? `"${this}"` : this.toString(); +}; + +/** + * Appropriately formats a a file path so it can be used as an argument for a command in a shell. + * E.g. if an argument contains a space, then it will be enclosed within double quotes. + */ +String.prototype.fileToCommandArgument = function (this: string): string { + if (!this) { + return this; + } + return this.toCommandArgument().replace(/\\/g, '/'); }; diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts index 485801a35f49..e606fd46bfbc 100644 --- a/src/client/common/markdown/restTextConverter.ts +++ b/src/client/common/markdown/restTextConverter.ts @@ -2,11 +2,18 @@ // Licensed under the MIT License. import { EOL } from 'os'; -import { isWhiteSpace } from '../../language/characters'; +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { isDecimal, isWhiteSpace } from '../../language/characters'; + +enum State { + Default, + Preformatted, + Code +} export class RestTextConverter { - private inPreBlock = false; - private inCodeBlock = false; + private state: State = State.Default; private md: string[] = []; // tslint:disable-next-line:cyclomatic-complexity @@ -16,14 +23,8 @@ export class RestTextConverter { // and documentation in the completion list. // See https://en.wikipedia.org/wiki/ReStructuredText - // Determine if this is actually a reStructruredText. - if (docstring.indexOf('::') < 0 && docstring.indexOf('..')) { - // If documentation contains markdown symbols such as ** (power of) in code, escape them. - return this.escapeMarkdown(docstring); - } const result = this.transformLines(docstring); - - this.inPreBlock = this.inPreBlock = false; + this.state = State.Default; this.md = []; return result; @@ -33,106 +34,150 @@ export class RestTextConverter { // Not complete escape list so it does not interfere // with subsequent code highlighting (see above). return text - .replace(/\\/g, '\\\\') - .replace(/\*/g, '\\*') - .replace(/\_/g, '\\_') - .replace(/\{/g, '\\{') - .replace(/\}/g, '\\}') - .replace(/\[/g, '\\[') - .replace(/\]/g, '\\]') - .replace(/\(/g, '\\(') - .replace(/\)/g, '\\)') .replace(/\#/g, '\\#') - .replace(/\+/g, '\\+') - .replace(/\-/g, '\\-') - .replace(/\!/g, '\\!'); + .replace(/\*/g, '\\*') + .replace(/\_/g, '\\_'); } - // tslint:disable-next-line:cyclomatic-complexity private transformLines(docstring: string): string { const lines = docstring.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { - let line = lines[i]; - + const line = lines[i]; // Avoid leading empty lines if (this.md.length === 0 && line.length === 0) { continue; } - if (this.handleCodeBlock(line)) { - continue; + switch (this.state) { + case State.Default: + i += this.inDefaultState(lines, i); + break; + case State.Preformatted: + i += this.inPreformattedState(lines, i); + break; + case State.Code: + this.inCodeState(line); + break; + default: + break; } + } - if (this.inPreBlock) { - // Preformatted block terminates by a line without leading - // whitespace or any special line like ..ABC::. - if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0))) { - this.endPreformattedBlock(); - } - } + this.endCodeBlock(); + this.endPreformattedBlock(); - if (this.handleSectionHeader(lines, i)) { - i += 1; // Eat line with === or --- - continue; - } + return this.md.join(EOL).trim(); + } - if (line.indexOf('generated/') >= 0) { - continue; // ignore generated content. - } - if (line.startsWith('===') || line.startsWith('---')) { - continue; // Eat standalone === or --- lines. - } + private inDefaultState(lines: string[], i: number): number { + let line = lines[i]; + if (line.startsWith('```')) { + this.startCodeBlock(); + return 0; + } - if (this.handleDoubleColon(line)) { - continue; - } - if (line.startsWith('..') && line.indexOf('::') > 0) { - // Ignore lines likes .. sectionauthor:: John Doe. - continue; - } + if (line.startsWith('===') || line.startsWith('---')) { + return 0; // Eat standalone === or --- lines. + } + if (this.handleDoubleColon(line)) { + return 0; + } + if (this.isIgnorable(line)) { + return 0; + } - line = this.convertEmphasis(line); - line = line.replace(/``/g, '`'); // Convert double backticks to single. + if (this.handleSectionHeader(lines, i)) { + return 1; // Eat line with === or --- + } - if (line.length > 0 && isWhiteSpace(line.charCodeAt(0))) { - // Keep hard line breaks for the pre-indented content. - line = ` ${line} `; - } + const result = this.checkPreContent(lines, i); + if (this.state !== State.Default) { + return result; // Handle line in the new state + } - const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; - if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { - continue; // Avoid more than one empty line in a row. - } + line = this.cleanup(line); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + line = this.escapeMarkdown(line); + this.md.push(line); - this.md.push(line); + return 0; + } + + private inPreformattedState(lines: string[], i: number): number { + let line = lines[i]; + if (this.isIgnorable(line)) { + return 0; + } + // Preformatted block terminates by a line without leading whitespace. + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + this.endPreformattedBlock(); + return -1; } - this.tryEndCodePreBlocks(); - return this.md.join(EOL).trim(); + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return 0; // Avoid more than one empty line in a row. + } + + // Since we use HTML blocks as preformatted text + // make sure we drop angle brackets since otherwise + // they will render as tags and attributes + line = line.replace(//g, ' '); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + // Keep hard line breaks for the preformatted content + this.md.push(`${line} `); + return 0; } - private handleCodeBlock(line: string): boolean { - if (!line.startsWith('```')) { - return false; + private inCodeState(line: string): void { + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return; // Avoid more than one empty line in a row. } - if (this.inCodeBlock) { + + if (line.startsWith('```')) { this.endCodeBlock(); } else { - this.startCodeBlock(); + this.md.push(line); } - return true; + } + + private isIgnorable(line: string): boolean { + if (line.indexOf('generated/') >= 0) { + return true; // Drop generated content. + } + const trimmed = line.trim(); + if (trimmed.startsWith('..') && trimmed.indexOf('::') > 0) { + // Ignore lines likes .. sectionauthor:: John Doe. + return true; + } + return false; + } + + private checkPreContent(lines: string[], i: number): number { + const line = lines[i]; + if (i === 0 || line.trim().length === 0) { + return 0; + } + + if (!isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + return 0; // regular line, nothing to do here. + } + // Indented content is considered to be preformatted. + this.startPreformattedBlock(); + return -1; } private handleSectionHeader(lines: string[], i: number): boolean { const line = lines[i]; if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { // Section title -> heading level 3. - this.md.push(`### ${this.convertEmphasis(line)}`); + this.md.push(`### ${this.cleanup(line)}`); return true; } if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { // Subsection title -> heading level 4. - this.md.push(`#### ${this.convertEmphasis(line)}`); + this.md.push(`#### ${this.cleanup(line)}`); return true; } return false; @@ -154,53 +199,52 @@ export class RestTextConverter { return true; } - private tryEndCodePreBlocks(): void { - if (this.inCodeBlock) { - this.endCodeBlock(); - } - if (this.inPreBlock) { - this.endPreformattedBlock(); - } - } - private startPreformattedBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); // Lie about the language since we don't want preformatted text // to be colorized as Python. HTML is more 'appropriate' as it does // not colorize -- or + or keywords like 'from'. this.md.push('```html'); - this.inPreBlock = true; + this.state = State.Preformatted; } private endPreformattedBlock(): void { - if (this.inPreBlock) { + if (this.state === State.Preformatted) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inPreBlock = false; + this.state = State.Default; } } private startCodeBlock(): void { // Remove previous empty line so we avoid double empties. - this.tryRemovePrecedingEmptyLine(); + this.tryRemovePrecedingEmptyLines(); this.md.push('```python'); - this.inCodeBlock = true; + this.state = State.Code; } private endCodeBlock(): void { - if (this.inCodeBlock) { + if (this.state === State.Code) { + this.tryRemovePrecedingEmptyLines(); this.md.push('```'); - this.inCodeBlock = false; + this.state = State.Default; } } - private tryRemovePrecedingEmptyLine(): void { - if (this.md.length > 0 && this.md[this.md.length - 1].length === 0) { + private tryRemovePrecedingEmptyLines(): void { + while (this.md.length > 0 && this.md[this.md.length - 1].trim().length === 0) { this.md.pop(); } } - private convertEmphasis(line: string): string { - return line.replace(/\:([\w\W]+)\:/g, '**$1**'); // Convert :word: to **word**. + private isListItem(line: string): boolean { + const trimmed = line.trim(); + const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0; + return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch); + } + + private cleanup(line: string): string { + return line.replace(/:mod:/g, 'module:'); } } diff --git a/src/client/common/terminal/helper.ts b/src/client/common/terminal/helper.ts index 07c679b86f4f..bc2f9c78a237 100644 --- a/src/client/common/terminal/helper.ts +++ b/src/client/common/terminal/helper.ts @@ -68,7 +68,7 @@ export class TerminalHelper implements ITerminalHelper { public buildCommandForTerminal(terminalShellType: TerminalShellType, command: string, args: string[]) { const isPowershell = terminalShellType === TerminalShellType.powershell || terminalShellType === TerminalShellType.powershellCore; const commandPrefix = isPowershell ? '& ' : ''; - return `${commandPrefix}${command.toCommandArgument()} ${args.join(' ')}`.trim(); + return `${commandPrefix}${command.fileToCommandArgument()} ${args.join(' ')}`.trim(); } public async getEnvironmentActivationCommands(terminalShellType: TerminalShellType, resource?: Uri): Promise { const settings = this.serviceContainer.get(IConfigurationService).getSettings(resource); diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index db147dbfd431..4159e9396f82 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -108,6 +108,7 @@ export class ItemInfoSource { if (signature.length > 0) { tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); } + const description = this.textConverter.toMarkdown(lines.join(EOL)); tooltip = tooltip.appendMarkdown(description); diff --git a/src/client/terminals/codeExecution/codeExecutionManager.ts b/src/client/terminals/codeExecution/codeExecutionManager.ts index fd967c56318f..5a6159a50aae 100644 --- a/src/client/terminals/codeExecution/codeExecutionManager.ts +++ b/src/client/terminals/codeExecution/codeExecutionManager.ts @@ -15,7 +15,7 @@ import { ICodeExecutionHelper, ICodeExecutionManager, ICodeExecutionService } fr @injectable() export class CodeExecutionManager implements ICodeExecutionManager { - constructor( @inject(ICommandManager) private commandManager: ICommandManager, + constructor(@inject(ICommandManager) private commandManager: ICommandManager, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IDisposableRegistry) private disposableRegistry: Disposable[], @inject(IServiceContainer) private serviceContainer: IServiceContainer) { @@ -28,8 +28,9 @@ export class CodeExecutionManager implements ICodeExecutionManager { this.disposableRegistry.push(this.commandManager.registerCommand(Commands.Exec_Selection_In_Django_Shell, this.executeSelectionInDjangoShell.bind(this))); } @captureTelemetry(EXECUTION_CODE, { scope: 'file' }, false) - private async executeFileInterTerminal(file: Uri) { + private async executeFileInterTerminal(file?: Uri) { const codeExecutionHelper = this.serviceContainer.get(ICodeExecutionHelper); + file = file instanceof Uri ? file : undefined; const fileToExecute = file ? file : await codeExecutionHelper.getFileToExecute(); if (!fileToExecute) { return; diff --git a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts index d188a2091547..5fbe2ef2d19f 100644 --- a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts +++ b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts @@ -40,7 +40,7 @@ export class DjangoShellCodeExecutionProvider extends TerminalCodeExecutionProvi const workspaceRoot = workspaceUri ? workspaceUri.uri.fsPath : defaultWorkspace; const managePyPath = workspaceRoot.length === 0 ? 'manage.py' : path.join(workspaceRoot, 'manage.py'); - args.push(managePyPath.toCommandArgument()); + args.push(managePyPath.fileToCommandArgument()); args.push('shell'); return { command, args }; } diff --git a/src/client/terminals/codeExecution/terminalCodeExecution.ts b/src/client/terminals/codeExecution/terminalCodeExecution.ts index a3aaedcc2584..4ed1d7da479c 100644 --- a/src/client/terminals/codeExecution/terminalCodeExecution.ts +++ b/src/client/terminals/codeExecution/terminalCodeExecution.ts @@ -34,7 +34,7 @@ export class TerminalCodeExecutionProvider implements ICodeExecutionService { const command = this.platformService.isWindows ? pythonSettings.pythonPath.replace(/\\/g, '/') : pythonSettings.pythonPath; const launchArgs = pythonSettings.terminal.launchArgs; - await this.getTerminalService(file).sendCommand(command, launchArgs.concat(file.fsPath.toCommandArgument())); + await this.getTerminalService(file).sendCommand(command, launchArgs.concat(file.fsPath.fileToCommandArgument())); } public async execute(code: string, resource?: Uri): Promise { @@ -47,7 +47,7 @@ export class TerminalCodeExecutionProvider implements ICodeExecutionService { } public async initializeRepl(resource?: Uri) { if (this.replActive && await this.replActive!) { - this._terminalService!.show(); + await this._terminalService!.show(); return; } this.replActive = new Promise(async resolve => { diff --git a/src/client/unittests/main.ts b/src/client/unittests/main.ts index c95777f78bcc..2e386edea0b0 100644 --- a/src/client/unittests/main.ts +++ b/src/client/unittests/main.ts @@ -158,7 +158,7 @@ async function selectAndRunTestMethod(cmdSource: CommandSource, resource: Uri, d return; } // tslint:disable-next-line:prefer-type-cast - await runTestsImpl(cmdSource, testManager.workspaceFolder, { testFunction: [selectedTestFn.testFunction] } as TestsToRun, debug); + await runTestsImpl(cmdSource, testManager.workspaceFolder, { testFunction: [selectedTestFn.testFunction] } as TestsToRun, false, debug); } async function selectAndRunTestFile(cmdSource: CommandSource) { const testManager = await getTestManager(true); diff --git a/src/test/common/extensions.test.ts b/src/test/common/extensions.test.ts new file mode 100644 index 000000000000..5724f3291274 --- /dev/null +++ b/src/test/common/extensions.test.ts @@ -0,0 +1,42 @@ +import { expect } from 'chai'; +import '../../client/common/extensions'; + +// Defines a Mocha test suite to group tests of similar kind together +suite('String Extensions', () => { + test('Should return empty string for empty arg', () => { + const argTotest = ''; + expect(argTotest.toCommandArgument()).to.be.equal(''); + }); + test('Should quote an empty space', () => { + const argTotest = ' '; + expect(argTotest.toCommandArgument()).to.be.equal('" "'); + }); + test('Should not quote command arguments without spaces', () => { + const argTotest = 'one.two.three'; + expect(argTotest.toCommandArgument()).to.be.equal(argTotest); + }); + test('Should quote command arguments with spaces', () => { + const argTotest = 'one two three'; + expect(argTotest.toCommandArgument()).to.be.equal(`"${argTotest}"`); + }); + test('Should return empty string for empty path', () => { + const fileToTest = ''; + expect(fileToTest.fileToCommandArgument()).to.be.equal(''); + }); + test('Should not quote file argument without spaces', () => { + const fileToTest = 'users/test/one'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(fileToTest); + }); + test('Should quote file argument with spaces', () => { + const fileToTest = 'one two three'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(`"${fileToTest}"`); + }); + test('Should replace all back slashes with forward slashes (irrespective of OS)', () => { + const fileToTest = 'c:\\users\\user\\conda\\scripts\\python.exe'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(fileToTest.replace(/\\/g, '/')); + }); + test('Should replace all back slashes with forward slashes (irrespective of OS) and quoted when file has spaces', () => { + const fileToTest = 'c:\\users\\user namne\\conda path\\scripts\\python.exe'; + expect(fileToTest.fileToCommandArgument()).to.be.equal(`"${fileToTest.replace(/\\/g, '/')}"`); + }); +}); diff --git a/src/test/common/terminals/activation.bash.test.ts b/src/test/common/terminals/activation.bash.test.ts index ee7d2829ea46..c321528140ea 100644 --- a/src/test/common/terminals/activation.bash.test.ts +++ b/src/test/common/terminals/activation.bash.test.ts @@ -5,6 +5,7 @@ import { expect } from 'chai'; import * as path from 'path'; import * as TypeMoq from 'typemoq'; import { EnumEx } from '../../../client/common/enumUtils'; +import '../../../client/common/extensions'; import { IFileSystem } from '../../../client/common/platform/types'; import { Bash } from '../../../client/common/terminal/environmentActivationProviders/bash'; import { TerminalShellType } from '../../../client/common/terminal/types'; @@ -13,7 +14,7 @@ import { IServiceContainer } from '../../../client/ioc/types'; // tslint:disable-next-line:max-func-body-length suite('Terminal Environment Activation (bash)', () => { - ['usr/bin/python', 'usr/bin/env with spaces/env more/python'].forEach(pythonPath => { + ['usr/bin/python', 'usr/bin/env with spaces/env more/python', 'c:\\users\\windows paths\\conda\\python.exe'].forEach(pythonPath => { const hasSpaces = pythonPath.indexOf(' ') > 0; const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; suite(suiteTitle, () => { @@ -83,8 +84,7 @@ suite('Terminal Environment Activation (bash)', () => { // Ensure the path is quoted if it contains any spaces. // Ensure it contains the name of the environment as an argument to the script file. - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`source ${quotedScriptFile}`.trim()], 'Invalid command'); + expect(command).to.be.deep.equal([`source ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); } else { expect(command).to.be.equal(undefined, 'Command should be undefined'); } diff --git a/src/test/common/terminals/activation.commandPrompt.test.ts b/src/test/common/terminals/activation.commandPrompt.test.ts index 47b610ea0997..f6c7509ce00d 100644 --- a/src/test/common/terminals/activation.commandPrompt.test.ts +++ b/src/test/common/terminals/activation.commandPrompt.test.ts @@ -15,20 +15,64 @@ import { IConfigurationService, IPythonSettings } from '../../../client/common/t import { IServiceContainer } from '../../../client/ioc/types'; suite('Terminal Environment Activation (cmd/powershell)', () => { - ['c:/programfiles/python/python', 'c:/program files/python/python'].forEach(pythonPath => { - const hasSpaces = pythonPath.indexOf(' ') > 0; - const resource = Uri.file('a'); - - const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; - suite(suiteTitle, () => { - ['activate', 'activate.sh', 'activate.csh', 'activate.fish', 'activate.bat', 'activate.ps1'].forEach(scriptFileName => { - suite(`and script file is ${scriptFileName}`, () => { + ['c:/programfiles/python/python', 'c:/program files/python/python', + 'c:\\users\\windows paths\\conda\\python.exe'].forEach(pythonPath => { + const hasSpaces = pythonPath.indexOf(' ') > 0; + const resource = Uri.file('a'); + + const suiteTitle = hasSpaces ? 'and there are spaces in the script file (pythonpath),' : 'and there are no spaces in the script file (pythonpath),'; + suite(suiteTitle, () => { + ['activate', 'activate.sh', 'activate.csh', 'activate.fish', 'activate.bat', 'activate.ps1'].forEach(scriptFileName => { + suite(`and script file is ${scriptFileName}`, () => { + let serviceContainer: TypeMoq.IMock; + let fileSystem: TypeMoq.IMock; + setup(() => { + serviceContainer = TypeMoq.Mock.ofType(); + fileSystem = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + + const configService = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); + const settings = TypeMoq.Mock.ofType(); + settings.setup(s => s.pythonPath).returns(() => pythonPath); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); + }); + + EnumEx.getNamesAndValues(TerminalShellType).forEach(shellType => { + const isScriptFileSupported = ['activate.bat', 'activate.ps1'].indexOf(scriptFileName) >= 0; + const titleTitle = isScriptFileSupported ? `Ensure terminal type is supported (Shell: ${shellType.name})` : + `Ensure terminal type is not supported (Shell: ${shellType.name})`; + + test(titleTitle, async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); + + const supported = bash.isShellSupported(shellType.value); + switch (shellType.value) { + case TerminalShellType.commandPrompt: + case TerminalShellType.powershellCore: + case TerminalShellType.powershell: { + expect(supported).to.be.equal(true, `${shellType.name} shell not supported (it should be)`); + break; + } + default: { + expect(supported).to.be.equal(false, `${shellType.name} incorrectly supported (should not be)`); + } + } + }); + }); + }); + }); + + suite('and script file is activate.bat', () => { let serviceContainer: TypeMoq.IMock; let fileSystem: TypeMoq.IMock; + let platform: TypeMoq.IMock; setup(() => { serviceContainer = TypeMoq.Mock.ofType(); fileSystem = TypeMoq.Mock.ofType(); + platform = TypeMoq.Mock.ofType(); serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); const configService = TypeMoq.Mock.ofType(); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); @@ -37,173 +81,125 @@ suite('Terminal Environment Activation (cmd/powershell)', () => { configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); }); - EnumEx.getNamesAndValues(TerminalShellType).forEach(shellType => { - const isScriptFileSupported = ['activate.bat', 'activate.ps1'].indexOf(scriptFileName) >= 0; - const titleTitle = isScriptFileSupported ? `Ensure terminal type is supported (Shell: ${shellType.name})` : - `Ensure terminal type is not supported (Shell: ${shellType.name})`; - - test(titleTitle, async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); - - const supported = bash.isShellSupported(shellType.value); - switch (shellType.value) { - case TerminalShellType.commandPrompt: - case TerminalShellType.powershellCore: - case TerminalShellType.powershell: { - expect(supported).to.be.equal(true, `${shellType.name} shell not supported (it should be)`); - break; - } - default: { - expect(supported).to.be.equal(false, `${shellType.name} incorrectly supported (should not be)`); - } - } - }); - }); - }); - }); - - suite('and script file is activate.bat', () => { - let serviceContainer: TypeMoq.IMock; - let fileSystem: TypeMoq.IMock; - let platform: TypeMoq.IMock; - setup(() => { - serviceContainer = TypeMoq.Mock.ofType(); - fileSystem = TypeMoq.Mock.ofType(); - platform = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); - serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); - - const configService = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); - const settings = TypeMoq.Mock.ofType(); - settings.setup(s => s.pythonPath).returns(() => pythonPath); - configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); - }); + test('Ensure batch files are supported by command prompt', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - test('Ensure batch files are supported by command prompt', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const commands = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const commands = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); + // Ensure the script file is of the following form: + // source "" + // Ensure the path is quoted if it contains any spaces. + // Ensure it contains the name of the environment as an argument to the script file. - // Ensure the script file is of the following form: - // source "" - // Ensure the path is quoted if it contains any spaces. - // Ensure it contains the name of the environment as an argument to the script file. - - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(commands).to.be.deep.equal([`${quotedScriptFile}`.trim()], 'Invalid command'); - }); + expect(commands).to.be.deep.equal([pathToScriptFile.fileToCommandArgument()], 'Invalid command'); + }); - test('Ensure batch files are supported by powershell (on windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are supported by powershell (on windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - // Executing batch files from powershell requires going back to cmd, then into powershell + // Executing batch files from powershell requires going back to cmd, then into powershell - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - const activationCommand = `${quotedScriptFile}`.trim(); - const commands = [`& cmd /k "${activationCommand} & powershell"`]; - expect(command).to.be.deep.equal(commands, 'Invalid command'); - }); + const activationCommand = pathToScriptFile.fileToCommandArgument(); + const commands = [`& cmd /k "${activationCommand} & powershell"`]; + expect(command).to.be.deep.equal(commands, 'Invalid command'); + }); - test('Ensure batch files are supported by powershell core (on windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are supported by powershell core (on windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - // Executing batch files from powershell requires going back to cmd, then into powershell + // Executing batch files from powershell requires going back to cmd, then into powershell - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - const activationCommand = `${quotedScriptFile}`.trim(); - const commands = [`& cmd /k "${activationCommand} & pwsh"`]; - expect(command).to.be.deep.equal(commands, 'Invalid command'); - }); + const activationCommand = pathToScriptFile.fileToCommandArgument(); + const commands = [`& cmd /k "${activationCommand} & pwsh"`]; + expect(command).to.be.deep.equal(commands, 'Invalid command'); + }); - test('Ensure batch files are not supported by powershell (on non-windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are not supported by powershell (on non-windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => false); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => false); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - expect(command).to.be.equal(undefined, 'Invalid command'); - }); + expect(command).to.be.equal(undefined, 'Invalid command'); + }); - test('Ensure batch files are not supported by powershell core (on non-windows)', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure batch files are not supported by powershell core (on non-windows)', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => false); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => false); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.bat'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - expect(command).to.be.equal(undefined, 'Invalid command'); + expect(command).to.be.equal(undefined, 'Invalid command'); + }); }); - }); - suite('and script file is activate.ps1', () => { - let serviceContainer: TypeMoq.IMock; - let fileSystem: TypeMoq.IMock; - let platform: TypeMoq.IMock; - setup(() => { - serviceContainer = TypeMoq.Mock.ofType(); - fileSystem = TypeMoq.Mock.ofType(); - platform = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); - serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); - - const configService = TypeMoq.Mock.ofType(); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); - const settings = TypeMoq.Mock.ofType(); - settings.setup(s => s.pythonPath).returns(() => pythonPath); - configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); - }); + suite('and script file is activate.ps1', () => { + let serviceContainer: TypeMoq.IMock; + let fileSystem: TypeMoq.IMock; + let platform: TypeMoq.IMock; + setup(() => { + serviceContainer = TypeMoq.Mock.ofType(); + fileSystem = TypeMoq.Mock.ofType(); + platform = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(IFileSystem)).returns(() => fileSystem.object); + serviceContainer.setup(c => c.get(IPlatformService)).returns(() => platform.object); + + const configService = TypeMoq.Mock.ofType(); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService))).returns(() => configService.object); + const settings = TypeMoq.Mock.ofType(); + settings.setup(s => s.pythonPath).returns(() => pythonPath); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => settings.object); + }); - test('Ensure powershell files are not supported by command prompt', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are not supported by command prompt', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.commandPrompt); - expect(command).to.be.deep.equal([], 'Invalid command (running powershell files are not supported on command prompt)'); - }); + expect(command).to.be.deep.equal([], 'Invalid command (running powershell files are not supported on command prompt)'); + }); - test('Ensure powershell files are supported by powershell', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are supported by powershell', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershell); - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`& ${quotedScriptFile}`.trim()], 'Invalid command'); - }); + expect(command).to.be.deep.equal([`& ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); + }); - test('Ensure powershell files are supported by powershell core', async () => { - const bash = new CommandPromptAndPowerShell(serviceContainer.object); + test('Ensure powershell files are supported by powershell core', async () => { + const bash = new CommandPromptAndPowerShell(serviceContainer.object); - platform.setup(p => p.isWindows).returns(() => true); - const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); - fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); - const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); + platform.setup(p => p.isWindows).returns(() => true); + const pathToScriptFile = path.join(path.dirname(pythonPath), 'activate.ps1'); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pathToScriptFile))).returns(() => Promise.resolve(true)); + const command = await bash.getActivationCommands(resource, TerminalShellType.powershellCore); - const quotedScriptFile = pathToScriptFile.indexOf(' ') > 0 ? `"${pathToScriptFile}"` : pathToScriptFile; - expect(command).to.be.deep.equal([`& ${quotedScriptFile}`.trim()], 'Invalid command'); + expect(command).to.be.deep.equal([`& ${pathToScriptFile.fileToCommandArgument()}`.trim()], 'Invalid command'); + }); }); }); }); - }); }); diff --git a/src/test/common/terminals/helper.test.ts b/src/test/common/terminals/helper.test.ts index dea22cd4eef5..e692f480a6ca 100644 --- a/src/test/common/terminals/helper.test.ts +++ b/src/test/common/terminals/helper.test.ts @@ -102,7 +102,7 @@ suite('Terminal Service helpers', () => { const command = 'c:\\python 3.7.exe'; const args = ['1', '2']; const commandPrefix = (item.value === TerminalShellType.powershell || item.value === TerminalShellType.powershellCore) ? '& ' : ''; - const expectedTerminalCommand = `${commandPrefix}"${command}" 1 2`; + const expectedTerminalCommand = `${commandPrefix}${command.fileToCommandArgument()} 1 2`; const terminalCommand = helper.buildCommandForTerminal(item.value, command, args); expect(terminalCommand).to.equal(expectedTerminalCommand, `Incorrect command for Shell ${item.name}`); @@ -126,7 +126,7 @@ suite('Terminal Service helpers', () => { const command = 'c:\\python 3.7.exe'; const args = []; const commandPrefix = (item.value === TerminalShellType.powershell || item.value === TerminalShellType.powershellCore) ? '& ' : ''; - const expectedTerminalCommand = `${commandPrefix}"${command}"`; + const expectedTerminalCommand = `${commandPrefix}${command.fileToCommandArgument()}`; const terminalCommand = helper.buildCommandForTerminal(item.value, command, args); expect(terminalCommand).to.equal(expectedTerminalCommand, `Incorrect command for Shell ${item.name}`); diff --git a/src/test/interpreters/condaEnvFileService.test.ts b/src/test/interpreters/condaEnvFileService.test.ts index 85844e19c022..2207be82732a 100644 --- a/src/test/interpreters/condaEnvFileService.test.ts +++ b/src/test/interpreters/condaEnvFileService.test.ts @@ -104,15 +104,20 @@ suite('Interpreters from Conda Environments Text File', () => { const interpreterPaths = [ path.join(environmentsPath, 'conda', 'envs', 'numpy') ]; + const pythonPath = path.join(interpreterPaths[0], 'pythonPath'); condaService.setup(c => c.condaEnvironmentsFile).returns(() => environmentsFilePath); + condaService.setup(c => c.getInterpreterPath(TypeMoq.It.isAny())).returns(() => pythonPath); + fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(pythonPath))).returns(() => Promise.resolve(true)); fileSystem.setup(fs => fs.fileExistsAsync(TypeMoq.It.isValue(environmentsFilePath))).returns(() => Promise.resolve(true)); fileSystem.setup(fs => fs.readFile(TypeMoq.It.isValue(environmentsFilePath))).returns(() => Promise.resolve(interpreterPaths.join(EOL))); - AnacondaCompanyNames.forEach(async companyDisplayName => { + for (const companyName of AnacondaCompanyNames) { + const versionWithCompanyName = `Mock Version :: ${companyName}`; + interpreterVersion.setup(c => c.getVersion(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve(versionWithCompanyName)); const interpreters = await condaFileProvider.getInterpreters(); assert.equal(interpreters.length, 1, 'Incorrect number of entries'); - assert.equal(interpreters[0].displayName, `${AnacondaDisplayName} Mock Version (numpy)`, 'Incorrect display name'); - }); + assert.equal(interpreters[0].displayName, `${AnacondaDisplayName} Mock Version`, 'Incorrect display name'); + } }); }); diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 727ce969dd09..86deb9282249 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -76,4 +76,11 @@ suite('Language.Tokenizer', () => { assert.equal(tokens.getItemAt(i).type, TokenType.Comment); } }); + test('Unknown token', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('.'); + assert.equal(tokens.count, 1); + + assert.equal(tokens.getItemAt(0).type, TokenType.Unknown); + }); }); diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 52cc7ea7c497..9b43d4d57657 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -15,7 +15,7 @@ function compareFiles(expectedContent: string, actualContent: string) { for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { const e = expectedLines[i]; const a = actualLines[i]; - expect(a, `Difference at line ${i}`).to.be.equal(e); + expect(e, `Difference at line ${i}`).to.be.equal(a); } expect(actualLines.length, @@ -36,5 +36,10 @@ async function testConversion(fileName: string): Promise { // tslint:disable-next-line:max-func-body-length suite('Hover - RestTextConverter', () => { + test('scipy', async () => await testConversion('scipy')); + test('scipy.spatial', async () => await testConversion('scipy.spatial')); test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); + test('anydbm', async () => await testConversion('anydbm')); + test('aifc', async () => await testConversion('aifc')); + test('astroid', async () => await testConversion('astroid')); }); diff --git a/src/test/pythonFiles/markdown/aifc.md b/src/test/pythonFiles/markdown/aifc.md new file mode 100644 index 000000000000..fff22dece1e5 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.md @@ -0,0 +1,142 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. +```html + +-----------------+ + | FORM | + +-----------------+ + | size | + +----+------------+ + | | AIFC | + | +------------+ + | | chunks | + | | . | + | | . | + | | . | + +----+------------+ +``` +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. +```html + FVER + version number of AIFF-C defining document (AIFF-C only). + MARK + # of markers (2 bytes) + list of markers: + marker ID (2 bytes, must be 0) + position (4 bytes) + marker name ("pstring") + COMM + # of channels (2 bytes) + # of sound frames (4 bytes) + size of the samples (2 bytes) + sampling frequency (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + compression type (4 bytes) + human-readable version of compression type ("pstring") + SSND + offset (4 bytes, not used by this program) + blocksize (4 bytes, not used by this program) + sound data +``` +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: +```html + f = aifc.open(file, 'r') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: +```html + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +``` +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: +```html + f = aifc.open(file, 'w') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: +```html + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +``` +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/aifc.pydoc b/src/test/pythonFiles/markdown/aifc.pydoc new file mode 100644 index 000000000000..a4cc346d5531 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.pydoc @@ -0,0 +1,134 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. + + +-----------------+ + | FORM | + +-----------------+ + | | + +----+------------+ + | | AIFC | + | +------------+ + | | | + | | . | + | | . | + | | . | + +----+------------+ + +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. + + FVER + (AIFF-C only). + MARK + <# of markers> (2 bytes) + list of markers: + (2 bytes, must be > 0) + (4 bytes) + ("pstring") + COMM + <# of channels> (2 bytes) + <# of sound frames> (4 bytes) + (2 bytes) + (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + (4 bytes) + ("pstring") + SSND + (4 bytes, not used by this program) + (4 bytes, not used by this program) + + +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: + f = aifc.open(file, 'r') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: + f = aifc.open(file, 'w') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.md b/src/test/pythonFiles/markdown/anydbm.md new file mode 100644 index 000000000000..e5914dcbadde --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.md @@ -0,0 +1,33 @@ +Generic interface to all dbm clones. + +Instead of +```html + import dbm + d = dbm.open(file, 'w', 0666) +``` +use +```html + import anydbm + d = anydbm.open(file, 'w') +``` +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): +```html + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) +``` +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.pydoc b/src/test/pythonFiles/markdown/anydbm.pydoc new file mode 100644 index 000000000000..2d46b5881789 --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.pydoc @@ -0,0 +1,33 @@ +Generic interface to all dbm clones. + +Instead of + + import dbm + d = dbm.open(file, 'w', 0666) + +use + + import anydbm + d = anydbm.open(file, 'w') + +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): + + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) + +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.md b/src/test/pythonFiles/markdown/astroid.md new file mode 100644 index 000000000000..b5ece21c1faf --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.md @@ -0,0 +1,24 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's \_ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: +```html +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.pydoc b/src/test/pythonFiles/markdown/astroid.pydoc new file mode 100644 index 000000000000..84d58487ead5 --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.pydoc @@ -0,0 +1,23 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: + +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.md b/src/test/pythonFiles/markdown/scipy.md new file mode 100644 index 000000000000..d28c1e290abe --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.md @@ -0,0 +1,47 @@ +### SciPy: A scientific computing package for Python + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +#### Contents +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +#### Subpackages +Using any of these subpackages requires an explicit import. For example, +`import scipy.cluster`. +```html + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions +``` +#### Utility tools +```html + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.pydoc b/src/test/pythonFiles/markdown/scipy.pydoc new file mode 100644 index 000000000000..293445fbea5b --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.pydoc @@ -0,0 +1,53 @@ +SciPy: A scientific computing package for Python +================================================ + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +Contents +-------- +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +Subpackages +----------- +Using any of these subpackages requires an explicit import. For example, +``import scipy.cluster``. + +:: + + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions + +Utility tools +------------- +:: + + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md index 125b19f6cdeb..276acddef787 100644 --- a/src/test/pythonFiles/markdown/scipy.spatial.distance.md +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -1,4 +1,4 @@ -### Distance computations (**mod**`scipy.spatial.distance`) +### Distance computations (module:`scipy.spatial.distance`) #### Function Reference @@ -6,53 +6,49 @@ Distance matrix computation from a collection of raw observation vectors stored in a rectangular array. ```html - pdist -- pairwise distances between observation vectors. - cdist -- distances between two collections of observation vectors - squareform -- convert distance matrix to a condensed one and vice versa - directed_hausdorff -- directed Hausdorff distance between arrays - + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays ``` Predicates for checking the validity of distance matrices, both condensed and redundant. Also contained in this module are functions for computing the number of observations in a distance matrix. ```html - is_valid_dm -- checks for a valid distance matrix - is_valid_y -- checks for a valid condensed distance matrix - num_obs_dm -- # of observations in a distance matrix - num_obs_y -- # of observations in a condensed distance matrix - + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix ``` Distance functions between two numeric vectors `u` and `v`. Computing distances over a large collection of vectors is inefficient for these functions. Use `pdist` for this purpose. ```html - braycurtis -- the Bray-Curtis distance. - canberra -- the Canberra distance. - chebyshev -- the Chebyshev distance. - cityblock -- the Manhattan distance. - correlation -- the Correlation distance. - cosine -- the Cosine distance. - euclidean -- the Euclidean distance. - mahalanobis -- the Mahalanobis distance. - minkowski -- the Minkowski distance. - seuclidean -- the normalized Euclidean distance. - sqeuclidean -- the squared Euclidean distance. - wminkowski -- (deprecated) alias of `minkowski`. - + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. ``` Distance functions between two boolean vectors (representing sets) `u` and `v`. As in the case of numerical vectors, `pdist` is more efficient for computing the distances between all pairs. ```html - dice -- the Dice dissimilarity. - hamming -- the Hamming distance. - jaccard -- the Jaccard distance. - kulsinski -- the Kulsinski distance. - rogerstanimoto -- the Rogers-Tanimoto dissimilarity. - russellrao -- the Russell-Rao dissimilarity. - sokalmichener -- the Sokal-Michener dissimilarity. - sokalsneath -- the Sokal-Sneath dissimilarity. - yule -- the Yule dissimilarity. - + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. ``` -**func**`hamming` also operates over discrete numerical vectors. \ No newline at end of file +:func:`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md new file mode 100644 index 000000000000..2d5e891db625 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -0,0 +1,65 @@ +### Spatial algorithms and data structures (module:`scipy.spatial`) + + +### Nearest-neighbor Queries +```html + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle +``` +### Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +```html + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces +``` +### Plotting Helpers +```html + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram +``` +### Simplex representation +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme: +```html + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells +``` +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: +```html + `tess.neighbors[i,j]` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. +``` +Convex hull facets also define a hyperplane equation: +```html + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 +``` +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +#### Functions +```html + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.pydoc b/src/test/pythonFiles/markdown/scipy.spatial.pydoc new file mode 100644 index 000000000000..1613b94384b7 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.pydoc @@ -0,0 +1,86 @@ +============================================================= +Spatial algorithms and data structures (:mod:`scipy.spatial`) +============================================================= + +.. currentmodule:: scipy.spatial + +Nearest-neighbor Queries +======================== +.. autosummary:: + :toctree: generated/ + + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle + +Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +========================================================= + +.. autosummary:: + :toctree: generated/ + + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces + +Plotting Helpers +================ + +.. autosummary:: + :toctree: generated/ + + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram + +.. seealso:: :ref:`Tutorial ` + + +Simplex representation +====================== +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme:: + + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells + +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: + + ``tess.neighbors[i,j]`` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. + +Convex hull facets also define a hyperplane equation:: + + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 + +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes \ No newline at end of file diff --git a/src/test/refactor/extension.refactor.extract.var.test.ts b/src/test/refactor/extension.refactor.extract.var.test.ts index 5ce6cc3f743c..d12283a74198 100644 --- a/src/test/refactor/extension.refactor.extract.var.test.ts +++ b/src/test/refactor/extension.refactor.extract.var.test.ts @@ -101,13 +101,13 @@ suite('Variable Extraction', () => { test('Extract Variable', async () => { const startPos = new vscode.Position(234, 29); const endPos = new vscode.Position(234, 38); - testingVariableExtraction(false, startPos, endPos); + await testingVariableExtraction(false, startPos, endPos); }); test('Extract Variable fails if whole string not selected', async () => { const startPos = new vscode.Position(234, 20); const endPos = new vscode.Position(234, 38); - testingVariableExtraction(true, startPos, endPos); + await testingVariableExtraction(true, startPos, endPos); }); function testingVariableExtractionEndToEnd(shouldError: boolean, startPos: Position, endPos: Position) { diff --git a/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts b/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts index 4778ffa55512..7b1dc4a55742 100644 --- a/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts +++ b/src/test/terminals/codeExecution/djangoShellCodeExect.test.ts @@ -150,7 +150,7 @@ suite('Terminal - Django Shell Code Execution', () => { const workspaceFolder: WorkspaceFolder = { index: 0, name: 'blah', uri: workspaceUri }; workspace.setup(w => w.getWorkspaceFolder(TypeMoq.It.isAny())).returns(() => undefined); workspace.setup(w => w.workspaceFolders).returns(() => [workspaceFolder]); - const expectedTerminalArgs = terminalArgs.concat(path.join(workspaceUri.fsPath, 'manage.py'), 'shell'); + const expectedTerminalArgs = terminalArgs.concat(path.join(workspaceUri.fsPath, 'manage.py').fileToCommandArgument(), 'shell'); testReplCommandArguments(true, pythonPath, pythonPath, terminalArgs, expectedTerminalArgs, Uri.file('x')); }); diff --git a/src/test/terminals/codeExecution/terminalCodeExec.test.ts b/src/test/terminals/codeExecution/terminalCodeExec.test.ts index 90c92560ee20..0a907c1ce548 100644 --- a/src/test/terminals/codeExecution/terminalCodeExec.test.ts +++ b/src/test/terminals/codeExecution/terminalCodeExec.test.ts @@ -154,7 +154,8 @@ suite('Terminal Code Execution', () => { terminalSettings.setup(t => t.launchArgs).returns(() => []); await executor.executeFile(file); - terminalService.verify(async t => await t.sendText(TypeMoq.It.isValue(`cd "${path.dirname(file.fsPath)}"`)), TypeMoq.Times.once()); + const dir = `"${path.dirname(file.fsPath)}"`.fileToCommandArgument(); + terminalService.verify(async t => await t.sendText(TypeMoq.It.isValue(`cd ${dir}`)), TypeMoq.Times.once()); } test('Ensure we set current directory (and quote it when containing spaces) before executing file (non windows)', async () => { @@ -213,7 +214,7 @@ suite('Terminal Code Execution', () => { await executor.executeFile(file); const expectedPythonPath = isWindows ? pythonPath.replace(/\\/g, '/') : pythonPath; - const expectedArgs = terminalArgs.concat(file.fsPath.indexOf(' ') > 0 ? `"${file.fsPath}"` : file.fsPath); + const expectedArgs = terminalArgs.concat(file.fsPath.fileToCommandArgument()); terminalService.verify(async t => await t.sendCommand(TypeMoq.It.isValue(expectedPythonPath), TypeMoq.It.isValue(expectedArgs)), TypeMoq.Times.once()); } diff --git a/src/test/unittests/debugger.test.ts b/src/test/unittests/debugger.test.ts index 2daba9a848e1..c526961527a9 100644 --- a/src/test/unittests/debugger.test.ts +++ b/src/test/unittests/debugger.test.ts @@ -71,15 +71,30 @@ suite('Unit Tests - debugging', () => { assert.equal(tests.testFunctions.length, 2, 'Incorrect number of test functions'); assert.equal(tests.testSuites.length, 2, 'Incorrect number of test suites'); + const deferred = createDeferred(); const testFunction = [tests.testFunctions[0].testFunction]; - testManager.runTest(CommandSource.commandPalette, { testFunction }, false, true); - const launched = await mockDebugLauncher.launched; - assert.isTrue(launched, 'Debugger not launched'); + const runningPromise = testManager.runTest(CommandSource.commandPalette, { testFunction }, false, true); + + // This promise should never resolve nor reject. + runningPromise + .then(() => deferred.reject('Debugger stopped when it shouldn\'t have')) + .catch(error => deferred.reject(error)); + + mockDebugLauncher.launched + .then((launched) => { + if (launched) { + deferred.resolve(''); + } else { + deferred.reject('Debugger not launched'); + } + }) .catch(error => deferred.reject(error)); + + await deferred.promise; } test('Debugger should start (unittest)', async () => { await updateSetting('unitTest.unittestArgs', ['-s=./tests', '-p=test_*.py'], rootWorkspaceUri, configTarget); - await testStartingDebugger('unittest'); + await testStartingDebugger('unittest'); }); test('Debugger should start (pytest)', async () => { @@ -105,9 +120,10 @@ suite('Unit Tests - debugging', () => { const launched = await mockDebugLauncher.launched; assert.isTrue(launched, 'Debugger not launched'); - testManager.discoverTests(CommandSource.commandPalette, true, true, true); - + const discoveryPromise = testManager.discoverTests(CommandSource.commandPalette, true, true, true); await expect(runningPromise).to.be.rejectedWith(CANCELLATION_REASON, 'Incorrect reason for ending the debugger'); + ioc.dispose(); // will cancel test discovery + await expect(discoveryPromise).to.be.rejectedWith(CANCELLATION_REASON, 'Incorrect reason for ending the debugger'); } test('Debugger should stop when user invokes a test discovery (unittest)', async () => { @@ -151,6 +167,7 @@ suite('Unit Tests - debugging', () => { runningPromise .then(() => 'Debugger stopped when it shouldn\'t have') .catch(() => 'Debugger crashed when it shouldn\'t have') + // tslint:disable-next-line: no-floating-promises .then(error => { deferred.reject(error); }); diff --git a/src/test/unittests/stoppingDiscoverAndTest.test.ts b/src/test/unittests/stoppingDiscoverAndTest.test.ts index 3386ee2b6955..3b3558f12bd2 100644 --- a/src/test/unittests/stoppingDiscoverAndTest.test.ts +++ b/src/test/unittests/stoppingDiscoverAndTest.test.ts @@ -5,6 +5,7 @@ import { expect, use } from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; import * as path from 'path'; import { Uri } from 'vscode'; +import {createDeferred} from '../../client/common/helpers'; import { Product } from '../../client/common/types'; import { CANCELLATION_REASON, CommandSource, UNITTEST_PROVIDER } from '../../client/unittests/common/constants'; import { ITestDiscoveryService } from '../../client/unittests/common/types'; @@ -60,9 +61,23 @@ suite('Unit Tests Stopping Discovery and Runner', () => { const discoveryPromise = mockTestManager.discoverTests(CommandSource.auto); mockTestManager.discoveryDeferred.resolve(EmptyTests); - mockTestManager.runTest(CommandSource.ui); + const runningPromise = mockTestManager.runTest(CommandSource.ui); + const deferred = createDeferred(); - await expect(discoveryPromise).to.eventually.equal(EmptyTests); + // This promise should never resolve nor reject. + runningPromise + .then(() => Promise.reject('Debugger stopped when it shouldn\'t have')) + .catch(error => deferred.reject(error)); + + discoveryPromise.then(result => { + if (result === EmptyTests) { + deferred.resolve(''); + } else { + deferred.reject('tests not empty'); + } + }).catch(error => deferred.reject(error)); + + await deferred.promise; }); test('Discovering tests should stop running tests', async () => { @@ -75,7 +90,7 @@ suite('Unit Tests Stopping Discovery and Runner', () => { await new Promise(resolve => setTimeout(resolve, 1000)); // User manually discovering tests will kill the existing test runner. - mockTestManager.discoverTests(CommandSource.ui, true, false, true); + await mockTestManager.discoverTests(CommandSource.ui, true, false, true); await expect(runPromise).to.eventually.be.rejectedWith(CANCELLATION_REASON); }); }); From 35838b9fb2a693b3f310eb37e32fa28c5f7a1c15 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 7 Feb 2018 14:51:52 -0800 Subject: [PATCH 49/49] Whitespace difference --- src/client/providers/itemInfoSource.ts | 4 ++-- src/test/definitions/hover.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 4159e9396f82..b78515c1822f 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -106,7 +106,7 @@ export class ItemInfoSource { // Tooltip is only used in hover if (signature.length > 0) { - tooltip = tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + tooltip = tooltip.appendMarkdown(['```python', signature, '```', ''].join(EOL)); } const description = this.textConverter.toMarkdown(lines.join(EOL)); @@ -127,7 +127,7 @@ export class ItemInfoSource { if (item.description) { if (signature.length > 0) { - tooltip.appendMarkdown(['```python', signature, '```', EOL].join(EOL)); + tooltip.appendMarkdown(['```python', signature, '```', ''].join(EOL)); } const description = this.textConverter.toMarkdown(item.description); tooltip.appendMarkdown(description); diff --git a/src/test/definitions/hover.test.ts b/src/test/definitions/hover.test.ts index 7e64db251464..0abda7ea51c8 100644 --- a/src/test/definitions/hover.test.ts +++ b/src/test/definitions/hover.test.ts @@ -157,8 +157,8 @@ suite('Hover Definition', () => { 'share state.' + EOL + '' + EOL + 'Class Random can also be subclassed if you want to use a different basic' + EOL + - 'generator of your own devising: in that case, override the following' + EOL + EOL + - '`methods` random(), seed(), getstate(), and setstate().' + EOL + EOL + + 'generator of your own devising: in that case, override the following' + EOL + + 'methods: random(), seed(), getstate(), and setstate().' + EOL + 'Optionally, implement a getrandbits() method so that randrange()' + EOL + 'can cover arbitrarily large ranges.';