diff --git a/src/client/common/markdown/restTextConverter.ts b/src/client/common/markdown/restTextConverter.ts new file mode 100644 index 000000000000..e606fd46bfbc --- /dev/null +++ b/src/client/common/markdown/restTextConverter.ts @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { EOL } from 'os'; +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { isDecimal, isWhiteSpace } from '../../language/characters'; + +enum State { + Default, + Preformatted, + Code +} + +export class RestTextConverter { + private state: State = State.Default; + private md: string[] = []; + + // tslint:disable-next-line:cyclomatic-complexity + public toMarkdown(docstring: string): string { + // Translates reStructruredText (Python doc syntax) to markdown. + // It only translates as much as needed to display tooltips + // and documentation in the completion list. + // See https://en.wikipedia.org/wiki/ReStructuredText + + const result = this.transformLines(docstring); + this.state = State.Default; + this.md = []; + + return result; + } + + public escapeMarkdown(text: string): string { + // Not complete escape list so it does not interfere + // with subsequent code highlighting (see above). + return text + .replace(/\#/g, '\\#') + .replace(/\*/g, '\\*') + .replace(/\_/g, '\\_'); + } + + private transformLines(docstring: string): string { + const lines = docstring.split(/\r?\n/); + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]; + // Avoid leading empty lines + if (this.md.length === 0 && line.length === 0) { + continue; + } + + switch (this.state) { + case State.Default: + i += this.inDefaultState(lines, i); + break; + case State.Preformatted: + i += this.inPreformattedState(lines, i); + break; + case State.Code: + this.inCodeState(line); + break; + default: + break; + } + } + + this.endCodeBlock(); + this.endPreformattedBlock(); + + return this.md.join(EOL).trim(); + } + + private inDefaultState(lines: string[], i: number): number { + let line = lines[i]; + if (line.startsWith('```')) { + this.startCodeBlock(); + return 0; + } + + if (line.startsWith('===') || line.startsWith('---')) { + return 0; // Eat standalone === or --- lines. + } + if (this.handleDoubleColon(line)) { + return 0; + } + if (this.isIgnorable(line)) { + return 0; + } + + if (this.handleSectionHeader(lines, i)) { + return 1; // Eat line with === or --- + } + + const result = this.checkPreContent(lines, i); + if (this.state !== State.Default) { + return result; // Handle line in the new state + } + + line = this.cleanup(line); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + line = this.escapeMarkdown(line); + this.md.push(line); + + return 0; + } + + private inPreformattedState(lines: string[], i: number): number { + let line = lines[i]; + if (this.isIgnorable(line)) { + return 0; + } + // Preformatted block terminates by a line without leading whitespace. + if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + this.endPreformattedBlock(); + return -1; + } + + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return 0; // Avoid more than one empty line in a row. + } + + // Since we use HTML blocks as preformatted text + // make sure we drop angle brackets since otherwise + // they will render as tags and attributes + line = line.replace(//g, ' '); + line = line.replace(/``/g, '`'); // Convert double backticks to single. + // Keep hard line breaks for the preformatted content + this.md.push(`${line} `); + return 0; + } + + private inCodeState(line: string): void { + const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined; + if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) { + return; // Avoid more than one empty line in a row. + } + + if (line.startsWith('```')) { + this.endCodeBlock(); + } else { + this.md.push(line); + } + } + + private isIgnorable(line: string): boolean { + if (line.indexOf('generated/') >= 0) { + return true; // Drop generated content. + } + const trimmed = line.trim(); + if (trimmed.startsWith('..') && trimmed.indexOf('::') > 0) { + // Ignore lines likes .. sectionauthor:: John Doe. + return true; + } + return false; + } + + private checkPreContent(lines: string[], i: number): number { + const line = lines[i]; + if (i === 0 || line.trim().length === 0) { + return 0; + } + + if (!isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) { + return 0; // regular line, nothing to do here. + } + // Indented content is considered to be preformatted. + this.startPreformattedBlock(); + return -1; + } + + private handleSectionHeader(lines: string[], i: number): boolean { + const line = lines[i]; + if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) { + // Section title -> heading level 3. + this.md.push(`### ${this.cleanup(line)}`); + return true; + } + if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) { + // Subsection title -> heading level 4. + this.md.push(`#### ${this.cleanup(line)}`); + return true; + } + return false; + } + + private handleDoubleColon(line: string): boolean { + if (!line.endsWith('::')) { + return false; + } + // Literal blocks begin with `::`. Such as sequence like + // '... as shown below::' that is followed by a preformatted text. + if (line.length > 2 && !line.startsWith('..')) { + // Ignore lines likes .. autosummary:: John Doe. + // Trim trailing : so :: turns into :. + this.md.push(line.substring(0, line.length - 1)); + } + + this.startPreformattedBlock(); + return true; + } + + private startPreformattedBlock(): void { + // Remove previous empty line so we avoid double empties. + this.tryRemovePrecedingEmptyLines(); + // Lie about the language since we don't want preformatted text + // to be colorized as Python. HTML is more 'appropriate' as it does + // not colorize -- or + or keywords like 'from'. + this.md.push('```html'); + this.state = State.Preformatted; + } + + private endPreformattedBlock(): void { + if (this.state === State.Preformatted) { + this.tryRemovePrecedingEmptyLines(); + this.md.push('```'); + this.state = State.Default; + } + } + + private startCodeBlock(): void { + // Remove previous empty line so we avoid double empties. + this.tryRemovePrecedingEmptyLines(); + this.md.push('```python'); + this.state = State.Code; + } + + private endCodeBlock(): void { + if (this.state === State.Code) { + this.tryRemovePrecedingEmptyLines(); + this.md.push('```'); + this.state = State.Default; + } + } + + private tryRemovePrecedingEmptyLines(): void { + while (this.md.length > 0 && this.md[this.md.length - 1].trim().length === 0) { + this.md.pop(); + } + } + + private isListItem(line: string): boolean { + const trimmed = line.trim(); + const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0; + return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch); + } + + private cleanup(line: string): string { + return line.replace(/:mod:/g, 'module:'); + } +} diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index b84a11b62fd1..0a2160fc15c5 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -300,7 +300,7 @@ export class Tokenizer implements ITokenizer { break; default: - break; + return false; } this.tokens.push(new Token(TokenType.Operator, this.cs.position, length)); this.cs.advance(length); diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index 3cb471959bac..b78515c1822f 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -4,6 +4,7 @@ import { EOL } from 'os'; import * as vscode from 'vscode'; +import { RestTextConverter } from '../common/markdown/restTextConverter'; import { JediFactory } from '../languageServices/jediProxyFactory'; import * as proxy from './jediProxy'; import { IHoverItem } from './jediProxy'; @@ -16,6 +17,7 @@ export class LanguageItemInfo { } export class ItemInfoSource { + private textConverter = new RestTextConverter(); constructor(private jediFactory: JediFactory) { } public async getItemInfoFromText(documentUri: vscode.Uri, fileName: string, range: vscode.Range, sourceText: string, token: vscode.CancellationToken) @@ -84,22 +86,8 @@ export class ItemInfoSource { const capturedInfo: string[] = []; data.items.forEach(item => { - let { signature } = item; - switch (item.kind) { - case vscode.SymbolKind.Constructor: - case vscode.SymbolKind.Function: - case vscode.SymbolKind.Method: { - signature = `def ${signature}`; - break; - } - case vscode.SymbolKind.Class: { - signature = `class ${signature}`; - break; - } - default: { - signature = typeof item.text === 'string' && item.text.length > 0 ? item.text : currentWord; - } - } + const signature = this.getSignature(item, currentWord); + let tooltip = new vscode.MarkdownString(); if (item.docstring) { let lines = item.docstring.split(/\r?\n/); const dnd = this.getDetailAndDescription(item, lines); @@ -116,9 +104,16 @@ export class ItemInfoSource { lines.shift(); } - const descriptionWithHighlightedCode = this.highlightCode(lines.join(EOL)); - const tooltip = new vscode.MarkdownString(['```python', signature, '```', descriptionWithHighlightedCode].join(EOL)); - infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(dnd[1]))); + // Tooltip is only used in hover + if (signature.length > 0) { + tooltip = tooltip.appendMarkdown(['```python', signature, '```', ''].join(EOL)); + } + + const description = this.textConverter.toMarkdown(lines.join(EOL)); + tooltip = tooltip.appendMarkdown(description); + + const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list + infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); // Sometimes we have duplicate documentation, one with a period at the end. @@ -131,13 +126,16 @@ export class ItemInfoSource { } if (item.description) { - const descriptionWithHighlightedCode = this.highlightCode(item.description); - // tslint:disable-next-line:prefer-template - const tooltip = new vscode.MarkdownString('```python' + `${EOL}${signature}${EOL}` + '```' + `${EOL}${descriptionWithHighlightedCode}`); + if (signature.length > 0) { + tooltip.appendMarkdown(['```python', signature, '```', ''].join(EOL)); + } + const description = this.textConverter.toMarkdown(item.description); + tooltip.appendMarkdown(description); const lines = item.description.split(EOL); const dd = this.getDetailAndDescription(item, lines); - infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(dd[1]))); + const documentation = this.textConverter.escapeMarkdown(dd[1]); + infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(documentation))); const key = signature + lines.join(''); // Sometimes we have duplicate documentation, one with a period at the end. @@ -157,7 +155,7 @@ export class ItemInfoSource { let detail: string; let description: string; - if (item.signature && item.signature.length > 0) { + if (item.signature && item.signature.length > 0 && lines.length > 0 && lines[0].indexOf(item.signature) >= 0) { detail = lines.length > 0 ? lines[0] : ''; description = lines.filter((line, index) => index > 0).join(EOL).trim(); } else { @@ -167,63 +165,29 @@ export class ItemInfoSource { return [detail, description]; } - private highlightCode(docstring: string): string { - /********** - * - * Magic. Do not touch. [What is the best comment in source code](https://stackoverflow.com/a/185106) - * - * This method uses several regexs to 'translate' reStructruedText syntax (Python doc syntax) to Markdown syntax. - * - * Let's just keep it unchanged unless a better solution becomes possible. - * - **********/ - // Add 2 line break before and after docstring (used to match a blank line) - docstring = EOL + EOL + docstring.trim() + EOL + EOL; - // Section title -> heading level 2 - docstring = docstring.replace(/(.+\r?\n)[-=]+\r?\n/g, `## $1${EOL}`); - // Directives: '.. directive::' -> '**directive**' - docstring = docstring.replace(/\.\. (.*)::/g, '**$1**'); - // Pattern of 'var : description' - const paramLinePattern = '[\\*\\w_]+ ?:[^:\r\n]+'; - // Add new line after and before param line - docstring = docstring.replace(new RegExp(`(${EOL + paramLinePattern})`, 'g'), `$1${EOL}`); - docstring = docstring.replace(new RegExp(`(${EOL + paramLinePattern + EOL})`, 'g'), `${EOL}$1`); - // 'var : description' -> '`var` description' - docstring = docstring.replace(/\r?\n([\*\w]+) ?: ?([^:\r\n]+\r?\n)/g, `${EOL}\`$1\` $2`); - // Doctest blocks: begin with `>>>` and end with blank line - // tslint:disable-next-line:prefer-template - docstring = docstring.replace(/(>>>[\w\W]+?\r?\n)\r?\n/g, `${'```python' + EOL}$1${'```' + EOL + EOL}`); - // Literal blocks: begin with `::` (literal blocks are indented or quoted; for simplicity, we end literal blocks with blank line) - // tslint:disable-next-line:prefer-template - docstring = docstring.replace(/(\r?\n[^\.]*)::\r?\n\r?\n([\w\W]+?\r?\n)\r?\n/g, `$1${EOL + '```' + EOL}$2${'```' + EOL + EOL}`); - // Remove indentation in Field lists and Literal blocks - let inCodeBlock = false; - let codeIndentation = 0; - const lines = docstring.split(/\r?\n/); - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i]; - if (line.startsWith('```')) { - inCodeBlock = !inCodeBlock; - if (inCodeBlock) { - const match = lines[i + 1].match(/^ */); - codeIndentation = match && match.length > 0 ? match[0].length : 0; - } - continue; + private getSignature(item: proxy.IHoverItem, currentWord: string): string { + let { signature } = item; + switch (item.kind) { + case vscode.SymbolKind.Constructor: + case vscode.SymbolKind.Function: + case vscode.SymbolKind.Method: { + signature = `def ${signature}`; + break; } - if (!inCodeBlock) { - lines[i] = line.replace(/^ {4,8}/, ''); - // Field lists: ':field:' -> '**field**' - lines[i] = lines[i].replace(/:(.+?):/g, '**$1** '); - } else { - if (codeIndentation !== 0) { - lines[i] = line.substring(codeIndentation); + case vscode.SymbolKind.Class: { + signature = `class ${signature}`; + break; + } + case vscode.SymbolKind.Module: { + if (signature.length > 0) { + signature = `module ${signature}`; } + break; + } + default: { + signature = typeof item.text === 'string' && item.text.length > 0 ? item.text : currentWord; } } - docstring = lines.join(EOL); - // Grid Tables - docstring = docstring.replace(/\r?\n[\+-]+\r?\n/g, EOL); - docstring = docstring.replace(/\r?\n[\+=]+\r?\n/g, s => s.replace(/\+/g, '|').replace(/=/g, '-')); - return docstring.trim(); + return signature; } } diff --git a/src/test/definitions/hover.test.ts b/src/test/definitions/hover.test.ts index 7e64db251464..0abda7ea51c8 100644 --- a/src/test/definitions/hover.test.ts +++ b/src/test/definitions/hover.test.ts @@ -157,8 +157,8 @@ suite('Hover Definition', () => { 'share state.' + EOL + '' + EOL + 'Class Random can also be subclassed if you want to use a different basic' + EOL + - 'generator of your own devising: in that case, override the following' + EOL + EOL + - '`methods` random(), seed(), getstate(), and setstate().' + EOL + EOL + + 'generator of your own devising: in that case, override the following' + EOL + + 'methods: random(), seed(), getstate(), and setstate().' + EOL + 'Optionally, implement a getrandbits() method so that randrange()' + EOL + 'can cover arbitrarily large ranges.'; diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 727ce969dd09..86deb9282249 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -76,4 +76,11 @@ suite('Language.Tokenizer', () => { assert.equal(tokens.getItemAt(i).type, TokenType.Comment); } }); + test('Unknown token', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('.'); + assert.equal(tokens.count, 1); + + assert.equal(tokens.getItemAt(0).type, TokenType.Unknown); + }); }); diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts new file mode 100644 index 000000000000..9b43d4d57657 --- /dev/null +++ b/src/test/markdown/restTextConverter.test.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { expect } from 'chai'; +import * as fs from 'fs-extra'; +import * as path from 'path'; +import { RestTextConverter } from '../../client/common/markdown/restTextConverter'; + +const srcPythoFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'markdown'); + +function compareFiles(expectedContent: string, actualContent: string) { + const expectedLines = expectedContent.split(/\r?\n/); + const actualLines = actualContent.split(/\r?\n/); + + for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { + const e = expectedLines[i]; + const a = actualLines[i]; + expect(e, `Difference at line ${i}`).to.be.equal(a); + } + + expect(actualLines.length, + expectedLines.length > actualLines.length + ? 'Actual contains more lines than expected' + : 'Expected contains more lines than the actual' + ).to.be.equal(expectedLines.length); +} + +async function testConversion(fileName: string): Promise { + const cvt = new RestTextConverter(); + const file = path.join(srcPythoFilesPath, fileName); + const source = await fs.readFile(`${file}.pydoc`, 'utf8'); + const actual = cvt.toMarkdown(source); + const expected = await fs.readFile(`${file}.md`, 'utf8'); + compareFiles(expected, actual); +} + +// tslint:disable-next-line:max-func-body-length +suite('Hover - RestTextConverter', () => { + test('scipy', async () => await testConversion('scipy')); + test('scipy.spatial', async () => await testConversion('scipy.spatial')); + test('scipy.spatial.distance', async () => await testConversion('scipy.spatial.distance')); + test('anydbm', async () => await testConversion('anydbm')); + test('aifc', async () => await testConversion('aifc')); + test('astroid', async () => await testConversion('astroid')); +}); diff --git a/src/test/pythonFiles/markdown/aifc.md b/src/test/pythonFiles/markdown/aifc.md new file mode 100644 index 000000000000..fff22dece1e5 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.md @@ -0,0 +1,142 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. +```html + +-----------------+ + | FORM | + +-----------------+ + | size | + +----+------------+ + | | AIFC | + | +------------+ + | | chunks | + | | . | + | | . | + | | . | + +----+------------+ +``` +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. +```html + FVER + version number of AIFF-C defining document (AIFF-C only). + MARK + # of markers (2 bytes) + list of markers: + marker ID (2 bytes, must be 0) + position (4 bytes) + marker name ("pstring") + COMM + # of channels (2 bytes) + # of sound frames (4 bytes) + size of the samples (2 bytes) + sampling frequency (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + compression type (4 bytes) + human-readable version of compression type ("pstring") + SSND + offset (4 bytes, not used by this program) + blocksize (4 bytes, not used by this program) + sound data +``` +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: +```html + f = aifc.open(file, 'r') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: +```html + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +``` +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: +```html + f = aifc.open(file, 'w') +``` +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: +```html + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +``` +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/aifc.pydoc b/src/test/pythonFiles/markdown/aifc.pydoc new file mode 100644 index 000000000000..a4cc346d5531 --- /dev/null +++ b/src/test/pythonFiles/markdown/aifc.pydoc @@ -0,0 +1,134 @@ +Stuff to parse AIFF-C and AIFF files. + +Unless explicitly stated otherwise, the description below is true +both for AIFF-C files and AIFF files. + +An AIFF-C file has the following structure. + + +-----------------+ + | FORM | + +-----------------+ + | | + +----+------------+ + | | AIFC | + | +------------+ + | | | + | | . | + | | . | + | | . | + +----+------------+ + +An AIFF file has the string "AIFF" instead of "AIFC". + +A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, +big endian order), followed by the data. The size field does not include +the size of the 8 byte header. + +The following chunk types are recognized. + + FVER + (AIFF-C only). + MARK + <# of markers> (2 bytes) + list of markers: + (2 bytes, must be > 0) + (4 bytes) + ("pstring") + COMM + <# of channels> (2 bytes) + <# of sound frames> (4 bytes) + (2 bytes) + (10 bytes, IEEE 80-bit extended + floating point) + in AIFF-C files only: + (4 bytes) + ("pstring") + SSND + (4 bytes, not used by this program) + (4 bytes, not used by this program) + + +A pstring consists of 1 byte length, a string of characters, and 0 or 1 +byte pad to make the total length even. + +Usage. + +Reading AIFF files: + f = aifc.open(file, 'r') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +In some types of audio files, if the setpos() method is not used, +the seek() method is not necessary. + +This returns an instance of a class with the following public methods: + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for AIFF files) + getcompname() -- returns human-readable version of + compression type ('not compressed' for AIFF files) + getparams() -- returns a tuple consisting of all of the + above in the above order + getmarkers() -- get the list of marks in the audio file or None + if there are no marks + getmark(id) -- get mark with the specified id (raises an error + if the mark does not exist) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +The position returned by tell(), the position given to setpos() and +the position of marks are all compatible and have nothing to do with +the actual position in the file. +The close() method is called automatically when the class instance +is destroyed. + +Writing AIFF files: + f = aifc.open(file, 'w') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: + aiff() -- create an AIFF file (AIFF-C default) + aifc() -- create an AIFF-C file + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + setmark(id, pos, name) + -- add specified mark to the list of marks + tell() -- return current position in output file (useful + in combination with setmark()) + writeframesraw(data) + -- write audio frames without pathing up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes('') or +close() to patch up the sizes in the header. +Marks can be added anytime. If there are any marks, you must call +close() after all frames have been written. +The close() method is called automatically when the class instance +is destroyed. + +When a file is opened with the extension '.aiff', an AIFF file is +written, otherwise an AIFF-C file is written. This default can be +changed by calling aiff() or aifc() before the first writeframes or +writeframesraw. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.md b/src/test/pythonFiles/markdown/anydbm.md new file mode 100644 index 000000000000..e5914dcbadde --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.md @@ -0,0 +1,33 @@ +Generic interface to all dbm clones. + +Instead of +```html + import dbm + d = dbm.open(file, 'w', 0666) +``` +use +```html + import anydbm + d = anydbm.open(file, 'w') +``` +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): +```html + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) +``` +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/anydbm.pydoc b/src/test/pythonFiles/markdown/anydbm.pydoc new file mode 100644 index 000000000000..2d46b5881789 --- /dev/null +++ b/src/test/pythonFiles/markdown/anydbm.pydoc @@ -0,0 +1,33 @@ +Generic interface to all dbm clones. + +Instead of + + import dbm + d = dbm.open(file, 'w', 0666) + +use + + import anydbm + d = anydbm.open(file, 'w') + +The returned object is a dbhash, gdbm, dbm or dumbdbm object, +dependent on the type of database being opened (determined by whichdb +module) in the case of an existing dbm. If the dbm does not exist and +the create or new flag ('c' or 'n') was specified, the dbm type will +be determined by the availability of the modules (tested in the above +order). + +It has the following interface (key and data are strings): + + d[key] = data # store data at key (may override data at + # existing key) + data = d[key] # retrieve data at key (raise KeyError if no + # such key) + del d[key] # delete data stored at key (raises KeyError + # if no such key) + flag = key in d # true if the key exists + list = d.keys() # return a list of all existing keys (slow!) + +Future versions may change the order in which implementations are +tested for existence, and add interfaces to other dbm-like +implementations. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.md b/src/test/pythonFiles/markdown/astroid.md new file mode 100644 index 000000000000..b5ece21c1faf --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.md @@ -0,0 +1,24 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's \_ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: +```html +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/astroid.pydoc b/src/test/pythonFiles/markdown/astroid.pydoc new file mode 100644 index 000000000000..84d58487ead5 --- /dev/null +++ b/src/test/pythonFiles/markdown/astroid.pydoc @@ -0,0 +1,23 @@ +Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: + +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.md b/src/test/pythonFiles/markdown/scipy.md new file mode 100644 index 000000000000..d28c1e290abe --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.md @@ -0,0 +1,47 @@ +### SciPy: A scientific computing package for Python + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +#### Contents +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +#### Subpackages +Using any of these subpackages requires an explicit import. For example, +`import scipy.cluster`. +```html + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions +``` +#### Utility tools +```html + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.pydoc b/src/test/pythonFiles/markdown/scipy.pydoc new file mode 100644 index 000000000000..293445fbea5b --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.pydoc @@ -0,0 +1,53 @@ +SciPy: A scientific computing package for Python +================================================ + +Documentation is available in the docstrings and +online at https://docs.scipy.org. + +Contents +-------- +SciPy imports all the functions from the NumPy namespace, and in +addition provides: + +Subpackages +----------- +Using any of these subpackages requires an explicit import. For example, +``import scipy.cluster``. + +:: + + cluster --- Vector Quantization / Kmeans + fftpack --- Discrete Fourier Transform algorithms + integrate --- Integration routines + interpolate --- Interpolation Tools + io --- Data input and output + linalg --- Linear algebra routines + linalg.blas --- Wrappers to BLAS library + linalg.lapack --- Wrappers to LAPACK library + misc --- Various utilities that don't have + another home. + ndimage --- n-dimensional image package + odr --- Orthogonal Distance Regression + optimize --- Optimization Tools + signal --- Signal Processing Tools + sparse --- Sparse Matrices + sparse.linalg --- Sparse Linear Algebra + sparse.linalg.dsolve --- Linear Solvers + sparse.linalg.dsolve.umfpack --- :Interface to the UMFPACK library: + Conjugate Gradient Method (LOBPCG) + sparse.linalg.eigen --- Sparse Eigenvalue Solvers + sparse.linalg.eigen.lobpcg --- Locally Optimal Block Preconditioned + Conjugate Gradient Method (LOBPCG) + spatial --- Spatial data structures and algorithms + special --- Special functions + stats --- Statistical Functions + +Utility tools +------------- +:: + + test --- Run scipy unittests + show_config --- Show scipy build configuration + show_numpy_config --- Show numpy build configuration + __version__ --- Scipy version string + __numpy_version__ --- Numpy version string \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.md b/src/test/pythonFiles/markdown/scipy.spatial.distance.md new file mode 100644 index 000000000000..276acddef787 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.md @@ -0,0 +1,54 @@ +### Distance computations (module:`scipy.spatial.distance`) + + +#### Function Reference + +Distance matrix computation from a collection of raw observation vectors +stored in a rectangular array. +```html + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays +``` +Predicates for checking the validity of distance matrices, both +condensed and redundant. Also contained in this module are functions +for computing the number of observations in a distance matrix. +```html + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix +``` +Distance functions between two numeric vectors `u` and `v`. Computing +distances over a large collection of vectors is inefficient for these +functions. Use `pdist` for this purpose. +```html + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. +``` +Distance functions between two boolean vectors (representing sets) `u` and +`v`. As in the case of numerical vectors, `pdist` is more efficient for +computing the distances between all pairs. +```html + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. +``` +:func:`hamming` also operates over discrete numerical vectors. \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc b/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc new file mode 100644 index 000000000000..cfc9b7008b99 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.distance.pydoc @@ -0,0 +1,71 @@ + +===================================================== +Distance computations (:mod:`scipy.spatial.distance`) +===================================================== + +.. sectionauthor:: Damian Eads + +Function Reference +------------------ + +Distance matrix computation from a collection of raw observation vectors +stored in a rectangular array. + +.. autosummary:: + :toctree: generated/ + + pdist -- pairwise distances between observation vectors. + cdist -- distances between two collections of observation vectors + squareform -- convert distance matrix to a condensed one and vice versa + directed_hausdorff -- directed Hausdorff distance between arrays + +Predicates for checking the validity of distance matrices, both +condensed and redundant. Also contained in this module are functions +for computing the number of observations in a distance matrix. + +.. autosummary:: + :toctree: generated/ + + is_valid_dm -- checks for a valid distance matrix + is_valid_y -- checks for a valid condensed distance matrix + num_obs_dm -- # of observations in a distance matrix + num_obs_y -- # of observations in a condensed distance matrix + +Distance functions between two numeric vectors ``u`` and ``v``. Computing +distances over a large collection of vectors is inefficient for these +functions. Use ``pdist`` for this purpose. + +.. autosummary:: + :toctree: generated/ + + braycurtis -- the Bray-Curtis distance. + canberra -- the Canberra distance. + chebyshev -- the Chebyshev distance. + cityblock -- the Manhattan distance. + correlation -- the Correlation distance. + cosine -- the Cosine distance. + euclidean -- the Euclidean distance. + mahalanobis -- the Mahalanobis distance. + minkowski -- the Minkowski distance. + seuclidean -- the normalized Euclidean distance. + sqeuclidean -- the squared Euclidean distance. + wminkowski -- (deprecated) alias of `minkowski`. + +Distance functions between two boolean vectors (representing sets) ``u`` and +``v``. As in the case of numerical vectors, ``pdist`` is more efficient for +computing the distances between all pairs. + +.. autosummary:: + :toctree: generated/ + + dice -- the Dice dissimilarity. + hamming -- the Hamming distance. + jaccard -- the Jaccard distance. + kulsinski -- the Kulsinski distance. + rogerstanimoto -- the Rogers-Tanimoto dissimilarity. + russellrao -- the Russell-Rao dissimilarity. + sokalmichener -- the Sokal-Michener dissimilarity. + sokalsneath -- the Sokal-Sneath dissimilarity. + yule -- the Yule dissimilarity. + +:func:`hamming` also operates over discrete numerical vectors. diff --git a/src/test/pythonFiles/markdown/scipy.spatial.md b/src/test/pythonFiles/markdown/scipy.spatial.md new file mode 100644 index 000000000000..2d5e891db625 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.md @@ -0,0 +1,65 @@ +### Spatial algorithms and data structures (module:`scipy.spatial`) + + +### Nearest-neighbor Queries +```html + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle +``` +### Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +```html + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces +``` +### Plotting Helpers +```html + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram +``` +### Simplex representation +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme: +```html + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells +``` +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: +```html + `tess.neighbors[i,j]` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. +``` +Convex hull facets also define a hyperplane equation: +```html + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 +``` +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +#### Functions +```html + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes +``` \ No newline at end of file diff --git a/src/test/pythonFiles/markdown/scipy.spatial.pydoc b/src/test/pythonFiles/markdown/scipy.spatial.pydoc new file mode 100644 index 000000000000..1613b94384b7 --- /dev/null +++ b/src/test/pythonFiles/markdown/scipy.spatial.pydoc @@ -0,0 +1,86 @@ +============================================================= +Spatial algorithms and data structures (:mod:`scipy.spatial`) +============================================================= + +.. currentmodule:: scipy.spatial + +Nearest-neighbor Queries +======================== +.. autosummary:: + :toctree: generated/ + + KDTree -- class for efficient nearest-neighbor queries + cKDTree -- class for efficient nearest-neighbor queries (faster impl.) + distance -- module containing many different distance measures + Rectangle + +Delaunay Triangulation, Convex Hulls and Voronoi Diagrams +========================================================= + +.. autosummary:: + :toctree: generated/ + + Delaunay -- compute Delaunay triangulation of input points + ConvexHull -- compute a convex hull for input points + Voronoi -- compute a Voronoi diagram hull from input points + SphericalVoronoi -- compute a Voronoi diagram from input points on the surface of a sphere + HalfspaceIntersection -- compute the intersection points of input halfspaces + +Plotting Helpers +================ + +.. autosummary:: + :toctree: generated/ + + delaunay_plot_2d -- plot 2-D triangulation + convex_hull_plot_2d -- plot 2-D convex hull + voronoi_plot_2d -- plot 2-D voronoi diagram + +.. seealso:: :ref:`Tutorial ` + + +Simplex representation +====================== +The simplices (triangles, tetrahedra, ...) appearing in the Delaunay +tesselation (N-dim simplices), convex hull facets, and Voronoi ridges +(N-1 dim simplices) are represented in the following scheme:: + + tess = Delaunay(points) + hull = ConvexHull(points) + voro = Voronoi(points) + + # coordinates of the j-th vertex of the i-th simplex + tess.points[tess.simplices[i, j], :] # tesselation element + hull.points[hull.simplices[i, j], :] # convex hull facet + voro.vertices[voro.ridge_vertices[i, j], :] # ridge between Voronoi cells + +For Delaunay triangulations and convex hulls, the neighborhood +structure of the simplices satisfies the condition: + + ``tess.neighbors[i,j]`` is the neighboring simplex of the i-th + simplex, opposite to the j-vertex. It is -1 in case of no + neighbor. + +Convex hull facets also define a hyperplane equation:: + + (hull.equations[i,:-1] * coord).sum() + hull.equations[i,-1] == 0 + +Similar hyperplane equations for the Delaunay triangulation correspond +to the convex hull facets on the corresponding N+1 dimensional +paraboloid. + +The Delaunay triangulation objects offer a method for locating the +simplex containing a given point, and barycentric coordinate +computations. + +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + tsearch + distance_matrix + minkowski_distance + minkowski_distance_p + procrustes \ No newline at end of file