-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
internal/lsp/lsppos: add helpers for mapping token positions
For use-cases that work only with token.Pos and protocol.Position, the span package is unnecessarily indirect, and inefficient. It also loses information about newline termination, and handles positions within CRLF line endings incorrectly. The lsppos package was written to bypass this complexity, but had limited use and lacked tests. Add tests, and an wrapper API that operates on token.Pos. Also fix source.TestTokenOffset to not panic, and add a temporary exemption of the new token.Offset usage. This change also fixes position calculation in the case of empty file content. The mapper now finds position (0, 0) at offset 0 of an empty file. Change-Id: I639bd3fac78a127b1c8eddad60b890449901c68c Reviewed-on: https://go-review.googlesource.com/c/tools/+/403678 Reviewed-by: Alan Donovan <adonovan@google.com> Run-TryBot: Robert Findley <rfindley@google.com> gopls-CI: kokoro <noreply+kokoro@google.com> TryBot-Result: Gopher Robot <gobot@golang.org>
- Loading branch information
Showing
6 changed files
with
309 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
// Copyright 2022 The Go Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style | ||
// license that can be found in the LICENSE file. | ||
|
||
package lsppos_test | ||
|
||
import ( | ||
"fmt" | ||
"strings" | ||
"testing" | ||
|
||
. "golang.org/x/tools/internal/lsp/lsppos" | ||
"golang.org/x/tools/internal/lsp/protocol" | ||
) | ||
|
||
type testCase struct { | ||
content string // input text | ||
substrOrOffset interface{} // explicit integer offset, or a substring | ||
wantLine, wantChar int // expected LSP position information | ||
} | ||
|
||
// offset returns the test case byte offset | ||
func (c testCase) offset() int { | ||
switch x := c.substrOrOffset.(type) { | ||
case int: | ||
return x | ||
case string: | ||
i := strings.Index(c.content, x) | ||
if i < 0 { | ||
panic(fmt.Sprintf("%q does not contain substring %q", c.content, x)) | ||
} | ||
return i | ||
} | ||
panic("substrOrIndex must be an integer or string") | ||
} | ||
|
||
var tests = []testCase{ | ||
{"a𐐀b", "a", 0, 0}, | ||
{"a𐐀b", "𐐀", 0, 1}, | ||
{"a𐐀b", "b", 0, 3}, | ||
{"a𐐀b\n", "\n", 0, 4}, | ||
{"a𐐀b\r\n", "\n", 0, 4}, // \r|\n is not a valid position, so we move back to the end of the first line. | ||
{"a𐐀b\r\nx", "x", 1, 0}, | ||
{"a𐐀b\r\nx\ny", "y", 2, 0}, | ||
|
||
// Testing EOL and EOF positions | ||
{"", 0, 0, 0}, // 0th position of an empty buffer is (0, 0) | ||
{"abc", "c", 0, 2}, | ||
{"abc", 3, 0, 3}, | ||
{"abc\n", "\n", 0, 3}, | ||
{"abc\n", 4, 1, 0}, // position after a newline is on the next line | ||
} | ||
|
||
func TestLineChar(t *testing.T) { | ||
for _, test := range tests { | ||
m := NewMapper([]byte(test.content)) | ||
offset := test.offset() | ||
gotLine, gotChar := m.LineColUTF16(offset) | ||
if gotLine != test.wantLine || gotChar != test.wantChar { | ||
t.Errorf("LineChar(%d) = (%d,%d), want (%d,%d)", offset, gotLine, gotChar, test.wantLine, test.wantChar) | ||
} | ||
} | ||
} | ||
|
||
func TestInvalidOffset(t *testing.T) { | ||
content := []byte("a𐐀b\r\nx\ny") | ||
m := NewMapper(content) | ||
for _, offset := range []int{-1, 100} { | ||
gotLine, gotChar := m.LineColUTF16(offset) | ||
if gotLine != -1 { | ||
t.Errorf("LineChar(%d) = (%d,%d), want (-1,-1)", offset, gotLine, gotChar) | ||
} | ||
} | ||
} | ||
|
||
func TestPosition(t *testing.T) { | ||
for _, test := range tests { | ||
m := NewMapper([]byte(test.content)) | ||
offset := test.offset() | ||
got, ok := m.Position(offset) | ||
if !ok { | ||
t.Error("invalid position for", test.substrOrOffset) | ||
continue | ||
} | ||
want := protocol.Position{Line: uint32(test.wantLine), Character: uint32(test.wantChar)} | ||
if got != want { | ||
t.Errorf("Position(%d) = %v, want %v", offset, got, want) | ||
} | ||
} | ||
} | ||
|
||
func TestRange(t *testing.T) { | ||
for _, test := range tests { | ||
m := NewMapper([]byte(test.content)) | ||
offset := test.offset() | ||
got, err := m.Range(0, offset) | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
want := protocol.Range{ | ||
End: protocol.Position{Line: uint32(test.wantLine), Character: uint32(test.wantChar)}, | ||
} | ||
if got != want { | ||
t.Errorf("Range(%d) = %v, want %v", offset, got, want) | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
// Copyright 2022 The Go Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style | ||
// license that can be found in the LICENSE file. | ||
|
||
package lsppos | ||
|
||
import ( | ||
"errors" | ||
"go/token" | ||
|
||
"golang.org/x/tools/internal/lsp/protocol" | ||
) | ||
|
||
// TokenMapper maps token.Pos to LSP positions for a single file. | ||
type TokenMapper struct { | ||
// file is used for computing offsets. | ||
file *token.File | ||
|
||
// For now, just delegate to a Mapper for position calculation. As an | ||
// optimization we could avoid building the mapper and just use the file, but | ||
// then have to correctly adjust for newline-terminated files. It is easier | ||
// to just delegate unless performance becomes a concern. | ||
mapper *Mapper | ||
} | ||
|
||
// NewMapper creates a new TokenMapper for the given content, using the | ||
// provided file to compute offsets. | ||
func NewTokenMapper(content []byte, file *token.File) *TokenMapper { | ||
return &TokenMapper{ | ||
file: file, | ||
mapper: NewMapper(content), | ||
} | ||
} | ||
|
||
// Position returns the protocol position corresponding to the given pos. It | ||
// returns false if pos is out of bounds for the file being mapped. | ||
func (m *TokenMapper) Position(pos token.Pos) (protocol.Position, bool) { | ||
if int(pos) < m.file.Base() || int(pos) > m.file.Base()+m.file.Size() { | ||
return protocol.Position{}, false | ||
} | ||
offset := m.file.Offset(pos) // usage of token.File.Offset is temporarily exempted | ||
return m.mapper.Position(offset) | ||
} | ||
|
||
// Range returns the protocol range corresponding to the given start and end | ||
// positions. It returns an error if start or end is out of bounds for the file | ||
// being mapped. | ||
func (m *TokenMapper) Range(start, end token.Pos) (protocol.Range, error) { | ||
startPos, ok := m.Position(start) | ||
if !ok { | ||
return protocol.Range{}, errors.New("invalid start position") | ||
} | ||
endPos, ok := m.Position(end) | ||
if !ok { | ||
return protocol.Range{}, errors.New("invalid end position") | ||
} | ||
|
||
return protocol.Range{Start: startPos, End: endPos}, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
// Copyright 2022 The Go Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style | ||
// license that can be found in the LICENSE file. | ||
|
||
package lsppos_test | ||
|
||
import ( | ||
"go/token" | ||
"testing" | ||
|
||
. "golang.org/x/tools/internal/lsp/lsppos" | ||
"golang.org/x/tools/internal/lsp/protocol" | ||
) | ||
|
||
func makeTokenMapper(content []byte) (*TokenMapper, *token.File) { | ||
file := token.NewFileSet().AddFile("p.go", -1, len(content)) | ||
file.SetLinesForContent(content) | ||
return NewTokenMapper(content, file), file | ||
} | ||
|
||
func TestInvalidPosition(t *testing.T) { | ||
content := []byte("a𐐀b\r\nx\ny") | ||
m, _ := makeTokenMapper(content) | ||
|
||
for _, pos := range []token.Pos{-1, 100} { | ||
posn, ok := m.Position(pos) | ||
if ok { | ||
t.Errorf("Position(%d) = %v, want error", pos, posn) | ||
} | ||
} | ||
} | ||
|
||
func TestTokenPosition(t *testing.T) { | ||
for _, test := range tests { | ||
m, f := makeTokenMapper([]byte(test.content)) | ||
pos := token.Pos(f.Base() + test.offset()) | ||
got, ok := m.Position(pos) | ||
if !ok { | ||
t.Error("invalid position for", test.substrOrOffset) | ||
continue | ||
} | ||
want := protocol.Position{Line: uint32(test.wantLine), Character: uint32(test.wantChar)} | ||
if got != want { | ||
t.Errorf("Position(%d) = %v, want %v", pos, got, want) | ||
} | ||
gotRange, err := m.Range(token.Pos(f.Base()), pos) | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
wantRange := protocol.Range{ | ||
End: want, | ||
} | ||
if gotRange != wantRange { | ||
t.Errorf("Range(%d) = %v, want %v", pos, got, want) | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.