Initial implementation of server support for training data

Change-Id: I4943b8c30575e78c1028d9d0be174f535db99303
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/97841
Reviewed-by: Ari Aye <ariaye@google.com>
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>

pkg/analysis_server/lib/protocol/protocol_generated.dart

@@ -21542,7 +21542,7 @@ class ServerStatusParams implements HasToJson {
  * {
  *   "lexeme": String
  *   "type": String
- *   "validElementKinds": List<ElementKind>
+ *   "validElementKinds": optional List<String>
  * }
  *
  * Clients may not extend, implement or mix-in this class.
@@ -21552,7 +21552,7 @@ class TokenDetails implements HasToJson {
 
   String _type;
 
-  List<ElementKind> _validElementKinds;
+  List<String> _validElementKinds;
 
   /**
    * The raw token text.
@@ -21583,18 +21583,16 @@ class TokenDetails implements HasToJson {
   /**
    * The kinds of elements which could validly replace this token.
    */
-  List<ElementKind> get validElementKinds => _validElementKinds;
+  List<String> get validElementKinds => _validElementKinds;
 
   /**
    * The kinds of elements which could validly replace this token.
    */
-  void set validElementKinds(List<ElementKind> value) {
-    assert(value != null);
+  void set validElementKinds(List<String> value) {
     this._validElementKinds = value;
   }
 
-  TokenDetails(
-      String lexeme, String type, List<ElementKind> validElementKinds) {
+  TokenDetails(String lexeme, String type, {List<String> validElementKinds}) {
     this.lexeme = lexeme;
     this.type = type;
     this.validElementKinds = validElementKinds;
@@ -21618,17 +21616,15 @@ class TokenDetails implements HasToJson {
       } else {
         throw jsonDecoder.mismatch(jsonPath, "type");
       }
-      List<ElementKind> validElementKinds;
+      List<String> validElementKinds;
       if (json.containsKey("validElementKinds")) {
         validElementKinds = jsonDecoder.decodeList(
             jsonPath + ".validElementKinds",
             json["validElementKinds"],
-            (String jsonPath, Object json) =>
-                new ElementKind.fromJson(jsonDecoder, jsonPath, json));
-      } else {
-        throw jsonDecoder.mismatch(jsonPath, "validElementKinds");
+            jsonDecoder.decodeString);
       }
-      return new TokenDetails(lexeme, type, validElementKinds);
+      return new TokenDetails(lexeme, type,
+          validElementKinds: validElementKinds);
     } else {
       throw jsonDecoder.mismatch(jsonPath, "TokenDetails", json);
     }
@@ -21639,8 +21635,9 @@ class TokenDetails implements HasToJson {
     Map<String, dynamic> result = {};
     result["lexeme"] = lexeme;
     result["type"] = type;
-    result["validElementKinds"] =
-        validElementKinds.map((ElementKind value) => value.toJson()).toList();
+    if (validElementKinds != null) {
+      result["validElementKinds"] = validElementKinds;
+    }
     return result;
   }
 
@@ -21653,7 +21650,7 @@ class TokenDetails implements HasToJson {
       return lexeme == other.lexeme &&
           type == other.type &&
           listEqual(validElementKinds, other.validElementKinds,
-              (ElementKind a, ElementKind b) => a == b);
+              (String a, String b) => a == b);
     }
     return false;
   }

pkg/analysis_server/lib/src/domain_completion.dart

@@ -16,6 +16,7 @@ import 'package:analysis_server/src/provisional/completion/completion_core.dart'
 import 'package:analysis_server/src/services/completion/completion_core.dart';
 import 'package:analysis_server/src/services/completion/completion_performance.dart';
 import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
+import 'package:analysis_server/src/services/completion/token_details/token_detail_builder.dart';
 import 'package:analyzer/dart/analysis/results.dart';
 import 'package:analyzer/dart/analysis/session.dart';
 import 'package:analyzer/src/dart/analysis/driver.dart';
@@ -260,6 +261,9 @@ class CompletionDomainHandler extends AbstractRequestHandler {
       } else if (requestName == COMPLETION_REQUEST_GET_SUGGESTIONS) {
         processRequest(request);
         return Response.DELAYED_RESPONSE;
+      } else if (requestName == COMPLETION_REQUEST_LIST_TOKEN_DETAILS) {
+        listTokenDetails(request);
+        return Response.DELAYED_RESPONSE;
       } else if (requestName == COMPLETION_REQUEST_SET_SUBSCRIPTIONS) {
         return setSubscriptions(request);
       }
@@ -278,6 +282,43 @@ class CompletionDomainHandler extends AbstractRequestHandler {
     }
   }
 
+  /**
+   * Process a `completion.listTokenDetails` request.
+   */
+  Future<void> listTokenDetails(Request request) async {
+    CompletionListTokenDetailsParams params =
+        CompletionListTokenDetailsParams.fromRequest(request);
+
+    String file = params.file;
+    if (server.sendResponseErrorIfInvalidFilePath(request, file)) {
+      return;
+    }
+
+    AnalysisDriver analysisDriver = server.getAnalysisDriver(file);
+    if (analysisDriver == null) {
+      server.sendResponse(Response.invalidParameter(
+        request,
+        'file',
+        'File is not being analyzed: $file',
+      ));
+    }
+    AnalysisSession session = analysisDriver.currentSession;
+    ResolvedUnitResult result = await session.getResolvedUnit(file);
+    if (result.state != ResultState.VALID) {
+      server.sendResponse(Response.invalidParameter(
+        request,
+        'file',
+        'File does not exist or cannot be read: $file',
+      ));
+    }
+
+    TokenDetailBuilder builder = new TokenDetailBuilder();
+    builder.visitNode(result.unit);
+    server.sendResponse(
+      CompletionListTokenDetailsResult(builder.details).toResponse(request.id),
+    );
+  }
+
   /**
    * Process a `completion.getSuggestions` request.
    */

pkg/analysis_server/lib/src/services/completion/token_details/token_detail_builder.dart

@@ -0,0 +1,44 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/protocol/protocol_generated.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/dart/ast/syntactic_entity.dart';
+import 'package:analyzer/dart/ast/token.dart';
+
+/// An object used to build the details for each token in the code being
+/// analyzed.
+class TokenDetailBuilder {
+  /// The list of details that were built.
+  List<TokenDetails> details = [];
+
+  /// Initialize a newly created builder.
+  TokenDetailBuilder();
+
+  /// Visit a [node] in the AST structure to build details for all of the tokens
+  /// contained by that node.
+  void visitNode(AstNode node) {
+    for (SyntacticEntity entity in node.childEntities) {
+      if (entity is Token) {
+        _createDetails(entity, null);
+      } else if (entity is SimpleIdentifier) {
+        List<String> kinds = [];
+        if (entity.inDeclarationContext()) {
+          kinds.add('declaration');
+        } else {
+          kinds.add('identifier');
+        }
+        _createDetails(entity.token, kinds);
+      } else if (entity is AstNode) {
+        visitNode(entity);
+      }
+    }
+  }
+
+  /// Create the details for a single [token], using the given list of [kinds].
+  void _createDetails(Token token, List<String> kinds) {
+    details.add(new TokenDetails(token.lexeme, token.type.name,
+        validElementKinds: kinds));
+  }
+}

pkg/analysis_server/test/domain_completion_test.dart

@@ -23,12 +23,14 @@ import 'mocks.dart';
 
 main() {
   defineReflectiveSuite(() {
+    defineReflectiveTests(CompletionDomainHandlerGetSuggestionsTest);
     defineReflectiveTests(CompletionDomainHandlerTest);
   });
 }
 
 @reflectiveTest
-class CompletionDomainHandlerTest extends AbstractCompletionDomainTest {
+class CompletionDomainHandlerGetSuggestionsTest
+    extends AbstractCompletionDomainTest {
   test_ArgumentList_constructor_named_fieldFormalParam() async {
     // https://github.com/dart-lang/sdk/issues/31023
     addTestFile('''
@@ -863,6 +865,81 @@ class B extends A {m() {^}}
   }
 }
 
+@reflectiveTest
+class CompletionDomainHandlerTest extends AbstractCompletionDomainTest {
+  test_listTokenDetails() async {
+    newFile(testFile, content: '''
+class A {
+  static A b(String s) {}
+  c(int i) {}
+}
+main() {
+  A.b('s').c(3);
+}
+''');
+    Request request =
+        new CompletionListTokenDetailsParams(testFile).toRequest('0');
+    Response response = await waitResponse(request);
+    List<Map<String, dynamic>> tokens = response.result['tokens'];
+    _expectTokens(tokens, [
+      _token('class', 'CLASS', null),
+      _token('A', 'STRING_INT', ['declaration']),
+      _token('{', 'OPEN_CURLY_BRACKET', null),
+      _token('static', 'STATIC', null),
+      _token('A', 'STRING_INT', ['identifier']),
+      _token('b', 'STRING_INT', ['declaration']),
+      _token('(', 'OPEN_PAREN', null),
+      _token('String', 'STRING_INT', ['identifier']),
+      _token('s', 'STRING_INT', ['declaration']),
+      _token(')', 'CLOSE_PAREN', null),
+      _token('{', 'OPEN_CURLY_BRACKET', null),
+      _token('}', 'CLOSE_CURLY_BRACKET', null),
+      _token('c', 'STRING_INT', ['declaration']),
+      _token('(', 'OPEN_PAREN', null),
+      _token('int', 'STRING_INT', ['identifier']),
+      _token('i', 'STRING_INT', ['declaration']),
+      _token(')', 'CLOSE_PAREN', null),
+      _token('{', 'OPEN_CURLY_BRACKET', null),
+      _token('}', 'CLOSE_CURLY_BRACKET', null),
+      _token('}', 'CLOSE_CURLY_BRACKET', null),
+      _token('main', 'STRING_INT', ['declaration']),
+      _token('(', 'OPEN_PAREN', null),
+      _token(')', 'CLOSE_PAREN', null),
+      _token('{', 'OPEN_CURLY_BRACKET', null),
+      _token('A', 'STRING_INT', ['identifier']),
+      _token('.', 'PERIOD', null),
+      _token('b', 'STRING_INT', ['identifier']),
+      _token('(', 'OPEN_PAREN', null),
+      _token("'s'", 'STRING', null),
+      _token(')', 'CLOSE_PAREN', null),
+      _token('.', 'PERIOD', null),
+      _token('c', 'STRING_INT', ['identifier']),
+      _token('(', 'OPEN_PAREN', null),
+      _token('3', 'INT', null),
+      _token(')', 'CLOSE_PAREN', null),
+      _token(';', 'SEMICOLON', null),
+      _token('}', 'CLOSE_CURLY_BRACKET', null),
+    ]);
+  }
+
+  void _expectTokens(List<Map<String, dynamic>> actualTokens,
+      List<TokenDetails> expectedTokens) {
+    int length = expectedTokens.length;
+    expect(actualTokens, hasLength(length));
+    for (int i = 0; i < length; i++) {
+      Map<String, dynamic> actual = actualTokens[i];
+      TokenDetails expected = expectedTokens[i];
+      expect(actual['lexeme'], expected.lexeme);
+      expect(actual['type'], expected.type);
+      expect(actual['validElementKinds'], expected.validElementKinds);
+    }
+  }
+
+  TokenDetails _token(String lexeme, String type, List<String> kinds) {
+    return new TokenDetails(lexeme, type, validElementKinds: kinds);
+  }
+}
+
 class MockRelevancySorter implements DartContributionSorter {
   bool enabled = true;
 

pkg/analysis_server/test/integration/support/protocol_matchers.dart

@@ -1619,15 +1619,12 @@ final Matcher isSourceFileEdit = new LazyMatcher(() => new MatchesJsonObject(
  * {
  *   "lexeme": String
  *   "type": String
- *   "validElementKinds": List<ElementKind>
+ *   "validElementKinds": optional List<String>
  * }
  */
-final Matcher isTokenDetails =
-    new LazyMatcher(() => new MatchesJsonObject("TokenDetails", {
-          "lexeme": isString,
-          "type": isString,
-          "validElementKinds": isListOf(isElementKind)
-        }));
+final Matcher isTokenDetails = new LazyMatcher(() => new MatchesJsonObject(
+    "TokenDetails", {"lexeme": isString, "type": isString},
+    optionalFields: {"validElementKinds": isListOf(isString)}));
 
 /**
  * TypeHierarchyItem

pkg/analysis_server/tool/spec/generated/java/types/TokenDetails.java

@@ -74,7 +74,7 @@ public boolean equals(Object obj) {
   public static TokenDetails fromJson(JsonObject jsonObject) {
     String lexeme = jsonObject.get("lexeme").getAsString();
     String type = jsonObject.get("type").getAsString();
-    List<String> validElementKinds = JsonUtilities.decodeStringList(jsonObject.get("validElementKinds").getAsJsonArray());
+    List<String> validElementKinds = jsonObject.get("validElementKinds") == null ? null : JsonUtilities.decodeStringList(jsonObject.get("validElementKinds").getAsJsonArray());
     return new TokenDetails(lexeme, type, validElementKinds);
   }
 
@@ -124,11 +124,13 @@ public JsonObject toJson() {
     JsonObject jsonObject = new JsonObject();
     jsonObject.addProperty("lexeme", lexeme);
     jsonObject.addProperty("type", type);
-    JsonArray jsonArrayValidElementKinds = new JsonArray();
-    for (String elt : validElementKinds) {
-      jsonArrayValidElementKinds.add(new JsonPrimitive(elt));
+    if (validElementKinds != null) {
+      JsonArray jsonArrayValidElementKinds = new JsonArray();
+      for (String elt : validElementKinds) {
+        jsonArrayValidElementKinds.add(new JsonPrimitive(elt));
+      }
+      jsonObject.add("validElementKinds", jsonArrayValidElementKinds);
     }
-    jsonObject.add("validElementKinds", jsonArrayValidElementKinds);
     return jsonObject;
   }
 

pkg/analysis_server/tool/spec/spec_input.html

@@ -3901,9 +3901,9 @@ <h2 class="domain"><a name="types">Types</a></h2>
           The type of this token.
         </p>
       </field>
-      <field name="validElementKinds">
+      <field name="validElementKinds" optional="true">
         <list>
-          <ref>ElementKind</ref>
+          <ref>String</ref>
         </list>
         <p>
           The kinds of elements which could validly replace this token.