Skip to content

Commit

Permalink
Scripting: Whitelist API spec gradle task (#66050) (#66115)
Browse files Browse the repository at this point in the history
Adds `generateContextApiSpec` gradle task that generates whitelist api
specs under `modules/lang-painless/src/main/generated/whitelist-json`.

The common classes are in `painless-common.json`, the specialized classes
per context are in `painless-$context.json`.

eg. `painless-aggs.json` has the specialization for the aggs contexts

Refs: #49879
 Backport: 2aa2224
  • Loading branch information
stu-elastic authored Dec 9, 2020
1 parent 3c9cf2b commit 5b8bac3
Show file tree
Hide file tree
Showing 5 changed files with 560 additions and 91 deletions.
20 changes: 20 additions & 0 deletions modules/lang-painless/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,26 @@ tasks.register("generateContextDoc", DefaultTestClustersTask) {
}.assertNormalExitValue()
}
}
/**********************************************
* Context JSON Generation *
**********************************************/
testClusters {
generateContextApiSpecCluster {
testDistribution = 'DEFAULT'
}
}

tasks.register("generateContextApiSpec", DefaultTestClustersTask) {
dependsOn sourceSets.doc.runtimeClasspath
useCluster testClusters.generateContextApiSpecCluster
doFirst {
project.javaexec {
main = 'org.elasticsearch.painless.ContextApiSpecGenerator'
classpath = sourceSets.doc.runtimeClasspath
systemProperty "cluster.uri", "${-> testClusters.generateContextApiSpecCluster.singleNode().getAllHttpSocketURI().get(0)}"
}.assertNormalExitValue()
}
}

/**********************************************
* Parser regeneration *
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.painless;

import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.painless.action.PainlessContextInfo;

import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;

public class ContextApiSpecGenerator {
public static void main(String[] args) throws IOException {
List<PainlessContextInfo> contexts = ContextGeneratorCommon.getContextInfos();
ContextGeneratorCommon.PainlessInfos infos = new ContextGeneratorCommon.PainlessInfos(contexts);
Path rootDir = resetRootDir();
Path json = rootDir.resolve("painless-common.json");
try (PrintStream jsonStream = new PrintStream(
Files.newOutputStream(json, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
false, StandardCharsets.UTF_8.name())) {

XContentBuilder builder = XContentFactory.jsonBuilder(jsonStream);
builder.startObject();
builder.field(PainlessContextInfo.CLASSES.getPreferredName(), infos.common);
builder.endObject();
builder.flush();
}

for (PainlessInfoJson.Context context : infos.contexts) {
json = rootDir.resolve("painless-" + context.getName() + ".json");
try (PrintStream jsonStream = new PrintStream(
Files.newOutputStream(json, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
false, StandardCharsets.UTF_8.name())) {

XContentBuilder builder = XContentFactory.jsonBuilder(jsonStream);
context.toXContent(builder, null);
builder.flush();
}
}
}

@SuppressForbidden(reason = "resolve context api directory with environment")
private static Path resetRootDir() throws IOException {
Path rootDir = PathUtils.get("./src/main/generated/whitelist-json");
IOUtils.rm(rootDir);
Files.createDirectories(rootDir);

return rootDir;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@

import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.painless.action.PainlessContextClassBindingInfo;
import org.elasticsearch.painless.action.PainlessContextClassInfo;
Expand All @@ -34,16 +32,12 @@

import java.io.IOException;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand All @@ -70,7 +64,7 @@ public final class ContextDocGenerator {
private static final String SHARED_NAME = "Shared";

public static void main(String[] args) throws IOException {
List<PainlessContextInfo> contextInfos = getContextInfos();
List<PainlessContextInfo> contextInfos = ContextGeneratorCommon.getContextInfos();
Set<Object> sharedStaticInfos = createSharedStatics(contextInfos);
Set<PainlessContextClassInfo> sharedClassInfos = createSharedClasses(contextInfos);

Expand Down Expand Up @@ -102,33 +96,6 @@ public static void main(String[] args) throws IOException {
printRootIndexPage(rootDir, contextInfos, isSpecialized);
}

@SuppressForbidden(reason = "retrieving data from an internal API not exposed as part of the REST client")
private static List<PainlessContextInfo> getContextInfos() throws IOException {
URLConnection getContextNames = new URL(
"http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context").openConnection();
XContentParser parser = JsonXContent.jsonXContent.createParser(null, null, getContextNames.getInputStream());
parser.nextToken();
parser.nextToken();
@SuppressWarnings("unchecked")
List<String> contextNames = (List<String>)(Object)parser.list();
parser.close();
((HttpURLConnection)getContextNames).disconnect();

List<PainlessContextInfo> contextInfos = new ArrayList<>();

for (String contextName : contextNames) {
URLConnection getContextInfo = new URL(
"http://" + System.getProperty("cluster.uri") + "/_scripts/painless/_context?context=" + contextName).openConnection();
parser = JsonXContent.jsonXContent.createParser(null, null, getContextInfo.getInputStream());
contextInfos.add(PainlessContextInfo.fromXContent(parser));
((HttpURLConnection)getContextInfo).disconnect();
}

contextInfos.sort(Comparator.comparing(PainlessContextInfo::getName));

return contextInfos;
}

private static Set<Object> createSharedStatics(List<PainlessContextInfo> contextInfos) {
Map<Object, Integer> staticInfoCounts = new HashMap<>();

Expand Down Expand Up @@ -291,7 +258,7 @@ private static void printIndex(PrintStream indexStream, String contextHeader, Ma
indexStream.println();
}

String className = getType(javaNamesToDisplayNames, classInfo.getName());
String className = ContextGeneratorCommon.getType(javaNamesToDisplayNames, classInfo.getName());
indexStream.println("* <<" + getClassHeader(contextHeader, className) + ", " + className + ">>");
}
}
Expand Down Expand Up @@ -354,7 +321,7 @@ private static void printPackages(PrintStream packagesStream, String contextName
"for a high-level overview of all packages and classes.");
}

String className = getType(javaNamesToDisplayNames, classInfo.getName());
String className = ContextGeneratorCommon.getType(javaNamesToDisplayNames, classInfo.getName());
packagesStream.println();
packagesStream.println("[[" + getClassHeader(contextHeader, className) + "]]");
packagesStream.println("==== " + className + "");
Expand Down Expand Up @@ -443,7 +410,7 @@ private static void printConstructor(
parameterIndex < constructorInfo.getParameters().size();
++parameterIndex) {

stream.print(getType(javaNamesToDisplayNames, constructorInfo.getParameters().get(parameterIndex)));
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, constructorInfo.getParameters().get(parameterIndex)));

if (parameterIndex + 1 < constructorInfo.getParameters().size()) {
stream.print(", ");
Expand All @@ -458,7 +425,7 @@ private static void printMethod(
boolean isStatic, PainlessContextMethodInfo methodInfo) {

stream.print("* " + (isStatic ? "static " : ""));
stream.print(getType(javaNamesToDisplayNames, methodInfo.getRtn()) + " ");
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, methodInfo.getRtn()) + " ");

if (methodInfo.getDeclaring().startsWith("java.")) {
stream.print(getMethodJavaDocLink(methodInfo) + "[" + methodInfo.getName() + "]");
Expand All @@ -472,7 +439,7 @@ private static void printMethod(
parameterIndex < methodInfo.getParameters().size();
++parameterIndex) {

stream.print(getType(javaNamesToDisplayNames, methodInfo.getParameters().get(parameterIndex)));
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, methodInfo.getParameters().get(parameterIndex)));

if (parameterIndex + 1 < methodInfo.getParameters().size()) {
stream.print(", ");
Expand All @@ -485,17 +452,21 @@ private static void printMethod(
private static void printClassBinding(
PrintStream stream, Map<String, String> javaNamesToDisplayNames, PainlessContextClassBindingInfo classBindingInfo) {

stream.print("* " + getType(javaNamesToDisplayNames, classBindingInfo.getRtn()) + " " + classBindingInfo.getName() + "(");
stream.print("* " +
ContextGeneratorCommon.getType(javaNamesToDisplayNames, classBindingInfo.getRtn()) +
" " +
classBindingInfo.getName() +
"(");

for (int parameterIndex = 0; parameterIndex < classBindingInfo.getParameters().size(); ++parameterIndex) {
// temporary fix to not print org.elasticsearch.script.ScoreScript parameter until
// class instance bindings are created and the information is appropriately added to the context info classes
if ("org.elasticsearch.script.ScoreScript".equals(
getType(javaNamesToDisplayNames, classBindingInfo.getParameters().get(parameterIndex)))) {
ContextGeneratorCommon.getType(javaNamesToDisplayNames, classBindingInfo.getParameters().get(parameterIndex)))) {
continue;
}

stream.print(getType(javaNamesToDisplayNames, classBindingInfo.getParameters().get(parameterIndex)));
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, classBindingInfo.getParameters().get(parameterIndex)));

if (parameterIndex < classBindingInfo.getReadOnly()) {
stream.print(" *");
Expand All @@ -512,10 +483,14 @@ private static void printClassBinding(
private static void printInstanceBinding(
PrintStream stream, Map<String, String> javaNamesToDisplayNames, PainlessContextInstanceBindingInfo instanceBindingInfo) {

stream.print("* " + getType(javaNamesToDisplayNames, instanceBindingInfo.getRtn()) + " " + instanceBindingInfo.getName() + "(");
stream.print("* " +
ContextGeneratorCommon.getType(javaNamesToDisplayNames, instanceBindingInfo.getRtn()) +
" " +
instanceBindingInfo.getName() +
"(");

for (int parameterIndex = 0; parameterIndex < instanceBindingInfo.getParameters().size(); ++parameterIndex) {
stream.print(getType(javaNamesToDisplayNames, instanceBindingInfo.getParameters().get(parameterIndex)));
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, instanceBindingInfo.getParameters().get(parameterIndex)));

if (parameterIndex + 1 < instanceBindingInfo.getParameters().size()) {
stream.print(", ");
Expand All @@ -530,7 +505,7 @@ private static void printField(
boolean isStatic, PainlessContextFieldInfo fieldInfo) {

stream.print("* " + (isStatic ? "static " : ""));
stream.print(getType(javaNamesToDisplayNames, fieldInfo.getType()) + " ");
stream.print(ContextGeneratorCommon.getType(javaNamesToDisplayNames, fieldInfo.getType()) + " ");

if (fieldInfo.getDeclaring().startsWith("java.")) {
stream.println(getFieldJavaDocLink(fieldInfo) + "[" + fieldInfo.getName() + "]");
Expand All @@ -539,52 +514,6 @@ private static void printField(
}
}

private static String getType(Map<String, String> javaNamesToDisplayNames, String javaType) {
int arrayDimensions = 0;

while (javaType.charAt(arrayDimensions) == '[') {
++arrayDimensions;
}

if (arrayDimensions > 0) {
if (javaType.charAt(javaType.length() - 1) == ';') {
javaType = javaType.substring(arrayDimensions + 1, javaType.length() - 1);
} else {
javaType = javaType.substring(arrayDimensions);
}
}

if ("Z".equals(javaType) || "boolean".equals(javaType)) {
javaType = "boolean";
} else if ("V".equals(javaType) || "void".equals(javaType)) {
javaType = "void";
} else if ("B".equals(javaType) || "byte".equals(javaType)) {
javaType = "byte";
} else if ("S".equals(javaType) || "short".equals(javaType)) {
javaType = "short";
} else if ("C".equals(javaType) || "char".equals(javaType)) {
javaType = "char";
} else if ("I".equals(javaType) || "int".equals(javaType)) {
javaType = "int";
} else if ("J".equals(javaType) || "long".equals(javaType)) {
javaType = "long";
} else if ("F".equals(javaType) || "float".equals(javaType)) {
javaType = "float";
} else if ("D".equals(javaType) || "double".equals(javaType)) {
javaType = "double";
} else if ("org.elasticsearch.painless.lookup.def".equals(javaType)) {
javaType = "def";
} else {
javaType = javaNamesToDisplayNames.get(javaType);
}

while (arrayDimensions-- > 0) {
javaType += "[]";
}

return javaType;
}

private static String getFieldJavaDocLink(PainlessContextFieldInfo fieldInfo) {
return "{java11-javadoc}/java.base/" + fieldInfo.getDeclaring().replace('.', '/') + ".html#" + fieldInfo.getName();
}
Expand Down
Loading

0 comments on commit 5b8bac3

Please sign in to comment.