From bf7fe0c833ae709874d1e20d23c90a3ebf95e58a Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 7 Sep 2022 15:03:14 -0700 Subject: [PATCH 01/50] adding wrapper --- .../samples/simpleWrapper/SimpleWrapper.java | 848 ++++++++++++++++++ 1 file changed, 848 insertions(+) create mode 100644 tritonserver/samples/simpleWrapper/SimpleWrapper.java diff --git a/tritonserver/samples/simpleWrapper/SimpleWrapper.java b/tritonserver/samples/simpleWrapper/SimpleWrapper.java new file mode 100644 index 00000000000..b89bcc1cd5f --- /dev/null +++ b/tritonserver/samples/simpleWrapper/SimpleWrapper.java @@ -0,0 +1,848 @@ +// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; +import com.google.gson.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.tritonserver.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritonserver.*; + +public class SimpleWrapper { + + static void FAIL(String MSG) { + System.err.println("Failure: " + MSG); + System.exit(1); + } + + static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { + if (err__ != null) { + System.err.println("error: " + MSG + ":" + + TRITONSERVER_ErrorCodeString(err__) + " - " + + TRITONSERVER_ErrorMessage(err__)); + TRITONSERVER_ErrorDelete(err__); + System.exit(1); + } + } + + static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; + + static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { + public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } + protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { + DeleteDeallocator(Pointer p) { super(p); } + @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } + } + } + + static void + Usage(String msg) + { + if (msg != null) { + System.err.println(msg); + } + + System.err.println("Usage: java " + SimpleCPUOnly.class.getSimpleName() + " [options]"); + System.err.println("\t-v Enable verbose logging"); + System.err.println("\t-r [model repository absolute path]"); + + System.exit(1); + } + + static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + long byte_size, int preferred_memory_type, + long preferred_memory_type_id, Pointer userp, PointerPointer buffer, + PointerPointer buffer_userp, IntPointer actual_memory_type, + LongPointer actual_memory_type_id) + { + // Initially attempt to make the actual memory type and id that we + // allocate be the same as preferred memory type + actual_memory_type.put(0, preferred_memory_type); + actual_memory_type_id.put(0, preferred_memory_type_id); + + // If 'byte_size' is zero just return 'buffer' == nullptr, we don't + // need to do any other book-keeping. + if (byte_size == 0) { + buffer.put(0, null); + buffer_userp.put(0, null); + System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); + } else { + Pointer allocated_ptr = new Pointer(); + actual_memory_type.put(0, requested_memory_type); + + actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); + allocated_ptr = Pointer.malloc(byte_size); + + // Pass the tensor name with buffer_userp so we can show it when + // releasing the buffer. + if (!allocated_ptr.isNull()) { + buffer.put(0, allocated_ptr); + buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); + System.out.println("allocated " + byte_size + " bytes in " + + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) + + " for result tensor " + tensor_name); + } + } + + return null; // Success + } + } + + static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + long byte_size, int memory_type, long memory_type_id) + { + String name = null; + if (buffer_userp != null) { + name = (String)Loader.accessGlobalRef(buffer_userp); + } else { + name = ""; + } + + System.out.println("Releasing buffer " + buffer + " of size " + byte_size + + " in " + TRITONSERVER_MemoryTypeString(memory_type) + + " for result '" + name + "'"); + Pointer.free(buffer); + Loader.deleteGlobalRef(buffer_userp); + + return null; // Success + } + } + + static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { + @Override public void call ( + TRITONSERVER_InferenceRequest request, int flags, Pointer userp) + { + // We reuse the request so we don't delete it here. + } + } + + static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { + @Override public void call ( + TRITONSERVER_InferenceResponse response, int flags, Pointer userp) + { + if (response != null) { + // Send 'response' to the future. + futures.get(userp).complete(response); + } + } + } + + static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); + static ResponseAlloc responseAlloc = new ResponseAlloc(); + static ResponseRelease responseRelease = new ResponseRelease(); + static InferRequestComplete inferRequestComplete = new InferRequestComplete(); + static InferResponseComplete inferResponseComplete = new InferResponseComplete(); + + static TRITONSERVER_Error + ParseModelMetadata( + JsonObject model_metadata, boolean[] is_int, + boolean[] is_torch_model) + { + String seen_data_type = null; + for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { + JsonObject input = input_element.getAsJsonObject(); + if (!input.get("datatype").getAsString().equals("INT32") && + !input.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } + if (seen_data_type == null) { + seen_data_type = input.get("datatype").getAsString(); + } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { + JsonObject output = output_element.getAsJsonObject(); + if (!output.get("datatype").getAsString().equals("INT32") && + !output.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + + is_int[0] = seen_data_type.equals("INT32"); + is_torch_model[0] = + model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); + return null; + } + + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + GenerateInputData( + FloatPointer[] input0_data, FloatPointer[] input1_data) + { + input0_data[0] = new FloatPointer(16); + input1_data[0] = new FloatPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + CompareResult( + String output0_name, String output1_name, + IntPointer input0, IntPointer input1, IntPointer output0, + IntPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + CompareResult( + String output0_name, String output1_name, + FloatPointer input0, FloatPointer input1, FloatPointer output0, + FloatPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + Check( + TRITONSERVER_InferenceResponse response, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + int[] output_count = {0}; + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutputCount(response, output_count), + "getting number of response outputs"); + if (output_count[0] != 2) { + FAIL("expecting 2 response outputs, got " + output_count[0]); + } + + for (int idx = 0; idx < output_count[0]; ++idx) { + BytePointer cname = new BytePointer((Pointer)null); + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutput( + response, idx, cname, datatype, shape, dim_count, base, + byte_size, memory_type, memory_type_id, userp), + "getting output info"); + + if (cname.isNull()) { + FAIL("unable to get output name"); + } + + String name = cname.getString(); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + static void + Check( + InferResult result, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + System.out.println("geting output counts"); + + int output_count = result.OutputNames().limit(); + + + if (output_count != 2) { + FAIL("expecting 2 response outputs, got " + output_count); + } + + for (int idx = 0; idx < output_count; ++idx) { + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + + System.out.println("geting output info"); + String name = result.OutputNames().position(idx); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + + + Tensor output = result.Ouput(name); + dim_count = + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + + public static void + RunInference(String model_repository_path, int verbose_level) throws Exception + { + // Check API version. + int[] api_version_major = {0}, api_version_minor = {0}; + FAIL_IF_ERR( + TRITONSERVER_ApiVersion(api_version_major, api_version_minor), + "getting Triton API version"); + if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || + (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { + FAIL("triton server API version mismatch"); + } + + // Create the server... + TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsNew(server_options), + "creating server options"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetModelRepositoryPath( + server_options, model_repository_path), + "setting model repository path"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), + "setting verbose logging level"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetBackendDirectory( + server_options, "/opt/tritonserver/backends"), + "setting backend directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + server_options, "/opt/tritonserver/repoagents"), + "setting repository agent directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), + "setting strict model configuration"); + + TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); + FAIL_IF_ERR( + TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsDelete(server_options), + "deleting server options"); + + TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); + + // Wait until the server is both live and ready. + int health_iters = 0; + while (true) { + boolean[] live = {false}, ready = {false}; + FAIL_IF_ERR( + TRITONSERVER_ServerIsLive(server, live), + "unable to get server liveness"); + FAIL_IF_ERR( + TRITONSERVER_ServerIsReady(server, ready), + "unable to get server readiness"); + System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); + if (live[0] && ready[0]) { + break; + } + + if (++health_iters >= 10) { + FAIL("failed to find healthy inference server"); + } + + Thread.sleep(500); + } + + // Print status of the server. + { + TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerMetadata(server, server_metadata_message), + "unable to get server metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + server_metadata_message, buffer, byte_size), + "unable to serialize server metadata message"); + + System.out.println("Server Status:"); + System.out.println(buffer.limit(byte_size.get()).getString()); + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(server_metadata_message), + "deleting status metadata"); + } + + String model_name = "simple"; + + // Wait for the model to become available. + boolean[] is_torch_model = {false}; + boolean[] is_int = {true}; + boolean[] is_ready = {false}; + health_iters = 0; + while (!is_ready[0]) { + FAIL_IF_ERR( + TRITONSERVER_ServerModelIsReady( + server, model_name, 1, is_ready), + "unable to get model readiness"); + if (!is_ready[0]) { + if (++health_iters >= 10) { + FAIL("model failed to be ready in 10 iterations"); + } + Thread.sleep(500); + continue; + } + + TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerModelMetadata( + server, model_name, 1, model_metadata_message), + "unable to get model metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + model_metadata_message, buffer, byte_size), + "unable to serialize model status protobuf"); + + JsonParser parser = new JsonParser(); + JsonObject model_metadata = null; + try { + model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); + } catch (Exception e) { + FAIL("error: failed to parse model metadata from JSON: " + e); + } + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(model_metadata_message), + "deleting status protobuf"); + + if (!model_metadata.get("name").getAsString().equals(model_name)) { + FAIL("unable to find metadata for model"); + } + + boolean found_version = false; + if (model_metadata.has("versions")) { + for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { + if (version.getAsString().equals("1")) { + found_version = true; + break; + } + } + } + if (!found_version) { + FAIL("unable to find version 1 status for model"); + } + + FAIL_IF_ERR( + ParseModelMetadata(model_metadata, is_int, is_torch_model), + "parsing model metadata"); + } + + // Create the allocator that will be used to allocate buffers for + // the result tensors. + TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorNew( + allocator, responseAlloc, responseRelease, null /* start_fn */), + "creating response allocator"); + + // Inference + TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestNew( + irequest, server, model_name, -1 /* model_version */), + "creating inference request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), + "setting ID for the request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, inferRequestComplete, null /* request_release_userp */), + "setting request release callback"); + + // Inputs + String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; + String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; + + long[] input0_shape = {1, 16}; + long[] input1_shape = {1, 16}; + + int datatype = + (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input0, datatype, input0_shape, input0_shape.length), + "setting input 0 meta-data for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input1, datatype, input1_shape, input1_shape.length), + "setting input 1 meta-data for the request"); + + String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; + String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), + "requesting output 0 for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), + "requesting output 1 for the request"); + + // Create the data for the two input tensors. Initialize the first + // to unique values and the second to all ones. + BytePointer input0_data; + BytePointer input1_data; + if (is_int[0]) { + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } else { + FloatPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } + + long input0_size = input0_data.limit(); + long input1_size = input1_data.limit(); + + Pointer input0_base = input0_data; + Pointer input1_base = input1_data; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input0_base, input0_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input1, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data"); + + // Perform inference... + { + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Modify some input data in place and then reuse the request + // object. + { + if (is_int[0]) { + new IntPointer(input0_data).put(0, 27); + } else { + new FloatPointer(input0_data).put(0, 27.0f); + } + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Remove input data and then add back different data. + { + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), + "removing INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data to INPUT0"); + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + // Both inputs are using input1_data... + Check( + completed_response, input1_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestDelete(irequest), + "deleting inference request"); + + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorDelete(allocator), + "deleting response allocator"); + } + + public static void + main(String[] args) throws Exception + { + String model_repository_path = null; + int verbose_level = 0; + + // Parse commandline... + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-r": + model_repository_path = args[++i]; + break; + case "-v": + verbose_level = 1; + break; + case "-?": + Usage(null); + break; + } + } + + if (model_repository_path == null) { + Usage("-r must be used to specify model repository path"); + } + + try (PointerScope scope = new PointerScope()) { + RunInference(model_repository_path, verbose_level); + } + + System.exit(0); + } +} From 79621106a732b5a7b2ea68abbee42c91f449a1ed Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 28 Oct 2022 17:34:16 -0700 Subject: [PATCH 02/50] add the tritonserver folder --- platform/pom.xml | 6 + pom.xml | 2 + tritonserver-wrapper/README.md | 2 + tritonserver-wrapper/platform/pom.xml | 126 ++++++++++++++++++ tritonserver-wrapper/platform/redist/pom.xml | 118 ++++++++++++++++ tritonserver-wrapper/pom.xml | 116 ++++++++++++++++ .../src/main/java9/module-info.java | 6 + 7 files changed, 376 insertions(+) create mode 100644 tritonserver-wrapper/README.md create mode 100644 tritonserver-wrapper/platform/pom.xml create mode 100644 tritonserver-wrapper/platform/redist/pom.xml create mode 100644 tritonserver-wrapper/pom.xml create mode 100644 tritonserver-wrapper/src/main/java9/module-info.java diff --git a/platform/pom.xml b/platform/pom.xml index 0514c065517..e96c6de893f 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,6 +61,7 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform + ../tritonserver-wrapper/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -313,6 +314,11 @@ tritonserver-platform 2.26-${project.version} + + org.bytedeco + tritonserver-wrapper-platform + 2.24-${project.version} + org.bytedeco ale-platform diff --git a/pom.xml b/pom.xml index bbd26497b9e..d88f3ef727e 100644 --- a/pom.xml +++ b/pom.xml @@ -623,6 +623,7 @@ tensorflow-lite tensorrt tritonserver + tritonserver-wrapper ale depthai onnx @@ -1406,6 +1407,7 @@ tensorflow-lite tensorrt tritonserver + tritonserver-wrapper ale depthai onnx diff --git a/tritonserver-wrapper/README.md b/tritonserver-wrapper/README.md new file mode 100644 index 00000000000..397e51e4d6c --- /dev/null +++ b/tritonserver-wrapper/README.md @@ -0,0 +1,2 @@ +JavaCPP Presets for Triton Inference Server +=========================================== diff --git a/tritonserver-wrapper/platform/pom.xml b/tritonserver-wrapper/platform/pom.xml new file mode 100644 index 00000000000..9b0e8606541 --- /dev/null +++ b/tritonserver-wrapper/platform/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + ../../ + + + org.bytedeco + tritonserver-wrapper-platform + 2.26-${project.parent.version} + JavaCPP Presets Platform for Triton Inference Server Wrapper + + + tritonserver-wrapper + + + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-x86_64} + + + com.google.code.gson + gson + 2.9.0 + + + + + + + maven-jar-plugin + + + default-jar + + + + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64.jar ${javacpp.moduleId}-linux-x86_64.jar ${javacpp.moduleId}-windows-x86_64.jar + + + + + + empty-javadoc-jar + + jar + + + javadoc + + + + empty-sources-jar + + jar + + + sources + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + none + + + add-platform-module-info + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}.jar + + module org.bytedeco.${javacpp.moduleId}.platform { +// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; + requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64; +// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64; + } + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + true + + + + + + + + diff --git a/tritonserver-wrapper/platform/redist/pom.xml b/tritonserver-wrapper/platform/redist/pom.xml new file mode 100644 index 00000000000..0bb28bb524f --- /dev/null +++ b/tritonserver-wrapper/platform/redist/pom.xml @@ -0,0 +1,118 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + ../../../ + + + org.bytedeco + tritonserver-wrapper-platform-redist + 2.26-${project.parent.version} + JavaCPP Presets Platform Redist for Triton Inference Server + + + tritonserver-wrapper + -redist + + + + + ${project.groupId} + ${javacpp.moduleId}-platform + ${project.version} + + + + + + + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-x86_64} + + + + + + + + + + + + + maven-jar-plugin + + + default-jar + + + + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64-redist.jar ${javacpp.moduleId}-linux-x86_64-redist.jar ${javacpp.moduleId}-windows-x86_64-redist.jar + + + + + + empty-javadoc-jar + + jar + + + javadoc + + + + empty-sources-jar + + jar + + + sources + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + none + + + add-platform-module-info + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}.jar + + module org.bytedeco.${javacpp.moduleId}.platform.redist { +// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64.redist; + requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64.redist; +// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64.redist; + } + + + + + + + + + + + diff --git a/tritonserver-wrapper/pom.xml b/tritonserver-wrapper/pom.xml new file mode 100644 index 00000000000..093fc778430 --- /dev/null +++ b/tritonserver-wrapper/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + + + org.bytedeco + tritonserver-wrapper + 2.26-${project.parent.version} + JavaCPP Presets for Triton Inference Server + + + + org.bytedeco + javacpp + + + + + + + maven-resources-plugin + + + maven-compiler-plugin + + + org.bytedeco + javacpp + + ISO-8859-1 + + + + maven-jar-plugin + + + javacpp-${javacpp.platform} + package + + jar + + + ${javacpp.platform} + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + META-INF/native-image/${javacpp.platform}/ + + + + + javacpp-${javacpp.platform}-redist + package + + jar + + + ${javacpp.platform}-redist + ${project.build.directory}/native + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/ + META-INF/native-image/${javacpp.platform}/ + + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + + + + + + + org.moditect + moditect-maven-plugin + + + add-module-info-redist + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}-${javacpp.platform}-redist.jar + + open module org.bytedeco.${javacpp.packageName}.${javacpp.platform.module}.redist { + requires transitive org.bytedeco.${javacpp.packageName}; + } + + + + + + + + + maven-dependency-plugin + + + maven-source-plugin + + + maven-javadoc-plugin + + ISO-8859-1 + + + + + + diff --git a/tritonserver-wrapper/src/main/java9/module-info.java b/tritonserver-wrapper/src/main/java9/module-info.java new file mode 100644 index 00000000000..6a1b64a9d46 --- /dev/null +++ b/tritonserver-wrapper/src/main/java9/module-info.java @@ -0,0 +1,6 @@ +module org.bytedeco.tritonserver { + requires transitive org.bytedeco.javacpp; + exports org.bytedeco.tritonserver.global; + exports org.bytedeco.tritonserver.presets; + exports org.bytedeco.tritonserver; +} From 210c516120f835747cc2fc1f04b967dc8f0e82c9 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 3 Nov 2022 12:22:17 -0700 Subject: [PATCH 03/50] add wrapper repo --- platform/pom.xml | 4 +- pom.xml | 4 +- .../src/main/java9/module-info.java | 6 - .../README.md | 0 tritonserverwrapper/cppbuild.sh | 30 + .../platform/pom.xml | 14 +- .../platform/redist/pom.xml | 16 +- .../pom.xml | 29 +- .../samples/simple/Simple.java | 764 ++++++++++++++++++ tritonserverwrapper/samples/simple/pom.xml | 22 + .../presets/tritonserverwrapper.java | 71 ++ .../src/main/java9/module-info.java | 6 + 12 files changed, 930 insertions(+), 36 deletions(-) delete mode 100644 tritonserver-wrapper/src/main/java9/module-info.java rename {tritonserver-wrapper => tritonserverwrapper}/README.md (100%) create mode 100755 tritonserverwrapper/cppbuild.sh rename {tritonserver-wrapper => tritonserverwrapper}/platform/pom.xml (92%) rename {tritonserver-wrapper => tritonserverwrapper}/platform/redist/pom.xml (83%) rename {tritonserver-wrapper => tritonserverwrapper}/pom.xml (77%) create mode 100644 tritonserverwrapper/samples/simple/Simple.java create mode 100644 tritonserverwrapper/samples/simple/pom.xml create mode 100644 tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java create mode 100644 tritonserverwrapper/src/main/java9/module-info.java diff --git a/platform/pom.xml b/platform/pom.xml index e96c6de893f..0078f97349d 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,7 +61,7 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform - ../tritonserver-wrapper/platform + ../tritonserverwrapper/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -316,7 +316,7 @@ org.bytedeco - tritonserver-wrapper-platform + tritonserverwrapper-platform 2.24-${project.version} diff --git a/pom.xml b/pom.xml index d88f3ef727e..74711303ea9 100644 --- a/pom.xml +++ b/pom.xml @@ -623,7 +623,7 @@ tensorflow-lite tensorrt tritonserver - tritonserver-wrapper + tritonserverwrapper ale depthai onnx @@ -1407,7 +1407,7 @@ tensorflow-lite tensorrt tritonserver - tritonserver-wrapper + tritonserverwrapper ale depthai onnx diff --git a/tritonserver-wrapper/src/main/java9/module-info.java b/tritonserver-wrapper/src/main/java9/module-info.java deleted file mode 100644 index 6a1b64a9d46..00000000000 --- a/tritonserver-wrapper/src/main/java9/module-info.java +++ /dev/null @@ -1,6 +0,0 @@ -module org.bytedeco.tritonserver { - requires transitive org.bytedeco.javacpp; - exports org.bytedeco.tritonserver.global; - exports org.bytedeco.tritonserver.presets; - exports org.bytedeco.tritonserver; -} diff --git a/tritonserver-wrapper/README.md b/tritonserverwrapper/README.md similarity index 100% rename from tritonserver-wrapper/README.md rename to tritonserverwrapper/README.md diff --git a/tritonserverwrapper/cppbuild.sh b/tritonserverwrapper/cppbuild.sh new file mode 100755 index 00000000000..87fe0b381f1 --- /dev/null +++ b/tritonserverwrapper/cppbuild.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# This file is meant to be included by the parent cppbuild.sh script +if [[ -z "$PLATFORM" ]]; then + pushd .. + bash cppbuild.sh "$@" tritonserverwrapper + popd + exit +fi + +case $PLATFORM in + linux-arm64) + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + echo "Please make sure library and include files exist" + exit 1 + fi + ;; + linux-x86_64) + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + echo "Please make sure library and include files exist" + exit 1 + fi + ;; + windows-x86_64) + echo "Windows is not supported yet" + exit 1 + ;; + *) + echo "Error: Platform \"$PLATFORM\" is not supported" + ;; +esac diff --git a/tritonserver-wrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml similarity index 92% rename from tritonserver-wrapper/platform/pom.xml rename to tritonserverwrapper/platform/pom.xml index 9b0e8606541..a177256db48 100644 --- a/tritonserver-wrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -11,15 +11,20 @@ org.bytedeco - tritonserver-wrapper-platform + tritonserverwrapper-platform 2.26-${project.parent.version} JavaCPP Presets Platform for Triton Inference Server Wrapper - tritonserver-wrapper + tritonserverwrapper + ${project.groupId} ${javacpp.moduleId} @@ -31,11 +36,6 @@ ${project.version} ${javacpp.platform.linux-x86_64} - - com.google.code.gson - gson - 2.9.0 - diff --git a/tritonserver-wrapper/platform/redist/pom.xml b/tritonserverwrapper/platform/redist/pom.xml similarity index 83% rename from tritonserver-wrapper/platform/redist/pom.xml rename to tritonserverwrapper/platform/redist/pom.xml index 0bb28bb524f..edc9651a14b 100644 --- a/tritonserver-wrapper/platform/redist/pom.xml +++ b/tritonserverwrapper/platform/redist/pom.xml @@ -11,12 +11,12 @@ org.bytedeco - tritonserver-wrapper-platform-redist + tritonserverwrapper-platform-redist 2.26-${project.parent.version} JavaCPP Presets Platform Redist for Triton Inference Server - tritonserver-wrapper + tritonserverwrapper -redist @@ -26,24 +26,12 @@ ${javacpp.moduleId}-platform ${project.version} - - - - - - ${project.groupId} ${javacpp.moduleId} ${project.version} ${javacpp.platform.linux-x86_64} - - - - - - diff --git a/tritonserver-wrapper/pom.xml b/tritonserverwrapper/pom.xml similarity index 77% rename from tritonserver-wrapper/pom.xml rename to tritonserverwrapper/pom.xml index 093fc778430..8f6cda3c6d5 100644 --- a/tritonserver-wrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -10,11 +10,16 @@ org.bytedeco - tritonserver-wrapper + tritonserverwrapper 2.26-${project.parent.version} - JavaCPP Presets for Triton Inference Server + JavaCPP Presets for Triton Inference Server Wrapper + org.bytedeco javacpp @@ -36,6 +41,20 @@ ISO-8859-1 + maven-jar-plugin @@ -48,7 +67,7 @@ ${javacpp.platform} - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* META-INF/native-image/${javacpp.platform}/ @@ -63,11 +82,11 @@ ${javacpp.platform}-redist ${project.build.directory}/native - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/ + org/bytedeco/tritonserverwrapper/${javacpp.platform}/ META-INF/native-image/${javacpp.platform}/ - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* diff --git a/tritonserverwrapper/samples/simple/Simple.java b/tritonserverwrapper/samples/simple/Simple.java new file mode 100644 index 00000000000..214e0a82096 --- /dev/null +++ b/tritonserverwrapper/samples/simple/Simple.java @@ -0,0 +1,764 @@ +// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; +import com.google.gson.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.tritonserver.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritonserver.*; +// import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; +// import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +public class Simple { + + static void FAIL(String MSG) { + System.err.println("Failure: " + MSG); + System.exit(1); + } + + static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { + if (err__ != null) { + System.err.println("error: " + MSG + ":" + + TRITONSERVER_ErrorCodeString(err__) + " - " + + TRITONSERVER_ErrorMessage(err__)); + TRITONSERVER_ErrorDelete(err__); + System.exit(1); + } + } + + static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; + + static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { + public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } + protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { + DeleteDeallocator(Pointer p) { super(p); } + @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } + } + } + + static void + Usage(String msg) + { + if (msg != null) { + System.err.println(msg); + } + + System.err.println("Usage: java " + Simple.class.getSimpleName() + " [options]"); + System.err.println("\t-v Enable verbose logging"); + System.err.println("\t-r [model repository absolute path]"); + + System.exit(1); + } + + static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + long byte_size, int preferred_memory_type, + long preferred_memory_type_id, Pointer userp, PointerPointer buffer, + PointerPointer buffer_userp, IntPointer actual_memory_type, + LongPointer actual_memory_type_id) + { + // Initially attempt to make the actual memory type and id that we + // allocate be the same as preferred memory type + actual_memory_type.put(0, preferred_memory_type); + actual_memory_type_id.put(0, preferred_memory_type_id); + + // If 'byte_size' is zero just return 'buffer' == nullptr, we don't + // need to do any other book-keeping. + if (byte_size == 0) { + buffer.put(0, null); + buffer_userp.put(0, null); + System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); + } else { + Pointer allocated_ptr = new Pointer(); + actual_memory_type.put(0, requested_memory_type); + + actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); + allocated_ptr = Pointer.malloc(byte_size); + + // Pass the tensor name with buffer_userp so we can show it when + // releasing the buffer. + if (!allocated_ptr.isNull()) { + buffer.put(0, allocated_ptr); + buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); + System.out.println("allocated " + byte_size + " bytes in " + + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) + + " for result tensor " + tensor_name); + } + } + + return null; // Success + } + } + + static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + long byte_size, int memory_type, long memory_type_id) + { + String name = null; + if (buffer_userp != null) { + name = (String)Loader.accessGlobalRef(buffer_userp); + } else { + name = ""; + } + + System.out.println("Releasing buffer " + buffer + " of size " + byte_size + + " in " + TRITONSERVER_MemoryTypeString(memory_type) + + " for result '" + name + "'"); + Pointer.free(buffer); + Loader.deleteGlobalRef(buffer_userp); + + return null; // Success + } + } + + static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { + @Override public void call ( + TRITONSERVER_InferenceRequest request, int flags, Pointer userp) + { + // We reuse the request so we don't delete it here. + } + } + + static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { + @Override public void call ( + TRITONSERVER_InferenceResponse response, int flags, Pointer userp) + { + if (response != null) { + // Send 'response' to the future. + futures.get(userp).complete(response); + } + } + } + + static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); + static ResponseAlloc responseAlloc = new ResponseAlloc(); + static ResponseRelease responseRelease = new ResponseRelease(); + static InferRequestComplete inferRequestComplete = new InferRequestComplete(); + static InferResponseComplete inferResponseComplete = new InferResponseComplete(); + + static TRITONSERVER_Error + ParseModelMetadata( + JsonObject model_metadata, boolean[] is_int, + boolean[] is_torch_model) + { + String seen_data_type = null; + for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { + JsonObject input = input_element.getAsJsonObject(); + if (!input.get("datatype").getAsString().equals("INT32") && + !input.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } + if (seen_data_type == null) { + seen_data_type = input.get("datatype").getAsString(); + } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { + JsonObject output = output_element.getAsJsonObject(); + if (!output.get("datatype").getAsString().equals("INT32") && + !output.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + + is_int[0] = seen_data_type.equals("INT32"); + is_torch_model[0] = + model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); + return null; + } + + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + GenerateInputData( + FloatPointer[] input0_data, FloatPointer[] input1_data) + { + input0_data[0] = new FloatPointer(16); + input1_data[0] = new FloatPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + CompareResult( + String output0_name, String output1_name, + IntPointer input0, IntPointer input1, IntPointer output0, + IntPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + CompareResult( + String output0_name, String output1_name, + FloatPointer input0, FloatPointer input1, FloatPointer output0, + FloatPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + Check( + TRITONSERVER_InferenceResponse response, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + int[] output_count = {0}; + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutputCount(response, output_count), + "getting number of response outputs"); + if (output_count[0] != 2) { + FAIL("expecting 2 response outputs, got " + output_count[0]); + } + + for (int idx = 0; idx < output_count[0]; ++idx) { + BytePointer cname = new BytePointer((Pointer)null); + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutput( + response, idx, cname, datatype, shape, dim_count, base, + byte_size, memory_type, memory_type_id, userp), + "getting output info"); + + if (cname.isNull()) { + FAIL("unable to get output name"); + } + + String name = cname.getString(); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + public static void + RunInference(String model_repository_path, int verbose_level) throws Exception + { + // Check API version. + int[] api_version_major = {0}, api_version_minor = {0}; + FAIL_IF_ERR( + TRITONSERVER_ApiVersion(api_version_major, api_version_minor), + "getting Triton API version"); + if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || + (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { + FAIL("triton server API version mismatch"); + } + + // Create the server... + TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsNew(server_options), + "creating server options"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetModelRepositoryPath( + server_options, model_repository_path), + "setting model repository path"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), + "setting verbose logging level"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetBackendDirectory( + server_options, "/opt/tritonserver/backends"), + "setting backend directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + server_options, "/opt/tritonserver/repoagents"), + "setting repository agent directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), + "setting strict model configuration"); + + TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); + FAIL_IF_ERR( + TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsDelete(server_options), + "deleting server options"); + + TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); + + // Wait until the server is both live and ready. + int health_iters = 0; + while (true) { + boolean[] live = {false}, ready = {false}; + FAIL_IF_ERR( + TRITONSERVER_ServerIsLive(server, live), + "unable to get server liveness"); + FAIL_IF_ERR( + TRITONSERVER_ServerIsReady(server, ready), + "unable to get server readiness"); + System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); + if (live[0] && ready[0]) { + break; + } + + if (++health_iters >= 10) { + FAIL("failed to find healthy inference server"); + } + + Thread.sleep(500); + } + + // Print status of the server. + { + TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerMetadata(server, server_metadata_message), + "unable to get server metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + server_metadata_message, buffer, byte_size), + "unable to serialize server metadata message"); + + System.out.println("Server Status:"); + System.out.println(buffer.limit(byte_size.get()).getString()); + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(server_metadata_message), + "deleting status metadata"); + } + + String model_name = "simple"; + + // Wait for the model to become available. + boolean[] is_torch_model = {false}; + boolean[] is_int = {true}; + boolean[] is_ready = {false}; + health_iters = 0; + while (!is_ready[0]) { + FAIL_IF_ERR( + TRITONSERVER_ServerModelIsReady( + server, model_name, 1, is_ready), + "unable to get model readiness"); + if (!is_ready[0]) { + if (++health_iters >= 10) { + FAIL("model failed to be ready in 10 iterations"); + } + Thread.sleep(500); + continue; + } + + TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerModelMetadata( + server, model_name, 1, model_metadata_message), + "unable to get model metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + model_metadata_message, buffer, byte_size), + "unable to serialize model status protobuf"); + + JsonParser parser = new JsonParser(); + JsonObject model_metadata = null; + try { + model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); + } catch (Exception e) { + FAIL("error: failed to parse model metadata from JSON: " + e); + } + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(model_metadata_message), + "deleting status protobuf"); + + if (!model_metadata.get("name").getAsString().equals(model_name)) { + FAIL("unable to find metadata for model"); + } + + boolean found_version = false; + if (model_metadata.has("versions")) { + for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { + if (version.getAsString().equals("1")) { + found_version = true; + break; + } + } + } + if (!found_version) { + FAIL("unable to find version 1 status for model"); + } + + FAIL_IF_ERR( + ParseModelMetadata(model_metadata, is_int, is_torch_model), + "parsing model metadata"); + } + + // Create the allocator that will be used to allocate buffers for + // the result tensors. + TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorNew( + allocator, responseAlloc, responseRelease, null /* start_fn */), + "creating response allocator"); + + // Inference + TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestNew( + irequest, server, model_name, -1 /* model_version */), + "creating inference request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), + "setting ID for the request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, inferRequestComplete, null /* request_release_userp */), + "setting request release callback"); + + // Inputs + String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; + String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; + + long[] input0_shape = {1, 16}; + long[] input1_shape = {1, 16}; + + int datatype = + (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input0, datatype, input0_shape, input0_shape.length), + "setting input 0 meta-data for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input1, datatype, input1_shape, input1_shape.length), + "setting input 1 meta-data for the request"); + + String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; + String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), + "requesting output 0 for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), + "requesting output 1 for the request"); + + // Create the data for the two input tensors. Initialize the first + // to unique values and the second to all ones. + BytePointer input0_data; + BytePointer input1_data; + if (is_int[0]) { + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } else { + FloatPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } + + long input0_size = input0_data.limit(); + long input1_size = input1_data.limit(); + + Pointer input0_base = input0_data; + Pointer input1_base = input1_data; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input0_base, input0_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input1, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data"); + + // Perform inference... + { + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Modify some input data in place and then reuse the request + // object. + { + if (is_int[0]) { + new IntPointer(input0_data).put(0, 27); + } else { + new FloatPointer(input0_data).put(0, 27.0f); + } + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Remove input data and then add back different data. + { + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), + "removing INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data to INPUT0"); + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + // Both inputs are using input1_data... + Check( + completed_response, input1_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestDelete(irequest), + "deleting inference request"); + + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorDelete(allocator), + "deleting response allocator"); + } + + public static void + main(String[] args) throws Exception + { + String model_repository_path = null; + int verbose_level = 0; + + // Parse commandline... + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-r": + model_repository_path = args[++i]; + break; + case "-v": + verbose_level = 1; + break; + case "-?": + Usage(null); + break; + } + } + + if (model_repository_path == null) { + Usage("-r must be used to specify model repository path"); + } + + try (PointerScope scope = new PointerScope()) { + RunInference(model_repository_path, verbose_level); + } + + System.exit(0); + } +} diff --git a/tritonserverwrapper/samples/simple/pom.xml b/tritonserverwrapper/samples/simple/pom.xml new file mode 100644 index 00000000000..59c5b1d5fe2 --- /dev/null +++ b/tritonserverwrapper/samples/simple/pom.xml @@ -0,0 +1,22 @@ + + 4.0.0 + org.bytedeco.tritonserverwrapper + simple + 1.5.8-SNAPSHOT + + Simple + 1.8 + 1.8 + + + + org.bytedeco + tritonserverwrapper-platform + 2.26-1.5.8-SNAPSHOT + shaded + + + + . + + diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java new file mode 100644 index 00000000000..0c0efebe33c --- /dev/null +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2021 Jack He, Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.tritonserverwrapper.presets; + +import java.util.List; +import org.bytedeco.javacpp.ClassProperties; +import org.bytedeco.javacpp.LoadEnabled; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; +// import org.bytedeco.tritonserver.presets.tritonserver; + +/** + * + * @author Katherine Yang + */ +@Properties( + value = { + @Platform( + value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, + include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h","server_wrapper.h", "infer_requested_output.h", "common.h"}, + link = "tritonserverwrapper", + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, + linkpath = {"/opt/tritonserver/lib/"} + ), + // @Platform( + // value = "windows-x86_64", + // includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", + // linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", + // preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" + // ) + }, + target = "org.bytedeco.tritonserverwrapper.tritonserverwrapper", + global = "org.bytedeco.tritonserverwrapper.global.tritonserverwrapper" +) +public class tritonserverwrapper implements InfoMapper { + static { Loader.checkVersion("org.bytedeco", "tritonserverwrapper"); } + public void map(InfoMap infoMap) { + infoMap.putFirst(new Info().enumerate(false)) + .put(new Info("bool").cast().valueTypes("boolean").pointerTypes("boolean[]", "BoolPointer")) + .put(new Info("const char").pointerTypes("String", "@Cast(\"const char*\") BytePointer")) + .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) + .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", + "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + ; + } +} diff --git a/tritonserverwrapper/src/main/java9/module-info.java b/tritonserverwrapper/src/main/java9/module-info.java new file mode 100644 index 00000000000..d6f16438d12 --- /dev/null +++ b/tritonserverwrapper/src/main/java9/module-info.java @@ -0,0 +1,6 @@ +module org.bytedeco.tritonserverwrapper { + requires transitive org.bytedeco.javacpp; + exports org.bytedeco.tritonserverwrapper.global; + exports org.bytedeco.tritonserverwrapper.presets; + exports org.bytedeco.tritonserverwrapper; +} From 05ed6ab35362082d4b533eab74201b64e5aba28d Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 3 Nov 2022 12:29:49 -0700 Subject: [PATCH 04/50] update tritonserver gen files and also start compiling with only wrapper --- .../tritonserver/global/tritonserver.java | 174 ++++++++++++++++-- .../TRITONBACKEND_BackendAttribute.java | 23 +++ .../TRITONBACKEND_ModelInstance.java | 6 - tritonserverwrapper/platform/pom.xml | 4 +- tritonserverwrapper/pom.xml | 4 +- .../presets/tritonserverwrapper.java | 4 +- 6 files changed, 191 insertions(+), 24 deletions(-) create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 0d486ae5aff..90f01e2fabf 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -130,7 +130,7 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver public static final int TRITONSERVER_API_VERSION_MAJOR = 1; /// -public static final int TRITONSERVER_API_VERSION_MINOR = 13; +public static final int TRITONSERVER_API_VERSION_MINOR = 17; /** Get the TRITONBACKEND API version supported by the Triton shared * library. This value can be compared against the @@ -240,7 +240,7 @@ public static native String TRITONSERVER_MemoryTypeString( TRITONSERVER_PARAMETER_BOOL = 2, TRITONSERVER_PARAMETER_BYTES = 3; -/** Get the string representation of a parmeter type. The returned +/** Get the string representation of a parameter type. The returned * string is not owned by the caller and so should not be modified or * freed. * @@ -328,6 +328,19 @@ public static native String TRITONSERVER_InstanceGroupKindString( TRITONSERVER_LOG_ERROR = 2, TRITONSERVER_LOG_VERBOSE = 3; +/** + * Format of logging. + * + * TRITONSERVER_LOG_DEFAULT: the log severity (L) and timestamp will be + * logged as "LMMDD hh:mm:ss.ssssss". + * + * TRITONSERVER_LOG_ISO8601: the log format will be "YYYY-MM-DDThh:mm:ssZ L". + * */ +/** enum TRITONSERVER_LogFormat */ +public static final int + TRITONSERVER_LOG_DEFAULT = 0, + TRITONSERVER_LOG_ISO8601 = 1; + /** Is a log level enabled? * * @param level The log level. @@ -1996,6 +2009,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout /** Set the number of threads used in buffer manager in a server options. * + * @param options The server options object. * @param thread_count The number of threads. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2003,6 +2017,30 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); +/** Set the number of threads to concurrently load models in a server options. + * + * @param options The server options object. + * @param thread_count The number of threads. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadThreadCount( + TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); + +/** Provide a log output file. + * + * @param options The server options object. + * @param file a string defining the file where the log outputs will be saved. + * An empty string for the file name will cause triton to direct logging + * facilities to the console + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, String file); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer file); + /** Enable or disable info level logging. * * @param options The server options object. @@ -2033,6 +2071,16 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogWarn( public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogError( TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); +/** Set the logging format. + * + * @param options The server options object. + * @param format The logging format. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFormat( + TRITONSERVER_ServerOptions options, @Cast("const TRITONSERVER_LogFormat") int format); + /** Set verbose logging level. Level zero disables verbose logging. * * @param options The server options object. @@ -2065,6 +2113,18 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetrics( public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetGpuMetrics( TRITONSERVER_ServerOptions options, @Cast("bool") boolean gpu_metrics); +/** Enable or disable CPU metrics collection in a server options. CPU + * metrics are collected if both this option and + * TRITONSERVER_ServerOptionsSetMetrics are true. + * + * @param options The server options object. + * @param cpu_metrics True to enable CPU metrics, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCpuMetrics( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean cpu_metrics); + /** Set the interval for metrics collection in a server options. * This is 2000 milliseconds by default. * @@ -2102,12 +2162,32 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDire * @param repoagent_dir The full path of the repository agent directory. * @return a TRITONSERVER_Error indicating success or failure. */ +/// /// public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( TRITONSERVER_ServerOptions options, String repoagent_dir); public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer repoagent_dir); +/** Specify the limit on memory usage as a fraction on the device identified by + * 'kind' and 'device_id'. If model loading on the device is requested and the + * current memory usage exceeds the limit, the load will be rejected. If not + * specified, the limit will not be set. + * + * Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU + * + * @param options The server options object. + * @param kind The kind of the device. + * @param device_id The id of the device. + * @param fraction The limit on memory usage as a fraction + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( + TRITONSERVER_ServerOptions options, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, int device_id, + double fraction); + /** Set a configuration setting for a named backend in a server * options. * @@ -2643,6 +2723,9 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer description); /** Delete a metric family object. + * A TRITONSERVER_MetricFamily* object should be deleted AFTER its + * corresponding TRITONSERVER_Metric* objects have been deleted. + * Attempting to delete a family before its metrics will return an error. * * @param family The metric family object to delete. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2653,7 +2736,10 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyDelete( /** Create a new metric object. The caller takes ownership of the * TRITONSERVER_Metric object and must call - * TRITONSERVER_MetricDelete to release the object. + * TRITONSERVER_MetricDelete to release the object. The caller is also + * responsible for ownership of the labels passed in. Each label can be deleted + * immediately after creating the metric with TRITONSERVER_ParameterDelete + * if not re-using the labels. * * @param metric Returns the new metric object. * @param family The metric family to add this new metric to. @@ -2670,6 +2756,9 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricNew( @Const @ByPtrPtr TRITONSERVER_Parameter labels, @Cast("const uint64_t") long label_count); /** Delete a metric object. + * All TRITONSERVER_Metric* objects should be deleted BEFORE their + * corresponding TRITONSERVER_MetricFamily* objects have been deleted. + * If a family is deleted before its metrics, an error will be returned. * * @param metric The metric object to delete. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2822,6 +2911,9 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( // Targeting ../tritonserver/TRITONBACKEND_ModelInstance.java +// Targeting ../tritonserver/TRITONBACKEND_BackendAttribute.java + + /** * TRITONBACKEND API Version @@ -2851,7 +2943,7 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; /// -public static final int TRITONBACKEND_API_VERSION_MINOR = 9; +public static final int TRITONBACKEND_API_VERSION_MINOR = 10; /** Get the TRITONBACKEND API version supported by Triton. This value * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and @@ -4155,12 +4247,16 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); /** Set the model configuration in Triton server. Only the inputs, outputs, - * and max batch size can be changed. Any other changes to the model - * configuration will be ignored by Triton. This function can only be called - * from TRITONBACKEND_ModelInitialize, calling in any other context will result - * in an error being returned. The function does not take ownership of the - * message object and so the caller should call TRITONSERVER_MessageDelete to - * release the object once the function returns. + * max batch size, and scheduling choice can be changed. A caveat being + * scheduling choice can only be changed if none is previously set. Any other + * changes to the model configuration will be ignored by Triton. This function + * can only be called from TRITONBACKEND_ModelInitialize, calling in any other + * context will result in an error being returned. Additionally, Triton server + * can add some of the missing fields in the provided config with this call. + * The backend must get the complete configuration again by using + * TRITONBACKEND_ModelConfig. TRITONBACKEND_ModelSetConfig does not take + * ownership of the message object and so the caller should call + * TRITONSERVER_MessageDelete to release the object once the function returns. * * @param model The model. * @param config_version The format version of the model configuration. @@ -4525,7 +4621,6 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportStatist * @return a TRITONSERVER_Error indicating success or failure. */ - /// /// /// @@ -4534,7 +4629,6 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportBatchSt @Cast("const uint64_t") long exec_start_ns, @Cast("const uint64_t") long compute_start_ns, @Cast("const uint64_t") long compute_end_ns, @Cast("const uint64_t") long exec_end_ns); - /** * The following functions can be implemented by a backend. Functions * indicated as required must be implemented or the backend will fail @@ -4648,6 +4742,8 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceFinalize( * @param requests The requests. * @param request_count The number of requests in the batch. * @return a TRITONSERVER_Error indicating success or failure. */ + +/// public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Request**") PointerPointer requests, @Cast("const uint32_t") int request_count); @@ -4655,6 +4751,60 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Request requests, @Cast("const uint32_t") int request_count); +/** Query the backend for different model attributes. This function is optional, + * a backend is not required to implement it. The backend is also not required + * to set all backend attribute listed. This function is called when + * Triton requires further backend / model information to perform operations. + * This function may be called multiple times within the lifetime of the + * backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize). + * The backend may return error to indicate failure to set the backend + * attributes, and the attributes specified in the same function call will be + * ignored. Triton will update the specified attributes if 'nullptr' is + * returned. + * + * @param backend The backend. + * @param backend_attributes Return the backend attribute. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_GetBackendAttribute( + TRITONBACKEND_Backend backend, + TRITONBACKEND_BackendAttribute backend_attributes); + +/** TRITONBACKEND_BackendAttribute + * + * API to modify attributes associated with a backend. + * +

+ * Add the preferred instance group of the backend. This function + * can be called multiple times to cover different instance group kinds that + * the backend supports, given the priority order that the first call describes + * the most preferred group. In the case where instance group are not + * explicitly provided, Triton will use this attribute to create model + * deployment that aligns more with the backend preference. + * + * @param backend_attributes The backend attributes object. + * @param kind The kind of the instance group. + * @param count The number of instances per device. Triton default will be used + * if 0 is provided. + * @param device_ids The devices where instances should be available. Triton + * default will be used if 'nullptr' is provided. + * @param id_count The number of devices in 'device_ids'. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongPointer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongBuffer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") long[] device_ids, @Cast("const uint64_t") long id_count); // #ifdef __cplusplus // #endif diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java new file mode 100644 index 00000000000..9269f4b92f0 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java @@ -0,0 +1,23 @@ +// Targeted by JavaCPP version 1.5.8-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + + +/// +/// +/// +/// +@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONBACKEND_BackendAttribute extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_BackendAttribute() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_BackendAttribute(Pointer p) { super(p); } +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java index 4bf992d1b6b..fc2bd9ad679 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java @@ -8,12 +8,6 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/// -/// -/// -/// @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONBACKEND_ModelInstance extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ diff --git a/tritonserverwrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml index a177256db48..34e767006ad 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -20,11 +20,11 @@ - + ${project.groupId} ${javacpp.moduleId} diff --git a/tritonserverwrapper/pom.xml b/tritonserverwrapper/pom.xml index 8f6cda3c6d5..ac90944fdc8 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -15,11 +15,11 @@ JavaCPP Presets for Triton Inference Server Wrapper - + org.bytedeco javacpp diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index 0c0efebe33c..a96e01713ec 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -31,7 +31,7 @@ import org.bytedeco.javacpp.tools.Info; import org.bytedeco.javacpp.tools.InfoMap; import org.bytedeco.javacpp.tools.InfoMapper; -// import org.bytedeco.tritonserver.presets.tritonserver; +import org.bytedeco.tritonserver.presets.tritonserver; /** * @@ -41,7 +41,7 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h","server_wrapper.h", "infer_requested_output.h", "common.h"}, + include = {"server_wrapper.h", "infer_requested_output.h", "common.h"}, link = "tritonserverwrapper", includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, linkpath = {"/opt/tritonserver/lib/"} From d3d3eb88b94f3cb267e663582d6ce3b4fdfb3a9e Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 15 Dec 2022 18:00:39 -0800 Subject: [PATCH 05/50] update container and keep getting the build working --- .../tritonserver/global/tritonserver.java | 25 +++++++++++-------- .../TRITONBACKEND_BackendAttribute.java | 2 +- .../presets/tritonserverwrapper.java | 11 ++++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 90f01e2fabf..1a24a782ba6 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -4246,17 +4246,20 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); -/** Set the model configuration in Triton server. Only the inputs, outputs, - * max batch size, and scheduling choice can be changed. A caveat being - * scheduling choice can only be changed if none is previously set. Any other - * changes to the model configuration will be ignored by Triton. This function - * can only be called from TRITONBACKEND_ModelInitialize, calling in any other - * context will result in an error being returned. Additionally, Triton server - * can add some of the missing fields in the provided config with this call. - * The backend must get the complete configuration again by using - * TRITONBACKEND_ModelConfig. TRITONBACKEND_ModelSetConfig does not take - * ownership of the message object and so the caller should call - * TRITONSERVER_MessageDelete to release the object once the function returns. +/** Set the model configuration in Triton server. This API should only be called + * when the backend implements the auto-completion of model configuration + * and TRITONBACKEND_ModelAutoCompleteConfig returns true in + * auto_complete_config. Only the inputs, outputs, max batch size, and + * scheduling choice can be changed. A caveat being scheduling choice can only + * be changed if none is previously set. Any other changes to the model + * configuration will be ignored by Triton. This function can only be called + * from TRITONBACKEND_ModelInitialize, calling in any other context will result + * in an error being returned. Additionally, Triton server can add some of the + * missing fields in the provided config with this call. The backend must get + * the complete configuration again by using TRITONBACKEND_ModelConfig. + * TRITONBACKEND_ModelSetConfig does not take ownership of the message object + * and so the caller should call TRITONSERVER_MessageDelete to release the + * object once the function returns. * * @param model The model. * @param config_version The format version of the model configuration. diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java index 9269f4b92f0..b2c4c9bad9a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.8-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.8: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index a96e01713ec..f667102987b 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -41,9 +41,9 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"server_wrapper.h", "infer_requested_output.h", "common.h"}, + include = {"common.h", "generic_server_wrapper.h"}, link = "tritonserverwrapper", - includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), // @Platform( @@ -66,6 +66,13 @@ public void map(InfoMap infoMap) { .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + .put(new Info("std::set").pointerTypes("StringSet").define()) + .put(new Info("std::vector").pointerTypes("StringVector").define()) + .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) + .put(new Info("TritonServer").purify(false).virtualize()) + // .put(new Info("server_wrapper.h").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) + // .put(new Info("server_wrapper.cc").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) + // .put(new Info("server_wrapper.h").linePatterns("std::future>*", ";", "std::unique_ptr>>*", ";").skip()) ; } } From cdb5b6455c6fe74c933bf815363d800a6e92d1b9 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 16 Dec 2022 18:10:06 -0800 Subject: [PATCH 06/50] update snapshot version --- tritonserverwrapper/platform/pom.xml | 2 +- tritonserverwrapper/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserverwrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml index 34e767006ad..4211a9555cc 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT ../../ diff --git a/tritonserverwrapper/pom.xml b/tritonserverwrapper/pom.xml index ac90944fdc8..b7e410f4598 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT org.bytedeco From 5ca8fce7617e0484a866d4bdc1bb21f90f7ea77f Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 16 Feb 2023 14:18:38 -0800 Subject: [PATCH 07/50] new update --- .../global/tritonserverwrapper.java | 183 +++++++++++++++++ .../tritonserverwrapper/Allocator.java | 37 ++++ .../tritonserverwrapper/BackendConfig.java | 43 ++++ .../CUDAMemoryPoolByteSize.java | 32 +++ .../GenericInferRequest.java | 56 ++++++ .../GenericInferResult.java | 68 +++++++ .../GenericTritonServer.java | 132 ++++++++++++ .../tritonserverwrapper/HostPolicy.java | 47 +++++ .../tritonserverwrapper/InferOptions.java | 109 ++++++++++ .../tritonserverwrapper/LoggingOptions.java | 67 +++++++ .../tritonserverwrapper/MetricsOptions.java | 49 +++++ .../ModelLoadGPULimit.java | 30 +++ .../tritonserverwrapper/NewModelRepo.java | 50 +++++ .../OutputBufferReleaseFn_t.java | 21 ++ .../RateLimitResource.java | 43 ++++ .../tritonserverwrapper/RepositoryIndex.java | 50 +++++ .../ResponseAllocatorAllocFn_t.java | 26 +++ .../ResponseAllocatorStartFn_t.java | 19 ++ .../tritonserverwrapper/ServerOptions.java | 189 ++++++++++++++++++ .../tritonserverwrapper/StringSet.java | 36 ++++ .../tritonserverwrapper/StringVector.java | 99 +++++++++ .../tritonserverwrapper/Tensor.java | 81 ++++++++ .../tritonserverwrapper/Trace.java | 66 ++++++ .../tritonserverwrapper/TritonException.java | 29 +++ 24 files changed, 1562 insertions(+) create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java new file mode 100644 index 00000000000..c0b4f403cbd --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java @@ -0,0 +1,183 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.global; + +import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper { + static { Loader.load(); } + +// Targeting ../tritonserverwrapper/StringSet.java + + +// Targeting ../tritonserverwrapper/StringVector.java + + +// Parsed from common.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once + +//============================================================================== +/** enum classes + * */ +/** enum class triton::developer_tools::server::ModelControlMode */ +public static final int NONE = 0, POLL = 1, EXPLICIT = 2; +/** enum class triton::developer_tools::server::MemoryType */ +public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; +/** enum class triton::developer_tools::server::DataType */ +public static final int + INVALID = 0, + BOOL = 1, + UINT8 = 2, + UINT16 = 3, + UINT32 = 4, + UINT64 = 5, + INT8 = 6, + INT16 = 7, + INT32 = 8, + INT64 = 9, + FP16 = 10, + FP32 = 11, + FP64 = 12, + BYTES = 13, + BF16 = 14; +/** enum class triton::developer_tools::server::ModelReadyState */ +public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; +// Targeting ../tritonserverwrapper/TritonException.java + + +// Targeting ../tritonserverwrapper/ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritonserverwrapper/OutputBufferReleaseFn_t.java + + +// Targeting ../tritonserverwrapper/ResponseAllocatorStartFn_t.java + + + + // namespace triton::developer_tools::server + + +// Parsed from generic_server_wrapper.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include +// #include +// #include "common.h" +// #include "../src/infer_requested_output.h" +// #include "../src/tracer.h" + +/// +// Targeting ../tritonserverwrapper/GenericTritonServer.java + + +// Targeting ../tritonserverwrapper/GenericInferResult.java + + +// Targeting ../tritonserverwrapper/GenericInferRequest.java + + +// Targeting ../tritonserverwrapper/LoggingOptions.java + + +// Targeting ../tritonserverwrapper/MetricsOptions.java + + +// Targeting ../tritonserverwrapper/RateLimitResource.java + + +// Targeting ../tritonserverwrapper/ModelLoadGPULimit.java + + +// Targeting ../tritonserverwrapper/Allocator.java + + +// Targeting ../tritonserverwrapper/BackendConfig.java + + +// Targeting ../tritonserverwrapper/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritonserverwrapper/HostPolicy.java + + +// Targeting ../tritonserverwrapper/Trace.java + + +// Targeting ../tritonserverwrapper/ServerOptions.java + + +// Targeting ../tritonserverwrapper/RepositoryIndex.java + + +// Targeting ../tritonserverwrapper/Tensor.java + + +// Targeting ../tritonserverwrapper/NewModelRepo.java + + +// Targeting ../tritonserverwrapper/InferOptions.java + + + + // namespace triton::developer_tools::server + +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java new file mode 100644 index 00000000000..2440e5ec4c4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java @@ -0,0 +1,37 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + + +//============================================================================== +/** Custom Allocator object for providing custom functions for allocator. + * If there is no custom allocator provided, will use the default allocator. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Allocator extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Allocator(Pointer p) { super(p); } + + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/) { super((Pointer)null); allocate(alloc_fn, release_fn, start_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/); + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn) { super((Pointer)null); allocate(alloc_fn, release_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn); + + public native ResponseAllocatorAllocFn_t AllocFn(); + public native OutputBufferReleaseFn_t ReleaseFn(); + public native ResponseAllocatorStartFn_t StartFn(); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java new file mode 100644 index 00000000000..f27cc1d4081 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold backend configuration for setting 'ServerOptions'. + * Different Triton-supported backends have different backend configuration + * options. Please refer to the 'Command line options' section in the + * documentation of each backend to see the options (e.g. Tensorflow Backend: + * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class BackendConfig extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public BackendConfig(Pointer p) { super(p); } + + public BackendConfig( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value); + public BackendConfig( + @StdString String name, @StdString String setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @StdString String setting, + @StdString String value); + + // The name of the backend. + public native @StdString BytePointer name_(); public native BackendConfig name_(BytePointer setter); + // The name of the setting. + public native @StdString BytePointer setting_(); public native BackendConfig setting_(BytePointer setter); + // The setting value. + public native @StdString BytePointer value_(); public native BackendConfig value_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java new file mode 100644 index 00000000000..97e0e60e4a2 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java @@ -0,0 +1,32 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold CUDA memory pool byte size for setting 'ServerOptions'. + * If GPU support is enabled, the server will allocate CUDA memory to minimize + * data transfer between host and devices until it exceeds the specified byte + * size. This will not affect the allocation conducted by the backend + * frameworks. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class CUDAMemoryPoolByteSize extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public CUDAMemoryPoolByteSize(Pointer p) { super(p); } + + public CUDAMemoryPoolByteSize(int gpu_device, @Cast("const uint64_t") long size) { super((Pointer)null); allocate(gpu_device, size); } + private native void allocate(int gpu_device, @Cast("const uint64_t") long size); + + // The GPU device ID to allocate the memory pool. + public native int gpu_device_(); public native CUDAMemoryPoolByteSize gpu_device_(int setter); + // The CUDA memory pool byte size that the server can allocate on given GPU + // device. Default is 64 MB. + public native @Cast("uint64_t") long size_(); public native CUDAMemoryPoolByteSize size_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java new file mode 100644 index 00000000000..54f9aa13a31 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java @@ -0,0 +1,56 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Object that describes an inflight inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericInferRequest extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferRequest(Pointer p) { super(p); } + + /** Create an InferRequest instance. */ + public static native @UniquePtr GenericInferRequest Create( + @Const @ByRef InferOptions infer_options); + + /** Add an input tensor to be sent within an InferRequest object. The input + * data buffer within the 'Tensor' object must not be modified until + * inference is completed and result is returned. + * @param name The name of the input tensor. + * @param input A Tensor object that describes an input tensor. */ + public native @NoException(true) void AddInput(@StdString BytePointer name, @Const @ByRef Tensor input); + public native @NoException(true) void AddInput(@StdString String name, @Const @ByRef Tensor input); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. Pre-allocated buffer for each output should be specified within + * the 'Tensor' object. + * @param name The name of the output tensor. + * @param output A Tensor object that describes an output tensor containing + * its pre-allocated buffer. */ + public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); + public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. + * @param name The name of the output tensor. */ + public native void AddRequestedOutput(@StdString BytePointer name); + public native void AddRequestedOutput(@StdString String name); + + /** Clear inputs and outputs of the request. This allows users to reuse the + * InferRequest object if needed. */ + public native void Reset(); + +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java new file mode 100644 index 00000000000..0929769fce4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java @@ -0,0 +1,68 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** An interface for InferResult object to interpret the response to an + * inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericInferResult extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferResult(Pointer p) { super(p); } + + + /** Get the name of the model which generated this response. + * @return Returns the name of the model. */ + public native @StdString @NoException(true) BytePointer ModelName(); + + /** Get the version of the model which generated this response. + * @return Returns the version of the model. */ + public native @StdString @NoException(true) BytePointer ModelVersion(); + + /** Get the id of the request which generated this response. + * @return Returns the id of the request. */ + public native @StdString @NoException(true) BytePointer Id(); + + /** Get the output names from the infer result + * @return Vector of output names */ + public native @ByVal StringVector OutputNames(); + /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' + * field of the output is owned by the returned 'Tensor' object itself. Note + * that for string data, need to use 'StringData' function for string data + * result. + * @param name The name of the output tensor to be retrieved. + * @return Returns the output result as a shared pointer of 'Tensor' object. */ + public native @SharedPtr Tensor Output(@StdString BytePointer name); + public native @SharedPtr Tensor Output(@StdString String name); + + /** Get the result data as a vector of strings. The vector will + * receive a copy of result data. An exception will be thrown if + * the data type of output is not 'BYTES'. + * @param output_name The name of the output to get result data. + * @return Returns the result data represented as a vector of strings. The + * strings are stored in the row-major order. */ + public native @ByVal StringVector StringData(@StdString BytePointer output_name); + public native @ByVal StringVector StringData(@StdString String output_name); + + /** Return the complete response as a user friendly string. + * @return The string describing the complete response. */ + public native @StdString BytePointer DebugString(); + + /** Return if there is an error within this result. + * @return True if this 'GenericInferResult' object has an error, false if no + * error. */ + public native @Cast("bool") boolean HasError(); + + /** Return the error message of the error. + * @return The messsage for the error. Empty if no error. */ + public native @StdString BytePointer ErrorMsg(); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java new file mode 100644 index 00000000000..7d0b8495a1d --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java @@ -0,0 +1,132 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Object that encapsulates in-process C API functionalities. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericTritonServer extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericTritonServer(Pointer p) { super(p); } + + /** Create a GenericTritonServer instance. */ + public static native @UniquePtr GenericTritonServer Create( + @Const @ByRef ServerOptions server_options); + + /** Load the requested model or reload the model if it is already loaded. + * @param model_name The name of the model. */ + public native void LoadModel(@StdString BytePointer model_name); + public native void LoadModel(@StdString String model_name); + + /** Unload the requested model. Unloading a model that is not loaded + * on server has no affect. + * @param model_name The name of the model. */ + public native void UnloadModel(@StdString BytePointer model_name); + public native void UnloadModel(@StdString String model_name); + + /** Get the set of names of models that are loaded and ready for inference. + * @return Returns the set of names of models that are + * loaded and ready for inference. */ + public native @ByVal StringSet LoadedModels(); + + /** Get the index of model repository contents. + * @return Returns a vector of 'RepositoryIndex' object + * representing the repository index. */ + public native @StdVector RepositoryIndex ModelIndex(); + + /** Get the metrics of the server. + * @return Returns a string representing the metrics. */ + public native @StdString BytePointer ServerMetrics(); + + /** Get the inference statistics of the specified model. + * @param model_name The name of the model. + * @param model_version the version of the model requested. + * @return Returns a json string representing the model metrics. */ + public native @StdString BytePointer ModelStatistics( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version); + public native @StdString String ModelStatistics( + @StdString String model_name, @Cast("const int64_t") long model_version); + + /** Is the server live? + * @return Returns true if server is live, false otherwise. */ + public native @Cast("bool") boolean IsServerLive(); + + /** Is the server ready? + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsServerReady(); + + /** Stop a server object. A server can't be restarted once it is + * stopped. */ + public native void ServerStop(); + + /** Is the model ready? + * @param model_name The name of the model to get readiness for. + * @param model_version The version of the model to get readiness + * for. If -1 then the server will choose a version based on the + * model's policy. This field is optional, default is -1. + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name); + + /** Get the configuration of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will + * choose a version based on the model and internal policy. This field is + * optional. @return Returns JSON representation of model configuration as a + * string. */ + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name); + public native @StdString String ModelConfig( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelConfig( + @StdString String model_name); + + /** Get the metadata of the server. + * @return Returns JSON representation of server metadata as a string. */ + public native @StdString BytePointer ServerMetadata(); + + /** Get the metadata of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will choose a version + * based on the model and internal policy. This field is optional. + * @return Returns JSON representation of model metadata as a string. */ + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name); + public native @StdString String ModelMetadata( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelMetadata( + @StdString String model_name); + + /** Register a new model repository. This function is not available in polling + * mode. + * @param new_model_repo The 'NewModelRepo' object contains the info of the + * new model repo to be registered. */ + public native void RegisterModelRepo(@Const @ByRef NewModelRepo new_model_repo); + + /** Unregister a model repository. This function is not available in polling + * mode. + * @param repo_path The full path to the model repository. */ + public native void UnregisterModelRepo(@StdString BytePointer repo_path); + public native void UnregisterModelRepo(@StdString String repo_path); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java new file mode 100644 index 00000000000..08f8326598d --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java @@ -0,0 +1,47 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold host policy for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class HostPolicy extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public HostPolicy(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::HostPolicy::Setting */ + public static final int NUMA_NODE = 0, CPU_CORES = 1; + + public HostPolicy( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value); + public HostPolicy( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value); + + // The name of the policy. + public native @StdString BytePointer name_(); public native HostPolicy name_(BytePointer setter); + // The kind of the host policy setting. Currently supported settings are + // 'NUMA_NODE', 'CPU_CORES'. Note that 'NUMA_NODE' setting will affect pinned + // memory pool behavior, see the comments of 'pinned_memory_pool_byte_size_' + // in 'ServerOptions' for more detail. + public native @Cast("triton::developer_tools::server::HostPolicy::Setting") int setting_(); public native HostPolicy setting_(int setter); + // The setting value. + public native @StdString BytePointer value_(); public native HostPolicy value_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java new file mode 100644 index 00000000000..fed63d6fcd6 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java @@ -0,0 +1,109 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold options for Inference Request. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class InferOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public InferOptions(Pointer p) { super(p); } + + public InferOptions(@StdString BytePointer model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString BytePointer model_name); + public InferOptions(@StdString String model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString String model_name); + + public InferOptions( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + public InferOptions( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + + // The name of the model to run inference. + public native @StdString BytePointer model_name_(); public native InferOptions model_name_(BytePointer setter); + // The version of the model to use while running inference. The default + // value is "-1" which means the server will select the + // version of the model based on its internal policy. + public native @Cast("int64_t") long model_version_(); public native InferOptions model_version_(long setter); + // An identifier for the request. If specified will be returned + // in the response. Default value is an empty string which means no + // request_id will be used. + public native @StdString BytePointer request_id_(); public native InferOptions request_id_(BytePointer setter); + // The correlation ID of the inference request to be an unsigned integer. + // Should be used exclusively with 'correlation_id_str_'. + // Default is 0, which indicates that the request has no correlation ID. + public native @Cast("uint64_t") long correlation_id_(); public native InferOptions correlation_id_(long setter); + // The correlation ID of the inference request to be a string. + // Should be used exclusively with 'correlation_id_'. + // Default value is "". + public native @StdString BytePointer correlation_id_str_(); public native InferOptions correlation_id_str_(BytePointer setter); + // Indicates whether the request being added marks the start of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_start_(); public native InferOptions sequence_start_(boolean setter); + // Indicates whether the request being added marks the end of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_end_(); public native InferOptions sequence_end_(boolean setter); + // Indicates the priority of the request. Priority value zero + // indicates that the default priority level should be used + // (i.e. same behavior as not specifying the priority parameter). + // Lower value priorities indicate higher priority levels. Thus + // the highest priority level is indicated by setting the parameter + // to 1, the next highest is 2, etc. If not provided, the server + // will handle the request using default setting for the model. + public native @Cast("uint64_t") long priority_(); public native InferOptions priority_(long setter); + // The timeout value for the request, in microseconds. If the request + // cannot be completed within the time by the server can take a + // model-specific action such as terminating the request. If not + // provided, the server will handle the request using default setting + // for the model. + public native @Cast("uint64_t") long request_timeout_(); public native InferOptions request_timeout_(long setter); + // User-provided custom reponse allocator object. Default is nullptr. + // If using custom allocator, the lifetime of this 'Allocator' object should + // be long enough until `InferResult` object goes out of scope as we need + // this `Allocator` object to call 'ResponseAllocatorReleaseFn_t' for + // releasing the response. + public native @SharedPtr Allocator custom_allocator_(); public native InferOptions custom_allocator_(Allocator setter); + // Update trace setting for the specified model. If not set, will use global + // trace setting in 'ServerOptions' for tracing if tracing is enabled in + // 'ServerOptions'. Default is nullptr. + public native @SharedPtr Trace trace_(); public native InferOptions trace_(Trace setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java new file mode 100644 index 00000000000..3a8084108fa --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java @@ -0,0 +1,67 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +//============================================================================== +/** Structure to hold logging options for setting 'ServerOptions'. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class LoggingOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public LoggingOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public LoggingOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public LoggingOptions position(long position) { + return (LoggingOptions)super.position(position); + } + @Override public LoggingOptions getPointer(long i) { + return new LoggingOptions((Pointer)this).offsetAddress(i); + } + + // The range of VerboseLevel is [0, INT_MAX]. + /** enum class triton::developer_tools::server::LoggingOptions::VerboseLevel */ + public static final int OFF = 0, MIN = 1, MAX = Integer.MAX_VALUE; + /** enum class triton::developer_tools::server::LoggingOptions::LogFormat */ + public static final int DEFAULT = 0, ISO8601 = 1; + + public LoggingOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file); + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file); + + // Verbose logging level. Default is OFF. + public native @Cast("triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose_(); public native LoggingOptions verbose_(int setter); + // Enable or disable info logging level. Default is true. + public native @Cast("bool") boolean info_(); public native LoggingOptions info_(boolean setter); + // Enable or disable warn logging level. Default is true. + public native @Cast("bool") boolean warn_(); public native LoggingOptions warn_(boolean setter); + // Enable or disable error logging level. Default is true. + public native @Cast("bool") boolean error_(); public native LoggingOptions error_(boolean setter); + // The format of logging. For "DEFAULT", the log severity (L) and + // timestamp will be logged as "LMMDD hh:mm:ss.ssssss". For "ISO8601", the + // log format will be "YYYY-MM-DDThh:mm:ssZ L". Default is 'DEFAULT'. + public native @Cast("triton::developer_tools::server::LoggingOptions::LogFormat") int format_(); public native LoggingOptions format_(int setter); + // Logging output file. If specified, log outputs will be saved to this file. + // If not specified, log outputs will stream to the console. Default is an + // empty string. + public native @StdString BytePointer log_file_(); public native LoggingOptions log_file_(BytePointer setter); // logging output file +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java new file mode 100644 index 00000000000..56378a66cb1 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java @@ -0,0 +1,49 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold metrics options for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class MetricsOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public MetricsOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public MetricsOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public MetricsOptions position(long position) { + return (MetricsOptions)super.position(position); + } + @Override public MetricsOptions getPointer(long i) { + return new MetricsOptions((Pointer)this).offsetAddress(i); + } + + public MetricsOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public MetricsOptions( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms) { super((Pointer)null); allocate(allow_metrics, allow_gpu_metrics, allow_cpu_metrics, metrics_interval_ms); } + private native void allocate( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms); + + // Enable or disable metrics. Default is true. + public native @Cast("bool") boolean allow_metrics_(); public native MetricsOptions allow_metrics_(boolean setter); + // Enable or disable GPU metrics. Default is true. + public native @Cast("bool") boolean allow_gpu_metrics_(); public native MetricsOptions allow_gpu_metrics_(boolean setter); + // Enable or disable CPU metrics. Default is true. + public native @Cast("bool") boolean allow_cpu_metrics_(); public native MetricsOptions allow_cpu_metrics_(boolean setter); + // The interval for metrics collection. Default is 2000. + public native @Cast("uint64_t") long metrics_interval_ms_(); public native MetricsOptions metrics_interval_ms_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java new file mode 100644 index 00000000000..e7f01eefbb3 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java @@ -0,0 +1,30 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold GPU limit of model loading for setting 'ServerOptions'. + * The limit on GPU memory usage is specified as a fraction. If model loading + * on the device is requested and the current memory usage exceeds the limit, + * the load will be rejected. If not specified, the limit will not be set. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ModelLoadGPULimit extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ModelLoadGPULimit(Pointer p) { super(p); } + + public ModelLoadGPULimit(int device_id, double fraction) { super((Pointer)null); allocate(device_id, fraction); } + private native void allocate(int device_id, double fraction); + + // The GPU device ID. + public native int device_id_(); public native ModelLoadGPULimit device_id_(int setter); + // The limit on memory usage as a fraction. + public native double fraction_(); public native ModelLoadGPULimit fraction_(double setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java new file mode 100644 index 00000000000..e28c3ba27c7 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold the full path to the model repository to be registered and + * the mapping from the original model name to the overriden one. This object + * is used for calling 'TritonServer::RegisterModelRepo' for registering + * model repository. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class NewModelRepo extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public NewModelRepo(Pointer p) { super(p); } + + public NewModelRepo(@StdString BytePointer path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString BytePointer path); + public NewModelRepo(@StdString String path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString String path); + + public NewModelRepo( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name); + public NewModelRepo( + @StdString String path, @StdString String original_name, + @StdString String override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString String path, @StdString String original_name, + @StdString String override_name); + + // The full path to the model repository. + public native @StdString BytePointer path_(); public native NewModelRepo path_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer original_name_(); public native NewModelRepo original_name_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer override_name_(); public native NewModelRepo override_name_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java new file mode 100644 index 00000000000..1871e81f3fa --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java @@ -0,0 +1,21 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class OutputBufferReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public OutputBufferReleaseFn_t(Pointer p) { super(p); } + protected OutputBufferReleaseFn_t() { allocate(); } + private native void allocate(); + public native void call( + Pointer buffer, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java new file mode 100644 index 00000000000..942337971ad --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold rate limit resource for setting 'ServerOptions'. See here + * for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class RateLimitResource extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RateLimitResource(Pointer p) { super(p); } + + public RateLimitResource(@StdString BytePointer name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString BytePointer name, int count); + public RateLimitResource(@StdString String name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString String name, int count); + + public RateLimitResource(@StdString BytePointer name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString BytePointer name, int count, int device); + public RateLimitResource(@StdString String name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString String name, int count, int device); + + // The name of the resource. + public native @StdString BytePointer name_(); public native RateLimitResource name_(BytePointer setter); + // The count of the resource. + public native int count_(); public native RateLimitResource count_(int setter); + // The device identifier for the resource. This field is optional and if not + // specified will be applied to every device. The device value is ignored for + // a global resource. The server will use the rate limiter configuration + // specified for instance groups in model config to determine whether resource + // is global. In case of conflicting resource type in different model + // configurations, server will raise an appropriate error while loading model. + public native int device_(); public native RateLimitResource device_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java new file mode 100644 index 00000000000..0beba9c95b4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold repository index for 'ModelIndex' function. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class RepositoryIndex extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RepositoryIndex(Pointer p) { super(p); } + + public RepositoryIndex( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + public RepositoryIndex( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + + // The name of the model. + public native @StdString BytePointer name_(); public native RepositoryIndex name_(BytePointer setter); + // The version of the model. + public native @StdString BytePointer version_(); public native RepositoryIndex version_(BytePointer setter); + // The state of the model. The states are + // * UNKNOWN: The model is in an unknown state. The model is not available for + // inferencing. + // * READY: The model is ready and available for inferencing. + // * UNAVAILABLE: The model is unavailable, indicating that the model failed + // to load or has been implicitly or explicitly unloaded. The model is not + // available for inferencing. + // * LOADING: The model is being loaded by the inference server. The model is + // not available for inferencing. + // * UNLOADING: The model is being unloaded by the inference server. The model + // is not available for inferencing. + public native @Cast("triton::developer_tools::server::ModelReadyState") int state_(); public native RepositoryIndex state_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java new file mode 100644 index 00000000000..079b2ce7446 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java @@ -0,0 +1,26 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Custom Response Allocator Callback function signatures. + * */ +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ResponseAllocatorAllocFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorAllocFn_t(Pointer p) { super(p); } + protected ResponseAllocatorAllocFn_t() { allocate(); } + private native void allocate(); + public native void call( + String tensor_name, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int preferred_memory_type, + @Cast("int64_t") long preferred_memory_type_id, @Cast("void**") PointerPointer buffer, + @Cast("triton::developer_tools::server::MemoryType*") IntPointer actual_memory_type, @Cast("int64_t*") LongPointer actual_memory_type_id); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java new file mode 100644 index 00000000000..c3e3e808569 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java @@ -0,0 +1,19 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ResponseAllocatorStartFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorStartFn_t(Pointer p) { super(p); } + protected ResponseAllocatorStartFn_t() { allocate(); } + private native void allocate(); + public native void call(Pointer userp); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java new file mode 100644 index 00000000000..b8bb019dd68 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java @@ -0,0 +1,189 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Server options that are used to initialize Triton Server. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ServerOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ServerOptions(Pointer p) { super(p); } + + public ServerOptions(@Const @ByRef StringVector model_repository_paths) { super((Pointer)null); allocate(model_repository_paths); } + private native void allocate(@Const @ByRef StringVector model_repository_paths); + + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + + // Paths to model repository directory. Note that if a model is not unique + // across all model repositories at any time, the model will not be available. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md. + public native @ByRef StringVector model_repository_paths_(); public native ServerOptions model_repository_paths_(StringVector setter); + // Logging options. See the 'LoggingOptions' structure for more information. + public native @ByRef LoggingOptions logging_(); public native ServerOptions logging_(LoggingOptions setter); + // Metrics options. See the 'MetricsOptions' structure for more information. + public native @ByRef MetricsOptions metrics_(); public native ServerOptions metrics_(MetricsOptions setter); + // Backend configuration. See the 'BackendConfig' structure for more + // information. + public native @StdVector BackendConfig be_config_(); public native ServerOptions be_config_(BackendConfig setter); + // The ID of the server. + public native @StdString BytePointer server_id_(); public native ServerOptions server_id_(BytePointer setter); + // The global directory searched for backend shared libraries. Default is + // "/opt/tritonserver/backends". See here for more information: + // https://github.com/triton-inference-server/backend#backends. + public native @StdString BytePointer backend_dir_(); public native ServerOptions backend_dir_(BytePointer setter); + // The global directory searched for repository agent shared libraries. + // Default is "/opt/tritonserver/repoagents". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/repository_agents.md. + public native @StdString BytePointer repo_agent_dir_(); public native ServerOptions repo_agent_dir_(BytePointer setter); + // If set, disables the triton and backends from auto completing model + // configuration files. Model configuration files must be provided and + // all required configuration settings must be specified. Default is false. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration. + public native @Cast("bool") boolean disable_auto_complete_config_(); public native ServerOptions disable_auto_complete_config_(boolean setter); + // Specify the mode for model management. Options are "NONE", "POLL" and + // "EXPLICIT". Default is "NONE". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md. + public native @Cast("triton::developer_tools::server::ModelControlMode") int model_control_mode_(); public native ServerOptions model_control_mode_(int setter); + // Interval in seconds between each poll of the model repository to check for + // changes. Valid only when 'model_control_mode_' is set to "POLL". Default + // is 15. + public native int repository_poll_secs_(); public native ServerOptions repository_poll_secs_(int setter); + // Specify the the models to be loaded on server startup. This will only take + // effect if 'model_control_mode_' is set to 'EXPLICIT'. + public native @ByRef StringSet startup_models_(); public native ServerOptions startup_models_(StringSet setter); + // The number of resources available to the server. Rate limiting is disabled + // by default, and can be enabled once 'rate_limit_resource_' is set. See the + // 'RateLimitResource' structure for more information. + public native @StdVector RateLimitResource rate_limit_resource_(); public native ServerOptions rate_limit_resource_(RateLimitResource setter); + // The total byte size that can be allocated as pinned system memory. If GPU + // support is enabled, the server will allocate pinned system memory to + // accelerate data transfer between host and devices until it exceeds the + // specified byte size. If 'NUMA_NODE' is configured via 'host_policy_', the + // pinned system memory of the pool size will be allocated on each numa node. + // This option will not affect the allocation conducted by the backend + // frameworks. Default is 256 MB. + public native @Cast("int64_t") long pinned_memory_pool_byte_size_(); public native ServerOptions pinned_memory_pool_byte_size_(long setter); + // The total byte size that can be allocated as CUDA memory for the GPU + // device. See the 'CUDAMemoryPoolByteSize' structure for more information. + public native @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size_(); public native ServerOptions cuda_memory_pool_byte_size_(CUDAMemoryPoolByteSize setter); + // The size in bytes to allocate for a request/response cache. When non-zero, + // Triton allocates the requested size in CPU memory and shares the cache + // across all inference requests and across all models. For a given model to + // use request caching, the model must enable request caching in the model + // configuration. See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#response-cache. + // By default, no model uses request caching even if the + // 'response_cache_byte_size_' is set. Default is 0. + public native @Cast("uint64_t") long response_cache_byte_size_(); public native ServerOptions response_cache_byte_size_(long setter); + // The minimum supported CUDA compute capability. GPUs that don't support this + // compute capability will not be used by the server. Default is 0. + public native double min_cuda_compute_capability_(); public native ServerOptions min_cuda_compute_capability_(double setter); + // If set, exit the inference server when an error occurs during + // initialization. Default is true. + public native @Cast("bool") boolean exit_on_error_(); public native ServerOptions exit_on_error_(boolean setter); + // Timeout (in seconds) when exiting to wait for in-flight inferences to + // finish. After the timeout expires the server exits even if inferences are + // still in flight. Default is 30 secs. + public native int exit_timeout_secs_(); public native ServerOptions exit_timeout_secs_(int setter); + // The number of threads used to accelerate copies and other operations + // required to manage input and output tensor contents. Default is 0. + public native int buffer_manager_thread_count_(); public native ServerOptions buffer_manager_thread_count_(int setter); + // The number of threads used to concurrently load models in model + // repositories. Default is 2*. + public native @Cast("uint32_t") int model_load_thread_count_(); public native ServerOptions model_load_thread_count_(int setter); + // The GPU limit of model loading. See the 'ModelLoadGPULimit' structure for + // more information. + public native @StdVector ModelLoadGPULimit model_load_gpu_limit_(); public native ServerOptions model_load_gpu_limit_(ModelLoadGPULimit setter); + // The host policy setting. See the 'HostPolicy' structure for more + // information. + public native @StdVector HostPolicy host_policy_(); public native ServerOptions host_policy_(HostPolicy setter); + // The global trace setting. Default is nullptr, meaning that tracing is not + // enabled. See the 'Trace' structure for more information. + public native @SharedPtr Trace trace_(); public native ServerOptions trace_(Trace setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java new file mode 100644 index 00000000000..9c1bb2e351f --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java @@ -0,0 +1,36 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Name("std::set") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class StringSet extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringSet(Pointer p) { super(p); } + public StringSet() { allocate(); } + private native void allocate(); + public native @Name("operator =") @ByRef StringSet put(@ByRef StringSet x); + + public boolean empty() { return size() == 0; } + public native long size(); + + public native void insert(@StdString BytePointer value); + public native void erase(@StdString BytePointer value); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } +} + diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java new file mode 100644 index 00000000000..bcba32a1f82 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java @@ -0,0 +1,99 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class StringVector extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringVector(Pointer p) { super(p); } + public StringVector(BytePointer value) { this(1); put(0, value); } + public StringVector(BytePointer ... array) { this(array.length); put(array); } + public StringVector(String value) { this(1); put(0, value); } + public StringVector(String ... array) { this(array.length); put(array); } + public StringVector() { allocate(); } + public StringVector(long n) { allocate(n); } + private native void allocate(); + private native void allocate(@Cast("size_t") long n); + public native @Name("operator =") @ByRef StringVector put(@ByRef StringVector x); + + public boolean empty() { return size() == 0; } + public native long size(); + public void clear() { resize(0); } + public native void resize(@Cast("size_t") long n); + + @Index(function = "at") public native @StdString BytePointer get(@Cast("size_t") long i); + public native StringVector put(@Cast("size_t") long i, BytePointer value); + @ValueSetter @Index(function = "at") public native StringVector put(@Cast("size_t") long i, @StdString String value); + + public native @ByVal Iterator insert(@ByVal Iterator pos, @StdString BytePointer value); + public native @ByVal Iterator erase(@ByVal Iterator pos); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } + + public BytePointer[] get() { + BytePointer[] array = new BytePointer[size() < Integer.MAX_VALUE ? (int)size() : Integer.MAX_VALUE]; + for (int i = 0; i < array.length; i++) { + array[i] = get(i); + } + return array; + } + @Override public String toString() { + return java.util.Arrays.toString(get()); + } + + public BytePointer pop_back() { + long size = size(); + BytePointer value = get(size - 1); + resize(size - 1); + return value; + } + public StringVector push_back(BytePointer value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(BytePointer value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(BytePointer ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } + + public StringVector push_back(String value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(String value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(String ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } +} + diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java new file mode 100644 index 00000000000..dd7d0c59cef --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java @@ -0,0 +1,81 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold information of a tensor. This object is used for adding + * input/requested output to an inference request, and retrieving the output + * result from inference result. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Tensor extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Tensor(Pointer p) { super(p); } + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + // The pointer to the start of the buffer. + public native @Cast("char*") BytePointer buffer_(); public native Tensor buffer_(BytePointer setter); + // The size of buffer in bytes. + public native @Cast("size_t") long byte_size_(); public native Tensor byte_size_(long setter); + // The data type of the tensor. + public native @Cast("triton::developer_tools::server::DataType") int data_type_(); public native Tensor data_type_(int setter); + // The shape of the tensor. + public native @Cast("int64_t*") @StdVector LongPointer shape_(); public native Tensor shape_(LongPointer setter); + // The memory type of the tensor. Valid memory types are "CPU", "CPU_PINNED" + // and "GPU". + public native @Cast("triton::developer_tools::server::MemoryType") int memory_type_(); public native Tensor memory_type_(int setter); + // The ID of the memory for the tensor. (e.g. '0' is the memory type id of + // 'GPU-0') + public native @Cast("int64_t") long memory_type_id_(); public native Tensor memory_type_id_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java new file mode 100644 index 00000000000..debbb257802 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java @@ -0,0 +1,66 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold global trace setting for 'ServerOptions' and + * model-specific trace setting for 'InferOptions'. See here for more + * information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Trace extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Trace(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::Trace::Level */ + public static final int OFF = 0, TIMESTAMPS = 1, TENSORS = 2; + + public Trace(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + public Trace(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + + public Trace( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + public Trace( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + + // The file where trace output will be saved. If 'log-frequency' is also + // specified, this argument value will be the prefix of the files to save the + // trace output. + public native @StdString BytePointer file_(); public native Trace file_(BytePointer setter); + // Specify a trace level. OFF to disable tracing, TIMESTAMPS to trace + // timestamps, TENSORS to trace tensors. + public native @Cast("triton::developer_tools::server::Trace::Level") int level_(); public native Trace level_(int setter); + // The trace sampling rate. The value represents how many requests will one + // trace be sampled from. For example, if the trace rate is "1000", 1 trace + // will be sampled for every 1000 requests. Default is 1000. + public native @Cast("uint32_t") int rate_(); public native Trace rate_(int setter); + // The number of traces to be sampled. If the value is -1, the number of + // traces to be sampled will not be limited. Default is -1. + public native int count_(); public native Trace count_(int setter); + // The trace log frequency. If the value is 0, Triton will only log the trace + // output to 'file_' when shutting down. Otherwise, Triton will log the trace + // output to 'file_.' when it collects the specified number of traces. + // For example, if the log frequency is 100, when Triton collects the 100-th + // trace, it logs the traces to file 'file_.0', and when it collects the + // 200-th trace, it logs the 101-th to the 200-th traces to file file_.1'. + // Default is 0. + public native @Cast("uint32_t") int log_frequency_(); public native Trace log_frequency_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java new file mode 100644 index 00000000000..5904afd5c77 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java @@ -0,0 +1,29 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +// TritonException +// +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class TritonException extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TritonException(Pointer p) { super(p); } + + public TritonException(@StdString BytePointer message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString BytePointer message); + public TritonException(@StdString String message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString String message); + + public native String what(); + + public native @StdString BytePointer message_(); public native TritonException message_(BytePointer setter); +} From 6a6678c12572c9543affd09c7ab662aef1a40870 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 16 Feb 2023 14:42:15 -0800 Subject: [PATCH 08/50] update linked library --- .../tritonserverwrapper/presets/tritonserverwrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index f667102987b..e06e12bdae9 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -42,7 +42,7 @@ @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, include = {"common.h", "generic_server_wrapper.h"}, - link = "tritonserverwrapper", + link = "tritondevelopertoolsserver", includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), From 4415908c9a329313ea18bb19b1090eaa1f6ef421 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 7 Sep 2022 15:03:14 -0700 Subject: [PATCH 09/50] adding wrapper --- .../samples/simpleWrapper/SimpleWrapper.java | 848 ++++++++++++++++++ 1 file changed, 848 insertions(+) create mode 100644 tritonserver/samples/simpleWrapper/SimpleWrapper.java diff --git a/tritonserver/samples/simpleWrapper/SimpleWrapper.java b/tritonserver/samples/simpleWrapper/SimpleWrapper.java new file mode 100644 index 00000000000..b89bcc1cd5f --- /dev/null +++ b/tritonserver/samples/simpleWrapper/SimpleWrapper.java @@ -0,0 +1,848 @@ +// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; +import com.google.gson.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.tritonserver.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritonserver.*; + +public class SimpleWrapper { + + static void FAIL(String MSG) { + System.err.println("Failure: " + MSG); + System.exit(1); + } + + static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { + if (err__ != null) { + System.err.println("error: " + MSG + ":" + + TRITONSERVER_ErrorCodeString(err__) + " - " + + TRITONSERVER_ErrorMessage(err__)); + TRITONSERVER_ErrorDelete(err__); + System.exit(1); + } + } + + static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; + + static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { + public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } + protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { + DeleteDeallocator(Pointer p) { super(p); } + @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } + } + } + + static void + Usage(String msg) + { + if (msg != null) { + System.err.println(msg); + } + + System.err.println("Usage: java " + SimpleCPUOnly.class.getSimpleName() + " [options]"); + System.err.println("\t-v Enable verbose logging"); + System.err.println("\t-r [model repository absolute path]"); + + System.exit(1); + } + + static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + long byte_size, int preferred_memory_type, + long preferred_memory_type_id, Pointer userp, PointerPointer buffer, + PointerPointer buffer_userp, IntPointer actual_memory_type, + LongPointer actual_memory_type_id) + { + // Initially attempt to make the actual memory type and id that we + // allocate be the same as preferred memory type + actual_memory_type.put(0, preferred_memory_type); + actual_memory_type_id.put(0, preferred_memory_type_id); + + // If 'byte_size' is zero just return 'buffer' == nullptr, we don't + // need to do any other book-keeping. + if (byte_size == 0) { + buffer.put(0, null); + buffer_userp.put(0, null); + System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); + } else { + Pointer allocated_ptr = new Pointer(); + actual_memory_type.put(0, requested_memory_type); + + actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); + allocated_ptr = Pointer.malloc(byte_size); + + // Pass the tensor name with buffer_userp so we can show it when + // releasing the buffer. + if (!allocated_ptr.isNull()) { + buffer.put(0, allocated_ptr); + buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); + System.out.println("allocated " + byte_size + " bytes in " + + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) + + " for result tensor " + tensor_name); + } + } + + return null; // Success + } + } + + static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + long byte_size, int memory_type, long memory_type_id) + { + String name = null; + if (buffer_userp != null) { + name = (String)Loader.accessGlobalRef(buffer_userp); + } else { + name = ""; + } + + System.out.println("Releasing buffer " + buffer + " of size " + byte_size + + " in " + TRITONSERVER_MemoryTypeString(memory_type) + + " for result '" + name + "'"); + Pointer.free(buffer); + Loader.deleteGlobalRef(buffer_userp); + + return null; // Success + } + } + + static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { + @Override public void call ( + TRITONSERVER_InferenceRequest request, int flags, Pointer userp) + { + // We reuse the request so we don't delete it here. + } + } + + static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { + @Override public void call ( + TRITONSERVER_InferenceResponse response, int flags, Pointer userp) + { + if (response != null) { + // Send 'response' to the future. + futures.get(userp).complete(response); + } + } + } + + static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); + static ResponseAlloc responseAlloc = new ResponseAlloc(); + static ResponseRelease responseRelease = new ResponseRelease(); + static InferRequestComplete inferRequestComplete = new InferRequestComplete(); + static InferResponseComplete inferResponseComplete = new InferResponseComplete(); + + static TRITONSERVER_Error + ParseModelMetadata( + JsonObject model_metadata, boolean[] is_int, + boolean[] is_torch_model) + { + String seen_data_type = null; + for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { + JsonObject input = input_element.getAsJsonObject(); + if (!input.get("datatype").getAsString().equals("INT32") && + !input.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } + if (seen_data_type == null) { + seen_data_type = input.get("datatype").getAsString(); + } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { + JsonObject output = output_element.getAsJsonObject(); + if (!output.get("datatype").getAsString().equals("INT32") && + !output.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + + is_int[0] = seen_data_type.equals("INT32"); + is_torch_model[0] = + model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); + return null; + } + + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + GenerateInputData( + FloatPointer[] input0_data, FloatPointer[] input1_data) + { + input0_data[0] = new FloatPointer(16); + input1_data[0] = new FloatPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + CompareResult( + String output0_name, String output1_name, + IntPointer input0, IntPointer input1, IntPointer output0, + IntPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + CompareResult( + String output0_name, String output1_name, + FloatPointer input0, FloatPointer input1, FloatPointer output0, + FloatPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + Check( + TRITONSERVER_InferenceResponse response, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + int[] output_count = {0}; + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutputCount(response, output_count), + "getting number of response outputs"); + if (output_count[0] != 2) { + FAIL("expecting 2 response outputs, got " + output_count[0]); + } + + for (int idx = 0; idx < output_count[0]; ++idx) { + BytePointer cname = new BytePointer((Pointer)null); + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutput( + response, idx, cname, datatype, shape, dim_count, base, + byte_size, memory_type, memory_type_id, userp), + "getting output info"); + + if (cname.isNull()) { + FAIL("unable to get output name"); + } + + String name = cname.getString(); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + static void + Check( + InferResult result, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + System.out.println("geting output counts"); + + int output_count = result.OutputNames().limit(); + + + if (output_count != 2) { + FAIL("expecting 2 response outputs, got " + output_count); + } + + for (int idx = 0; idx < output_count; ++idx) { + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + + System.out.println("geting output info"); + String name = result.OutputNames().position(idx); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + + + Tensor output = result.Ouput(name); + dim_count = + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + + public static void + RunInference(String model_repository_path, int verbose_level) throws Exception + { + // Check API version. + int[] api_version_major = {0}, api_version_minor = {0}; + FAIL_IF_ERR( + TRITONSERVER_ApiVersion(api_version_major, api_version_minor), + "getting Triton API version"); + if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || + (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { + FAIL("triton server API version mismatch"); + } + + // Create the server... + TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsNew(server_options), + "creating server options"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetModelRepositoryPath( + server_options, model_repository_path), + "setting model repository path"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), + "setting verbose logging level"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetBackendDirectory( + server_options, "/opt/tritonserver/backends"), + "setting backend directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + server_options, "/opt/tritonserver/repoagents"), + "setting repository agent directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), + "setting strict model configuration"); + + TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); + FAIL_IF_ERR( + TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsDelete(server_options), + "deleting server options"); + + TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); + + // Wait until the server is both live and ready. + int health_iters = 0; + while (true) { + boolean[] live = {false}, ready = {false}; + FAIL_IF_ERR( + TRITONSERVER_ServerIsLive(server, live), + "unable to get server liveness"); + FAIL_IF_ERR( + TRITONSERVER_ServerIsReady(server, ready), + "unable to get server readiness"); + System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); + if (live[0] && ready[0]) { + break; + } + + if (++health_iters >= 10) { + FAIL("failed to find healthy inference server"); + } + + Thread.sleep(500); + } + + // Print status of the server. + { + TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerMetadata(server, server_metadata_message), + "unable to get server metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + server_metadata_message, buffer, byte_size), + "unable to serialize server metadata message"); + + System.out.println("Server Status:"); + System.out.println(buffer.limit(byte_size.get()).getString()); + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(server_metadata_message), + "deleting status metadata"); + } + + String model_name = "simple"; + + // Wait for the model to become available. + boolean[] is_torch_model = {false}; + boolean[] is_int = {true}; + boolean[] is_ready = {false}; + health_iters = 0; + while (!is_ready[0]) { + FAIL_IF_ERR( + TRITONSERVER_ServerModelIsReady( + server, model_name, 1, is_ready), + "unable to get model readiness"); + if (!is_ready[0]) { + if (++health_iters >= 10) { + FAIL("model failed to be ready in 10 iterations"); + } + Thread.sleep(500); + continue; + } + + TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerModelMetadata( + server, model_name, 1, model_metadata_message), + "unable to get model metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + model_metadata_message, buffer, byte_size), + "unable to serialize model status protobuf"); + + JsonParser parser = new JsonParser(); + JsonObject model_metadata = null; + try { + model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); + } catch (Exception e) { + FAIL("error: failed to parse model metadata from JSON: " + e); + } + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(model_metadata_message), + "deleting status protobuf"); + + if (!model_metadata.get("name").getAsString().equals(model_name)) { + FAIL("unable to find metadata for model"); + } + + boolean found_version = false; + if (model_metadata.has("versions")) { + for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { + if (version.getAsString().equals("1")) { + found_version = true; + break; + } + } + } + if (!found_version) { + FAIL("unable to find version 1 status for model"); + } + + FAIL_IF_ERR( + ParseModelMetadata(model_metadata, is_int, is_torch_model), + "parsing model metadata"); + } + + // Create the allocator that will be used to allocate buffers for + // the result tensors. + TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorNew( + allocator, responseAlloc, responseRelease, null /* start_fn */), + "creating response allocator"); + + // Inference + TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestNew( + irequest, server, model_name, -1 /* model_version */), + "creating inference request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), + "setting ID for the request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, inferRequestComplete, null /* request_release_userp */), + "setting request release callback"); + + // Inputs + String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; + String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; + + long[] input0_shape = {1, 16}; + long[] input1_shape = {1, 16}; + + int datatype = + (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input0, datatype, input0_shape, input0_shape.length), + "setting input 0 meta-data for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input1, datatype, input1_shape, input1_shape.length), + "setting input 1 meta-data for the request"); + + String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; + String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), + "requesting output 0 for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), + "requesting output 1 for the request"); + + // Create the data for the two input tensors. Initialize the first + // to unique values and the second to all ones. + BytePointer input0_data; + BytePointer input1_data; + if (is_int[0]) { + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } else { + FloatPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } + + long input0_size = input0_data.limit(); + long input1_size = input1_data.limit(); + + Pointer input0_base = input0_data; + Pointer input1_base = input1_data; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input0_base, input0_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input1, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data"); + + // Perform inference... + { + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Modify some input data in place and then reuse the request + // object. + { + if (is_int[0]) { + new IntPointer(input0_data).put(0, 27); + } else { + new FloatPointer(input0_data).put(0, 27.0f); + } + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Remove input data and then add back different data. + { + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), + "removing INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data to INPUT0"); + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + // Both inputs are using input1_data... + Check( + completed_response, input1_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestDelete(irequest), + "deleting inference request"); + + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorDelete(allocator), + "deleting response allocator"); + } + + public static void + main(String[] args) throws Exception + { + String model_repository_path = null; + int verbose_level = 0; + + // Parse commandline... + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-r": + model_repository_path = args[++i]; + break; + case "-v": + verbose_level = 1; + break; + case "-?": + Usage(null); + break; + } + } + + if (model_repository_path == null) { + Usage("-r must be used to specify model repository path"); + } + + try (PointerScope scope = new PointerScope()) { + RunInference(model_repository_path, verbose_level); + } + + System.exit(0); + } +} From bf057aaa81871f450da412c47b51c0310f8d0d39 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 28 Oct 2022 17:34:16 -0700 Subject: [PATCH 10/50] add the tritonserver folder --- platform/pom.xml | 6 + pom.xml | 2 + tritonserver-wrapper/README.md | 2 + tritonserver-wrapper/platform/pom.xml | 126 ++++++++++++++++++ tritonserver-wrapper/platform/redist/pom.xml | 118 ++++++++++++++++ tritonserver-wrapper/pom.xml | 116 ++++++++++++++++ .../src/main/java9/module-info.java | 6 + 7 files changed, 376 insertions(+) create mode 100644 tritonserver-wrapper/README.md create mode 100644 tritonserver-wrapper/platform/pom.xml create mode 100644 tritonserver-wrapper/platform/redist/pom.xml create mode 100644 tritonserver-wrapper/pom.xml create mode 100644 tritonserver-wrapper/src/main/java9/module-info.java diff --git a/platform/pom.xml b/platform/pom.xml index 3624223291f..ba30924159b 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,6 +61,7 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform + ../tritonserver-wrapper/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -313,6 +314,11 @@ tritonserver-platform 2.26-${project.version} + + org.bytedeco + tritonserver-wrapper-platform + 2.24-${project.version} + org.bytedeco ale-platform diff --git a/pom.xml b/pom.xml index bbd26497b9e..d88f3ef727e 100644 --- a/pom.xml +++ b/pom.xml @@ -623,6 +623,7 @@ tensorflow-lite tensorrt tritonserver + tritonserver-wrapper ale depthai onnx @@ -1406,6 +1407,7 @@ tensorflow-lite tensorrt tritonserver + tritonserver-wrapper ale depthai onnx diff --git a/tritonserver-wrapper/README.md b/tritonserver-wrapper/README.md new file mode 100644 index 00000000000..397e51e4d6c --- /dev/null +++ b/tritonserver-wrapper/README.md @@ -0,0 +1,2 @@ +JavaCPP Presets for Triton Inference Server +=========================================== diff --git a/tritonserver-wrapper/platform/pom.xml b/tritonserver-wrapper/platform/pom.xml new file mode 100644 index 00000000000..9b0e8606541 --- /dev/null +++ b/tritonserver-wrapper/platform/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + ../../ + + + org.bytedeco + tritonserver-wrapper-platform + 2.26-${project.parent.version} + JavaCPP Presets Platform for Triton Inference Server Wrapper + + + tritonserver-wrapper + + + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-x86_64} + + + com.google.code.gson + gson + 2.9.0 + + + + + + + maven-jar-plugin + + + default-jar + + + + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64.jar ${javacpp.moduleId}-linux-x86_64.jar ${javacpp.moduleId}-windows-x86_64.jar + + + + + + empty-javadoc-jar + + jar + + + javadoc + + + + empty-sources-jar + + jar + + + sources + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + none + + + add-platform-module-info + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}.jar + + module org.bytedeco.${javacpp.moduleId}.platform { +// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; + requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64; +// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64; + } + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + true + + + + + + + + diff --git a/tritonserver-wrapper/platform/redist/pom.xml b/tritonserver-wrapper/platform/redist/pom.xml new file mode 100644 index 00000000000..0bb28bb524f --- /dev/null +++ b/tritonserver-wrapper/platform/redist/pom.xml @@ -0,0 +1,118 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + ../../../ + + + org.bytedeco + tritonserver-wrapper-platform-redist + 2.26-${project.parent.version} + JavaCPP Presets Platform Redist for Triton Inference Server + + + tritonserver-wrapper + -redist + + + + + ${project.groupId} + ${javacpp.moduleId}-platform + ${project.version} + + + + + + + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-x86_64} + + + + + + + + + + + + + maven-jar-plugin + + + default-jar + + + + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64-redist.jar ${javacpp.moduleId}-linux-x86_64-redist.jar ${javacpp.moduleId}-windows-x86_64-redist.jar + + + + + + empty-javadoc-jar + + jar + + + javadoc + + + + empty-sources-jar + + jar + + + sources + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + none + + + add-platform-module-info + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}.jar + + module org.bytedeco.${javacpp.moduleId}.platform.redist { +// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64.redist; + requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64.redist; +// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64.redist; + } + + + + + + + + + + + diff --git a/tritonserver-wrapper/pom.xml b/tritonserver-wrapper/pom.xml new file mode 100644 index 00000000000..093fc778430 --- /dev/null +++ b/tritonserver-wrapper/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.5.8-SNAPSHOT + + + org.bytedeco + tritonserver-wrapper + 2.26-${project.parent.version} + JavaCPP Presets for Triton Inference Server + + + + org.bytedeco + javacpp + + + + + + + maven-resources-plugin + + + maven-compiler-plugin + + + org.bytedeco + javacpp + + ISO-8859-1 + + + + maven-jar-plugin + + + javacpp-${javacpp.platform} + package + + jar + + + ${javacpp.platform} + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + META-INF/native-image/${javacpp.platform}/ + + + + + javacpp-${javacpp.platform}-redist + package + + jar + + + ${javacpp.platform}-redist + ${project.build.directory}/native + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/ + META-INF/native-image/${javacpp.platform}/ + + + org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + + + + + + + org.moditect + moditect-maven-plugin + + + add-module-info-redist + package + + add-module-info + + + + + ${project.build.directory}/${project.artifactId}-${javacpp.platform}-redist.jar + + open module org.bytedeco.${javacpp.packageName}.${javacpp.platform.module}.redist { + requires transitive org.bytedeco.${javacpp.packageName}; + } + + + + + + + + + maven-dependency-plugin + + + maven-source-plugin + + + maven-javadoc-plugin + + ISO-8859-1 + + + + + + diff --git a/tritonserver-wrapper/src/main/java9/module-info.java b/tritonserver-wrapper/src/main/java9/module-info.java new file mode 100644 index 00000000000..6a1b64a9d46 --- /dev/null +++ b/tritonserver-wrapper/src/main/java9/module-info.java @@ -0,0 +1,6 @@ +module org.bytedeco.tritonserver { + requires transitive org.bytedeco.javacpp; + exports org.bytedeco.tritonserver.global; + exports org.bytedeco.tritonserver.presets; + exports org.bytedeco.tritonserver; +} From 17c233719f4878a19c85595c7864e7ac75120594 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 3 Nov 2022 12:22:17 -0700 Subject: [PATCH 11/50] add wrapper repo --- platform/pom.xml | 4 +- pom.xml | 4 +- .../src/main/java9/module-info.java | 6 - .../README.md | 0 tritonserverwrapper/cppbuild.sh | 30 + .../platform/pom.xml | 14 +- .../platform/redist/pom.xml | 16 +- .../pom.xml | 29 +- .../samples/simple/Simple.java | 764 ++++++++++++++++++ tritonserverwrapper/samples/simple/pom.xml | 22 + .../presets/tritonserverwrapper.java | 71 ++ .../src/main/java9/module-info.java | 6 + 12 files changed, 930 insertions(+), 36 deletions(-) delete mode 100644 tritonserver-wrapper/src/main/java9/module-info.java rename {tritonserver-wrapper => tritonserverwrapper}/README.md (100%) create mode 100755 tritonserverwrapper/cppbuild.sh rename {tritonserver-wrapper => tritonserverwrapper}/platform/pom.xml (92%) rename {tritonserver-wrapper => tritonserverwrapper}/platform/redist/pom.xml (83%) rename {tritonserver-wrapper => tritonserverwrapper}/pom.xml (77%) create mode 100644 tritonserverwrapper/samples/simple/Simple.java create mode 100644 tritonserverwrapper/samples/simple/pom.xml create mode 100644 tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java create mode 100644 tritonserverwrapper/src/main/java9/module-info.java diff --git a/platform/pom.xml b/platform/pom.xml index ba30924159b..11ca0e656c1 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,7 +61,7 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform - ../tritonserver-wrapper/platform + ../tritonserverwrapper/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -316,7 +316,7 @@ org.bytedeco - tritonserver-wrapper-platform + tritonserverwrapper-platform 2.24-${project.version} diff --git a/pom.xml b/pom.xml index d88f3ef727e..74711303ea9 100644 --- a/pom.xml +++ b/pom.xml @@ -623,7 +623,7 @@ tensorflow-lite tensorrt tritonserver - tritonserver-wrapper + tritonserverwrapper ale depthai onnx @@ -1407,7 +1407,7 @@ tensorflow-lite tensorrt tritonserver - tritonserver-wrapper + tritonserverwrapper ale depthai onnx diff --git a/tritonserver-wrapper/src/main/java9/module-info.java b/tritonserver-wrapper/src/main/java9/module-info.java deleted file mode 100644 index 6a1b64a9d46..00000000000 --- a/tritonserver-wrapper/src/main/java9/module-info.java +++ /dev/null @@ -1,6 +0,0 @@ -module org.bytedeco.tritonserver { - requires transitive org.bytedeco.javacpp; - exports org.bytedeco.tritonserver.global; - exports org.bytedeco.tritonserver.presets; - exports org.bytedeco.tritonserver; -} diff --git a/tritonserver-wrapper/README.md b/tritonserverwrapper/README.md similarity index 100% rename from tritonserver-wrapper/README.md rename to tritonserverwrapper/README.md diff --git a/tritonserverwrapper/cppbuild.sh b/tritonserverwrapper/cppbuild.sh new file mode 100755 index 00000000000..87fe0b381f1 --- /dev/null +++ b/tritonserverwrapper/cppbuild.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# This file is meant to be included by the parent cppbuild.sh script +if [[ -z "$PLATFORM" ]]; then + pushd .. + bash cppbuild.sh "$@" tritonserverwrapper + popd + exit +fi + +case $PLATFORM in + linux-arm64) + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + echo "Please make sure library and include files exist" + exit 1 + fi + ;; + linux-x86_64) + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + echo "Please make sure library and include files exist" + exit 1 + fi + ;; + windows-x86_64) + echo "Windows is not supported yet" + exit 1 + ;; + *) + echo "Error: Platform \"$PLATFORM\" is not supported" + ;; +esac diff --git a/tritonserver-wrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml similarity index 92% rename from tritonserver-wrapper/platform/pom.xml rename to tritonserverwrapper/platform/pom.xml index 9b0e8606541..a177256db48 100644 --- a/tritonserver-wrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -11,15 +11,20 @@ org.bytedeco - tritonserver-wrapper-platform + tritonserverwrapper-platform 2.26-${project.parent.version} JavaCPP Presets Platform for Triton Inference Server Wrapper - tritonserver-wrapper + tritonserverwrapper + ${project.groupId} ${javacpp.moduleId} @@ -31,11 +36,6 @@ ${project.version} ${javacpp.platform.linux-x86_64} - - com.google.code.gson - gson - 2.9.0 - diff --git a/tritonserver-wrapper/platform/redist/pom.xml b/tritonserverwrapper/platform/redist/pom.xml similarity index 83% rename from tritonserver-wrapper/platform/redist/pom.xml rename to tritonserverwrapper/platform/redist/pom.xml index 0bb28bb524f..edc9651a14b 100644 --- a/tritonserver-wrapper/platform/redist/pom.xml +++ b/tritonserverwrapper/platform/redist/pom.xml @@ -11,12 +11,12 @@ org.bytedeco - tritonserver-wrapper-platform-redist + tritonserverwrapper-platform-redist 2.26-${project.parent.version} JavaCPP Presets Platform Redist for Triton Inference Server - tritonserver-wrapper + tritonserverwrapper -redist @@ -26,24 +26,12 @@ ${javacpp.moduleId}-platform ${project.version} - - - - - - ${project.groupId} ${javacpp.moduleId} ${project.version} ${javacpp.platform.linux-x86_64} - - - - - - diff --git a/tritonserver-wrapper/pom.xml b/tritonserverwrapper/pom.xml similarity index 77% rename from tritonserver-wrapper/pom.xml rename to tritonserverwrapper/pom.xml index 093fc778430..8f6cda3c6d5 100644 --- a/tritonserver-wrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -10,11 +10,16 @@ org.bytedeco - tritonserver-wrapper + tritonserverwrapper 2.26-${project.parent.version} - JavaCPP Presets for Triton Inference Server + JavaCPP Presets for Triton Inference Server Wrapper + org.bytedeco javacpp @@ -36,6 +41,20 @@ ISO-8859-1 + maven-jar-plugin @@ -48,7 +67,7 @@ ${javacpp.platform} - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* META-INF/native-image/${javacpp.platform}/ @@ -63,11 +82,11 @@ ${javacpp.platform}-redist ${project.build.directory}/native - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/ + org/bytedeco/tritonserverwrapper/${javacpp.platform}/ META-INF/native-image/${javacpp.platform}/ - org/bytedeco/tritonserver-wrapper/${javacpp.platform}/*jni* + org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* diff --git a/tritonserverwrapper/samples/simple/Simple.java b/tritonserverwrapper/samples/simple/Simple.java new file mode 100644 index 00000000000..214e0a82096 --- /dev/null +++ b/tritonserverwrapper/samples/simple/Simple.java @@ -0,0 +1,764 @@ +// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; +import com.google.gson.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.tritonserver.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritonserver.*; +// import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; +// import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +public class Simple { + + static void FAIL(String MSG) { + System.err.println("Failure: " + MSG); + System.exit(1); + } + + static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { + if (err__ != null) { + System.err.println("error: " + MSG + ":" + + TRITONSERVER_ErrorCodeString(err__) + " - " + + TRITONSERVER_ErrorMessage(err__)); + TRITONSERVER_ErrorDelete(err__); + System.exit(1); + } + } + + static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; + + static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { + public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } + protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { + DeleteDeallocator(Pointer p) { super(p); } + @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } + } + } + + static void + Usage(String msg) + { + if (msg != null) { + System.err.println(msg); + } + + System.err.println("Usage: java " + Simple.class.getSimpleName() + " [options]"); + System.err.println("\t-v Enable verbose logging"); + System.err.println("\t-r [model repository absolute path]"); + + System.exit(1); + } + + static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + long byte_size, int preferred_memory_type, + long preferred_memory_type_id, Pointer userp, PointerPointer buffer, + PointerPointer buffer_userp, IntPointer actual_memory_type, + LongPointer actual_memory_type_id) + { + // Initially attempt to make the actual memory type and id that we + // allocate be the same as preferred memory type + actual_memory_type.put(0, preferred_memory_type); + actual_memory_type_id.put(0, preferred_memory_type_id); + + // If 'byte_size' is zero just return 'buffer' == nullptr, we don't + // need to do any other book-keeping. + if (byte_size == 0) { + buffer.put(0, null); + buffer_userp.put(0, null); + System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); + } else { + Pointer allocated_ptr = new Pointer(); + actual_memory_type.put(0, requested_memory_type); + + actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); + allocated_ptr = Pointer.malloc(byte_size); + + // Pass the tensor name with buffer_userp so we can show it when + // releasing the buffer. + if (!allocated_ptr.isNull()) { + buffer.put(0, allocated_ptr); + buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); + System.out.println("allocated " + byte_size + " bytes in " + + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) + + " for result tensor " + tensor_name); + } + } + + return null; // Success + } + } + + static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { + @Override public TRITONSERVER_Error call ( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + long byte_size, int memory_type, long memory_type_id) + { + String name = null; + if (buffer_userp != null) { + name = (String)Loader.accessGlobalRef(buffer_userp); + } else { + name = ""; + } + + System.out.println("Releasing buffer " + buffer + " of size " + byte_size + + " in " + TRITONSERVER_MemoryTypeString(memory_type) + + " for result '" + name + "'"); + Pointer.free(buffer); + Loader.deleteGlobalRef(buffer_userp); + + return null; // Success + } + } + + static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { + @Override public void call ( + TRITONSERVER_InferenceRequest request, int flags, Pointer userp) + { + // We reuse the request so we don't delete it here. + } + } + + static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { + @Override public void call ( + TRITONSERVER_InferenceResponse response, int flags, Pointer userp) + { + if (response != null) { + // Send 'response' to the future. + futures.get(userp).complete(response); + } + } + } + + static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); + static ResponseAlloc responseAlloc = new ResponseAlloc(); + static ResponseRelease responseRelease = new ResponseRelease(); + static InferRequestComplete inferRequestComplete = new InferRequestComplete(); + static InferResponseComplete inferResponseComplete = new InferResponseComplete(); + + static TRITONSERVER_Error + ParseModelMetadata( + JsonObject model_metadata, boolean[] is_int, + boolean[] is_torch_model) + { + String seen_data_type = null; + for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { + JsonObject input = input_element.getAsJsonObject(); + if (!input.get("datatype").getAsString().equals("INT32") && + !input.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } + if (seen_data_type == null) { + seen_data_type = input.get("datatype").getAsString(); + } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { + JsonObject output = output_element.getAsJsonObject(); + if (!output.get("datatype").getAsString().equals("INT32") && + !output.get("datatype").getAsString().equals("FP32")) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "simple lib example only supports model with data type INT32 or " + + "FP32"); + } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + "the inputs and outputs of 'simple' model must have the data type"); + } + } + + is_int[0] = seen_data_type.equals("INT32"); + is_torch_model[0] = + model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); + return null; + } + + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + GenerateInputData( + FloatPointer[] input0_data, FloatPointer[] input1_data) + { + input0_data[0] = new FloatPointer(16); + input1_data[0] = new FloatPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, i); + input1_data[0].put(i, 1); + } + } + + static void + CompareResult( + String output0_name, String output1_name, + IntPointer input0, IntPointer input1, IntPointer output0, + IntPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + CompareResult( + String output0_name, String output1_name, + FloatPointer input0, FloatPointer input1, FloatPointer output0, + FloatPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static void + Check( + TRITONSERVER_InferenceResponse response, + Pointer input0_data, Pointer input1_data, + String output0, String output1, + long expected_byte_size, + int expected_datatype, boolean is_int) + { + HashMap output_data = new HashMap<>(); + + int[] output_count = {0}; + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutputCount(response, output_count), + "getting number of response outputs"); + if (output_count[0] != 2) { + FAIL("expecting 2 response outputs, got " + output_count[0]); + } + + for (int idx = 0; idx < output_count[0]; ++idx) { + BytePointer cname = new BytePointer((Pointer)null); + IntPointer datatype = new IntPointer(1); + LongPointer shape = new LongPointer((Pointer)null); + LongPointer dim_count = new LongPointer(1); + Pointer base = new Pointer(); + SizeTPointer byte_size = new SizeTPointer(1); + IntPointer memory_type = new IntPointer(1); + LongPointer memory_type_id = new LongPointer(1); + Pointer userp = new Pointer(); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseOutput( + response, idx, cname, datatype, shape, dim_count, base, + byte_size, memory_type, memory_type_id, userp), + "getting output info"); + + if (cname.isNull()) { + FAIL("unable to get output name"); + } + + String name = cname.getString(); + if ((!name.equals(output0)) && (!name.equals(output1))) { + FAIL("unexpected output '" + name + "'"); + } + + if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { + FAIL("unexpected shape for '" + name + "'"); + } + + if (datatype.get() != expected_datatype) { + FAIL( + "unexpected datatype '" + + TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + + name + "'"); + } + + if (byte_size.get() != expected_byte_size) { + FAIL( + "unexpected byte-size, expected " + + expected_byte_size + ", got " + + byte_size.get() + " for " + name); + } + + if (memory_type.get() != requested_memory_type) { + FAIL( + "unexpected memory type, expected to be allocated in " + + TRITONSERVER_MemoryTypeString(requested_memory_type) + + ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + + ", id " + memory_type_id.get() + " for " + name); + } + + // We make a copy of the data here... which we could avoid for + // performance reasons but ok for this simple example. + BytePointer odata = new BytePointer(byte_size.get()); + output_data.put(name, odata); + System.out.println(name + " is stored in system memory"); + odata.put(base.limit(byte_size.get())); + } + + if (is_int) { + CompareResult( + output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), + new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); + } else { + CompareResult( + output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), + new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); + } + } + + public static void + RunInference(String model_repository_path, int verbose_level) throws Exception + { + // Check API version. + int[] api_version_major = {0}, api_version_minor = {0}; + FAIL_IF_ERR( + TRITONSERVER_ApiVersion(api_version_major, api_version_minor), + "getting Triton API version"); + if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || + (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { + FAIL("triton server API version mismatch"); + } + + // Create the server... + TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsNew(server_options), + "creating server options"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetModelRepositoryPath( + server_options, model_repository_path), + "setting model repository path"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), + "setting verbose logging level"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetBackendDirectory( + server_options, "/opt/tritonserver/backends"), + "setting backend directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + server_options, "/opt/tritonserver/repoagents"), + "setting repository agent directory"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), + "setting strict model configuration"); + + TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); + FAIL_IF_ERR( + TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); + FAIL_IF_ERR( + TRITONSERVER_ServerOptionsDelete(server_options), + "deleting server options"); + + TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); + + // Wait until the server is both live and ready. + int health_iters = 0; + while (true) { + boolean[] live = {false}, ready = {false}; + FAIL_IF_ERR( + TRITONSERVER_ServerIsLive(server, live), + "unable to get server liveness"); + FAIL_IF_ERR( + TRITONSERVER_ServerIsReady(server, ready), + "unable to get server readiness"); + System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); + if (live[0] && ready[0]) { + break; + } + + if (++health_iters >= 10) { + FAIL("failed to find healthy inference server"); + } + + Thread.sleep(500); + } + + // Print status of the server. + { + TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerMetadata(server, server_metadata_message), + "unable to get server metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + server_metadata_message, buffer, byte_size), + "unable to serialize server metadata message"); + + System.out.println("Server Status:"); + System.out.println(buffer.limit(byte_size.get()).getString()); + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(server_metadata_message), + "deleting status metadata"); + } + + String model_name = "simple"; + + // Wait for the model to become available. + boolean[] is_torch_model = {false}; + boolean[] is_int = {true}; + boolean[] is_ready = {false}; + health_iters = 0; + while (!is_ready[0]) { + FAIL_IF_ERR( + TRITONSERVER_ServerModelIsReady( + server, model_name, 1, is_ready), + "unable to get model readiness"); + if (!is_ready[0]) { + if (++health_iters >= 10) { + FAIL("model failed to be ready in 10 iterations"); + } + Thread.sleep(500); + continue; + } + + TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); + FAIL_IF_ERR( + TRITONSERVER_ServerModelMetadata( + server, model_name, 1, model_metadata_message), + "unable to get model metadata message"); + BytePointer buffer = new BytePointer((Pointer)null); + SizeTPointer byte_size = new SizeTPointer(1); + FAIL_IF_ERR( + TRITONSERVER_MessageSerializeToJson( + model_metadata_message, buffer, byte_size), + "unable to serialize model status protobuf"); + + JsonParser parser = new JsonParser(); + JsonObject model_metadata = null; + try { + model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); + } catch (Exception e) { + FAIL("error: failed to parse model metadata from JSON: " + e); + } + + FAIL_IF_ERR( + TRITONSERVER_MessageDelete(model_metadata_message), + "deleting status protobuf"); + + if (!model_metadata.get("name").getAsString().equals(model_name)) { + FAIL("unable to find metadata for model"); + } + + boolean found_version = false; + if (model_metadata.has("versions")) { + for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { + if (version.getAsString().equals("1")) { + found_version = true; + break; + } + } + } + if (!found_version) { + FAIL("unable to find version 1 status for model"); + } + + FAIL_IF_ERR( + ParseModelMetadata(model_metadata, is_int, is_torch_model), + "parsing model metadata"); + } + + // Create the allocator that will be used to allocate buffers for + // the result tensors. + TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorNew( + allocator, responseAlloc, responseRelease, null /* start_fn */), + "creating response allocator"); + + // Inference + TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestNew( + irequest, server, model_name, -1 /* model_version */), + "creating inference request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), + "setting ID for the request"); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, inferRequestComplete, null /* request_release_userp */), + "setting request release callback"); + + // Inputs + String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; + String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; + + long[] input0_shape = {1, 16}; + long[] input1_shape = {1, 16}; + + int datatype = + (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input0, datatype, input0_shape, input0_shape.length), + "setting input 0 meta-data for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddInput( + irequest, input1, datatype, input1_shape, input1_shape.length), + "setting input 1 meta-data for the request"); + + String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; + String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), + "requesting output 0 for the request"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), + "requesting output 1 for the request"); + + // Create the data for the two input tensors. Initialize the first + // to unique values and the second to all ones. + BytePointer input0_data; + BytePointer input1_data; + if (is_int[0]) { + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } else { + FloatPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + } + + long input0_size = input0_data.limit(); + long input1_size = input1_data.limit(); + + Pointer input0_base = input0_data; + Pointer input1_base = input1_data; + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input0_base, input0_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input1, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data"); + + // Perform inference... + { + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Modify some input data in place and then reuse the request + // object. + { + if (is_int[0]) { + new IntPointer(input0_data).put(0, 27); + } else { + new FloatPointer(input0_data).put(0, 27.0f); + } + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + Check( + completed_response, input0_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + // Remove input data and then add back different data. + { + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), + "removing INPUT0 data"); + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestAppendInputData( + irequest, input0, input1_base, input1_size, requested_memory_type, + 0 /* memory_type_id */), + "assigning INPUT1 data to INPUT0"); + + CompletableFuture completed = new CompletableFuture<>(); + futures.put(irequest, completed); + + // Using a new promise so have to re-register the callback to set + // the promise as the userp. + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator, null /* response_allocator_userp */, + inferResponseComplete, irequest), + "setting response callback"); + + FAIL_IF_ERR( + TRITONSERVER_ServerInferAsync( + server, irequest, null /* trace */), + "running inference"); + + // Wait for the inference to complete. + TRITONSERVER_InferenceResponse completed_response = completed.get(); + futures.remove(irequest); + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseError(completed_response), + "response status"); + + // Both inputs are using input1_data... + Check( + completed_response, input1_data, input1_data, output0, output1, + input0_size, datatype, is_int[0]); + + FAIL_IF_ERR( + TRITONSERVER_InferenceResponseDelete(completed_response), + "deleting inference response"); + } + + FAIL_IF_ERR( + TRITONSERVER_InferenceRequestDelete(irequest), + "deleting inference request"); + + FAIL_IF_ERR( + TRITONSERVER_ResponseAllocatorDelete(allocator), + "deleting response allocator"); + } + + public static void + main(String[] args) throws Exception + { + String model_repository_path = null; + int verbose_level = 0; + + // Parse commandline... + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-r": + model_repository_path = args[++i]; + break; + case "-v": + verbose_level = 1; + break; + case "-?": + Usage(null); + break; + } + } + + if (model_repository_path == null) { + Usage("-r must be used to specify model repository path"); + } + + try (PointerScope scope = new PointerScope()) { + RunInference(model_repository_path, verbose_level); + } + + System.exit(0); + } +} diff --git a/tritonserverwrapper/samples/simple/pom.xml b/tritonserverwrapper/samples/simple/pom.xml new file mode 100644 index 00000000000..59c5b1d5fe2 --- /dev/null +++ b/tritonserverwrapper/samples/simple/pom.xml @@ -0,0 +1,22 @@ + + 4.0.0 + org.bytedeco.tritonserverwrapper + simple + 1.5.8-SNAPSHOT + + Simple + 1.8 + 1.8 + + + + org.bytedeco + tritonserverwrapper-platform + 2.26-1.5.8-SNAPSHOT + shaded + + + + . + + diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java new file mode 100644 index 00000000000..0c0efebe33c --- /dev/null +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2021 Jack He, Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.tritonserverwrapper.presets; + +import java.util.List; +import org.bytedeco.javacpp.ClassProperties; +import org.bytedeco.javacpp.LoadEnabled; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; +// import org.bytedeco.tritonserver.presets.tritonserver; + +/** + * + * @author Katherine Yang + */ +@Properties( + value = { + @Platform( + value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, + include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h","server_wrapper.h", "infer_requested_output.h", "common.h"}, + link = "tritonserverwrapper", + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, + linkpath = {"/opt/tritonserver/lib/"} + ), + // @Platform( + // value = "windows-x86_64", + // includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", + // linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", + // preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" + // ) + }, + target = "org.bytedeco.tritonserverwrapper.tritonserverwrapper", + global = "org.bytedeco.tritonserverwrapper.global.tritonserverwrapper" +) +public class tritonserverwrapper implements InfoMapper { + static { Loader.checkVersion("org.bytedeco", "tritonserverwrapper"); } + public void map(InfoMap infoMap) { + infoMap.putFirst(new Info().enumerate(false)) + .put(new Info("bool").cast().valueTypes("boolean").pointerTypes("boolean[]", "BoolPointer")) + .put(new Info("const char").pointerTypes("String", "@Cast(\"const char*\") BytePointer")) + .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) + .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", + "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + ; + } +} diff --git a/tritonserverwrapper/src/main/java9/module-info.java b/tritonserverwrapper/src/main/java9/module-info.java new file mode 100644 index 00000000000..d6f16438d12 --- /dev/null +++ b/tritonserverwrapper/src/main/java9/module-info.java @@ -0,0 +1,6 @@ +module org.bytedeco.tritonserverwrapper { + requires transitive org.bytedeco.javacpp; + exports org.bytedeco.tritonserverwrapper.global; + exports org.bytedeco.tritonserverwrapper.presets; + exports org.bytedeco.tritonserverwrapper; +} From adeb77283fcd140bad8ba620547c446cb701340e Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 3 Nov 2022 12:29:49 -0700 Subject: [PATCH 12/50] update tritonserver gen files and also start compiling with only wrapper --- .../tritonserver/global/tritonserver.java | 174 ++++++++++++++++-- .../TRITONBACKEND_BackendAttribute.java | 23 +++ .../TRITONBACKEND_ModelInstance.java | 6 - tritonserverwrapper/platform/pom.xml | 4 +- tritonserverwrapper/pom.xml | 4 +- .../presets/tritonserverwrapper.java | 4 +- 6 files changed, 191 insertions(+), 24 deletions(-) create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 0d486ae5aff..90f01e2fabf 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -130,7 +130,7 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver public static final int TRITONSERVER_API_VERSION_MAJOR = 1; /// -public static final int TRITONSERVER_API_VERSION_MINOR = 13; +public static final int TRITONSERVER_API_VERSION_MINOR = 17; /** Get the TRITONBACKEND API version supported by the Triton shared * library. This value can be compared against the @@ -240,7 +240,7 @@ public static native String TRITONSERVER_MemoryTypeString( TRITONSERVER_PARAMETER_BOOL = 2, TRITONSERVER_PARAMETER_BYTES = 3; -/** Get the string representation of a parmeter type. The returned +/** Get the string representation of a parameter type. The returned * string is not owned by the caller and so should not be modified or * freed. * @@ -328,6 +328,19 @@ public static native String TRITONSERVER_InstanceGroupKindString( TRITONSERVER_LOG_ERROR = 2, TRITONSERVER_LOG_VERBOSE = 3; +/** + * Format of logging. + * + * TRITONSERVER_LOG_DEFAULT: the log severity (L) and timestamp will be + * logged as "LMMDD hh:mm:ss.ssssss". + * + * TRITONSERVER_LOG_ISO8601: the log format will be "YYYY-MM-DDThh:mm:ssZ L". + * */ +/** enum TRITONSERVER_LogFormat */ +public static final int + TRITONSERVER_LOG_DEFAULT = 0, + TRITONSERVER_LOG_ISO8601 = 1; + /** Is a log level enabled? * * @param level The log level. @@ -1996,6 +2009,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout /** Set the number of threads used in buffer manager in a server options. * + * @param options The server options object. * @param thread_count The number of threads. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2003,6 +2017,30 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); +/** Set the number of threads to concurrently load models in a server options. + * + * @param options The server options object. + * @param thread_count The number of threads. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadThreadCount( + TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); + +/** Provide a log output file. + * + * @param options The server options object. + * @param file a string defining the file where the log outputs will be saved. + * An empty string for the file name will cause triton to direct logging + * facilities to the console + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, String file); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer file); + /** Enable or disable info level logging. * * @param options The server options object. @@ -2033,6 +2071,16 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogWarn( public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogError( TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); +/** Set the logging format. + * + * @param options The server options object. + * @param format The logging format. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFormat( + TRITONSERVER_ServerOptions options, @Cast("const TRITONSERVER_LogFormat") int format); + /** Set verbose logging level. Level zero disables verbose logging. * * @param options The server options object. @@ -2065,6 +2113,18 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetrics( public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetGpuMetrics( TRITONSERVER_ServerOptions options, @Cast("bool") boolean gpu_metrics); +/** Enable or disable CPU metrics collection in a server options. CPU + * metrics are collected if both this option and + * TRITONSERVER_ServerOptionsSetMetrics are true. + * + * @param options The server options object. + * @param cpu_metrics True to enable CPU metrics, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCpuMetrics( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean cpu_metrics); + /** Set the interval for metrics collection in a server options. * This is 2000 milliseconds by default. * @@ -2102,12 +2162,32 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDire * @param repoagent_dir The full path of the repository agent directory. * @return a TRITONSERVER_Error indicating success or failure. */ +/// /// public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( TRITONSERVER_ServerOptions options, String repoagent_dir); public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer repoagent_dir); +/** Specify the limit on memory usage as a fraction on the device identified by + * 'kind' and 'device_id'. If model loading on the device is requested and the + * current memory usage exceeds the limit, the load will be rejected. If not + * specified, the limit will not be set. + * + * Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU + * + * @param options The server options object. + * @param kind The kind of the device. + * @param device_id The id of the device. + * @param fraction The limit on memory usage as a fraction + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( + TRITONSERVER_ServerOptions options, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, int device_id, + double fraction); + /** Set a configuration setting for a named backend in a server * options. * @@ -2643,6 +2723,9 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer description); /** Delete a metric family object. + * A TRITONSERVER_MetricFamily* object should be deleted AFTER its + * corresponding TRITONSERVER_Metric* objects have been deleted. + * Attempting to delete a family before its metrics will return an error. * * @param family The metric family object to delete. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2653,7 +2736,10 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyDelete( /** Create a new metric object. The caller takes ownership of the * TRITONSERVER_Metric object and must call - * TRITONSERVER_MetricDelete to release the object. + * TRITONSERVER_MetricDelete to release the object. The caller is also + * responsible for ownership of the labels passed in. Each label can be deleted + * immediately after creating the metric with TRITONSERVER_ParameterDelete + * if not re-using the labels. * * @param metric Returns the new metric object. * @param family The metric family to add this new metric to. @@ -2670,6 +2756,9 @@ public static native TRITONSERVER_Error TRITONSERVER_MetricNew( @Const @ByPtrPtr TRITONSERVER_Parameter labels, @Cast("const uint64_t") long label_count); /** Delete a metric object. + * All TRITONSERVER_Metric* objects should be deleted BEFORE their + * corresponding TRITONSERVER_MetricFamily* objects have been deleted. + * If a family is deleted before its metrics, an error will be returned. * * @param metric The metric object to delete. * @return a TRITONSERVER_Error indicating success or failure. */ @@ -2822,6 +2911,9 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( // Targeting ../tritonserver/TRITONBACKEND_ModelInstance.java +// Targeting ../tritonserver/TRITONBACKEND_BackendAttribute.java + + /** * TRITONBACKEND API Version @@ -2851,7 +2943,7 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; /// -public static final int TRITONBACKEND_API_VERSION_MINOR = 9; +public static final int TRITONBACKEND_API_VERSION_MINOR = 10; /** Get the TRITONBACKEND API version supported by Triton. This value * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and @@ -4155,12 +4247,16 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); /** Set the model configuration in Triton server. Only the inputs, outputs, - * and max batch size can be changed. Any other changes to the model - * configuration will be ignored by Triton. This function can only be called - * from TRITONBACKEND_ModelInitialize, calling in any other context will result - * in an error being returned. The function does not take ownership of the - * message object and so the caller should call TRITONSERVER_MessageDelete to - * release the object once the function returns. + * max batch size, and scheduling choice can be changed. A caveat being + * scheduling choice can only be changed if none is previously set. Any other + * changes to the model configuration will be ignored by Triton. This function + * can only be called from TRITONBACKEND_ModelInitialize, calling in any other + * context will result in an error being returned. Additionally, Triton server + * can add some of the missing fields in the provided config with this call. + * The backend must get the complete configuration again by using + * TRITONBACKEND_ModelConfig. TRITONBACKEND_ModelSetConfig does not take + * ownership of the message object and so the caller should call + * TRITONSERVER_MessageDelete to release the object once the function returns. * * @param model The model. * @param config_version The format version of the model configuration. @@ -4525,7 +4621,6 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportStatist * @return a TRITONSERVER_Error indicating success or failure. */ - /// /// /// @@ -4534,7 +4629,6 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportBatchSt @Cast("const uint64_t") long exec_start_ns, @Cast("const uint64_t") long compute_start_ns, @Cast("const uint64_t") long compute_end_ns, @Cast("const uint64_t") long exec_end_ns); - /** * The following functions can be implemented by a backend. Functions * indicated as required must be implemented or the backend will fail @@ -4648,6 +4742,8 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceFinalize( * @param requests The requests. * @param request_count The number of requests in the batch. * @return a TRITONSERVER_Error indicating success or failure. */ + +/// public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Request**") PointerPointer requests, @Cast("const uint32_t") int request_count); @@ -4655,6 +4751,60 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Request requests, @Cast("const uint32_t") int request_count); +/** Query the backend for different model attributes. This function is optional, + * a backend is not required to implement it. The backend is also not required + * to set all backend attribute listed. This function is called when + * Triton requires further backend / model information to perform operations. + * This function may be called multiple times within the lifetime of the + * backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize). + * The backend may return error to indicate failure to set the backend + * attributes, and the attributes specified in the same function call will be + * ignored. Triton will update the specified attributes if 'nullptr' is + * returned. + * + * @param backend The backend. + * @param backend_attributes Return the backend attribute. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_GetBackendAttribute( + TRITONBACKEND_Backend backend, + TRITONBACKEND_BackendAttribute backend_attributes); + +/** TRITONBACKEND_BackendAttribute + * + * API to modify attributes associated with a backend. + * +

+ * Add the preferred instance group of the backend. This function + * can be called multiple times to cover different instance group kinds that + * the backend supports, given the priority order that the first call describes + * the most preferred group. In the case where instance group are not + * explicitly provided, Triton will use this attribute to create model + * deployment that aligns more with the backend preference. + * + * @param backend_attributes The backend attributes object. + * @param kind The kind of the instance group. + * @param count The number of instances per device. Triton default will be used + * if 0 is provided. + * @param device_ids The devices where instances should be available. Triton + * default will be used if 'nullptr' is provided. + * @param id_count The number of devices in 'device_ids'. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongPointer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongBuffer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") long[] device_ids, @Cast("const uint64_t") long id_count); // #ifdef __cplusplus // #endif diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java new file mode 100644 index 00000000000..9269f4b92f0 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java @@ -0,0 +1,23 @@ +// Targeted by JavaCPP version 1.5.8-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + + +/// +/// +/// +/// +@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONBACKEND_BackendAttribute extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_BackendAttribute() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_BackendAttribute(Pointer p) { super(p); } +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java index 4bf992d1b6b..fc2bd9ad679 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java @@ -8,12 +8,6 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/// -/// -/// -/// @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONBACKEND_ModelInstance extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ diff --git a/tritonserverwrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml index a177256db48..34e767006ad 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -20,11 +20,11 @@ - + ${project.groupId} ${javacpp.moduleId} diff --git a/tritonserverwrapper/pom.xml b/tritonserverwrapper/pom.xml index 8f6cda3c6d5..ac90944fdc8 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -15,11 +15,11 @@ JavaCPP Presets for Triton Inference Server Wrapper - + org.bytedeco javacpp diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index 0c0efebe33c..a96e01713ec 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -31,7 +31,7 @@ import org.bytedeco.javacpp.tools.Info; import org.bytedeco.javacpp.tools.InfoMap; import org.bytedeco.javacpp.tools.InfoMapper; -// import org.bytedeco.tritonserver.presets.tritonserver; +import org.bytedeco.tritonserver.presets.tritonserver; /** * @@ -41,7 +41,7 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h","server_wrapper.h", "infer_requested_output.h", "common.h"}, + include = {"server_wrapper.h", "infer_requested_output.h", "common.h"}, link = "tritonserverwrapper", includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, linkpath = {"/opt/tritonserver/lib/"} From 86437e82c26f8ef57886b2c04818e2505ea0b35d Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 15 Dec 2022 18:00:39 -0800 Subject: [PATCH 13/50] update container and keep getting the build working --- .../tritonserver/global/tritonserver.java | 25 +++++++++++-------- .../TRITONBACKEND_BackendAttribute.java | 2 +- .../presets/tritonserverwrapper.java | 11 ++++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 90f01e2fabf..1a24a782ba6 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -4246,17 +4246,20 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); -/** Set the model configuration in Triton server. Only the inputs, outputs, - * max batch size, and scheduling choice can be changed. A caveat being - * scheduling choice can only be changed if none is previously set. Any other - * changes to the model configuration will be ignored by Triton. This function - * can only be called from TRITONBACKEND_ModelInitialize, calling in any other - * context will result in an error being returned. Additionally, Triton server - * can add some of the missing fields in the provided config with this call. - * The backend must get the complete configuration again by using - * TRITONBACKEND_ModelConfig. TRITONBACKEND_ModelSetConfig does not take - * ownership of the message object and so the caller should call - * TRITONSERVER_MessageDelete to release the object once the function returns. +/** Set the model configuration in Triton server. This API should only be called + * when the backend implements the auto-completion of model configuration + * and TRITONBACKEND_ModelAutoCompleteConfig returns true in + * auto_complete_config. Only the inputs, outputs, max batch size, and + * scheduling choice can be changed. A caveat being scheduling choice can only + * be changed if none is previously set. Any other changes to the model + * configuration will be ignored by Triton. This function can only be called + * from TRITONBACKEND_ModelInitialize, calling in any other context will result + * in an error being returned. Additionally, Triton server can add some of the + * missing fields in the provided config with this call. The backend must get + * the complete configuration again by using TRITONBACKEND_ModelConfig. + * TRITONBACKEND_ModelSetConfig does not take ownership of the message object + * and so the caller should call TRITONSERVER_MessageDelete to release the + * object once the function returns. * * @param model The model. * @param config_version The format version of the model configuration. diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java index 9269f4b92f0..b2c4c9bad9a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.8-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.8: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index a96e01713ec..f667102987b 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -41,9 +41,9 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"server_wrapper.h", "infer_requested_output.h", "common.h"}, + include = {"common.h", "generic_server_wrapper.h"}, link = "tritonserverwrapper", - includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools"}, + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), // @Platform( @@ -66,6 +66,13 @@ public void map(InfoMap infoMap) { .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + .put(new Info("std::set").pointerTypes("StringSet").define()) + .put(new Info("std::vector").pointerTypes("StringVector").define()) + .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) + .put(new Info("TritonServer").purify(false).virtualize()) + // .put(new Info("server_wrapper.h").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) + // .put(new Info("server_wrapper.cc").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) + // .put(new Info("server_wrapper.h").linePatterns("std::future>*", ";", "std::unique_ptr>>*", ";").skip()) ; } } From 49f76b095eeef7fcbea945d8a70dadc59099ebc0 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 16 Dec 2022 18:10:06 -0800 Subject: [PATCH 14/50] update snapshot version --- tritonserverwrapper/platform/pom.xml | 2 +- tritonserverwrapper/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserverwrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml index 34e767006ad..4211a9555cc 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT ../../ diff --git a/tritonserverwrapper/pom.xml b/tritonserverwrapper/pom.xml index ac90944fdc8..b7e410f4598 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT org.bytedeco From aa6e0ce2857355ea39932a0cc249b847ca8f25f8 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 16 Feb 2023 14:18:38 -0800 Subject: [PATCH 15/50] new update --- .../global/tritonserverwrapper.java | 183 +++++++++++++++++ .../tritonserverwrapper/Allocator.java | 37 ++++ .../tritonserverwrapper/BackendConfig.java | 43 ++++ .../CUDAMemoryPoolByteSize.java | 32 +++ .../GenericInferRequest.java | 56 ++++++ .../GenericInferResult.java | 68 +++++++ .../GenericTritonServer.java | 132 ++++++++++++ .../tritonserverwrapper/HostPolicy.java | 47 +++++ .../tritonserverwrapper/InferOptions.java | 109 ++++++++++ .../tritonserverwrapper/LoggingOptions.java | 67 +++++++ .../tritonserverwrapper/MetricsOptions.java | 49 +++++ .../ModelLoadGPULimit.java | 30 +++ .../tritonserverwrapper/NewModelRepo.java | 50 +++++ .../OutputBufferReleaseFn_t.java | 21 ++ .../RateLimitResource.java | 43 ++++ .../tritonserverwrapper/RepositoryIndex.java | 50 +++++ .../ResponseAllocatorAllocFn_t.java | 26 +++ .../ResponseAllocatorStartFn_t.java | 19 ++ .../tritonserverwrapper/ServerOptions.java | 189 ++++++++++++++++++ .../tritonserverwrapper/StringSet.java | 36 ++++ .../tritonserverwrapper/StringVector.java | 99 +++++++++ .../tritonserverwrapper/Tensor.java | 81 ++++++++ .../tritonserverwrapper/Trace.java | 66 ++++++ .../tritonserverwrapper/TritonException.java | 29 +++ 24 files changed, 1562 insertions(+) create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java create mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java new file mode 100644 index 00000000000..c0b4f403cbd --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java @@ -0,0 +1,183 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.global; + +import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper { + static { Loader.load(); } + +// Targeting ../tritonserverwrapper/StringSet.java + + +// Targeting ../tritonserverwrapper/StringVector.java + + +// Parsed from common.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once + +//============================================================================== +/** enum classes + * */ +/** enum class triton::developer_tools::server::ModelControlMode */ +public static final int NONE = 0, POLL = 1, EXPLICIT = 2; +/** enum class triton::developer_tools::server::MemoryType */ +public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; +/** enum class triton::developer_tools::server::DataType */ +public static final int + INVALID = 0, + BOOL = 1, + UINT8 = 2, + UINT16 = 3, + UINT32 = 4, + UINT64 = 5, + INT8 = 6, + INT16 = 7, + INT32 = 8, + INT64 = 9, + FP16 = 10, + FP32 = 11, + FP64 = 12, + BYTES = 13, + BF16 = 14; +/** enum class triton::developer_tools::server::ModelReadyState */ +public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; +// Targeting ../tritonserverwrapper/TritonException.java + + +// Targeting ../tritonserverwrapper/ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritonserverwrapper/OutputBufferReleaseFn_t.java + + +// Targeting ../tritonserverwrapper/ResponseAllocatorStartFn_t.java + + + + // namespace triton::developer_tools::server + + +// Parsed from generic_server_wrapper.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include +// #include +// #include "common.h" +// #include "../src/infer_requested_output.h" +// #include "../src/tracer.h" + +/// +// Targeting ../tritonserverwrapper/GenericTritonServer.java + + +// Targeting ../tritonserverwrapper/GenericInferResult.java + + +// Targeting ../tritonserverwrapper/GenericInferRequest.java + + +// Targeting ../tritonserverwrapper/LoggingOptions.java + + +// Targeting ../tritonserverwrapper/MetricsOptions.java + + +// Targeting ../tritonserverwrapper/RateLimitResource.java + + +// Targeting ../tritonserverwrapper/ModelLoadGPULimit.java + + +// Targeting ../tritonserverwrapper/Allocator.java + + +// Targeting ../tritonserverwrapper/BackendConfig.java + + +// Targeting ../tritonserverwrapper/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritonserverwrapper/HostPolicy.java + + +// Targeting ../tritonserverwrapper/Trace.java + + +// Targeting ../tritonserverwrapper/ServerOptions.java + + +// Targeting ../tritonserverwrapper/RepositoryIndex.java + + +// Targeting ../tritonserverwrapper/Tensor.java + + +// Targeting ../tritonserverwrapper/NewModelRepo.java + + +// Targeting ../tritonserverwrapper/InferOptions.java + + + + // namespace triton::developer_tools::server + +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java new file mode 100644 index 00000000000..2440e5ec4c4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java @@ -0,0 +1,37 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + + +//============================================================================== +/** Custom Allocator object for providing custom functions for allocator. + * If there is no custom allocator provided, will use the default allocator. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Allocator extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Allocator(Pointer p) { super(p); } + + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/) { super((Pointer)null); allocate(alloc_fn, release_fn, start_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/); + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn) { super((Pointer)null); allocate(alloc_fn, release_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn); + + public native ResponseAllocatorAllocFn_t AllocFn(); + public native OutputBufferReleaseFn_t ReleaseFn(); + public native ResponseAllocatorStartFn_t StartFn(); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java new file mode 100644 index 00000000000..f27cc1d4081 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold backend configuration for setting 'ServerOptions'. + * Different Triton-supported backends have different backend configuration + * options. Please refer to the 'Command line options' section in the + * documentation of each backend to see the options (e.g. Tensorflow Backend: + * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class BackendConfig extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public BackendConfig(Pointer p) { super(p); } + + public BackendConfig( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value); + public BackendConfig( + @StdString String name, @StdString String setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @StdString String setting, + @StdString String value); + + // The name of the backend. + public native @StdString BytePointer name_(); public native BackendConfig name_(BytePointer setter); + // The name of the setting. + public native @StdString BytePointer setting_(); public native BackendConfig setting_(BytePointer setter); + // The setting value. + public native @StdString BytePointer value_(); public native BackendConfig value_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java new file mode 100644 index 00000000000..97e0e60e4a2 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java @@ -0,0 +1,32 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold CUDA memory pool byte size for setting 'ServerOptions'. + * If GPU support is enabled, the server will allocate CUDA memory to minimize + * data transfer between host and devices until it exceeds the specified byte + * size. This will not affect the allocation conducted by the backend + * frameworks. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class CUDAMemoryPoolByteSize extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public CUDAMemoryPoolByteSize(Pointer p) { super(p); } + + public CUDAMemoryPoolByteSize(int gpu_device, @Cast("const uint64_t") long size) { super((Pointer)null); allocate(gpu_device, size); } + private native void allocate(int gpu_device, @Cast("const uint64_t") long size); + + // The GPU device ID to allocate the memory pool. + public native int gpu_device_(); public native CUDAMemoryPoolByteSize gpu_device_(int setter); + // The CUDA memory pool byte size that the server can allocate on given GPU + // device. Default is 64 MB. + public native @Cast("uint64_t") long size_(); public native CUDAMemoryPoolByteSize size_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java new file mode 100644 index 00000000000..54f9aa13a31 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java @@ -0,0 +1,56 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Object that describes an inflight inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericInferRequest extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferRequest(Pointer p) { super(p); } + + /** Create an InferRequest instance. */ + public static native @UniquePtr GenericInferRequest Create( + @Const @ByRef InferOptions infer_options); + + /** Add an input tensor to be sent within an InferRequest object. The input + * data buffer within the 'Tensor' object must not be modified until + * inference is completed and result is returned. + * @param name The name of the input tensor. + * @param input A Tensor object that describes an input tensor. */ + public native @NoException(true) void AddInput(@StdString BytePointer name, @Const @ByRef Tensor input); + public native @NoException(true) void AddInput(@StdString String name, @Const @ByRef Tensor input); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. Pre-allocated buffer for each output should be specified within + * the 'Tensor' object. + * @param name The name of the output tensor. + * @param output A Tensor object that describes an output tensor containing + * its pre-allocated buffer. */ + public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); + public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. + * @param name The name of the output tensor. */ + public native void AddRequestedOutput(@StdString BytePointer name); + public native void AddRequestedOutput(@StdString String name); + + /** Clear inputs and outputs of the request. This allows users to reuse the + * InferRequest object if needed. */ + public native void Reset(); + +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java new file mode 100644 index 00000000000..0929769fce4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java @@ -0,0 +1,68 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** An interface for InferResult object to interpret the response to an + * inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericInferResult extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferResult(Pointer p) { super(p); } + + + /** Get the name of the model which generated this response. + * @return Returns the name of the model. */ + public native @StdString @NoException(true) BytePointer ModelName(); + + /** Get the version of the model which generated this response. + * @return Returns the version of the model. */ + public native @StdString @NoException(true) BytePointer ModelVersion(); + + /** Get the id of the request which generated this response. + * @return Returns the id of the request. */ + public native @StdString @NoException(true) BytePointer Id(); + + /** Get the output names from the infer result + * @return Vector of output names */ + public native @ByVal StringVector OutputNames(); + /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' + * field of the output is owned by the returned 'Tensor' object itself. Note + * that for string data, need to use 'StringData' function for string data + * result. + * @param name The name of the output tensor to be retrieved. + * @return Returns the output result as a shared pointer of 'Tensor' object. */ + public native @SharedPtr Tensor Output(@StdString BytePointer name); + public native @SharedPtr Tensor Output(@StdString String name); + + /** Get the result data as a vector of strings. The vector will + * receive a copy of result data. An exception will be thrown if + * the data type of output is not 'BYTES'. + * @param output_name The name of the output to get result data. + * @return Returns the result data represented as a vector of strings. The + * strings are stored in the row-major order. */ + public native @ByVal StringVector StringData(@StdString BytePointer output_name); + public native @ByVal StringVector StringData(@StdString String output_name); + + /** Return the complete response as a user friendly string. + * @return The string describing the complete response. */ + public native @StdString BytePointer DebugString(); + + /** Return if there is an error within this result. + * @return True if this 'GenericInferResult' object has an error, false if no + * error. */ + public native @Cast("bool") boolean HasError(); + + /** Return the error message of the error. + * @return The messsage for the error. Empty if no error. */ + public native @StdString BytePointer ErrorMsg(); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java new file mode 100644 index 00000000000..7d0b8495a1d --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java @@ -0,0 +1,132 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Object that encapsulates in-process C API functionalities. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class GenericTritonServer extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericTritonServer(Pointer p) { super(p); } + + /** Create a GenericTritonServer instance. */ + public static native @UniquePtr GenericTritonServer Create( + @Const @ByRef ServerOptions server_options); + + /** Load the requested model or reload the model if it is already loaded. + * @param model_name The name of the model. */ + public native void LoadModel(@StdString BytePointer model_name); + public native void LoadModel(@StdString String model_name); + + /** Unload the requested model. Unloading a model that is not loaded + * on server has no affect. + * @param model_name The name of the model. */ + public native void UnloadModel(@StdString BytePointer model_name); + public native void UnloadModel(@StdString String model_name); + + /** Get the set of names of models that are loaded and ready for inference. + * @return Returns the set of names of models that are + * loaded and ready for inference. */ + public native @ByVal StringSet LoadedModels(); + + /** Get the index of model repository contents. + * @return Returns a vector of 'RepositoryIndex' object + * representing the repository index. */ + public native @StdVector RepositoryIndex ModelIndex(); + + /** Get the metrics of the server. + * @return Returns a string representing the metrics. */ + public native @StdString BytePointer ServerMetrics(); + + /** Get the inference statistics of the specified model. + * @param model_name The name of the model. + * @param model_version the version of the model requested. + * @return Returns a json string representing the model metrics. */ + public native @StdString BytePointer ModelStatistics( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version); + public native @StdString String ModelStatistics( + @StdString String model_name, @Cast("const int64_t") long model_version); + + /** Is the server live? + * @return Returns true if server is live, false otherwise. */ + public native @Cast("bool") boolean IsServerLive(); + + /** Is the server ready? + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsServerReady(); + + /** Stop a server object. A server can't be restarted once it is + * stopped. */ + public native void ServerStop(); + + /** Is the model ready? + * @param model_name The name of the model to get readiness for. + * @param model_version The version of the model to get readiness + * for. If -1 then the server will choose a version based on the + * model's policy. This field is optional, default is -1. + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name); + + /** Get the configuration of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will + * choose a version based on the model and internal policy. This field is + * optional. @return Returns JSON representation of model configuration as a + * string. */ + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name); + public native @StdString String ModelConfig( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelConfig( + @StdString String model_name); + + /** Get the metadata of the server. + * @return Returns JSON representation of server metadata as a string. */ + public native @StdString BytePointer ServerMetadata(); + + /** Get the metadata of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will choose a version + * based on the model and internal policy. This field is optional. + * @return Returns JSON representation of model metadata as a string. */ + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name); + public native @StdString String ModelMetadata( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelMetadata( + @StdString String model_name); + + /** Register a new model repository. This function is not available in polling + * mode. + * @param new_model_repo The 'NewModelRepo' object contains the info of the + * new model repo to be registered. */ + public native void RegisterModelRepo(@Const @ByRef NewModelRepo new_model_repo); + + /** Unregister a model repository. This function is not available in polling + * mode. + * @param repo_path The full path to the model repository. */ + public native void UnregisterModelRepo(@StdString BytePointer repo_path); + public native void UnregisterModelRepo(@StdString String repo_path); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java new file mode 100644 index 00000000000..08f8326598d --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java @@ -0,0 +1,47 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold host policy for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class HostPolicy extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public HostPolicy(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::HostPolicy::Setting */ + public static final int NUMA_NODE = 0, CPU_CORES = 1; + + public HostPolicy( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value); + public HostPolicy( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value); + + // The name of the policy. + public native @StdString BytePointer name_(); public native HostPolicy name_(BytePointer setter); + // The kind of the host policy setting. Currently supported settings are + // 'NUMA_NODE', 'CPU_CORES'. Note that 'NUMA_NODE' setting will affect pinned + // memory pool behavior, see the comments of 'pinned_memory_pool_byte_size_' + // in 'ServerOptions' for more detail. + public native @Cast("triton::developer_tools::server::HostPolicy::Setting") int setting_(); public native HostPolicy setting_(int setter); + // The setting value. + public native @StdString BytePointer value_(); public native HostPolicy value_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java new file mode 100644 index 00000000000..fed63d6fcd6 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java @@ -0,0 +1,109 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold options for Inference Request. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class InferOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public InferOptions(Pointer p) { super(p); } + + public InferOptions(@StdString BytePointer model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString BytePointer model_name); + public InferOptions(@StdString String model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString String model_name); + + public InferOptions( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + public InferOptions( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + + // The name of the model to run inference. + public native @StdString BytePointer model_name_(); public native InferOptions model_name_(BytePointer setter); + // The version of the model to use while running inference. The default + // value is "-1" which means the server will select the + // version of the model based on its internal policy. + public native @Cast("int64_t") long model_version_(); public native InferOptions model_version_(long setter); + // An identifier for the request. If specified will be returned + // in the response. Default value is an empty string which means no + // request_id will be used. + public native @StdString BytePointer request_id_(); public native InferOptions request_id_(BytePointer setter); + // The correlation ID of the inference request to be an unsigned integer. + // Should be used exclusively with 'correlation_id_str_'. + // Default is 0, which indicates that the request has no correlation ID. + public native @Cast("uint64_t") long correlation_id_(); public native InferOptions correlation_id_(long setter); + // The correlation ID of the inference request to be a string. + // Should be used exclusively with 'correlation_id_'. + // Default value is "". + public native @StdString BytePointer correlation_id_str_(); public native InferOptions correlation_id_str_(BytePointer setter); + // Indicates whether the request being added marks the start of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_start_(); public native InferOptions sequence_start_(boolean setter); + // Indicates whether the request being added marks the end of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_end_(); public native InferOptions sequence_end_(boolean setter); + // Indicates the priority of the request. Priority value zero + // indicates that the default priority level should be used + // (i.e. same behavior as not specifying the priority parameter). + // Lower value priorities indicate higher priority levels. Thus + // the highest priority level is indicated by setting the parameter + // to 1, the next highest is 2, etc. If not provided, the server + // will handle the request using default setting for the model. + public native @Cast("uint64_t") long priority_(); public native InferOptions priority_(long setter); + // The timeout value for the request, in microseconds. If the request + // cannot be completed within the time by the server can take a + // model-specific action such as terminating the request. If not + // provided, the server will handle the request using default setting + // for the model. + public native @Cast("uint64_t") long request_timeout_(); public native InferOptions request_timeout_(long setter); + // User-provided custom reponse allocator object. Default is nullptr. + // If using custom allocator, the lifetime of this 'Allocator' object should + // be long enough until `InferResult` object goes out of scope as we need + // this `Allocator` object to call 'ResponseAllocatorReleaseFn_t' for + // releasing the response. + public native @SharedPtr Allocator custom_allocator_(); public native InferOptions custom_allocator_(Allocator setter); + // Update trace setting for the specified model. If not set, will use global + // trace setting in 'ServerOptions' for tracing if tracing is enabled in + // 'ServerOptions'. Default is nullptr. + public native @SharedPtr Trace trace_(); public native InferOptions trace_(Trace setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java new file mode 100644 index 00000000000..3a8084108fa --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java @@ -0,0 +1,67 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +//============================================================================== +/** Structure to hold logging options for setting 'ServerOptions'. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class LoggingOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public LoggingOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public LoggingOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public LoggingOptions position(long position) { + return (LoggingOptions)super.position(position); + } + @Override public LoggingOptions getPointer(long i) { + return new LoggingOptions((Pointer)this).offsetAddress(i); + } + + // The range of VerboseLevel is [0, INT_MAX]. + /** enum class triton::developer_tools::server::LoggingOptions::VerboseLevel */ + public static final int OFF = 0, MIN = 1, MAX = Integer.MAX_VALUE; + /** enum class triton::developer_tools::server::LoggingOptions::LogFormat */ + public static final int DEFAULT = 0, ISO8601 = 1; + + public LoggingOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file); + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file); + + // Verbose logging level. Default is OFF. + public native @Cast("triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose_(); public native LoggingOptions verbose_(int setter); + // Enable or disable info logging level. Default is true. + public native @Cast("bool") boolean info_(); public native LoggingOptions info_(boolean setter); + // Enable or disable warn logging level. Default is true. + public native @Cast("bool") boolean warn_(); public native LoggingOptions warn_(boolean setter); + // Enable or disable error logging level. Default is true. + public native @Cast("bool") boolean error_(); public native LoggingOptions error_(boolean setter); + // The format of logging. For "DEFAULT", the log severity (L) and + // timestamp will be logged as "LMMDD hh:mm:ss.ssssss". For "ISO8601", the + // log format will be "YYYY-MM-DDThh:mm:ssZ L". Default is 'DEFAULT'. + public native @Cast("triton::developer_tools::server::LoggingOptions::LogFormat") int format_(); public native LoggingOptions format_(int setter); + // Logging output file. If specified, log outputs will be saved to this file. + // If not specified, log outputs will stream to the console. Default is an + // empty string. + public native @StdString BytePointer log_file_(); public native LoggingOptions log_file_(BytePointer setter); // logging output file +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java new file mode 100644 index 00000000000..56378a66cb1 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java @@ -0,0 +1,49 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold metrics options for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class MetricsOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public MetricsOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public MetricsOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public MetricsOptions position(long position) { + return (MetricsOptions)super.position(position); + } + @Override public MetricsOptions getPointer(long i) { + return new MetricsOptions((Pointer)this).offsetAddress(i); + } + + public MetricsOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public MetricsOptions( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms) { super((Pointer)null); allocate(allow_metrics, allow_gpu_metrics, allow_cpu_metrics, metrics_interval_ms); } + private native void allocate( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms); + + // Enable or disable metrics. Default is true. + public native @Cast("bool") boolean allow_metrics_(); public native MetricsOptions allow_metrics_(boolean setter); + // Enable or disable GPU metrics. Default is true. + public native @Cast("bool") boolean allow_gpu_metrics_(); public native MetricsOptions allow_gpu_metrics_(boolean setter); + // Enable or disable CPU metrics. Default is true. + public native @Cast("bool") boolean allow_cpu_metrics_(); public native MetricsOptions allow_cpu_metrics_(boolean setter); + // The interval for metrics collection. Default is 2000. + public native @Cast("uint64_t") long metrics_interval_ms_(); public native MetricsOptions metrics_interval_ms_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java new file mode 100644 index 00000000000..e7f01eefbb3 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java @@ -0,0 +1,30 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold GPU limit of model loading for setting 'ServerOptions'. + * The limit on GPU memory usage is specified as a fraction. If model loading + * on the device is requested and the current memory usage exceeds the limit, + * the load will be rejected. If not specified, the limit will not be set. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ModelLoadGPULimit extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ModelLoadGPULimit(Pointer p) { super(p); } + + public ModelLoadGPULimit(int device_id, double fraction) { super((Pointer)null); allocate(device_id, fraction); } + private native void allocate(int device_id, double fraction); + + // The GPU device ID. + public native int device_id_(); public native ModelLoadGPULimit device_id_(int setter); + // The limit on memory usage as a fraction. + public native double fraction_(); public native ModelLoadGPULimit fraction_(double setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java new file mode 100644 index 00000000000..e28c3ba27c7 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold the full path to the model repository to be registered and + * the mapping from the original model name to the overriden one. This object + * is used for calling 'TritonServer::RegisterModelRepo' for registering + * model repository. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class NewModelRepo extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public NewModelRepo(Pointer p) { super(p); } + + public NewModelRepo(@StdString BytePointer path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString BytePointer path); + public NewModelRepo(@StdString String path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString String path); + + public NewModelRepo( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name); + public NewModelRepo( + @StdString String path, @StdString String original_name, + @StdString String override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString String path, @StdString String original_name, + @StdString String override_name); + + // The full path to the model repository. + public native @StdString BytePointer path_(); public native NewModelRepo path_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer original_name_(); public native NewModelRepo original_name_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer override_name_(); public native NewModelRepo override_name_(BytePointer setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java new file mode 100644 index 00000000000..1871e81f3fa --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java @@ -0,0 +1,21 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class OutputBufferReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public OutputBufferReleaseFn_t(Pointer p) { super(p); } + protected OutputBufferReleaseFn_t() { allocate(); } + private native void allocate(); + public native void call( + Pointer buffer, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java new file mode 100644 index 00000000000..942337971ad --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold rate limit resource for setting 'ServerOptions'. See here + * for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class RateLimitResource extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RateLimitResource(Pointer p) { super(p); } + + public RateLimitResource(@StdString BytePointer name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString BytePointer name, int count); + public RateLimitResource(@StdString String name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString String name, int count); + + public RateLimitResource(@StdString BytePointer name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString BytePointer name, int count, int device); + public RateLimitResource(@StdString String name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString String name, int count, int device); + + // The name of the resource. + public native @StdString BytePointer name_(); public native RateLimitResource name_(BytePointer setter); + // The count of the resource. + public native int count_(); public native RateLimitResource count_(int setter); + // The device identifier for the resource. This field is optional and if not + // specified will be applied to every device. The device value is ignored for + // a global resource. The server will use the rate limiter configuration + // specified for instance groups in model config to determine whether resource + // is global. In case of conflicting resource type in different model + // configurations, server will raise an appropriate error while loading model. + public native int device_(); public native RateLimitResource device_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java new file mode 100644 index 00000000000..0beba9c95b4 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold repository index for 'ModelIndex' function. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class RepositoryIndex extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RepositoryIndex(Pointer p) { super(p); } + + public RepositoryIndex( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + public RepositoryIndex( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + + // The name of the model. + public native @StdString BytePointer name_(); public native RepositoryIndex name_(BytePointer setter); + // The version of the model. + public native @StdString BytePointer version_(); public native RepositoryIndex version_(BytePointer setter); + // The state of the model. The states are + // * UNKNOWN: The model is in an unknown state. The model is not available for + // inferencing. + // * READY: The model is ready and available for inferencing. + // * UNAVAILABLE: The model is unavailable, indicating that the model failed + // to load or has been implicitly or explicitly unloaded. The model is not + // available for inferencing. + // * LOADING: The model is being loaded by the inference server. The model is + // not available for inferencing. + // * UNLOADING: The model is being unloaded by the inference server. The model + // is not available for inferencing. + public native @Cast("triton::developer_tools::server::ModelReadyState") int state_(); public native RepositoryIndex state_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java new file mode 100644 index 00000000000..079b2ce7446 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java @@ -0,0 +1,26 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Custom Response Allocator Callback function signatures. + * */ +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ResponseAllocatorAllocFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorAllocFn_t(Pointer p) { super(p); } + protected ResponseAllocatorAllocFn_t() { allocate(); } + private native void allocate(); + public native void call( + String tensor_name, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int preferred_memory_type, + @Cast("int64_t") long preferred_memory_type_id, @Cast("void**") PointerPointer buffer, + @Cast("triton::developer_tools::server::MemoryType*") IntPointer actual_memory_type, @Cast("int64_t*") LongPointer actual_memory_type_id); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java new file mode 100644 index 00000000000..c3e3e808569 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java @@ -0,0 +1,19 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ResponseAllocatorStartFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorStartFn_t(Pointer p) { super(p); } + protected ResponseAllocatorStartFn_t() { allocate(); } + private native void allocate(); + public native void call(Pointer userp); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java new file mode 100644 index 00000000000..b8bb019dd68 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java @@ -0,0 +1,189 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Server options that are used to initialize Triton Server. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class ServerOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ServerOptions(Pointer p) { super(p); } + + public ServerOptions(@Const @ByRef StringVector model_repository_paths) { super((Pointer)null); allocate(model_repository_paths); } + private native void allocate(@Const @ByRef StringVector model_repository_paths); + + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + + // Paths to model repository directory. Note that if a model is not unique + // across all model repositories at any time, the model will not be available. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md. + public native @ByRef StringVector model_repository_paths_(); public native ServerOptions model_repository_paths_(StringVector setter); + // Logging options. See the 'LoggingOptions' structure for more information. + public native @ByRef LoggingOptions logging_(); public native ServerOptions logging_(LoggingOptions setter); + // Metrics options. See the 'MetricsOptions' structure for more information. + public native @ByRef MetricsOptions metrics_(); public native ServerOptions metrics_(MetricsOptions setter); + // Backend configuration. See the 'BackendConfig' structure for more + // information. + public native @StdVector BackendConfig be_config_(); public native ServerOptions be_config_(BackendConfig setter); + // The ID of the server. + public native @StdString BytePointer server_id_(); public native ServerOptions server_id_(BytePointer setter); + // The global directory searched for backend shared libraries. Default is + // "/opt/tritonserver/backends". See here for more information: + // https://github.com/triton-inference-server/backend#backends. + public native @StdString BytePointer backend_dir_(); public native ServerOptions backend_dir_(BytePointer setter); + // The global directory searched for repository agent shared libraries. + // Default is "/opt/tritonserver/repoagents". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/repository_agents.md. + public native @StdString BytePointer repo_agent_dir_(); public native ServerOptions repo_agent_dir_(BytePointer setter); + // If set, disables the triton and backends from auto completing model + // configuration files. Model configuration files must be provided and + // all required configuration settings must be specified. Default is false. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration. + public native @Cast("bool") boolean disable_auto_complete_config_(); public native ServerOptions disable_auto_complete_config_(boolean setter); + // Specify the mode for model management. Options are "NONE", "POLL" and + // "EXPLICIT". Default is "NONE". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md. + public native @Cast("triton::developer_tools::server::ModelControlMode") int model_control_mode_(); public native ServerOptions model_control_mode_(int setter); + // Interval in seconds between each poll of the model repository to check for + // changes. Valid only when 'model_control_mode_' is set to "POLL". Default + // is 15. + public native int repository_poll_secs_(); public native ServerOptions repository_poll_secs_(int setter); + // Specify the the models to be loaded on server startup. This will only take + // effect if 'model_control_mode_' is set to 'EXPLICIT'. + public native @ByRef StringSet startup_models_(); public native ServerOptions startup_models_(StringSet setter); + // The number of resources available to the server. Rate limiting is disabled + // by default, and can be enabled once 'rate_limit_resource_' is set. See the + // 'RateLimitResource' structure for more information. + public native @StdVector RateLimitResource rate_limit_resource_(); public native ServerOptions rate_limit_resource_(RateLimitResource setter); + // The total byte size that can be allocated as pinned system memory. If GPU + // support is enabled, the server will allocate pinned system memory to + // accelerate data transfer between host and devices until it exceeds the + // specified byte size. If 'NUMA_NODE' is configured via 'host_policy_', the + // pinned system memory of the pool size will be allocated on each numa node. + // This option will not affect the allocation conducted by the backend + // frameworks. Default is 256 MB. + public native @Cast("int64_t") long pinned_memory_pool_byte_size_(); public native ServerOptions pinned_memory_pool_byte_size_(long setter); + // The total byte size that can be allocated as CUDA memory for the GPU + // device. See the 'CUDAMemoryPoolByteSize' structure for more information. + public native @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size_(); public native ServerOptions cuda_memory_pool_byte_size_(CUDAMemoryPoolByteSize setter); + // The size in bytes to allocate for a request/response cache. When non-zero, + // Triton allocates the requested size in CPU memory and shares the cache + // across all inference requests and across all models. For a given model to + // use request caching, the model must enable request caching in the model + // configuration. See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#response-cache. + // By default, no model uses request caching even if the + // 'response_cache_byte_size_' is set. Default is 0. + public native @Cast("uint64_t") long response_cache_byte_size_(); public native ServerOptions response_cache_byte_size_(long setter); + // The minimum supported CUDA compute capability. GPUs that don't support this + // compute capability will not be used by the server. Default is 0. + public native double min_cuda_compute_capability_(); public native ServerOptions min_cuda_compute_capability_(double setter); + // If set, exit the inference server when an error occurs during + // initialization. Default is true. + public native @Cast("bool") boolean exit_on_error_(); public native ServerOptions exit_on_error_(boolean setter); + // Timeout (in seconds) when exiting to wait for in-flight inferences to + // finish. After the timeout expires the server exits even if inferences are + // still in flight. Default is 30 secs. + public native int exit_timeout_secs_(); public native ServerOptions exit_timeout_secs_(int setter); + // The number of threads used to accelerate copies and other operations + // required to manage input and output tensor contents. Default is 0. + public native int buffer_manager_thread_count_(); public native ServerOptions buffer_manager_thread_count_(int setter); + // The number of threads used to concurrently load models in model + // repositories. Default is 2*. + public native @Cast("uint32_t") int model_load_thread_count_(); public native ServerOptions model_load_thread_count_(int setter); + // The GPU limit of model loading. See the 'ModelLoadGPULimit' structure for + // more information. + public native @StdVector ModelLoadGPULimit model_load_gpu_limit_(); public native ServerOptions model_load_gpu_limit_(ModelLoadGPULimit setter); + // The host policy setting. See the 'HostPolicy' structure for more + // information. + public native @StdVector HostPolicy host_policy_(); public native ServerOptions host_policy_(HostPolicy setter); + // The global trace setting. Default is nullptr, meaning that tracing is not + // enabled. See the 'Trace' structure for more information. + public native @SharedPtr Trace trace_(); public native ServerOptions trace_(Trace setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java new file mode 100644 index 00000000000..9c1bb2e351f --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java @@ -0,0 +1,36 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Name("std::set") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class StringSet extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringSet(Pointer p) { super(p); } + public StringSet() { allocate(); } + private native void allocate(); + public native @Name("operator =") @ByRef StringSet put(@ByRef StringSet x); + + public boolean empty() { return size() == 0; } + public native long size(); + + public native void insert(@StdString BytePointer value); + public native void erase(@StdString BytePointer value); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } +} + diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java new file mode 100644 index 00000000000..bcba32a1f82 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java @@ -0,0 +1,99 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class StringVector extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringVector(Pointer p) { super(p); } + public StringVector(BytePointer value) { this(1); put(0, value); } + public StringVector(BytePointer ... array) { this(array.length); put(array); } + public StringVector(String value) { this(1); put(0, value); } + public StringVector(String ... array) { this(array.length); put(array); } + public StringVector() { allocate(); } + public StringVector(long n) { allocate(n); } + private native void allocate(); + private native void allocate(@Cast("size_t") long n); + public native @Name("operator =") @ByRef StringVector put(@ByRef StringVector x); + + public boolean empty() { return size() == 0; } + public native long size(); + public void clear() { resize(0); } + public native void resize(@Cast("size_t") long n); + + @Index(function = "at") public native @StdString BytePointer get(@Cast("size_t") long i); + public native StringVector put(@Cast("size_t") long i, BytePointer value); + @ValueSetter @Index(function = "at") public native StringVector put(@Cast("size_t") long i, @StdString String value); + + public native @ByVal Iterator insert(@ByVal Iterator pos, @StdString BytePointer value); + public native @ByVal Iterator erase(@ByVal Iterator pos); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } + + public BytePointer[] get() { + BytePointer[] array = new BytePointer[size() < Integer.MAX_VALUE ? (int)size() : Integer.MAX_VALUE]; + for (int i = 0; i < array.length; i++) { + array[i] = get(i); + } + return array; + } + @Override public String toString() { + return java.util.Arrays.toString(get()); + } + + public BytePointer pop_back() { + long size = size(); + BytePointer value = get(size - 1); + resize(size - 1); + return value; + } + public StringVector push_back(BytePointer value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(BytePointer value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(BytePointer ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } + + public StringVector push_back(String value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(String value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(String ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } +} + diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java new file mode 100644 index 00000000000..dd7d0c59cef --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java @@ -0,0 +1,81 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold information of a tensor. This object is used for adding + * input/requested output to an inference request, and retrieving the output + * result from inference result. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Tensor extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Tensor(Pointer p) { super(p); } + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + // The pointer to the start of the buffer. + public native @Cast("char*") BytePointer buffer_(); public native Tensor buffer_(BytePointer setter); + // The size of buffer in bytes. + public native @Cast("size_t") long byte_size_(); public native Tensor byte_size_(long setter); + // The data type of the tensor. + public native @Cast("triton::developer_tools::server::DataType") int data_type_(); public native Tensor data_type_(int setter); + // The shape of the tensor. + public native @Cast("int64_t*") @StdVector LongPointer shape_(); public native Tensor shape_(LongPointer setter); + // The memory type of the tensor. Valid memory types are "CPU", "CPU_PINNED" + // and "GPU". + public native @Cast("triton::developer_tools::server::MemoryType") int memory_type_(); public native Tensor memory_type_(int setter); + // The ID of the memory for the tensor. (e.g. '0' is the memory type id of + // 'GPU-0') + public native @Cast("int64_t") long memory_type_id_(); public native Tensor memory_type_id_(long setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java new file mode 100644 index 00000000000..debbb257802 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java @@ -0,0 +1,66 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +/** Structure to hold global trace setting for 'ServerOptions' and + * model-specific trace setting for 'InferOptions'. See here for more + * information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class Trace extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Trace(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::Trace::Level */ + public static final int OFF = 0, TIMESTAMPS = 1, TENSORS = 2; + + public Trace(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + public Trace(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + + public Trace( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + public Trace( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + + // The file where trace output will be saved. If 'log-frequency' is also + // specified, this argument value will be the prefix of the files to save the + // trace output. + public native @StdString BytePointer file_(); public native Trace file_(BytePointer setter); + // Specify a trace level. OFF to disable tracing, TIMESTAMPS to trace + // timestamps, TENSORS to trace tensors. + public native @Cast("triton::developer_tools::server::Trace::Level") int level_(); public native Trace level_(int setter); + // The trace sampling rate. The value represents how many requests will one + // trace be sampled from. For example, if the trace rate is "1000", 1 trace + // will be sampled for every 1000 requests. Default is 1000. + public native @Cast("uint32_t") int rate_(); public native Trace rate_(int setter); + // The number of traces to be sampled. If the value is -1, the number of + // traces to be sampled will not be limited. Default is -1. + public native int count_(); public native Trace count_(int setter); + // The trace log frequency. If the value is 0, Triton will only log the trace + // output to 'file_' when shutting down. Otherwise, Triton will log the trace + // output to 'file_.' when it collects the specified number of traces. + // For example, if the log frequency is 100, when Triton collects the 100-th + // trace, it logs the traces to file 'file_.0', and when it collects the + // 200-th trace, it logs the 101-th to the 200-th traces to file file_.1'. + // Default is 0. + public native @Cast("uint32_t") int log_frequency_(); public native Trace log_frequency_(int setter); +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java new file mode 100644 index 00000000000..5904afd5c77 --- /dev/null +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java @@ -0,0 +1,29 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserverwrapper.tritonserverwrapper; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + + +//============================================================================== +// TritonException +// +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +public class TritonException extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TritonException(Pointer p) { super(p); } + + public TritonException(@StdString BytePointer message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString BytePointer message); + public TritonException(@StdString String message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString String message); + + public native String what(); + + public native @StdString BytePointer message_(); public native TritonException message_(BytePointer setter); +} From 92dd58ae7890d563f011c40c82a7e4a92367f263 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Mon, 3 Apr 2023 05:56:03 -0700 Subject: [PATCH 16/50] adding simpletest and compiling wrapper to newer version of wrapper --- .../samples/simple/Simple.java | 764 ------------------ .../samples/simpletest/SimpleTest.java | 140 ++++ .../samples/{simple => simpletest}/pom.xml | 8 +- .../global/tritonserverwrapper.java | 2 +- .../tritonserverwrapper/LoggingOptions.java | 2 + .../tritonserverwrapper/ServerOptions.java | 2 + 6 files changed, 149 insertions(+), 769 deletions(-) delete mode 100644 tritonserverwrapper/samples/simple/Simple.java create mode 100644 tritonserverwrapper/samples/simpletest/SimpleTest.java rename tritonserverwrapper/samples/{simple => simpletest}/pom.xml (76%) diff --git a/tritonserverwrapper/samples/simple/Simple.java b/tritonserverwrapper/samples/simple/Simple.java deleted file mode 100644 index 214e0a82096..00000000000 --- a/tritonserverwrapper/samples/simple/Simple.java +++ /dev/null @@ -1,764 +0,0 @@ -// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import java.io.*; -import java.util.*; -import java.util.concurrent.*; -import com.google.gson.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.tritonserver.tritonserver.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; -// import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; -// import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; - -public class Simple { - - static void FAIL(String MSG) { - System.err.println("Failure: " + MSG); - System.exit(1); - } - - static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { - if (err__ != null) { - System.err.println("error: " + MSG + ":" - + TRITONSERVER_ErrorCodeString(err__) + " - " - + TRITONSERVER_ErrorMessage(err__)); - TRITONSERVER_ErrorDelete(err__); - System.exit(1); - } - } - - static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; - - static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { - public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } - protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { - DeleteDeallocator(Pointer p) { super(p); } - @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } - } - } - - static void - Usage(String msg) - { - if (msg != null) { - System.err.println(msg); - } - - System.err.println("Usage: java " + Simple.class.getSimpleName() + " [options]"); - System.err.println("\t-v Enable verbose logging"); - System.err.println("\t-r [model repository absolute path]"); - - System.exit(1); - } - - static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { - @Override public TRITONSERVER_Error call ( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - long byte_size, int preferred_memory_type, - long preferred_memory_type_id, Pointer userp, PointerPointer buffer, - PointerPointer buffer_userp, IntPointer actual_memory_type, - LongPointer actual_memory_type_id) - { - // Initially attempt to make the actual memory type and id that we - // allocate be the same as preferred memory type - actual_memory_type.put(0, preferred_memory_type); - actual_memory_type_id.put(0, preferred_memory_type_id); - - // If 'byte_size' is zero just return 'buffer' == nullptr, we don't - // need to do any other book-keeping. - if (byte_size == 0) { - buffer.put(0, null); - buffer_userp.put(0, null); - System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); - } else { - Pointer allocated_ptr = new Pointer(); - actual_memory_type.put(0, requested_memory_type); - - actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); - allocated_ptr = Pointer.malloc(byte_size); - - // Pass the tensor name with buffer_userp so we can show it when - // releasing the buffer. - if (!allocated_ptr.isNull()) { - buffer.put(0, allocated_ptr); - buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); - System.out.println("allocated " + byte_size + " bytes in " - + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) - + " for result tensor " + tensor_name); - } - } - - return null; // Success - } - } - - static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { - @Override public TRITONSERVER_Error call ( - TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, - long byte_size, int memory_type, long memory_type_id) - { - String name = null; - if (buffer_userp != null) { - name = (String)Loader.accessGlobalRef(buffer_userp); - } else { - name = ""; - } - - System.out.println("Releasing buffer " + buffer + " of size " + byte_size - + " in " + TRITONSERVER_MemoryTypeString(memory_type) - + " for result '" + name + "'"); - Pointer.free(buffer); - Loader.deleteGlobalRef(buffer_userp); - - return null; // Success - } - } - - static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { - @Override public void call ( - TRITONSERVER_InferenceRequest request, int flags, Pointer userp) - { - // We reuse the request so we don't delete it here. - } - } - - static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { - @Override public void call ( - TRITONSERVER_InferenceResponse response, int flags, Pointer userp) - { - if (response != null) { - // Send 'response' to the future. - futures.get(userp).complete(response); - } - } - } - - static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); - static ResponseAlloc responseAlloc = new ResponseAlloc(); - static ResponseRelease responseRelease = new ResponseRelease(); - static InferRequestComplete inferRequestComplete = new InferRequestComplete(); - static InferResponseComplete inferResponseComplete = new InferResponseComplete(); - - static TRITONSERVER_Error - ParseModelMetadata( - JsonObject model_metadata, boolean[] is_int, - boolean[] is_torch_model) - { - String seen_data_type = null; - for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { - JsonObject input = input_element.getAsJsonObject(); - if (!input.get("datatype").getAsString().equals("INT32") && - !input.get("datatype").getAsString().equals("FP32")) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "simple lib example only supports model with data type INT32 or " + - "FP32"); - } - if (seen_data_type == null) { - seen_data_type = input.get("datatype").getAsString(); - } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - "the inputs and outputs of 'simple' model must have the data type"); - } - } - for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { - JsonObject output = output_element.getAsJsonObject(); - if (!output.get("datatype").getAsString().equals("INT32") && - !output.get("datatype").getAsString().equals("FP32")) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "simple lib example only supports model with data type INT32 or " + - "FP32"); - } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - "the inputs and outputs of 'simple' model must have the data type"); - } - } - - is_int[0] = seen_data_type.equals("INT32"); - is_torch_model[0] = - model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); - return null; - } - - static void - GenerateInputData( - IntPointer[] input0_data, IntPointer[] input1_data) - { - input0_data[0] = new IntPointer(16); - input1_data[0] = new IntPointer(16); - for (int i = 0; i < 16; ++i) { - input0_data[0].put(i, i); - input1_data[0].put(i, 1); - } - } - - static void - GenerateInputData( - FloatPointer[] input0_data, FloatPointer[] input1_data) - { - input0_data[0] = new FloatPointer(16); - input1_data[0] = new FloatPointer(16); - for (int i = 0; i < 16; ++i) { - input0_data[0].put(i, i); - input1_data[0].put(i, 1); - } - } - - static void - CompareResult( - String output0_name, String output1_name, - IntPointer input0, IntPointer input1, IntPointer output0, - IntPointer output1) - { - for (int i = 0; i < 16; ++i) { - System.out.println(input0.get(i) + " + " + input1.get(i) + " = " - + output0.get(i)); - System.out.println(input0.get(i) + " - " + input1.get(i) + " = " - + output1.get(i)); - - if ((input0.get(i) + input1.get(i)) != output0.get(i)) { - FAIL("incorrect sum in " + output0_name); - } - if ((input0.get(i) - input1.get(i)) != output1.get(i)) { - FAIL("incorrect difference in " + output1_name); - } - } - } - - static void - CompareResult( - String output0_name, String output1_name, - FloatPointer input0, FloatPointer input1, FloatPointer output0, - FloatPointer output1) - { - for (int i = 0; i < 16; ++i) { - System.out.println(input0.get(i) + " + " + input1.get(i) + " = " - + output0.get(i)); - System.out.println(input0.get(i) + " - " + input1.get(i) + " = " - + output1.get(i)); - - if ((input0.get(i) + input1.get(i)) != output0.get(i)) { - FAIL("incorrect sum in " + output0_name); - } - if ((input0.get(i) - input1.get(i)) != output1.get(i)) { - FAIL("incorrect difference in " + output1_name); - } - } - } - - static void - Check( - TRITONSERVER_InferenceResponse response, - Pointer input0_data, Pointer input1_data, - String output0, String output1, - long expected_byte_size, - int expected_datatype, boolean is_int) - { - HashMap output_data = new HashMap<>(); - - int[] output_count = {0}; - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseOutputCount(response, output_count), - "getting number of response outputs"); - if (output_count[0] != 2) { - FAIL("expecting 2 response outputs, got " + output_count[0]); - } - - for (int idx = 0; idx < output_count[0]; ++idx) { - BytePointer cname = new BytePointer((Pointer)null); - IntPointer datatype = new IntPointer(1); - LongPointer shape = new LongPointer((Pointer)null); - LongPointer dim_count = new LongPointer(1); - Pointer base = new Pointer(); - SizeTPointer byte_size = new SizeTPointer(1); - IntPointer memory_type = new IntPointer(1); - LongPointer memory_type_id = new LongPointer(1); - Pointer userp = new Pointer(); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseOutput( - response, idx, cname, datatype, shape, dim_count, base, - byte_size, memory_type, memory_type_id, userp), - "getting output info"); - - if (cname.isNull()) { - FAIL("unable to get output name"); - } - - String name = cname.getString(); - if ((!name.equals(output0)) && (!name.equals(output1))) { - FAIL("unexpected output '" + name + "'"); - } - - if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { - FAIL("unexpected shape for '" + name + "'"); - } - - if (datatype.get() != expected_datatype) { - FAIL( - "unexpected datatype '" + - TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + - name + "'"); - } - - if (byte_size.get() != expected_byte_size) { - FAIL( - "unexpected byte-size, expected " + - expected_byte_size + ", got " + - byte_size.get() + " for " + name); - } - - if (memory_type.get() != requested_memory_type) { - FAIL( - "unexpected memory type, expected to be allocated in " + - TRITONSERVER_MemoryTypeString(requested_memory_type) + - ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + - ", id " + memory_type_id.get() + " for " + name); - } - - // We make a copy of the data here... which we could avoid for - // performance reasons but ok for this simple example. - BytePointer odata = new BytePointer(byte_size.get()); - output_data.put(name, odata); - System.out.println(name + " is stored in system memory"); - odata.put(base.limit(byte_size.get())); - } - - if (is_int) { - CompareResult( - output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), - new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); - } else { - CompareResult( - output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), - new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); - } - } - - public static void - RunInference(String model_repository_path, int verbose_level) throws Exception - { - // Check API version. - int[] api_version_major = {0}, api_version_minor = {0}; - FAIL_IF_ERR( - TRITONSERVER_ApiVersion(api_version_major, api_version_minor), - "getting Triton API version"); - if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || - (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { - FAIL("triton server API version mismatch"); - } - - // Create the server... - TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsNew(server_options), - "creating server options"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetModelRepositoryPath( - server_options, model_repository_path), - "setting model repository path"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), - "setting verbose logging level"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetBackendDirectory( - server_options, "/opt/tritonserver/backends"), - "setting backend directory"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetRepoAgentDirectory( - server_options, "/opt/tritonserver/repoagents"), - "setting repository agent directory"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), - "setting strict model configuration"); - - TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); - FAIL_IF_ERR( - TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsDelete(server_options), - "deleting server options"); - - TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); - - // Wait until the server is both live and ready. - int health_iters = 0; - while (true) { - boolean[] live = {false}, ready = {false}; - FAIL_IF_ERR( - TRITONSERVER_ServerIsLive(server, live), - "unable to get server liveness"); - FAIL_IF_ERR( - TRITONSERVER_ServerIsReady(server, ready), - "unable to get server readiness"); - System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); - if (live[0] && ready[0]) { - break; - } - - if (++health_iters >= 10) { - FAIL("failed to find healthy inference server"); - } - - Thread.sleep(500); - } - - // Print status of the server. - { - TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); - FAIL_IF_ERR( - TRITONSERVER_ServerMetadata(server, server_metadata_message), - "unable to get server metadata message"); - BytePointer buffer = new BytePointer((Pointer)null); - SizeTPointer byte_size = new SizeTPointer(1); - FAIL_IF_ERR( - TRITONSERVER_MessageSerializeToJson( - server_metadata_message, buffer, byte_size), - "unable to serialize server metadata message"); - - System.out.println("Server Status:"); - System.out.println(buffer.limit(byte_size.get()).getString()); - - FAIL_IF_ERR( - TRITONSERVER_MessageDelete(server_metadata_message), - "deleting status metadata"); - } - - String model_name = "simple"; - - // Wait for the model to become available. - boolean[] is_torch_model = {false}; - boolean[] is_int = {true}; - boolean[] is_ready = {false}; - health_iters = 0; - while (!is_ready[0]) { - FAIL_IF_ERR( - TRITONSERVER_ServerModelIsReady( - server, model_name, 1, is_ready), - "unable to get model readiness"); - if (!is_ready[0]) { - if (++health_iters >= 10) { - FAIL("model failed to be ready in 10 iterations"); - } - Thread.sleep(500); - continue; - } - - TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); - FAIL_IF_ERR( - TRITONSERVER_ServerModelMetadata( - server, model_name, 1, model_metadata_message), - "unable to get model metadata message"); - BytePointer buffer = new BytePointer((Pointer)null); - SizeTPointer byte_size = new SizeTPointer(1); - FAIL_IF_ERR( - TRITONSERVER_MessageSerializeToJson( - model_metadata_message, buffer, byte_size), - "unable to serialize model status protobuf"); - - JsonParser parser = new JsonParser(); - JsonObject model_metadata = null; - try { - model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); - } catch (Exception e) { - FAIL("error: failed to parse model metadata from JSON: " + e); - } - - FAIL_IF_ERR( - TRITONSERVER_MessageDelete(model_metadata_message), - "deleting status protobuf"); - - if (!model_metadata.get("name").getAsString().equals(model_name)) { - FAIL("unable to find metadata for model"); - } - - boolean found_version = false; - if (model_metadata.has("versions")) { - for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { - if (version.getAsString().equals("1")) { - found_version = true; - break; - } - } - } - if (!found_version) { - FAIL("unable to find version 1 status for model"); - } - - FAIL_IF_ERR( - ParseModelMetadata(model_metadata, is_int, is_torch_model), - "parsing model metadata"); - } - - // Create the allocator that will be used to allocate buffers for - // the result tensors. - TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); - FAIL_IF_ERR( - TRITONSERVER_ResponseAllocatorNew( - allocator, responseAlloc, responseRelease, null /* start_fn */), - "creating response allocator"); - - // Inference - TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestNew( - irequest, server, model_name, -1 /* model_version */), - "creating inference request"); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), - "setting ID for the request"); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetReleaseCallback( - irequest, inferRequestComplete, null /* request_release_userp */), - "setting request release callback"); - - // Inputs - String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; - String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; - - long[] input0_shape = {1, 16}; - long[] input1_shape = {1, 16}; - - int datatype = - (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddInput( - irequest, input0, datatype, input0_shape, input0_shape.length), - "setting input 0 meta-data for the request"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddInput( - irequest, input1, datatype, input1_shape, input1_shape.length), - "setting input 1 meta-data for the request"); - - String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; - String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), - "requesting output 0 for the request"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), - "requesting output 1 for the request"); - - // Create the data for the two input tensors. Initialize the first - // to unique values and the second to all ones. - BytePointer input0_data; - BytePointer input1_data; - if (is_int[0]) { - IntPointer[] p0 = {null}, p1 = {null}; - GenerateInputData(p0, p1); - input0_data = p0[0].getPointer(BytePointer.class); - input1_data = p1[0].getPointer(BytePointer.class); - } else { - FloatPointer[] p0 = {null}, p1 = {null}; - GenerateInputData(p0, p1); - input0_data = p0[0].getPointer(BytePointer.class); - input1_data = p1[0].getPointer(BytePointer.class); - } - - long input0_size = input0_data.limit(); - long input1_size = input1_data.limit(); - - Pointer input0_base = input0_data; - Pointer input1_base = input1_data; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input0, input0_base, input0_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT0 data"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input1, input1_base, input1_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT1 data"); - - // Perform inference... - { - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - Check( - completed_response, input0_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - // Modify some input data in place and then reuse the request - // object. - { - if (is_int[0]) { - new IntPointer(input0_data).put(0, 27); - } else { - new FloatPointer(input0_data).put(0, 27.0f); - } - - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - // Using a new promise so have to re-register the callback to set - // the promise as the userp. - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - Check( - completed_response, input0_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - // Remove input data and then add back different data. - { - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), - "removing INPUT0 data"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input0, input1_base, input1_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT1 data to INPUT0"); - - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - // Using a new promise so have to re-register the callback to set - // the promise as the userp. - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - // Both inputs are using input1_data... - Check( - completed_response, input1_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestDelete(irequest), - "deleting inference request"); - - FAIL_IF_ERR( - TRITONSERVER_ResponseAllocatorDelete(allocator), - "deleting response allocator"); - } - - public static void - main(String[] args) throws Exception - { - String model_repository_path = null; - int verbose_level = 0; - - // Parse commandline... - for (int i = 0; i < args.length; i++) { - switch (args[i]) { - case "-r": - model_repository_path = args[++i]; - break; - case "-v": - verbose_level = 1; - break; - case "-?": - Usage(null); - break; - } - } - - if (model_repository_path == null) { - Usage("-r must be used to specify model repository path"); - } - - try (PointerScope scope = new PointerScope()) { - RunInference(model_repository_path, verbose_level); - } - - System.exit(0); - } -} diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java new file mode 100644 index 00000000000..638daaa951f --- /dev/null +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -0,0 +1,140 @@ +// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; +import com.google.gson.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; +import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +public class SimpleTest { + // Helper functions + static void FAIL(String MSG) { + System.err.println("Failure: " + MSG); + System.exit(1); + } + + static void Usage(String msg) + { + if (msg != null) { + System.err.println(msg); + } + + System.err.println("Usage: java " + SimpleTest.class.getSimpleName() +" [options]"); + System.err.println("\t-v Enable verbose logging"); + System.err.println("\t-r [model repository absolute path]"); + System.exit(1); + } + + // static void + // GenerateInputData( + // StringVector input0_data, StringVector input1_data) + // { + // input0_data = new StringVector(16); + // input1_data = new StringVector(16); + // for (int i = 0; i < 16; ++i) { + // String input = "" + i; + // input0_data.put(i, input); + // input1_data.put(i, input); + // } + // } + + static void + CompareResult( + String output0_name, String output1_name, + IntPointer input0, IntPointer input1, IntPointer output0, + IntPointer output1) + { + for (int i = 0; i < 16; ++i) { + System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + + output0.get(i)); + System.out.println(input0.get(i) + " - " + input1.get(i) + " = " + + output1.get(i)); + + if ((input0.get(i) + input1.get(i)) != output0.get(i)) { + FAIL("incorrect sum in " + output0_name); + } + if ((input0.get(i) - input1.get(i)) != output1.get(i)) { + FAIL("incorrect difference in " + output1_name); + } + } + } + + static int RunInference(int verbose_level, String model_repository_path, String model_name) { + StringVector model_repository_paths = new StringVector(model_repository_path); + ServerOptions options = new ServerOptions(model_repository_paths); + LoggingOptions logging_options = options.logging_(); + logging_options.SetVerboseLevel(verbose_level); + options.SetLoggingOptions(logging_options); + + // GenericTritonServer server = new GenericTritonServer(options); + // StringSet loaded_models = server.LoadedModels(); + // GenericInferRequest request = new GenericInferRequest(); + // InferOptions infer_options = new InferOptions(model_name); + // request.Create(infer_options); + + + return 0; + } + + public static void + main(String[] args) throws Exception { + String model_repository_path = "./models"; + int verbose_level = 0; + + // Parse commandline... + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-r": + model_repository_path = args[++i]; + break; + case "-v": + verbose_level = 1; + break; + case "-?": + Usage(null); + break; + } + } + + // We use a simple model that takes 2 input tensors of 16 strings + // each and returns 2 output tensors of 16 strings each. The input + // strings must represent integers. One output tensor is the + // element-wise sum of the inputs and one output is the element-wise + // difference. + String model_name = "add_sub_str"; + if (model_repository_path == null) { + Usage("-r must be used to specify model repository path"); + } + + RunInference(verbose_level, model_repository_path, model_name); + + System.exit(0); + } + +} \ No newline at end of file diff --git a/tritonserverwrapper/samples/simple/pom.xml b/tritonserverwrapper/samples/simpletest/pom.xml similarity index 76% rename from tritonserverwrapper/samples/simple/pom.xml rename to tritonserverwrapper/samples/simpletest/pom.xml index 59c5b1d5fe2..87430483a61 100644 --- a/tritonserverwrapper/samples/simple/pom.xml +++ b/tritonserverwrapper/samples/simpletest/pom.xml @@ -1,10 +1,10 @@ 4.0.0 org.bytedeco.tritonserverwrapper - simple - 1.5.8-SNAPSHOT + simpletest + 1.5.9-SNAPSHOT - Simple + SimpleTest 1.8 1.8 @@ -12,7 +12,7 @@ org.bytedeco tritonserverwrapper-platform - 2.26-1.5.8-SNAPSHOT + 2.26-1.5.9-SNAPSHOT shaded diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java index c0b4f403cbd..93e970fd3dd 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java @@ -90,7 +90,7 @@ public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.preset // Parsed from generic_server_wrapper.h -// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java index 3a8084108fa..aeb7d621758 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java @@ -48,6 +48,8 @@ private native void allocate( @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file); + public native void SetVerboseLevel(int verbose_level); + // Verbose logging level. Default is OFF. public native @Cast("triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose_(); public native LoggingOptions verbose_(int setter); // Enable or disable info logging level. Default is true. diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java index b8bb019dd68..0c5201f1fc8 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java @@ -98,6 +98,8 @@ private native void allocate( @StdVector ModelLoadGPULimit model_load_gpu_limit, @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + public native void SetLoggingOptions(@Const @ByRef LoggingOptions logging); + // Paths to model repository directory. Note that if a model is not unique // across all model repositories at any time, the model will not be available. // See here for more information: From 513b33de7a69d9401305fca9a3db8f4bef9693e8 Mon Sep 17 00:00:00 2001 From: Baojun Liu Date: Tue, 4 Apr 2023 00:39:35 +0000 Subject: [PATCH 17/50] Test GenericTritonServer Create --- tritonserverwrapper/samples/simpletest/SimpleTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java index 638daaa951f..cbe101034ca 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -92,7 +92,7 @@ static int RunInference(int verbose_level, String model_repository_path, String logging_options.SetVerboseLevel(verbose_level); options.SetLoggingOptions(logging_options); - // GenericTritonServer server = new GenericTritonServer(options); + GenericTritonServer server = GenericTritonServer.Create(options); // StringSet loaded_models = server.LoadedModels(); // GenericInferRequest request = new GenericInferRequest(); // InferOptions infer_options = new InferOptions(model_name); @@ -137,4 +137,4 @@ static int RunInference(int verbose_level, String model_repository_path, String System.exit(0); } -} \ No newline at end of file +} From 215d9ed4564779427785e63d0bf875ee1d724177 Mon Sep 17 00:00:00 2001 From: Baojun Liu Date: Tue, 4 Apr 2023 03:55:04 +0000 Subject: [PATCH 18/50] Add more test --- tritonserverwrapper/samples/simpletest/SimpleTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java index cbe101034ca..6529815df26 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -93,10 +93,12 @@ static int RunInference(int verbose_level, String model_repository_path, String options.SetLoggingOptions(logging_options); GenericTritonServer server = GenericTritonServer.Create(options); - // StringSet loaded_models = server.LoadedModels(); - // GenericInferRequest request = new GenericInferRequest(); - // InferOptions infer_options = new InferOptions(model_name); - // request.Create(infer_options); + StringSet loaded_models = server.LoadedModels(); + System.out.println("Loaded_models count : " + loaded_models.size()); + + model_name = "simple"; + InferOptions infer_options = new InferOptions(model_name); + GenericInferRequest request = GenericInferRequest.Create(infer_options); return 0; From fe688477327260c8d5f477a8a0b5eaabb6d0d7d5 Mon Sep 17 00:00:00 2001 From: Baojun Liu Date: Tue, 4 Apr 2023 08:19:08 +0000 Subject: [PATCH 19/50] Add infer test --- .../samples/simpletest/SimpleTest.java | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java index 6529815df26..8c3c9ca673c 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -85,6 +85,18 @@ static void Usage(String msg) } } + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, 1 + i); + input1_data[0].put(i, 2); + } + } + static int RunInference(int verbose_level, String model_repository_path, String model_name) { StringVector model_repository_paths = new StringVector(model_repository_path); ServerOptions options = new ServerOptions(model_repository_paths); @@ -94,13 +106,41 @@ static int RunInference(int verbose_level, String model_repository_path, String GenericTritonServer server = GenericTritonServer.Create(options); StringSet loaded_models = server.LoadedModels(); - System.out.println("Loaded_models count : " + loaded_models.size()); - - model_name = "simple"; + System.out.println("Loaded_models count : " + loaded_models.size()); + InferOptions infer_options = new InferOptions(model_name); GenericInferRequest request = GenericInferRequest.Create(infer_options); - + BytePointer input0_data; + BytePointer input1_data; + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + + LongPointer shape0 = new LongPointer(2); + LongPointer shape1 = new LongPointer(2); + shape0.put(0, 1); + shape0.put(1, 16); + shape1.put(0, 1); + shape1.put(1, 16); + Tensor tensor0 = new Tensor(input0_data, 4, 8, shape0, 0, 0); + Tensor tensor1 = new Tensor(input1_data, 4, 8, shape1, 0, 0); + request.AddInput("INPUT0", tensor0); + request.AddInput("INPUT1", tensor1); + GenericInferResult result = server.Infer(request); + + Tensor output = result.Output("OUTPUT0"); + BytePointer buffer = output.buffer_(); + + System.out.println("output val at index 0 : " + buffer.getInt(0)); + System.out.println("output val at index 1 : " + buffer.getInt(1)); + System.out.println("output val at index 2 : " + buffer.getInt(2)); + System.out.println("output val at index 3 : " + buffer.getInt(3)); + System.out.println("output val at index 4 : " + buffer.getInt(4)); + System.out.println("output val at index 5 : " + buffer.getInt(5)); + System.out.println("output val at index 6 : " + buffer.getInt(6)); + System.out.println("output val at index 7 : " + buffer.getInt(7)); return 0; } @@ -129,7 +169,7 @@ static int RunInference(int verbose_level, String model_repository_path, String // strings must represent integers. One output tensor is the // element-wise sum of the inputs and one output is the element-wise // difference. - String model_name = "add_sub_str"; + String model_name = "simple"; if (model_repository_path == null) { Usage("-r must be used to specify model repository path"); } From ae7b1da3389fcdf88f171e8d7cfa19c2042f28b1 Mon Sep 17 00:00:00 2001 From: Baojun Liu Date: Tue, 4 Apr 2023 22:58:17 +0000 Subject: [PATCH 20/50] Make tensor add sub work --- .../samples/simpletest/SimpleTest.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java index 8c3c9ca673c..c6e08ae4854 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -92,8 +92,8 @@ static void Usage(String msg) input0_data[0] = new IntPointer(16); input1_data[0] = new IntPointer(16); for (int i = 0; i < 16; ++i) { - input0_data[0].put(i, 1 + i); - input1_data[0].put(i, 2); + input0_data[0].put(i, 2); + input1_data[0].put(i, 1 * i); } } @@ -124,8 +124,8 @@ static int RunInference(int verbose_level, String model_repository_path, String shape0.put(1, 16); shape1.put(0, 1); shape1.put(1, 16); - Tensor tensor0 = new Tensor(input0_data, 4, 8, shape0, 0, 0); - Tensor tensor1 = new Tensor(input1_data, 4, 8, shape1, 0, 0); + Tensor tensor0 = new Tensor(input0_data, 4 * 16, 8, shape0, 0, 1); + Tensor tensor1 = new Tensor(input1_data, 4 * 16, 8, shape1, 0, 1); request.AddInput("INPUT0", tensor0); request.AddInput("INPUT1", tensor1); GenericInferResult result = server.Infer(request); @@ -133,14 +133,16 @@ static int RunInference(int verbose_level, String model_repository_path, String Tensor output = result.Output("OUTPUT0"); BytePointer buffer = output.buffer_(); + System.out.println("buffer to string : " + buffer.toString()); + System.out.println("output val at index 0 : " + buffer.getInt(0)); - System.out.println("output val at index 1 : " + buffer.getInt(1)); - System.out.println("output val at index 2 : " + buffer.getInt(2)); - System.out.println("output val at index 3 : " + buffer.getInt(3)); - System.out.println("output val at index 4 : " + buffer.getInt(4)); - System.out.println("output val at index 5 : " + buffer.getInt(5)); - System.out.println("output val at index 6 : " + buffer.getInt(6)); - System.out.println("output val at index 7 : " + buffer.getInt(7)); + System.out.println("output val at index 1 : " + buffer.getInt(1 * 4)); + System.out.println("output val at index 2 : " + buffer.getInt(2 * 4)); + System.out.println("output val at index 3 : " + buffer.getInt(3 * 4)); + System.out.println("output val at index 4 : " + buffer.getInt(4 * 4)); + System.out.println("output val at index 5 : " + buffer.getInt(5 * 4)); + System.out.println("output val at index 6 : " + buffer.getInt(6 * 4)); + System.out.println("output val at index 7 : " + buffer.getInt(7 * 4)); return 0; } From d94dfe5c40a89bb4c9bff5eb80ea982ab38c88aa Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Tue, 23 May 2023 16:38:58 -0700 Subject: [PATCH 21/50] completed simple test --- .../samples/simpletest/SimpleTest.java | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simpletest/SimpleTest.java index 638daaa951f..f15949ce14c 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simpletest/SimpleTest.java @@ -51,19 +51,6 @@ static void Usage(String msg) System.exit(1); } - // static void - // GenerateInputData( - // StringVector input0_data, StringVector input1_data) - // { - // input0_data = new StringVector(16); - // input1_data = new StringVector(16); - // for (int i = 0; i < 16; ++i) { - // String input = "" + i; - // input0_data.put(i, input); - // input1_data.put(i, input); - // } - // } - static void CompareResult( String output0_name, String output1_name, @@ -85,6 +72,18 @@ static void Usage(String msg) } } + static void + GenerateInputData( + IntPointer[] input0_data, IntPointer[] input1_data) + { + input0_data[0] = new IntPointer(16); + input1_data[0] = new IntPointer(16); + for (int i = 0; i < 16; ++i) { + input0_data[0].put(i, 2); + input1_data[0].put(i, 1 * i); + } + } + static int RunInference(int verbose_level, String model_repository_path, String model_name) { StringVector model_repository_paths = new StringVector(model_repository_path); ServerOptions options = new ServerOptions(model_repository_paths); @@ -92,13 +91,45 @@ static int RunInference(int verbose_level, String model_repository_path, String logging_options.SetVerboseLevel(verbose_level); options.SetLoggingOptions(logging_options); - // GenericTritonServer server = new GenericTritonServer(options); - // StringSet loaded_models = server.LoadedModels(); - // GenericInferRequest request = new GenericInferRequest(); - // InferOptions infer_options = new InferOptions(model_name); - // request.Create(infer_options); + GenericTritonServer server = GenericTritonServer.Create(options); + StringSet loaded_models = server.LoadedModels(); + System.out.println("Loaded_models count : " + loaded_models.size()); + + InferOptions infer_options = new InferOptions(model_name); + GenericInferRequest request = GenericInferRequest.Create(infer_options); + + BytePointer input0_data; + BytePointer input1_data; + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + + LongPointer shape0 = new LongPointer(2); + LongPointer shape1 = new LongPointer(2); + shape0.put(0, 1); + shape0.put(1, 16); + shape1.put(0, 1); + shape1.put(1, 16); + Tensor tensor0 = new Tensor(input0_data, 4 * 16, 8, shape0, 0, 1); + Tensor tensor1 = new Tensor(input1_data, 4 * 16, 8, shape1, 0, 1); + request.AddInput("INPUT0", tensor0); + request.AddInput("INPUT1", tensor1); + GenericInferResult result = server.Infer(request); + + Tensor output = result.Output("OUTPUT0"); + BytePointer buffer = output.buffer_(); + System.out.println("buffer to string : " + buffer.toString()); + System.out.println("output val at index 0 : " + buffer.getInt(0)); + System.out.println("output val at index 1 : " + buffer.getInt(1 * 4)); + System.out.println("output val at index 2 : " + buffer.getInt(2 * 4)); + System.out.println("output val at index 3 : " + buffer.getInt(3 * 4)); + System.out.println("output val at index 4 : " + buffer.getInt(4 * 4)); + System.out.println("output val at index 5 : " + buffer.getInt(5 * 4)); + System.out.println("output val at index 6 : " + buffer.getInt(6 * 4)); + System.out.println("output val at index 7 : " + buffer.getInt(7 * 4)); return 0; } @@ -127,7 +158,7 @@ static int RunInference(int verbose_level, String model_repository_path, String // strings must represent integers. One output tensor is the // element-wise sum of the inputs and one output is the element-wise // difference. - String model_name = "add_sub_str"; + String model_name = "simple"; if (model_repository_path == null) { Usage("-r must be used to specify model repository path"); } From c5a1a9cb33ce16ddfdae74408340ff4734cb8cdf Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 24 May 2023 18:49:32 -0700 Subject: [PATCH 22/50] renamed example and cleaned up wrapper code --- tritonserver/samples/simple/pom.xml | 4 +- .../samples/simpleWrapper/SimpleWrapper.java | 848 ------------------ tritonserverwrapper/cppbuild.sh | 4 +- tritonserverwrapper/platform/pom.xml | 6 +- tritonserverwrapper/pom.xml | 19 - .../SimpleTest.java => simple/Simple.java} | 4 +- .../samples/{simpletest => simple}/pom.xml | 4 +- .../presets/tritonserverwrapper.java | 14 +- 8 files changed, 13 insertions(+), 890 deletions(-) delete mode 100644 tritonserver/samples/simpleWrapper/SimpleWrapper.java rename tritonserverwrapper/samples/{simpletest/SimpleTest.java => simple/Simple.java} (98%) rename tritonserverwrapper/samples/{simpletest => simple}/pom.xml (87%) diff --git a/tritonserver/samples/simple/pom.xml b/tritonserver/samples/simple/pom.xml index b635baebdb4..02e761820a8 100644 --- a/tritonserver/samples/simple/pom.xml +++ b/tritonserver/samples/simple/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.bytedeco.tritonserver simple - 1.5.8 + 1.5.9-SNAPSHOT Simple 1.8 @@ -12,7 +12,7 @@ org.bytedeco tritonserver-platform - 2.26-1.5.8 + 2.26-1.5.9-SNAPSHOT shaded diff --git a/tritonserver/samples/simpleWrapper/SimpleWrapper.java b/tritonserver/samples/simpleWrapper/SimpleWrapper.java deleted file mode 100644 index b89bcc1cd5f..00000000000 --- a/tritonserver/samples/simpleWrapper/SimpleWrapper.java +++ /dev/null @@ -1,848 +0,0 @@ -// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import java.io.*; -import java.util.*; -import java.util.concurrent.*; -import com.google.gson.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.tritonserver.tritonserver.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; - -public class SimpleWrapper { - - static void FAIL(String MSG) { - System.err.println("Failure: " + MSG); - System.exit(1); - } - - static void FAIL_IF_ERR(TRITONSERVER_Error err__, String MSG) { - if (err__ != null) { - System.err.println("error: " + MSG + ":" - + TRITONSERVER_ErrorCodeString(err__) + " - " - + TRITONSERVER_ErrorMessage(err__)); - TRITONSERVER_ErrorDelete(err__); - System.exit(1); - } - } - - static final int requested_memory_type = TRITONSERVER_MEMORY_CPU; - - static class TRITONSERVER_ServerDeleter extends TRITONSERVER_Server { - public TRITONSERVER_ServerDeleter(TRITONSERVER_Server p) { super(p); deallocator(new DeleteDeallocator(this)); } - protected static class DeleteDeallocator extends TRITONSERVER_Server implements Deallocator { - DeleteDeallocator(Pointer p) { super(p); } - @Override public void deallocate() { TRITONSERVER_ServerDelete(this); } - } - } - - static void - Usage(String msg) - { - if (msg != null) { - System.err.println(msg); - } - - System.err.println("Usage: java " + SimpleCPUOnly.class.getSimpleName() + " [options]"); - System.err.println("\t-v Enable verbose logging"); - System.err.println("\t-r [model repository absolute path]"); - - System.exit(1); - } - - static class ResponseAlloc extends TRITONSERVER_ResponseAllocatorAllocFn_t { - @Override public TRITONSERVER_Error call ( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - long byte_size, int preferred_memory_type, - long preferred_memory_type_id, Pointer userp, PointerPointer buffer, - PointerPointer buffer_userp, IntPointer actual_memory_type, - LongPointer actual_memory_type_id) - { - // Initially attempt to make the actual memory type and id that we - // allocate be the same as preferred memory type - actual_memory_type.put(0, preferred_memory_type); - actual_memory_type_id.put(0, preferred_memory_type_id); - - // If 'byte_size' is zero just return 'buffer' == nullptr, we don't - // need to do any other book-keeping. - if (byte_size == 0) { - buffer.put(0, null); - buffer_userp.put(0, null); - System.out.println("allocated " + byte_size + " bytes for result tensor " + tensor_name); - } else { - Pointer allocated_ptr = new Pointer(); - actual_memory_type.put(0, requested_memory_type); - - actual_memory_type.put(0, TRITONSERVER_MEMORY_CPU); - allocated_ptr = Pointer.malloc(byte_size); - - // Pass the tensor name with buffer_userp so we can show it when - // releasing the buffer. - if (!allocated_ptr.isNull()) { - buffer.put(0, allocated_ptr); - buffer_userp.put(0, Loader.newGlobalRef(tensor_name)); - System.out.println("allocated " + byte_size + " bytes in " - + TRITONSERVER_MemoryTypeString(actual_memory_type.get()) - + " for result tensor " + tensor_name); - } - } - - return null; // Success - } - } - - static class ResponseRelease extends TRITONSERVER_ResponseAllocatorReleaseFn_t { - @Override public TRITONSERVER_Error call ( - TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, - long byte_size, int memory_type, long memory_type_id) - { - String name = null; - if (buffer_userp != null) { - name = (String)Loader.accessGlobalRef(buffer_userp); - } else { - name = ""; - } - - System.out.println("Releasing buffer " + buffer + " of size " + byte_size - + " in " + TRITONSERVER_MemoryTypeString(memory_type) - + " for result '" + name + "'"); - Pointer.free(buffer); - Loader.deleteGlobalRef(buffer_userp); - - return null; // Success - } - } - - static class InferRequestComplete extends TRITONSERVER_InferenceRequestReleaseFn_t { - @Override public void call ( - TRITONSERVER_InferenceRequest request, int flags, Pointer userp) - { - // We reuse the request so we don't delete it here. - } - } - - static class InferResponseComplete extends TRITONSERVER_InferenceResponseCompleteFn_t { - @Override public void call ( - TRITONSERVER_InferenceResponse response, int flags, Pointer userp) - { - if (response != null) { - // Send 'response' to the future. - futures.get(userp).complete(response); - } - } - } - - static ConcurrentHashMap> futures = new ConcurrentHashMap<>(); - static ResponseAlloc responseAlloc = new ResponseAlloc(); - static ResponseRelease responseRelease = new ResponseRelease(); - static InferRequestComplete inferRequestComplete = new InferRequestComplete(); - static InferResponseComplete inferResponseComplete = new InferResponseComplete(); - - static TRITONSERVER_Error - ParseModelMetadata( - JsonObject model_metadata, boolean[] is_int, - boolean[] is_torch_model) - { - String seen_data_type = null; - for (JsonElement input_element : model_metadata.get("inputs").getAsJsonArray()) { - JsonObject input = input_element.getAsJsonObject(); - if (!input.get("datatype").getAsString().equals("INT32") && - !input.get("datatype").getAsString().equals("FP32")) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "simple lib example only supports model with data type INT32 or " + - "FP32"); - } - if (seen_data_type == null) { - seen_data_type = input.get("datatype").getAsString(); - } else if (!seen_data_type.equals(input.get("datatype").getAsString())) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - "the inputs and outputs of 'simple' model must have the data type"); - } - } - for (JsonElement output_element : model_metadata.get("outputs").getAsJsonArray()) { - JsonObject output = output_element.getAsJsonObject(); - if (!output.get("datatype").getAsString().equals("INT32") && - !output.get("datatype").getAsString().equals("FP32")) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "simple lib example only supports model with data type INT32 or " + - "FP32"); - } else if (!seen_data_type.equals(output.get("datatype").getAsString())) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - "the inputs and outputs of 'simple' model must have the data type"); - } - } - - is_int[0] = seen_data_type.equals("INT32"); - is_torch_model[0] = - model_metadata.get("platform").getAsString().equals("pytorch_libtorch"); - return null; - } - - static void - GenerateInputData( - IntPointer[] input0_data, IntPointer[] input1_data) - { - input0_data[0] = new IntPointer(16); - input1_data[0] = new IntPointer(16); - for (int i = 0; i < 16; ++i) { - input0_data[0].put(i, i); - input1_data[0].put(i, 1); - } - } - - static void - GenerateInputData( - FloatPointer[] input0_data, FloatPointer[] input1_data) - { - input0_data[0] = new FloatPointer(16); - input1_data[0] = new FloatPointer(16); - for (int i = 0; i < 16; ++i) { - input0_data[0].put(i, i); - input1_data[0].put(i, 1); - } - } - - static void - CompareResult( - String output0_name, String output1_name, - IntPointer input0, IntPointer input1, IntPointer output0, - IntPointer output1) - { - for (int i = 0; i < 16; ++i) { - System.out.println(input0.get(i) + " + " + input1.get(i) + " = " - + output0.get(i)); - System.out.println(input0.get(i) + " - " + input1.get(i) + " = " - + output1.get(i)); - - if ((input0.get(i) + input1.get(i)) != output0.get(i)) { - FAIL("incorrect sum in " + output0_name); - } - if ((input0.get(i) - input1.get(i)) != output1.get(i)) { - FAIL("incorrect difference in " + output1_name); - } - } - } - - static void - CompareResult( - String output0_name, String output1_name, - FloatPointer input0, FloatPointer input1, FloatPointer output0, - FloatPointer output1) - { - for (int i = 0; i < 16; ++i) { - System.out.println(input0.get(i) + " + " + input1.get(i) + " = " - + output0.get(i)); - System.out.println(input0.get(i) + " - " + input1.get(i) + " = " - + output1.get(i)); - - if ((input0.get(i) + input1.get(i)) != output0.get(i)) { - FAIL("incorrect sum in " + output0_name); - } - if ((input0.get(i) - input1.get(i)) != output1.get(i)) { - FAIL("incorrect difference in " + output1_name); - } - } - } - - static void - Check( - TRITONSERVER_InferenceResponse response, - Pointer input0_data, Pointer input1_data, - String output0, String output1, - long expected_byte_size, - int expected_datatype, boolean is_int) - { - HashMap output_data = new HashMap<>(); - - int[] output_count = {0}; - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseOutputCount(response, output_count), - "getting number of response outputs"); - if (output_count[0] != 2) { - FAIL("expecting 2 response outputs, got " + output_count[0]); - } - - for (int idx = 0; idx < output_count[0]; ++idx) { - BytePointer cname = new BytePointer((Pointer)null); - IntPointer datatype = new IntPointer(1); - LongPointer shape = new LongPointer((Pointer)null); - LongPointer dim_count = new LongPointer(1); - Pointer base = new Pointer(); - SizeTPointer byte_size = new SizeTPointer(1); - IntPointer memory_type = new IntPointer(1); - LongPointer memory_type_id = new LongPointer(1); - Pointer userp = new Pointer(); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseOutput( - response, idx, cname, datatype, shape, dim_count, base, - byte_size, memory_type, memory_type_id, userp), - "getting output info"); - - if (cname.isNull()) { - FAIL("unable to get output name"); - } - - String name = cname.getString(); - if ((!name.equals(output0)) && (!name.equals(output1))) { - FAIL("unexpected output '" + name + "'"); - } - - if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { - FAIL("unexpected shape for '" + name + "'"); - } - - if (datatype.get() != expected_datatype) { - FAIL( - "unexpected datatype '" + - TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + - name + "'"); - } - - if (byte_size.get() != expected_byte_size) { - FAIL( - "unexpected byte-size, expected " + - expected_byte_size + ", got " + - byte_size.get() + " for " + name); - } - - if (memory_type.get() != requested_memory_type) { - FAIL( - "unexpected memory type, expected to be allocated in " + - TRITONSERVER_MemoryTypeString(requested_memory_type) + - ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + - ", id " + memory_type_id.get() + " for " + name); - } - - // We make a copy of the data here... which we could avoid for - // performance reasons but ok for this simple example. - BytePointer odata = new BytePointer(byte_size.get()); - output_data.put(name, odata); - System.out.println(name + " is stored in system memory"); - odata.put(base.limit(byte_size.get())); - } - - if (is_int) { - CompareResult( - output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), - new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); - } else { - CompareResult( - output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), - new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); - } - } - - static void - Check( - InferResult result, - Pointer input0_data, Pointer input1_data, - String output0, String output1, - long expected_byte_size, - int expected_datatype, boolean is_int) - { - HashMap output_data = new HashMap<>(); - - System.out.println("geting output counts"); - - int output_count = result.OutputNames().limit(); - - - if (output_count != 2) { - FAIL("expecting 2 response outputs, got " + output_count); - } - - for (int idx = 0; idx < output_count; ++idx) { - IntPointer datatype = new IntPointer(1); - LongPointer shape = new LongPointer((Pointer)null); - LongPointer dim_count = new LongPointer(1); - Pointer base = new Pointer(); - SizeTPointer byte_size = new SizeTPointer(1); - IntPointer memory_type = new IntPointer(1); - LongPointer memory_type_id = new LongPointer(1); - Pointer userp = new Pointer(); - - - System.out.println("geting output info"); - String name = result.OutputNames().position(idx); - if ((!name.equals(output0)) && (!name.equals(output1))) { - FAIL("unexpected output '" + name + "'"); - } - - - - Tensor output = result.Ouput(name); - dim_count = - if ((dim_count.get() != 2) || (shape.get(0) != 1) || (shape.get(1) != 16)) { - FAIL("unexpected shape for '" + name + "'"); - } - - if (datatype.get() != expected_datatype) { - FAIL( - "unexpected datatype '" + - TRITONSERVER_DataTypeString(datatype.get()) + "' for '" + - name + "'"); - } - - if (byte_size.get() != expected_byte_size) { - FAIL( - "unexpected byte-size, expected " + - expected_byte_size + ", got " + - byte_size.get() + " for " + name); - } - - if (memory_type.get() != requested_memory_type) { - FAIL( - "unexpected memory type, expected to be allocated in " + - TRITONSERVER_MemoryTypeString(requested_memory_type) + - ", got " + TRITONSERVER_MemoryTypeString(memory_type.get()) + - ", id " + memory_type_id.get() + " for " + name); - } - - // We make a copy of the data here... which we could avoid for - // performance reasons but ok for this simple example. - BytePointer odata = new BytePointer(byte_size.get()); - output_data.put(name, odata); - System.out.println(name + " is stored in system memory"); - odata.put(base.limit(byte_size.get())); - } - - if (is_int) { - CompareResult( - output0, output1, new IntPointer(input0_data), new IntPointer(input1_data), - new IntPointer(output_data.get(output0)), new IntPointer(output_data.get(output1))); - } else { - CompareResult( - output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), - new FloatPointer(output_data.get(output0)), new FloatPointer(output_data.get(output1))); - } - } - - - public static void - RunInference(String model_repository_path, int verbose_level) throws Exception - { - // Check API version. - int[] api_version_major = {0}, api_version_minor = {0}; - FAIL_IF_ERR( - TRITONSERVER_ApiVersion(api_version_major, api_version_minor), - "getting Triton API version"); - if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || - (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { - FAIL("triton server API version mismatch"); - } - - // Create the server... - TRITONSERVER_ServerOptions server_options = new TRITONSERVER_ServerOptions(null); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsNew(server_options), - "creating server options"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetModelRepositoryPath( - server_options, model_repository_path), - "setting model repository path"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), - "setting verbose logging level"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetBackendDirectory( - server_options, "/opt/tritonserver/backends"), - "setting backend directory"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetRepoAgentDirectory( - server_options, "/opt/tritonserver/repoagents"), - "setting repository agent directory"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), - "setting strict model configuration"); - - TRITONSERVER_Server server_ptr = new TRITONSERVER_Server(null); - FAIL_IF_ERR( - TRITONSERVER_ServerNew(server_ptr, server_options), "creating server"); - FAIL_IF_ERR( - TRITONSERVER_ServerOptionsDelete(server_options), - "deleting server options"); - - TRITONSERVER_ServerDeleter server = new TRITONSERVER_ServerDeleter(server_ptr); - - // Wait until the server is both live and ready. - int health_iters = 0; - while (true) { - boolean[] live = {false}, ready = {false}; - FAIL_IF_ERR( - TRITONSERVER_ServerIsLive(server, live), - "unable to get server liveness"); - FAIL_IF_ERR( - TRITONSERVER_ServerIsReady(server, ready), - "unable to get server readiness"); - System.out.println("Server Health: live " + live[0] + ", ready " + ready[0]); - if (live[0] && ready[0]) { - break; - } - - if (++health_iters >= 10) { - FAIL("failed to find healthy inference server"); - } - - Thread.sleep(500); - } - - // Print status of the server. - { - TRITONSERVER_Message server_metadata_message = new TRITONSERVER_Message(null); - FAIL_IF_ERR( - TRITONSERVER_ServerMetadata(server, server_metadata_message), - "unable to get server metadata message"); - BytePointer buffer = new BytePointer((Pointer)null); - SizeTPointer byte_size = new SizeTPointer(1); - FAIL_IF_ERR( - TRITONSERVER_MessageSerializeToJson( - server_metadata_message, buffer, byte_size), - "unable to serialize server metadata message"); - - System.out.println("Server Status:"); - System.out.println(buffer.limit(byte_size.get()).getString()); - - FAIL_IF_ERR( - TRITONSERVER_MessageDelete(server_metadata_message), - "deleting status metadata"); - } - - String model_name = "simple"; - - // Wait for the model to become available. - boolean[] is_torch_model = {false}; - boolean[] is_int = {true}; - boolean[] is_ready = {false}; - health_iters = 0; - while (!is_ready[0]) { - FAIL_IF_ERR( - TRITONSERVER_ServerModelIsReady( - server, model_name, 1, is_ready), - "unable to get model readiness"); - if (!is_ready[0]) { - if (++health_iters >= 10) { - FAIL("model failed to be ready in 10 iterations"); - } - Thread.sleep(500); - continue; - } - - TRITONSERVER_Message model_metadata_message = new TRITONSERVER_Message(null); - FAIL_IF_ERR( - TRITONSERVER_ServerModelMetadata( - server, model_name, 1, model_metadata_message), - "unable to get model metadata message"); - BytePointer buffer = new BytePointer((Pointer)null); - SizeTPointer byte_size = new SizeTPointer(1); - FAIL_IF_ERR( - TRITONSERVER_MessageSerializeToJson( - model_metadata_message, buffer, byte_size), - "unable to serialize model status protobuf"); - - JsonParser parser = new JsonParser(); - JsonObject model_metadata = null; - try { - model_metadata = parser.parse(buffer.limit(byte_size.get()).getString()).getAsJsonObject(); - } catch (Exception e) { - FAIL("error: failed to parse model metadata from JSON: " + e); - } - - FAIL_IF_ERR( - TRITONSERVER_MessageDelete(model_metadata_message), - "deleting status protobuf"); - - if (!model_metadata.get("name").getAsString().equals(model_name)) { - FAIL("unable to find metadata for model"); - } - - boolean found_version = false; - if (model_metadata.has("versions")) { - for (JsonElement version : model_metadata.get("versions").getAsJsonArray()) { - if (version.getAsString().equals("1")) { - found_version = true; - break; - } - } - } - if (!found_version) { - FAIL("unable to find version 1 status for model"); - } - - FAIL_IF_ERR( - ParseModelMetadata(model_metadata, is_int, is_torch_model), - "parsing model metadata"); - } - - // Create the allocator that will be used to allocate buffers for - // the result tensors. - TRITONSERVER_ResponseAllocator allocator = new TRITONSERVER_ResponseAllocator(null); - FAIL_IF_ERR( - TRITONSERVER_ResponseAllocatorNew( - allocator, responseAlloc, responseRelease, null /* start_fn */), - "creating response allocator"); - - // Inference - TRITONSERVER_InferenceRequest irequest = new TRITONSERVER_InferenceRequest(null); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestNew( - irequest, server, model_name, -1 /* model_version */), - "creating inference request"); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), - "setting ID for the request"); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetReleaseCallback( - irequest, inferRequestComplete, null /* request_release_userp */), - "setting request release callback"); - - // Inputs - String input0 = is_torch_model[0] ? "INPUT__0" : "INPUT0"; - String input1 = is_torch_model[0] ? "INPUT__1" : "INPUT1"; - - long[] input0_shape = {1, 16}; - long[] input1_shape = {1, 16}; - - int datatype = - (is_int[0]) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddInput( - irequest, input0, datatype, input0_shape, input0_shape.length), - "setting input 0 meta-data for the request"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddInput( - irequest, input1, datatype, input1_shape, input1_shape.length), - "setting input 1 meta-data for the request"); - - String output0 = is_torch_model[0] ? "OUTPUT__0" : "OUTPUT0"; - String output1 = is_torch_model[0] ? "OUTPUT__1" : "OUTPUT1"; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), - "requesting output 0 for the request"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), - "requesting output 1 for the request"); - - // Create the data for the two input tensors. Initialize the first - // to unique values and the second to all ones. - BytePointer input0_data; - BytePointer input1_data; - if (is_int[0]) { - IntPointer[] p0 = {null}, p1 = {null}; - GenerateInputData(p0, p1); - input0_data = p0[0].getPointer(BytePointer.class); - input1_data = p1[0].getPointer(BytePointer.class); - } else { - FloatPointer[] p0 = {null}, p1 = {null}; - GenerateInputData(p0, p1); - input0_data = p0[0].getPointer(BytePointer.class); - input1_data = p1[0].getPointer(BytePointer.class); - } - - long input0_size = input0_data.limit(); - long input1_size = input1_data.limit(); - - Pointer input0_base = input0_data; - Pointer input1_base = input1_data; - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input0, input0_base, input0_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT0 data"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input1, input1_base, input1_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT1 data"); - - // Perform inference... - { - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - Check( - completed_response, input0_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - // Modify some input data in place and then reuse the request - // object. - { - if (is_int[0]) { - new IntPointer(input0_data).put(0, 27); - } else { - new FloatPointer(input0_data).put(0, 27.0f); - } - - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - // Using a new promise so have to re-register the callback to set - // the promise as the userp. - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - Check( - completed_response, input0_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - // Remove input data and then add back different data. - { - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), - "removing INPUT0 data"); - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestAppendInputData( - irequest, input0, input1_base, input1_size, requested_memory_type, - 0 /* memory_type_id */), - "assigning INPUT1 data to INPUT0"); - - CompletableFuture completed = new CompletableFuture<>(); - futures.put(irequest, completed); - - // Using a new promise so have to re-register the callback to set - // the promise as the userp. - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator, null /* response_allocator_userp */, - inferResponseComplete, irequest), - "setting response callback"); - - FAIL_IF_ERR( - TRITONSERVER_ServerInferAsync( - server, irequest, null /* trace */), - "running inference"); - - // Wait for the inference to complete. - TRITONSERVER_InferenceResponse completed_response = completed.get(); - futures.remove(irequest); - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseError(completed_response), - "response status"); - - // Both inputs are using input1_data... - Check( - completed_response, input1_data, input1_data, output0, output1, - input0_size, datatype, is_int[0]); - - FAIL_IF_ERR( - TRITONSERVER_InferenceResponseDelete(completed_response), - "deleting inference response"); - } - - FAIL_IF_ERR( - TRITONSERVER_InferenceRequestDelete(irequest), - "deleting inference request"); - - FAIL_IF_ERR( - TRITONSERVER_ResponseAllocatorDelete(allocator), - "deleting response allocator"); - } - - public static void - main(String[] args) throws Exception - { - String model_repository_path = null; - int verbose_level = 0; - - // Parse commandline... - for (int i = 0; i < args.length; i++) { - switch (args[i]) { - case "-r": - model_repository_path = args[++i]; - break; - case "-v": - verbose_level = 1; - break; - case "-?": - Usage(null); - break; - } - } - - if (model_repository_path == null) { - Usage("-r must be used to specify model repository path"); - } - - try (PointerScope scope = new PointerScope()) { - RunInference(model_repository_path, verbose_level); - } - - System.exit(0); - } -} diff --git a/tritonserverwrapper/cppbuild.sh b/tritonserverwrapper/cppbuild.sh index 87fe0b381f1..712186b0138 100755 --- a/tritonserverwrapper/cppbuild.sh +++ b/tritonserverwrapper/cppbuild.sh @@ -9,13 +9,13 @@ fi case $PLATFORM in linux-arm64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + if [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/developer_tools/" ]]; then echo "Please make sure library and include files exist" exit 1 fi ;; linux-x86_64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + if [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/developer_tools/" ]]; then echo "Please make sure library and include files exist" exit 1 fi diff --git a/tritonserverwrapper/platform/pom.xml b/tritonserverwrapper/platform/pom.xml index 4211a9555cc..e514ce47c7d 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritonserverwrapper/platform/pom.xml @@ -20,11 +20,11 @@ - + ${project.groupId} ${javacpp.moduleId} @@ -48,7 +48,7 @@ - ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64.jar ${javacpp.moduleId}-linux-x86_64.jar ${javacpp.moduleId}-windows-x86_64.jar + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64.jar ${javacpp.moduleId}-linux-x86_64.jar diff --git a/tritonserverwrapper/pom.xml b/tritonserverwrapper/pom.xml index b7e410f4598..dad9346d56a 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritonserverwrapper/pom.xml @@ -15,11 +15,6 @@ JavaCPP Presets for Triton Inference Server Wrapper - - org.bytedeco - tritonserver - 2.26-${project.parent.version} - org.bytedeco javacpp @@ -41,20 +36,6 @@ ISO-8859-1 - maven-jar-plugin diff --git a/tritonserverwrapper/samples/simpletest/SimpleTest.java b/tritonserverwrapper/samples/simple/Simple.java similarity index 98% rename from tritonserverwrapper/samples/simpletest/SimpleTest.java rename to tritonserverwrapper/samples/simple/Simple.java index 1c88e4e46de..8c7abb5069a 100644 --- a/tritonserverwrapper/samples/simpletest/SimpleTest.java +++ b/tritonserverwrapper/samples/simple/Simple.java @@ -32,7 +32,7 @@ import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; -public class SimpleTest { +public class Simple { // Helper functions static void FAIL(String MSG) { System.err.println("Failure: " + MSG); @@ -45,7 +45,7 @@ static void Usage(String msg) System.err.println(msg); } - System.err.println("Usage: java " + SimpleTest.class.getSimpleName() +" [options]"); + System.err.println("Usage: java " + Simple.class.getSimpleName() +" [options]"); System.err.println("\t-v Enable verbose logging"); System.err.println("\t-r [model repository absolute path]"); System.exit(1); diff --git a/tritonserverwrapper/samples/simpletest/pom.xml b/tritonserverwrapper/samples/simple/pom.xml similarity index 87% rename from tritonserverwrapper/samples/simpletest/pom.xml rename to tritonserverwrapper/samples/simple/pom.xml index 87430483a61..1fc3f5107ab 100644 --- a/tritonserverwrapper/samples/simpletest/pom.xml +++ b/tritonserverwrapper/samples/simple/pom.xml @@ -1,10 +1,10 @@ 4.0.0 org.bytedeco.tritonserverwrapper - simpletest + simple 1.5.9-SNAPSHOT - SimpleTest + Simple 1.8 1.8 diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index e06e12bdae9..3f80af0db79 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Jack He, Samuel Audet + * Copyright (C) 2023 * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -46,12 +46,6 @@ includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), - // @Platform( - // value = "windows-x86_64", - // includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", - // linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", - // preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" - // ) }, target = "org.bytedeco.tritonserverwrapper.tritonserverwrapper", global = "org.bytedeco.tritonserverwrapper.global.tritonserverwrapper" @@ -69,10 +63,6 @@ public void map(InfoMap infoMap) { .put(new Info("std::set").pointerTypes("StringSet").define()) .put(new Info("std::vector").pointerTypes("StringVector").define()) .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) - .put(new Info("TritonServer").purify(false).virtualize()) - // .put(new Info("server_wrapper.h").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) - // .put(new Info("server_wrapper.cc").linePatterns(" // START_JAVA_CUSTOM_FUNCTIONS", " // END_JAVA_CUSTOM_FUNCTIONS").skip()) - // .put(new Info("server_wrapper.h").linePatterns("std::future>*", ";", "std::unique_ptr>>*", ";").skip()) - ; + .put(new Info("TritonServer").purify(false).virtualize()); } } From c5c1e92a3bc4219f0983de1f71841e77dca47e55 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 24 May 2023 19:14:36 -0700 Subject: [PATCH 23/50] rerun build --- .../samples/simple/Simple.java | 2 +- .../global/tritonserverwrapper.java | 95 ++++++++++--------- .../GenericInferRequest.java | 11 ++- .../GenericInferResult.java | 7 +- .../GenericTritonServer.java | 4 +- .../tritonserverwrapper/LoggingOptions.java | 1 + .../ResponseAllocatorStartFn_t.java | 2 + .../presets/tritonserverwrapper.java | 1 - 8 files changed, 67 insertions(+), 56 deletions(-) diff --git a/tritonserverwrapper/samples/simple/Simple.java b/tritonserverwrapper/samples/simple/Simple.java index 8c7abb5069a..837e22b1977 100644 --- a/tritonserverwrapper/samples/simple/Simple.java +++ b/tritonserverwrapper/samples/simple/Simple.java @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java index 93e970fd3dd..db0ef98fba3 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java @@ -45,6 +45,9 @@ public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.preset // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once +// #include +// #include +// #include //============================================================================== /** enum classes @@ -84,6 +87,48 @@ public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.preset // Targeting ../tritonserverwrapper/ResponseAllocatorStartFn_t.java +// Targeting ../tritonserverwrapper/LoggingOptions.java + + +// Targeting ../tritonserverwrapper/MetricsOptions.java + + +// Targeting ../tritonserverwrapper/RateLimitResource.java + + +// Targeting ../tritonserverwrapper/ModelLoadGPULimit.java + + +// Targeting ../tritonserverwrapper/Allocator.java + + +// Targeting ../tritonserverwrapper/BackendConfig.java + + +// Targeting ../tritonserverwrapper/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritonserverwrapper/HostPolicy.java + + +// Targeting ../tritonserverwrapper/Trace.java + + +// Targeting ../tritonserverwrapper/ServerOptions.java + + +// Targeting ../tritonserverwrapper/RepositoryIndex.java + + +// Targeting ../tritonserverwrapper/Tensor.java + + +// Targeting ../tritonserverwrapper/NewModelRepo.java + + +// Targeting ../tritonserverwrapper/InferOptions.java + + // namespace triton::developer_tools::server @@ -116,14 +161,13 @@ public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.preset // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once +// #include // #include -// #include -// #include // #include -// #include -// #include "common.h" +// #include // #include "../src/infer_requested_output.h" // #include "../src/tracer.h" +// #include "common.h" /// // Targeting ../tritonserverwrapper/GenericTritonServer.java @@ -135,49 +179,8 @@ public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.preset // Targeting ../tritonserverwrapper/GenericInferRequest.java -// Targeting ../tritonserverwrapper/LoggingOptions.java - - -// Targeting ../tritonserverwrapper/MetricsOptions.java - - -// Targeting ../tritonserverwrapper/RateLimitResource.java - - -// Targeting ../tritonserverwrapper/ModelLoadGPULimit.java - - -// Targeting ../tritonserverwrapper/Allocator.java - - -// Targeting ../tritonserverwrapper/BackendConfig.java - - -// Targeting ../tritonserverwrapper/CUDAMemoryPoolByteSize.java - - -// Targeting ../tritonserverwrapper/HostPolicy.java - - -// Targeting ../tritonserverwrapper/Trace.java - - -// Targeting ../tritonserverwrapper/ServerOptions.java - - -// Targeting ../tritonserverwrapper/RepositoryIndex.java - - -// Targeting ../tritonserverwrapper/Tensor.java - - -// Targeting ../tritonserverwrapper/NewModelRepo.java - - -// Targeting ../tritonserverwrapper/InferOptions.java - - // namespace triton::developer_tools::server + } diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java index 54f9aa13a31..0aa12fe979b 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java @@ -27,8 +27,10 @@ public class GenericInferRequest extends Pointer { * inference is completed and result is returned. * @param name The name of the input tensor. * @param input A Tensor object that describes an input tensor. */ - public native @NoException(true) void AddInput(@StdString BytePointer name, @Const @ByRef Tensor input); - public native @NoException(true) void AddInput(@StdString String name, @Const @ByRef Tensor input); + public native @NoException(true) void AddInput( + @StdString BytePointer name, @Const @ByRef Tensor input); + public native @NoException(true) void AddInput( + @StdString String name, @Const @ByRef Tensor input); /** Add a requested output to be sent within an InferRequest object. * Calling this function is optional. If no output(s) are specifically @@ -38,8 +40,8 @@ public class GenericInferRequest extends Pointer { * @param name The name of the output tensor. * @param output A Tensor object that describes an output tensor containing * its pre-allocated buffer. */ - public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); - public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); + public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); + public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); /** Add a requested output to be sent within an InferRequest object. * Calling this function is optional. If no output(s) are specifically @@ -52,5 +54,4 @@ public class GenericInferRequest extends Pointer { /** Clear inputs and outputs of the request. This allows users to reuse the * InferRequest object if needed. */ public native void Reset(); - } diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java index 0929769fce4..368d542e85f 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java @@ -35,6 +35,7 @@ public class GenericInferResult extends Pointer { /** Get the output names from the infer result * @return Vector of output names */ public native @ByVal StringVector OutputNames(); + /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' * field of the output is owned by the returned 'Tensor' object itself. Note * that for string data, need to use 'StringData' function for string data @@ -50,8 +51,10 @@ public class GenericInferResult extends Pointer { * @param output_name The name of the output to get result data. * @return Returns the result data represented as a vector of strings. The * strings are stored in the row-major order. */ - public native @ByVal StringVector StringData(@StdString BytePointer output_name); - public native @ByVal StringVector StringData(@StdString String output_name); + public native @ByVal StringVector StringData( + @StdString BytePointer output_name); + public native @ByVal StringVector StringData( + @StdString String output_name); /** Return the complete response as a user friendly string. * @return The string describing the complete response. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java index 7d0b8495a1d..dcfce89868e 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java @@ -107,7 +107,7 @@ public class GenericTritonServer extends Pointer { * @param model_name The name of the model. * @param model_version The version of the model to get configuration. * The default value is -1 which means then the server will choose a version - * based on the model and internal policy. This field is optional. + * based on the model and internal policy. This field is optional. * @return Returns JSON representation of model metadata as a string. */ public native @StdString BytePointer ModelMetadata( @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); @@ -129,4 +129,6 @@ public class GenericTritonServer extends Pointer { * @param repo_path The full path to the model repository. */ public native void UnregisterModelRepo(@StdString BytePointer repo_path); public native void UnregisterModelRepo(@StdString String repo_path); + + public native @UniquePtr GenericInferResult Infer(@ByRef GenericInferRequest infer_request); } diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java index aeb7d621758..5ebf7a51519 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java @@ -8,6 +8,7 @@ import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + //============================================================================== /** Structure to hold logging options for setting 'ServerOptions'. * */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java index c3e3e808569..7a53ad140b3 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java +++ b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java @@ -8,6 +8,8 @@ import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; + +/// @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) public class ResponseAllocatorStartFn_t extends FunctionPointer { static { Loader.load(); } diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java index 3f80af0db79..8e4aece0ffa 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java @@ -31,7 +31,6 @@ import org.bytedeco.javacpp.tools.Info; import org.bytedeco.javacpp.tools.InfoMap; import org.bytedeco.javacpp.tools.InfoMapper; -import org.bytedeco.tritonserver.presets.tritonserver; /** * From a4690b10cf29a4f3fa2d95e574f45ef87b96db6d Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 25 May 2023 20:27:16 -0700 Subject: [PATCH 24/50] renamed tritonserverwrapper to tritondevelopertoolsserver --- platform/pom.xml | 6 +- pom.xml | 4 +- .../README.md | 0 .../cppbuild.sh | 2 +- .../platform/pom.xml | 6 +- .../platform/redist/pom.xml | 8 +- .../pom.xml | 10 +- .../samples/simple/Simple.java | 4 +- .../samples/simple/pom.xml | 6 +- .../global/tritondevelopertoolsserver.java | 5767 +++++++++++++++++ .../Allocator.java | 6 +- .../BackendConfig.java | 6 +- .../CUDAMemoryPoolByteSize.java | 6 +- .../GenericInferRequest.java | 6 +- .../GenericInferResult.java | 8 +- .../GenericTritonServer.java | 11 +- .../HostPolicy.java | 6 +- .../InferOptions.java | 6 +- .../LoggingOptions.java | 6 +- .../MetricsOptions.java | 6 +- .../ModelLoadGPULimit.java | 6 +- .../NewModelRepo.java | 6 +- .../OutputBufferReleaseFn_t.java | 6 +- .../RateLimitResource.java | 6 +- .../RepositoryIndex.java | 6 +- .../ResponseAllocatorAllocFn_t.java | 6 +- .../ResponseAllocatorStartFn_t.java | 6 +- .../ServerOptions.java | 6 +- .../StringSet.java | 6 +- .../StringVector.java | 6 +- .../TRITONBACKEND_Backend.java | 17 + .../TRITONBACKEND_BackendAttribute.java | 17 + .../TRITONBACKEND_Batcher.java | 23 + .../TRITONBACKEND_Input.java | 17 + .../TRITONBACKEND_MemoryManager.java | 20 + .../TRITONBACKEND_Model.java | 17 + .../TRITONBACKEND_ModelInstance.java | 17 + .../TRITONBACKEND_Output.java | 17 + .../TRITONBACKEND_Request.java | 17 + .../TRITONBACKEND_Response.java | 17 + .../TRITONBACKEND_ResponseFactory.java | 17 + .../TRITONBACKEND_State.java | 17 + .../TRITONREPOAGENT_Agent.java | 20 + .../TRITONREPOAGENT_AgentModel.java | 23 + .../TRITONSERVER_BufferAttributes.java | 20 + .../TRITONSERVER_Error.java | 17 + .../TRITONSERVER_InferenceRequest.java | 17 + ...TONSERVER_InferenceRequestReleaseFn_t.java | 46 + .../TRITONSERVER_InferenceResponse.java | 17 + ...NSERVER_InferenceResponseCompleteFn_t.java | 41 + .../TRITONSERVER_InferenceTrace.java | 17 + ...ITONSERVER_InferenceTraceActivityFn_t.java | 29 + ...RITONSERVER_InferenceTraceReleaseFn_t.java | 30 + ...RVER_InferenceTraceTensorActivityFn_t.java | 31 + .../TRITONSERVER_Message.java | 17 + .../TRITONSERVER_Metric.java | 17 + .../TRITONSERVER_MetricFamily.java | 23 + .../TRITONSERVER_Metrics.java | 17 + .../TRITONSERVER_Parameter.java | 17 + .../TRITONSERVER_ResponseAllocator.java | 17 + ...ITONSERVER_ResponseAllocatorAllocFn_t.java | 63 + ...ResponseAllocatorBufferAttributesFn_t.java | 47 + ...ITONSERVER_ResponseAllocatorQueryFn_t.java | 49 + ...ONSERVER_ResponseAllocatorReleaseFn_t.java | 42 + ...ITONSERVER_ResponseAllocatorStartFn_t.java | 37 + .../TRITONSERVER_Server.java | 17 + .../TRITONSERVER_ServerOptions.java | 17 + .../tritondevelopertoolsserver}/Tensor.java | 6 +- .../tritondevelopertoolsserver}/Trace.java | 6 +- .../TritonException.java | 6 +- .../presets/tritondevelopertoolsserver.java | 14 +- .../src/main/java9/module-info.java | 6 + .../global/tritonserverwrapper.java | 186 - .../src/main/java9/module-info.java | 6 - 74 files changed, 6777 insertions(+), 294 deletions(-) rename {tritonserverwrapper => tritondevelopertoolsserver}/README.md (100%) rename {tritonserverwrapper => tritondevelopertoolsserver}/cppbuild.sh (93%) mode change 100755 => 100644 rename {tritonserverwrapper => tritondevelopertoolsserver}/platform/pom.xml (95%) rename {tritonserverwrapper => tritondevelopertoolsserver}/platform/redist/pom.xml (93%) rename {tritonserverwrapper => tritondevelopertoolsserver}/pom.xml (89%) rename {tritonserverwrapper => tritondevelopertoolsserver}/samples/simple/Simple.java (97%) rename {tritonserverwrapper => tritondevelopertoolsserver}/samples/simple/pom.xml (75%) create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/Allocator.java (85%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/BackendConfig.java (88%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/CUDAMemoryPoolByteSize.java (84%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/GenericInferRequest.java (90%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/GenericInferResult.java (92%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/GenericTritonServer.java (94%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/HostPolicy.java (89%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/InferOptions.java (96%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/LoggingOptions.java (94%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/MetricsOptions.java (90%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/ModelLoadGPULimit.java (82%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/NewModelRepo.java (90%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/OutputBufferReleaseFn_t.java (71%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/RateLimitResource.java (89%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/RepositoryIndex.java (90%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/ResponseAllocatorAllocFn_t.java (79%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/ResponseAllocatorStartFn_t.java (67%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/ServerOptions.java (98%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/StringSet.java (84%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/StringVector.java (93%) create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java create mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/Tensor.java (95%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/Trace.java (94%) rename {tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper => tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver}/TritonException.java (79%) rename tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java => tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java (83%) create mode 100644 tritondevelopertoolsserver/src/main/java9/module-info.java delete mode 100644 tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java delete mode 100644 tritonserverwrapper/src/main/java9/module-info.java diff --git a/platform/pom.xml b/platform/pom.xml index e506964640f..3417c26647d 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,7 +61,7 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform - ../tritonserverwrapper/platform + ../tritondevelopertoolsserver/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -316,8 +316,8 @@ org.bytedeco - tritonserverwrapper-platform - 2.24-${project.version} + tritondevelopertoolsserver-platform + 2.35-${project.version} org.bytedeco diff --git a/pom.xml b/pom.xml index 74711303ea9..a1959109e4c 100644 --- a/pom.xml +++ b/pom.xml @@ -623,7 +623,7 @@ tensorflow-lite tensorrt tritonserver - tritonserverwrapper + tritondevelopertoolsserver ale depthai onnx @@ -1407,7 +1407,7 @@ tensorflow-lite tensorrt tritonserver - tritonserverwrapper + tritondevelopertoolsserver ale depthai onnx diff --git a/tritonserverwrapper/README.md b/tritondevelopertoolsserver/README.md similarity index 100% rename from tritonserverwrapper/README.md rename to tritondevelopertoolsserver/README.md diff --git a/tritonserverwrapper/cppbuild.sh b/tritondevelopertoolsserver/cppbuild.sh old mode 100755 new mode 100644 similarity index 93% rename from tritonserverwrapper/cppbuild.sh rename to tritondevelopertoolsserver/cppbuild.sh index 712186b0138..958f7730390 --- a/tritonserverwrapper/cppbuild.sh +++ b/tritondevelopertoolsserver/cppbuild.sh @@ -2,7 +2,7 @@ # This file is meant to be included by the parent cppbuild.sh script if [[ -z "$PLATFORM" ]]; then pushd .. - bash cppbuild.sh "$@" tritonserverwrapper + bash cppbuild.sh "$@" tritondevelopertoolsserver popd exit fi diff --git a/tritonserverwrapper/platform/pom.xml b/tritondevelopertoolsserver/platform/pom.xml similarity index 95% rename from tritonserverwrapper/platform/pom.xml rename to tritondevelopertoolsserver/platform/pom.xml index e514ce47c7d..ee34c8727d8 100644 --- a/tritonserverwrapper/platform/pom.xml +++ b/tritondevelopertoolsserver/platform/pom.xml @@ -11,12 +11,12 @@ org.bytedeco - tritonserverwrapper-platform - 2.26-${project.parent.version} + tritondevelopertoolsserver-platform + 2.35-${project.parent.version} JavaCPP Presets Platform for Triton Inference Server Wrapper - tritonserverwrapper + tritondevelopertoolsserver diff --git a/tritonserverwrapper/platform/redist/pom.xml b/tritondevelopertoolsserver/platform/redist/pom.xml similarity index 93% rename from tritonserverwrapper/platform/redist/pom.xml rename to tritondevelopertoolsserver/platform/redist/pom.xml index edc9651a14b..28a8c1f2c75 100644 --- a/tritonserverwrapper/platform/redist/pom.xml +++ b/tritondevelopertoolsserver/platform/redist/pom.xml @@ -6,17 +6,17 @@ org.bytedeco javacpp-presets - 1.5.8-SNAPSHOT + 1.5.9-SNAPSHOT ../../../ org.bytedeco - tritonserverwrapper-platform-redist - 2.26-${project.parent.version} + tritondevelopertoolsserver-platform-redist + 2.35-${project.parent.version} JavaCPP Presets Platform Redist for Triton Inference Server - tritonserverwrapper + tritondevelopertoolsserver -redist diff --git a/tritonserverwrapper/pom.xml b/tritondevelopertoolsserver/pom.xml similarity index 89% rename from tritonserverwrapper/pom.xml rename to tritondevelopertoolsserver/pom.xml index dad9346d56a..42dcde40f77 100644 --- a/tritonserverwrapper/pom.xml +++ b/tritondevelopertoolsserver/pom.xml @@ -10,8 +10,8 @@ org.bytedeco - tritonserverwrapper - 2.26-${project.parent.version} + tritondevelopertoolsserver + 2.35-${project.parent.version} JavaCPP Presets for Triton Inference Server Wrapper @@ -48,7 +48,7 @@ ${javacpp.platform} - org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* + org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/*jni* META-INF/native-image/${javacpp.platform}/ @@ -63,11 +63,11 @@ ${javacpp.platform}-redist ${project.build.directory}/native - org/bytedeco/tritonserverwrapper/${javacpp.platform}/ + org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/ META-INF/native-image/${javacpp.platform}/ - org/bytedeco/tritonserverwrapper/${javacpp.platform}/*jni* + org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/*jni* diff --git a/tritonserverwrapper/samples/simple/Simple.java b/tritondevelopertoolsserver/samples/simple/Simple.java similarity index 97% rename from tritonserverwrapper/samples/simple/Simple.java rename to tritondevelopertoolsserver/samples/simple/Simple.java index 837e22b1977..9187ece0f80 100644 --- a/tritonserverwrapper/samples/simple/Simple.java +++ b/tritondevelopertoolsserver/samples/simple/Simple.java @@ -29,8 +29,8 @@ import java.util.concurrent.*; import com.google.gson.*; import org.bytedeco.javacpp.*; -import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; public class Simple { // Helper functions diff --git a/tritonserverwrapper/samples/simple/pom.xml b/tritondevelopertoolsserver/samples/simple/pom.xml similarity index 75% rename from tritonserverwrapper/samples/simple/pom.xml rename to tritondevelopertoolsserver/samples/simple/pom.xml index 1fc3f5107ab..325ee49f7db 100644 --- a/tritonserverwrapper/samples/simple/pom.xml +++ b/tritondevelopertoolsserver/samples/simple/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - org.bytedeco.tritonserverwrapper + org.bytedeco.tritondevelopertoolsserver simple 1.5.9-SNAPSHOT @@ -11,8 +11,8 @@ org.bytedeco - tritonserverwrapper-platform - 2.26-1.5.9-SNAPSHOT + tritondevelopertoolsserver-platform + 2.35-1.5.9-SNAPSHOT shaded diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java new file mode 100644 index 00000000000..eb34d14bad0 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java @@ -0,0 +1,5767 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.global; + +import org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver.*; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +public class tritondevelopertoolsserver extends org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver { + static { Loader.load(); } + +// Targeting ../tritondevelopertoolsserver/StringSet.java + + +// Targeting ../tritondevelopertoolsserver/StringVector.java + + +// Parsed from common.h + +// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include + +//============================================================================== +/** enum classes + * */ +/** enum class triton::developer_tools::server::ModelControlMode */ +public static final int NONE = 0, POLL = 1, EXPLICIT = 2; +/** enum class triton::developer_tools::server::MemoryType */ +public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; +/** enum class triton::developer_tools::server::DataType */ +public static final int + INVALID = 0, + BOOL = 1, + UINT8 = 2, + UINT16 = 3, + UINT32 = 4, + UINT64 = 5, + INT8 = 6, + INT16 = 7, + INT32 = 8, + INT64 = 9, + FP16 = 10, + FP32 = 11, + FP64 = 12, + BYTES = 13, + BF16 = 14; +/** enum class triton::developer_tools::server::ModelReadyState */ +public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; +// Targeting ../tritondevelopertoolsserver/TritonException.java + + +// Targeting ../tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritondevelopertoolsserver/OutputBufferReleaseFn_t.java + + +// Targeting ../tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java + + +// Targeting ../tritondevelopertoolsserver/LoggingOptions.java + + +// Targeting ../tritondevelopertoolsserver/MetricsOptions.java + + +// Targeting ../tritondevelopertoolsserver/RateLimitResource.java + + +// Targeting ../tritondevelopertoolsserver/ModelLoadGPULimit.java + + +// Targeting ../tritondevelopertoolsserver/Allocator.java + + +// Targeting ../tritondevelopertoolsserver/BackendConfig.java + + +// Targeting ../tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritondevelopertoolsserver/HostPolicy.java + + +// Targeting ../tritondevelopertoolsserver/Trace.java + + +// Targeting ../tritondevelopertoolsserver/ServerOptions.java + + +// Targeting ../tritondevelopertoolsserver/RepositoryIndex.java + + +// Targeting ../tritondevelopertoolsserver/Tensor.java + + +// Targeting ../tritondevelopertoolsserver/NewModelRepo.java + + +// Targeting ../tritondevelopertoolsserver/InferOptions.java + + + + // namespace triton::developer_tools::server + + +// Parsed from generic_server_wrapper.h + +// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include +// #include "../src/infer_requested_output.h" +// #include "../src/tracer.h" +// #include "common.h" + +/// +// Targeting ../tritondevelopertoolsserver/GenericTritonServer.java + + +// Targeting ../tritondevelopertoolsserver/GenericInferResult.java + + +// Targeting ../tritondevelopertoolsserver/GenericInferRequest.java + + + + // namespace triton::developer_tools::server + + +// Parsed from tritonserver.h + +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once + +/** \file */ + +// #include +// #include +// #include + +// #ifdef __cplusplus +// #endif + +// #ifdef _COMPILING_TRITONSERVER +// #if defined(_MSC_VER) +// #define TRITONSERVER_DECLSPEC __declspec(dllexport) +// #elif defined(__GNUC__) +// #define TRITONSERVER_DECLSPEC __attribute__((__visibility__("default"))) +// #else +// #define TRITONSERVER_DECLSPEC +// #endif +// #else +// #if defined(_MSC_VER) +// #define TRITONSERVER_DECLSPEC __declspec(dllimport) +// #else +// #define TRITONSERVER_DECLSPEC +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Error.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Message.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Metrics.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Parameter.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Server.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Metric.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java + + + +/** + * TRITONSERVER API Version + * + * The TRITONSERVER API is versioned with major and minor version + * numbers. Any change to the API that does not impact backwards + * compatibility (for example, adding a non-required function) + * increases the minor version number. Any change that breaks + * backwards compatibility (for example, deleting or changing the + * behavior of a function) increases the major version number. A + * client should check that the API version used to compile the + * client is compatible with the API version of the Triton shared + * library that it is linking against. This is typically done by code + * similar to the following which makes sure that the major versions + * are equal and that the minor version of the Triton shared library + * is >= the minor version used to build the client. + * + * uint32_t api_version_major, api_version_minor; + * TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor); + * if ((api_version_major != TRITONSERVER_API_VERSION_MAJOR) || + * (api_version_minor < TRITONSERVER_API_VERSION_MINOR)) { + * return TRITONSERVER_ErrorNew( + * TRITONSERVER_ERROR_UNSUPPORTED, + * "triton server API version does not support this client"); + * } + * */ +public static final int TRITONSERVER_API_VERSION_MAJOR = 1; + +/// +public static final int TRITONSERVER_API_VERSION_MINOR = 22; + +/** Get the TRITONBACKEND API version supported by the Triton shared + * library. This value can be compared against the + * TRITONSERVER_API_VERSION_MAJOR and TRITONSERVER_API_VERSION_MINOR + * used to build the client to ensure that Triton shared library is + * compatible with the client. + * + * @param major Returns the TRITONSERVER API major version supported + * by Triton. + * @param minor Returns the TRITONSERVER API minor version supported + * by Triton. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( + @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); +public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( + @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); +public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( + @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); + +/** TRITONSERVER_DataType + * + * Tensor data types recognized by TRITONSERVER. + * */ +/** enum TRITONSERVER_DataType */ +public static final int + TRITONSERVER_TYPE_INVALID = 0, + TRITONSERVER_TYPE_BOOL = 1, + TRITONSERVER_TYPE_UINT8 = 2, + TRITONSERVER_TYPE_UINT16 = 3, + TRITONSERVER_TYPE_UINT32 = 4, + TRITONSERVER_TYPE_UINT64 = 5, + TRITONSERVER_TYPE_INT8 = 6, + TRITONSERVER_TYPE_INT16 = 7, + TRITONSERVER_TYPE_INT32 = 8, + TRITONSERVER_TYPE_INT64 = 9, + TRITONSERVER_TYPE_FP16 = 10, + TRITONSERVER_TYPE_FP32 = 11, + TRITONSERVER_TYPE_FP64 = 12, + TRITONSERVER_TYPE_BYTES = 13, + TRITONSERVER_TYPE_BF16 = 14; + +/** Get the string representation of a data type. The returned string + * is not owned by the caller and so should not be modified or freed. + * + * @param datatype The data type. + * @return The string representation of the data type. */ + +/// +public static native String TRITONSERVER_DataTypeString( + @Cast("TRITONSERVER_DataType") int datatype); + +/** Get the Triton datatype corresponding to a string representation + * of a datatype. + * + * @param dtype The datatype string representation. + * @return The Triton data type or TRITONSERVER_TYPE_INVALID if the + * string does not represent a data type. */ + +/// +public static native @Cast("TRITONSERVER_DataType") int TRITONSERVER_StringToDataType(String dtype); +public static native @Cast("TRITONSERVER_DataType") int TRITONSERVER_StringToDataType(@Cast("const char*") BytePointer dtype); + +/** Get the size of a Triton datatype in bytes. Zero is returned for + * TRITONSERVER_TYPE_BYTES because it have variable size. Zero is + * returned for TRITONSERVER_TYPE_INVALID. + * + * @param dtype The datatype. + * @return The size of the datatype. */ + +/// +/// +public static native @Cast("uint32_t") int TRITONSERVER_DataTypeByteSize(@Cast("TRITONSERVER_DataType") int datatype); + +/** TRITONSERVER_MemoryType + * + * Types of memory recognized by TRITONSERVER. + * */ +/** enum TRITONSERVER_MemoryType */ +public static final int + TRITONSERVER_MEMORY_CPU = 0, + TRITONSERVER_MEMORY_CPU_PINNED = 1, + TRITONSERVER_MEMORY_GPU = 2; + +/** Get the string representation of a memory type. The returned + * string is not owned by the caller and so should not be modified or + * freed. + * + * @param memtype The memory type. + * @return The string representation of the memory type. */ + +/// +/// +public static native String TRITONSERVER_MemoryTypeString( + @Cast("TRITONSERVER_MemoryType") int memtype); + +/** TRITONSERVER_ParameterType + * + * Types of parameters recognized by TRITONSERVER. + * */ +/** enum TRITONSERVER_ParameterType */ +public static final int + TRITONSERVER_PARAMETER_STRING = 0, + TRITONSERVER_PARAMETER_INT = 1, + TRITONSERVER_PARAMETER_BOOL = 2, + TRITONSERVER_PARAMETER_BYTES = 3; + +/** Get the string representation of a parameter type. The returned + * string is not owned by the caller and so should not be modified or + * freed. + * + * @param paramtype The parameter type. + * @return The string representation of the parameter type. */ + +/// +public static native String TRITONSERVER_ParameterTypeString( + @Cast("TRITONSERVER_ParameterType") int paramtype); + +/** Create a new parameter object. The caller takes ownership of the + * TRITONSERVER_Parameter object and must call TRITONSERVER_ParameterDelete to + * release the object. The object will maintain its own copy of the 'value' + * + * @param name The parameter name. + * @param type The parameter type. + * @param value The pointer to the value. + * @return A new TRITONSERVER_Parameter object. 'nullptr' will be returned if + * 'type' is 'TRITONSERVER_PARAMETER_BYTES'. The caller should use + * TRITONSERVER_ParameterBytesNew to create parameter with bytes type. */ + +/// +public static native TRITONSERVER_Parameter TRITONSERVER_ParameterNew( + String name, @Cast("const TRITONSERVER_ParameterType") int type, @Const Pointer value); +public static native TRITONSERVER_Parameter TRITONSERVER_ParameterNew( + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_ParameterType") int type, @Const Pointer value); + +/** Create a new parameter object with type TRITONSERVER_PARAMETER_BYTES. + * The caller takes ownership of the TRITONSERVER_Parameter object and must + * call TRITONSERVER_ParameterDelete to release the object. The object only + * maintains a shallow copy of the 'byte_ptr' so the data content must be + * valid until the parameter object is deleted. + * + * @param name The parameter name. + * @param byte_ptr The pointer to the data content. + * @param size The size of the data content. + * @return A new TRITONSERVER_Error object. */ + +/// +public static native TRITONSERVER_Parameter TRITONSERVER_ParameterBytesNew( + String name, @Const Pointer byte_ptr, @Cast("const uint64_t") long size); +public static native TRITONSERVER_Parameter TRITONSERVER_ParameterBytesNew( + @Cast("const char*") BytePointer name, @Const Pointer byte_ptr, @Cast("const uint64_t") long size); + +/** Delete an parameter object. + * + * @param parameter The parameter object. */ + +/// +/// +public static native void TRITONSERVER_ParameterDelete( + TRITONSERVER_Parameter parameter); + +/** TRITONSERVER_InstanceGroupKind + * + * Kinds of instance groups recognized by TRITONSERVER. + * */ +/** enum TRITONSERVER_InstanceGroupKind */ +public static final int + TRITONSERVER_INSTANCEGROUPKIND_AUTO = 0, + TRITONSERVER_INSTANCEGROUPKIND_CPU = 1, + TRITONSERVER_INSTANCEGROUPKIND_GPU = 2, + TRITONSERVER_INSTANCEGROUPKIND_MODEL = 3; + +/** Get the string representation of an instance-group kind. The + * returned string is not owned by the caller and so should not be + * modified or freed. + * + * @param kind The instance-group kind. + * @return The string representation of the kind. */ + +/// +/// +public static native String TRITONSERVER_InstanceGroupKindString( + @Cast("TRITONSERVER_InstanceGroupKind") int kind); + +/** TRITONSERVER_Logging + * + * Types/levels of logging. + * */ +/** enum TRITONSERVER_LogLevel */ +public static final int + TRITONSERVER_LOG_INFO = 0, + TRITONSERVER_LOG_WARN = 1, + TRITONSERVER_LOG_ERROR = 2, + TRITONSERVER_LOG_VERBOSE = 3; + +/** + * Format of logging. + * + * TRITONSERVER_LOG_DEFAULT: the log severity (L) and timestamp will be + * logged as "LMMDD hh:mm:ss.ssssss". + * + * TRITONSERVER_LOG_ISO8601: the log format will be "YYYY-MM-DDThh:mm:ssZ L". + * */ +/** enum TRITONSERVER_LogFormat */ +public static final int + TRITONSERVER_LOG_DEFAULT = 0, + TRITONSERVER_LOG_ISO8601 = 1; + +/** Is a log level enabled? + * + * @param level The log level. + * @return True if the log level is enabled, false if not enabled. */ + +/// +public static native @Cast("bool") boolean TRITONSERVER_LogIsEnabled( + @Cast("TRITONSERVER_LogLevel") int level); + +/** Log a message at a given log level if that level is enabled. + * + * @param level The log level. + * @param filename The file name of the location of the log message. + * @param line The line number of the log message. + * @param msg The log message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_LogMessage( + @Cast("TRITONSERVER_LogLevel") int level, String filename, int line, + String msg); +public static native TRITONSERVER_Error TRITONSERVER_LogMessage( + @Cast("TRITONSERVER_LogLevel") int level, @Cast("const char*") BytePointer filename, int line, + @Cast("const char*") BytePointer msg); + +/** TRITONSERVER_Error + * + * Errors are reported by a TRITONSERVER_Error object. A NULL + * TRITONSERVER_Error indicates no error, a non-NULL TRITONSERVER_Error + * indicates error and the code and message for the error can be + * retrieved from the object. + * + * The caller takes ownership of a TRITONSERVER_Error object returned by + * the API and must call TRITONSERVER_ErrorDelete to release the object. + * +

+ * The TRITONSERVER_Error error codes */ +/** enum TRITONSERVER_Error_Code */ +public static final int + TRITONSERVER_ERROR_UNKNOWN = 0, + TRITONSERVER_ERROR_INTERNAL = 1, + TRITONSERVER_ERROR_NOT_FOUND = 2, + TRITONSERVER_ERROR_INVALID_ARG = 3, + TRITONSERVER_ERROR_UNAVAILABLE = 4, + TRITONSERVER_ERROR_UNSUPPORTED = 5, + TRITONSERVER_ERROR_ALREADY_EXISTS = 6; + +/** Create a new error object. The caller takes ownership of the + * TRITONSERVER_Error object and must call TRITONSERVER_ErrorDelete to + * release the object. + * + * @param code The error code. + * @param msg The error message. + * @return A new TRITONSERVER_Error object. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ErrorNew( + @Cast("TRITONSERVER_Error_Code") int code, String msg); +public static native TRITONSERVER_Error TRITONSERVER_ErrorNew( + @Cast("TRITONSERVER_Error_Code") int code, @Cast("const char*") BytePointer msg); + +/** Delete an error object. + * + * @param error The error object. */ + +/// +public static native void TRITONSERVER_ErrorDelete(TRITONSERVER_Error error); + +/** Get the error code. + * + * @param error The error object. + * @return The error code. */ + +/// +public static native @Cast("TRITONSERVER_Error_Code") int TRITONSERVER_ErrorCode(TRITONSERVER_Error error); + +/** Get the string representation of an error code. The returned + * string is not owned by the caller and so should not be modified or + * freed. The lifetime of the returned string extends only as long as + * 'error' and must not be accessed once 'error' is deleted. + * + * @param error The error object. + * @return The string representation of the error code. */ + +/// +public static native String TRITONSERVER_ErrorCodeString( + TRITONSERVER_Error error); + +/** Get the error message. The returned string is not owned by the + * caller and so should not be modified or freed. The lifetime of the + * returned string extends only as long as 'error' and must not be + * accessed once 'error' is deleted. + * + * @param error The error object. + * @return The error message. */ + +/// +/// +/// +public static native String TRITONSERVER_ErrorMessage( + TRITONSERVER_Error error); +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java + + + +/** Create a new response allocator object. + * + * The response allocator object is used by Triton to allocate + * buffers to hold the output tensors in inference responses. Most + * models generate a single response for each inference request + * (TRITONSERVER_TXN_ONE_TO_ONE). For these models the order of + * callbacks will be: + * + * TRITONSERVER_ServerInferAsync called + * - start_fn : optional (and typically not required) + * - alloc_fn : called once for each output tensor in response + * TRITONSERVER_InferenceResponseDelete called + * - release_fn: called once for each output tensor in response + * + * For models that generate multiple responses for each inference + * request (TRITONSERVER_TXN_DECOUPLED), the start_fn callback can be + * used to determine sets of alloc_fn callbacks that belong to the + * same response: + * + * TRITONSERVER_ServerInferAsync called + * - start_fn + * - alloc_fn : called once for each output tensor in response + * - start_fn + * - alloc_fn : called once for each output tensor in response + * ... + * For each response, TRITONSERVER_InferenceResponseDelete called + * - release_fn: called once for each output tensor in the response + * + * In all cases the start_fn, alloc_fn and release_fn callback + * functions must be thread-safe. Typically making these functions + * thread-safe does not require explicit locking. The recommended way + * to implement these functions is to have each inference request + * provide a 'response_allocator_userp' object that is unique to that + * request with TRITONSERVER_InferenceRequestSetResponseCallback. The + * callback functions then operate only on this unique state. Locking + * is required only when the callback function needs to access state + * that is shared across inference requests (for example, a common + * allocation pool). + * + * @param allocator Returns the new response allocator object. + * @param alloc_fn The function to call to allocate buffers for result + * tensors. + * @param release_fn The function to call when the server no longer + * holds a reference to an allocated buffer. + * @param start_fn The function to call to indicate that the + * subsequent 'alloc_fn' calls are for a new response. This callback + * is optional (use nullptr to indicate that it should not be + * invoked). + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( + @Cast("TRITONSERVER_ResponseAllocator**") PointerPointer allocator, + TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + TRITONSERVER_ResponseAllocatorStartFn_t start_fn); +public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( + @ByPtrPtr TRITONSERVER_ResponseAllocator allocator, + TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + TRITONSERVER_ResponseAllocatorStartFn_t start_fn); + +/** Set the buffer attributes function for a response allocator object. + * The function will be called after alloc_fn to set the buffer attributes + * associated with the output buffer. + * + * The thread-safy requirement for buffer_attributes_fn is the same as other + * allocator callbacks. + * + * @param allocator The response allocator object. + * @param buffer_attributes_fn The function to call to get the buffer + * attributes information for an allocated buffer. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( + TRITONSERVER_ResponseAllocator allocator, + TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); + +/** Set the query function to a response allocator object. Usually the + * function will be called before alloc_fn to understand what is the + * allocator's preferred memory type and memory type ID at the current + * situation to make different execution decision. + * + * The thread-safy requirement for query_fn is the same as other allocator + * callbacks. + * + * @param allocator The response allocator object. + * @param query_fn The function to call to query allocator's preferred memory + * type and memory type ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetQueryFunction( + TRITONSERVER_ResponseAllocator allocator, + TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); + +/** Delete a response allocator. + * + * @param allocator The response allocator object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorDelete( + TRITONSERVER_ResponseAllocator allocator); + +/** TRITONSERVER_Message + * + * Object representing a Triton Server message. + * +

+ * Create a new message object from serialized JSON string. + * + * @param message The message object. + * @param base The base of the serialized JSON. + * @param byte_size The size, in bytes, of the serialized message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( + @Cast("TRITONSERVER_Message**") PointerPointer message, String base, @Cast("size_t") long byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( + @ByPtrPtr TRITONSERVER_Message message, String base, @Cast("size_t") long byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( + @ByPtrPtr TRITONSERVER_Message message, @Cast("const char*") BytePointer base, @Cast("size_t") long byte_size); + +/** Delete a message object. + * + * @param message The message object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MessageDelete( + TRITONSERVER_Message message); + +/** Get the base and size of the buffer containing the serialized + * message in JSON format. The buffer is owned by the + * TRITONSERVER_Message object and should not be modified or freed by + * the caller. The lifetime of the buffer extends only as long as + * 'message' and must not be accessed once 'message' is deleted. + * + * @param message The message object. + * @param base Returns the base of the serialized message. + * @param byte_size Returns the size, in bytes, of the serialized + * message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( + TRITONSERVER_Message message, @Cast("const char**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( + TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr BytePointer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( + TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr ByteBuffer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( + TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr byte[] base, @Cast("size_t*") SizeTPointer byte_size); + +/** TRITONSERVER_Metrics + * + * Object representing metrics. + * +

+ * Metric format types */ +/** enum TRITONSERVER_MetricFormat */ +public static final int + TRITONSERVER_METRIC_PROMETHEUS = 0; + +/** Delete a metrics object. + * + * @param metrics The metrics object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricsDelete( + TRITONSERVER_Metrics metrics); + +/** Get a buffer containing the metrics in the specified format. For + * each format the buffer contains the following: + * + * TRITONSERVER_METRIC_PROMETHEUS: 'base' points to a single multiline + * string (char*) that gives a text representation of the metrics in + * prometheus format. 'byte_size' returns the length of the string + * in bytes. + * + * The buffer is owned by the 'metrics' object and should not be + * modified or freed by the caller. The lifetime of the buffer + * extends only as long as 'metrics' and must not be accessed once + * 'metrics' is deleted. + * + * @param metrics The metrics object. + * @param format The format to use for the returned metrics. + * @param base Returns a pointer to the base of the formatted + * metrics, as described above. + * @param byte_size Returns the size, in bytes, of the formatted + * metrics. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( + TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, + @Cast("const char**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( + TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, + @Cast("const char**") @ByPtrPtr BytePointer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( + TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, + @Cast("const char**") @ByPtrPtr ByteBuffer base, @Cast("size_t*") SizeTPointer byte_size); +public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( + TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, + @Cast("const char**") @ByPtrPtr byte[] base, @Cast("size_t*") SizeTPointer byte_size); + +/** TRITONSERVER_InferenceTrace + * + * Object that represents tracing for an inference request. + * +

+ * Trace levels. The trace level controls the type of trace + * activities that are reported for an inference request. + * + * Trace level values are power-of-2 and can be combined to trace + * multiple types of activities. For example, use + * (TRITONSERVER_TRACE_LEVEL_TIMESTAMPS | + * TRITONSERVER_TRACE_LEVEL_TENSORS) to trace both timestamps and + * tensors for an inference request. + * + * TRITONSERVER_TRACE_LEVEL_MIN and TRITONSERVER_TRACE_LEVEL_MAX are + * deprecated and should not be used. */ +/** enum TRITONSERVER_InferenceTraceLevel */ +public static final int + /** Tracing disabled. No trace activities are reported. */ + TRITONSERVER_TRACE_LEVEL_DISABLED = 0, + /** Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. */ + TRITONSERVER_TRACE_LEVEL_MIN = 1, + /** Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. */ + TRITONSERVER_TRACE_LEVEL_MAX = 2, + /** Record timestamps for the inference request. */ + TRITONSERVER_TRACE_LEVEL_TIMESTAMPS = 0x4, + /** Record input and output tensor values for the inference request. */ + TRITONSERVER_TRACE_LEVEL_TENSORS = 0x8; + +/** Get the string representation of a trace level. The returned + * string is not owned by the caller and so should not be modified or + * freed. + * + * @param level The trace level. + * @return The string representation of the trace level. */ +public static native String TRITONSERVER_InferenceTraceLevelString( + @Cast("TRITONSERVER_InferenceTraceLevel") int level); + +/** Trace activities */ +/** enum TRITONSERVER_InferenceTraceActivity */ +public static final int + TRITONSERVER_TRACE_REQUEST_START = 0, + TRITONSERVER_TRACE_QUEUE_START = 1, + TRITONSERVER_TRACE_COMPUTE_START = 2, + TRITONSERVER_TRACE_COMPUTE_INPUT_END = 3, + TRITONSERVER_TRACE_COMPUTE_OUTPUT_START = 4, + TRITONSERVER_TRACE_COMPUTE_END = 5, + TRITONSERVER_TRACE_REQUEST_END = 6, + TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT = 7, + TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT = 8, + TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT = 9; + +/** Get the string representation of a trace activity. The returned + * string is not owned by the caller and so should not be modified or + * freed. + * + * @param activity The trace activity. + * @return The string representation of the trace activity. */ +public static native String TRITONSERVER_InferenceTraceActivityString( + @Cast("TRITONSERVER_InferenceTraceActivity") int activity); +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java + + + +/** Create a new inference trace object. The caller takes ownership of + * the TRITONSERVER_InferenceTrace object and must call + * TRITONSERVER_InferenceTraceDelete to release the object. + * + * The activity callback function will be called to report activity + * for 'trace' as well as for any child traces that are spawned by + * 'trace', and so the activity callback must check the trace object + * to determine specifically what activity is being reported. + * + * The release callback is called for both 'trace' and for any child + * traces spawned by 'trace'. + * + * @param trace Returns the new inference trace object. + * @param level The tracing level. + * @param parent_id The parent trace id for this trace. A value of 0 + * indicates that there is not parent trace. + * @param activity_fn The callback function where activity for the + * trace is reported. + * @param release_fn The callback function called when all activity + * is complete for the trace. + * @param trace_userp User-provided pointer that is delivered to + * the activity and release callback functions. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceNew( + @Cast("TRITONSERVER_InferenceTrace**") PointerPointer trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, + @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, + TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceNew( + @ByPtrPtr TRITONSERVER_InferenceTrace trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, + @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, + TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); + +/** Create a new inference trace object. The caller takes ownership of + * the TRITONSERVER_InferenceTrace object and must call + * TRITONSERVER_InferenceTraceDelete to release the object. + * + * The timeline and tensor activity callback function will be called to report + * activity for 'trace' as well as for any child traces that are spawned by + * 'trace', and so the activity callback must check the trace object + * to determine specifically what activity is being reported. + * + * The release callback is called for both 'trace' and for any child + * traces spawned by 'trace'. + * + * @param trace Returns the new inference trace object. + * @param level The tracing level. + * @param parent_id The parent trace id for this trace. A value of 0 + * indicates that there is not parent trace. + * @param activity_fn The callback function where timeline activity for the + * trace is reported. + * @param tensor_activity_fn The callback function where tensor activity for + * the trace is reported. + * @param release_fn The callback function called when all activity + * is complete for the trace. + * @param trace_userp User-provided pointer that is delivered to + * the activity and release callback functions. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceTensorNew( + @Cast("TRITONSERVER_InferenceTrace**") PointerPointer trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, + @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, + TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, + TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceTensorNew( + @ByPtrPtr TRITONSERVER_InferenceTrace trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, + @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, + TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, + TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); + +/** Delete a trace object. + * + * @param trace The trace object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceDelete( + TRITONSERVER_InferenceTrace trace); + +/** Get the id associated with a trace. Every trace is assigned an id + * that is unique across all traces created for a Triton server. + * + * @param trace The trace. + * @param id Returns the id associated with the trace. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongPointer id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongBuffer id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") long[] id); + +/** Get the parent id associated with a trace. The parent id indicates + * a parent-child relationship between two traces. A parent id value + * of 0 indicates that there is no parent trace. + * + * @param trace The trace. + * @param id Returns the parent id associated with the trace. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongPointer parent_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongBuffer parent_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( + TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") long[] parent_id); + +/** Get the name of the model associated with a trace. The caller does + * not own the returned string and must not modify or delete it. The + * lifetime of the returned string extends only as long as 'trace'. + * + * @param trace The trace. + * @param model_name Returns the name of the model associated with + * the trace. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") PointerPointer model_name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr BytePointer model_name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr ByteBuffer model_name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr byte[] model_name); + +/** Get the version of the model associated with a trace. + * + * @param trace The trace. + * @param model_version Returns the version of the model associated + * with the trace. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( + TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") LongPointer model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( + TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") LongBuffer model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( + TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") long[] model_version); + +/** Get the request id associated with a trace. The caller does + * not own the returned string and must not modify or delete it. The + * lifetime of the returned string extends only as long as 'trace'. + * + * @param trace The trace. + * @param request_id Returns the version of the model associated + * with the trace. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") PointerPointer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr BytePointer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr ByteBuffer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( + TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr byte[] request_id); + +/** TRITONSERVER_InferenceRequest + * + * Object representing an inference request. The inference request + * provides the meta-data and input tensor values needed for an + * inference and returns the inference result meta-data and output + * tensors. An inference request object can be modified and reused + * multiple times. + * +

+ * Inference request flags. The enum values must be power-of-2 values. */ +/** enum TRITONSERVER_RequestFlag */ +public static final int + TRITONSERVER_REQUEST_FLAG_SEQUENCE_START = 1, + TRITONSERVER_REQUEST_FLAG_SEQUENCE_END = 2; + +/** Inference request release flags. The enum values must be + * power-of-2 values. */ +/** enum TRITONSERVER_RequestReleaseFlag */ +public static final int + TRITONSERVER_REQUEST_RELEASE_ALL = 1; + +/** Inference response complete flags. The enum values must be + * power-of-2 values. */ +/** enum TRITONSERVER_ResponseCompleteFlag */ +public static final int + TRITONSERVER_RESPONSE_COMPLETE_FINAL = 1; +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java + + +// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java + + + +/** Create a new inference request object. + * + * @param inference_request Returns the new request object. + * @param server the inference server object. + * @param model_name The name of the model to use for the request. + * @param model_version The version of the model to use for the + * request. If -1 then the server will choose a version based on the + * model's policy. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( + @Cast("TRITONSERVER_InferenceRequest**") PointerPointer inference_request, + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( + @ByPtrPtr TRITONSERVER_InferenceRequest inference_request, + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( + @ByPtrPtr TRITONSERVER_InferenceRequest inference_request, + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version); + +/** Delete an inference request object. + * + * @param inference_request The request object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestDelete( + TRITONSERVER_InferenceRequest inference_request); + +/** Get the ID for a request. The returned ID is owned by + * 'inference_request' and must not be modified or freed by the + * caller. + * + * @param inference_request The request object. + * @param id Returns the ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") PointerPointer id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr BytePointer id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr ByteBuffer id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr byte[] id); + +/** Set the ID for a request. + * + * @param inference_request The request object. + * @param id The ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetId( + TRITONSERVER_InferenceRequest inference_request, String id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetId( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer id); + +/** Get the flag(s) associated with a request. On return 'flags' holds + * a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for + * available flags. + * + * @param inference_request The request object. + * @param flags Returns the flags. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntPointer flags); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntBuffer flags); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] flags); + +/** Set the flag(s) associated with a request. 'flags' should hold a + * bitwise-or of all flag values, see TRITONSERVER_RequestFlag for + * available flags. + * + * @param inference_request The request object. + * @param flags The flags. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetFlags( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int flags); + +/** Get the correlation ID of the inference request as an unsigned integer. + * Default is 0, which indicates that the request has no correlation ID. + * If the correlation id associated with the inference request is a string, + * this function will return a failure. The correlation ID is used + * to indicate two or more inference request are related to each other. + * How this relationship is handled by the inference server is determined by + * the model's scheduling policy. + * + * @param inference_request The request object. + * @param correlation_id Returns the correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongPointer correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongBuffer correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") long[] correlation_id); + +/** Get the correlation ID of the inference request as a string. + * Default is empty "", which indicates that the request has no correlation ID. + * If the correlation id associated with the inference request is an unsigned + * integer, then this function will return a failure. The correlation ID + * is used to indicate two or more inference request are related to each other. + * How this relationship is handled by the inference server is determined by + * the model's scheduling policy. + * + * @param inference_request The request object. + * @param correlation_id Returns the correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + @Cast("const char**") PointerPointer correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + @Cast("const char**") @ByPtrPtr BytePointer correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + @Cast("const char**") @ByPtrPtr ByteBuffer correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + @Cast("const char**") @ByPtrPtr byte[] correlation_id); + +/** Set the correlation ID of the inference request to be an unsigned integer. + * Default is 0, which indicates that the request has no correlation ID. + * The correlation ID is used to indicate two or more inference request + * are related to each other. How this relationship is handled by the + * inference server is determined by the model's scheduling policy. + * + * @param inference_request The request object. + * @param correlation_id The correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationId( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long correlation_id); + +/** Set the correlation ID of the inference request to be a string. + * The correlation ID is used to indicate two or more inference + * request are related to each other. How this relationship is + * handled by the inference server is determined by the model's + * scheduling policy. + * + * @param inference_request The request object. + * @param correlation_id The correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + String correlation_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( + TRITONSERVER_InferenceRequest inference_request, + @Cast("const char*") BytePointer correlation_id); + +/** Get the priority for a request. The default is 0 indicating that + * the request does not specify a priority and so will use the + * model's default priority. + * + * @param inference_request The request object. + * @param priority Returns the priority level. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntPointer priority); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntBuffer priority); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] priority); + +/** Set the priority for a request. The default is 0 indicating that + * the request does not specify a priority and so will use the + * model's default priority. + * + * @param inference_request The request object. + * @param priority The priority level. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriority( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int priority); + +/** Get the timeout for a request, in microseconds. The default is 0 + * which indicates that the request has no timeout. + * + * @param inference_request The request object. + * @param timeout_us Returns the timeout, in microseconds. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongPointer timeout_us); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongBuffer timeout_us); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") long[] timeout_us); + +/** Set the timeout for a request, in microseconds. The default is 0 + * which indicates that the request has no timeout. + * + * @param inference_request The request object. + * @param timeout_us The timeout, in microseconds. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long timeout_us); + +/** Add an input to a request. + * + * @param inference_request The request object. + * @param name The name of the input. + * @param datatype The type of the input. Valid type names are BOOL, + * UINT8, UINT16, UINT32, UINT64, INT8, INT16, INT32, INT64, FP16, + * FP32, FP64, and BYTES. + * @param shape The shape of the input. + * @param dim_count The number of dimensions of 'shape'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, String name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongPointer shape, + @Cast("uint64_t") long dim_count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongBuffer shape, + @Cast("uint64_t") long dim_count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, String name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") long[] shape, + @Cast("uint64_t") long dim_count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongPointer shape, + @Cast("uint64_t") long dim_count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, String name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongBuffer shape, + @Cast("uint64_t") long dim_count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") long[] shape, + @Cast("uint64_t") long dim_count); + +/** Add a raw input to a request. The name recognized by the model, data type + * and shape of the input will be deduced from model configuration. + * This function must be called at most once on request with no other input to + * ensure the deduction is accurate. + * + * @param inference_request The request object. + * @param name The name of the input. This name is only used as a reference + * of the raw input in other Tritonserver APIs. It doesn't assoicate with the + * name used in the model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRawInput( + TRITONSERVER_InferenceRequest inference_request, String name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRawInput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); + +/** Remove an input from a request. + * + * @param inference_request The request object. + * @param name The name of the input. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveInput( + TRITONSERVER_InferenceRequest inference_request, String name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveInput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); + +/** Remove all inputs from a request. + * + * @param inference_request The request object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputs( + TRITONSERVER_InferenceRequest inference_request); + +/** Assign a buffer of data to an input. The buffer will be appended + * to any existing buffers for that input. The 'inference_request' + * object takes ownership of the buffer and so the caller should not + * modify or free the buffer until that ownership is released by + * 'inference_request' being deleted or by the input being removed + * from 'inference_request'. + * + * @param inference_request The request object. + * @param name The name of the input. + * @param base The base address of the input data. + * @param byte_size The size, in bytes, of the input data. + * @param memory_type The memory type of the input data. + * @param memory_type_id The memory type id of the input data. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputData( + TRITONSERVER_InferenceRequest inference_request, String name, + @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputData( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); + +/** Assign a buffer of data to an input for execution on all model instances + * with the specified host policy. The buffer will be appended to any existing + * buffers for that input on all devices with this host policy. The + * 'inference_request' object takes ownership of the buffer and so the caller + * should not modify or free the buffer until that ownership is released by + * 'inference_request' being deleted or by the input being removed from + * 'inference_request'. If the execution is scheduled on a device that does not + * have a input buffer specified using this function, then the input buffer + * specified with TRITONSERVER_InferenceRequestAppendInputData will be used so + * a non-host policy specific version of data must be added using that API. + * @param inference_request The request object. + * @param name The name of the input. + * @param base The base address of the input data. + * @param byte_size The size, in bytes, of the input data. + * @param memory_type The memory type of the input data. + * @param memory_type_id The memory type id of the input data. + * @param host_policy_name All model instances executing with this host_policy + * will use this input buffer for execution. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( + TRITONSERVER_InferenceRequest inference_request, String name, + @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id, String host_policy_name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id, @Cast("const char*") BytePointer host_policy_name); + +/** Assign a buffer of data to an input. The buffer will be appended + * to any existing buffers for that input. The 'inference_request' + * object takes ownership of the buffer and so the caller should not + * modify or free the buffer until that ownership is released by + * 'inference_request' being deleted or by the input being removed + * from 'inference_request'. + * + * @param inference_request The request object. + * @param name The name of the input. + * @param base The base address of the input data. + * @param buffer_attributes The buffer attrubutes of the input. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( + TRITONSERVER_InferenceRequest inference_request, String name, + @Const Pointer base, TRITONSERVER_BufferAttributes buffer_attributes); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, + @Const Pointer base, TRITONSERVER_BufferAttributes buffer_attributes); + +/** Clear all input data from an input, releasing ownership of the + * buffer(s) that were appended to the input with + * TRITONSERVER_InferenceRequestAppendInputData or + * TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy + * @param inference_request The request object. + * @param name The name of the input. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputData( + TRITONSERVER_InferenceRequest inference_request, String name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputData( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); + +/** Add an output request to an inference request. + * + * @param inference_request The request object. + * @param name The name of the output. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRequestedOutput( + TRITONSERVER_InferenceRequest inference_request, String name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRequestedOutput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); + +/** Remove an output request from an inference request. + * + * @param inference_request The request object. + * @param name The name of the output. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveRequestedOutput( + TRITONSERVER_InferenceRequest inference_request, String name); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveRequestedOutput( + TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); + +/** Remove all output requests from an inference request. + * + * @param inference_request The request object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs( + TRITONSERVER_InferenceRequest inference_request); + +/** Set the release callback for an inference request. The release + * callback is called by Triton to return ownership of the request + * object. + * + * @param inference_request The request object. + * @param request_release_fn The function called to return ownership + * of the 'inference_request' object. + * @param request_release_userp User-provided pointer that is + * delivered to the 'request_release_fn' callback. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetReleaseCallback( + TRITONSERVER_InferenceRequest inference_request, + TRITONSERVER_InferenceRequestReleaseFn_t request_release_fn, + Pointer request_release_userp); + +/** Set the allocator and response callback for an inference + * request. The allocator is used to allocate buffers for any output + * tensors included in responses that are produced for this + * request. The response callback is called to return response + * objects representing responses produced for this request. + * + * @param inference_request The request object. + * @param response_allocator The TRITONSERVER_ResponseAllocator to use + * to allocate buffers to hold inference results. + * @param response_allocator_userp User-provided pointer that is + * delivered to the response allocator's start and allocation functions. + * @param response_fn The function called to deliver an inference + * response for this request. + * @param response_userp User-provided pointer that is delivered to + * the 'response_fn' callback. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetResponseCallback( + TRITONSERVER_InferenceRequest inference_request, + TRITONSERVER_ResponseAllocator response_allocator, + Pointer response_allocator_userp, + TRITONSERVER_InferenceResponseCompleteFn_t response_fn, + Pointer response_userp); + +/** Set a string parameter in the request. + * + * @param request The request. + * @param key The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetStringParameter( + TRITONSERVER_InferenceRequest request, String key, String value); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetStringParameter( + TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, @Cast("const char*") BytePointer value); + +/** Set an integer parameter in the request. + * + * @param request The request. + * @param key The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( + TRITONSERVER_InferenceRequest request, String key, + @Cast("const int64_t") long value); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( + TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, + @Cast("const int64_t") long value); + +/** Set a boolean parameter in the request. + * + * @param request The request. + * @param key The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetBoolParameter( + TRITONSERVER_InferenceRequest request, String key, @Cast("const bool") boolean value); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetBoolParameter( + TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, @Cast("const bool") boolean value); + +/** TRITONSERVER_InferenceResponse + * + * Object representing an inference response. The inference response + * provides the meta-data and output tensor values calculated by the + * inference. + * +

+ * Delete an inference response object. + * + * @param inference_response The response object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseDelete( + TRITONSERVER_InferenceResponse inference_response); + +/** Return the error status of an inference response. Return a + * TRITONSERVER_Error object on failure, return nullptr on success. + * The returned error object is owned by 'inference_response' and so + * should not be deleted by the caller. + * + * @param inference_response The response object. + * @return a TRITONSERVER_Error indicating the success or failure + * status of the response. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseError( + TRITONSERVER_InferenceResponse inference_response); + +/** Get model used to produce a response. The caller does not own the + * returned model name value and must not modify or delete it. The + * lifetime of all returned values extends until 'inference_response' + * is deleted. + * + * @param inference_response The response object. + * @param model_name Returns the name of the model. + * @param model_version Returns the version of the model. + * this response. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") PointerPointer model_name, + @Cast("int64_t*") LongPointer model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr BytePointer model_name, + @Cast("int64_t*") LongPointer model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr ByteBuffer model_name, + @Cast("int64_t*") LongBuffer model_version); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr byte[] model_name, + @Cast("int64_t*") long[] model_version); + +/** Get the ID of the request corresponding to a response. The caller + * does not own the returned ID and must not modify or delete it. The + * lifetime of all returned values extends until 'inference_response' + * is deleted. + * + * @param inference_response The response object. + * @param request_id Returns the ID of the request corresponding to + * this response. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( + TRITONSERVER_InferenceResponse inference_response, + @Cast("const char**") PointerPointer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( + TRITONSERVER_InferenceResponse inference_response, + @Cast("const char**") @ByPtrPtr BytePointer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( + TRITONSERVER_InferenceResponse inference_response, + @Cast("const char**") @ByPtrPtr ByteBuffer request_id); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( + TRITONSERVER_InferenceResponse inference_response, + @Cast("const char**") @ByPtrPtr byte[] request_id); + +/** Get the number of parameters available in the response. + * + * @param inference_response The response object. + * @param count Returns the number of parameters. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") int[] count); + +/** Get all information about a parameter. The caller does not own any + * of the returned values and must not modify or delete them. The + * lifetime of all returned values extends until 'inference_response' + * is deleted. + * + * The 'vvalue' returns a void* pointer that must be cast + * appropriately based on 'type'. For example: + * + * void* vvalue; + * TRITONSERVER_ParameterType type; + * TRITONSERVER_InferenceResponseParameter( + * response, index, &name, &type, &vvalue); + * switch (type) { + * case TRITONSERVER_PARAMETER_BOOL: + * bool value = *(reinterpret_cast(vvalue)); + * ... + * case TRITONSERVER_PARAMETER_INT: + * int64_t value = *(reinterpret_cast(vvalue)); + * ... + * case TRITONSERVER_PARAMETER_STRING: + * const char* value = reinterpret_cast(vvalue); + * ... + * + * @param inference_response The response object. + * @param index The index of the parameter, must be 0 <= index < + * count, where 'count' is the value returned by + * TRITONSERVER_InferenceResponseParameterCount. + * @param name Returns the name of the parameter. + * @param type Returns the type of the parameter. + * @param vvalue Returns a pointer to the parameter value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") PointerPointer name, @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") PointerPointer vvalue); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr BytePointer name, @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr ByteBuffer name, @Cast("TRITONSERVER_ParameterType*") IntBuffer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr byte[] name, @Cast("TRITONSERVER_ParameterType*") int[] type, @Cast("const void**") @ByPtrPtr Pointer vvalue); + +/** Get the number of outputs available in the response. + * + * @param inference_response The response object. + * @param count Returns the number of output tensors. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( + TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") int[] count); + +/** Get all information about an output tensor. The tensor data is + * returned as the base pointer to the data and the size, in bytes, + * of the data. The caller does not own any of the returned values + * and must not modify or delete them. The lifetime of all returned + * values extends until 'inference_response' is deleted. + * + * @param inference_response The response object. + * @param index The index of the output tensor, must be 0 <= index < + * count, where 'count' is the value returned by + * TRITONSERVER_InferenceResponseOutputCount. + * @param name Returns the name of the output. + * @param datatype Returns the type of the output. + * @param shape Returns the shape of the output. + * @param dim_count Returns the number of dimensions of the returned + * shape. + * @param base Returns the tensor data for the output. + * @param byte_size Returns the size, in bytes, of the data. + * @param memory_type Returns the memory type of the data. + * @param memory_type_id Returns the memory type id of the data. + * @param userp The user-specified value associated with the buffer + * in TRITONSERVER_ResponseAllocatorAllocFn_t. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") PointerPointer name, @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, + @Cast("uint64_t*") LongPointer dim_count, @Cast("const void**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id, + @Cast("void**") PointerPointer userp); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr BytePointer name, @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, + @Cast("uint64_t*") LongPointer dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id, + @Cast("void**") @ByPtrPtr Pointer userp); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr ByteBuffer name, @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, + @Cast("uint64_t*") LongBuffer dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id, + @Cast("void**") @ByPtrPtr Pointer userp); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr byte[] name, @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, + @Cast("uint64_t*") long[] dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id, + @Cast("void**") @ByPtrPtr Pointer userp); + +/** Get a classification label associated with an output for a given + * index. The caller does not own the returned label and must not + * modify or delete it. The lifetime of all returned label extends + * until 'inference_response' is deleted. + * + * @param inference_response The response object. + * @param index The index of the output tensor, must be 0 <= index < + * count, where 'count' is the value returned by + * TRITONSERVER_InferenceResponseOutputCount. + * @param class_index The index of the class. + * @param name Returns the label corresponding to 'class_index' or + * nullptr if no label. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const size_t") long class_index, @Cast("const char**") PointerPointer label); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr BytePointer label); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr ByteBuffer label); +public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( + TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, + @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr byte[] label); + +/** TRITONSERVER_BufferAttributes + * + * API to create, modify, or retrieve attributes associated with a buffer. + * +

+ * Create a new buffer attributes object. The caller takes ownership of + * the TRITONSERVER_BufferAttributes object and must call + * TRITONSERVER_BufferAttributesDelete to release the object. + * + * @param buffer_attributes Returns the new buffer attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesNew( + @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesNew( + @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); + +/** Delete a buffer attributes object. + * + * @param buffer_attributes The buffer_attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesDelete( + TRITONSERVER_BufferAttributes buffer_attributes); + +/** Set the memory type id field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param memory_type_id Memory type id to assign to the buffer attributes + * object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetMemoryTypeId( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t") long memory_type_id); + +/** Set the memory type field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param memory_type Memory type to assign to the buffer attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetMemoryType( + TRITONSERVER_BufferAttributes buffer_attributes, + @Cast("TRITONSERVER_MemoryType") int memory_type); + +/** Set the CudaIpcHandle field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param cuda_ipc_handle The CudaIpcHandle to assign to the buffer attributes + * object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetCudaIpcHandle( + TRITONSERVER_BufferAttributes buffer_attributes, Pointer cuda_ipc_handle); + +/** Set the byte size field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param byte_size Byte size to assign to the buffer attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetByteSize( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("size_t") long byte_size); + +/** Get the memory type id field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param memory_type_id Returns the memory type id associated with the buffer + * attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") long[] memory_type_id); + +/** Get the memory type field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param memory_type Returns the memory type associated with the buffer + * attributes object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( + TRITONSERVER_BufferAttributes buffer_attributes, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( + TRITONSERVER_BufferAttributes buffer_attributes, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( + TRITONSERVER_BufferAttributes buffer_attributes, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type); + +/** Get the CudaIpcHandle field of the buffer attributes object. + * + * @param buffer_attributes The buffer attributes object. + * @param cuda_ipc_handle Returns the memory type associated with the buffer + * attributes object. If the cudaIpcHandle does not exist for the buffer, + * nullptr will be returned. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesCudaIpcHandle( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("void**") PointerPointer cuda_ipc_handle); +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesCudaIpcHandle( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("void**") @ByPtrPtr Pointer cuda_ipc_handle); + +/** Get the byte size field of the buffer attributes. + * + * @param buffer_attributes The buffer attributes object. + * @param byte_size Returns the byte size associated with the buffer attributes + * object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesByteSize( + TRITONSERVER_BufferAttributes buffer_attributes, @Cast("size_t*") SizeTPointer byte_size); + + +/** TRITONSERVER_ServerOptions + * + * Options to use when creating an inference server. + * +

+ * Model control modes */ +/** enum TRITONSERVER_ModelControlMode */ +public static final int + TRITONSERVER_MODEL_CONTROL_NONE = 0, + TRITONSERVER_MODEL_CONTROL_POLL = 1, + TRITONSERVER_MODEL_CONTROL_EXPLICIT = 2; + +/** Rate limit modes */ +/** enum TRITONSERVER_RateLimitMode */ +public static final int + TRITONSERVER_RATE_LIMIT_OFF = 0, + TRITONSERVER_RATE_LIMIT_EXEC_COUNT = 1; + +/** Create a new server options object. The caller takes ownership of + * the TRITONSERVER_ServerOptions object and must call + * TRITONSERVER_ServerOptionsDelete to release the object. + * + * @param options Returns the new server options object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsNew( + @Cast("TRITONSERVER_ServerOptions**") PointerPointer options); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsNew( + @ByPtrPtr TRITONSERVER_ServerOptions options); + +/** Delete a server options object. + * + * @param options The server options object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsDelete( + TRITONSERVER_ServerOptions options); + +/** Set the textual ID for the server in a server options. The ID is a + * name that identifies the server. + * + * @param options The server options object. + * @param server_id The server identifier. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetServerId( + TRITONSERVER_ServerOptions options, String server_id); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetServerId( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer server_id); + +/** Set the model repository path in a server options. The path must be + * the full absolute path to the model repository. This function can be called + * multiple times with different paths to set multiple model repositories. + * Note that if a model is not unique across all model repositories + * at any time, the model will not be available. + * + * @param options The server options object. + * @param model_repository_path The full path to the model repository. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelRepositoryPath( + TRITONSERVER_ServerOptions options, String model_repository_path); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelRepositoryPath( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer model_repository_path); + +/** Set the model control mode in a server options. For each mode the models + * will be managed as the following: + * + * TRITONSERVER_MODEL_CONTROL_NONE: the models in model repository will be + * loaded on startup. After startup any changes to the model repository will + * be ignored. Calling TRITONSERVER_ServerPollModelRepository will result in + * an error. + * + * TRITONSERVER_MODEL_CONTROL_POLL: the models in model repository will be + * loaded on startup. The model repository can be polled periodically using + * TRITONSERVER_ServerPollModelRepository and the server will load, unload, + * and updated models according to changes in the model repository. + * + * TRITONSERVER_MODEL_CONTROL_EXPLICIT: the models in model repository will + * not be loaded on startup. The corresponding model control APIs must be + * called to load / unload a model in the model repository. + * + * @param options The server options object. + * @param mode The mode to use for the model control. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelControlMode( + TRITONSERVER_ServerOptions options, @Cast("TRITONSERVER_ModelControlMode") int mode); + +/** Set the model to be loaded at startup in a server options. The model must be + * present in one, and only one, of the specified model repositories. + * This function can be called multiple times with different model name + * to set multiple startup models. + * Note that it only takes affect on TRITONSERVER_MODEL_CONTROL_EXPLICIT mode. + * + * @param options The server options object. + * @param mode_name The name of the model to load on startup. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStartupModel( + TRITONSERVER_ServerOptions options, String model_name); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStartupModel( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer model_name); + +/** Enable or disable strict model configuration handling in a server + * options. + * + * @param options The server options object. + * @param strict True to enable strict model configuration handling, + * false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStrictModelConfig( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean strict); + +/** Set the rate limit mode in a server options. + * + * TRITONSERVER_RATE_LIMIT_EXEC_COUNT: The rate limiting prioritizes the + * inference execution using the number of times each instance has got a + * chance to run. The execution gets to run only when its resource + * constraints are satisfied. + * + * TRITONSERVER_RATE_LIMIT_OFF: The rate limiting is turned off and the + * inference gets executed whenever an instance is available. + * + * @param options The server options object. + * @param mode The mode to use for the rate limiting. By default, execution + * count is used to determine the priorities. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRateLimiterMode( + TRITONSERVER_ServerOptions options, @Cast("TRITONSERVER_RateLimitMode") int mode); + +/** Add resource count for rate limiting. + * + * @param options The server options object. + * @param name The name of the resource. + * @param count The count of the resource. + * @param device The device identifier for the resource. A value of -1 + * indicates that the specified number of resources are available on every + * device. The device value is ignored for a global resource. The server + * will use the rate limiter configuration specified for instance groups + * in model config to determine whether resource is global. In case of + * conflicting resource type in different model configurations, server + * will raise an appropriate error while loading model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsAddRateLimiterResource( + TRITONSERVER_ServerOptions options, String resource_name, + @Cast("const size_t") long resource_count, int device); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsAddRateLimiterResource( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer resource_name, + @Cast("const size_t") long resource_count, int device); + +/** Set the total pinned memory byte size that the server can allocate + * in a server options. The pinned memory pool will be shared across + * Triton itself and the backends that use + * TRITONBACKEND_MemoryManager to allocate memory. + * + * @param options The server options object. + * @param size The pinned memory pool byte size. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( + TRITONSERVER_ServerOptions options, @Cast("uint64_t") long size); + +/** Set the total CUDA memory byte size that the server can allocate + * on given GPU device in a server options. The pinned memory pool + * will be shared across Triton itself and the backends that use + * TRITONBACKEND_MemoryManager to allocate memory. + * + * @param options The server options object. + * @param gpu_device The GPU device to allocate the memory pool. + * @param size The CUDA memory pool byte size. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( + TRITONSERVER_ServerOptions options, int gpu_device, @Cast("uint64_t") long size); + +/** Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead. + * + * Set the total response cache byte size that the server can allocate in CPU + * memory. The response cache will be shared across all inference requests and + * across all models. + * + * @param options The server options object. + * @param size The total response cache byte size. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetResponseCacheByteSize( + TRITONSERVER_ServerOptions options, @Cast("uint64_t") long size); + +/** Set the cache config that will be used to initialize the cache + * implementation for "cache_name". + * + * It is expected that the "cache_name" provided matches a directory inside + * the "cache_dir" used for TRITONSERVER_ServerOptionsSetCacheDirectory. The + * default "cache_dir" is "/opt/tritonserver/caches", so for a "cache_name" of + * "local", Triton would expect to find the "local" cache implementation at + * "/opt/tritonserver/caches/local/libtritoncache_local.so" + * + * Altogether an example for the "local" cache implementation would look like: + * std::string cache_name = "local"; + * std::string config_json = R"({"size": 1048576})" + * auto err = TRITONSERVER_ServerOptionsSetCacheConfig( + * options, cache_name, config_json); + * + * @param options The server options object. + * @param cache_name The name of the cache. Example names would be + * "local", "redis", or the name of a custom cache implementation. + * @param config_json The string representation of config JSON that is + * used to initialize the cache implementation. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheConfig( + TRITONSERVER_ServerOptions options, String cache_name, + String config_json); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheConfig( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer cache_name, + @Cast("const char*") BytePointer config_json); + +/** Set the directory containing cache shared libraries. This + * directory is searched when looking for cache implementations. + * + * @param options The server options object. + * @param cache_dir The full path of the cache directory. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheDirectory( + TRITONSERVER_ServerOptions options, String cache_dir); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheDirectory( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer cache_dir); + +/** Set the minimum support CUDA compute capability in a server + * options. + * + * @param options The server options object. + * @param cc The minimum CUDA compute capability. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( + TRITONSERVER_ServerOptions options, double cc); + +/** Enable or disable exit-on-error in a server options. + * + * @param options The server options object. + * @param exit True to enable exiting on intialization error, false + * to continue. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitOnError( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean exit); + +/** Enable or disable strict readiness handling in a server options. + * + * @param options The server options object. + * @param strict True to enable strict readiness handling, false to + * disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStrictReadiness( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean strict); + +/** Set the exit timeout, in seconds, for the server in a server + * options. + * + * @param options The server options object. + * @param timeout The exit timeout, in seconds. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout( + TRITONSERVER_ServerOptions options, @Cast("unsigned int") int timeout); + +/** Set the number of threads used in buffer manager in a server options. + * + * @param options The server options object. + * @param thread_count The number of threads. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( + TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); + +/** Set the number of threads to concurrently load models in a server options. + * + * @param options The server options object. + * @param thread_count The number of threads. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadThreadCount( + TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); + +/** Enable model namespacing to allow serving models with the same name if + * they are in different namespaces. + * + * @param options The server options object. + * @param enable_namespace Whether to enable model namespacing or not. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelNamespacing( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean enable_namespace); + +/** Provide a log output file. + * + * @param options The server options object. + * @param file a string defining the file where the log outputs will be saved. + * An empty string for the file name will cause triton to direct logging + * facilities to the console + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, String file); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer file); + +/** Enable or disable info level logging. + * + * @param options The server options object. + * @param log True to enable info logging, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogInfo( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); + +/** Enable or disable warning level logging. + * + * @param options The server options object. + * @param log True to enable warning logging, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogWarn( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); + +/** Enable or disable error level logging. + * + * @param options The server options object. + * @param log True to enable error logging, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogError( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); + +/** Set the logging format. + * + * @param options The server options object. + * @param format The logging format. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFormat( + TRITONSERVER_ServerOptions options, @Cast("const TRITONSERVER_LogFormat") int format); + +/** Set verbose logging level. Level zero disables verbose logging. + * + * @param options The server options object. + * @param level The verbose logging level. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogVerbose( + TRITONSERVER_ServerOptions options, int level); + +/** Enable or disable metrics collection in a server options. + * + * @param options The server options object. + * @param metrics True to enable metrics, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetrics( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean metrics); + +/** Enable or disable GPU metrics collection in a server options. GPU + * metrics are collected if both this option and + * TRITONSERVER_ServerOptionsSetMetrics are true. + * + * @param options The server options object. + * @param gpu_metrics True to enable GPU metrics, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetGpuMetrics( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean gpu_metrics); + +/** Enable or disable CPU metrics collection in a server options. CPU + * metrics are collected if both this option and + * TRITONSERVER_ServerOptionsSetMetrics are true. + * + * @param options The server options object. + * @param cpu_metrics True to enable CPU metrics, false to disable. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCpuMetrics( + TRITONSERVER_ServerOptions options, @Cast("bool") boolean cpu_metrics); + +/** Set the interval for metrics collection in a server options. + * This is 2000 milliseconds by default. + * + * @param options The server options object. + * @param metrics_interval_ms The time interval in ms between + * successive metrics updates. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsInterval( + TRITONSERVER_ServerOptions options, @Cast("uint64_t") long metrics_interval_ms); + +/** Set the directory containing backend shared libraries. This + * directory is searched last after the version and model directory + * in the model repository when looking for the backend shared + * library for a model. If the backend is named 'be' the directory + * searched is 'backend_dir'/be/libtriton_be.so. + * + * @param options The server options object. + * @param backend_dir The full path of the backend directory. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDirectory( + TRITONSERVER_ServerOptions options, String backend_dir); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDirectory( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer backend_dir); + +/** Set the directory containing repository agent shared libraries. This + * directory is searched when looking for the repository agent shared + * library for a model. If the repo agent is named 'ra' the directory + * searched is 'repoagent_dir'/ra/libtritonrepoagent_ra.so. + * + * @param options The server options object. + * @param repoagent_dir The full path of the repository agent directory. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + TRITONSERVER_ServerOptions options, String repoagent_dir); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer repoagent_dir); + +/** Specify the limit on memory usage as a fraction on the device identified by + * 'kind' and 'device_id'. If model loading on the device is requested and the + * current memory usage exceeds the limit, the load will be rejected. If not + * specified, the limit will not be set. + * + * Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU + * + * @param options The server options object. + * @param kind The kind of the device. + * @param device_id The id of the device. + * @param fraction The limit on memory usage as a fraction + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( + TRITONSERVER_ServerOptions options, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, int device_id, + double fraction); + +/** Set a configuration setting for a named backend in a server + * options. + * + * @param options The server options object. + * @param backend_name The name of the backend. + * @param setting The name of the setting. + * @param value The setting value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendConfig( + TRITONSERVER_ServerOptions options, String backend_name, + String setting, String value); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendConfig( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer backend_name, + @Cast("const char*") BytePointer setting, @Cast("const char*") BytePointer value); + +/** Set a host policy setting for a given policy name in a server options. + * + * @param options The server options object. + * @param policy_name The name of the policy. + * @param setting The name of the setting. + * @param value The setting value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( + TRITONSERVER_ServerOptions options, String policy_name, + String setting, String value); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer policy_name, + @Cast("const char*") BytePointer setting, @Cast("const char*") BytePointer value); + +/** Set a configuration setting for metrics in server options. + * + * @param options The server options object. + * @param name The name of the configuration group. An empty string indicates + * a global configuration option. + * @param setting The name of the setting. + * @param value The setting value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( + TRITONSERVER_ServerOptions options, String name, String setting, + String value); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer setting, + @Cast("const char*") BytePointer value); + +/** TRITONSERVER_Server + * + * An inference server. + * +

+ * Model batch flags. The enum values must be power-of-2 values. */ +/** enum TRITONSERVER_ModelBatchFlag */ +public static final int + TRITONSERVER_BATCH_UNKNOWN = 1, + TRITONSERVER_BATCH_FIRST_DIM = 2; + +/** Model index flags. The enum values must be power-of-2 values. */ +/** enum TRITONSERVER_ModelIndexFlag */ +public static final int + TRITONSERVER_INDEX_FLAG_READY = 1; + +/** Model transaction policy flags. The enum values must be + * power-of-2 values. */ +/** enum TRITONSERVER_ModelTxnPropertyFlag */ +public static final int + TRITONSERVER_TXN_ONE_TO_ONE = 1, + TRITONSERVER_TXN_DECOUPLED = 2; + +/** Create a new server object. The caller takes ownership of the + * TRITONSERVER_Server object and must call TRITONSERVER_ServerDelete + * to release the object. + * + * @param server Returns the new inference server object. + * @param options The inference server options object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerNew( + @Cast("TRITONSERVER_Server**") PointerPointer server, TRITONSERVER_ServerOptions options); +public static native TRITONSERVER_Error TRITONSERVER_ServerNew( + @ByPtrPtr TRITONSERVER_Server server, TRITONSERVER_ServerOptions options); + +/** Delete a server object. If server is not already stopped it is + * stopped before being deleted. + * + * @param server The inference server object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerDelete( + TRITONSERVER_Server server); + +/** Stop a server object. A server can't be restarted once it is + * stopped. + * + * @param server The inference server object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerStop( + TRITONSERVER_Server server); + +/** Register a new model repository. Not available in polling mode. + * + * @param server The inference server object. + * @param repository_path The full path to the model repository. + * @param name_mapping List of name_mapping parameters. Each mapping has + * the model directory name as its key, overriden model name as its value. + * @param model_count Number of mappings provided. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( + TRITONSERVER_Server server, String repository_path, + @Cast("const TRITONSERVER_Parameter**") PointerPointer name_mapping, @Cast("const uint32_t") int mapping_count); +public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( + TRITONSERVER_Server server, String repository_path, + @Const @ByPtrPtr TRITONSERVER_Parameter name_mapping, @Cast("const uint32_t") int mapping_count); +public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( + TRITONSERVER_Server server, @Cast("const char*") BytePointer repository_path, + @Const @ByPtrPtr TRITONSERVER_Parameter name_mapping, @Cast("const uint32_t") int mapping_count); + +/** Unregister a model repository. Not available in polling mode. + * + * @param server The inference server object. + * @param repository_path The full path to the model repository. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerUnregisterModelRepository( + TRITONSERVER_Server server, String repository_path); +public static native TRITONSERVER_Error TRITONSERVER_ServerUnregisterModelRepository( + TRITONSERVER_Server server, @Cast("const char*") BytePointer repository_path); + +/** Check the model repository for changes and update server state + * based on those changes. + * + * @param server The inference server object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerPollModelRepository(TRITONSERVER_Server server); + +/** Is the server live? + * + * @param server The inference server object. + * @param live Returns true if server is live, false otherwise. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerIsLive( + TRITONSERVER_Server server, @Cast("bool*") boolean[] live); +public static native TRITONSERVER_Error TRITONSERVER_ServerIsLive( + TRITONSERVER_Server server, @Cast("bool*") BoolPointer live); + +/** Is the server ready? + * + * @param server The inference server object. + * @param ready Returns true if server is ready, false otherwise. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerIsReady( + TRITONSERVER_Server server, @Cast("bool*") boolean[] ready); +public static native TRITONSERVER_Error TRITONSERVER_ServerIsReady( + TRITONSERVER_Server server, @Cast("bool*") BoolPointer ready); + +/** Is the model ready? + * + * @param server The inference server object. + * @param model_name The name of the model to get readiness for. + * @param model_version The version of the model to get readiness + * for. If -1 then the server will choose a version based on the + * model's policy. + * @param ready Returns true if server is ready, false otherwise. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelIsReady( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("bool*") boolean[] ready); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelIsReady( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("bool*") BoolPointer ready); + +/** Get the batch properties of the model. The properties are + * communicated by a flags value and an (optional) object returned by + * 'voidp'. + * + * - TRITONSERVER_BATCH_UNKNOWN: Triton cannot determine the + * batching properties of the model. This means that the model + * does not support batching in any way that is useable by + * Triton. The returned 'voidp' value is nullptr. + * + * - TRITONSERVER_BATCH_FIRST_DIM: The model supports batching + * along the first dimension of every input and output + * tensor. Triton schedulers that perform batching can + * automatically batch inference requests along this dimension. + * The returned 'voidp' value is nullptr. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @param model_version The version of the model. If -1 then the + * server will choose a version based on the model's policy. + * @param flags Returns flags indicating the batch properties of the + * model. + * @param voidp If non-nullptr, returns a point specific to the + * 'flags' value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") PointerPointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] flags, @Cast("void**") @ByPtrPtr Pointer voidp); + +/** Get the transaction policy of the model. The policy is + * communicated by a flags value. + * + * - TRITONSERVER_TXN_ONE_TO_ONE: The model generates exactly + * one response per request. + * + * - TRITONSERVER_TXN_DECOUPLED: The model may generate zero + * to many responses per request. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @param model_version The version of the model. If -1 then the + * server will choose a version based on the model's policy. + * @param txn_flags Returns flags indicating the transaction policy of the + * model. + * @param voidp If non-nullptr, returns a point specific to the 'flags' value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") PointerPointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); + +/** Get the metadata of the server as a TRITONSERVER_Message object. + * The caller takes ownership of the message object and must call + * TRITONSERVER_MessageDelete to release the object. + * + * @param server The inference server object. + * @param server_metadata Returns the server metadata message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerMetadata( + TRITONSERVER_Server server, @Cast("TRITONSERVER_Message**") PointerPointer server_metadata); +public static native TRITONSERVER_Error TRITONSERVER_ServerMetadata( + TRITONSERVER_Server server, @ByPtrPtr TRITONSERVER_Message server_metadata); + +/** Get the metadata of a model as a TRITONSERVER_Message + * object. The caller takes ownership of the message object and must + * call TRITONSERVER_MessageDelete to release the object. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @param model_version The version of the model. + * If -1 then the server will choose a version based on the model's + * policy. + * @param model_metadata Returns the model metadata message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("TRITONSERVER_Message**") PointerPointer model_metadata); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_metadata); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_metadata); + +/** Get the statistics of a model as a TRITONSERVER_Message + * object. The caller takes ownership of the object and must call + * TRITONSERVER_MessageDelete to release the object. + * + * @param server The inference server object. + * @param model_name The name of the model. + * If empty, then statistics for all available models will be returned, + * and the server will choose a version based on those models' policies. + * @param model_version The version of the model. If -1 then the + * server will choose a version based on the model's policy. + * @param model_stats Returns the model statistics message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("TRITONSERVER_Message**") PointerPointer model_stats); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_stats); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_stats); + +/** Get the configuration of a model as a TRITONSERVER_Message object. + * The caller takes ownership of the message object and must call + * TRITONSERVER_MessageDelete to release the object. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @param model_version The version of the model. If -1 then the + * server will choose a version based on the model's policy. + * @param config_version The model configuration will be returned in + * a format matching this version. If the configuration cannot be + * represented in the requested version's format then an error will + * be returned. Currently only version 1 is supported. + * @param model_config Returns the model config message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, + @Cast("TRITONSERVER_Message**") PointerPointer model_config); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( + TRITONSERVER_Server server, String model_name, + @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, + @ByPtrPtr TRITONSERVER_Message model_config); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, + @ByPtrPtr TRITONSERVER_Message model_config); + +/** Get the index of all unique models in the model repositories as a + * TRITONSERVER_Message object. The caller takes ownership of the + * message object and must call TRITONSERVER_MessageDelete to release + * the object. + * + * If TRITONSERVER_INDEX_FLAG_READY is set in 'flags' only the models + * that are loaded into the server and ready for inferencing are + * returned. + * + * @param server The inference server object. + * @param flags TRITONSERVER_ModelIndexFlag flags that control how to + * collect the index. + * @param model_index Return the model index message that holds the + * index of all models contained in the server's model repository(s). + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerModelIndex( + TRITONSERVER_Server server, @Cast("uint32_t") int flags, + @Cast("TRITONSERVER_Message**") PointerPointer model_index); +public static native TRITONSERVER_Error TRITONSERVER_ServerModelIndex( + TRITONSERVER_Server server, @Cast("uint32_t") int flags, + @ByPtrPtr TRITONSERVER_Message model_index); + +/** Load the requested model or reload the model if it is already + * loaded. The function does not return until the model is loaded or + * fails to load. Returned error indicates if model loaded + * successfully or not. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModel( + TRITONSERVER_Server server, String model_name); +public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModel( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); + +/** Load the requested model or reload the model if it is already + * loaded, with load parameters provided. The function does not return until + * the model is loaded or fails to load. Returned error indicates if model + * loaded successfully or not. + * Currently the below parameter names are recognized: + * - "config" : string parameter that contains a JSON representation of the + * model configuration. This config will be used for loading the model instead + * of the one in the model directory. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @param parameters The array of load parameters. + * @param parameter_count The number of parameters. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( + TRITONSERVER_Server server, String model_name, + @Cast("const TRITONSERVER_Parameter**") PointerPointer parameters, @Cast("const uint64_t") long parameter_count); +public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( + TRITONSERVER_Server server, String model_name, + @Const @ByPtrPtr TRITONSERVER_Parameter parameters, @Cast("const uint64_t") long parameter_count); +public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, + @Const @ByPtrPtr TRITONSERVER_Parameter parameters, @Cast("const uint64_t") long parameter_count); + +/** Unload the requested model. Unloading a model that is not loaded + * on server has no affect and success code will be returned. + * The function does not wait for the requested model to be fully unload + * and success code will be returned. + * Returned error indicates if model unloaded successfully or not. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModel( + TRITONSERVER_Server server, String model_name); +public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModel( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); + +/** Unload the requested model, and also unload any dependent model that + * was loaded along with the requested model (for example, the models composing + * an ensemble). Unloading a model that is not loaded + * on server has no affect and success code will be returned. + * The function does not wait for the requested model and all dependent + * models to be fully unload and success code will be returned. + * Returned error indicates if model unloaded successfully or not. + * + * @param server The inference server object. + * @param model_name The name of the model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModelAndDependents( + TRITONSERVER_Server server, String model_name); +public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModelAndDependents( + TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); + +/** Get the current metrics for the server. The caller takes ownership + * of the metrics object and must call TRITONSERVER_MetricsDelete to + * release the object. + * + * @param server The inference server object. + * @param metrics Returns the metrics. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerMetrics( + TRITONSERVER_Server server, @Cast("TRITONSERVER_Metrics**") PointerPointer metrics); +public static native TRITONSERVER_Error TRITONSERVER_ServerMetrics( + TRITONSERVER_Server server, @ByPtrPtr TRITONSERVER_Metrics metrics); + +/** Perform inference using the meta-data and inputs supplied by the + * 'inference_request'. If the function returns success, then the + * caller releases ownership of 'inference_request' and must not + * access it in any way after this call, until ownership is returned + * via the 'request_release_fn' callback registered in the request + * object with TRITONSERVER_InferenceRequestSetReleaseCallback. + * + * The function unconditionally takes ownership of 'trace' and so the + * caller must not access it in any way after this call (except in + * the trace activity callbacks) until ownership is returned via the + * trace's release_fn callback. + * + * Responses produced for this request are returned using the + * allocator and callback registered with the request by + * TRITONSERVER_InferenceRequestSetResponseCallback. + * + * @param server The inference server object. + * @param inference_request The request object. + * @param trace The trace object for this request, or nullptr if no + * tracing. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerInferAsync( + TRITONSERVER_Server server, + TRITONSERVER_InferenceRequest inference_request, + TRITONSERVER_InferenceTrace trace); + +/** TRITONSERVER_MetricKind + * + * Types of metrics recognized by TRITONSERVER. + * */ +/** enum TRITONSERVER_MetricKind */ +public static final int + TRITONSERVER_METRIC_KIND_COUNTER = 0, + TRITONSERVER_METRIC_KIND_GAUGE = 1; + +/** Create a new metric family object. The caller takes ownership of the + * TRITONSERVER_MetricFamily object and must call + * TRITONSERVER_MetricFamilyDelete to release the object. + * + * @param family Returns the new metric family object. + * @param kind The type of metric family to create. + * @param name The name of the metric family seen when calling the metrics + * endpoint. + * @param description The description of the metric family seen when + * calling the metrics endpoint. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( + @Cast("TRITONSERVER_MetricFamily**") PointerPointer family, @Cast("const TRITONSERVER_MetricKind") int kind, + String name, String description); +public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( + @ByPtrPtr TRITONSERVER_MetricFamily family, @Cast("const TRITONSERVER_MetricKind") int kind, + String name, String description); +public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( + @ByPtrPtr TRITONSERVER_MetricFamily family, @Cast("const TRITONSERVER_MetricKind") int kind, + @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer description); + +/** Delete a metric family object. + * A TRITONSERVER_MetricFamily* object should be deleted AFTER its + * corresponding TRITONSERVER_Metric* objects have been deleted. + * Attempting to delete a family before its metrics will return an error. + * + * @param family The metric family object to delete. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyDelete( + TRITONSERVER_MetricFamily family); + +/** Create a new metric object. The caller takes ownership of the + * TRITONSERVER_Metric object and must call + * TRITONSERVER_MetricDelete to release the object. The caller is also + * responsible for ownership of the labels passed in. Each label can be deleted + * immediately after creating the metric with TRITONSERVER_ParameterDelete + * if not re-using the labels. + * + * @param metric Returns the new metric object. + * @param family The metric family to add this new metric to. + * @param labels The array of labels to associate with this new metric. + * @param label_count The number of labels. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricNew( + @Cast("TRITONSERVER_Metric**") PointerPointer metric, TRITONSERVER_MetricFamily family, + @Cast("const TRITONSERVER_Parameter**") PointerPointer labels, @Cast("const uint64_t") long label_count); +public static native TRITONSERVER_Error TRITONSERVER_MetricNew( + @ByPtrPtr TRITONSERVER_Metric metric, TRITONSERVER_MetricFamily family, + @Const @ByPtrPtr TRITONSERVER_Parameter labels, @Cast("const uint64_t") long label_count); + +/** Delete a metric object. + * All TRITONSERVER_Metric* objects should be deleted BEFORE their + * corresponding TRITONSERVER_MetricFamily* objects have been deleted. + * If a family is deleted before its metrics, an error will be returned. + * + * @param metric The metric object to delete. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricDelete( + TRITONSERVER_Metric metric); + +/** Get the current value of a metric object. + * Supports metrics of kind TRITONSERVER_METRIC_KIND_COUNTER + * and TRITONSERVER_METRIC_KIND_GAUGE, and returns + * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. + * + * @param metric The metric object to query. + * @param value Returns the current value of the metric object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricValue( + TRITONSERVER_Metric metric, DoublePointer value); +public static native TRITONSERVER_Error TRITONSERVER_MetricValue( + TRITONSERVER_Metric metric, DoubleBuffer value); +public static native TRITONSERVER_Error TRITONSERVER_MetricValue( + TRITONSERVER_Metric metric, double[] value); + +/** Increment the current value of metric by value. + * Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE for any value, + * and TRITONSERVER_METRIC_KIND_COUNTER for non-negative values. Returns + * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind + * and TRITONSERVER_ERROR_INVALID_ARG for negative values on a + * TRITONSERVER_METRIC_KIND_COUNTER metric. + * + * @param metric The metric object to update. + * @param value The amount to increment the metric's value by. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricIncrement( + TRITONSERVER_Metric metric, double value); + +/** Set the current value of metric to value. + * Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and returns + * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. + * + * @param metric The metric object to update. + * @param value The amount to set metric's value to. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_MetricSet( + TRITONSERVER_Metric metric, double value); + +/** Get the TRITONSERVER_MetricKind of metric and its corresponding family. + * + * @param metric The metric object to query. + * @param kind Returns the TRITONSERVER_MetricKind of metric. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( + TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") IntPointer kind); +public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( + TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") IntBuffer kind); +public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( + TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") int[] kind); + +// #ifdef __cplusplus +// #endif + + +// Parsed from tritonbackend.h + +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once + +// #include +// #include + +// #include "triton/core/tritonserver.h" + +// #ifdef __cplusplus +// #endif + +// #ifdef _COMPILING_TRITONBACKEND +// #if defined(_MSC_VER) +// #define TRITONBACKEND_DECLSPEC __declspec(dllexport) +// #define TRITONBACKEND_ISPEC __declspec(dllimport) +// #elif defined(__GNUC__) +// #define TRITONBACKEND_DECLSPEC __attribute__((__visibility__("default"))) +// #define TRITONBACKEND_ISPEC +// #else +// #define TRITONBACKEND_DECLSPEC +// #define TRITONBACKEND_ISPEC +// #endif +// #else +// #if defined(_MSC_VER) +// #define TRITONBACKEND_DECLSPEC __declspec(dllimport) +// #define TRITONBACKEND_ISPEC __declspec(dllexport) +// #else +// #define TRITONBACKEND_DECLSPEC +// #define TRITONBACKEND_ISPEC +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Input.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Output.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_State.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Request.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Response.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Backend.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Model.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java + + +// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Batcher.java + + + +/** + * TRITONBACKEND API Version + * + * The TRITONBACKEND API is versioned with major and minor version + * numbers. Any change to the API that does not impact backwards + * compatibility (for example, adding a non-required function) + * increases the minor version number. Any change that breaks + * backwards compatibility (for example, deleting or changing the + * behavior of a function) increases the major version number. A + * backend should check that the API version used to compile the + * backend is compatible with the API version of the Triton server + * that it is running in. This is typically done by code similar to + * the following which makes sure that the major versions are equal + * and that the minor version of Triton is >= the minor version used + * to build the backend. + * + * uint32_t api_version_major, api_version_minor; + * TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor); + * if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || + * (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { + * return TRITONSERVER_ErrorNew( + * TRITONSERVER_ERROR_UNSUPPORTED, + * "triton backend API version does not support this backend"); + * } + * */ +public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; + +/// +public static final int TRITONBACKEND_API_VERSION_MINOR = 12; + +/** Get the TRITONBACKEND API version supported by Triton. This value + * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and + * TRITONBACKEND_API_VERSION_MINOR used to build the backend to + * ensure that Triton is compatible with the backend. + * + * @param major Returns the TRITONBACKEND API major version supported + * by Triton. + * @param minor Returns the TRITONBACKEND API minor version supported + * by Triton. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( + @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); +public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( + @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); +public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( + @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); + +/** TRITONBACKEND_ArtifactType + * + * The ways that the files that make up a backend or model are + * communicated to the backend. + * + * TRITONBACKEND_ARTIFACT_FILESYSTEM: The model or backend + * artifacts are made available to Triton via a locally + * accessible filesystem. The backend can access these files + * using an appropriate system API. + * */ +/** enum TRITONBACKEND_ArtifactType */ +public static final int + TRITONBACKEND_ARTIFACT_FILESYSTEM = 0; + + +/** + * TRITONBACKEND_MemoryManager + * + * Object representing an memory manager that is capable of + * allocating and otherwise managing different memory types. For + * improved performance Triton maintains pools for GPU and CPU-pinned + * memory and the memory manager allows backends to access those + * pools. + * +

+ * Allocate a contiguous block of memory of a specific type using a + * memory manager. Two error codes have specific interpretations for + * this function: + * + * TRITONSERVER_ERROR_UNSUPPORTED: Indicates that Triton is + * incapable of allocating the requested memory type and memory + * type ID. Requests for the memory type and ID will always fail + * no matter 'byte_size' of the request. + * + * TRITONSERVER_ERROR_UNAVAILABLE: Indicates that Triton can + * allocate the memory type and ID but that currently it cannot + * allocate a contiguous block of memory of the requested + * 'byte_size'. + * + * @param manager The memory manager. + * @param buffer Returns the allocated memory. + * @param memory_type The type of memory to allocate. + * @param memory_type_id The ID associated with the memory type to + * allocate. For GPU memory this indicates the device ID of the GPU + * to allocate from. + * @param byte_size The size of memory to allocate, in bytes. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerAllocate( + TRITONBACKEND_MemoryManager manager, @Cast("void**") PointerPointer buffer, + @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id, + @Cast("const uint64_t") long byte_size); +public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerAllocate( + TRITONBACKEND_MemoryManager manager, @Cast("void**") @ByPtrPtr Pointer buffer, + @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id, + @Cast("const uint64_t") long byte_size); + +/** Free a buffer that was previously allocated with + * TRITONBACKEND_MemoryManagerAllocate. The call must provide the + * same values for 'memory_type' and 'memory_type_id' as were used + * when the buffer was allocate or else the behavior is undefined. + * + * @param manager The memory manager. + * @param buffer The allocated memory buffer to free. + * @param memory_type The type of memory of the buffer. + * @param memory_type_id The ID associated with the memory type of + * the buffer. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerFree( + TRITONBACKEND_MemoryManager manager, Pointer buffer, + @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id); + +/** + * TRITONBACKEND_Input + * + * Object representing an input tensor. + * +

+ * Get the name and properties of an input tensor. The returned + * strings and other properties are owned by the input, not the + * caller, and so should not be modified or freed. + * + * @param input The input tensor. + * @param name If non-nullptr, returns the tensor name. + * @param datatype If non-nullptr, returns the tensor datatype. + * @param shape If non-nullptr, returns the tensor shape. + * @param dim_count If non-nullptr, returns the number of dimensions + * in the tensor shape. + * @param byte_size If non-nullptr, returns the size of the available + * data for the tensor, in bytes. This size reflects the actual data + * available, and does not necessarily match what is + * expected/required for the tensor given its shape and datatype. It + * is the responsibility of the backend to handle mismatches in these + * sizes appropriately. + * @param buffer_count If non-nullptr, returns the number of buffers + * holding the contents of the tensor. These buffers are accessed + * using TRITONBACKEND_InputBuffer. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( + TRITONBACKEND_Input input, @Cast("const char**") PointerPointer name, + @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, + @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( + TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr BytePointer name, + @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, + @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( + TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr ByteBuffer name, + @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, + @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( + TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr byte[] name, + @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, + @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); + +/** Get the name and properties of an input tensor associated with a given + * host policy. If there are no input buffers for the specified host policy, + * the properties of the fallback input buffers are returned. The returned + * strings and other properties are owned by the input, not the caller, and so + * should not be modified or freed. + * + * @param input The input tensor. + * @param host_policy_name The host policy name. Fallback input properties + * will be return if nullptr is provided. + * @param name If non-nullptr, returns the tensor name. + * @param datatype If non-nullptr, returns the tensor datatype. + * @param shape If non-nullptr, returns the tensor shape. + * @param dim_count If non-nullptr, returns the number of dimensions + * in the tensor shape. + * @param byte_size If non-nullptr, returns the size of the available + * data for the tensor, in bytes. This size reflects the actual data + * available, and does not necessarily match what is + * expected/required for the tensor given its shape and datatype. It + * is the responsibility of the backend to handle mismatches in these + * sizes appropriately. + * @param buffer_count If non-nullptr, returns the number of buffers + * holding the contents of the tensor. These buffers are accessed + * using TRITONBACKEND_InputBufferForHostPolicy. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") PointerPointer name, + @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, + @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr BytePointer name, + @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, + @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr ByteBuffer name, + @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, + @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr byte[] name, + @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, + @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr BytePointer name, + @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, + @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr ByteBuffer name, + @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, + @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); +public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr byte[] name, + @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, + @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); + +/** Get a buffer holding (part of) the tensor data for an input. For a + * given input the number of buffers composing the input are found + * from 'buffer_count' returned by TRITONBACKEND_InputProperties. The + * returned buffer is owned by the input and so should not be + * modified or freed by the caller. The lifetime of the buffer + * matches that of the input and so the buffer should not be accessed + * after the input tensor object is released. + * + * @param input The input tensor. + * @param index The index of the buffer. Must be 0 <= index < + * buffer_count, where buffer_count is the value returned by + * TRITONBACKEND_InputProperties. + * @param buffer Returns a pointer to a contiguous block of data for + * the named input. + * @param buffer_byte_size Returns the size, in bytes, of 'buffer'. + * @param memory_type Acts as both input and output. On input gives + * the buffer memory type preferred by the function caller. Returns + * the actual memory type of 'buffer'. + * @param memory_type_id Acts as both input and output. On input + * gives the buffer memory type id preferred by the function caller. + * Returns the actual memory type id of 'buffer'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, + @Cast("uint64_t*") LongPointer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, + @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, + @Cast("uint64_t*") LongPointer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, + @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, + @Cast("uint64_t*") LongBuffer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, + @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, + @Cast("uint64_t*") long[] buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") int[] memory_type, + @Cast("int64_t*") long[] memory_type_id); + +/** Get a buffer holding (part of) the tensor data for an input for a specific + * host policy. If there are no input buffers specified for this host policy, + * the fallback input buffer is returned. + * For a given input the number of buffers composing the input are found + * from 'buffer_count' returned by TRITONBACKEND_InputPropertiesForHostPolicy. + * The returned buffer is owned by the input and so should not be modified or + * freed by the caller. The lifetime of the buffer matches that of the input + * and so the buffer should not be accessed after the input tensor object is + * released. + * + * @param input The input tensor. + * @param host_policy_name The host policy name. Fallback input buffer + * will be return if nullptr is provided. + * @param index The index of the buffer. Must be 0 <= index < + * buffer_count, where buffer_count is the value returned by + * TRITONBACKEND_InputPropertiesForHostPolicy. + * @param buffer Returns a pointer to a contiguous block of data for + * the named input. + * @param buffer_byte_size Returns the size, in bytes, of 'buffer'. + * @param memory_type Acts as both input and output. On input gives + * the buffer memory type preferred by the function caller. Returns + * the actual memory type of 'buffer'. + * @param memory_type_id Acts as both input and output. On input + * gives the buffer memory type id preferred by the function caller. + * Returns the actual memory type id of 'buffer'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongBuffer buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") long[] buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, String host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongBuffer buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( + TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, + @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") long[] buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); + +/** Get the buffer attributes associated with the given input buffer. For a + * given input the number of buffers composing the input are found from + * 'buffer_count' returned by TRITONBACKEND_InputProperties. The returned + * 'buffer_attributes' is owned by the input and so should not be modified or + * freed by the caller. The lifetime of the 'buffer_attributes' matches that of + * the input and so the 'buffer_attributes' should not be accessed after the + * input tensor object is released. + * + * @param input The input tensor. + * @param index The index of the buffer. Must be 0 <= index < buffer_count, + * where buffer_count is the value returned by TRITONBACKEND_InputProperties. + * @param buffer Returns a pointer to a contiguous block of data for + * the named input. + * @param buffer_attributes Returns the attributes for the given buffer. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferAttributes( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, + @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); +public static native TRITONSERVER_Error TRITONBACKEND_InputBufferAttributes( + TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, + @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); + +/** + * TRITONBACKEND_Output + * + * Object representing a response output tensor. + * +

+ * Get a buffer to use to hold the tensor data for the output. The + * returned buffer is owned by the output and so should not be freed + * by the caller. The caller can and should fill the buffer with the + * output data for the tensor. The lifetime of the buffer matches + * that of the output and so the buffer should not be accessed after + * the output tensor object is released. + * + * @param buffer Returns a pointer to a buffer where the contents of + * the output tensor should be placed. + * @param buffer_byte_size The size, in bytes, of the buffer required + * by the caller. + * @param memory_type Acts as both input and output. On input gives + * the buffer memory type preferred by the caller. Returns the + * actual memory type of 'buffer'. + * @param memory_type_id Acts as both input and output. On input + * gives the buffer memory type id preferred by the caller. Returns + * the actual memory type id of 'buffer'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( + TRITONBACKEND_Output output, @Cast("void**") PointerPointer buffer, + @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, + @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( + TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, + @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, + @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( + TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, + @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, + @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( + TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, + @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") int[] memory_type, + @Cast("int64_t*") long[] memory_type_id); + +/** Get the buffer attributes associated with the given output buffer. The + * returned 'buffer_attributes' is owned by the output and so should not be + * modified or freed by the caller. The lifetime of the 'buffer_attributes' + * matches that of the output and so the 'buffer_attributes' should not be + * accessed after the output tensor object is released. This function must be + * called after the TRITONBACKEND_OutputBuffer otherwise it might contain + * incorrect data. + * + * @param output The output tensor. + * @param buffer_attributes Returns the attributes for the output buffer. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_OutputBufferAttributes( + TRITONBACKEND_Output output, + @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); +public static native TRITONSERVER_Error TRITONBACKEND_OutputBufferAttributes( + TRITONBACKEND_Output output, + @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); + +/** + * TRITONBACKEND_Request + * + * Object representing an inference request. + * +

+ * Get the ID of the request. Can be nullptr if request doesn't have + * an ID. The returned string is owned by the request, not the + * caller, and so should not be modified or freed. + * + * @param request The inference request. + * @param id Returns the ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestId( + TRITONBACKEND_Request request, @Cast("const char**") PointerPointer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestId( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr BytePointer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestId( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr ByteBuffer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestId( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr byte[] id); + +/** Get the correlation ID of the request if it is an unsigned integer. + * Zero indicates that the request does not have a correlation ID. + * Returns failure if correlation ID for given request is not an unsigned + * integer. + * + * @param request The inference request. + * @param id Returns the correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( + TRITONBACKEND_Request request, @Cast("uint64_t*") LongPointer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( + TRITONBACKEND_Request request, @Cast("uint64_t*") LongBuffer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( + TRITONBACKEND_Request request, @Cast("uint64_t*") long[] id); + +/** Get the correlation ID of the request if it is a string. + * Empty string indicates that the request does not have a correlation ID. + * Returns error if correlation ID for given request is not a string. + * + * @param request The inference request. + * @param id Returns the correlation ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( + TRITONBACKEND_Request request, @Cast("const char**") PointerPointer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr BytePointer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr ByteBuffer id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( + TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr byte[] id); + +/** Get the flag(s) associated with a request. On return 'flags' holds + * a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for + * available flags. + * + * @param request The inference request. + * @param flags Returns the flags. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer flags); +public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer flags); +public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( + TRITONBACKEND_Request request, @Cast("uint32_t*") int[] flags); + +/** Get the number of parameters specified in the inference request. + * + * @param request The inference request. + * @param count Returns the number of parameters. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); + +/** Get a request parameters by index. The order of parameters in a given + * request is not necessarily consistent with other requests, even if + * the requests are in the same batch. As a result, you can not + * assume that an index obtained from one request will point to the + * same parameter in a different request. + * + * The lifetime of the returned parameter object matches that of the + * request and so the parameter object should not be accessed after the + * request object is released. + * + * @param request The inference request. + * @param index The index of the parameter. Must be 0 <= index < + * count, where count is the value returned by + * TRITONBACKEND_RequestParameterCount. + * @param key Returns the key of the parameter. + * @param type Returns the type of the parameter. + * @param vvalue Returns a pointer to the parameter value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") PointerPointer key, + @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") PointerPointer vvalue); +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer key, + @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer key, + @Cast("TRITONSERVER_ParameterType*") IntBuffer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); +public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] key, + @Cast("TRITONSERVER_ParameterType*") int[] type, @Cast("const void**") @ByPtrPtr Pointer vvalue); + +/** Get the number of input tensors specified in the request. + * + * @param request The inference request. + * @param count Returns the number of input tensors. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); + +/** Get the name of an input tensor. The caller does not own + * the returned string and must not modify or delete it. The lifetime + * of the returned string extends only as long as 'request'. + * + * @param request The inference request. + * @param index The index of the input tensor. Must be 0 <= index < + * count, where count is the value returned by + * TRITONBACKEND_RequestInputCount. + * @param input_name Returns the name of the input tensor + * corresponding to the index. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") PointerPointer input_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr BytePointer input_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr ByteBuffer input_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr byte[] input_name); + +/** Get a named request input. The lifetime of the returned input + * object matches that of the request and so the input object should + * not be accessed after the request object is released. + * + * @param request The inference request. + * @param name The name of the input. + * @param input Returns the input corresponding to the name. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( + TRITONBACKEND_Request request, String name, + @Cast("TRITONBACKEND_Input**") PointerPointer input); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( + TRITONBACKEND_Request request, String name, + @ByPtrPtr TRITONBACKEND_Input input); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( + TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, + @ByPtrPtr TRITONBACKEND_Input input); + +/** Get a request input by index. The order of inputs in a given + * request is not necessarily consistent with other requests, even if + * the requests are in the same batch. As a result, you can not + * assume that an index obtained from one request will point to the + * same input in a different request. + * + * The lifetime of the returned input object matches that of the + * request and so the input object should not be accessed after the + * request object is released. + * + * @param request The inference request. + * @param index The index of the input tensor. Must be 0 <= index < + * count, where count is the value returned by + * TRITONBACKEND_RequestInputCount. + * @param input Returns the input corresponding to the index. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputByIndex( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("TRITONBACKEND_Input**") PointerPointer input); +public static native TRITONSERVER_Error TRITONBACKEND_RequestInputByIndex( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @ByPtrPtr TRITONBACKEND_Input input); + +/** Get the number of output tensors requested to be returned in the + * request. + * + * @param request The inference request. + * @param count Returns the number of output tensors. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( + TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); + +/** Get the name of a requested output tensor. The caller does not own + * the returned string and must not modify or delete it. The lifetime + * of the returned string extends only as long as 'request'. + * + * @param request The inference request. + * @param index The index of the requested output tensor. Must be 0 + * <= index < count, where count is the value returned by + * TRITONBACKEND_RequestOutputCount. + * @param output_name Returns the name of the requested output tensor + * corresponding to the index. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") PointerPointer output_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr BytePointer output_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr ByteBuffer output_name); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( + TRITONBACKEND_Request request, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr byte[] output_name); + +/** Returns the preferred memory type and memory type ID of the output buffer + * for the request. As much as possible, Triton will attempt to return + * the same memory_type and memory_type_id values that will be returned by + * the subsequent call to TRITONBACKEND_OutputBuffer, however, the backend must + * be capable of handling cases where the values differ. + * + * @param request The request. + * @param name The name of the output tensor. This is optional + * and it should be set to nullptr to indicate that the tensor name has + * not determined. + * @param byte_size The expected size of the buffer. This is optional + * and it should be set to nullptr to indicate that the byte size has + * not determined. + * @param memory_type Acts as both input and output. On input gives + * the memory type preferred by the caller. Returns memory type preferred + * by Triton, taken account of the caller preferred type. + * @param memory_type_id Acts as both input and output. On input gives + * the memory type ID preferred by the caller. Returns memory type ID preferred + * by Triton, taken account of the caller preferred type ID. + * @return a TRITONSERVER_Error object if a failure occurs. + * A TRITONSERVER_ERROR_UNAVAILABLE error indicates that the properties are not + * available, other error codes indicate an error. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( + TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); + +/** Release the request. The request should be released when it is no + * longer needed by the backend. If this call returns with an error + * (i.e. non-nullptr) then the request was not released and ownership + * remains with the backend. If this call returns with success, the + * 'request' object is no longer owned by the backend and must not be + * used. Any tensor names, data types, shapes, input tensors, + * etc. returned by TRITONBACKEND_Request* functions for this request + * are no longer valid. If a persistent copy of that data is required + * it must be created before calling this function. + * + * @param request The inference request. + * @param release_flags Flags indicating what type of request release + * should be performed. @see TRITONSERVER_RequestReleaseFlag. @see + * TRITONSERVER_InferenceRequestReleaseFn_t. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_RequestRelease( + TRITONBACKEND_Request request, @Cast("uint32_t") int release_flags); + +/** + * TRITONBACKEND_ResponseFactory + * + * Object representing an inference response factory. Using a + * response factory is not required; instead a response can be + * generated directly from a TRITONBACKEND_Request object using + * TRITONBACKEND_ResponseNew(). A response factory allows a request + * to be released before all responses have been sent. Releasing a + * request as early as possible releases all input tensor data and + * therefore may be desirable in some cases. +

+ * Create the response factory associated with a request. + * + * @param factory Returns the new response factory. + * @param request The inference request. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryNew( + @Cast("TRITONBACKEND_ResponseFactory**") PointerPointer factory, TRITONBACKEND_Request request); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryNew( + @ByPtrPtr TRITONBACKEND_ResponseFactory factory, TRITONBACKEND_Request request); + +/** Destroy a response factory. + * + * @param factory The response factory. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryDelete( + TRITONBACKEND_ResponseFactory factory); + +/** Send response flags without a corresponding response. + * + * @param factory The response factory. + * @param send_flags Flags to send. @see + * TRITONSERVER_ResponseCompleteFlag. @see + * TRITONSERVER_InferenceResponseCompleteFn_t. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactorySendFlags( + TRITONBACKEND_ResponseFactory factory, @Cast("const uint32_t") int send_flags); + +/** + * TRITONBACKEND_Response + * + * Object representing an inference response. For a given request, + * the backend must carefully manage the lifecycle of responses + * generated for that request to ensure that the output tensor + * buffers are allocated correctly. When a response is created with + * TRITONBACKEND_ResponseNew or TRITONBACKEND_ResponseNewFromFactory, + * all the outputs and corresponding buffers must be created for that + * response using TRITONBACKEND_ResponseOutput and + * TRITONBACKEND_OutputBuffer *before* another response is created + * for the request. For a given response, outputs can be created in + * any order but they must be created sequentially/sychronously (for + * example, the backend cannot use multiple threads to simultaneously + * add multiple outputs to a response). + * + * The above requirement applies only to responses being generated + * for a given request. The backend may generate responses in + * parallel on multiple threads as long as those responses are for + * different requests. + * + * This order of response creation must be strictly followed. But, + * once response(s) are created they do not need to be sent + * immediately, nor do they need to be sent in the order they were + * created. The backend may even delete a created response instead of + * sending it by using TRITONBACKEND_ResponseDelete. +

+ * Create a response for a request. + * + * @param response Returns the new response. + * @param request The request. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseNew( + @Cast("TRITONBACKEND_Response**") PointerPointer response, TRITONBACKEND_Request request); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseNew( + @ByPtrPtr TRITONBACKEND_Response response, TRITONBACKEND_Request request); + +/** Create a response using a factory. + * + * @param response Returns the new response. + * @param factory The response factory. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseNewFromFactory( + @Cast("TRITONBACKEND_Response**") PointerPointer response, TRITONBACKEND_ResponseFactory factory); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseNewFromFactory( + @ByPtrPtr TRITONBACKEND_Response response, TRITONBACKEND_ResponseFactory factory); + +/** Destroy a response. It is not necessary to delete a response if + * TRITONBACKEND_ResponseSend is called as that function transfers + * ownership of the response object to Triton. + * + * @param response The response. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseDelete( + TRITONBACKEND_Response response); + +/** Set a string parameter in the response. + * + * @param response The response. + * @param name The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetStringParameter( + TRITONBACKEND_Response response, String name, String value); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetStringParameter( + TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer value); + +/** Set an integer parameter in the response. + * + * @param response The response. + * @param name The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetIntParameter( + TRITONBACKEND_Response response, String name, @Cast("const int64_t") long value); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetIntParameter( + TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const int64_t") long value); + +/** Set a boolean parameter in the response. + * + * @param response The response. + * @param name The name of the parameter. + * @param value The value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetBoolParameter( + TRITONBACKEND_Response response, String name, @Cast("const bool") boolean value); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetBoolParameter( + TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const bool") boolean value); + +/** Create an output tensor in the response. The lifetime of the + * returned output tensor object matches that of the response and so + * the output tensor object should not be accessed after the response + * object is deleted. + * + * @param response The response. + * @param output Returns the new response output. + * @param name The name of the output tensor. + * @param datatype The datatype of the output tensor. + * @param shape The shape of the output tensor. + * @param dims_count The number of dimensions in the output tensor + * shape. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @Cast("TRITONBACKEND_Output**") PointerPointer output, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( + TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); + +/** Send a response. Calling this function transfers ownership of the + * response object to Triton. The caller must not access or delete + * the response object after calling this function. + * + * @param response The response. + * @param send_flags Flags associated with the response. @see + * TRITONSERVER_ResponseCompleteFlag. @see + * TRITONSERVER_InferenceResponseCompleteFn_t. + * @param error The TRITONSERVER_Error to send if the response is an + * error, or nullptr if the response is successful. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ResponseSend( + TRITONBACKEND_Response response, @Cast("const uint32_t") int send_flags, + TRITONSERVER_Error error); + +/** + * TRITONBACKEND_State + * + * Object representing a state. + * +

+ * Create a state in the request. The returned state object is only valid + * before the TRITONBACKEND_StateUpdate is called. The state should not be + * freed by the caller. If TRITONBACKEND_StateUpdate is not called, the + * lifetime of the state matches the lifetime of the request. If the state name + * does not exist in the "state" section of the model configuration, the state + * will not be created and an error will be returned. If this function is + * called when sequence batching is not enabled or there is no 'states' section + * in the sequence batching section of the model configuration, this call will + * return an error. + * + * @param state Returns the new state. + * @param request The request. + * @param name The name of the state. + * @param datatype The datatype of the state. + * @param shape The shape of the state. + * @param dims_count The number of dimensions in the state shape. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @Cast("TRITONBACKEND_State**") PointerPointer state, TRITONBACKEND_Request request, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + String name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); +public static native TRITONSERVER_Error TRITONBACKEND_StateNew( + @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, + @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, + @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); + +/** Update the state for the sequence. Calling this function will replace the + * state stored for this seqeunce in Triton with 'state' provided in the + * function argument. If this function is called when sequence batching is not + * enabled or there is no 'states' section in the sequence batching section of + * the model configuration, this call will return an error. The backend is not + * required to call this function. If the backend doesn't call + * TRITONBACKEND_StateUpdate function, this particular state for the sequence + * will not be updated and the next inference request in the sequence will use + * the same state as the current inference request. + * + * @param state The state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_StateUpdate( + TRITONBACKEND_State state); + +/** Get a buffer to use to hold the tensor data for the state. The returned + * buffer is owned by the state and so should not be freed by the caller. The + * caller can and should fill the buffer with the state data. The buffer must + * not be accessed by the backend after TRITONBACKEND_StateUpdate is called. + * The caller should fill the buffer before calling TRITONBACKEND_StateUpdate. + * + * @param state The state. + * @param buffer Returns a pointer to a buffer where the contents of the state + * should be placed. + * @param buffer_byte_size The size, in bytes, of the buffer required + * by the caller. + * @param memory_type Acts as both input and output. On input gives + * the buffer memory type preferred by the caller. Returns the + * actual memory type of 'buffer'. + * @param memory_type_id Acts as both input and output. On input + * gives the buffer memory type id preferred by the caller. Returns + * the actual memory type id of 'buffer'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( + TRITONBACKEND_State state, @Cast("void**") PointerPointer buffer, @Cast("const uint64_t") long buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( + TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( + TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); +public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( + TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, + @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); + +/** Get the buffer attributes associated with the given state buffer. + * The returned 'buffer_attributes' is owned by the state and so should not be + * modified or freed by the caller. The lifetime of the 'buffer_attributes' + * matches that of the state. + * + * @param state The state. + * @param buffer_attributes Returns the buffer attributes for the given state. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_StateBufferAttributes( + TRITONBACKEND_State state, + @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); +public static native TRITONSERVER_Error TRITONBACKEND_StateBufferAttributes( + TRITONBACKEND_State state, + @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); + +/** + * TRITONBACKEND_Backend + * + * Object representing a backend. + * +

+ * TRITONBACKEND_ExecutionPolicy + * + * Types of execution policy that can be implemented by a backend. + * + * TRITONBACKEND_EXECUTION_BLOCKING: An instance of the model + * blocks in TRITONBACKEND_ModelInstanceExecute until it is ready + * to handle another inference. Upon returning from + * TRITONBACKEND_ModelInstanceExecute, Triton may immediately + * call TRITONBACKEND_ModelInstanceExecute for the same instance + * to execute a new batch of requests. Thus, most backends using + * this policy will not return from + * TRITONBACKEND_ModelInstanceExecute until all responses have + * been sent and all requests have been released. This is the + * default execution policy. + * + * TRITONBACKEND_EXECUTION_DEVICE_BLOCKING: An instance, A, of the + * model blocks in TRITONBACKEND_ModelInstanceExecute if the + * device associated with the instance is unable to handle + * another inference. Even if another instance, B, associated + * with the device, is available and ready to perform an + * inference, Triton will not invoke + * TRITONBACKEND_ModeInstanceExecute for B until A returns from + * TRITONBACKEND_ModelInstanceExecute. Triton will not be blocked + * from calling TRITONBACKEND_ModelInstanceExecute for instance + * C, which is associated with a different device than A and B, + * even if A or B has not returned from + * TRITONBACKEND_ModelInstanceExecute. This execution policy is + * typically used by a backend that can cooperatively execute + * multiple model instances on the same device. + * */ +/** enum TRITONBACKEND_ExecutionPolicy */ +public static final int + TRITONBACKEND_EXECUTION_BLOCKING = 0, + TRITONBACKEND_EXECUTION_DEVICE_BLOCKING = 1; + +/** Get the name of the backend. The caller does not own the returned + * string and must not modify or delete it. The lifetime of the + * returned string extends only as long as 'backend'. + * + * @param backend The backend. + * @param name Returns the name of the backend. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendName( + TRITONBACKEND_Backend backend, @Cast("const char**") PointerPointer name); +public static native TRITONSERVER_Error TRITONBACKEND_BackendName( + TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr BytePointer name); +public static native TRITONSERVER_Error TRITONBACKEND_BackendName( + TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr ByteBuffer name); +public static native TRITONSERVER_Error TRITONBACKEND_BackendName( + TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr byte[] name); + +/** Get the backend configuration. The 'backend_config' message is + * owned by Triton and should not be modified or freed by the caller. + * + * The backend configuration, as JSON, is: + * + * { + * "cmdline" : { + * "" : "", + * ... + * } + * } + * + * @param backend The backend. + * @param backend_config Returns the backend configuration as a message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendConfig( + TRITONBACKEND_Backend backend, @Cast("TRITONSERVER_Message**") PointerPointer backend_config); +public static native TRITONSERVER_Error TRITONBACKEND_BackendConfig( + TRITONBACKEND_Backend backend, @ByPtrPtr TRITONSERVER_Message backend_config); + +/** Get the execution policy for this backend. By default the + * execution policy is TRITONBACKEND_EXECUTION_BLOCKING. + * + * @param backend The backend. + * @param policy Returns the execution policy. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") IntPointer policy); +public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") IntBuffer policy); +public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") int[] policy); + +/** Set the execution policy for this backend. By default the + * execution policy is TRITONBACKEND_EXECUTION_BLOCKING. Triton reads + * the backend's execution policy after calling + * TRITONBACKEND_Initialize, so to be recognized changes to the + * execution policy must be made in TRITONBACKEND_Initialize. + * Also, note that if using sequence batcher for the model, Triton will + * use TRITONBACKEND_EXECUTION_BLOCKING policy irrespective of the + * policy specified by this setter function. + * + * @param backend The backend. + * @param policy The execution policy. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendSetExecutionPolicy( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy") int policy); + +/** Get the location of the files that make up the backend + * implementation. This location contains the backend shared library + * and any other files located with the shared library. The + * 'location' communicated depends on how the backend is being + * communicated to Triton as indicated by 'artifact_type'. + * + * TRITONBACKEND_ARTIFACT_FILESYSTEM: The backend artifacts are + * made available to Triton via the local filesytem. 'location' + * returns the full path to the directory containing this + * backend's artifacts. The returned string is owned by Triton, + * not the caller, and so should not be modified or freed. + * + * @param backend The backend. + * @param artifact_type Returns the artifact type for the backend. + * @param path Returns the location. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, + @Cast("const char**") PointerPointer location); +public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, + @Cast("const char**") @ByPtrPtr BytePointer location); +public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntBuffer artifact_type, + @Cast("const char**") @ByPtrPtr ByteBuffer location); +public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") int[] artifact_type, + @Cast("const char**") @ByPtrPtr byte[] location); + +/** Get the memory manager associated with a backend. + * + * @param backend The backend. + * @param manager Returns the memory manager. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendMemoryManager( + TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_MemoryManager**") PointerPointer manager); +public static native TRITONSERVER_Error TRITONBACKEND_BackendMemoryManager( + TRITONBACKEND_Backend backend, @ByPtrPtr TRITONBACKEND_MemoryManager manager); + +/** Get the user-specified state associated with the backend. The + * state is completely owned and managed by the backend. + * + * @param backend The backend. + * @param state Returns the user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendState( + TRITONBACKEND_Backend backend, @Cast("void**") PointerPointer state); +public static native TRITONSERVER_Error TRITONBACKEND_BackendState( + TRITONBACKEND_Backend backend, @Cast("void**") @ByPtrPtr Pointer state); + +/** Set the user-specified state associated with the backend. The + * state is completely owned and managed by the backend. + * + * @param backend The backend. + * @param state The user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendSetState( + TRITONBACKEND_Backend backend, Pointer state); + +/** + * TRITONBACKEND_Model + * + * Object representing a model implemented using the backend. + * +

+ * Get the name of the model. The returned string is owned by the + * model object, not the caller, and so should not be modified or + * freed. + * + * @param model The model. + * @param name Returns the model name. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelName( + TRITONBACKEND_Model model, @Cast("const char**") PointerPointer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelName( + TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr BytePointer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelName( + TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr ByteBuffer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelName( + TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr byte[] name); + +/** Get the version of the model. + * + * @param model The model. + * @param version Returns the model version. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( + TRITONBACKEND_Model model, @Cast("uint64_t*") LongPointer version); +public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( + TRITONBACKEND_Model model, @Cast("uint64_t*") LongBuffer version); +public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( + TRITONBACKEND_Model model, @Cast("uint64_t*") long[] version); + +/** Get the location of the files that make up the model. The + * 'location' communicated depends on how the model is being + * communicated to Triton as indicated by 'artifact_type'. + * + * TRITONBACKEND_ARTIFACT_FILESYSTEM: The model artifacts are made + * available to Triton via the local filesytem. 'location' + * returns the full path to the directory in the model repository + * that contains this model's artifacts. The returned string is + * owned by Triton, not the caller, and so should not be modified + * or freed. + * + * @param model The model. + * @param artifact_type Returns the artifact type for the model. + * @param path Returns the location. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( + TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, + @Cast("const char**") PointerPointer location); +public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( + TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, + @Cast("const char**") @ByPtrPtr BytePointer location); +public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( + TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntBuffer artifact_type, + @Cast("const char**") @ByPtrPtr ByteBuffer location); +public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( + TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") int[] artifact_type, + @Cast("const char**") @ByPtrPtr byte[] location); + +/** Get the model configuration. The caller takes ownership of the + * message object and must call TRITONSERVER_MessageDelete to release + * the object. The configuration is available via this call even + * before the model is loaded and so can be used in + * TRITONBACKEND_ModelInitialize. TRITONSERVER_ServerModelConfig + * returns equivalent information but is not useable until after the + * model loads. + * + * @param model The model. + * @param config_version The model configuration will be returned in + * a format matching this version. If the configuration cannot be + * represented in the requested version's format then an error will + * be returned. Currently only version 1 is supported. + * @param model_config Returns the model configuration as a message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelConfig( + TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, + @Cast("TRITONSERVER_Message**") PointerPointer model_config); +public static native TRITONSERVER_Error TRITONBACKEND_ModelConfig( + TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, + @ByPtrPtr TRITONSERVER_Message model_config); + +/** Whether the backend should attempt to auto-complete the model configuration. + * If true, the model should fill the inputs, outputs, and max batch size in + * the model configuration if incomplete. If the model configuration is + * changed, the new configuration must be reported to Triton using + * TRITONBACKEND_ModelSetConfig. + * + * @param model The model. + * @param auto_complete_config Returns whether the backend should auto-complete + * the model configuration. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( + TRITONBACKEND_Model model, @Cast("bool*") boolean[] auto_complete_config); +public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( + TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); + +/** Set the model configuration in Triton server. This API should only be called + * when the backend implements the auto-completion of model configuration + * and TRITONBACKEND_ModelAutoCompleteConfig returns true in + * auto_complete_config. Only the inputs, outputs, max batch size, and + * scheduling choice can be changed. A caveat being scheduling choice can only + * be changed if none is previously set. Any other changes to the model + * configuration will be ignored by Triton. This function can only be called + * from TRITONBACKEND_ModelInitialize, calling in any other context will result + * in an error being returned. Additionally, Triton server can add some of the + * missing fields in the provided config with this call. The backend must get + * the complete configuration again by using TRITONBACKEND_ModelConfig. + * TRITONBACKEND_ModelSetConfig does not take ownership of the message object + * and so the caller should call TRITONSERVER_MessageDelete to release the + * object once the function returns. + * + * @param model The model. + * @param config_version The format version of the model configuration. + * If the configuration is not represented in the version's format + * then an error will be returned. Currently only version 1 is supported. + * @param model_config The updated model configuration as a message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelSetConfig( + TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, + TRITONSERVER_Message model_config); + +/** Get the TRITONSERVER_Server object that this model is being served + * by. + * + * @param model The model. + * @param server Returns the server. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelServer( + TRITONBACKEND_Model model, @Cast("TRITONSERVER_Server**") PointerPointer server); +public static native TRITONSERVER_Error TRITONBACKEND_ModelServer( + TRITONBACKEND_Model model, @ByPtrPtr TRITONSERVER_Server server); + +/** Get the backend used by the model. + * + * @param model The model. + * @param model Returns the backend object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelBackend( + TRITONBACKEND_Model model, @Cast("TRITONBACKEND_Backend**") PointerPointer backend); +public static native TRITONSERVER_Error TRITONBACKEND_ModelBackend( + TRITONBACKEND_Model model, @ByPtrPtr TRITONBACKEND_Backend backend); + +/** Get the user-specified state associated with the model. The + * state is completely owned and managed by the backend. + * + * @param model The model. + * @param state Returns the user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelState( + TRITONBACKEND_Model model, @Cast("void**") PointerPointer state); +public static native TRITONSERVER_Error TRITONBACKEND_ModelState( + TRITONBACKEND_Model model, @Cast("void**") @ByPtrPtr Pointer state); + +/** Set the user-specified state associated with the model. The + * state is completely owned and managed by the backend. + * + * @param model The model. + * @param state The user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( + TRITONBACKEND_Model model, Pointer state); + +/** + * TRITONBACKEND_ModelInstance + * + * Object representing a model instance implemented using the + * backend. + * +

+ * Get the name of the model instance. The returned string is owned by the + * model object, not the caller, and so should not be modified or + * freed. + * + * @param instance The model instance. + * @param name Returns the instance name. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( + TRITONBACKEND_ModelInstance instance, @Cast("const char**") PointerPointer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( + TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr BytePointer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( + TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr ByteBuffer name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( + TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr byte[] name); + +/** Get the kind of the model instance. + * + * @param instance The model instance. + * @param kind Returns the instance kind. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( + TRITONBACKEND_ModelInstance instance, + @Cast("TRITONSERVER_InstanceGroupKind*") IntPointer kind); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( + TRITONBACKEND_ModelInstance instance, + @Cast("TRITONSERVER_InstanceGroupKind*") IntBuffer kind); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( + TRITONBACKEND_ModelInstance instance, + @Cast("TRITONSERVER_InstanceGroupKind*") int[] kind); + +/** Get the device ID of the model instance. + * + * @param instance The model instance. + * @param device_id Returns the instance device ID. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( + TRITONBACKEND_ModelInstance instance, IntPointer device_id); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( + TRITONBACKEND_ModelInstance instance, IntBuffer device_id); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( + TRITONBACKEND_ModelInstance instance, int[] device_id); + +/** Get the host policy setting. The 'host_policy' message is + * owned by Triton and should not be modified or freed by the caller. + * + * The host policy setting, as JSON, is: + * + * { + * "" : { + * "" : "", + * ... + * } + * } + * + * @param instance The model instance. + * @param host_policy Returns the host policy setting as a message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceHostPolicy( + TRITONBACKEND_ModelInstance instance, @Cast("TRITONSERVER_Message**") PointerPointer host_policy); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceHostPolicy( + TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONSERVER_Message host_policy); + +/** Whether the model instance is passive. + * + * @param instance The model instance. + * @param is_passive Returns true if the instance is passive, false otherwise + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceIsPassive( + TRITONBACKEND_ModelInstance instance, @Cast("bool*") boolean[] is_passive); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceIsPassive( + TRITONBACKEND_ModelInstance instance, @Cast("bool*") BoolPointer is_passive); + +/** Get the number of optimization profiles to be loaded for the instance. + * + * @param instance The model instance. + * @param count Returns the number of optimization profiles. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") int[] count); + +/** Get the name of optimization profile. The caller does not own + * the returned string and must not modify or delete it. The lifetime + * of the returned string extends only as long as 'instance'. + * + * @param instance The model instance. + * @param index The index of the optimization profile. Must be 0 + * <= index < count, where count is the value returned by + * TRITONBACKEND_ModelInstanceProfileCount. + * @param profile_name Returns the name of the optimization profile + * corresponding to the index. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( + TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, + @Cast("const char**") PointerPointer profile_name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( + TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr BytePointer profile_name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( + TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr ByteBuffer profile_name); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( + TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, + @Cast("const char**") @ByPtrPtr byte[] profile_name); + +/** Get the number of secondary devices configured for the instance. + * + * @param instance The model instance. + * @param count Returns the number of secondary devices. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") int[] count); + +/** Get the properties of indexed secondary device. The returned + * strings and other properties are owned by the instance, not the + * caller, and so should not be modified or freed. + * + * @param instance The model instance. + * @param index The index of the secondary device. Must be 0 + * <= index < count, where count is the value returned by + * TRITONBACKEND_ModelInstanceSecondaryDeviceCount. + * @param kind Returns the kind of secondary device corresponding + * to the index. + * @param id Returns the id of secondary device corresponding to the index. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") PointerPointer kind, + @Cast("int64_t*") LongPointer id); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer kind, + @Cast("int64_t*") LongPointer id); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer kind, + @Cast("int64_t*") LongBuffer id); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( + TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] kind, + @Cast("int64_t*") long[] id); + +/** Get the model associated with a model instance. + * + * @param instance The model instance. + * @param backend Returns the model object. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceModel( + TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Model**") PointerPointer model); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceModel( + TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Model model); + +/** Get the user-specified state associated with the model + * instance. The state is completely owned and managed by the + * backend. + * + * @param instance The model instance. + * @param state Returns the user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( + TRITONBACKEND_ModelInstance instance, @Cast("void**") PointerPointer state); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( + TRITONBACKEND_ModelInstance instance, @Cast("void**") @ByPtrPtr Pointer state); + +/** Set the user-specified state associated with the model + * instance. The state is completely owned and managed by the + * backend. + * + * @param instance The model instance. + * @param state The user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( + TRITONBACKEND_ModelInstance instance, Pointer state); + +/** Record statistics for an inference request. + * + * Set 'success' true to indicate that the inference request + * completed successfully. In this case all timestamps should be + * non-zero values reported in nanoseconds and should be collected + * using std::chrono::steady_clock::now().time_since_epoch() or the equivalent. + * Set 'success' to false to indicate that the inference request failed + * to complete successfully. In this case all timestamps values are + * ignored. + * + * For consistency of measurement across different backends, the + * timestamps should be collected at the following points during + * TRITONBACKEND_ModelInstanceExecute. + * + * TRITONBACKEND_ModelInstanceExecute() + * CAPTURE TIMESPACE (exec_start_ns) + * < process input tensors to prepare them for inference + * execution, including copying the tensors to/from GPU if + * necessary> + * CAPTURE TIMESPACE (compute_start_ns) + * < perform inference computations to produce outputs > + * CAPTURE TIMESPACE (compute_end_ns) + * < allocate output buffers and extract output tensors, including + * copying the tensors to/from GPU if necessary> + * CAPTURE TIMESPACE (exec_end_ns) + * return + * + * Note that these statistics are associated with a valid + * TRITONBACKEND_Request object and so must be reported before the + * request is released. For backends that release the request before + * all response(s) are sent, these statistics cannot capture + * information about the time required to produce the response. + * + * @param instance The model instance. + * @param request The inference request that statistics are being + * reported for. + * @param success True if the inference request completed + * successfully, false if it failed to complete. + * @param exec_start_ns Timestamp for the start of execution. + * @param compute_start_ns Timestamp for the start of execution + * computations. + * @param compute_end_ns Timestamp for the end of execution + * computations. + * @param exec_end_ns Timestamp for the end of execution. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportStatistics( + TRITONBACKEND_ModelInstance instance, TRITONBACKEND_Request request, + @Cast("const bool") boolean success, @Cast("const uint64_t") long exec_start_ns, + @Cast("const uint64_t") long compute_start_ns, @Cast("const uint64_t") long compute_end_ns, + @Cast("const uint64_t") long exec_end_ns); + +/** Record statistics for the execution of an entire batch of + * inference requests. + * + * All timestamps should be non-zero values reported in nanoseconds + * and should be collected using + * std::chrono::steady_clock::now().time_since_epoch() or the equivalent. + * See TRITONBACKEND_ModelInstanceReportStatistics for more information about + * the timestamps. + * + * 'batch_size' is the sum of the batch sizes for the individual + * requests that were delivered together in the call to + * TRITONBACKEND_ModelInstanceExecute. For example, if three requests + * are passed to TRITONBACKEND_ModelInstanceExecute and those + * requests have batch size 1, 2, and 3; then 'batch_size' should be + * set to 6. + * + * @param instance The model instance. + * @param batch_size Combined batch size of all the individual + * requests executed in the batch. + * @param exec_start_ns Timestamp for the start of execution. + * @param compute_start_ns Timestamp for the start of execution + * computations. + * @param compute_end_ns Timestamp for the end of execution + * computations. + * @param exec_end_ns Timestamp for the end of execution. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportBatchStatistics( + TRITONBACKEND_ModelInstance instance, @Cast("const uint64_t") long batch_size, + @Cast("const uint64_t") long exec_start_ns, @Cast("const uint64_t") long compute_start_ns, + @Cast("const uint64_t") long compute_end_ns, @Cast("const uint64_t") long exec_end_ns); + +/** + * The following functions can be implemented by a backend. Functions + * indicated as required must be implemented or the backend will fail + * to load. + * +

+ * Initialize a backend. This function is optional, a backend is not + * required to implement it. This function is called once when a + * backend is loaded to allow the backend to initialize any state + * associated with the backend. A backend has a single state that is + * shared across all models that use the backend. + * + * @param backend The backend. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_Initialize( + TRITONBACKEND_Backend backend); + +/** Finalize for a backend. This function is optional, a backend is + * not required to implement it. This function is called once, just + * before the backend is unloaded. All state associated with the + * backend should be freed and any threads created for the backend + * should be exited/joined before returning from this function. + * + * @param backend The backend. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_Finalize( + TRITONBACKEND_Backend backend); + +/** Initialize for a model. This function is optional, a backend is + * not required to implement it. This function is called once when a + * model that uses the backend is loaded to allow the backend to + * initialize any state associated with the model. The backend should + * also examine the model configuration to determine if the + * configuration is suitable for the backend. Any errors reported by + * this function will prevent the model from loading. + * + * @param model The model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInitialize( + TRITONBACKEND_Model model); + +/** Finalize for a model. This function is optional, a backend is not + * required to implement it. This function is called once for a + * model, just before the model is unloaded from Triton. All state + * associated with the model should be freed and any threads created + * for the model should be exited/joined before returning from this + * function. + * + * @param model The model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelFinalize( + TRITONBACKEND_Model model); + +/** Initialize for a model instance. This function is optional, a + * backend is not required to implement it. This function is called + * once when a model instance is created to allow the backend to + * initialize any state associated with the instance. + * + * @param instance The model instance. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance instance); + +/** Finalize for a model instance. This function is optional, a + * backend is not required to implement it. This function is called + * once for an instance, just before the corresponding model is + * unloaded from Triton. All state associated with the instance + * should be freed and any threads created for the instance should be + * exited/joined before returning from this function. + * + * @param instance The model instance. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance instance); + +/** Execute a batch of one or more requests on a model instance. This + * function is required. Triton will not perform multiple + * simultaneous calls to this function for a given model 'instance'; + * however, there may be simultaneous calls for different model + * instances (for the same or different models). + * + * If an error is returned the ownership of the request objects + * remains with Triton and the backend must not retain references to + * the request objects or access them in any way. + * + * If success is returned, ownership of the request objects is + * transferred to the backend and it is then responsible for creating + * responses and releasing the request objects. Note that even though + * ownership of the request objects is transferred to the backend, the + * ownership of the buffer holding request pointers is returned back + * to Triton upon return from TRITONBACKEND_ModelInstanceExecute. If + * any request objects need to be maintained beyond + * TRITONBACKEND_ModelInstanceExecute, then the pointers must be copied + * out of the array within TRITONBACKEND_ModelInstanceExecute. + * + * @param instance The model instance. + * @param requests The requests. + * @param request_count The number of requests in the batch. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Request**") PointerPointer requests, + @Cast("const uint32_t") int request_count); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Request requests, + @Cast("const uint32_t") int request_count); + +/** Query the backend for different model attributes. This function is optional, + * a backend is not required to implement it. The backend is also not required + * to set all backend attribute listed. This function is called when + * Triton requires further backend / model information to perform operations. + * This function may be called multiple times within the lifetime of the + * backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize). + * The backend may return error to indicate failure to set the backend + * attributes, and the attributes specified in the same function call will be + * ignored. Triton will update the specified attributes if 'nullptr' is + * returned. + * + * @param backend The backend. + * @param backend_attributes Return the backend attribute. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_GetBackendAttribute( + TRITONBACKEND_Backend backend, + TRITONBACKEND_BackendAttribute backend_attributes); + +/** TRITONBACKEND_BackendAttribute + * + * API to modify attributes associated with a backend. + * +

+ * Add the preferred instance group of the backend. This function + * can be called multiple times to cover different instance group kinds that + * the backend supports, given the priority order that the first call describes + * the most preferred group. In the case where instance group are not + * explicitly provided, Triton will use this attribute to create model + * deployment that aligns more with the backend preference. + * + * @param backend_attributes The backend attributes object. + * @param kind The kind of the instance group. + * @param count The number of instances per device. Triton default will be used + * if 0 is provided. + * @param device_ids The devices where instances should be available. Triton + * default will be used if 'nullptr' is provided. + * @param id_count The number of devices in 'device_ids'. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongPointer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") LongBuffer device_ids, @Cast("const uint64_t") long id_count); +public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( + TRITONBACKEND_BackendAttribute backend_attributes, + @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, + @Cast("const uint64_t*") long[] device_ids, @Cast("const uint64_t") long id_count); + +/** TRITONBACKEND Batching + * + * API to add custom batching strategy + * + * The following functions can be implemented by a backend to add custom + * batching conditionals on top of the existing Triton batching strategy. The + * functions are optional but all or none must be implemented. + * +

+ * Create a new batcher for use with custom batching. This is called during + * model loading. The batcher will point to a user-defined data structure that + * holds read-only data used for custom batching. + * + * @param batcher User-defined placeholder for backend to store and + * retrieve information about the batching strategy for this + * model.RITONBACKEND_ISPEC return a TRITONSERVER_Error indicating success or + * failure. @param model The backend model for which Triton is forming a batch. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherInitialize( + @Cast("TRITONBACKEND_Batcher**") PointerPointer batcher, TRITONBACKEND_Model model); +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherInitialize( + @ByPtrPtr TRITONBACKEND_Batcher batcher, TRITONBACKEND_Model model); + +/** Free memory associated with batcher. This is called during model unloading. + * + * @param batcher User-defined placeholder for backend to store and + * retrieve information about the batching strategy for this model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherFinalize( + TRITONBACKEND_Batcher batcher); + +/** Check whether a request should be added to the pending model batch. + * + * @param request The request to be added to the pending batch. + * @param userp The placeholder for backend to store and retrieve information + * about this pending batch. When the callback returns, this should reflect + * the latest batch information. + * @param should_include The pointer to be updated on whether the request + * should be included in the batch. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchIncludeRequest( + TRITONBACKEND_Request request, Pointer userp, @Cast("bool*") boolean[] should_include); +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchIncludeRequest( + TRITONBACKEND_Request request, Pointer userp, @Cast("bool*") BoolPointer should_include); + +/** Callback to be invoked when Triton has begun forming a batch. + * + * @param batcher The read-only placeholder for backend to retrieve */ +// information about the batching strategy for this model. +/** @param userp The placeholder for backend to store and retrieve information +/** about this pending batch. +/** @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchInitialize( + @Const TRITONBACKEND_Batcher batcher, @Cast("void**") PointerPointer userp); +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchInitialize( + @Const TRITONBACKEND_Batcher batcher, @Cast("void**") @ByPtrPtr Pointer userp); + +/** Callback to be invoked when Triton has finishing forming a batch. + * + * @param userp The placeholder for backend to store and retrieve information + * about this pending batch. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchFinalize( + Pointer userp); + +// #ifdef __cplusplus +// #endif + + +// Parsed from tritonrepoagent.h + +// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once + +// #include +// #include +// #include "triton/core/tritonserver.h" + +// #ifdef __cplusplus +// #endif + +// #ifdef _COMPILING_TRITONREPOAGENT +// #if defined(_MSC_VER) +// #define TRITONREPOAGENT_DECLSPEC __declspec(dllexport) +// #define TRITONREPOAGENT_ISPEC __declspec(dllimport) +// #elif defined(__GNUC__) +// #define TRITONREPOAGENT_DECLSPEC __attribute__((__visibility__("default"))) +// #define TRITONREPOAGENT_ISPEC +// #else +// #define TRITONREPOAGENT_DECLSPEC +// #define TRITONREPOAGENT_ISPEC +// #endif +// #else +// #if defined(_MSC_VER) +// #define TRITONREPOAGENT_DECLSPEC __declspec(dllimport) +// #define TRITONREPOAGENT_ISPEC __declspec(dllexport) +// #else +// #define TRITONREPOAGENT_DECLSPEC +// #define TRITONREPOAGENT_ISPEC +// Targeting ../tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java + + +// Targeting ../tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java + + + +/** + * TRITONREPOAGENT API Version + * + * The TRITONREPOAGENT API is versioned with major and minor version + * numbers. Any change to the API that does not impact backwards + * compatibility (for example, adding a non-required function) + * increases the minor version number. Any change that breaks + * backwards compatibility (for example, deleting or changing the + * behavior of a function) increases the major version number. A + * repository agent should check that the API version used to compile + * the agent is compatible with the API version of the Triton server + * that it is running in. This is typically done by code similar to + * the following which makes sure that the major versions are equal + * and that the minor version of Triton is >= the minor version used + * to build the agent. + * + * uint32_t api_version_major, api_version_minor; + * TRITONREPOAGENT_ApiVersion(&api_version_major, &api_version_minor); + * if ((api_version_major != TRITONREPOAGENT_API_VERSION_MAJOR) || + * (api_version_minor < TRITONREPOAGENT_API_VERSION_MINOR)) { + * return TRITONSERVER_ErrorNew( + * TRITONSERVER_ERROR_UNSUPPORTED, + * "triton repository agent API version does not support this agent"); + * } + * */ +public static final int TRITONREPOAGENT_API_VERSION_MAJOR = 0; + +/// +public static final int TRITONREPOAGENT_API_VERSION_MINOR = 1; + +/** Get the TRITONREPOAGENT API version supported by Triton. This + * value can be compared against the + * TRITONREPOAGENT_API_VERSION_MAJOR and + * TRITONREPOAGENT_API_VERSION_MINOR used to build the agent to + * ensure that Triton is compatible with the agent. + * + * @param major Returns the TRITONREPOAGENT API major version supported + * by Triton. + * @param minor Returns the TRITONREPOAGENT API minor version supported + * by Triton. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( + @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); +public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( + @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); +public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( + @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); + +/** TRITONREPOAGENT_ArtifactType + * + * The ways that the files that make up a model's repository content + * are communicated between Triton and the agent. + * + * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are + * communicated to and from the repository agent via a locally + * accessible filesystem. The agent can access these files using + * an appropriate filesystem API. + * + * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are + * communicated to and from the repository agent via a remote filesystem. + * The remote filesystem path follows the same convention as is used for + * repository paths, for example, "s3://" prefix indicates an S3 path. + * */ +/** enum TRITONREPOAGENT_ArtifactType */ +public static final int + TRITONREPOAGENT_ARTIFACT_FILESYSTEM = 0, + TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM = 1; + +/** TRITONREPOAGENT_ActionType + * + * Types of repository actions that can be handled by an agent. + * The lifecycle of a TRITONREPOAGENT_AgentModel begins with a call to + * TRITONREPOAGENT_ModelInitialize and ends with a call to + * TRITONREPOAGENT_ModelFinalize. Between those calls the current lifecycle + * state of the model is communicated by calls to TRITONREPOAGENT_ModelAction. + * Possible lifecycles are: + * + * LOAD -> LOAD_COMPLETE -> UNLOAD -> UNLOAD_COMPLETE + * LOAD -> LOAD_FAIL + * + * TRITONREPOAGENT_ACTION_LOAD: A model is being loaded. + * + * TRITONREPOAGENT_ACTION_LOAD_COMPLETE: The model load completed + * successfully and the model is now loaded. + * + * TRITONREPOAGENT_ACTION_LOAD_FAIL: The model load did not complete + * successfully. The model is not loaded. + * + * TRITONREPOAGENT_ACTION_UNLOAD: The model is being unloaded. + * + * TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: The model unload is complete. + * */ +/** enum TRITONREPOAGENT_ActionType */ +public static final int + TRITONREPOAGENT_ACTION_LOAD = 0, + TRITONREPOAGENT_ACTION_LOAD_COMPLETE = 1, + TRITONREPOAGENT_ACTION_LOAD_FAIL = 2, + TRITONREPOAGENT_ACTION_UNLOAD = 3, + TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE = 4; + +/** Get the location of the files that make up the model. The + * 'location' communicated depends on how the model is being + * communicated to the agent as indicated by 'artifact_type'. + * + * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are + * made available to the agent via the local + * filesytem. 'location' returns the full path to the directory + * in the model repository that contains the model's + * artifacts. The returned location string is owned by Triton, + * not the caller, and so should not be modified or freed. The + * contents of the directory are owned by Triton, not the agent, + * and so the agent should not delete or modify the contents. Use + * TRITONREPOAGENT_RepositoryAcquire to get a location that can be + * used to modify the model repository contents. + * + * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are + * made available to the agent via a remote filesystem. + * 'location' returns the full path to the remote directory that contains + * the model's artifacts. The returned location string is owned by Triton, + * not the caller, and so should not be modified or freed. The contents of + * the remote directory are owned by Triton, not the agent, + * and so the agent should not delete or modify the contents. + * Use TRITONREPOAGENT_ModelRepositoryLocationAcquire to get a location + * that can be used to write updated model repository contents. + * + * @param agent The agent. + * @param model The model. + * @param artifact_type Returns the artifact type for the location. + * @param path Returns the location. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("TRITONREPOAGENT_ArtifactType*") IntPointer artifact_type, @Cast("const char**") PointerPointer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("TRITONREPOAGENT_ArtifactType*") IntPointer artifact_type, @Cast("const char**") @ByPtrPtr BytePointer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("TRITONREPOAGENT_ArtifactType*") IntBuffer artifact_type, @Cast("const char**") @ByPtrPtr ByteBuffer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("TRITONREPOAGENT_ArtifactType*") int[] artifact_type, @Cast("const char**") @ByPtrPtr byte[] location); + +/** Acquire a location where the agent can produce a new version of + * the model repository files. This is a convenience method to create + * a temporary directory for the agent. The agent is responsible for + * calling TRITONREPOAGENT_ModelRepositoryLocationDelete in + * TRITONREPOAGENT_ModelFinalize to delete the location. Initially the + * acquired location is empty. The 'location' communicated depends on + * the requested 'artifact_type'. + * + * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The location is a directory + * on the local filesystem. 'location' returns the full path to + * an empty directory that the agent should populate with the + * model's artifacts. The returned location string is owned by + * Triton, not the agent, and so should not be modified or freed. + * + * @param agent The agent. + * @param model The model. + * @param artifact_type The artifact type for the location. + * @param path Returns the location. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") PointerPointer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr BytePointer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr ByteBuffer location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr byte[] location); + +/** Discard and release ownership of a previously acquired location + * and its contents. The agent must not access or modify the location + * or its contents after this call. + * + * @param agent The agent. + * @param model The model. + * @param path The location to release. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationRelease( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + String location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationRelease( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const char*") BytePointer location); + +/** Inform Triton that the specified repository location should be used for + * the model in place of the original model repository. This method can only be + * called when TRITONREPOAGENT_ModelAction is invoked with + * TRITONREPOAGENT_ACTION_LOAD. The 'location' The 'location' + * communicated depends on how the repository is being + * communicated to Triton as indicated by 'artifact_type'. + * + * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are + * made available to Triton via the local filesytem. 'location' returns + * the full path to the directory. Ownership of the contents of the + * returned directory are transferred to Triton and the agent should not + * modified or freed the contents until TRITONREPOAGENT_ModelFinalize. + * The local filesystem directory can be created using + * TRITONREPOAGENT_ModelReopsitroyLocationAcquire or the agent can use + * its own local filesystem API. + * + * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are + * made available to Triton via a remote filesystem. 'location' returns + * the full path to the remote filesystem directory. Ownership of the + * contents of the returned directory are transferred to Triton and + * the agent should not modified or freed the contents until + * TRITONREPOAGENT_ModelFinalize. + * + * @param agent The agent. + * @param model The model. + * @param artifact_type The artifact type for the location. + * @param path Returns the location. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryUpdate( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, String location); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryUpdate( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char*") BytePointer location); + +/** Get the number of agent parameters defined for a model. + * + * @param agent The agent. + * @param model The model. + * @param count Returns the number of input tensors. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("uint32_t*") IntPointer count); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("uint32_t*") IntBuffer count); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("uint32_t*") int[] count); + +/** Get a parameter name and value. The caller does not own the + * returned strings and must not modify or delete them. + * + * @param agent The agent. + * @param model The model. + * @param index The index of the parameter. Must be 0 <= index < + * count, where count is the value returned by + * TRITONREPOAGENT_ModelParameterCount. + * @param parameter_name Returns the name of the parameter. + * @param parameter_value Returns the value of the parameter. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int index, @Cast("const char**") PointerPointer parameter_name, + @Cast("const char**") PointerPointer parameter_value); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer parameter_name, + @Cast("const char**") @ByPtrPtr BytePointer parameter_value); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer parameter_name, + @Cast("const char**") @ByPtrPtr ByteBuffer parameter_value); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] parameter_name, + @Cast("const char**") @ByPtrPtr byte[] parameter_value); + +/** Get the model configuration. The caller takes ownership of the + * message object and must call TRITONSERVER_MessageDelete to release + * the object. If the model repository does not contain a + * config.pbtxt file then 'model_config' is returned as nullptr. + * + * @param agent The agent. + * @param model The model. + * @param config_version The model configuration will be returned in + * a format matching this version. If the configuration cannot be + * represented in the requested version's format then an error will + * be returned. Currently only version 1 is supported. + * @param model_config Returns the model configuration as a message. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelConfig( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int config_version, @Cast("TRITONSERVER_Message**") PointerPointer model_config); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelConfig( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const uint32_t") int config_version, @ByPtrPtr TRITONSERVER_Message model_config); + +/** Get the user-specified state associated with the model. + * + * @param model The agent model. + * @param state Returns the user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelState( + TRITONREPOAGENT_AgentModel model, @Cast("void**") PointerPointer state); +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelState( + TRITONREPOAGENT_AgentModel model, @Cast("void**") @ByPtrPtr Pointer state); + +/** Set the user-specified state associated with the model. + * + * @param model The agent model. + * @param state The user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelSetState( + TRITONREPOAGENT_AgentModel model, Pointer state); + +/** Get the user-specified state associated with the agent. + * + * @param agent The agent. + * @param state Returns the user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_State( + TRITONREPOAGENT_Agent agent, @Cast("void**") PointerPointer state); +public static native TRITONSERVER_Error TRITONREPOAGENT_State( + TRITONREPOAGENT_Agent agent, @Cast("void**") @ByPtrPtr Pointer state); + +/** Set the user-specified state associated with the agent. + * + * @param agent The agent. + * @param state The user state, or nullptr if no user state. + * @return a TRITONSERVER_Error indicating success or failure. */ + + +/// +/// +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_SetState( + TRITONREPOAGENT_Agent agent, Pointer state); + +/** + * The following functions can be implemented by an agent. Functions + * indicated as required must be implemented or the agent will fail + * to load. + * +

+ * Initialize an agent. This function is optional. This function is + * called once when an agent is loaded to allow the agent to + * initialize any state associated with the agent. An agent has a + * single state that is shared across all invocations of the agent. + * + * @param agent The agent. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_Initialize( + TRITONREPOAGENT_Agent agent); + +/** Finalize for an agent. This function is optional. This function is + * called once, just before the agent is unloaded. All state + * associated with the agent should be freed and any threads created + * for the agent should be exited/joined before returning from this + * function. + * + * @param agent The agent. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_Finalize( + TRITONREPOAGENT_Agent agent); + +/** Initialize a model associated with an agent. This function is optional. + * This function is called once when an agent model's lifecycle begins to allow + * the agent model to initialize any state associated with it. An agent model + * has a single state that is shared across all the lifecycle of the agent + * model. + * + * @param agent The agent to be associated with the model. + * @param model The model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelInitialize( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model); + +/** Finalize for a model. This function is optional. This function is + * called once, just before the end of the agent model's lifecycle. All state + * associated with the agent model should be freed and any threads created + * for the agent model should be exited/joined before returning from this + * function. If the model acquired a model location using + * TRITONREPOAGENT_ModelRepositoryLocationAcquire, it must call + * TRITONREPOAGENT_ModelRepositoryLocationRelease to release that location. + * + * @param agent The agent associated with the model. + * @param model The model. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +/// +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelFinalize( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model); + +/** Handle an action for a specified model. This function is + * required. Triton will not perform multiple simultaneous calls to + * this function for a given agent and model; however, there may be + * simultaneous calls for the agent for different models. + * + * If the agent does not handle the action the agent should + * immediately return success (nullptr). + * + * Any modification to the model's repository must be made when 'action_type' + * is TRITONREPOAGENT_ACTION_LOAD. + * To modify the model's repository the agent must either acquire a mutable + * location via TRITONREPOAGENT_ModelRepositoryLocationAcquire + * or its own managed location, report the location to Triton via + * TRITONREPOAGENT_ModelRepositoryUpdate, and then return + * success (nullptr). If the agent does not need to make any changes + * to the model repository it should not call + * TRITONREPOAGENT_ModelRepositoryUpdate and then return success. + * To indicate that a model load should fail return a non-success status. + * + * @param agent The agent. + * @param model The model that is the target of the action. + * \action_type The type of action the agent should handle for the model. + * @return a TRITONSERVER_Error indicating success or failure. */ +public static native TRITONSERVER_Error TRITONREPOAGENT_ModelAction( + TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, + @Cast("const TRITONREPOAGENT_ActionType") int action_type); + +// #ifdef __cplusplus +// #endif + + +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java similarity index 85% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java index 2440e5ec4c4..e211ad88475 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Allocator.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; @@ -14,7 +14,7 @@ /** Custom Allocator object for providing custom functions for allocator. * If there is no custom allocator provided, will use the default allocator. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class Allocator extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java similarity index 88% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java index f27cc1d4081..27cfc52cc0e 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/BackendConfig.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * options. Please refer to the 'Command line options' section in the * documentation of each backend to see the options (e.g. Tensorflow Backend: * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class BackendConfig extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java similarity index 84% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java index 97e0e60e4a2..811f9cb1615 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/CUDAMemoryPoolByteSize.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * data transfer between host and devices until it exceeds the specified byte * size. This will not affect the allocation conducted by the backend * frameworks. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class CUDAMemoryPoolByteSize extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java similarity index 90% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java index 0aa12fe979b..7b999d259dd 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferRequest.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Object that describes an inflight inference request. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class GenericInferRequest extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java similarity index 92% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java index 368d542e85f..ffdf4f46f81 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericInferResult.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** An interface for InferResult object to interpret the response to an * inference request. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class GenericInferResult extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ @@ -35,7 +35,7 @@ public class GenericInferResult extends Pointer { /** Get the output names from the infer result * @return Vector of output names */ public native @ByVal StringVector OutputNames(); - + /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' * field of the output is owned by the returned 'Tensor' object itself. Note * that for string data, need to use 'StringData' function for string data diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java similarity index 94% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java index dcfce89868e..a15dd744917 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/GenericTritonServer.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Object that encapsulates in-process C API functionalities. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class GenericTritonServer extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ @@ -129,6 +129,7 @@ public class GenericTritonServer extends Pointer { * @param repo_path The full path to the model repository. */ public native void UnregisterModelRepo(@StdString BytePointer repo_path); public native void UnregisterModelRepo(@StdString String repo_path); - - public native @UniquePtr GenericInferResult Infer(@ByRef GenericInferRequest infer_request); + + public native @UniquePtr GenericInferResult Infer( + @ByRef GenericInferRequest infer_request); } diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java similarity index 89% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java index 08f8326598d..6cb1418f4b2 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/HostPolicy.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold host policy for setting 'ServerOptions'. * See here for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class HostPolicy extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java similarity index 96% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java index fed63d6fcd6..8463f677c32 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/InferOptions.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold options for Inference Request. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class InferOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java similarity index 94% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java index 5ebf7a51519..828b87092ee 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/LoggingOptions.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold logging options for setting 'ServerOptions'. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class LoggingOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java similarity index 90% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java index 56378a66cb1..d4d2643323c 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/MetricsOptions.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold metrics options for setting 'ServerOptions'. * See here for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class MetricsOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java similarity index 82% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java index e7f01eefbb3..8a5dcabd2d8 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ModelLoadGPULimit.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * The limit on GPU memory usage is specified as a fraction. If model loading * on the device is requested and the current memory usage exceeds the limit, * the load will be rejected. If not specified, the limit will not be set. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class ModelLoadGPULimit extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java similarity index 90% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java index e28c3ba27c7..1b1b1a5b8b2 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/NewModelRepo.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * is used for calling 'TritonServer::RegisterModelRepo' for registering * model repository. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class NewModelRepo extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java similarity index 71% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java index 1871e81f3fa..0981c12bf1d 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/OutputBufferReleaseFn_t.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; -@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class OutputBufferReleaseFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java similarity index 89% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java index 942337971ad..1913b0beab2 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RateLimitResource.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold rate limit resource for setting 'ServerOptions'. See here * for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class RateLimitResource extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java similarity index 90% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java index 0beba9c95b4..c35ad10f30f 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/RepositoryIndex.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold repository index for 'ModelIndex' function. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class RepositoryIndex extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java similarity index 79% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java index 079b2ce7446..dff5f68ac31 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorAllocFn_t.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Custom Response Allocator Callback function signatures. * */ -@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class ResponseAllocatorAllocFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java similarity index 67% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java index 7a53ad140b3..9a857db06db 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ResponseAllocatorStartFn_t.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java @@ -1,16 +1,16 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; /// -@Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class ResponseAllocatorStartFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java similarity index 98% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java index 0c5201f1fc8..34d257b286d 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/ServerOptions.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Server options that are used to initialize Triton Server. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class ServerOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java similarity index 84% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java index 9c1bb2e351f..d01cc4cdd17 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringSet.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; -@Name("std::set") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Name("std::set") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class StringSet extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java similarity index 93% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java index bcba32a1f82..242088d67dd 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/StringVector.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; -@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Name("std::vector") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class StringVector extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java new file mode 100644 index 00000000000..574cfc7ef4f --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Backend extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Backend() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Backend(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java new file mode 100644 index 00000000000..2da20895083 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_BackendAttribute extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_BackendAttribute() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_BackendAttribute(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java new file mode 100644 index 00000000000..3e627e2f39c --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java @@ -0,0 +1,23 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + + +/// +/// +/// +/// +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Batcher extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Batcher() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Batcher(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java new file mode 100644 index 00000000000..73812b97187 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Input extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Input() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Input(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java new file mode 100644 index 00000000000..1cf35b87293 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java @@ -0,0 +1,20 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +// #endif +// #endif + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_MemoryManager extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_MemoryManager() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_MemoryManager(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java new file mode 100644 index 00000000000..783af23deba --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Model extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Model() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Model(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java new file mode 100644 index 00000000000..2319222a839 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_ModelInstance extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_ModelInstance() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_ModelInstance(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java new file mode 100644 index 00000000000..2645a2d3a82 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Output extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Output() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Output(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java new file mode 100644 index 00000000000..1e7e5af8b3f --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Request extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Request() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Request(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java new file mode 100644 index 00000000000..a60df8359c4 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_Response extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_Response() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_Response(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java new file mode 100644 index 00000000000..14ef5ac2cbd --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_ResponseFactory extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_ResponseFactory() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_ResponseFactory(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java new file mode 100644 index 00000000000..ad77aa48b96 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONBACKEND_State extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONBACKEND_State() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONBACKEND_State(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java new file mode 100644 index 00000000000..8379f8e4dbc --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java @@ -0,0 +1,20 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +// #endif +// #endif + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONREPOAGENT_Agent extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONREPOAGENT_Agent() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONREPOAGENT_Agent(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java new file mode 100644 index 00000000000..74d8a32d3ab --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java @@ -0,0 +1,23 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + + +/// +/// +/// +/// +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONREPOAGENT_AgentModel extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONREPOAGENT_AgentModel() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONREPOAGENT_AgentModel(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java new file mode 100644 index 00000000000..427fb6aaf21 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java @@ -0,0 +1,20 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +// #endif +// #endif + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_BufferAttributes extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_BufferAttributes() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_BufferAttributes(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java new file mode 100644 index 00000000000..0278510d8bd --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Error extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Error() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Error(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java new file mode 100644 index 00000000000..8b80120a13f --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceRequest extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_InferenceRequest() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceRequest(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java new file mode 100644 index 00000000000..3d5b2484589 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java @@ -0,0 +1,46 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for inference request release callback function. The callback + * indicates what type of release is being performed on the request + * and for some of these the callback function takes ownership of the + * TRITONSERVER_InferenceRequest object. The 'userp' data is the data + * provided as 'request_release_userp' in the call to + * TRITONSERVER_InferenceRequestSetReleaseCallback. + * + * One or more flags will be specified when the callback is invoked, + * and the callback must take the following actions: + * + * - TRITONSERVER_REQUEST_RELEASE_ALL: The entire inference request + * is being released and ownership is passed to the callback + * function. Triton will not longer access the 'request' object + * itself nor any input tensor data associated with the + * request. The callback should free or otherwise manage the + * 'request' object and all associated tensor data. + * + * Note that currently TRITONSERVER_REQUEST_RELEASE_ALL should always + * be set when the callback is invoked but in the future that may + * change, so the callback should explicitly check for the flag + * before taking ownership of the request object. + * */ + +/// +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceRequestReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceRequestReleaseFn_t(Pointer p) { super(p); } + protected TRITONSERVER_InferenceRequestReleaseFn_t() { allocate(); } + private native void allocate(); + public native void call( + TRITONSERVER_InferenceRequest request, @Cast("const uint32_t") int flags, Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java new file mode 100644 index 00000000000..3f3e0c4f65e --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceResponse extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_InferenceResponse() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceResponse(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java new file mode 100644 index 00000000000..caada929b0c --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java @@ -0,0 +1,41 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for callback function indicating that an inference response + * has completed. The callback function takes ownership of the + * TRITONSERVER_InferenceResponse object. The 'userp' data is the + * data provided as 'response_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * + * One or more flags may be specified when the callback is invoked: + * + * - TRITONSERVER_RESPONSE_COMPLETE_FINAL: Indicates that no more + * responses will be generated for a given request (more + * specifically, that no more responses will be generated for the + * inference request that set this callback and 'userp'). When + * this flag is set 'response' may be a response object or may be + * nullptr. If 'response' is not nullptr, then 'response' is the + * last response that Triton will produce for the request. If + * 'response' is nullptr then Triton is indicating that no more + * responses will be produced for the request. */ + +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceResponseCompleteFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceResponseCompleteFn_t(Pointer p) { super(p); } + protected TRITONSERVER_InferenceResponseCompleteFn_t() { allocate(); } + private native void allocate(); + public native void call( + TRITONSERVER_InferenceResponse response, @Cast("const uint32_t") int flags, + Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java new file mode 100644 index 00000000000..307baec38c8 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceTrace extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_InferenceTrace() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceTrace(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java new file mode 100644 index 00000000000..4190dbe2f65 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java @@ -0,0 +1,29 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for trace timeline activity callback function. This callback function + * is used to report activity occurring for a trace. This function + * does not take ownership of 'trace' and so any information needed + * from that object must be copied before returning. The 'userp' data + * is the same as what is supplied in the call to + * TRITONSERVER_InferenceTraceNew. */ +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceTraceActivityFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceTraceActivityFn_t(Pointer p) { super(p); } + protected TRITONSERVER_InferenceTraceActivityFn_t() { allocate(); } + private native void allocate(); + public native void call( + TRITONSERVER_InferenceTrace trace, + @Cast("TRITONSERVER_InferenceTraceActivity") int activity, @Cast("uint64_t") long timestamp_ns, + Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java new file mode 100644 index 00000000000..29df3a4476d --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java @@ -0,0 +1,30 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for trace release callback function. This callback function + * is called when all activity for the trace has completed. The + * callback function takes ownership of the + * TRITONSERVER_InferenceTrace object. The 'userp' data is the same + * as what is supplied in the call to TRITONSERVER_InferenceTraceNew. */ + +/// +/// +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceTraceReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceTraceReleaseFn_t(Pointer p) { super(p); } + protected TRITONSERVER_InferenceTraceReleaseFn_t() { allocate(); } + private native void allocate(); + public native void call( + TRITONSERVER_InferenceTrace trace, Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java new file mode 100644 index 00000000000..913d36cd6d0 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java @@ -0,0 +1,31 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for trace tensor activity callback function. This callback function + * is used to report tensor activity occurring for a trace. This function + * does not take ownership of 'trace' and so any information needed + * from that object must be copied before returning. The 'userp' data + * is the same as what is supplied in the call to + * TRITONSERVER_InferenceTraceTensorNew. */ +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_InferenceTraceTensorActivityFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_InferenceTraceTensorActivityFn_t(Pointer p) { super(p); } + protected TRITONSERVER_InferenceTraceTensorActivityFn_t() { allocate(); } + private native void allocate(); + public native void call( + TRITONSERVER_InferenceTrace trace, + @Cast("TRITONSERVER_InferenceTraceActivity") int activity, String name, + @Cast("TRITONSERVER_DataType") int datatype, @Const Pointer base, @Cast("size_t") long byte_size, + @Cast("const int64_t*") LongPointer shape, @Cast("uint64_t") long dim_count, + @Cast("TRITONSERVER_MemoryType") int memory_type, @Cast("int64_t") long memory_type_id, Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java new file mode 100644 index 00000000000..cc2537c5153 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Message extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Message() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Message(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java new file mode 100644 index 00000000000..242a48e645e --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Metric extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Metric() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Metric(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java new file mode 100644 index 00000000000..f8aa886086a --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java @@ -0,0 +1,23 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + + +/// +/// +/// +/// +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_MetricFamily extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_MetricFamily() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_MetricFamily(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java new file mode 100644 index 00000000000..8c7f8157d19 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Metrics extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Metrics() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Metrics(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java new file mode 100644 index 00000000000..52ddd9ee2f5 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Parameter extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Parameter() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Parameter(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java new file mode 100644 index 00000000000..d58b84f1534 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocator extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_ResponseAllocator() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocator(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java new file mode 100644 index 00000000000..99ea92db9d7 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java @@ -0,0 +1,63 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** TRITONSERVER_ResponseAllocator + * + * Object representing a memory allocator for output tensors in an + * inference response. + * +

+ * Type for allocation function that allocates a buffer to hold an + * output tensor. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param byte_size The size of the buffer to allocate. + * @param memory_type The type of memory that the caller prefers for + * the buffer allocation. + * @param memory_type_id The ID of the memory that the caller prefers + * for the buffer allocation. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Returns a pointer to the allocated memory. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @param actual_memory_type Returns the type of memory where the + * allocation resides. May be different than the type of memory + * requested by 'memory_type'. + * @param actual_memory_type_id Returns the ID of the memory where + * the allocation resides. May be different than the ID of the memory + * requested by 'memory_type_id'. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocatorAllocFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorAllocFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorAllocFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id, Pointer userp, @Cast("void**") PointerPointer buffer, @Cast("void**") PointerPointer buffer_userp, + @Cast("TRITONSERVER_MemoryType*") IntPointer actual_memory_type, + @Cast("int64_t*") LongPointer actual_memory_type_id); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java new file mode 100644 index 00000000000..123a31eb2a2 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java @@ -0,0 +1,47 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for allocation function that allocates a buffer to hold an + * output tensor with buffer attributes. The callback function must fill in the + * appropriate buffer attributes information related to this buffer. If set, + * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t + * function. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param buffer_attributes The buffer attributes associated with the buffer. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocatorBufferAttributesFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorBufferAttributesFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorBufferAttributesFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + TRITONSERVER_BufferAttributes buffer_attributes, Pointer userp, + Pointer buffer_userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java new file mode 100644 index 00000000000..8b2af5b67aa --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java @@ -0,0 +1,49 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for function that is called to query the allocator's preferred memory + * type and memory type ID. As much as possible, the allocator should attempt + * to return the same memory_type and memory_type_id values that will be + * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. + * But the allocator is not required to do so. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor. This is optional + * and it should be set to nullptr to indicate that the tensor name has + * not determined. + * @param byte_size The expected size of the buffer. This is optional + * and it should be set to nullptr to indicate that the byte size has + * not determined. + * @param memory_type Acts as both input and output. On input gives + * the memory type preferred by the caller. Returns memory type preferred + * by the allocator, taken account of the caller preferred type. + * @param memory_type_id Acts as both input and output. On input gives + * the memory type ID preferred by the caller. Returns memory type ID preferred + * by the allocator, taken account of the caller preferred type ID. + * @return a TRITONSERVER_Error object if a failure occurs. */ + +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocatorQueryFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorQueryFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorQueryFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer userp, + String tensor_name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java new file mode 100644 index 00000000000..bd07e12eddc --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java @@ -0,0 +1,42 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for function that is called when the server no longer holds + * any reference to a buffer allocated by + * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function + * is typically called when the response object associated with the + * buffer is deleted by TRITONSERVER_InferenceResponseDelete. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Pointer to the buffer to be freed. + * @param buffer_userp The user-specified value associated + * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. + * @param byte_size The size of the buffer. + * @param memory_type The type of memory holding the buffer. + * @param memory_type_id The ID of the memory holding the buffer. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting the release. If an error is returned Triton will not + * attempt to release the buffer again. */ + +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocatorReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorReleaseFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorReleaseFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java new file mode 100644 index 00000000000..c4f4ff05455 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java @@ -0,0 +1,37 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + + +/** Type for function that is called to indicate that subsequent + * allocation requests will refer to a new response. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @return a TRITONSERVER_Error object if a failure occurs. */ + +/// +/// +/// +/// +/// +/// +@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ResponseAllocatorStartFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorStartFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorStartFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer userp); +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java new file mode 100644 index 00000000000..8c2b4c140fd --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_Server extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_Server() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_Server(Pointer p) { super(p); } +} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java new file mode 100644 index 00000000000..2ddd5858995 --- /dev/null +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java @@ -0,0 +1,17 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; + +@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) +public class TRITONSERVER_ServerOptions extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public TRITONSERVER_ServerOptions() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ServerOptions(Pointer p) { super(p); } +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java similarity index 95% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java index dd7d0c59cef..1fbd27c8812 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Tensor.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * input/requested output to an inference request, and retrieving the output * result from inference result. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class Tensor extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java similarity index 94% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java index debbb257802..b532c293c78 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/Trace.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * model-specific trace setting for 'InferOptions'. See here for more * information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class Trace extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java similarity index 79% rename from tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java rename to tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java index 5904afd5c77..b147ef88e9f 100644 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/tritonserverwrapper/TritonException.java +++ b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserverwrapper.tritonserverwrapper; +package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserverwrapper.global.tritonserverwrapper.*; +import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; //============================================================================== // TritonException // -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) public class TritonException extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java b/tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java similarity index 83% rename from tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java rename to tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java index 8e4aece0ffa..f30b7d91d3f 100644 --- a/tritonserverwrapper/src/main/java/org/bytedeco/tritonserverwrapper/presets/tritonserverwrapper.java +++ b/tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java @@ -20,7 +20,7 @@ * limitations under the License. */ -package org.bytedeco.tritonserverwrapper.presets; +package org.bytedeco.tritondevelopertoolsserver.presets; import java.util.List; import org.bytedeco.javacpp.ClassProperties; @@ -40,17 +40,17 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"common.h", "generic_server_wrapper.h"}, - link = "tritondevelopertoolsserver", + include = {"common.h", "generic_server_wrapper.h", "tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, + link = {"tritondevelopertoolsserver", "tritonserver"}, includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), }, - target = "org.bytedeco.tritonserverwrapper.tritonserverwrapper", - global = "org.bytedeco.tritonserverwrapper.global.tritonserverwrapper" + target = "org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver", + global = "org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver" ) -public class tritonserverwrapper implements InfoMapper { - static { Loader.checkVersion("org.bytedeco", "tritonserverwrapper"); } +public class tritondevelopertoolsserver implements InfoMapper { + static { Loader.checkVersion("org.bytedeco", "tritondevelopertoolsserver"); } public void map(InfoMap infoMap) { infoMap.putFirst(new Info().enumerate(false)) .put(new Info("bool").cast().valueTypes("boolean").pointerTypes("boolean[]", "BoolPointer")) diff --git a/tritondevelopertoolsserver/src/main/java9/module-info.java b/tritondevelopertoolsserver/src/main/java9/module-info.java new file mode 100644 index 00000000000..0ec0a919305 --- /dev/null +++ b/tritondevelopertoolsserver/src/main/java9/module-info.java @@ -0,0 +1,6 @@ +module org.bytedeco.tritondevelopertoolsserver { + requires transitive org.bytedeco.javacpp; + exports org.bytedeco.tritondevelopertoolsserver.global; + exports org.bytedeco.tritondevelopertoolsserver.presets; + exports org.bytedeco.tritondevelopertoolsserver; +} diff --git a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java b/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java deleted file mode 100644 index db0ef98fba3..00000000000 --- a/tritonserverwrapper/src/gen/java/org/bytedeco/tritonserverwrapper/global/tritonserverwrapper.java +++ /dev/null @@ -1,186 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserverwrapper.global; - -import org.bytedeco.tritonserverwrapper.tritonserverwrapper.*; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -public class tritonserverwrapper extends org.bytedeco.tritonserverwrapper.presets.tritonserverwrapper { - static { Loader.load(); } - -// Targeting ../tritonserverwrapper/StringSet.java - - -// Targeting ../tritonserverwrapper/StringVector.java - - -// Parsed from common.h - -// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include - -//============================================================================== -/** enum classes - * */ -/** enum class triton::developer_tools::server::ModelControlMode */ -public static final int NONE = 0, POLL = 1, EXPLICIT = 2; -/** enum class triton::developer_tools::server::MemoryType */ -public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; -/** enum class triton::developer_tools::server::DataType */ -public static final int - INVALID = 0, - BOOL = 1, - UINT8 = 2, - UINT16 = 3, - UINT32 = 4, - UINT64 = 5, - INT8 = 6, - INT16 = 7, - INT32 = 8, - INT64 = 9, - FP16 = 10, - FP32 = 11, - FP64 = 12, - BYTES = 13, - BF16 = 14; -/** enum class triton::developer_tools::server::ModelReadyState */ -public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; -// Targeting ../tritonserverwrapper/TritonException.java - - -// Targeting ../tritonserverwrapper/ResponseAllocatorAllocFn_t.java - - -// Targeting ../tritonserverwrapper/OutputBufferReleaseFn_t.java - - -// Targeting ../tritonserverwrapper/ResponseAllocatorStartFn_t.java - - -// Targeting ../tritonserverwrapper/LoggingOptions.java - - -// Targeting ../tritonserverwrapper/MetricsOptions.java - - -// Targeting ../tritonserverwrapper/RateLimitResource.java - - -// Targeting ../tritonserverwrapper/ModelLoadGPULimit.java - - -// Targeting ../tritonserverwrapper/Allocator.java - - -// Targeting ../tritonserverwrapper/BackendConfig.java - - -// Targeting ../tritonserverwrapper/CUDAMemoryPoolByteSize.java - - -// Targeting ../tritonserverwrapper/HostPolicy.java - - -// Targeting ../tritonserverwrapper/Trace.java - - -// Targeting ../tritonserverwrapper/ServerOptions.java - - -// Targeting ../tritonserverwrapper/RepositoryIndex.java - - -// Targeting ../tritonserverwrapper/Tensor.java - - -// Targeting ../tritonserverwrapper/NewModelRepo.java - - -// Targeting ../tritonserverwrapper/InferOptions.java - - - - // namespace triton::developer_tools::server - - -// Parsed from generic_server_wrapper.h - -// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include -// #include -// #include "../src/infer_requested_output.h" -// #include "../src/tracer.h" -// #include "common.h" - -/// -// Targeting ../tritonserverwrapper/GenericTritonServer.java - - -// Targeting ../tritonserverwrapper/GenericInferResult.java - - -// Targeting ../tritonserverwrapper/GenericInferRequest.java - - - - // namespace triton::developer_tools::server - - -} diff --git a/tritonserverwrapper/src/main/java9/module-info.java b/tritonserverwrapper/src/main/java9/module-info.java deleted file mode 100644 index d6f16438d12..00000000000 --- a/tritonserverwrapper/src/main/java9/module-info.java +++ /dev/null @@ -1,6 +0,0 @@ -module org.bytedeco.tritonserverwrapper { - requires transitive org.bytedeco.javacpp; - exports org.bytedeco.tritonserverwrapper.global; - exports org.bytedeco.tritonserverwrapper.presets; - exports org.bytedeco.tritonserverwrapper; -} From 6eb48fc8d54125c150bd2d70dd72cb0e0648c08f Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Tue, 30 May 2023 18:17:05 -0700 Subject: [PATCH 25/50] removed developer tools bindings folder and added developer tools build --- platform/pom.xml | 6 - pom.xml | 2 - tritondevelopertoolsserver/README.md | 2 - tritondevelopertoolsserver/cppbuild.sh | 30 - tritondevelopertoolsserver/platform/pom.xml | 126 - .../platform/redist/pom.xml | 106 - tritondevelopertoolsserver/pom.xml | 116 - .../global/tritondevelopertoolsserver.java | 5767 ----------------- .../tritondevelopertoolsserver/Allocator.java | 37 - .../BackendConfig.java | 43 - .../CUDAMemoryPoolByteSize.java | 32 - .../GenericInferRequest.java | 57 - .../GenericInferResult.java | 71 - .../GenericTritonServer.java | 135 - .../HostPolicy.java | 47 - .../InferOptions.java | 109 - .../LoggingOptions.java | 70 - .../MetricsOptions.java | 49 - .../ModelLoadGPULimit.java | 30 - .../NewModelRepo.java | 50 - .../OutputBufferReleaseFn_t.java | 21 - .../RateLimitResource.java | 43 - .../RepositoryIndex.java | 50 - .../ResponseAllocatorAllocFn_t.java | 26 - .../ResponseAllocatorStartFn_t.java | 21 - .../ServerOptions.java | 191 - .../tritondevelopertoolsserver/StringSet.java | 36 - .../StringVector.java | 99 - .../TRITONBACKEND_Backend.java | 17 - .../TRITONBACKEND_BackendAttribute.java | 17 - .../TRITONBACKEND_Batcher.java | 23 - .../TRITONBACKEND_Input.java | 17 - .../TRITONBACKEND_MemoryManager.java | 20 - .../TRITONBACKEND_Model.java | 17 - .../TRITONBACKEND_ModelInstance.java | 17 - .../TRITONBACKEND_Output.java | 17 - .../TRITONBACKEND_Request.java | 17 - .../TRITONBACKEND_Response.java | 17 - .../TRITONBACKEND_ResponseFactory.java | 17 - .../TRITONBACKEND_State.java | 17 - .../TRITONREPOAGENT_Agent.java | 20 - .../TRITONREPOAGENT_AgentModel.java | 23 - .../TRITONSERVER_BufferAttributes.java | 20 - .../TRITONSERVER_Error.java | 17 - .../TRITONSERVER_InferenceRequest.java | 17 - ...TONSERVER_InferenceRequestReleaseFn_t.java | 46 - .../TRITONSERVER_InferenceResponse.java | 17 - ...NSERVER_InferenceResponseCompleteFn_t.java | 41 - .../TRITONSERVER_InferenceTrace.java | 17 - ...ITONSERVER_InferenceTraceActivityFn_t.java | 29 - ...RITONSERVER_InferenceTraceReleaseFn_t.java | 30 - ...RVER_InferenceTraceTensorActivityFn_t.java | 31 - .../TRITONSERVER_Message.java | 17 - .../TRITONSERVER_Metric.java | 17 - .../TRITONSERVER_MetricFamily.java | 23 - .../TRITONSERVER_Metrics.java | 17 - .../TRITONSERVER_Parameter.java | 17 - .../TRITONSERVER_ResponseAllocator.java | 17 - ...ITONSERVER_ResponseAllocatorAllocFn_t.java | 63 - ...ResponseAllocatorBufferAttributesFn_t.java | 47 - ...ITONSERVER_ResponseAllocatorQueryFn_t.java | 49 - ...ONSERVER_ResponseAllocatorReleaseFn_t.java | 42 - ...ITONSERVER_ResponseAllocatorStartFn_t.java | 37 - .../TRITONSERVER_Server.java | 17 - .../TRITONSERVER_ServerOptions.java | 17 - .../tritondevelopertoolsserver/Tensor.java | 81 - .../tritondevelopertoolsserver/Trace.java | 66 - .../TritonException.java | 29 - .../presets/tritondevelopertoolsserver.java | 67 - .../src/main/java9/module-info.java | 6 - .../samples/simplecpp/SimpleCPP.java | 8 +- .../samples/simplecpp}/pom.xml | 13 +- .../tritonserver/presets/tritonserver.java | 21 +- 73 files changed, 22 insertions(+), 8542 deletions(-) delete mode 100644 tritondevelopertoolsserver/README.md delete mode 100644 tritondevelopertoolsserver/cppbuild.sh delete mode 100644 tritondevelopertoolsserver/platform/pom.xml delete mode 100644 tritondevelopertoolsserver/platform/redist/pom.xml delete mode 100644 tritondevelopertoolsserver/pom.xml delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java delete mode 100644 tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java delete mode 100644 tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java delete mode 100644 tritondevelopertoolsserver/src/main/java9/module-info.java rename tritondevelopertoolsserver/samples/simple/Simple.java => tritonserver/samples/simplecpp/SimpleCPP.java (95%) rename {tritondevelopertoolsserver/samples/simple => tritonserver/samples/simplecpp}/pom.xml (59%) diff --git a/platform/pom.xml b/platform/pom.xml index 3417c26647d..44907ba6a54 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -61,7 +61,6 @@ ../tensorflow-lite/platform ../tensorrt/platform ../tritonserver/platform - ../tritondevelopertoolsserver/platform ../ale/platform ../depthai/platform ../onnx/platform @@ -314,11 +313,6 @@ tritonserver-platform 2.33-${project.version} - - org.bytedeco - tritondevelopertoolsserver-platform - 2.35-${project.version} - org.bytedeco ale-platform diff --git a/pom.xml b/pom.xml index a1959109e4c..bbd26497b9e 100644 --- a/pom.xml +++ b/pom.xml @@ -623,7 +623,6 @@ tensorflow-lite tensorrt tritonserver - tritondevelopertoolsserver ale depthai onnx @@ -1407,7 +1406,6 @@ tensorflow-lite tensorrt tritonserver - tritondevelopertoolsserver ale depthai onnx diff --git a/tritondevelopertoolsserver/README.md b/tritondevelopertoolsserver/README.md deleted file mode 100644 index 397e51e4d6c..00000000000 --- a/tritondevelopertoolsserver/README.md +++ /dev/null @@ -1,2 +0,0 @@ -JavaCPP Presets for Triton Inference Server -=========================================== diff --git a/tritondevelopertoolsserver/cppbuild.sh b/tritondevelopertoolsserver/cppbuild.sh deleted file mode 100644 index 958f7730390..00000000000 --- a/tritondevelopertoolsserver/cppbuild.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# This file is meant to be included by the parent cppbuild.sh script -if [[ -z "$PLATFORM" ]]; then - pushd .. - bash cppbuild.sh "$@" tritondevelopertoolsserver - popd - exit -fi - -case $PLATFORM in - linux-arm64) - if [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/developer_tools/" ]]; then - echo "Please make sure library and include files exist" - exit 1 - fi - ;; - linux-x86_64) - if [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/developer_tools/" ]]; then - echo "Please make sure library and include files exist" - exit 1 - fi - ;; - windows-x86_64) - echo "Windows is not supported yet" - exit 1 - ;; - *) - echo "Error: Platform \"$PLATFORM\" is not supported" - ;; -esac diff --git a/tritondevelopertoolsserver/platform/pom.xml b/tritondevelopertoolsserver/platform/pom.xml deleted file mode 100644 index ee34c8727d8..00000000000 --- a/tritondevelopertoolsserver/platform/pom.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - 4.0.0 - - - org.bytedeco - javacpp-presets - 1.5.9-SNAPSHOT - ../../ - - - org.bytedeco - tritondevelopertoolsserver-platform - 2.35-${project.parent.version} - JavaCPP Presets Platform for Triton Inference Server Wrapper - - - tritondevelopertoolsserver - - - - - - ${project.groupId} - ${javacpp.moduleId} - ${project.version} - - - ${project.groupId} - ${javacpp.moduleId} - ${project.version} - ${javacpp.platform.linux-x86_64} - - - - - - - maven-jar-plugin - - - default-jar - - - - ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64.jar ${javacpp.moduleId}-linux-x86_64.jar - - - - - - empty-javadoc-jar - - jar - - - javadoc - - - - empty-sources-jar - - jar - - - sources - - - - - - org.moditect - moditect-maven-plugin - - - add-module-infos - none - - - add-platform-module-info - package - - add-module-info - - - - - ${project.build.directory}/${project.artifactId}.jar - - module org.bytedeco.${javacpp.moduleId}.platform { -// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; - requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64; -// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64; - } - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.4 - - - package - - shade - - - true - - - - - - - - diff --git a/tritondevelopertoolsserver/platform/redist/pom.xml b/tritondevelopertoolsserver/platform/redist/pom.xml deleted file mode 100644 index 28a8c1f2c75..00000000000 --- a/tritondevelopertoolsserver/platform/redist/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - 4.0.0 - - - org.bytedeco - javacpp-presets - 1.5.9-SNAPSHOT - ../../../ - - - org.bytedeco - tritondevelopertoolsserver-platform-redist - 2.35-${project.parent.version} - JavaCPP Presets Platform Redist for Triton Inference Server - - - tritondevelopertoolsserver - -redist - - - - - ${project.groupId} - ${javacpp.moduleId}-platform - ${project.version} - - - ${project.groupId} - ${javacpp.moduleId} - ${project.version} - ${javacpp.platform.linux-x86_64} - - - - - - - maven-jar-plugin - - - default-jar - - - - ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-arm64-redist.jar ${javacpp.moduleId}-linux-x86_64-redist.jar ${javacpp.moduleId}-windows-x86_64-redist.jar - - - - - - empty-javadoc-jar - - jar - - - javadoc - - - - empty-sources-jar - - jar - - - sources - - - - - - org.moditect - moditect-maven-plugin - - - add-module-infos - none - - - add-platform-module-info - package - - add-module-info - - - - - ${project.build.directory}/${project.artifactId}.jar - - module org.bytedeco.${javacpp.moduleId}.platform.redist { -// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64.redist; - requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64.redist; -// requires static org.bytedeco.${javacpp.moduleId}.windows.x86_64.redist; - } - - - - - - - - - - - diff --git a/tritondevelopertoolsserver/pom.xml b/tritondevelopertoolsserver/pom.xml deleted file mode 100644 index 42dcde40f77..00000000000 --- a/tritondevelopertoolsserver/pom.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - 4.0.0 - - - org.bytedeco - javacpp-presets - 1.5.9-SNAPSHOT - - - org.bytedeco - tritondevelopertoolsserver - 2.35-${project.parent.version} - JavaCPP Presets for Triton Inference Server Wrapper - - - - org.bytedeco - javacpp - - - - - - - maven-resources-plugin - - - maven-compiler-plugin - - - org.bytedeco - javacpp - - ISO-8859-1 - - - - maven-jar-plugin - - - javacpp-${javacpp.platform} - package - - jar - - - ${javacpp.platform} - - org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/*jni* - META-INF/native-image/${javacpp.platform}/ - - - - - javacpp-${javacpp.platform}-redist - package - - jar - - - ${javacpp.platform}-redist - ${project.build.directory}/native - - org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/ - META-INF/native-image/${javacpp.platform}/ - - - org/bytedeco/tritondevelopertoolsserver/${javacpp.platform}/*jni* - - - - - - - org.moditect - moditect-maven-plugin - - - add-module-info-redist - package - - add-module-info - - - - - ${project.build.directory}/${project.artifactId}-${javacpp.platform}-redist.jar - - open module org.bytedeco.${javacpp.packageName}.${javacpp.platform.module}.redist { - requires transitive org.bytedeco.${javacpp.packageName}; - } - - - - - - - - - maven-dependency-plugin - - - maven-source-plugin - - - maven-javadoc-plugin - - ISO-8859-1 - - - - - - diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java deleted file mode 100644 index eb34d14bad0..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/global/tritondevelopertoolsserver.java +++ /dev/null @@ -1,5767 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.global; - -import org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver.*; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -public class tritondevelopertoolsserver extends org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver { - static { Loader.load(); } - -// Targeting ../tritondevelopertoolsserver/StringSet.java - - -// Targeting ../tritondevelopertoolsserver/StringVector.java - - -// Parsed from common.h - -// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include - -//============================================================================== -/** enum classes - * */ -/** enum class triton::developer_tools::server::ModelControlMode */ -public static final int NONE = 0, POLL = 1, EXPLICIT = 2; -/** enum class triton::developer_tools::server::MemoryType */ -public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; -/** enum class triton::developer_tools::server::DataType */ -public static final int - INVALID = 0, - BOOL = 1, - UINT8 = 2, - UINT16 = 3, - UINT32 = 4, - UINT64 = 5, - INT8 = 6, - INT16 = 7, - INT32 = 8, - INT64 = 9, - FP16 = 10, - FP32 = 11, - FP64 = 12, - BYTES = 13, - BF16 = 14; -/** enum class triton::developer_tools::server::ModelReadyState */ -public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; -// Targeting ../tritondevelopertoolsserver/TritonException.java - - -// Targeting ../tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java - - -// Targeting ../tritondevelopertoolsserver/OutputBufferReleaseFn_t.java - - -// Targeting ../tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java - - -// Targeting ../tritondevelopertoolsserver/LoggingOptions.java - - -// Targeting ../tritondevelopertoolsserver/MetricsOptions.java - - -// Targeting ../tritondevelopertoolsserver/RateLimitResource.java - - -// Targeting ../tritondevelopertoolsserver/ModelLoadGPULimit.java - - -// Targeting ../tritondevelopertoolsserver/Allocator.java - - -// Targeting ../tritondevelopertoolsserver/BackendConfig.java - - -// Targeting ../tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java - - -// Targeting ../tritondevelopertoolsserver/HostPolicy.java - - -// Targeting ../tritondevelopertoolsserver/Trace.java - - -// Targeting ../tritondevelopertoolsserver/ServerOptions.java - - -// Targeting ../tritondevelopertoolsserver/RepositoryIndex.java - - -// Targeting ../tritondevelopertoolsserver/Tensor.java - - -// Targeting ../tritondevelopertoolsserver/NewModelRepo.java - - -// Targeting ../tritondevelopertoolsserver/InferOptions.java - - - - // namespace triton::developer_tools::server - - -// Parsed from generic_server_wrapper.h - -// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include -// #include -// #include "../src/infer_requested_output.h" -// #include "../src/tracer.h" -// #include "common.h" - -/// -// Targeting ../tritondevelopertoolsserver/GenericTritonServer.java - - -// Targeting ../tritondevelopertoolsserver/GenericInferResult.java - - -// Targeting ../tritondevelopertoolsserver/GenericInferRequest.java - - - - // namespace triton::developer_tools::server - - -// Parsed from tritonserver.h - -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once - -/** \file */ - -// #include -// #include -// #include - -// #ifdef __cplusplus -// #endif - -// #ifdef _COMPILING_TRITONSERVER -// #if defined(_MSC_VER) -// #define TRITONSERVER_DECLSPEC __declspec(dllexport) -// #elif defined(__GNUC__) -// #define TRITONSERVER_DECLSPEC __attribute__((__visibility__("default"))) -// #else -// #define TRITONSERVER_DECLSPEC -// #endif -// #else -// #if defined(_MSC_VER) -// #define TRITONSERVER_DECLSPEC __declspec(dllimport) -// #else -// #define TRITONSERVER_DECLSPEC -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Error.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Message.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Metrics.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Parameter.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Server.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_Metric.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java - - - -/** - * TRITONSERVER API Version - * - * The TRITONSERVER API is versioned with major and minor version - * numbers. Any change to the API that does not impact backwards - * compatibility (for example, adding a non-required function) - * increases the minor version number. Any change that breaks - * backwards compatibility (for example, deleting or changing the - * behavior of a function) increases the major version number. A - * client should check that the API version used to compile the - * client is compatible with the API version of the Triton shared - * library that it is linking against. This is typically done by code - * similar to the following which makes sure that the major versions - * are equal and that the minor version of the Triton shared library - * is >= the minor version used to build the client. - * - * uint32_t api_version_major, api_version_minor; - * TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor); - * if ((api_version_major != TRITONSERVER_API_VERSION_MAJOR) || - * (api_version_minor < TRITONSERVER_API_VERSION_MINOR)) { - * return TRITONSERVER_ErrorNew( - * TRITONSERVER_ERROR_UNSUPPORTED, - * "triton server API version does not support this client"); - * } - * */ -public static final int TRITONSERVER_API_VERSION_MAJOR = 1; - -/// -public static final int TRITONSERVER_API_VERSION_MINOR = 22; - -/** Get the TRITONBACKEND API version supported by the Triton shared - * library. This value can be compared against the - * TRITONSERVER_API_VERSION_MAJOR and TRITONSERVER_API_VERSION_MINOR - * used to build the client to ensure that Triton shared library is - * compatible with the client. - * - * @param major Returns the TRITONSERVER API major version supported - * by Triton. - * @param minor Returns the TRITONSERVER API minor version supported - * by Triton. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( - @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); -public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( - @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); -public static native TRITONSERVER_Error TRITONSERVER_ApiVersion( - @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); - -/** TRITONSERVER_DataType - * - * Tensor data types recognized by TRITONSERVER. - * */ -/** enum TRITONSERVER_DataType */ -public static final int - TRITONSERVER_TYPE_INVALID = 0, - TRITONSERVER_TYPE_BOOL = 1, - TRITONSERVER_TYPE_UINT8 = 2, - TRITONSERVER_TYPE_UINT16 = 3, - TRITONSERVER_TYPE_UINT32 = 4, - TRITONSERVER_TYPE_UINT64 = 5, - TRITONSERVER_TYPE_INT8 = 6, - TRITONSERVER_TYPE_INT16 = 7, - TRITONSERVER_TYPE_INT32 = 8, - TRITONSERVER_TYPE_INT64 = 9, - TRITONSERVER_TYPE_FP16 = 10, - TRITONSERVER_TYPE_FP32 = 11, - TRITONSERVER_TYPE_FP64 = 12, - TRITONSERVER_TYPE_BYTES = 13, - TRITONSERVER_TYPE_BF16 = 14; - -/** Get the string representation of a data type. The returned string - * is not owned by the caller and so should not be modified or freed. - * - * @param datatype The data type. - * @return The string representation of the data type. */ - -/// -public static native String TRITONSERVER_DataTypeString( - @Cast("TRITONSERVER_DataType") int datatype); - -/** Get the Triton datatype corresponding to a string representation - * of a datatype. - * - * @param dtype The datatype string representation. - * @return The Triton data type or TRITONSERVER_TYPE_INVALID if the - * string does not represent a data type. */ - -/// -public static native @Cast("TRITONSERVER_DataType") int TRITONSERVER_StringToDataType(String dtype); -public static native @Cast("TRITONSERVER_DataType") int TRITONSERVER_StringToDataType(@Cast("const char*") BytePointer dtype); - -/** Get the size of a Triton datatype in bytes. Zero is returned for - * TRITONSERVER_TYPE_BYTES because it have variable size. Zero is - * returned for TRITONSERVER_TYPE_INVALID. - * - * @param dtype The datatype. - * @return The size of the datatype. */ - -/// -/// -public static native @Cast("uint32_t") int TRITONSERVER_DataTypeByteSize(@Cast("TRITONSERVER_DataType") int datatype); - -/** TRITONSERVER_MemoryType - * - * Types of memory recognized by TRITONSERVER. - * */ -/** enum TRITONSERVER_MemoryType */ -public static final int - TRITONSERVER_MEMORY_CPU = 0, - TRITONSERVER_MEMORY_CPU_PINNED = 1, - TRITONSERVER_MEMORY_GPU = 2; - -/** Get the string representation of a memory type. The returned - * string is not owned by the caller and so should not be modified or - * freed. - * - * @param memtype The memory type. - * @return The string representation of the memory type. */ - -/// -/// -public static native String TRITONSERVER_MemoryTypeString( - @Cast("TRITONSERVER_MemoryType") int memtype); - -/** TRITONSERVER_ParameterType - * - * Types of parameters recognized by TRITONSERVER. - * */ -/** enum TRITONSERVER_ParameterType */ -public static final int - TRITONSERVER_PARAMETER_STRING = 0, - TRITONSERVER_PARAMETER_INT = 1, - TRITONSERVER_PARAMETER_BOOL = 2, - TRITONSERVER_PARAMETER_BYTES = 3; - -/** Get the string representation of a parameter type. The returned - * string is not owned by the caller and so should not be modified or - * freed. - * - * @param paramtype The parameter type. - * @return The string representation of the parameter type. */ - -/// -public static native String TRITONSERVER_ParameterTypeString( - @Cast("TRITONSERVER_ParameterType") int paramtype); - -/** Create a new parameter object. The caller takes ownership of the - * TRITONSERVER_Parameter object and must call TRITONSERVER_ParameterDelete to - * release the object. The object will maintain its own copy of the 'value' - * - * @param name The parameter name. - * @param type The parameter type. - * @param value The pointer to the value. - * @return A new TRITONSERVER_Parameter object. 'nullptr' will be returned if - * 'type' is 'TRITONSERVER_PARAMETER_BYTES'. The caller should use - * TRITONSERVER_ParameterBytesNew to create parameter with bytes type. */ - -/// -public static native TRITONSERVER_Parameter TRITONSERVER_ParameterNew( - String name, @Cast("const TRITONSERVER_ParameterType") int type, @Const Pointer value); -public static native TRITONSERVER_Parameter TRITONSERVER_ParameterNew( - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_ParameterType") int type, @Const Pointer value); - -/** Create a new parameter object with type TRITONSERVER_PARAMETER_BYTES. - * The caller takes ownership of the TRITONSERVER_Parameter object and must - * call TRITONSERVER_ParameterDelete to release the object. The object only - * maintains a shallow copy of the 'byte_ptr' so the data content must be - * valid until the parameter object is deleted. - * - * @param name The parameter name. - * @param byte_ptr The pointer to the data content. - * @param size The size of the data content. - * @return A new TRITONSERVER_Error object. */ - -/// -public static native TRITONSERVER_Parameter TRITONSERVER_ParameterBytesNew( - String name, @Const Pointer byte_ptr, @Cast("const uint64_t") long size); -public static native TRITONSERVER_Parameter TRITONSERVER_ParameterBytesNew( - @Cast("const char*") BytePointer name, @Const Pointer byte_ptr, @Cast("const uint64_t") long size); - -/** Delete an parameter object. - * - * @param parameter The parameter object. */ - -/// -/// -public static native void TRITONSERVER_ParameterDelete( - TRITONSERVER_Parameter parameter); - -/** TRITONSERVER_InstanceGroupKind - * - * Kinds of instance groups recognized by TRITONSERVER. - * */ -/** enum TRITONSERVER_InstanceGroupKind */ -public static final int - TRITONSERVER_INSTANCEGROUPKIND_AUTO = 0, - TRITONSERVER_INSTANCEGROUPKIND_CPU = 1, - TRITONSERVER_INSTANCEGROUPKIND_GPU = 2, - TRITONSERVER_INSTANCEGROUPKIND_MODEL = 3; - -/** Get the string representation of an instance-group kind. The - * returned string is not owned by the caller and so should not be - * modified or freed. - * - * @param kind The instance-group kind. - * @return The string representation of the kind. */ - -/// -/// -public static native String TRITONSERVER_InstanceGroupKindString( - @Cast("TRITONSERVER_InstanceGroupKind") int kind); - -/** TRITONSERVER_Logging - * - * Types/levels of logging. - * */ -/** enum TRITONSERVER_LogLevel */ -public static final int - TRITONSERVER_LOG_INFO = 0, - TRITONSERVER_LOG_WARN = 1, - TRITONSERVER_LOG_ERROR = 2, - TRITONSERVER_LOG_VERBOSE = 3; - -/** - * Format of logging. - * - * TRITONSERVER_LOG_DEFAULT: the log severity (L) and timestamp will be - * logged as "LMMDD hh:mm:ss.ssssss". - * - * TRITONSERVER_LOG_ISO8601: the log format will be "YYYY-MM-DDThh:mm:ssZ L". - * */ -/** enum TRITONSERVER_LogFormat */ -public static final int - TRITONSERVER_LOG_DEFAULT = 0, - TRITONSERVER_LOG_ISO8601 = 1; - -/** Is a log level enabled? - * - * @param level The log level. - * @return True if the log level is enabled, false if not enabled. */ - -/// -public static native @Cast("bool") boolean TRITONSERVER_LogIsEnabled( - @Cast("TRITONSERVER_LogLevel") int level); - -/** Log a message at a given log level if that level is enabled. - * - * @param level The log level. - * @param filename The file name of the location of the log message. - * @param line The line number of the log message. - * @param msg The log message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_LogMessage( - @Cast("TRITONSERVER_LogLevel") int level, String filename, int line, - String msg); -public static native TRITONSERVER_Error TRITONSERVER_LogMessage( - @Cast("TRITONSERVER_LogLevel") int level, @Cast("const char*") BytePointer filename, int line, - @Cast("const char*") BytePointer msg); - -/** TRITONSERVER_Error - * - * Errors are reported by a TRITONSERVER_Error object. A NULL - * TRITONSERVER_Error indicates no error, a non-NULL TRITONSERVER_Error - * indicates error and the code and message for the error can be - * retrieved from the object. - * - * The caller takes ownership of a TRITONSERVER_Error object returned by - * the API and must call TRITONSERVER_ErrorDelete to release the object. - * -

- * The TRITONSERVER_Error error codes */ -/** enum TRITONSERVER_Error_Code */ -public static final int - TRITONSERVER_ERROR_UNKNOWN = 0, - TRITONSERVER_ERROR_INTERNAL = 1, - TRITONSERVER_ERROR_NOT_FOUND = 2, - TRITONSERVER_ERROR_INVALID_ARG = 3, - TRITONSERVER_ERROR_UNAVAILABLE = 4, - TRITONSERVER_ERROR_UNSUPPORTED = 5, - TRITONSERVER_ERROR_ALREADY_EXISTS = 6; - -/** Create a new error object. The caller takes ownership of the - * TRITONSERVER_Error object and must call TRITONSERVER_ErrorDelete to - * release the object. - * - * @param code The error code. - * @param msg The error message. - * @return A new TRITONSERVER_Error object. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ErrorNew( - @Cast("TRITONSERVER_Error_Code") int code, String msg); -public static native TRITONSERVER_Error TRITONSERVER_ErrorNew( - @Cast("TRITONSERVER_Error_Code") int code, @Cast("const char*") BytePointer msg); - -/** Delete an error object. - * - * @param error The error object. */ - -/// -public static native void TRITONSERVER_ErrorDelete(TRITONSERVER_Error error); - -/** Get the error code. - * - * @param error The error object. - * @return The error code. */ - -/// -public static native @Cast("TRITONSERVER_Error_Code") int TRITONSERVER_ErrorCode(TRITONSERVER_Error error); - -/** Get the string representation of an error code. The returned - * string is not owned by the caller and so should not be modified or - * freed. The lifetime of the returned string extends only as long as - * 'error' and must not be accessed once 'error' is deleted. - * - * @param error The error object. - * @return The string representation of the error code. */ - -/// -public static native String TRITONSERVER_ErrorCodeString( - TRITONSERVER_Error error); - -/** Get the error message. The returned string is not owned by the - * caller and so should not be modified or freed. The lifetime of the - * returned string extends only as long as 'error' and must not be - * accessed once 'error' is deleted. - * - * @param error The error object. - * @return The error message. */ - -/// -/// -/// -public static native String TRITONSERVER_ErrorMessage( - TRITONSERVER_Error error); -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java - - - -/** Create a new response allocator object. - * - * The response allocator object is used by Triton to allocate - * buffers to hold the output tensors in inference responses. Most - * models generate a single response for each inference request - * (TRITONSERVER_TXN_ONE_TO_ONE). For these models the order of - * callbacks will be: - * - * TRITONSERVER_ServerInferAsync called - * - start_fn : optional (and typically not required) - * - alloc_fn : called once for each output tensor in response - * TRITONSERVER_InferenceResponseDelete called - * - release_fn: called once for each output tensor in response - * - * For models that generate multiple responses for each inference - * request (TRITONSERVER_TXN_DECOUPLED), the start_fn callback can be - * used to determine sets of alloc_fn callbacks that belong to the - * same response: - * - * TRITONSERVER_ServerInferAsync called - * - start_fn - * - alloc_fn : called once for each output tensor in response - * - start_fn - * - alloc_fn : called once for each output tensor in response - * ... - * For each response, TRITONSERVER_InferenceResponseDelete called - * - release_fn: called once for each output tensor in the response - * - * In all cases the start_fn, alloc_fn and release_fn callback - * functions must be thread-safe. Typically making these functions - * thread-safe does not require explicit locking. The recommended way - * to implement these functions is to have each inference request - * provide a 'response_allocator_userp' object that is unique to that - * request with TRITONSERVER_InferenceRequestSetResponseCallback. The - * callback functions then operate only on this unique state. Locking - * is required only when the callback function needs to access state - * that is shared across inference requests (for example, a common - * allocation pool). - * - * @param allocator Returns the new response allocator object. - * @param alloc_fn The function to call to allocate buffers for result - * tensors. - * @param release_fn The function to call when the server no longer - * holds a reference to an allocated buffer. - * @param start_fn The function to call to indicate that the - * subsequent 'alloc_fn' calls are for a new response. This callback - * is optional (use nullptr to indicate that it should not be - * invoked). - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( - @Cast("TRITONSERVER_ResponseAllocator**") PointerPointer allocator, - TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - TRITONSERVER_ResponseAllocatorStartFn_t start_fn); -public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( - @ByPtrPtr TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - TRITONSERVER_ResponseAllocatorStartFn_t start_fn); - -/** Set the buffer attributes function for a response allocator object. - * The function will be called after alloc_fn to set the buffer attributes - * associated with the output buffer. - * - * The thread-safy requirement for buffer_attributes_fn is the same as other - * allocator callbacks. - * - * @param allocator The response allocator object. - * @param buffer_attributes_fn The function to call to get the buffer - * attributes information for an allocated buffer. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( - TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); - -/** Set the query function to a response allocator object. Usually the - * function will be called before alloc_fn to understand what is the - * allocator's preferred memory type and memory type ID at the current - * situation to make different execution decision. - * - * The thread-safy requirement for query_fn is the same as other allocator - * callbacks. - * - * @param allocator The response allocator object. - * @param query_fn The function to call to query allocator's preferred memory - * type and memory type ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetQueryFunction( - TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); - -/** Delete a response allocator. - * - * @param allocator The response allocator object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorDelete( - TRITONSERVER_ResponseAllocator allocator); - -/** TRITONSERVER_Message - * - * Object representing a Triton Server message. - * -

- * Create a new message object from serialized JSON string. - * - * @param message The message object. - * @param base The base of the serialized JSON. - * @param byte_size The size, in bytes, of the serialized message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( - @Cast("TRITONSERVER_Message**") PointerPointer message, String base, @Cast("size_t") long byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( - @ByPtrPtr TRITONSERVER_Message message, String base, @Cast("size_t") long byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MessageNewFromSerializedJson( - @ByPtrPtr TRITONSERVER_Message message, @Cast("const char*") BytePointer base, @Cast("size_t") long byte_size); - -/** Delete a message object. - * - * @param message The message object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MessageDelete( - TRITONSERVER_Message message); - -/** Get the base and size of the buffer containing the serialized - * message in JSON format. The buffer is owned by the - * TRITONSERVER_Message object and should not be modified or freed by - * the caller. The lifetime of the buffer extends only as long as - * 'message' and must not be accessed once 'message' is deleted. - * - * @param message The message object. - * @param base Returns the base of the serialized message. - * @param byte_size Returns the size, in bytes, of the serialized - * message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( - TRITONSERVER_Message message, @Cast("const char**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( - TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr BytePointer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( - TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr ByteBuffer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MessageSerializeToJson( - TRITONSERVER_Message message, @Cast("const char**") @ByPtrPtr byte[] base, @Cast("size_t*") SizeTPointer byte_size); - -/** TRITONSERVER_Metrics - * - * Object representing metrics. - * -

- * Metric format types */ -/** enum TRITONSERVER_MetricFormat */ -public static final int - TRITONSERVER_METRIC_PROMETHEUS = 0; - -/** Delete a metrics object. - * - * @param metrics The metrics object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricsDelete( - TRITONSERVER_Metrics metrics); - -/** Get a buffer containing the metrics in the specified format. For - * each format the buffer contains the following: - * - * TRITONSERVER_METRIC_PROMETHEUS: 'base' points to a single multiline - * string (char*) that gives a text representation of the metrics in - * prometheus format. 'byte_size' returns the length of the string - * in bytes. - * - * The buffer is owned by the 'metrics' object and should not be - * modified or freed by the caller. The lifetime of the buffer - * extends only as long as 'metrics' and must not be accessed once - * 'metrics' is deleted. - * - * @param metrics The metrics object. - * @param format The format to use for the returned metrics. - * @param base Returns a pointer to the base of the formatted - * metrics, as described above. - * @param byte_size Returns the size, in bytes, of the formatted - * metrics. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( - TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, - @Cast("const char**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( - TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, - @Cast("const char**") @ByPtrPtr BytePointer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( - TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, - @Cast("const char**") @ByPtrPtr ByteBuffer base, @Cast("size_t*") SizeTPointer byte_size); -public static native TRITONSERVER_Error TRITONSERVER_MetricsFormatted( - TRITONSERVER_Metrics metrics, @Cast("TRITONSERVER_MetricFormat") int format, - @Cast("const char**") @ByPtrPtr byte[] base, @Cast("size_t*") SizeTPointer byte_size); - -/** TRITONSERVER_InferenceTrace - * - * Object that represents tracing for an inference request. - * -

- * Trace levels. The trace level controls the type of trace - * activities that are reported for an inference request. - * - * Trace level values are power-of-2 and can be combined to trace - * multiple types of activities. For example, use - * (TRITONSERVER_TRACE_LEVEL_TIMESTAMPS | - * TRITONSERVER_TRACE_LEVEL_TENSORS) to trace both timestamps and - * tensors for an inference request. - * - * TRITONSERVER_TRACE_LEVEL_MIN and TRITONSERVER_TRACE_LEVEL_MAX are - * deprecated and should not be used. */ -/** enum TRITONSERVER_InferenceTraceLevel */ -public static final int - /** Tracing disabled. No trace activities are reported. */ - TRITONSERVER_TRACE_LEVEL_DISABLED = 0, - /** Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. */ - TRITONSERVER_TRACE_LEVEL_MIN = 1, - /** Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. */ - TRITONSERVER_TRACE_LEVEL_MAX = 2, - /** Record timestamps for the inference request. */ - TRITONSERVER_TRACE_LEVEL_TIMESTAMPS = 0x4, - /** Record input and output tensor values for the inference request. */ - TRITONSERVER_TRACE_LEVEL_TENSORS = 0x8; - -/** Get the string representation of a trace level. The returned - * string is not owned by the caller and so should not be modified or - * freed. - * - * @param level The trace level. - * @return The string representation of the trace level. */ -public static native String TRITONSERVER_InferenceTraceLevelString( - @Cast("TRITONSERVER_InferenceTraceLevel") int level); - -/** Trace activities */ -/** enum TRITONSERVER_InferenceTraceActivity */ -public static final int - TRITONSERVER_TRACE_REQUEST_START = 0, - TRITONSERVER_TRACE_QUEUE_START = 1, - TRITONSERVER_TRACE_COMPUTE_START = 2, - TRITONSERVER_TRACE_COMPUTE_INPUT_END = 3, - TRITONSERVER_TRACE_COMPUTE_OUTPUT_START = 4, - TRITONSERVER_TRACE_COMPUTE_END = 5, - TRITONSERVER_TRACE_REQUEST_END = 6, - TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT = 7, - TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT = 8, - TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT = 9; - -/** Get the string representation of a trace activity. The returned - * string is not owned by the caller and so should not be modified or - * freed. - * - * @param activity The trace activity. - * @return The string representation of the trace activity. */ -public static native String TRITONSERVER_InferenceTraceActivityString( - @Cast("TRITONSERVER_InferenceTraceActivity") int activity); -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java - - - -/** Create a new inference trace object. The caller takes ownership of - * the TRITONSERVER_InferenceTrace object and must call - * TRITONSERVER_InferenceTraceDelete to release the object. - * - * The activity callback function will be called to report activity - * for 'trace' as well as for any child traces that are spawned by - * 'trace', and so the activity callback must check the trace object - * to determine specifically what activity is being reported. - * - * The release callback is called for both 'trace' and for any child - * traces spawned by 'trace'. - * - * @param trace Returns the new inference trace object. - * @param level The tracing level. - * @param parent_id The parent trace id for this trace. A value of 0 - * indicates that there is not parent trace. - * @param activity_fn The callback function where activity for the - * trace is reported. - * @param release_fn The callback function called when all activity - * is complete for the trace. - * @param trace_userp User-provided pointer that is delivered to - * the activity and release callback functions. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceNew( - @Cast("TRITONSERVER_InferenceTrace**") PointerPointer trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, - @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, - TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceNew( - @ByPtrPtr TRITONSERVER_InferenceTrace trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, - @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, - TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); - -/** Create a new inference trace object. The caller takes ownership of - * the TRITONSERVER_InferenceTrace object and must call - * TRITONSERVER_InferenceTraceDelete to release the object. - * - * The timeline and tensor activity callback function will be called to report - * activity for 'trace' as well as for any child traces that are spawned by - * 'trace', and so the activity callback must check the trace object - * to determine specifically what activity is being reported. - * - * The release callback is called for both 'trace' and for any child - * traces spawned by 'trace'. - * - * @param trace Returns the new inference trace object. - * @param level The tracing level. - * @param parent_id The parent trace id for this trace. A value of 0 - * indicates that there is not parent trace. - * @param activity_fn The callback function where timeline activity for the - * trace is reported. - * @param tensor_activity_fn The callback function where tensor activity for - * the trace is reported. - * @param release_fn The callback function called when all activity - * is complete for the trace. - * @param trace_userp User-provided pointer that is delivered to - * the activity and release callback functions. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceTensorNew( - @Cast("TRITONSERVER_InferenceTrace**") PointerPointer trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, - @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, - TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, - TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceTensorNew( - @ByPtrPtr TRITONSERVER_InferenceTrace trace, @Cast("TRITONSERVER_InferenceTraceLevel") int level, - @Cast("uint64_t") long parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, - TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, - TRITONSERVER_InferenceTraceReleaseFn_t release_fn, Pointer trace_userp); - -/** Delete a trace object. - * - * @param trace The trace object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceDelete( - TRITONSERVER_InferenceTrace trace); - -/** Get the id associated with a trace. Every trace is assigned an id - * that is unique across all traces created for a Triton server. - * - * @param trace The trace. - * @param id Returns the id associated with the trace. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongPointer id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongBuffer id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") long[] id); - -/** Get the parent id associated with a trace. The parent id indicates - * a parent-child relationship between two traces. A parent id value - * of 0 indicates that there is no parent trace. - * - * @param trace The trace. - * @param id Returns the parent id associated with the trace. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongPointer parent_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") LongBuffer parent_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceParentId( - TRITONSERVER_InferenceTrace trace, @Cast("uint64_t*") long[] parent_id); - -/** Get the name of the model associated with a trace. The caller does - * not own the returned string and must not modify or delete it. The - * lifetime of the returned string extends only as long as 'trace'. - * - * @param trace The trace. - * @param model_name Returns the name of the model associated with - * the trace. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") PointerPointer model_name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr BytePointer model_name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr ByteBuffer model_name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelName( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr byte[] model_name); - -/** Get the version of the model associated with a trace. - * - * @param trace The trace. - * @param model_version Returns the version of the model associated - * with the trace. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( - TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") LongPointer model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( - TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") LongBuffer model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceModelVersion( - TRITONSERVER_InferenceTrace trace, @Cast("int64_t*") long[] model_version); - -/** Get the request id associated with a trace. The caller does - * not own the returned string and must not modify or delete it. The - * lifetime of the returned string extends only as long as 'trace'. - * - * @param trace The trace. - * @param request_id Returns the version of the model associated - * with the trace. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") PointerPointer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr BytePointer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr ByteBuffer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceTraceRequestId( - TRITONSERVER_InferenceTrace trace, @Cast("const char**") @ByPtrPtr byte[] request_id); - -/** TRITONSERVER_InferenceRequest - * - * Object representing an inference request. The inference request - * provides the meta-data and input tensor values needed for an - * inference and returns the inference result meta-data and output - * tensors. An inference request object can be modified and reused - * multiple times. - * -

- * Inference request flags. The enum values must be power-of-2 values. */ -/** enum TRITONSERVER_RequestFlag */ -public static final int - TRITONSERVER_REQUEST_FLAG_SEQUENCE_START = 1, - TRITONSERVER_REQUEST_FLAG_SEQUENCE_END = 2; - -/** Inference request release flags. The enum values must be - * power-of-2 values. */ -/** enum TRITONSERVER_RequestReleaseFlag */ -public static final int - TRITONSERVER_REQUEST_RELEASE_ALL = 1; - -/** Inference response complete flags. The enum values must be - * power-of-2 values. */ -/** enum TRITONSERVER_ResponseCompleteFlag */ -public static final int - TRITONSERVER_RESPONSE_COMPLETE_FINAL = 1; -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java - - -// Targeting ../tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java - - - -/** Create a new inference request object. - * - * @param inference_request Returns the new request object. - * @param server the inference server object. - * @param model_name The name of the model to use for the request. - * @param model_version The version of the model to use for the - * request. If -1 then the server will choose a version based on the - * model's policy. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( - @Cast("TRITONSERVER_InferenceRequest**") PointerPointer inference_request, - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( - @ByPtrPtr TRITONSERVER_InferenceRequest inference_request, - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestNew( - @ByPtrPtr TRITONSERVER_InferenceRequest inference_request, - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version); - -/** Delete an inference request object. - * - * @param inference_request The request object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestDelete( - TRITONSERVER_InferenceRequest inference_request); - -/** Get the ID for a request. The returned ID is owned by - * 'inference_request' and must not be modified or freed by the - * caller. - * - * @param inference_request The request object. - * @param id Returns the ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") PointerPointer id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr BytePointer id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr ByteBuffer id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestId( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char**") @ByPtrPtr byte[] id); - -/** Set the ID for a request. - * - * @param inference_request The request object. - * @param id The ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetId( - TRITONSERVER_InferenceRequest inference_request, String id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetId( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer id); - -/** Get the flag(s) associated with a request. On return 'flags' holds - * a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for - * available flags. - * - * @param inference_request The request object. - * @param flags Returns the flags. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntPointer flags); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntBuffer flags); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestFlags( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] flags); - -/** Set the flag(s) associated with a request. 'flags' should hold a - * bitwise-or of all flag values, see TRITONSERVER_RequestFlag for - * available flags. - * - * @param inference_request The request object. - * @param flags The flags. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetFlags( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int flags); - -/** Get the correlation ID of the inference request as an unsigned integer. - * Default is 0, which indicates that the request has no correlation ID. - * If the correlation id associated with the inference request is a string, - * this function will return a failure. The correlation ID is used - * to indicate two or more inference request are related to each other. - * How this relationship is handled by the inference server is determined by - * the model's scheduling policy. - * - * @param inference_request The request object. - * @param correlation_id Returns the correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongPointer correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongBuffer correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationId( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") long[] correlation_id); - -/** Get the correlation ID of the inference request as a string. - * Default is empty "", which indicates that the request has no correlation ID. - * If the correlation id associated with the inference request is an unsigned - * integer, then this function will return a failure. The correlation ID - * is used to indicate two or more inference request are related to each other. - * How this relationship is handled by the inference server is determined by - * the model's scheduling policy. - * - * @param inference_request The request object. - * @param correlation_id Returns the correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - @Cast("const char**") PointerPointer correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - @Cast("const char**") @ByPtrPtr BytePointer correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - @Cast("const char**") @ByPtrPtr ByteBuffer correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - @Cast("const char**") @ByPtrPtr byte[] correlation_id); - -/** Set the correlation ID of the inference request to be an unsigned integer. - * Default is 0, which indicates that the request has no correlation ID. - * The correlation ID is used to indicate two or more inference request - * are related to each other. How this relationship is handled by the - * inference server is determined by the model's scheduling policy. - * - * @param inference_request The request object. - * @param correlation_id The correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationId( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long correlation_id); - -/** Set the correlation ID of the inference request to be a string. - * The correlation ID is used to indicate two or more inference - * request are related to each other. How this relationship is - * handled by the inference server is determined by the model's - * scheduling policy. - * - * @param inference_request The request object. - * @param correlation_id The correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - String correlation_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( - TRITONSERVER_InferenceRequest inference_request, - @Cast("const char*") BytePointer correlation_id); - -/** Get the priority for a request. The default is 0 indicating that - * the request does not specify a priority and so will use the - * model's default priority. - * - * @param inference_request The request object. - * @param priority Returns the priority level. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntPointer priority); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") IntBuffer priority); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] priority); - -/** Set the priority for a request. The default is 0 indicating that - * the request does not specify a priority and so will use the - * model's default priority. - * - * @param inference_request The request object. - * @param priority The priority level. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriority( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int priority); - -/** Get the timeout for a request, in microseconds. The default is 0 - * which indicates that the request has no timeout. - * - * @param inference_request The request object. - * @param timeout_us Returns the timeout, in microseconds. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongPointer timeout_us); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") LongBuffer timeout_us); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestTimeoutMicroseconds( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t*") long[] timeout_us); - -/** Set the timeout for a request, in microseconds. The default is 0 - * which indicates that the request has no timeout. - * - * @param inference_request The request object. - * @param timeout_us The timeout, in microseconds. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long timeout_us); - -/** Add an input to a request. - * - * @param inference_request The request object. - * @param name The name of the input. - * @param datatype The type of the input. Valid type names are BOOL, - * UINT8, UINT16, UINT32, UINT64, INT8, INT16, INT32, INT64, FP16, - * FP32, FP64, and BYTES. - * @param shape The shape of the input. - * @param dim_count The number of dimensions of 'shape'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, String name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongPointer shape, - @Cast("uint64_t") long dim_count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongBuffer shape, - @Cast("uint64_t") long dim_count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, String name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") long[] shape, - @Cast("uint64_t") long dim_count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongPointer shape, - @Cast("uint64_t") long dim_count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, String name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") LongBuffer shape, - @Cast("uint64_t") long dim_count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddInput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Cast("const TRITONSERVER_DataType") int datatype, @Cast("const int64_t*") long[] shape, - @Cast("uint64_t") long dim_count); - -/** Add a raw input to a request. The name recognized by the model, data type - * and shape of the input will be deduced from model configuration. - * This function must be called at most once on request with no other input to - * ensure the deduction is accurate. - * - * @param inference_request The request object. - * @param name The name of the input. This name is only used as a reference - * of the raw input in other Tritonserver APIs. It doesn't assoicate with the - * name used in the model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRawInput( - TRITONSERVER_InferenceRequest inference_request, String name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRawInput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); - -/** Remove an input from a request. - * - * @param inference_request The request object. - * @param name The name of the input. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveInput( - TRITONSERVER_InferenceRequest inference_request, String name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveInput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); - -/** Remove all inputs from a request. - * - * @param inference_request The request object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputs( - TRITONSERVER_InferenceRequest inference_request); - -/** Assign a buffer of data to an input. The buffer will be appended - * to any existing buffers for that input. The 'inference_request' - * object takes ownership of the buffer and so the caller should not - * modify or free the buffer until that ownership is released by - * 'inference_request' being deleted or by the input being removed - * from 'inference_request'. - * - * @param inference_request The request object. - * @param name The name of the input. - * @param base The base address of the input data. - * @param byte_size The size, in bytes, of the input data. - * @param memory_type The memory type of the input data. - * @param memory_type_id The memory type id of the input data. - * @return a TRITONSERVER_Error indicating success or failure. */ -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputData( - TRITONSERVER_InferenceRequest inference_request, String name, - @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputData( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id); - -/** Assign a buffer of data to an input for execution on all model instances - * with the specified host policy. The buffer will be appended to any existing - * buffers for that input on all devices with this host policy. The - * 'inference_request' object takes ownership of the buffer and so the caller - * should not modify or free the buffer until that ownership is released by - * 'inference_request' being deleted or by the input being removed from - * 'inference_request'. If the execution is scheduled on a device that does not - * have a input buffer specified using this function, then the input buffer - * specified with TRITONSERVER_InferenceRequestAppendInputData will be used so - * a non-host policy specific version of data must be added using that API. - * @param inference_request The request object. - * @param name The name of the input. - * @param base The base address of the input data. - * @param byte_size The size, in bytes, of the input data. - * @param memory_type The memory type of the input data. - * @param memory_type_id The memory type id of the input data. - * @param host_policy_name All model instances executing with this host_policy - * will use this input buffer for execution. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( - TRITONSERVER_InferenceRequest inference_request, String name, - @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id, String host_policy_name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Const Pointer base, @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id, @Cast("const char*") BytePointer host_policy_name); - -/** Assign a buffer of data to an input. The buffer will be appended - * to any existing buffers for that input. The 'inference_request' - * object takes ownership of the buffer and so the caller should not - * modify or free the buffer until that ownership is released by - * 'inference_request' being deleted or by the input being removed - * from 'inference_request'. - * - * @param inference_request The request object. - * @param name The name of the input. - * @param base The base address of the input data. - * @param buffer_attributes The buffer attrubutes of the input. - * @return a TRITONSERVER_Error indicating success or failure. */ -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( - TRITONSERVER_InferenceRequest inference_request, String name, - @Const Pointer base, TRITONSERVER_BufferAttributes buffer_attributes); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name, - @Const Pointer base, TRITONSERVER_BufferAttributes buffer_attributes); - -/** Clear all input data from an input, releasing ownership of the - * buffer(s) that were appended to the input with - * TRITONSERVER_InferenceRequestAppendInputData or - * TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy - * @param inference_request The request object. - * @param name The name of the input. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputData( - TRITONSERVER_InferenceRequest inference_request, String name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllInputData( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); - -/** Add an output request to an inference request. - * - * @param inference_request The request object. - * @param name The name of the output. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRequestedOutput( - TRITONSERVER_InferenceRequest inference_request, String name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestAddRequestedOutput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); - -/** Remove an output request from an inference request. - * - * @param inference_request The request object. - * @param name The name of the output. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveRequestedOutput( - TRITONSERVER_InferenceRequest inference_request, String name); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveRequestedOutput( - TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer name); - -/** Remove all output requests from an inference request. - * - * @param inference_request The request object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs( - TRITONSERVER_InferenceRequest inference_request); - -/** Set the release callback for an inference request. The release - * callback is called by Triton to return ownership of the request - * object. - * - * @param inference_request The request object. - * @param request_release_fn The function called to return ownership - * of the 'inference_request' object. - * @param request_release_userp User-provided pointer that is - * delivered to the 'request_release_fn' callback. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetReleaseCallback( - TRITONSERVER_InferenceRequest inference_request, - TRITONSERVER_InferenceRequestReleaseFn_t request_release_fn, - Pointer request_release_userp); - -/** Set the allocator and response callback for an inference - * request. The allocator is used to allocate buffers for any output - * tensors included in responses that are produced for this - * request. The response callback is called to return response - * objects representing responses produced for this request. - * - * @param inference_request The request object. - * @param response_allocator The TRITONSERVER_ResponseAllocator to use - * to allocate buffers to hold inference results. - * @param response_allocator_userp User-provided pointer that is - * delivered to the response allocator's start and allocation functions. - * @param response_fn The function called to deliver an inference - * response for this request. - * @param response_userp User-provided pointer that is delivered to - * the 'response_fn' callback. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetResponseCallback( - TRITONSERVER_InferenceRequest inference_request, - TRITONSERVER_ResponseAllocator response_allocator, - Pointer response_allocator_userp, - TRITONSERVER_InferenceResponseCompleteFn_t response_fn, - Pointer response_userp); - -/** Set a string parameter in the request. - * - * @param request The request. - * @param key The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetStringParameter( - TRITONSERVER_InferenceRequest request, String key, String value); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetStringParameter( - TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, @Cast("const char*") BytePointer value); - -/** Set an integer parameter in the request. - * - * @param request The request. - * @param key The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( - TRITONSERVER_InferenceRequest request, String key, - @Cast("const int64_t") long value); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( - TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, - @Cast("const int64_t") long value); - -/** Set a boolean parameter in the request. - * - * @param request The request. - * @param key The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetBoolParameter( - TRITONSERVER_InferenceRequest request, String key, @Cast("const bool") boolean value); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetBoolParameter( - TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, @Cast("const bool") boolean value); - -/** TRITONSERVER_InferenceResponse - * - * Object representing an inference response. The inference response - * provides the meta-data and output tensor values calculated by the - * inference. - * -

- * Delete an inference response object. - * - * @param inference_response The response object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseDelete( - TRITONSERVER_InferenceResponse inference_response); - -/** Return the error status of an inference response. Return a - * TRITONSERVER_Error object on failure, return nullptr on success. - * The returned error object is owned by 'inference_response' and so - * should not be deleted by the caller. - * - * @param inference_response The response object. - * @return a TRITONSERVER_Error indicating the success or failure - * status of the response. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseError( - TRITONSERVER_InferenceResponse inference_response); - -/** Get model used to produce a response. The caller does not own the - * returned model name value and must not modify or delete it. The - * lifetime of all returned values extends until 'inference_response' - * is deleted. - * - * @param inference_response The response object. - * @param model_name Returns the name of the model. - * @param model_version Returns the version of the model. - * this response. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") PointerPointer model_name, - @Cast("int64_t*") LongPointer model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr BytePointer model_name, - @Cast("int64_t*") LongPointer model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr ByteBuffer model_name, - @Cast("int64_t*") LongBuffer model_version); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseModel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const char**") @ByPtrPtr byte[] model_name, - @Cast("int64_t*") long[] model_version); - -/** Get the ID of the request corresponding to a response. The caller - * does not own the returned ID and must not modify or delete it. The - * lifetime of all returned values extends until 'inference_response' - * is deleted. - * - * @param inference_response The response object. - * @param request_id Returns the ID of the request corresponding to - * this response. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( - TRITONSERVER_InferenceResponse inference_response, - @Cast("const char**") PointerPointer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( - TRITONSERVER_InferenceResponse inference_response, - @Cast("const char**") @ByPtrPtr BytePointer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( - TRITONSERVER_InferenceResponse inference_response, - @Cast("const char**") @ByPtrPtr ByteBuffer request_id); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseId( - TRITONSERVER_InferenceResponse inference_response, - @Cast("const char**") @ByPtrPtr byte[] request_id); - -/** Get the number of parameters available in the response. - * - * @param inference_response The response object. - * @param count Returns the number of parameters. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameterCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") int[] count); - -/** Get all information about a parameter. The caller does not own any - * of the returned values and must not modify or delete them. The - * lifetime of all returned values extends until 'inference_response' - * is deleted. - * - * The 'vvalue' returns a void* pointer that must be cast - * appropriately based on 'type'. For example: - * - * void* vvalue; - * TRITONSERVER_ParameterType type; - * TRITONSERVER_InferenceResponseParameter( - * response, index, &name, &type, &vvalue); - * switch (type) { - * case TRITONSERVER_PARAMETER_BOOL: - * bool value = *(reinterpret_cast(vvalue)); - * ... - * case TRITONSERVER_PARAMETER_INT: - * int64_t value = *(reinterpret_cast(vvalue)); - * ... - * case TRITONSERVER_PARAMETER_STRING: - * const char* value = reinterpret_cast(vvalue); - * ... - * - * @param inference_response The response object. - * @param index The index of the parameter, must be 0 <= index < - * count, where 'count' is the value returned by - * TRITONSERVER_InferenceResponseParameterCount. - * @param name Returns the name of the parameter. - * @param type Returns the type of the parameter. - * @param vvalue Returns a pointer to the parameter value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") PointerPointer name, @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") PointerPointer vvalue); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr BytePointer name, @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr ByteBuffer name, @Cast("TRITONSERVER_ParameterType*") IntBuffer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseParameter( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr byte[] name, @Cast("TRITONSERVER_ParameterType*") int[] type, @Cast("const void**") @ByPtrPtr Pointer vvalue); - -/** Get the number of outputs available in the response. - * - * @param inference_response The response object. - * @param count Returns the number of output tensors. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputCount( - TRITONSERVER_InferenceResponse inference_response, @Cast("uint32_t*") int[] count); - -/** Get all information about an output tensor. The tensor data is - * returned as the base pointer to the data and the size, in bytes, - * of the data. The caller does not own any of the returned values - * and must not modify or delete them. The lifetime of all returned - * values extends until 'inference_response' is deleted. - * - * @param inference_response The response object. - * @param index The index of the output tensor, must be 0 <= index < - * count, where 'count' is the value returned by - * TRITONSERVER_InferenceResponseOutputCount. - * @param name Returns the name of the output. - * @param datatype Returns the type of the output. - * @param shape Returns the shape of the output. - * @param dim_count Returns the number of dimensions of the returned - * shape. - * @param base Returns the tensor data for the output. - * @param byte_size Returns the size, in bytes, of the data. - * @param memory_type Returns the memory type of the data. - * @param memory_type_id Returns the memory type id of the data. - * @param userp The user-specified value associated with the buffer - * in TRITONSERVER_ResponseAllocatorAllocFn_t. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") PointerPointer name, @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, - @Cast("uint64_t*") LongPointer dim_count, @Cast("const void**") PointerPointer base, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id, - @Cast("void**") PointerPointer userp); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr BytePointer name, @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, - @Cast("uint64_t*") LongPointer dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id, - @Cast("void**") @ByPtrPtr Pointer userp); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr ByteBuffer name, @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, - @Cast("uint64_t*") LongBuffer dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id, - @Cast("void**") @ByPtrPtr Pointer userp); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutput( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr byte[] name, @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, - @Cast("uint64_t*") long[] dim_count, @Cast("const void**") @ByPtrPtr Pointer base, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id, - @Cast("void**") @ByPtrPtr Pointer userp); - -/** Get a classification label associated with an output for a given - * index. The caller does not own the returned label and must not - * modify or delete it. The lifetime of all returned label extends - * until 'inference_response' is deleted. - * - * @param inference_response The response object. - * @param index The index of the output tensor, must be 0 <= index < - * count, where 'count' is the value returned by - * TRITONSERVER_InferenceResponseOutputCount. - * @param class_index The index of the class. - * @param name Returns the label corresponding to 'class_index' or - * nullptr if no label. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const size_t") long class_index, @Cast("const char**") PointerPointer label); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr BytePointer label); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr ByteBuffer label); -public static native TRITONSERVER_Error TRITONSERVER_InferenceResponseOutputClassificationLabel( - TRITONSERVER_InferenceResponse inference_response, @Cast("const uint32_t") int index, - @Cast("const size_t") long class_index, @Cast("const char**") @ByPtrPtr byte[] label); - -/** TRITONSERVER_BufferAttributes - * - * API to create, modify, or retrieve attributes associated with a buffer. - * -

- * Create a new buffer attributes object. The caller takes ownership of - * the TRITONSERVER_BufferAttributes object and must call - * TRITONSERVER_BufferAttributesDelete to release the object. - * - * @param buffer_attributes Returns the new buffer attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesNew( - @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesNew( - @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); - -/** Delete a buffer attributes object. - * - * @param buffer_attributes The buffer_attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesDelete( - TRITONSERVER_BufferAttributes buffer_attributes); - -/** Set the memory type id field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param memory_type_id Memory type id to assign to the buffer attributes - * object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetMemoryTypeId( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t") long memory_type_id); - -/** Set the memory type field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param memory_type Memory type to assign to the buffer attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetMemoryType( - TRITONSERVER_BufferAttributes buffer_attributes, - @Cast("TRITONSERVER_MemoryType") int memory_type); - -/** Set the CudaIpcHandle field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param cuda_ipc_handle The CudaIpcHandle to assign to the buffer attributes - * object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetCudaIpcHandle( - TRITONSERVER_BufferAttributes buffer_attributes, Pointer cuda_ipc_handle); - -/** Set the byte size field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param byte_size Byte size to assign to the buffer attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesSetByteSize( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("size_t") long byte_size); - -/** Get the memory type id field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param memory_type_id Returns the memory type id associated with the buffer - * attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryTypeId( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("int64_t*") long[] memory_type_id); - -/** Get the memory type field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param memory_type Returns the memory type associated with the buffer - * attributes object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( - TRITONSERVER_BufferAttributes buffer_attributes, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( - TRITONSERVER_BufferAttributes buffer_attributes, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesMemoryType( - TRITONSERVER_BufferAttributes buffer_attributes, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type); - -/** Get the CudaIpcHandle field of the buffer attributes object. - * - * @param buffer_attributes The buffer attributes object. - * @param cuda_ipc_handle Returns the memory type associated with the buffer - * attributes object. If the cudaIpcHandle does not exist for the buffer, - * nullptr will be returned. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesCudaIpcHandle( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("void**") PointerPointer cuda_ipc_handle); -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesCudaIpcHandle( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("void**") @ByPtrPtr Pointer cuda_ipc_handle); - -/** Get the byte size field of the buffer attributes. - * - * @param buffer_attributes The buffer attributes object. - * @param byte_size Returns the byte size associated with the buffer attributes - * object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_BufferAttributesByteSize( - TRITONSERVER_BufferAttributes buffer_attributes, @Cast("size_t*") SizeTPointer byte_size); - - -/** TRITONSERVER_ServerOptions - * - * Options to use when creating an inference server. - * -

- * Model control modes */ -/** enum TRITONSERVER_ModelControlMode */ -public static final int - TRITONSERVER_MODEL_CONTROL_NONE = 0, - TRITONSERVER_MODEL_CONTROL_POLL = 1, - TRITONSERVER_MODEL_CONTROL_EXPLICIT = 2; - -/** Rate limit modes */ -/** enum TRITONSERVER_RateLimitMode */ -public static final int - TRITONSERVER_RATE_LIMIT_OFF = 0, - TRITONSERVER_RATE_LIMIT_EXEC_COUNT = 1; - -/** Create a new server options object. The caller takes ownership of - * the TRITONSERVER_ServerOptions object and must call - * TRITONSERVER_ServerOptionsDelete to release the object. - * - * @param options Returns the new server options object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsNew( - @Cast("TRITONSERVER_ServerOptions**") PointerPointer options); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsNew( - @ByPtrPtr TRITONSERVER_ServerOptions options); - -/** Delete a server options object. - * - * @param options The server options object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsDelete( - TRITONSERVER_ServerOptions options); - -/** Set the textual ID for the server in a server options. The ID is a - * name that identifies the server. - * - * @param options The server options object. - * @param server_id The server identifier. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetServerId( - TRITONSERVER_ServerOptions options, String server_id); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetServerId( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer server_id); - -/** Set the model repository path in a server options. The path must be - * the full absolute path to the model repository. This function can be called - * multiple times with different paths to set multiple model repositories. - * Note that if a model is not unique across all model repositories - * at any time, the model will not be available. - * - * @param options The server options object. - * @param model_repository_path The full path to the model repository. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelRepositoryPath( - TRITONSERVER_ServerOptions options, String model_repository_path); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelRepositoryPath( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer model_repository_path); - -/** Set the model control mode in a server options. For each mode the models - * will be managed as the following: - * - * TRITONSERVER_MODEL_CONTROL_NONE: the models in model repository will be - * loaded on startup. After startup any changes to the model repository will - * be ignored. Calling TRITONSERVER_ServerPollModelRepository will result in - * an error. - * - * TRITONSERVER_MODEL_CONTROL_POLL: the models in model repository will be - * loaded on startup. The model repository can be polled periodically using - * TRITONSERVER_ServerPollModelRepository and the server will load, unload, - * and updated models according to changes in the model repository. - * - * TRITONSERVER_MODEL_CONTROL_EXPLICIT: the models in model repository will - * not be loaded on startup. The corresponding model control APIs must be - * called to load / unload a model in the model repository. - * - * @param options The server options object. - * @param mode The mode to use for the model control. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelControlMode( - TRITONSERVER_ServerOptions options, @Cast("TRITONSERVER_ModelControlMode") int mode); - -/** Set the model to be loaded at startup in a server options. The model must be - * present in one, and only one, of the specified model repositories. - * This function can be called multiple times with different model name - * to set multiple startup models. - * Note that it only takes affect on TRITONSERVER_MODEL_CONTROL_EXPLICIT mode. - * - * @param options The server options object. - * @param mode_name The name of the model to load on startup. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStartupModel( - TRITONSERVER_ServerOptions options, String model_name); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStartupModel( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer model_name); - -/** Enable or disable strict model configuration handling in a server - * options. - * - * @param options The server options object. - * @param strict True to enable strict model configuration handling, - * false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStrictModelConfig( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean strict); - -/** Set the rate limit mode in a server options. - * - * TRITONSERVER_RATE_LIMIT_EXEC_COUNT: The rate limiting prioritizes the - * inference execution using the number of times each instance has got a - * chance to run. The execution gets to run only when its resource - * constraints are satisfied. - * - * TRITONSERVER_RATE_LIMIT_OFF: The rate limiting is turned off and the - * inference gets executed whenever an instance is available. - * - * @param options The server options object. - * @param mode The mode to use for the rate limiting. By default, execution - * count is used to determine the priorities. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRateLimiterMode( - TRITONSERVER_ServerOptions options, @Cast("TRITONSERVER_RateLimitMode") int mode); - -/** Add resource count for rate limiting. - * - * @param options The server options object. - * @param name The name of the resource. - * @param count The count of the resource. - * @param device The device identifier for the resource. A value of -1 - * indicates that the specified number of resources are available on every - * device. The device value is ignored for a global resource. The server - * will use the rate limiter configuration specified for instance groups - * in model config to determine whether resource is global. In case of - * conflicting resource type in different model configurations, server - * will raise an appropriate error while loading model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsAddRateLimiterResource( - TRITONSERVER_ServerOptions options, String resource_name, - @Cast("const size_t") long resource_count, int device); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsAddRateLimiterResource( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer resource_name, - @Cast("const size_t") long resource_count, int device); - -/** Set the total pinned memory byte size that the server can allocate - * in a server options. The pinned memory pool will be shared across - * Triton itself and the backends that use - * TRITONBACKEND_MemoryManager to allocate memory. - * - * @param options The server options object. - * @param size The pinned memory pool byte size. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( - TRITONSERVER_ServerOptions options, @Cast("uint64_t") long size); - -/** Set the total CUDA memory byte size that the server can allocate - * on given GPU device in a server options. The pinned memory pool - * will be shared across Triton itself and the backends that use - * TRITONBACKEND_MemoryManager to allocate memory. - * - * @param options The server options object. - * @param gpu_device The GPU device to allocate the memory pool. - * @param size The CUDA memory pool byte size. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( - TRITONSERVER_ServerOptions options, int gpu_device, @Cast("uint64_t") long size); - -/** Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead. - * - * Set the total response cache byte size that the server can allocate in CPU - * memory. The response cache will be shared across all inference requests and - * across all models. - * - * @param options The server options object. - * @param size The total response cache byte size. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetResponseCacheByteSize( - TRITONSERVER_ServerOptions options, @Cast("uint64_t") long size); - -/** Set the cache config that will be used to initialize the cache - * implementation for "cache_name". - * - * It is expected that the "cache_name" provided matches a directory inside - * the "cache_dir" used for TRITONSERVER_ServerOptionsSetCacheDirectory. The - * default "cache_dir" is "/opt/tritonserver/caches", so for a "cache_name" of - * "local", Triton would expect to find the "local" cache implementation at - * "/opt/tritonserver/caches/local/libtritoncache_local.so" - * - * Altogether an example for the "local" cache implementation would look like: - * std::string cache_name = "local"; - * std::string config_json = R"({"size": 1048576})" - * auto err = TRITONSERVER_ServerOptionsSetCacheConfig( - * options, cache_name, config_json); - * - * @param options The server options object. - * @param cache_name The name of the cache. Example names would be - * "local", "redis", or the name of a custom cache implementation. - * @param config_json The string representation of config JSON that is - * used to initialize the cache implementation. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheConfig( - TRITONSERVER_ServerOptions options, String cache_name, - String config_json); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheConfig( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer cache_name, - @Cast("const char*") BytePointer config_json); - -/** Set the directory containing cache shared libraries. This - * directory is searched when looking for cache implementations. - * - * @param options The server options object. - * @param cache_dir The full path of the cache directory. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheDirectory( - TRITONSERVER_ServerOptions options, String cache_dir); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCacheDirectory( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer cache_dir); - -/** Set the minimum support CUDA compute capability in a server - * options. - * - * @param options The server options object. - * @param cc The minimum CUDA compute capability. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( - TRITONSERVER_ServerOptions options, double cc); - -/** Enable or disable exit-on-error in a server options. - * - * @param options The server options object. - * @param exit True to enable exiting on intialization error, false - * to continue. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitOnError( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean exit); - -/** Enable or disable strict readiness handling in a server options. - * - * @param options The server options object. - * @param strict True to enable strict readiness handling, false to - * disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetStrictReadiness( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean strict); - -/** Set the exit timeout, in seconds, for the server in a server - * options. - * - * @param options The server options object. - * @param timeout The exit timeout, in seconds. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetExitTimeout( - TRITONSERVER_ServerOptions options, @Cast("unsigned int") int timeout); - -/** Set the number of threads used in buffer manager in a server options. - * - * @param options The server options object. - * @param thread_count The number of threads. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( - TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); - -/** Set the number of threads to concurrently load models in a server options. - * - * @param options The server options object. - * @param thread_count The number of threads. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadThreadCount( - TRITONSERVER_ServerOptions options, @Cast("unsigned int") int thread_count); - -/** Enable model namespacing to allow serving models with the same name if - * they are in different namespaces. - * - * @param options The server options object. - * @param enable_namespace Whether to enable model namespacing or not. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelNamespacing( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean enable_namespace); - -/** Provide a log output file. - * - * @param options The server options object. - * @param file a string defining the file where the log outputs will be saved. - * An empty string for the file name will cause triton to direct logging - * facilities to the console - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( - TRITONSERVER_ServerOptions options, String file); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFile( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer file); - -/** Enable or disable info level logging. - * - * @param options The server options object. - * @param log True to enable info logging, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogInfo( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); - -/** Enable or disable warning level logging. - * - * @param options The server options object. - * @param log True to enable warning logging, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogWarn( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); - -/** Enable or disable error level logging. - * - * @param options The server options object. - * @param log True to enable error logging, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogError( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean log); - -/** Set the logging format. - * - * @param options The server options object. - * @param format The logging format. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogFormat( - TRITONSERVER_ServerOptions options, @Cast("const TRITONSERVER_LogFormat") int format); - -/** Set verbose logging level. Level zero disables verbose logging. - * - * @param options The server options object. - * @param level The verbose logging level. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetLogVerbose( - TRITONSERVER_ServerOptions options, int level); - -/** Enable or disable metrics collection in a server options. - * - * @param options The server options object. - * @param metrics True to enable metrics, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetrics( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean metrics); - -/** Enable or disable GPU metrics collection in a server options. GPU - * metrics are collected if both this option and - * TRITONSERVER_ServerOptionsSetMetrics are true. - * - * @param options The server options object. - * @param gpu_metrics True to enable GPU metrics, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetGpuMetrics( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean gpu_metrics); - -/** Enable or disable CPU metrics collection in a server options. CPU - * metrics are collected if both this option and - * TRITONSERVER_ServerOptionsSetMetrics are true. - * - * @param options The server options object. - * @param cpu_metrics True to enable CPU metrics, false to disable. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetCpuMetrics( - TRITONSERVER_ServerOptions options, @Cast("bool") boolean cpu_metrics); - -/** Set the interval for metrics collection in a server options. - * This is 2000 milliseconds by default. - * - * @param options The server options object. - * @param metrics_interval_ms The time interval in ms between - * successive metrics updates. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsInterval( - TRITONSERVER_ServerOptions options, @Cast("uint64_t") long metrics_interval_ms); - -/** Set the directory containing backend shared libraries. This - * directory is searched last after the version and model directory - * in the model repository when looking for the backend shared - * library for a model. If the backend is named 'be' the directory - * searched is 'backend_dir'/be/libtriton_be.so. - * - * @param options The server options object. - * @param backend_dir The full path of the backend directory. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDirectory( - TRITONSERVER_ServerOptions options, String backend_dir); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendDirectory( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer backend_dir); - -/** Set the directory containing repository agent shared libraries. This - * directory is searched when looking for the repository agent shared - * library for a model. If the repo agent is named 'ra' the directory - * searched is 'repoagent_dir'/ra/libtritonrepoagent_ra.so. - * - * @param options The server options object. - * @param repoagent_dir The full path of the repository agent directory. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( - TRITONSERVER_ServerOptions options, String repoagent_dir); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetRepoAgentDirectory( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer repoagent_dir); - -/** Specify the limit on memory usage as a fraction on the device identified by - * 'kind' and 'device_id'. If model loading on the device is requested and the - * current memory usage exceeds the limit, the load will be rejected. If not - * specified, the limit will not be set. - * - * Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU - * - * @param options The server options object. - * @param kind The kind of the device. - * @param device_id The id of the device. - * @param fraction The limit on memory usage as a fraction - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( - TRITONSERVER_ServerOptions options, - @Cast("const TRITONSERVER_InstanceGroupKind") int kind, int device_id, - double fraction); - -/** Set a configuration setting for a named backend in a server - * options. - * - * @param options The server options object. - * @param backend_name The name of the backend. - * @param setting The name of the setting. - * @param value The setting value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendConfig( - TRITONSERVER_ServerOptions options, String backend_name, - String setting, String value); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendConfig( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer backend_name, - @Cast("const char*") BytePointer setting, @Cast("const char*") BytePointer value); - -/** Set a host policy setting for a given policy name in a server options. - * - * @param options The server options object. - * @param policy_name The name of the policy. - * @param setting The name of the setting. - * @param value The setting value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( - TRITONSERVER_ServerOptions options, String policy_name, - String setting, String value); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer policy_name, - @Cast("const char*") BytePointer setting, @Cast("const char*") BytePointer value); - -/** Set a configuration setting for metrics in server options. - * - * @param options The server options object. - * @param name The name of the configuration group. An empty string indicates - * a global configuration option. - * @param setting The name of the setting. - * @param value The setting value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( - TRITONSERVER_ServerOptions options, String name, String setting, - String value); -public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( - TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer setting, - @Cast("const char*") BytePointer value); - -/** TRITONSERVER_Server - * - * An inference server. - * -

- * Model batch flags. The enum values must be power-of-2 values. */ -/** enum TRITONSERVER_ModelBatchFlag */ -public static final int - TRITONSERVER_BATCH_UNKNOWN = 1, - TRITONSERVER_BATCH_FIRST_DIM = 2; - -/** Model index flags. The enum values must be power-of-2 values. */ -/** enum TRITONSERVER_ModelIndexFlag */ -public static final int - TRITONSERVER_INDEX_FLAG_READY = 1; - -/** Model transaction policy flags. The enum values must be - * power-of-2 values. */ -/** enum TRITONSERVER_ModelTxnPropertyFlag */ -public static final int - TRITONSERVER_TXN_ONE_TO_ONE = 1, - TRITONSERVER_TXN_DECOUPLED = 2; - -/** Create a new server object. The caller takes ownership of the - * TRITONSERVER_Server object and must call TRITONSERVER_ServerDelete - * to release the object. - * - * @param server Returns the new inference server object. - * @param options The inference server options object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerNew( - @Cast("TRITONSERVER_Server**") PointerPointer server, TRITONSERVER_ServerOptions options); -public static native TRITONSERVER_Error TRITONSERVER_ServerNew( - @ByPtrPtr TRITONSERVER_Server server, TRITONSERVER_ServerOptions options); - -/** Delete a server object. If server is not already stopped it is - * stopped before being deleted. - * - * @param server The inference server object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerDelete( - TRITONSERVER_Server server); - -/** Stop a server object. A server can't be restarted once it is - * stopped. - * - * @param server The inference server object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerStop( - TRITONSERVER_Server server); - -/** Register a new model repository. Not available in polling mode. - * - * @param server The inference server object. - * @param repository_path The full path to the model repository. - * @param name_mapping List of name_mapping parameters. Each mapping has - * the model directory name as its key, overriden model name as its value. - * @param model_count Number of mappings provided. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( - TRITONSERVER_Server server, String repository_path, - @Cast("const TRITONSERVER_Parameter**") PointerPointer name_mapping, @Cast("const uint32_t") int mapping_count); -public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( - TRITONSERVER_Server server, String repository_path, - @Const @ByPtrPtr TRITONSERVER_Parameter name_mapping, @Cast("const uint32_t") int mapping_count); -public static native TRITONSERVER_Error TRITONSERVER_ServerRegisterModelRepository( - TRITONSERVER_Server server, @Cast("const char*") BytePointer repository_path, - @Const @ByPtrPtr TRITONSERVER_Parameter name_mapping, @Cast("const uint32_t") int mapping_count); - -/** Unregister a model repository. Not available in polling mode. - * - * @param server The inference server object. - * @param repository_path The full path to the model repository. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerUnregisterModelRepository( - TRITONSERVER_Server server, String repository_path); -public static native TRITONSERVER_Error TRITONSERVER_ServerUnregisterModelRepository( - TRITONSERVER_Server server, @Cast("const char*") BytePointer repository_path); - -/** Check the model repository for changes and update server state - * based on those changes. - * - * @param server The inference server object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerPollModelRepository(TRITONSERVER_Server server); - -/** Is the server live? - * - * @param server The inference server object. - * @param live Returns true if server is live, false otherwise. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerIsLive( - TRITONSERVER_Server server, @Cast("bool*") boolean[] live); -public static native TRITONSERVER_Error TRITONSERVER_ServerIsLive( - TRITONSERVER_Server server, @Cast("bool*") BoolPointer live); - -/** Is the server ready? - * - * @param server The inference server object. - * @param ready Returns true if server is ready, false otherwise. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerIsReady( - TRITONSERVER_Server server, @Cast("bool*") boolean[] ready); -public static native TRITONSERVER_Error TRITONSERVER_ServerIsReady( - TRITONSERVER_Server server, @Cast("bool*") BoolPointer ready); - -/** Is the model ready? - * - * @param server The inference server object. - * @param model_name The name of the model to get readiness for. - * @param model_version The version of the model to get readiness - * for. If -1 then the server will choose a version based on the - * model's policy. - * @param ready Returns true if server is ready, false otherwise. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelIsReady( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("bool*") boolean[] ready); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelIsReady( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("bool*") BoolPointer ready); - -/** Get the batch properties of the model. The properties are - * communicated by a flags value and an (optional) object returned by - * 'voidp'. - * - * - TRITONSERVER_BATCH_UNKNOWN: Triton cannot determine the - * batching properties of the model. This means that the model - * does not support batching in any way that is useable by - * Triton. The returned 'voidp' value is nullptr. - * - * - TRITONSERVER_BATCH_FIRST_DIM: The model supports batching - * along the first dimension of every input and output - * tensor. Triton schedulers that perform batching can - * automatically batch inference requests along this dimension. - * The returned 'voidp' value is nullptr. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @param model_version The version of the model. If -1 then the - * server will choose a version based on the model's policy. - * @param flags Returns flags indicating the batch properties of the - * model. - * @param voidp If non-nullptr, returns a point specific to the - * 'flags' value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") PointerPointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelBatchProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] flags, @Cast("void**") @ByPtrPtr Pointer voidp); - -/** Get the transaction policy of the model. The policy is - * communicated by a flags value. - * - * - TRITONSERVER_TXN_ONE_TO_ONE: The model generates exactly - * one response per request. - * - * - TRITONSERVER_TXN_DECOUPLED: The model may generate zero - * to many responses per request. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @param model_version The version of the model. If -1 then the - * server will choose a version based on the model's policy. - * @param txn_flags Returns flags indicating the transaction policy of the - * model. - * @param voidp If non-nullptr, returns a point specific to the 'flags' value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") PointerPointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntPointer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") IntBuffer txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelTransactionProperties( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("uint32_t*") int[] txn_flags, @Cast("void**") @ByPtrPtr Pointer voidp); - -/** Get the metadata of the server as a TRITONSERVER_Message object. - * The caller takes ownership of the message object and must call - * TRITONSERVER_MessageDelete to release the object. - * - * @param server The inference server object. - * @param server_metadata Returns the server metadata message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerMetadata( - TRITONSERVER_Server server, @Cast("TRITONSERVER_Message**") PointerPointer server_metadata); -public static native TRITONSERVER_Error TRITONSERVER_ServerMetadata( - TRITONSERVER_Server server, @ByPtrPtr TRITONSERVER_Message server_metadata); - -/** Get the metadata of a model as a TRITONSERVER_Message - * object. The caller takes ownership of the message object and must - * call TRITONSERVER_MessageDelete to release the object. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @param model_version The version of the model. - * If -1 then the server will choose a version based on the model's - * policy. - * @param model_metadata Returns the model metadata message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("TRITONSERVER_Message**") PointerPointer model_metadata); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_metadata); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelMetadata( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_metadata); - -/** Get the statistics of a model as a TRITONSERVER_Message - * object. The caller takes ownership of the object and must call - * TRITONSERVER_MessageDelete to release the object. - * - * @param server The inference server object. - * @param model_name The name of the model. - * If empty, then statistics for all available models will be returned, - * and the server will choose a version based on those models' policies. - * @param model_version The version of the model. If -1 then the - * server will choose a version based on the model's policy. - * @param model_stats Returns the model statistics message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("TRITONSERVER_Message**") PointerPointer model_stats); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_stats); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelStatistics( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @ByPtrPtr TRITONSERVER_Message model_stats); - -/** Get the configuration of a model as a TRITONSERVER_Message object. - * The caller takes ownership of the message object and must call - * TRITONSERVER_MessageDelete to release the object. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @param model_version The version of the model. If -1 then the - * server will choose a version based on the model's policy. - * @param config_version The model configuration will be returned in - * a format matching this version. If the configuration cannot be - * represented in the requested version's format then an error will - * be returned. Currently only version 1 is supported. - * @param model_config Returns the model config message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, - @Cast("TRITONSERVER_Message**") PointerPointer model_config); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( - TRITONSERVER_Server server, String model_name, - @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, - @ByPtrPtr TRITONSERVER_Message model_config); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelConfig( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Cast("const int64_t") long model_version, @Cast("const uint32_t") int config_version, - @ByPtrPtr TRITONSERVER_Message model_config); - -/** Get the index of all unique models in the model repositories as a - * TRITONSERVER_Message object. The caller takes ownership of the - * message object and must call TRITONSERVER_MessageDelete to release - * the object. - * - * If TRITONSERVER_INDEX_FLAG_READY is set in 'flags' only the models - * that are loaded into the server and ready for inferencing are - * returned. - * - * @param server The inference server object. - * @param flags TRITONSERVER_ModelIndexFlag flags that control how to - * collect the index. - * @param model_index Return the model index message that holds the - * index of all models contained in the server's model repository(s). - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerModelIndex( - TRITONSERVER_Server server, @Cast("uint32_t") int flags, - @Cast("TRITONSERVER_Message**") PointerPointer model_index); -public static native TRITONSERVER_Error TRITONSERVER_ServerModelIndex( - TRITONSERVER_Server server, @Cast("uint32_t") int flags, - @ByPtrPtr TRITONSERVER_Message model_index); - -/** Load the requested model or reload the model if it is already - * loaded. The function does not return until the model is loaded or - * fails to load. Returned error indicates if model loaded - * successfully or not. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModel( - TRITONSERVER_Server server, String model_name); -public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModel( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); - -/** Load the requested model or reload the model if it is already - * loaded, with load parameters provided. The function does not return until - * the model is loaded or fails to load. Returned error indicates if model - * loaded successfully or not. - * Currently the below parameter names are recognized: - * - "config" : string parameter that contains a JSON representation of the - * model configuration. This config will be used for loading the model instead - * of the one in the model directory. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @param parameters The array of load parameters. - * @param parameter_count The number of parameters. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( - TRITONSERVER_Server server, String model_name, - @Cast("const TRITONSERVER_Parameter**") PointerPointer parameters, @Cast("const uint64_t") long parameter_count); -public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( - TRITONSERVER_Server server, String model_name, - @Const @ByPtrPtr TRITONSERVER_Parameter parameters, @Cast("const uint64_t") long parameter_count); -public static native TRITONSERVER_Error TRITONSERVER_ServerLoadModelWithParameters( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name, - @Const @ByPtrPtr TRITONSERVER_Parameter parameters, @Cast("const uint64_t") long parameter_count); - -/** Unload the requested model. Unloading a model that is not loaded - * on server has no affect and success code will be returned. - * The function does not wait for the requested model to be fully unload - * and success code will be returned. - * Returned error indicates if model unloaded successfully or not. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModel( - TRITONSERVER_Server server, String model_name); -public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModel( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); - -/** Unload the requested model, and also unload any dependent model that - * was loaded along with the requested model (for example, the models composing - * an ensemble). Unloading a model that is not loaded - * on server has no affect and success code will be returned. - * The function does not wait for the requested model and all dependent - * models to be fully unload and success code will be returned. - * Returned error indicates if model unloaded successfully or not. - * - * @param server The inference server object. - * @param model_name The name of the model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModelAndDependents( - TRITONSERVER_Server server, String model_name); -public static native TRITONSERVER_Error TRITONSERVER_ServerUnloadModelAndDependents( - TRITONSERVER_Server server, @Cast("const char*") BytePointer model_name); - -/** Get the current metrics for the server. The caller takes ownership - * of the metrics object and must call TRITONSERVER_MetricsDelete to - * release the object. - * - * @param server The inference server object. - * @param metrics Returns the metrics. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerMetrics( - TRITONSERVER_Server server, @Cast("TRITONSERVER_Metrics**") PointerPointer metrics); -public static native TRITONSERVER_Error TRITONSERVER_ServerMetrics( - TRITONSERVER_Server server, @ByPtrPtr TRITONSERVER_Metrics metrics); - -/** Perform inference using the meta-data and inputs supplied by the - * 'inference_request'. If the function returns success, then the - * caller releases ownership of 'inference_request' and must not - * access it in any way after this call, until ownership is returned - * via the 'request_release_fn' callback registered in the request - * object with TRITONSERVER_InferenceRequestSetReleaseCallback. - * - * The function unconditionally takes ownership of 'trace' and so the - * caller must not access it in any way after this call (except in - * the trace activity callbacks) until ownership is returned via the - * trace's release_fn callback. - * - * Responses produced for this request are returned using the - * allocator and callback registered with the request by - * TRITONSERVER_InferenceRequestSetResponseCallback. - * - * @param server The inference server object. - * @param inference_request The request object. - * @param trace The trace object for this request, or nullptr if no - * tracing. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_ServerInferAsync( - TRITONSERVER_Server server, - TRITONSERVER_InferenceRequest inference_request, - TRITONSERVER_InferenceTrace trace); - -/** TRITONSERVER_MetricKind - * - * Types of metrics recognized by TRITONSERVER. - * */ -/** enum TRITONSERVER_MetricKind */ -public static final int - TRITONSERVER_METRIC_KIND_COUNTER = 0, - TRITONSERVER_METRIC_KIND_GAUGE = 1; - -/** Create a new metric family object. The caller takes ownership of the - * TRITONSERVER_MetricFamily object and must call - * TRITONSERVER_MetricFamilyDelete to release the object. - * - * @param family Returns the new metric family object. - * @param kind The type of metric family to create. - * @param name The name of the metric family seen when calling the metrics - * endpoint. - * @param description The description of the metric family seen when - * calling the metrics endpoint. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( - @Cast("TRITONSERVER_MetricFamily**") PointerPointer family, @Cast("const TRITONSERVER_MetricKind") int kind, - String name, String description); -public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( - @ByPtrPtr TRITONSERVER_MetricFamily family, @Cast("const TRITONSERVER_MetricKind") int kind, - String name, String description); -public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyNew( - @ByPtrPtr TRITONSERVER_MetricFamily family, @Cast("const TRITONSERVER_MetricKind") int kind, - @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer description); - -/** Delete a metric family object. - * A TRITONSERVER_MetricFamily* object should be deleted AFTER its - * corresponding TRITONSERVER_Metric* objects have been deleted. - * Attempting to delete a family before its metrics will return an error. - * - * @param family The metric family object to delete. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricFamilyDelete( - TRITONSERVER_MetricFamily family); - -/** Create a new metric object. The caller takes ownership of the - * TRITONSERVER_Metric object and must call - * TRITONSERVER_MetricDelete to release the object. The caller is also - * responsible for ownership of the labels passed in. Each label can be deleted - * immediately after creating the metric with TRITONSERVER_ParameterDelete - * if not re-using the labels. - * - * @param metric Returns the new metric object. - * @param family The metric family to add this new metric to. - * @param labels The array of labels to associate with this new metric. - * @param label_count The number of labels. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricNew( - @Cast("TRITONSERVER_Metric**") PointerPointer metric, TRITONSERVER_MetricFamily family, - @Cast("const TRITONSERVER_Parameter**") PointerPointer labels, @Cast("const uint64_t") long label_count); -public static native TRITONSERVER_Error TRITONSERVER_MetricNew( - @ByPtrPtr TRITONSERVER_Metric metric, TRITONSERVER_MetricFamily family, - @Const @ByPtrPtr TRITONSERVER_Parameter labels, @Cast("const uint64_t") long label_count); - -/** Delete a metric object. - * All TRITONSERVER_Metric* objects should be deleted BEFORE their - * corresponding TRITONSERVER_MetricFamily* objects have been deleted. - * If a family is deleted before its metrics, an error will be returned. - * - * @param metric The metric object to delete. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricDelete( - TRITONSERVER_Metric metric); - -/** Get the current value of a metric object. - * Supports metrics of kind TRITONSERVER_METRIC_KIND_COUNTER - * and TRITONSERVER_METRIC_KIND_GAUGE, and returns - * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. - * - * @param metric The metric object to query. - * @param value Returns the current value of the metric object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricValue( - TRITONSERVER_Metric metric, DoublePointer value); -public static native TRITONSERVER_Error TRITONSERVER_MetricValue( - TRITONSERVER_Metric metric, DoubleBuffer value); -public static native TRITONSERVER_Error TRITONSERVER_MetricValue( - TRITONSERVER_Metric metric, double[] value); - -/** Increment the current value of metric by value. - * Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE for any value, - * and TRITONSERVER_METRIC_KIND_COUNTER for non-negative values. Returns - * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind - * and TRITONSERVER_ERROR_INVALID_ARG for negative values on a - * TRITONSERVER_METRIC_KIND_COUNTER metric. - * - * @param metric The metric object to update. - * @param value The amount to increment the metric's value by. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricIncrement( - TRITONSERVER_Metric metric, double value); - -/** Set the current value of metric to value. - * Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and returns - * TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. - * - * @param metric The metric object to update. - * @param value The amount to set metric's value to. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_MetricSet( - TRITONSERVER_Metric metric, double value); - -/** Get the TRITONSERVER_MetricKind of metric and its corresponding family. - * - * @param metric The metric object to query. - * @param kind Returns the TRITONSERVER_MetricKind of metric. - * @return a TRITONSERVER_Error indicating success or failure. */ -public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( - TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") IntPointer kind); -public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( - TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") IntBuffer kind); -public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( - TRITONSERVER_Metric metric, @Cast("TRITONSERVER_MetricKind*") int[] kind); - -// #ifdef __cplusplus -// #endif - - -// Parsed from tritonbackend.h - -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once - -// #include -// #include - -// #include "triton/core/tritonserver.h" - -// #ifdef __cplusplus -// #endif - -// #ifdef _COMPILING_TRITONBACKEND -// #if defined(_MSC_VER) -// #define TRITONBACKEND_DECLSPEC __declspec(dllexport) -// #define TRITONBACKEND_ISPEC __declspec(dllimport) -// #elif defined(__GNUC__) -// #define TRITONBACKEND_DECLSPEC __attribute__((__visibility__("default"))) -// #define TRITONBACKEND_ISPEC -// #else -// #define TRITONBACKEND_DECLSPEC -// #define TRITONBACKEND_ISPEC -// #endif -// #else -// #if defined(_MSC_VER) -// #define TRITONBACKEND_DECLSPEC __declspec(dllimport) -// #define TRITONBACKEND_ISPEC __declspec(dllexport) -// #else -// #define TRITONBACKEND_DECLSPEC -// #define TRITONBACKEND_ISPEC -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Input.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Output.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_State.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Request.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Response.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Backend.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Model.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java - - -// Targeting ../tritondevelopertoolsserver/TRITONBACKEND_Batcher.java - - - -/** - * TRITONBACKEND API Version - * - * The TRITONBACKEND API is versioned with major and minor version - * numbers. Any change to the API that does not impact backwards - * compatibility (for example, adding a non-required function) - * increases the minor version number. Any change that breaks - * backwards compatibility (for example, deleting or changing the - * behavior of a function) increases the major version number. A - * backend should check that the API version used to compile the - * backend is compatible with the API version of the Triton server - * that it is running in. This is typically done by code similar to - * the following which makes sure that the major versions are equal - * and that the minor version of Triton is >= the minor version used - * to build the backend. - * - * uint32_t api_version_major, api_version_minor; - * TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor); - * if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || - * (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { - * return TRITONSERVER_ErrorNew( - * TRITONSERVER_ERROR_UNSUPPORTED, - * "triton backend API version does not support this backend"); - * } - * */ -public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; - -/// -public static final int TRITONBACKEND_API_VERSION_MINOR = 12; - -/** Get the TRITONBACKEND API version supported by Triton. This value - * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and - * TRITONBACKEND_API_VERSION_MINOR used to build the backend to - * ensure that Triton is compatible with the backend. - * - * @param major Returns the TRITONBACKEND API major version supported - * by Triton. - * @param minor Returns the TRITONBACKEND API minor version supported - * by Triton. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( - @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); -public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( - @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); -public static native TRITONSERVER_Error TRITONBACKEND_ApiVersion( - @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); - -/** TRITONBACKEND_ArtifactType - * - * The ways that the files that make up a backend or model are - * communicated to the backend. - * - * TRITONBACKEND_ARTIFACT_FILESYSTEM: The model or backend - * artifacts are made available to Triton via a locally - * accessible filesystem. The backend can access these files - * using an appropriate system API. - * */ -/** enum TRITONBACKEND_ArtifactType */ -public static final int - TRITONBACKEND_ARTIFACT_FILESYSTEM = 0; - - -/** - * TRITONBACKEND_MemoryManager - * - * Object representing an memory manager that is capable of - * allocating and otherwise managing different memory types. For - * improved performance Triton maintains pools for GPU and CPU-pinned - * memory and the memory manager allows backends to access those - * pools. - * -

- * Allocate a contiguous block of memory of a specific type using a - * memory manager. Two error codes have specific interpretations for - * this function: - * - * TRITONSERVER_ERROR_UNSUPPORTED: Indicates that Triton is - * incapable of allocating the requested memory type and memory - * type ID. Requests for the memory type and ID will always fail - * no matter 'byte_size' of the request. - * - * TRITONSERVER_ERROR_UNAVAILABLE: Indicates that Triton can - * allocate the memory type and ID but that currently it cannot - * allocate a contiguous block of memory of the requested - * 'byte_size'. - * - * @param manager The memory manager. - * @param buffer Returns the allocated memory. - * @param memory_type The type of memory to allocate. - * @param memory_type_id The ID associated with the memory type to - * allocate. For GPU memory this indicates the device ID of the GPU - * to allocate from. - * @param byte_size The size of memory to allocate, in bytes. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerAllocate( - TRITONBACKEND_MemoryManager manager, @Cast("void**") PointerPointer buffer, - @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id, - @Cast("const uint64_t") long byte_size); -public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerAllocate( - TRITONBACKEND_MemoryManager manager, @Cast("void**") @ByPtrPtr Pointer buffer, - @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id, - @Cast("const uint64_t") long byte_size); - -/** Free a buffer that was previously allocated with - * TRITONBACKEND_MemoryManagerAllocate. The call must provide the - * same values for 'memory_type' and 'memory_type_id' as were used - * when the buffer was allocate or else the behavior is undefined. - * - * @param manager The memory manager. - * @param buffer The allocated memory buffer to free. - * @param memory_type The type of memory of the buffer. - * @param memory_type_id The ID associated with the memory type of - * the buffer. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_MemoryManagerFree( - TRITONBACKEND_MemoryManager manager, Pointer buffer, - @Cast("const TRITONSERVER_MemoryType") int memory_type, @Cast("const int64_t") long memory_type_id); - -/** - * TRITONBACKEND_Input - * - * Object representing an input tensor. - * -

- * Get the name and properties of an input tensor. The returned - * strings and other properties are owned by the input, not the - * caller, and so should not be modified or freed. - * - * @param input The input tensor. - * @param name If non-nullptr, returns the tensor name. - * @param datatype If non-nullptr, returns the tensor datatype. - * @param shape If non-nullptr, returns the tensor shape. - * @param dim_count If non-nullptr, returns the number of dimensions - * in the tensor shape. - * @param byte_size If non-nullptr, returns the size of the available - * data for the tensor, in bytes. This size reflects the actual data - * available, and does not necessarily match what is - * expected/required for the tensor given its shape and datatype. It - * is the responsibility of the backend to handle mismatches in these - * sizes appropriately. - * @param buffer_count If non-nullptr, returns the number of buffers - * holding the contents of the tensor. These buffers are accessed - * using TRITONBACKEND_InputBuffer. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( - TRITONBACKEND_Input input, @Cast("const char**") PointerPointer name, - @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, - @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( - TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr BytePointer name, - @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, - @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( - TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr ByteBuffer name, - @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, - @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputProperties( - TRITONBACKEND_Input input, @Cast("const char**") @ByPtrPtr byte[] name, - @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, - @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); - -/** Get the name and properties of an input tensor associated with a given - * host policy. If there are no input buffers for the specified host policy, - * the properties of the fallback input buffers are returned. The returned - * strings and other properties are owned by the input, not the caller, and so - * should not be modified or freed. - * - * @param input The input tensor. - * @param host_policy_name The host policy name. Fallback input properties - * will be return if nullptr is provided. - * @param name If non-nullptr, returns the tensor name. - * @param datatype If non-nullptr, returns the tensor datatype. - * @param shape If non-nullptr, returns the tensor shape. - * @param dim_count If non-nullptr, returns the number of dimensions - * in the tensor shape. - * @param byte_size If non-nullptr, returns the size of the available - * data for the tensor, in bytes. This size reflects the actual data - * available, and does not necessarily match what is - * expected/required for the tensor given its shape and datatype. It - * is the responsibility of the backend to handle mismatches in these - * sizes appropriately. - * @param buffer_count If non-nullptr, returns the number of buffers - * holding the contents of the tensor. These buffers are accessed - * using TRITONBACKEND_InputBufferForHostPolicy. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") PointerPointer name, - @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") PointerPointer shape, - @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr BytePointer name, - @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, - @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr ByteBuffer name, - @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, - @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr byte[] name, - @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, - @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr BytePointer name, - @Cast("TRITONSERVER_DataType*") IntPointer datatype, @Cast("const int64_t**") @ByPtrPtr LongPointer shape, - @Cast("uint32_t*") IntPointer dims_count, @Cast("uint64_t*") LongPointer byte_size, @Cast("uint32_t*") IntPointer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, @Cast("const char**") @ByPtrPtr ByteBuffer name, - @Cast("TRITONSERVER_DataType*") IntBuffer datatype, @Cast("const int64_t**") @ByPtrPtr LongBuffer shape, - @Cast("uint32_t*") IntBuffer dims_count, @Cast("uint64_t*") LongBuffer byte_size, @Cast("uint32_t*") IntBuffer buffer_count); -public static native TRITONSERVER_Error TRITONBACKEND_InputPropertiesForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, @Cast("const char**") @ByPtrPtr byte[] name, - @Cast("TRITONSERVER_DataType*") int[] datatype, @Cast("const int64_t**") @ByPtrPtr long[] shape, - @Cast("uint32_t*") int[] dims_count, @Cast("uint64_t*") long[] byte_size, @Cast("uint32_t*") int[] buffer_count); - -/** Get a buffer holding (part of) the tensor data for an input. For a - * given input the number of buffers composing the input are found - * from 'buffer_count' returned by TRITONBACKEND_InputProperties. The - * returned buffer is owned by the input and so should not be - * modified or freed by the caller. The lifetime of the buffer - * matches that of the input and so the buffer should not be accessed - * after the input tensor object is released. - * - * @param input The input tensor. - * @param index The index of the buffer. Must be 0 <= index < - * buffer_count, where buffer_count is the value returned by - * TRITONBACKEND_InputProperties. - * @param buffer Returns a pointer to a contiguous block of data for - * the named input. - * @param buffer_byte_size Returns the size, in bytes, of 'buffer'. - * @param memory_type Acts as both input and output. On input gives - * the buffer memory type preferred by the function caller. Returns - * the actual memory type of 'buffer'. - * @param memory_type_id Acts as both input and output. On input - * gives the buffer memory type id preferred by the function caller. - * Returns the actual memory type id of 'buffer'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, - @Cast("uint64_t*") LongPointer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, - @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, - @Cast("uint64_t*") LongPointer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, - @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, - @Cast("uint64_t*") LongBuffer buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, - @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBuffer( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, - @Cast("uint64_t*") long[] buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") int[] memory_type, - @Cast("int64_t*") long[] memory_type_id); - -/** Get a buffer holding (part of) the tensor data for an input for a specific - * host policy. If there are no input buffers specified for this host policy, - * the fallback input buffer is returned. - * For a given input the number of buffers composing the input are found - * from 'buffer_count' returned by TRITONBACKEND_InputPropertiesForHostPolicy. - * The returned buffer is owned by the input and so should not be modified or - * freed by the caller. The lifetime of the buffer matches that of the input - * and so the buffer should not be accessed after the input tensor object is - * released. - * - * @param input The input tensor. - * @param host_policy_name The host policy name. Fallback input buffer - * will be return if nullptr is provided. - * @param index The index of the buffer. Must be 0 <= index < - * buffer_count, where buffer_count is the value returned by - * TRITONBACKEND_InputPropertiesForHostPolicy. - * @param buffer Returns a pointer to a contiguous block of data for - * the named input. - * @param buffer_byte_size Returns the size, in bytes, of 'buffer'. - * @param memory_type Acts as both input and output. On input gives - * the buffer memory type preferred by the function caller. Returns - * the actual memory type of 'buffer'. - * @param memory_type_id Acts as both input and output. On input - * gives the buffer memory type id preferred by the function caller. - * Returns the actual memory type id of 'buffer'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongBuffer buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") long[] buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongPointer buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, String host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") LongBuffer buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferForHostPolicy( - TRITONBACKEND_Input input, @Cast("const char*") BytePointer host_policy_name, - @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, @Cast("uint64_t*") long[] buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); - -/** Get the buffer attributes associated with the given input buffer. For a - * given input the number of buffers composing the input are found from - * 'buffer_count' returned by TRITONBACKEND_InputProperties. The returned - * 'buffer_attributes' is owned by the input and so should not be modified or - * freed by the caller. The lifetime of the 'buffer_attributes' matches that of - * the input and so the 'buffer_attributes' should not be accessed after the - * input tensor object is released. - * - * @param input The input tensor. - * @param index The index of the buffer. Must be 0 <= index < buffer_count, - * where buffer_count is the value returned by TRITONBACKEND_InputProperties. - * @param buffer Returns a pointer to a contiguous block of data for - * the named input. - * @param buffer_attributes Returns the attributes for the given buffer. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferAttributes( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") PointerPointer buffer, - @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); -public static native TRITONSERVER_Error TRITONBACKEND_InputBufferAttributes( - TRITONBACKEND_Input input, @Cast("const uint32_t") int index, @Cast("const void**") @ByPtrPtr Pointer buffer, - @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); - -/** - * TRITONBACKEND_Output - * - * Object representing a response output tensor. - * -

- * Get a buffer to use to hold the tensor data for the output. The - * returned buffer is owned by the output and so should not be freed - * by the caller. The caller can and should fill the buffer with the - * output data for the tensor. The lifetime of the buffer matches - * that of the output and so the buffer should not be accessed after - * the output tensor object is released. - * - * @param buffer Returns a pointer to a buffer where the contents of - * the output tensor should be placed. - * @param buffer_byte_size The size, in bytes, of the buffer required - * by the caller. - * @param memory_type Acts as both input and output. On input gives - * the buffer memory type preferred by the caller. Returns the - * actual memory type of 'buffer'. - * @param memory_type_id Acts as both input and output. On input - * gives the buffer memory type id preferred by the caller. Returns - * the actual memory type id of 'buffer'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( - TRITONBACKEND_Output output, @Cast("void**") PointerPointer buffer, - @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, - @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( - TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, - @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, - @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( - TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, - @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, - @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_OutputBuffer( - TRITONBACKEND_Output output, @Cast("void**") @ByPtrPtr Pointer buffer, - @Cast("const uint64_t") long buffer_byte_size, @Cast("TRITONSERVER_MemoryType*") int[] memory_type, - @Cast("int64_t*") long[] memory_type_id); - -/** Get the buffer attributes associated with the given output buffer. The - * returned 'buffer_attributes' is owned by the output and so should not be - * modified or freed by the caller. The lifetime of the 'buffer_attributes' - * matches that of the output and so the 'buffer_attributes' should not be - * accessed after the output tensor object is released. This function must be - * called after the TRITONBACKEND_OutputBuffer otherwise it might contain - * incorrect data. - * - * @param output The output tensor. - * @param buffer_attributes Returns the attributes for the output buffer. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_OutputBufferAttributes( - TRITONBACKEND_Output output, - @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); -public static native TRITONSERVER_Error TRITONBACKEND_OutputBufferAttributes( - TRITONBACKEND_Output output, - @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); - -/** - * TRITONBACKEND_Request - * - * Object representing an inference request. - * -

- * Get the ID of the request. Can be nullptr if request doesn't have - * an ID. The returned string is owned by the request, not the - * caller, and so should not be modified or freed. - * - * @param request The inference request. - * @param id Returns the ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestId( - TRITONBACKEND_Request request, @Cast("const char**") PointerPointer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestId( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr BytePointer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestId( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr ByteBuffer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestId( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr byte[] id); - -/** Get the correlation ID of the request if it is an unsigned integer. - * Zero indicates that the request does not have a correlation ID. - * Returns failure if correlation ID for given request is not an unsigned - * integer. - * - * @param request The inference request. - * @param id Returns the correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( - TRITONBACKEND_Request request, @Cast("uint64_t*") LongPointer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( - TRITONBACKEND_Request request, @Cast("uint64_t*") LongBuffer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationId( - TRITONBACKEND_Request request, @Cast("uint64_t*") long[] id); - -/** Get the correlation ID of the request if it is a string. - * Empty string indicates that the request does not have a correlation ID. - * Returns error if correlation ID for given request is not a string. - * - * @param request The inference request. - * @param id Returns the correlation ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( - TRITONBACKEND_Request request, @Cast("const char**") PointerPointer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr BytePointer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr ByteBuffer id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestCorrelationIdString( - TRITONBACKEND_Request request, @Cast("const char**") @ByPtrPtr byte[] id); - -/** Get the flag(s) associated with a request. On return 'flags' holds - * a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for - * available flags. - * - * @param request The inference request. - * @param flags Returns the flags. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer flags); -public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer flags); -public static native TRITONSERVER_Error TRITONBACKEND_RequestFlags( - TRITONBACKEND_Request request, @Cast("uint32_t*") int[] flags); - -/** Get the number of parameters specified in the inference request. - * - * @param request The inference request. - * @param count Returns the number of parameters. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameterCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); - -/** Get a request parameters by index. The order of parameters in a given - * request is not necessarily consistent with other requests, even if - * the requests are in the same batch. As a result, you can not - * assume that an index obtained from one request will point to the - * same parameter in a different request. - * - * The lifetime of the returned parameter object matches that of the - * request and so the parameter object should not be accessed after the - * request object is released. - * - * @param request The inference request. - * @param index The index of the parameter. Must be 0 <= index < - * count, where count is the value returned by - * TRITONBACKEND_RequestParameterCount. - * @param key Returns the key of the parameter. - * @param type Returns the type of the parameter. - * @param vvalue Returns a pointer to the parameter value. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") PointerPointer key, - @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") PointerPointer vvalue); -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer key, - @Cast("TRITONSERVER_ParameterType*") IntPointer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer key, - @Cast("TRITONSERVER_ParameterType*") IntBuffer type, @Cast("const void**") @ByPtrPtr Pointer vvalue); -public static native TRITONSERVER_Error TRITONBACKEND_RequestParameter( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] key, - @Cast("TRITONSERVER_ParameterType*") int[] type, @Cast("const void**") @ByPtrPtr Pointer vvalue); - -/** Get the number of input tensors specified in the request. - * - * @param request The inference request. - * @param count Returns the number of input tensors. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); - -/** Get the name of an input tensor. The caller does not own - * the returned string and must not modify or delete it. The lifetime - * of the returned string extends only as long as 'request'. - * - * @param request The inference request. - * @param index The index of the input tensor. Must be 0 <= index < - * count, where count is the value returned by - * TRITONBACKEND_RequestInputCount. - * @param input_name Returns the name of the input tensor - * corresponding to the index. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") PointerPointer input_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr BytePointer input_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr ByteBuffer input_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr byte[] input_name); - -/** Get a named request input. The lifetime of the returned input - * object matches that of the request and so the input object should - * not be accessed after the request object is released. - * - * @param request The inference request. - * @param name The name of the input. - * @param input Returns the input corresponding to the name. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( - TRITONBACKEND_Request request, String name, - @Cast("TRITONBACKEND_Input**") PointerPointer input); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( - TRITONBACKEND_Request request, String name, - @ByPtrPtr TRITONBACKEND_Input input); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInput( - TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, - @ByPtrPtr TRITONBACKEND_Input input); - -/** Get a request input by index. The order of inputs in a given - * request is not necessarily consistent with other requests, even if - * the requests are in the same batch. As a result, you can not - * assume that an index obtained from one request will point to the - * same input in a different request. - * - * The lifetime of the returned input object matches that of the - * request and so the input object should not be accessed after the - * request object is released. - * - * @param request The inference request. - * @param index The index of the input tensor. Must be 0 <= index < - * count, where count is the value returned by - * TRITONBACKEND_RequestInputCount. - * @param input Returns the input corresponding to the index. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputByIndex( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("TRITONBACKEND_Input**") PointerPointer input); -public static native TRITONSERVER_Error TRITONBACKEND_RequestInputByIndex( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @ByPtrPtr TRITONBACKEND_Input input); - -/** Get the number of output tensors requested to be returned in the - * request. - * - * @param request The inference request. - * @param count Returns the number of output tensors. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputCount( - TRITONBACKEND_Request request, @Cast("uint32_t*") int[] count); - -/** Get the name of a requested output tensor. The caller does not own - * the returned string and must not modify or delete it. The lifetime - * of the returned string extends only as long as 'request'. - * - * @param request The inference request. - * @param index The index of the requested output tensor. Must be 0 - * <= index < count, where count is the value returned by - * TRITONBACKEND_RequestOutputCount. - * @param output_name Returns the name of the requested output tensor - * corresponding to the index. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") PointerPointer output_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr BytePointer output_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr ByteBuffer output_name); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputName( - TRITONBACKEND_Request request, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr byte[] output_name); - -/** Returns the preferred memory type and memory type ID of the output buffer - * for the request. As much as possible, Triton will attempt to return - * the same memory_type and memory_type_id values that will be returned by - * the subsequent call to TRITONBACKEND_OutputBuffer, however, the backend must - * be capable of handling cases where the values differ. - * - * @param request The request. - * @param name The name of the output tensor. This is optional - * and it should be set to nullptr to indicate that the tensor name has - * not determined. - * @param byte_size The expected size of the buffer. This is optional - * and it should be set to nullptr to indicate that the byte size has - * not determined. - * @param memory_type Acts as both input and output. On input gives - * the memory type preferred by the caller. Returns memory type preferred - * by Triton, taken account of the caller preferred type. - * @param memory_type_id Acts as both input and output. On input gives - * the memory type ID preferred by the caller. Returns memory type ID preferred - * by Triton, taken account of the caller preferred type ID. - * @return a TRITONSERVER_Error object if a failure occurs. - * A TRITONSERVER_ERROR_UNAVAILABLE error indicates that the properties are not - * available, other error codes indicate an error. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, String name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_RequestOutputBufferProperties( - TRITONBACKEND_Request request, @Cast("const char*") BytePointer name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); - -/** Release the request. The request should be released when it is no - * longer needed by the backend. If this call returns with an error - * (i.e. non-nullptr) then the request was not released and ownership - * remains with the backend. If this call returns with success, the - * 'request' object is no longer owned by the backend and must not be - * used. Any tensor names, data types, shapes, input tensors, - * etc. returned by TRITONBACKEND_Request* functions for this request - * are no longer valid. If a persistent copy of that data is required - * it must be created before calling this function. - * - * @param request The inference request. - * @param release_flags Flags indicating what type of request release - * should be performed. @see TRITONSERVER_RequestReleaseFlag. @see - * TRITONSERVER_InferenceRequestReleaseFn_t. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_RequestRelease( - TRITONBACKEND_Request request, @Cast("uint32_t") int release_flags); - -/** - * TRITONBACKEND_ResponseFactory - * - * Object representing an inference response factory. Using a - * response factory is not required; instead a response can be - * generated directly from a TRITONBACKEND_Request object using - * TRITONBACKEND_ResponseNew(). A response factory allows a request - * to be released before all responses have been sent. Releasing a - * request as early as possible releases all input tensor data and - * therefore may be desirable in some cases. -

- * Create the response factory associated with a request. - * - * @param factory Returns the new response factory. - * @param request The inference request. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryNew( - @Cast("TRITONBACKEND_ResponseFactory**") PointerPointer factory, TRITONBACKEND_Request request); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryNew( - @ByPtrPtr TRITONBACKEND_ResponseFactory factory, TRITONBACKEND_Request request); - -/** Destroy a response factory. - * - * @param factory The response factory. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactoryDelete( - TRITONBACKEND_ResponseFactory factory); - -/** Send response flags without a corresponding response. - * - * @param factory The response factory. - * @param send_flags Flags to send. @see - * TRITONSERVER_ResponseCompleteFlag. @see - * TRITONSERVER_InferenceResponseCompleteFn_t. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseFactorySendFlags( - TRITONBACKEND_ResponseFactory factory, @Cast("const uint32_t") int send_flags); - -/** - * TRITONBACKEND_Response - * - * Object representing an inference response. For a given request, - * the backend must carefully manage the lifecycle of responses - * generated for that request to ensure that the output tensor - * buffers are allocated correctly. When a response is created with - * TRITONBACKEND_ResponseNew or TRITONBACKEND_ResponseNewFromFactory, - * all the outputs and corresponding buffers must be created for that - * response using TRITONBACKEND_ResponseOutput and - * TRITONBACKEND_OutputBuffer *before* another response is created - * for the request. For a given response, outputs can be created in - * any order but they must be created sequentially/sychronously (for - * example, the backend cannot use multiple threads to simultaneously - * add multiple outputs to a response). - * - * The above requirement applies only to responses being generated - * for a given request. The backend may generate responses in - * parallel on multiple threads as long as those responses are for - * different requests. - * - * This order of response creation must be strictly followed. But, - * once response(s) are created they do not need to be sent - * immediately, nor do they need to be sent in the order they were - * created. The backend may even delete a created response instead of - * sending it by using TRITONBACKEND_ResponseDelete. -

- * Create a response for a request. - * - * @param response Returns the new response. - * @param request The request. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseNew( - @Cast("TRITONBACKEND_Response**") PointerPointer response, TRITONBACKEND_Request request); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseNew( - @ByPtrPtr TRITONBACKEND_Response response, TRITONBACKEND_Request request); - -/** Create a response using a factory. - * - * @param response Returns the new response. - * @param factory The response factory. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseNewFromFactory( - @Cast("TRITONBACKEND_Response**") PointerPointer response, TRITONBACKEND_ResponseFactory factory); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseNewFromFactory( - @ByPtrPtr TRITONBACKEND_Response response, TRITONBACKEND_ResponseFactory factory); - -/** Destroy a response. It is not necessary to delete a response if - * TRITONBACKEND_ResponseSend is called as that function transfers - * ownership of the response object to Triton. - * - * @param response The response. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseDelete( - TRITONBACKEND_Response response); - -/** Set a string parameter in the response. - * - * @param response The response. - * @param name The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetStringParameter( - TRITONBACKEND_Response response, String name, String value); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetStringParameter( - TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer value); - -/** Set an integer parameter in the response. - * - * @param response The response. - * @param name The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetIntParameter( - TRITONBACKEND_Response response, String name, @Cast("const int64_t") long value); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetIntParameter( - TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const int64_t") long value); - -/** Set a boolean parameter in the response. - * - * @param response The response. - * @param name The name of the parameter. - * @param value The value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetBoolParameter( - TRITONBACKEND_Response response, String name, @Cast("const bool") boolean value); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSetBoolParameter( - TRITONBACKEND_Response response, @Cast("const char*") BytePointer name, @Cast("const bool") boolean value); - -/** Create an output tensor in the response. The lifetime of the - * returned output tensor object matches that of the response and so - * the output tensor object should not be accessed after the response - * object is deleted. - * - * @param response The response. - * @param output Returns the new response output. - * @param name The name of the output tensor. - * @param datatype The datatype of the output tensor. - * @param shape The shape of the output tensor. - * @param dims_count The number of dimensions in the output tensor - * shape. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @Cast("TRITONBACKEND_Output**") PointerPointer output, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_ResponseOutput( - TRITONBACKEND_Response response, @ByPtrPtr TRITONBACKEND_Output output, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); - -/** Send a response. Calling this function transfers ownership of the - * response object to Triton. The caller must not access or delete - * the response object after calling this function. - * - * @param response The response. - * @param send_flags Flags associated with the response. @see - * TRITONSERVER_ResponseCompleteFlag. @see - * TRITONSERVER_InferenceResponseCompleteFn_t. - * @param error The TRITONSERVER_Error to send if the response is an - * error, or nullptr if the response is successful. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ResponseSend( - TRITONBACKEND_Response response, @Cast("const uint32_t") int send_flags, - TRITONSERVER_Error error); - -/** - * TRITONBACKEND_State - * - * Object representing a state. - * -

- * Create a state in the request. The returned state object is only valid - * before the TRITONBACKEND_StateUpdate is called. The state should not be - * freed by the caller. If TRITONBACKEND_StateUpdate is not called, the - * lifetime of the state matches the lifetime of the request. If the state name - * does not exist in the "state" section of the model configuration, the state - * will not be created and an error will be returned. If this function is - * called when sequence batching is not enabled or there is no 'states' section - * in the sequence batching section of the model configuration, this call will - * return an error. - * - * @param state Returns the new state. - * @param request The request. - * @param name The name of the state. - * @param datatype The datatype of the state. - * @param shape The shape of the state. - * @param dims_count The number of dimensions in the state shape. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @Cast("TRITONBACKEND_State**") PointerPointer state, TRITONBACKEND_Request request, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongPointer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - String name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") LongBuffer shape, @Cast("const uint32_t") int dims_count); -public static native TRITONSERVER_Error TRITONBACKEND_StateNew( - @ByPtrPtr TRITONBACKEND_State state, TRITONBACKEND_Request request, - @Cast("const char*") BytePointer name, @Cast("const TRITONSERVER_DataType") int datatype, - @Cast("const int64_t*") long[] shape, @Cast("const uint32_t") int dims_count); - -/** Update the state for the sequence. Calling this function will replace the - * state stored for this seqeunce in Triton with 'state' provided in the - * function argument. If this function is called when sequence batching is not - * enabled or there is no 'states' section in the sequence batching section of - * the model configuration, this call will return an error. The backend is not - * required to call this function. If the backend doesn't call - * TRITONBACKEND_StateUpdate function, this particular state for the sequence - * will not be updated and the next inference request in the sequence will use - * the same state as the current inference request. - * - * @param state The state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_StateUpdate( - TRITONBACKEND_State state); - -/** Get a buffer to use to hold the tensor data for the state. The returned - * buffer is owned by the state and so should not be freed by the caller. The - * caller can and should fill the buffer with the state data. The buffer must - * not be accessed by the backend after TRITONBACKEND_StateUpdate is called. - * The caller should fill the buffer before calling TRITONBACKEND_StateUpdate. - * - * @param state The state. - * @param buffer Returns a pointer to a buffer where the contents of the state - * should be placed. - * @param buffer_byte_size The size, in bytes, of the buffer required - * by the caller. - * @param memory_type Acts as both input and output. On input gives - * the buffer memory type preferred by the caller. Returns the - * actual memory type of 'buffer'. - * @param memory_type_id Acts as both input and output. On input - * gives the buffer memory type id preferred by the caller. Returns - * the actual memory type id of 'buffer'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( - TRITONBACKEND_State state, @Cast("void**") PointerPointer buffer, @Cast("const uint64_t") long buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( - TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( - TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") IntBuffer memory_type, @Cast("int64_t*") LongBuffer memory_type_id); -public static native TRITONSERVER_Error TRITONBACKEND_StateBuffer( - TRITONBACKEND_State state, @Cast("void**") @ByPtrPtr Pointer buffer, @Cast("const uint64_t") long buffer_byte_size, - @Cast("TRITONSERVER_MemoryType*") int[] memory_type, @Cast("int64_t*") long[] memory_type_id); - -/** Get the buffer attributes associated with the given state buffer. - * The returned 'buffer_attributes' is owned by the state and so should not be - * modified or freed by the caller. The lifetime of the 'buffer_attributes' - * matches that of the state. - * - * @param state The state. - * @param buffer_attributes Returns the buffer attributes for the given state. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_StateBufferAttributes( - TRITONBACKEND_State state, - @Cast("TRITONSERVER_BufferAttributes**") PointerPointer buffer_attributes); -public static native TRITONSERVER_Error TRITONBACKEND_StateBufferAttributes( - TRITONBACKEND_State state, - @ByPtrPtr TRITONSERVER_BufferAttributes buffer_attributes); - -/** - * TRITONBACKEND_Backend - * - * Object representing a backend. - * -

- * TRITONBACKEND_ExecutionPolicy - * - * Types of execution policy that can be implemented by a backend. - * - * TRITONBACKEND_EXECUTION_BLOCKING: An instance of the model - * blocks in TRITONBACKEND_ModelInstanceExecute until it is ready - * to handle another inference. Upon returning from - * TRITONBACKEND_ModelInstanceExecute, Triton may immediately - * call TRITONBACKEND_ModelInstanceExecute for the same instance - * to execute a new batch of requests. Thus, most backends using - * this policy will not return from - * TRITONBACKEND_ModelInstanceExecute until all responses have - * been sent and all requests have been released. This is the - * default execution policy. - * - * TRITONBACKEND_EXECUTION_DEVICE_BLOCKING: An instance, A, of the - * model blocks in TRITONBACKEND_ModelInstanceExecute if the - * device associated with the instance is unable to handle - * another inference. Even if another instance, B, associated - * with the device, is available and ready to perform an - * inference, Triton will not invoke - * TRITONBACKEND_ModeInstanceExecute for B until A returns from - * TRITONBACKEND_ModelInstanceExecute. Triton will not be blocked - * from calling TRITONBACKEND_ModelInstanceExecute for instance - * C, which is associated with a different device than A and B, - * even if A or B has not returned from - * TRITONBACKEND_ModelInstanceExecute. This execution policy is - * typically used by a backend that can cooperatively execute - * multiple model instances on the same device. - * */ -/** enum TRITONBACKEND_ExecutionPolicy */ -public static final int - TRITONBACKEND_EXECUTION_BLOCKING = 0, - TRITONBACKEND_EXECUTION_DEVICE_BLOCKING = 1; - -/** Get the name of the backend. The caller does not own the returned - * string and must not modify or delete it. The lifetime of the - * returned string extends only as long as 'backend'. - * - * @param backend The backend. - * @param name Returns the name of the backend. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendName( - TRITONBACKEND_Backend backend, @Cast("const char**") PointerPointer name); -public static native TRITONSERVER_Error TRITONBACKEND_BackendName( - TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr BytePointer name); -public static native TRITONSERVER_Error TRITONBACKEND_BackendName( - TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr ByteBuffer name); -public static native TRITONSERVER_Error TRITONBACKEND_BackendName( - TRITONBACKEND_Backend backend, @Cast("const char**") @ByPtrPtr byte[] name); - -/** Get the backend configuration. The 'backend_config' message is - * owned by Triton and should not be modified or freed by the caller. - * - * The backend configuration, as JSON, is: - * - * { - * "cmdline" : { - * "" : "", - * ... - * } - * } - * - * @param backend The backend. - * @param backend_config Returns the backend configuration as a message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendConfig( - TRITONBACKEND_Backend backend, @Cast("TRITONSERVER_Message**") PointerPointer backend_config); -public static native TRITONSERVER_Error TRITONBACKEND_BackendConfig( - TRITONBACKEND_Backend backend, @ByPtrPtr TRITONSERVER_Message backend_config); - -/** Get the execution policy for this backend. By default the - * execution policy is TRITONBACKEND_EXECUTION_BLOCKING. - * - * @param backend The backend. - * @param policy Returns the execution policy. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") IntPointer policy); -public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") IntBuffer policy); -public static native TRITONSERVER_Error TRITONBACKEND_BackendExecutionPolicy( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy*") int[] policy); - -/** Set the execution policy for this backend. By default the - * execution policy is TRITONBACKEND_EXECUTION_BLOCKING. Triton reads - * the backend's execution policy after calling - * TRITONBACKEND_Initialize, so to be recognized changes to the - * execution policy must be made in TRITONBACKEND_Initialize. - * Also, note that if using sequence batcher for the model, Triton will - * use TRITONBACKEND_EXECUTION_BLOCKING policy irrespective of the - * policy specified by this setter function. - * - * @param backend The backend. - * @param policy The execution policy. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendSetExecutionPolicy( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ExecutionPolicy") int policy); - -/** Get the location of the files that make up the backend - * implementation. This location contains the backend shared library - * and any other files located with the shared library. The - * 'location' communicated depends on how the backend is being - * communicated to Triton as indicated by 'artifact_type'. - * - * TRITONBACKEND_ARTIFACT_FILESYSTEM: The backend artifacts are - * made available to Triton via the local filesytem. 'location' - * returns the full path to the directory containing this - * backend's artifacts. The returned string is owned by Triton, - * not the caller, and so should not be modified or freed. - * - * @param backend The backend. - * @param artifact_type Returns the artifact type for the backend. - * @param path Returns the location. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, - @Cast("const char**") PointerPointer location); -public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, - @Cast("const char**") @ByPtrPtr BytePointer location); -public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") IntBuffer artifact_type, - @Cast("const char**") @ByPtrPtr ByteBuffer location); -public static native TRITONSERVER_Error TRITONBACKEND_BackendArtifacts( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_ArtifactType*") int[] artifact_type, - @Cast("const char**") @ByPtrPtr byte[] location); - -/** Get the memory manager associated with a backend. - * - * @param backend The backend. - * @param manager Returns the memory manager. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendMemoryManager( - TRITONBACKEND_Backend backend, @Cast("TRITONBACKEND_MemoryManager**") PointerPointer manager); -public static native TRITONSERVER_Error TRITONBACKEND_BackendMemoryManager( - TRITONBACKEND_Backend backend, @ByPtrPtr TRITONBACKEND_MemoryManager manager); - -/** Get the user-specified state associated with the backend. The - * state is completely owned and managed by the backend. - * - * @param backend The backend. - * @param state Returns the user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendState( - TRITONBACKEND_Backend backend, @Cast("void**") PointerPointer state); -public static native TRITONSERVER_Error TRITONBACKEND_BackendState( - TRITONBACKEND_Backend backend, @Cast("void**") @ByPtrPtr Pointer state); - -/** Set the user-specified state associated with the backend. The - * state is completely owned and managed by the backend. - * - * @param backend The backend. - * @param state The user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendSetState( - TRITONBACKEND_Backend backend, Pointer state); - -/** - * TRITONBACKEND_Model - * - * Object representing a model implemented using the backend. - * -

- * Get the name of the model. The returned string is owned by the - * model object, not the caller, and so should not be modified or - * freed. - * - * @param model The model. - * @param name Returns the model name. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelName( - TRITONBACKEND_Model model, @Cast("const char**") PointerPointer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelName( - TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr BytePointer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelName( - TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr ByteBuffer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelName( - TRITONBACKEND_Model model, @Cast("const char**") @ByPtrPtr byte[] name); - -/** Get the version of the model. - * - * @param model The model. - * @param version Returns the model version. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( - TRITONBACKEND_Model model, @Cast("uint64_t*") LongPointer version); -public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( - TRITONBACKEND_Model model, @Cast("uint64_t*") LongBuffer version); -public static native TRITONSERVER_Error TRITONBACKEND_ModelVersion( - TRITONBACKEND_Model model, @Cast("uint64_t*") long[] version); - -/** Get the location of the files that make up the model. The - * 'location' communicated depends on how the model is being - * communicated to Triton as indicated by 'artifact_type'. - * - * TRITONBACKEND_ARTIFACT_FILESYSTEM: The model artifacts are made - * available to Triton via the local filesytem. 'location' - * returns the full path to the directory in the model repository - * that contains this model's artifacts. The returned string is - * owned by Triton, not the caller, and so should not be modified - * or freed. - * - * @param model The model. - * @param artifact_type Returns the artifact type for the model. - * @param path Returns the location. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( - TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, - @Cast("const char**") PointerPointer location); -public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( - TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntPointer artifact_type, - @Cast("const char**") @ByPtrPtr BytePointer location); -public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( - TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") IntBuffer artifact_type, - @Cast("const char**") @ByPtrPtr ByteBuffer location); -public static native TRITONSERVER_Error TRITONBACKEND_ModelRepository( - TRITONBACKEND_Model model, @Cast("TRITONBACKEND_ArtifactType*") int[] artifact_type, - @Cast("const char**") @ByPtrPtr byte[] location); - -/** Get the model configuration. The caller takes ownership of the - * message object and must call TRITONSERVER_MessageDelete to release - * the object. The configuration is available via this call even - * before the model is loaded and so can be used in - * TRITONBACKEND_ModelInitialize. TRITONSERVER_ServerModelConfig - * returns equivalent information but is not useable until after the - * model loads. - * - * @param model The model. - * @param config_version The model configuration will be returned in - * a format matching this version. If the configuration cannot be - * represented in the requested version's format then an error will - * be returned. Currently only version 1 is supported. - * @param model_config Returns the model configuration as a message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelConfig( - TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, - @Cast("TRITONSERVER_Message**") PointerPointer model_config); -public static native TRITONSERVER_Error TRITONBACKEND_ModelConfig( - TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, - @ByPtrPtr TRITONSERVER_Message model_config); - -/** Whether the backend should attempt to auto-complete the model configuration. - * If true, the model should fill the inputs, outputs, and max batch size in - * the model configuration if incomplete. If the model configuration is - * changed, the new configuration must be reported to Triton using - * TRITONBACKEND_ModelSetConfig. - * - * @param model The model. - * @param auto_complete_config Returns whether the backend should auto-complete - * the model configuration. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( - TRITONBACKEND_Model model, @Cast("bool*") boolean[] auto_complete_config); -public static native TRITONSERVER_Error TRITONBACKEND_ModelAutoCompleteConfig( - TRITONBACKEND_Model model, @Cast("bool*") BoolPointer auto_complete_config); - -/** Set the model configuration in Triton server. This API should only be called - * when the backend implements the auto-completion of model configuration - * and TRITONBACKEND_ModelAutoCompleteConfig returns true in - * auto_complete_config. Only the inputs, outputs, max batch size, and - * scheduling choice can be changed. A caveat being scheduling choice can only - * be changed if none is previously set. Any other changes to the model - * configuration will be ignored by Triton. This function can only be called - * from TRITONBACKEND_ModelInitialize, calling in any other context will result - * in an error being returned. Additionally, Triton server can add some of the - * missing fields in the provided config with this call. The backend must get - * the complete configuration again by using TRITONBACKEND_ModelConfig. - * TRITONBACKEND_ModelSetConfig does not take ownership of the message object - * and so the caller should call TRITONSERVER_MessageDelete to release the - * object once the function returns. - * - * @param model The model. - * @param config_version The format version of the model configuration. - * If the configuration is not represented in the version's format - * then an error will be returned. Currently only version 1 is supported. - * @param model_config The updated model configuration as a message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelSetConfig( - TRITONBACKEND_Model model, @Cast("const uint32_t") int config_version, - TRITONSERVER_Message model_config); - -/** Get the TRITONSERVER_Server object that this model is being served - * by. - * - * @param model The model. - * @param server Returns the server. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelServer( - TRITONBACKEND_Model model, @Cast("TRITONSERVER_Server**") PointerPointer server); -public static native TRITONSERVER_Error TRITONBACKEND_ModelServer( - TRITONBACKEND_Model model, @ByPtrPtr TRITONSERVER_Server server); - -/** Get the backend used by the model. - * - * @param model The model. - * @param model Returns the backend object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelBackend( - TRITONBACKEND_Model model, @Cast("TRITONBACKEND_Backend**") PointerPointer backend); -public static native TRITONSERVER_Error TRITONBACKEND_ModelBackend( - TRITONBACKEND_Model model, @ByPtrPtr TRITONBACKEND_Backend backend); - -/** Get the user-specified state associated with the model. The - * state is completely owned and managed by the backend. - * - * @param model The model. - * @param state Returns the user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelState( - TRITONBACKEND_Model model, @Cast("void**") PointerPointer state); -public static native TRITONSERVER_Error TRITONBACKEND_ModelState( - TRITONBACKEND_Model model, @Cast("void**") @ByPtrPtr Pointer state); - -/** Set the user-specified state associated with the model. The - * state is completely owned and managed by the backend. - * - * @param model The model. - * @param state The user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( - TRITONBACKEND_Model model, Pointer state); - -/** - * TRITONBACKEND_ModelInstance - * - * Object representing a model instance implemented using the - * backend. - * -

- * Get the name of the model instance. The returned string is owned by the - * model object, not the caller, and so should not be modified or - * freed. - * - * @param instance The model instance. - * @param name Returns the instance name. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( - TRITONBACKEND_ModelInstance instance, @Cast("const char**") PointerPointer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( - TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr BytePointer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( - TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr ByteBuffer name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceName( - TRITONBACKEND_ModelInstance instance, @Cast("const char**") @ByPtrPtr byte[] name); - -/** Get the kind of the model instance. - * - * @param instance The model instance. - * @param kind Returns the instance kind. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( - TRITONBACKEND_ModelInstance instance, - @Cast("TRITONSERVER_InstanceGroupKind*") IntPointer kind); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( - TRITONBACKEND_ModelInstance instance, - @Cast("TRITONSERVER_InstanceGroupKind*") IntBuffer kind); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceKind( - TRITONBACKEND_ModelInstance instance, - @Cast("TRITONSERVER_InstanceGroupKind*") int[] kind); - -/** Get the device ID of the model instance. - * - * @param instance The model instance. - * @param device_id Returns the instance device ID. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( - TRITONBACKEND_ModelInstance instance, IntPointer device_id); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( - TRITONBACKEND_ModelInstance instance, IntBuffer device_id); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceDeviceId( - TRITONBACKEND_ModelInstance instance, int[] device_id); - -/** Get the host policy setting. The 'host_policy' message is - * owned by Triton and should not be modified or freed by the caller. - * - * The host policy setting, as JSON, is: - * - * { - * "" : { - * "" : "", - * ... - * } - * } - * - * @param instance The model instance. - * @param host_policy Returns the host policy setting as a message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceHostPolicy( - TRITONBACKEND_ModelInstance instance, @Cast("TRITONSERVER_Message**") PointerPointer host_policy); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceHostPolicy( - TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONSERVER_Message host_policy); - -/** Whether the model instance is passive. - * - * @param instance The model instance. - * @param is_passive Returns true if the instance is passive, false otherwise - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceIsPassive( - TRITONBACKEND_ModelInstance instance, @Cast("bool*") boolean[] is_passive); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceIsPassive( - TRITONBACKEND_ModelInstance instance, @Cast("bool*") BoolPointer is_passive); - -/** Get the number of optimization profiles to be loaded for the instance. - * - * @param instance The model instance. - * @param count Returns the number of optimization profiles. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") int[] count); - -/** Get the name of optimization profile. The caller does not own - * the returned string and must not modify or delete it. The lifetime - * of the returned string extends only as long as 'instance'. - * - * @param instance The model instance. - * @param index The index of the optimization profile. Must be 0 - * <= index < count, where count is the value returned by - * TRITONBACKEND_ModelInstanceProfileCount. - * @param profile_name Returns the name of the optimization profile - * corresponding to the index. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( - TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, - @Cast("const char**") PointerPointer profile_name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( - TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr BytePointer profile_name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( - TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr ByteBuffer profile_name); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceProfileName( - TRITONBACKEND_ModelInstance instance, @Cast("const uint32_t") int index, - @Cast("const char**") @ByPtrPtr byte[] profile_name); - -/** Get the number of secondary devices configured for the instance. - * - * @param instance The model instance. - * @param count Returns the number of secondary devices. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceCount( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t*") int[] count); - -/** Get the properties of indexed secondary device. The returned - * strings and other properties are owned by the instance, not the - * caller, and so should not be modified or freed. - * - * @param instance The model instance. - * @param index The index of the secondary device. Must be 0 - * <= index < count, where count is the value returned by - * TRITONBACKEND_ModelInstanceSecondaryDeviceCount. - * @param kind Returns the kind of secondary device corresponding - * to the index. - * @param id Returns the id of secondary device corresponding to the index. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") PointerPointer kind, - @Cast("int64_t*") LongPointer id); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer kind, - @Cast("int64_t*") LongPointer id); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer kind, - @Cast("int64_t*") LongBuffer id); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( - TRITONBACKEND_ModelInstance instance, @Cast("uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] kind, - @Cast("int64_t*") long[] id); - -/** Get the model associated with a model instance. - * - * @param instance The model instance. - * @param backend Returns the model object. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceModel( - TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Model**") PointerPointer model); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceModel( - TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Model model); - -/** Get the user-specified state associated with the model - * instance. The state is completely owned and managed by the - * backend. - * - * @param instance The model instance. - * @param state Returns the user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( - TRITONBACKEND_ModelInstance instance, @Cast("void**") PointerPointer state); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( - TRITONBACKEND_ModelInstance instance, @Cast("void**") @ByPtrPtr Pointer state); - -/** Set the user-specified state associated with the model - * instance. The state is completely owned and managed by the - * backend. - * - * @param instance The model instance. - * @param state The user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( - TRITONBACKEND_ModelInstance instance, Pointer state); - -/** Record statistics for an inference request. - * - * Set 'success' true to indicate that the inference request - * completed successfully. In this case all timestamps should be - * non-zero values reported in nanoseconds and should be collected - * using std::chrono::steady_clock::now().time_since_epoch() or the equivalent. - * Set 'success' to false to indicate that the inference request failed - * to complete successfully. In this case all timestamps values are - * ignored. - * - * For consistency of measurement across different backends, the - * timestamps should be collected at the following points during - * TRITONBACKEND_ModelInstanceExecute. - * - * TRITONBACKEND_ModelInstanceExecute() - * CAPTURE TIMESPACE (exec_start_ns) - * < process input tensors to prepare them for inference - * execution, including copying the tensors to/from GPU if - * necessary> - * CAPTURE TIMESPACE (compute_start_ns) - * < perform inference computations to produce outputs > - * CAPTURE TIMESPACE (compute_end_ns) - * < allocate output buffers and extract output tensors, including - * copying the tensors to/from GPU if necessary> - * CAPTURE TIMESPACE (exec_end_ns) - * return - * - * Note that these statistics are associated with a valid - * TRITONBACKEND_Request object and so must be reported before the - * request is released. For backends that release the request before - * all response(s) are sent, these statistics cannot capture - * information about the time required to produce the response. - * - * @param instance The model instance. - * @param request The inference request that statistics are being - * reported for. - * @param success True if the inference request completed - * successfully, false if it failed to complete. - * @param exec_start_ns Timestamp for the start of execution. - * @param compute_start_ns Timestamp for the start of execution - * computations. - * @param compute_end_ns Timestamp for the end of execution - * computations. - * @param exec_end_ns Timestamp for the end of execution. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportStatistics( - TRITONBACKEND_ModelInstance instance, TRITONBACKEND_Request request, - @Cast("const bool") boolean success, @Cast("const uint64_t") long exec_start_ns, - @Cast("const uint64_t") long compute_start_ns, @Cast("const uint64_t") long compute_end_ns, - @Cast("const uint64_t") long exec_end_ns); - -/** Record statistics for the execution of an entire batch of - * inference requests. - * - * All timestamps should be non-zero values reported in nanoseconds - * and should be collected using - * std::chrono::steady_clock::now().time_since_epoch() or the equivalent. - * See TRITONBACKEND_ModelInstanceReportStatistics for more information about - * the timestamps. - * - * 'batch_size' is the sum of the batch sizes for the individual - * requests that were delivered together in the call to - * TRITONBACKEND_ModelInstanceExecute. For example, if three requests - * are passed to TRITONBACKEND_ModelInstanceExecute and those - * requests have batch size 1, 2, and 3; then 'batch_size' should be - * set to 6. - * - * @param instance The model instance. - * @param batch_size Combined batch size of all the individual - * requests executed in the batch. - * @param exec_start_ns Timestamp for the start of execution. - * @param compute_start_ns Timestamp for the start of execution - * computations. - * @param compute_end_ns Timestamp for the end of execution - * computations. - * @param exec_end_ns Timestamp for the end of execution. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportBatchStatistics( - TRITONBACKEND_ModelInstance instance, @Cast("const uint64_t") long batch_size, - @Cast("const uint64_t") long exec_start_ns, @Cast("const uint64_t") long compute_start_ns, - @Cast("const uint64_t") long compute_end_ns, @Cast("const uint64_t") long exec_end_ns); - -/** - * The following functions can be implemented by a backend. Functions - * indicated as required must be implemented or the backend will fail - * to load. - * -

- * Initialize a backend. This function is optional, a backend is not - * required to implement it. This function is called once when a - * backend is loaded to allow the backend to initialize any state - * associated with the backend. A backend has a single state that is - * shared across all models that use the backend. - * - * @param backend The backend. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_Initialize( - TRITONBACKEND_Backend backend); - -/** Finalize for a backend. This function is optional, a backend is - * not required to implement it. This function is called once, just - * before the backend is unloaded. All state associated with the - * backend should be freed and any threads created for the backend - * should be exited/joined before returning from this function. - * - * @param backend The backend. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_Finalize( - TRITONBACKEND_Backend backend); - -/** Initialize for a model. This function is optional, a backend is - * not required to implement it. This function is called once when a - * model that uses the backend is loaded to allow the backend to - * initialize any state associated with the model. The backend should - * also examine the model configuration to determine if the - * configuration is suitable for the backend. Any errors reported by - * this function will prevent the model from loading. - * - * @param model The model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInitialize( - TRITONBACKEND_Model model); - -/** Finalize for a model. This function is optional, a backend is not - * required to implement it. This function is called once for a - * model, just before the model is unloaded from Triton. All state - * associated with the model should be freed and any threads created - * for the model should be exited/joined before returning from this - * function. - * - * @param model The model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelFinalize( - TRITONBACKEND_Model model); - -/** Initialize for a model instance. This function is optional, a - * backend is not required to implement it. This function is called - * once when a model instance is created to allow the backend to - * initialize any state associated with the instance. - * - * @param instance The model instance. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceInitialize( - TRITONBACKEND_ModelInstance instance); - -/** Finalize for a model instance. This function is optional, a - * backend is not required to implement it. This function is called - * once for an instance, just before the corresponding model is - * unloaded from Triton. All state associated with the instance - * should be freed and any threads created for the instance should be - * exited/joined before returning from this function. - * - * @param instance The model instance. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceFinalize( - TRITONBACKEND_ModelInstance instance); - -/** Execute a batch of one or more requests on a model instance. This - * function is required. Triton will not perform multiple - * simultaneous calls to this function for a given model 'instance'; - * however, there may be simultaneous calls for different model - * instances (for the same or different models). - * - * If an error is returned the ownership of the request objects - * remains with Triton and the backend must not retain references to - * the request objects or access them in any way. - * - * If success is returned, ownership of the request objects is - * transferred to the backend and it is then responsible for creating - * responses and releasing the request objects. Note that even though - * ownership of the request objects is transferred to the backend, the - * ownership of the buffer holding request pointers is returned back - * to Triton upon return from TRITONBACKEND_ModelInstanceExecute. If - * any request objects need to be maintained beyond - * TRITONBACKEND_ModelInstanceExecute, then the pointers must be copied - * out of the array within TRITONBACKEND_ModelInstanceExecute. - * - * @param instance The model instance. - * @param requests The requests. - * @param request_count The number of requests in the batch. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( - TRITONBACKEND_ModelInstance instance, @Cast("TRITONBACKEND_Request**") PointerPointer requests, - @Cast("const uint32_t") int request_count); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceExecute( - TRITONBACKEND_ModelInstance instance, @ByPtrPtr TRITONBACKEND_Request requests, - @Cast("const uint32_t") int request_count); - -/** Query the backend for different model attributes. This function is optional, - * a backend is not required to implement it. The backend is also not required - * to set all backend attribute listed. This function is called when - * Triton requires further backend / model information to perform operations. - * This function may be called multiple times within the lifetime of the - * backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize). - * The backend may return error to indicate failure to set the backend - * attributes, and the attributes specified in the same function call will be - * ignored. Triton will update the specified attributes if 'nullptr' is - * returned. - * - * @param backend The backend. - * @param backend_attributes Return the backend attribute. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_GetBackendAttribute( - TRITONBACKEND_Backend backend, - TRITONBACKEND_BackendAttribute backend_attributes); - -/** TRITONBACKEND_BackendAttribute - * - * API to modify attributes associated with a backend. - * -

- * Add the preferred instance group of the backend. This function - * can be called multiple times to cover different instance group kinds that - * the backend supports, given the priority order that the first call describes - * the most preferred group. In the case where instance group are not - * explicitly provided, Triton will use this attribute to create model - * deployment that aligns more with the backend preference. - * - * @param backend_attributes The backend attributes object. - * @param kind The kind of the instance group. - * @param count The number of instances per device. Triton default will be used - * if 0 is provided. - * @param device_ids The devices where instances should be available. Triton - * default will be used if 'nullptr' is provided. - * @param id_count The number of devices in 'device_ids'. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( - TRITONBACKEND_BackendAttribute backend_attributes, - @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, - @Cast("const uint64_t*") LongPointer device_ids, @Cast("const uint64_t") long id_count); -public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( - TRITONBACKEND_BackendAttribute backend_attributes, - @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, - @Cast("const uint64_t*") LongBuffer device_ids, @Cast("const uint64_t") long id_count); -public static native TRITONSERVER_Error TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( - TRITONBACKEND_BackendAttribute backend_attributes, - @Cast("const TRITONSERVER_InstanceGroupKind") int kind, @Cast("const uint64_t") long count, - @Cast("const uint64_t*") long[] device_ids, @Cast("const uint64_t") long id_count); - -/** TRITONBACKEND Batching - * - * API to add custom batching strategy - * - * The following functions can be implemented by a backend to add custom - * batching conditionals on top of the existing Triton batching strategy. The - * functions are optional but all or none must be implemented. - * -

- * Create a new batcher for use with custom batching. This is called during - * model loading. The batcher will point to a user-defined data structure that - * holds read-only data used for custom batching. - * - * @param batcher User-defined placeholder for backend to store and - * retrieve information about the batching strategy for this - * model.RITONBACKEND_ISPEC return a TRITONSERVER_Error indicating success or - * failure. @param model The backend model for which Triton is forming a batch. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherInitialize( - @Cast("TRITONBACKEND_Batcher**") PointerPointer batcher, TRITONBACKEND_Model model); -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherInitialize( - @ByPtrPtr TRITONBACKEND_Batcher batcher, TRITONBACKEND_Model model); - -/** Free memory associated with batcher. This is called during model unloading. - * - * @param batcher User-defined placeholder for backend to store and - * retrieve information about the batching strategy for this model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatcherFinalize( - TRITONBACKEND_Batcher batcher); - -/** Check whether a request should be added to the pending model batch. - * - * @param request The request to be added to the pending batch. - * @param userp The placeholder for backend to store and retrieve information - * about this pending batch. When the callback returns, this should reflect - * the latest batch information. - * @param should_include The pointer to be updated on whether the request - * should be included in the batch. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchIncludeRequest( - TRITONBACKEND_Request request, Pointer userp, @Cast("bool*") boolean[] should_include); -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchIncludeRequest( - TRITONBACKEND_Request request, Pointer userp, @Cast("bool*") BoolPointer should_include); - -/** Callback to be invoked when Triton has begun forming a batch. - * - * @param batcher The read-only placeholder for backend to retrieve */ -// information about the batching strategy for this model. -/** @param userp The placeholder for backend to store and retrieve information -/** about this pending batch. -/** @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchInitialize( - @Const TRITONBACKEND_Batcher batcher, @Cast("void**") PointerPointer userp); -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchInitialize( - @Const TRITONBACKEND_Batcher batcher, @Cast("void**") @ByPtrPtr Pointer userp); - -/** Callback to be invoked when Triton has finishing forming a batch. - * - * @param userp The placeholder for backend to store and retrieve information - * about this pending batch. - * @return a TRITONSERVER_Error indicating success or failure. */ -public static native TRITONSERVER_Error TRITONBACKEND_ModelBatchFinalize( - Pointer userp); - -// #ifdef __cplusplus -// #endif - - -// Parsed from tritonrepoagent.h - -// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once - -// #include -// #include -// #include "triton/core/tritonserver.h" - -// #ifdef __cplusplus -// #endif - -// #ifdef _COMPILING_TRITONREPOAGENT -// #if defined(_MSC_VER) -// #define TRITONREPOAGENT_DECLSPEC __declspec(dllexport) -// #define TRITONREPOAGENT_ISPEC __declspec(dllimport) -// #elif defined(__GNUC__) -// #define TRITONREPOAGENT_DECLSPEC __attribute__((__visibility__("default"))) -// #define TRITONREPOAGENT_ISPEC -// #else -// #define TRITONREPOAGENT_DECLSPEC -// #define TRITONREPOAGENT_ISPEC -// #endif -// #else -// #if defined(_MSC_VER) -// #define TRITONREPOAGENT_DECLSPEC __declspec(dllimport) -// #define TRITONREPOAGENT_ISPEC __declspec(dllexport) -// #else -// #define TRITONREPOAGENT_DECLSPEC -// #define TRITONREPOAGENT_ISPEC -// Targeting ../tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java - - -// Targeting ../tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java - - - -/** - * TRITONREPOAGENT API Version - * - * The TRITONREPOAGENT API is versioned with major and minor version - * numbers. Any change to the API that does not impact backwards - * compatibility (for example, adding a non-required function) - * increases the minor version number. Any change that breaks - * backwards compatibility (for example, deleting or changing the - * behavior of a function) increases the major version number. A - * repository agent should check that the API version used to compile - * the agent is compatible with the API version of the Triton server - * that it is running in. This is typically done by code similar to - * the following which makes sure that the major versions are equal - * and that the minor version of Triton is >= the minor version used - * to build the agent. - * - * uint32_t api_version_major, api_version_minor; - * TRITONREPOAGENT_ApiVersion(&api_version_major, &api_version_minor); - * if ((api_version_major != TRITONREPOAGENT_API_VERSION_MAJOR) || - * (api_version_minor < TRITONREPOAGENT_API_VERSION_MINOR)) { - * return TRITONSERVER_ErrorNew( - * TRITONSERVER_ERROR_UNSUPPORTED, - * "triton repository agent API version does not support this agent"); - * } - * */ -public static final int TRITONREPOAGENT_API_VERSION_MAJOR = 0; - -/// -public static final int TRITONREPOAGENT_API_VERSION_MINOR = 1; - -/** Get the TRITONREPOAGENT API version supported by Triton. This - * value can be compared against the - * TRITONREPOAGENT_API_VERSION_MAJOR and - * TRITONREPOAGENT_API_VERSION_MINOR used to build the agent to - * ensure that Triton is compatible with the agent. - * - * @param major Returns the TRITONREPOAGENT API major version supported - * by Triton. - * @param minor Returns the TRITONREPOAGENT API minor version supported - * by Triton. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( - @Cast("uint32_t*") IntPointer major, @Cast("uint32_t*") IntPointer minor); -public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( - @Cast("uint32_t*") IntBuffer major, @Cast("uint32_t*") IntBuffer minor); -public static native TRITONSERVER_Error TRITONREPOAGENT_ApiVersion( - @Cast("uint32_t*") int[] major, @Cast("uint32_t*") int[] minor); - -/** TRITONREPOAGENT_ArtifactType - * - * The ways that the files that make up a model's repository content - * are communicated between Triton and the agent. - * - * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are - * communicated to and from the repository agent via a locally - * accessible filesystem. The agent can access these files using - * an appropriate filesystem API. - * - * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are - * communicated to and from the repository agent via a remote filesystem. - * The remote filesystem path follows the same convention as is used for - * repository paths, for example, "s3://" prefix indicates an S3 path. - * */ -/** enum TRITONREPOAGENT_ArtifactType */ -public static final int - TRITONREPOAGENT_ARTIFACT_FILESYSTEM = 0, - TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM = 1; - -/** TRITONREPOAGENT_ActionType - * - * Types of repository actions that can be handled by an agent. - * The lifecycle of a TRITONREPOAGENT_AgentModel begins with a call to - * TRITONREPOAGENT_ModelInitialize and ends with a call to - * TRITONREPOAGENT_ModelFinalize. Between those calls the current lifecycle - * state of the model is communicated by calls to TRITONREPOAGENT_ModelAction. - * Possible lifecycles are: - * - * LOAD -> LOAD_COMPLETE -> UNLOAD -> UNLOAD_COMPLETE - * LOAD -> LOAD_FAIL - * - * TRITONREPOAGENT_ACTION_LOAD: A model is being loaded. - * - * TRITONREPOAGENT_ACTION_LOAD_COMPLETE: The model load completed - * successfully and the model is now loaded. - * - * TRITONREPOAGENT_ACTION_LOAD_FAIL: The model load did not complete - * successfully. The model is not loaded. - * - * TRITONREPOAGENT_ACTION_UNLOAD: The model is being unloaded. - * - * TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: The model unload is complete. - * */ -/** enum TRITONREPOAGENT_ActionType */ -public static final int - TRITONREPOAGENT_ACTION_LOAD = 0, - TRITONREPOAGENT_ACTION_LOAD_COMPLETE = 1, - TRITONREPOAGENT_ACTION_LOAD_FAIL = 2, - TRITONREPOAGENT_ACTION_UNLOAD = 3, - TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE = 4; - -/** Get the location of the files that make up the model. The - * 'location' communicated depends on how the model is being - * communicated to the agent as indicated by 'artifact_type'. - * - * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are - * made available to the agent via the local - * filesytem. 'location' returns the full path to the directory - * in the model repository that contains the model's - * artifacts. The returned location string is owned by Triton, - * not the caller, and so should not be modified or freed. The - * contents of the directory are owned by Triton, not the agent, - * and so the agent should not delete or modify the contents. Use - * TRITONREPOAGENT_RepositoryAcquire to get a location that can be - * used to modify the model repository contents. - * - * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are - * made available to the agent via a remote filesystem. - * 'location' returns the full path to the remote directory that contains - * the model's artifacts. The returned location string is owned by Triton, - * not the caller, and so should not be modified or freed. The contents of - * the remote directory are owned by Triton, not the agent, - * and so the agent should not delete or modify the contents. - * Use TRITONREPOAGENT_ModelRepositoryLocationAcquire to get a location - * that can be used to write updated model repository contents. - * - * @param agent The agent. - * @param model The model. - * @param artifact_type Returns the artifact type for the location. - * @param path Returns the location. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("TRITONREPOAGENT_ArtifactType*") IntPointer artifact_type, @Cast("const char**") PointerPointer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("TRITONREPOAGENT_ArtifactType*") IntPointer artifact_type, @Cast("const char**") @ByPtrPtr BytePointer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("TRITONREPOAGENT_ArtifactType*") IntBuffer artifact_type, @Cast("const char**") @ByPtrPtr ByteBuffer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocation( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("TRITONREPOAGENT_ArtifactType*") int[] artifact_type, @Cast("const char**") @ByPtrPtr byte[] location); - -/** Acquire a location where the agent can produce a new version of - * the model repository files. This is a convenience method to create - * a temporary directory for the agent. The agent is responsible for - * calling TRITONREPOAGENT_ModelRepositoryLocationDelete in - * TRITONREPOAGENT_ModelFinalize to delete the location. Initially the - * acquired location is empty. The 'location' communicated depends on - * the requested 'artifact_type'. - * - * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The location is a directory - * on the local filesystem. 'location' returns the full path to - * an empty directory that the agent should populate with the - * model's artifacts. The returned location string is owned by - * Triton, not the agent, and so should not be modified or freed. - * - * @param agent The agent. - * @param model The model. - * @param artifact_type The artifact type for the location. - * @param path Returns the location. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") PointerPointer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr BytePointer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr ByteBuffer location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationAcquire( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char**") @ByPtrPtr byte[] location); - -/** Discard and release ownership of a previously acquired location - * and its contents. The agent must not access or modify the location - * or its contents after this call. - * - * @param agent The agent. - * @param model The model. - * @param path The location to release. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationRelease( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - String location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryLocationRelease( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const char*") BytePointer location); - -/** Inform Triton that the specified repository location should be used for - * the model in place of the original model repository. This method can only be - * called when TRITONREPOAGENT_ModelAction is invoked with - * TRITONREPOAGENT_ACTION_LOAD. The 'location' The 'location' - * communicated depends on how the repository is being - * communicated to Triton as indicated by 'artifact_type'. - * - * TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are - * made available to Triton via the local filesytem. 'location' returns - * the full path to the directory. Ownership of the contents of the - * returned directory are transferred to Triton and the agent should not - * modified or freed the contents until TRITONREPOAGENT_ModelFinalize. - * The local filesystem directory can be created using - * TRITONREPOAGENT_ModelReopsitroyLocationAcquire or the agent can use - * its own local filesystem API. - * - * TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are - * made available to Triton via a remote filesystem. 'location' returns - * the full path to the remote filesystem directory. Ownership of the - * contents of the returned directory are transferred to Triton and - * the agent should not modified or freed the contents until - * TRITONREPOAGENT_ModelFinalize. - * - * @param agent The agent. - * @param model The model. - * @param artifact_type The artifact type for the location. - * @param path Returns the location. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryUpdate( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, String location); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelRepositoryUpdate( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ArtifactType") int artifact_type, @Cast("const char*") BytePointer location); - -/** Get the number of agent parameters defined for a model. - * - * @param agent The agent. - * @param model The model. - * @param count Returns the number of input tensors. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("uint32_t*") IntPointer count); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("uint32_t*") IntBuffer count); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameterCount( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("uint32_t*") int[] count); - -/** Get a parameter name and value. The caller does not own the - * returned strings and must not modify or delete them. - * - * @param agent The agent. - * @param model The model. - * @param index The index of the parameter. Must be 0 <= index < - * count, where count is the value returned by - * TRITONREPOAGENT_ModelParameterCount. - * @param parameter_name Returns the name of the parameter. - * @param parameter_value Returns the value of the parameter. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int index, @Cast("const char**") PointerPointer parameter_name, - @Cast("const char**") PointerPointer parameter_value); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr BytePointer parameter_name, - @Cast("const char**") @ByPtrPtr BytePointer parameter_value); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr ByteBuffer parameter_name, - @Cast("const char**") @ByPtrPtr ByteBuffer parameter_value); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelParameter( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int index, @Cast("const char**") @ByPtrPtr byte[] parameter_name, - @Cast("const char**") @ByPtrPtr byte[] parameter_value); - -/** Get the model configuration. The caller takes ownership of the - * message object and must call TRITONSERVER_MessageDelete to release - * the object. If the model repository does not contain a - * config.pbtxt file then 'model_config' is returned as nullptr. - * - * @param agent The agent. - * @param model The model. - * @param config_version The model configuration will be returned in - * a format matching this version. If the configuration cannot be - * represented in the requested version's format then an error will - * be returned. Currently only version 1 is supported. - * @param model_config Returns the model configuration as a message. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelConfig( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int config_version, @Cast("TRITONSERVER_Message**") PointerPointer model_config); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelConfig( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const uint32_t") int config_version, @ByPtrPtr TRITONSERVER_Message model_config); - -/** Get the user-specified state associated with the model. - * - * @param model The agent model. - * @param state Returns the user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelState( - TRITONREPOAGENT_AgentModel model, @Cast("void**") PointerPointer state); -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelState( - TRITONREPOAGENT_AgentModel model, @Cast("void**") @ByPtrPtr Pointer state); - -/** Set the user-specified state associated with the model. - * - * @param model The agent model. - * @param state The user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelSetState( - TRITONREPOAGENT_AgentModel model, Pointer state); - -/** Get the user-specified state associated with the agent. - * - * @param agent The agent. - * @param state Returns the user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_State( - TRITONREPOAGENT_Agent agent, @Cast("void**") PointerPointer state); -public static native TRITONSERVER_Error TRITONREPOAGENT_State( - TRITONREPOAGENT_Agent agent, @Cast("void**") @ByPtrPtr Pointer state); - -/** Set the user-specified state associated with the agent. - * - * @param agent The agent. - * @param state The user state, or nullptr if no user state. - * @return a TRITONSERVER_Error indicating success or failure. */ - - -/// -/// -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_SetState( - TRITONREPOAGENT_Agent agent, Pointer state); - -/** - * The following functions can be implemented by an agent. Functions - * indicated as required must be implemented or the agent will fail - * to load. - * -

- * Initialize an agent. This function is optional. This function is - * called once when an agent is loaded to allow the agent to - * initialize any state associated with the agent. An agent has a - * single state that is shared across all invocations of the agent. - * - * @param agent The agent. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_Initialize( - TRITONREPOAGENT_Agent agent); - -/** Finalize for an agent. This function is optional. This function is - * called once, just before the agent is unloaded. All state - * associated with the agent should be freed and any threads created - * for the agent should be exited/joined before returning from this - * function. - * - * @param agent The agent. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_Finalize( - TRITONREPOAGENT_Agent agent); - -/** Initialize a model associated with an agent. This function is optional. - * This function is called once when an agent model's lifecycle begins to allow - * the agent model to initialize any state associated with it. An agent model - * has a single state that is shared across all the lifecycle of the agent - * model. - * - * @param agent The agent to be associated with the model. - * @param model The model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelInitialize( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model); - -/** Finalize for a model. This function is optional. This function is - * called once, just before the end of the agent model's lifecycle. All state - * associated with the agent model should be freed and any threads created - * for the agent model should be exited/joined before returning from this - * function. If the model acquired a model location using - * TRITONREPOAGENT_ModelRepositoryLocationAcquire, it must call - * TRITONREPOAGENT_ModelRepositoryLocationRelease to release that location. - * - * @param agent The agent associated with the model. - * @param model The model. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -/// -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelFinalize( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model); - -/** Handle an action for a specified model. This function is - * required. Triton will not perform multiple simultaneous calls to - * this function for a given agent and model; however, there may be - * simultaneous calls for the agent for different models. - * - * If the agent does not handle the action the agent should - * immediately return success (nullptr). - * - * Any modification to the model's repository must be made when 'action_type' - * is TRITONREPOAGENT_ACTION_LOAD. - * To modify the model's repository the agent must either acquire a mutable - * location via TRITONREPOAGENT_ModelRepositoryLocationAcquire - * or its own managed location, report the location to Triton via - * TRITONREPOAGENT_ModelRepositoryUpdate, and then return - * success (nullptr). If the agent does not need to make any changes - * to the model repository it should not call - * TRITONREPOAGENT_ModelRepositoryUpdate and then return success. - * To indicate that a model load should fail return a non-success status. - * - * @param agent The agent. - * @param model The model that is the target of the action. - * \action_type The type of action the agent should handle for the model. - * @return a TRITONSERVER_Error indicating success or failure. */ -public static native TRITONSERVER_Error TRITONREPOAGENT_ModelAction( - TRITONREPOAGENT_Agent agent, TRITONREPOAGENT_AgentModel model, - @Cast("const TRITONREPOAGENT_ActionType") int action_type); - -// #ifdef __cplusplus -// #endif - - -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java deleted file mode 100644 index e211ad88475..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Allocator.java +++ /dev/null @@ -1,37 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - - -//============================================================================== -/** Custom Allocator object for providing custom functions for allocator. - * If there is no custom allocator provided, will use the default allocator. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class Allocator extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public Allocator(Pointer p) { super(p); } - - public Allocator( - ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, - ResponseAllocatorStartFn_t start_fn/*=nullptr*/) { super((Pointer)null); allocate(alloc_fn, release_fn, start_fn); } - private native void allocate( - ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, - ResponseAllocatorStartFn_t start_fn/*=nullptr*/); - public Allocator( - ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn) { super((Pointer)null); allocate(alloc_fn, release_fn); } - private native void allocate( - ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn); - - public native ResponseAllocatorAllocFn_t AllocFn(); - public native OutputBufferReleaseFn_t ReleaseFn(); - public native ResponseAllocatorStartFn_t StartFn(); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java deleted file mode 100644 index 27cfc52cc0e..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/BackendConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold backend configuration for setting 'ServerOptions'. - * Different Triton-supported backends have different backend configuration - * options. Please refer to the 'Command line options' section in the - * documentation of each backend to see the options (e.g. Tensorflow Backend: - * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class BackendConfig extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public BackendConfig(Pointer p) { super(p); } - - public BackendConfig( - @StdString BytePointer name, @StdString BytePointer setting, - @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } - private native void allocate( - @StdString BytePointer name, @StdString BytePointer setting, - @StdString BytePointer value); - public BackendConfig( - @StdString String name, @StdString String setting, - @StdString String value) { super((Pointer)null); allocate(name, setting, value); } - private native void allocate( - @StdString String name, @StdString String setting, - @StdString String value); - - // The name of the backend. - public native @StdString BytePointer name_(); public native BackendConfig name_(BytePointer setter); - // The name of the setting. - public native @StdString BytePointer setting_(); public native BackendConfig setting_(BytePointer setter); - // The setting value. - public native @StdString BytePointer value_(); public native BackendConfig value_(BytePointer setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java deleted file mode 100644 index 811f9cb1615..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java +++ /dev/null @@ -1,32 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold CUDA memory pool byte size for setting 'ServerOptions'. - * If GPU support is enabled, the server will allocate CUDA memory to minimize - * data transfer between host and devices until it exceeds the specified byte - * size. This will not affect the allocation conducted by the backend - * frameworks. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class CUDAMemoryPoolByteSize extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public CUDAMemoryPoolByteSize(Pointer p) { super(p); } - - public CUDAMemoryPoolByteSize(int gpu_device, @Cast("const uint64_t") long size) { super((Pointer)null); allocate(gpu_device, size); } - private native void allocate(int gpu_device, @Cast("const uint64_t") long size); - - // The GPU device ID to allocate the memory pool. - public native int gpu_device_(); public native CUDAMemoryPoolByteSize gpu_device_(int setter); - // The CUDA memory pool byte size that the server can allocate on given GPU - // device. Default is 64 MB. - public native @Cast("uint64_t") long size_(); public native CUDAMemoryPoolByteSize size_(long setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java deleted file mode 100644 index 7b999d259dd..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferRequest.java +++ /dev/null @@ -1,57 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Object that describes an inflight inference request. - * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class GenericInferRequest extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public GenericInferRequest(Pointer p) { super(p); } - - /** Create an InferRequest instance. */ - public static native @UniquePtr GenericInferRequest Create( - @Const @ByRef InferOptions infer_options); - - /** Add an input tensor to be sent within an InferRequest object. The input - * data buffer within the 'Tensor' object must not be modified until - * inference is completed and result is returned. - * @param name The name of the input tensor. - * @param input A Tensor object that describes an input tensor. */ - public native @NoException(true) void AddInput( - @StdString BytePointer name, @Const @ByRef Tensor input); - public native @NoException(true) void AddInput( - @StdString String name, @Const @ByRef Tensor input); - - /** Add a requested output to be sent within an InferRequest object. - * Calling this function is optional. If no output(s) are specifically - * requested then all outputs defined by the model will be calculated and - * returned. Pre-allocated buffer for each output should be specified within - * the 'Tensor' object. - * @param name The name of the output tensor. - * @param output A Tensor object that describes an output tensor containing - * its pre-allocated buffer. */ - public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); - public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); - - /** Add a requested output to be sent within an InferRequest object. - * Calling this function is optional. If no output(s) are specifically - * requested then all outputs defined by the model will be calculated and - * returned. - * @param name The name of the output tensor. */ - public native void AddRequestedOutput(@StdString BytePointer name); - public native void AddRequestedOutput(@StdString String name); - - /** Clear inputs and outputs of the request. This allows users to reuse the - * InferRequest object if needed. */ - public native void Reset(); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java deleted file mode 100644 index ffdf4f46f81..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericInferResult.java +++ /dev/null @@ -1,71 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** An interface for InferResult object to interpret the response to an - * inference request. - * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class GenericInferResult extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public GenericInferResult(Pointer p) { super(p); } - - - /** Get the name of the model which generated this response. - * @return Returns the name of the model. */ - public native @StdString @NoException(true) BytePointer ModelName(); - - /** Get the version of the model which generated this response. - * @return Returns the version of the model. */ - public native @StdString @NoException(true) BytePointer ModelVersion(); - - /** Get the id of the request which generated this response. - * @return Returns the id of the request. */ - public native @StdString @NoException(true) BytePointer Id(); - - /** Get the output names from the infer result - * @return Vector of output names */ - public native @ByVal StringVector OutputNames(); - - /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' - * field of the output is owned by the returned 'Tensor' object itself. Note - * that for string data, need to use 'StringData' function for string data - * result. - * @param name The name of the output tensor to be retrieved. - * @return Returns the output result as a shared pointer of 'Tensor' object. */ - public native @SharedPtr Tensor Output(@StdString BytePointer name); - public native @SharedPtr Tensor Output(@StdString String name); - - /** Get the result data as a vector of strings. The vector will - * receive a copy of result data. An exception will be thrown if - * the data type of output is not 'BYTES'. - * @param output_name The name of the output to get result data. - * @return Returns the result data represented as a vector of strings. The - * strings are stored in the row-major order. */ - public native @ByVal StringVector StringData( - @StdString BytePointer output_name); - public native @ByVal StringVector StringData( - @StdString String output_name); - - /** Return the complete response as a user friendly string. - * @return The string describing the complete response. */ - public native @StdString BytePointer DebugString(); - - /** Return if there is an error within this result. - * @return True if this 'GenericInferResult' object has an error, false if no - * error. */ - public native @Cast("bool") boolean HasError(); - - /** Return the error message of the error. - * @return The messsage for the error. Empty if no error. */ - public native @StdString BytePointer ErrorMsg(); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java deleted file mode 100644 index a15dd744917..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/GenericTritonServer.java +++ /dev/null @@ -1,135 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Object that encapsulates in-process C API functionalities. - * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class GenericTritonServer extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public GenericTritonServer(Pointer p) { super(p); } - - /** Create a GenericTritonServer instance. */ - public static native @UniquePtr GenericTritonServer Create( - @Const @ByRef ServerOptions server_options); - - /** Load the requested model or reload the model if it is already loaded. - * @param model_name The name of the model. */ - public native void LoadModel(@StdString BytePointer model_name); - public native void LoadModel(@StdString String model_name); - - /** Unload the requested model. Unloading a model that is not loaded - * on server has no affect. - * @param model_name The name of the model. */ - public native void UnloadModel(@StdString BytePointer model_name); - public native void UnloadModel(@StdString String model_name); - - /** Get the set of names of models that are loaded and ready for inference. - * @return Returns the set of names of models that are - * loaded and ready for inference. */ - public native @ByVal StringSet LoadedModels(); - - /** Get the index of model repository contents. - * @return Returns a vector of 'RepositoryIndex' object - * representing the repository index. */ - public native @StdVector RepositoryIndex ModelIndex(); - - /** Get the metrics of the server. - * @return Returns a string representing the metrics. */ - public native @StdString BytePointer ServerMetrics(); - - /** Get the inference statistics of the specified model. - * @param model_name The name of the model. - * @param model_version the version of the model requested. - * @return Returns a json string representing the model metrics. */ - public native @StdString BytePointer ModelStatistics( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version); - public native @StdString String ModelStatistics( - @StdString String model_name, @Cast("const int64_t") long model_version); - - /** Is the server live? - * @return Returns true if server is live, false otherwise. */ - public native @Cast("bool") boolean IsServerLive(); - - /** Is the server ready? - * @return Returns true if server is ready, false otherwise. */ - public native @Cast("bool") boolean IsServerReady(); - - /** Stop a server object. A server can't be restarted once it is - * stopped. */ - public native void ServerStop(); - - /** Is the model ready? - * @param model_name The name of the model to get readiness for. - * @param model_version The version of the model to get readiness - * for. If -1 then the server will choose a version based on the - * model's policy. This field is optional, default is -1. - * @return Returns true if server is ready, false otherwise. */ - public native @Cast("bool") boolean IsModelReady( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @Cast("bool") boolean IsModelReady( - @StdString BytePointer model_name); - public native @Cast("bool") boolean IsModelReady( - @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @Cast("bool") boolean IsModelReady( - @StdString String model_name); - - /** Get the configuration of specified model. - * @param model_name The name of the model. - * @param model_version The version of the model to get configuration. - * The default value is -1 which means then the server will - * choose a version based on the model and internal policy. This field is - * optional. @return Returns JSON representation of model configuration as a - * string. */ - public native @StdString BytePointer ModelConfig( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @StdString BytePointer ModelConfig( - @StdString BytePointer model_name); - public native @StdString String ModelConfig( - @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @StdString String ModelConfig( - @StdString String model_name); - - /** Get the metadata of the server. - * @return Returns JSON representation of server metadata as a string. */ - public native @StdString BytePointer ServerMetadata(); - - /** Get the metadata of specified model. - * @param model_name The name of the model. - * @param model_version The version of the model to get configuration. - * The default value is -1 which means then the server will choose a version - * based on the model and internal policy. This field is optional. - * @return Returns JSON representation of model metadata as a string. */ - public native @StdString BytePointer ModelMetadata( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @StdString BytePointer ModelMetadata( - @StdString BytePointer model_name); - public native @StdString String ModelMetadata( - @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); - public native @StdString String ModelMetadata( - @StdString String model_name); - - /** Register a new model repository. This function is not available in polling - * mode. - * @param new_model_repo The 'NewModelRepo' object contains the info of the - * new model repo to be registered. */ - public native void RegisterModelRepo(@Const @ByRef NewModelRepo new_model_repo); - - /** Unregister a model repository. This function is not available in polling - * mode. - * @param repo_path The full path to the model repository. */ - public native void UnregisterModelRepo(@StdString BytePointer repo_path); - public native void UnregisterModelRepo(@StdString String repo_path); - - public native @UniquePtr GenericInferResult Infer( - @ByRef GenericInferRequest infer_request); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java deleted file mode 100644 index 6cb1418f4b2..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/HostPolicy.java +++ /dev/null @@ -1,47 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold host policy for setting 'ServerOptions'. - * See here for more information: - * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class HostPolicy extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public HostPolicy(Pointer p) { super(p); } - - /** enum class triton::developer_tools::server::HostPolicy::Setting */ - public static final int NUMA_NODE = 0, CPU_CORES = 1; - - public HostPolicy( - @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, - @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } - private native void allocate( - @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, - @StdString BytePointer value); - public HostPolicy( - @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, - @StdString String value) { super((Pointer)null); allocate(name, setting, value); } - private native void allocate( - @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, - @StdString String value); - - // The name of the policy. - public native @StdString BytePointer name_(); public native HostPolicy name_(BytePointer setter); - // The kind of the host policy setting. Currently supported settings are - // 'NUMA_NODE', 'CPU_CORES'. Note that 'NUMA_NODE' setting will affect pinned - // memory pool behavior, see the comments of 'pinned_memory_pool_byte_size_' - // in 'ServerOptions' for more detail. - public native @Cast("triton::developer_tools::server::HostPolicy::Setting") int setting_(); public native HostPolicy setting_(int setter); - // The setting value. - public native @StdString BytePointer value_(); public native HostPolicy value_(BytePointer setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java deleted file mode 100644 index 8463f677c32..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/InferOptions.java +++ /dev/null @@ -1,109 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold options for Inference Request. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class InferOptions extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public InferOptions(Pointer p) { super(p); } - - public InferOptions(@StdString BytePointer model_name) { super((Pointer)null); allocate(model_name); } - private native void allocate(@StdString BytePointer model_name); - public InferOptions(@StdString String model_name) { super((Pointer)null); allocate(model_name); } - private native void allocate(@StdString String model_name); - - public InferOptions( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version, - @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, - @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, - @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, - @Cast("const uint64_t") long request_timeout, - @SharedPtr Allocator custom_allocator, - @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } - private native void allocate( - @StdString BytePointer model_name, @Cast("const int64_t") long model_version, - @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, - @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, - @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, - @Cast("const uint64_t") long request_timeout, - @SharedPtr Allocator custom_allocator, - @SharedPtr Trace trace); - public InferOptions( - @StdString String model_name, @Cast("const int64_t") long model_version, - @StdString String request_id, @Cast("const uint64_t") long correlation_id, - @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, - @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, - @Cast("const uint64_t") long request_timeout, - @SharedPtr Allocator custom_allocator, - @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } - private native void allocate( - @StdString String model_name, @Cast("const int64_t") long model_version, - @StdString String request_id, @Cast("const uint64_t") long correlation_id, - @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, - @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, - @Cast("const uint64_t") long request_timeout, - @SharedPtr Allocator custom_allocator, - @SharedPtr Trace trace); - - // The name of the model to run inference. - public native @StdString BytePointer model_name_(); public native InferOptions model_name_(BytePointer setter); - // The version of the model to use while running inference. The default - // value is "-1" which means the server will select the - // version of the model based on its internal policy. - public native @Cast("int64_t") long model_version_(); public native InferOptions model_version_(long setter); - // An identifier for the request. If specified will be returned - // in the response. Default value is an empty string which means no - // request_id will be used. - public native @StdString BytePointer request_id_(); public native InferOptions request_id_(BytePointer setter); - // The correlation ID of the inference request to be an unsigned integer. - // Should be used exclusively with 'correlation_id_str_'. - // Default is 0, which indicates that the request has no correlation ID. - public native @Cast("uint64_t") long correlation_id_(); public native InferOptions correlation_id_(long setter); - // The correlation ID of the inference request to be a string. - // Should be used exclusively with 'correlation_id_'. - // Default value is "". - public native @StdString BytePointer correlation_id_str_(); public native InferOptions correlation_id_str_(BytePointer setter); - // Indicates whether the request being added marks the start of the - // sequence. Default value is False. This argument is ignored if - // 'sequence_id' is 0. - public native @Cast("bool") boolean sequence_start_(); public native InferOptions sequence_start_(boolean setter); - // Indicates whether the request being added marks the end of the - // sequence. Default value is False. This argument is ignored if - // 'sequence_id' is 0. - public native @Cast("bool") boolean sequence_end_(); public native InferOptions sequence_end_(boolean setter); - // Indicates the priority of the request. Priority value zero - // indicates that the default priority level should be used - // (i.e. same behavior as not specifying the priority parameter). - // Lower value priorities indicate higher priority levels. Thus - // the highest priority level is indicated by setting the parameter - // to 1, the next highest is 2, etc. If not provided, the server - // will handle the request using default setting for the model. - public native @Cast("uint64_t") long priority_(); public native InferOptions priority_(long setter); - // The timeout value for the request, in microseconds. If the request - // cannot be completed within the time by the server can take a - // model-specific action such as terminating the request. If not - // provided, the server will handle the request using default setting - // for the model. - public native @Cast("uint64_t") long request_timeout_(); public native InferOptions request_timeout_(long setter); - // User-provided custom reponse allocator object. Default is nullptr. - // If using custom allocator, the lifetime of this 'Allocator' object should - // be long enough until `InferResult` object goes out of scope as we need - // this `Allocator` object to call 'ResponseAllocatorReleaseFn_t' for - // releasing the response. - public native @SharedPtr Allocator custom_allocator_(); public native InferOptions custom_allocator_(Allocator setter); - // Update trace setting for the specified model. If not set, will use global - // trace setting in 'ServerOptions' for tracing if tracing is enabled in - // 'ServerOptions'. Default is nullptr. - public native @SharedPtr Trace trace_(); public native InferOptions trace_(Trace setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java deleted file mode 100644 index 828b87092ee..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/LoggingOptions.java +++ /dev/null @@ -1,70 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold logging options for setting 'ServerOptions'. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class LoggingOptions extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public LoggingOptions(Pointer p) { super(p); } - /** Native array allocator. Access with {@link Pointer#position(long)}. */ - public LoggingOptions(long size) { super((Pointer)null); allocateArray(size); } - private native void allocateArray(long size); - @Override public LoggingOptions position(long position) { - return (LoggingOptions)super.position(position); - } - @Override public LoggingOptions getPointer(long i) { - return new LoggingOptions((Pointer)this).offsetAddress(i); - } - - // The range of VerboseLevel is [0, INT_MAX]. - /** enum class triton::developer_tools::server::LoggingOptions::VerboseLevel */ - public static final int OFF = 0, MIN = 1, MAX = Integer.MAX_VALUE; - /** enum class triton::developer_tools::server::LoggingOptions::LogFormat */ - public static final int DEFAULT = 0, ISO8601 = 1; - - public LoggingOptions() { super((Pointer)null); allocate(); } - private native void allocate(); - - public LoggingOptions( - @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, - @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } - private native void allocate( - @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, - @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file); - public LoggingOptions( - @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, - @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } - private native void allocate( - @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, - @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file); - - public native void SetVerboseLevel(int verbose_level); - - // Verbose logging level. Default is OFF. - public native @Cast("triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose_(); public native LoggingOptions verbose_(int setter); - // Enable or disable info logging level. Default is true. - public native @Cast("bool") boolean info_(); public native LoggingOptions info_(boolean setter); - // Enable or disable warn logging level. Default is true. - public native @Cast("bool") boolean warn_(); public native LoggingOptions warn_(boolean setter); - // Enable or disable error logging level. Default is true. - public native @Cast("bool") boolean error_(); public native LoggingOptions error_(boolean setter); - // The format of logging. For "DEFAULT", the log severity (L) and - // timestamp will be logged as "LMMDD hh:mm:ss.ssssss". For "ISO8601", the - // log format will be "YYYY-MM-DDThh:mm:ssZ L". Default is 'DEFAULT'. - public native @Cast("triton::developer_tools::server::LoggingOptions::LogFormat") int format_(); public native LoggingOptions format_(int setter); - // Logging output file. If specified, log outputs will be saved to this file. - // If not specified, log outputs will stream to the console. Default is an - // empty string. - public native @StdString BytePointer log_file_(); public native LoggingOptions log_file_(BytePointer setter); // logging output file -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java deleted file mode 100644 index d4d2643323c..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/MetricsOptions.java +++ /dev/null @@ -1,49 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold metrics options for setting 'ServerOptions'. - * See here for more information: - * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class MetricsOptions extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public MetricsOptions(Pointer p) { super(p); } - /** Native array allocator. Access with {@link Pointer#position(long)}. */ - public MetricsOptions(long size) { super((Pointer)null); allocateArray(size); } - private native void allocateArray(long size); - @Override public MetricsOptions position(long position) { - return (MetricsOptions)super.position(position); - } - @Override public MetricsOptions getPointer(long i) { - return new MetricsOptions((Pointer)this).offsetAddress(i); - } - - public MetricsOptions() { super((Pointer)null); allocate(); } - private native void allocate(); - - public MetricsOptions( - @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, - @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms) { super((Pointer)null); allocate(allow_metrics, allow_gpu_metrics, allow_cpu_metrics, metrics_interval_ms); } - private native void allocate( - @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, - @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms); - - // Enable or disable metrics. Default is true. - public native @Cast("bool") boolean allow_metrics_(); public native MetricsOptions allow_metrics_(boolean setter); - // Enable or disable GPU metrics. Default is true. - public native @Cast("bool") boolean allow_gpu_metrics_(); public native MetricsOptions allow_gpu_metrics_(boolean setter); - // Enable or disable CPU metrics. Default is true. - public native @Cast("bool") boolean allow_cpu_metrics_(); public native MetricsOptions allow_cpu_metrics_(boolean setter); - // The interval for metrics collection. Default is 2000. - public native @Cast("uint64_t") long metrics_interval_ms_(); public native MetricsOptions metrics_interval_ms_(long setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java deleted file mode 100644 index 8a5dcabd2d8..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ModelLoadGPULimit.java +++ /dev/null @@ -1,30 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold GPU limit of model loading for setting 'ServerOptions'. - * The limit on GPU memory usage is specified as a fraction. If model loading - * on the device is requested and the current memory usage exceeds the limit, - * the load will be rejected. If not specified, the limit will not be set. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class ModelLoadGPULimit extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public ModelLoadGPULimit(Pointer p) { super(p); } - - public ModelLoadGPULimit(int device_id, double fraction) { super((Pointer)null); allocate(device_id, fraction); } - private native void allocate(int device_id, double fraction); - - // The GPU device ID. - public native int device_id_(); public native ModelLoadGPULimit device_id_(int setter); - // The limit on memory usage as a fraction. - public native double fraction_(); public native ModelLoadGPULimit fraction_(double setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java deleted file mode 100644 index 1b1b1a5b8b2..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/NewModelRepo.java +++ /dev/null @@ -1,50 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold the full path to the model repository to be registered and - * the mapping from the original model name to the overriden one. This object - * is used for calling 'TritonServer::RegisterModelRepo' for registering - * model repository. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class NewModelRepo extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public NewModelRepo(Pointer p) { super(p); } - - public NewModelRepo(@StdString BytePointer path) { super((Pointer)null); allocate(path); } - private native void allocate(@StdString BytePointer path); - public NewModelRepo(@StdString String path) { super((Pointer)null); allocate(path); } - private native void allocate(@StdString String path); - - public NewModelRepo( - @StdString BytePointer path, @StdString BytePointer original_name, - @StdString BytePointer override_name) { super((Pointer)null); allocate(path, original_name, override_name); } - private native void allocate( - @StdString BytePointer path, @StdString BytePointer original_name, - @StdString BytePointer override_name); - public NewModelRepo( - @StdString String path, @StdString String original_name, - @StdString String override_name) { super((Pointer)null); allocate(path, original_name, override_name); } - private native void allocate( - @StdString String path, @StdString String original_name, - @StdString String override_name); - - // The full path to the model repository. - public native @StdString BytePointer path_(); public native NewModelRepo path_(BytePointer setter); - // The original name of the model. This field is optional when there is no - // name mapping needed. - public native @StdString BytePointer original_name_(); public native NewModelRepo original_name_(BytePointer setter); - // The original name of the model. This field is optional when there is no - // name mapping needed. - public native @StdString BytePointer override_name_(); public native NewModelRepo override_name_(BytePointer setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java deleted file mode 100644 index 0981c12bf1d..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java +++ /dev/null @@ -1,21 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class OutputBufferReleaseFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public OutputBufferReleaseFn_t(Pointer p) { super(p); } - protected OutputBufferReleaseFn_t() { allocate(); } - private native void allocate(); - public native void call( - Pointer buffer, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java deleted file mode 100644 index 1913b0beab2..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RateLimitResource.java +++ /dev/null @@ -1,43 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold rate limit resource for setting 'ServerOptions'. See here - * for more information: - * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class RateLimitResource extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public RateLimitResource(Pointer p) { super(p); } - - public RateLimitResource(@StdString BytePointer name, int count) { super((Pointer)null); allocate(name, count); } - private native void allocate(@StdString BytePointer name, int count); - public RateLimitResource(@StdString String name, int count) { super((Pointer)null); allocate(name, count); } - private native void allocate(@StdString String name, int count); - - public RateLimitResource(@StdString BytePointer name, int count, int device) { super((Pointer)null); allocate(name, count, device); } - private native void allocate(@StdString BytePointer name, int count, int device); - public RateLimitResource(@StdString String name, int count, int device) { super((Pointer)null); allocate(name, count, device); } - private native void allocate(@StdString String name, int count, int device); - - // The name of the resource. - public native @StdString BytePointer name_(); public native RateLimitResource name_(BytePointer setter); - // The count of the resource. - public native int count_(); public native RateLimitResource count_(int setter); - // The device identifier for the resource. This field is optional and if not - // specified will be applied to every device. The device value is ignored for - // a global resource. The server will use the rate limiter configuration - // specified for instance groups in model config to determine whether resource - // is global. In case of conflicting resource type in different model - // configurations, server will raise an appropriate error while loading model. - public native int device_(); public native RateLimitResource device_(int setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java deleted file mode 100644 index c35ad10f30f..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/RepositoryIndex.java +++ /dev/null @@ -1,50 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold repository index for 'ModelIndex' function. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class RepositoryIndex extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public RepositoryIndex(Pointer p) { super(p); } - - public RepositoryIndex( - @StdString BytePointer name, @StdString BytePointer version, - @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } - private native void allocate( - @StdString BytePointer name, @StdString BytePointer version, - @Cast("const triton::developer_tools::server::ModelReadyState") int state); - public RepositoryIndex( - @StdString String name, @StdString String version, - @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } - private native void allocate( - @StdString String name, @StdString String version, - @Cast("const triton::developer_tools::server::ModelReadyState") int state); - - // The name of the model. - public native @StdString BytePointer name_(); public native RepositoryIndex name_(BytePointer setter); - // The version of the model. - public native @StdString BytePointer version_(); public native RepositoryIndex version_(BytePointer setter); - // The state of the model. The states are - // * UNKNOWN: The model is in an unknown state. The model is not available for - // inferencing. - // * READY: The model is ready and available for inferencing. - // * UNAVAILABLE: The model is unavailable, indicating that the model failed - // to load or has been implicitly or explicitly unloaded. The model is not - // available for inferencing. - // * LOADING: The model is being loaded by the inference server. The model is - // not available for inferencing. - // * UNLOADING: The model is being unloaded by the inference server. The model - // is not available for inferencing. - public native @Cast("triton::developer_tools::server::ModelReadyState") int state_(); public native RepositoryIndex state_(int setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java deleted file mode 100644 index dff5f68ac31..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java +++ /dev/null @@ -1,26 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Custom Response Allocator Callback function signatures. - * */ -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class ResponseAllocatorAllocFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public ResponseAllocatorAllocFn_t(Pointer p) { super(p); } - protected ResponseAllocatorAllocFn_t() { allocate(); } - private native void allocate(); - public native void call( - String tensor_name, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int preferred_memory_type, - @Cast("int64_t") long preferred_memory_type_id, @Cast("void**") PointerPointer buffer, - @Cast("triton::developer_tools::server::MemoryType*") IntPointer actual_memory_type, @Cast("int64_t*") LongPointer actual_memory_type_id); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java deleted file mode 100644 index 9a857db06db..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java +++ /dev/null @@ -1,21 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class ResponseAllocatorStartFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public ResponseAllocatorStartFn_t(Pointer p) { super(p); } - protected ResponseAllocatorStartFn_t() { allocate(); } - private native void allocate(); - public native void call(Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java deleted file mode 100644 index 34d257b286d..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/ServerOptions.java +++ /dev/null @@ -1,191 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Server options that are used to initialize Triton Server. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class ServerOptions extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public ServerOptions(Pointer p) { super(p); } - - public ServerOptions(@Const @ByRef StringVector model_repository_paths) { super((Pointer)null); allocate(model_repository_paths); } - private native void allocate(@Const @ByRef StringVector model_repository_paths); - - public ServerOptions( - @Const @ByRef StringVector model_repository_paths, - @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, - @StdVector BackendConfig be_config, @StdString BytePointer server_id, - @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, - @Cast("const bool") boolean disable_auto_complete_config, - @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, - int repository_poll_secs, - @Const @ByRef StringSet startup_models, - @StdVector RateLimitResource rate_limit_resource, - @Cast("const int64_t") long pinned_memory_pool_byte_size, - @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, - @Cast("const uint64_t") long response_cache_byte_size, - double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, - int exit_timeout_secs, - int buffer_manager_thread_count, - @Cast("const uint32_t") int model_load_thread_count, - @StdVector ModelLoadGPULimit model_load_gpu_limit, - @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } - private native void allocate( - @Const @ByRef StringVector model_repository_paths, - @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, - @StdVector BackendConfig be_config, @StdString BytePointer server_id, - @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, - @Cast("const bool") boolean disable_auto_complete_config, - @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, - int repository_poll_secs, - @Const @ByRef StringSet startup_models, - @StdVector RateLimitResource rate_limit_resource, - @Cast("const int64_t") long pinned_memory_pool_byte_size, - @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, - @Cast("const uint64_t") long response_cache_byte_size, - double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, - int exit_timeout_secs, - int buffer_manager_thread_count, - @Cast("const uint32_t") int model_load_thread_count, - @StdVector ModelLoadGPULimit model_load_gpu_limit, - @StdVector HostPolicy host_policy, @SharedPtr Trace trace); - public ServerOptions( - @Const @ByRef StringVector model_repository_paths, - @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, - @StdVector BackendConfig be_config, @StdString String server_id, - @StdString String backend_dir, @StdString String repo_agent_dir, - @Cast("const bool") boolean disable_auto_complete_config, - @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, - int repository_poll_secs, - @Const @ByRef StringSet startup_models, - @StdVector RateLimitResource rate_limit_resource, - @Cast("const int64_t") long pinned_memory_pool_byte_size, - @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, - @Cast("const uint64_t") long response_cache_byte_size, - double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, - int exit_timeout_secs, - int buffer_manager_thread_count, - @Cast("const uint32_t") int model_load_thread_count, - @StdVector ModelLoadGPULimit model_load_gpu_limit, - @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } - private native void allocate( - @Const @ByRef StringVector model_repository_paths, - @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, - @StdVector BackendConfig be_config, @StdString String server_id, - @StdString String backend_dir, @StdString String repo_agent_dir, - @Cast("const bool") boolean disable_auto_complete_config, - @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, - int repository_poll_secs, - @Const @ByRef StringSet startup_models, - @StdVector RateLimitResource rate_limit_resource, - @Cast("const int64_t") long pinned_memory_pool_byte_size, - @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, - @Cast("const uint64_t") long response_cache_byte_size, - double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, - int exit_timeout_secs, - int buffer_manager_thread_count, - @Cast("const uint32_t") int model_load_thread_count, - @StdVector ModelLoadGPULimit model_load_gpu_limit, - @StdVector HostPolicy host_policy, @SharedPtr Trace trace); - - public native void SetLoggingOptions(@Const @ByRef LoggingOptions logging); - - // Paths to model repository directory. Note that if a model is not unique - // across all model repositories at any time, the model will not be available. - // See here for more information: - // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md. - public native @ByRef StringVector model_repository_paths_(); public native ServerOptions model_repository_paths_(StringVector setter); - // Logging options. See the 'LoggingOptions' structure for more information. - public native @ByRef LoggingOptions logging_(); public native ServerOptions logging_(LoggingOptions setter); - // Metrics options. See the 'MetricsOptions' structure for more information. - public native @ByRef MetricsOptions metrics_(); public native ServerOptions metrics_(MetricsOptions setter); - // Backend configuration. See the 'BackendConfig' structure for more - // information. - public native @StdVector BackendConfig be_config_(); public native ServerOptions be_config_(BackendConfig setter); - // The ID of the server. - public native @StdString BytePointer server_id_(); public native ServerOptions server_id_(BytePointer setter); - // The global directory searched for backend shared libraries. Default is - // "/opt/tritonserver/backends". See here for more information: - // https://github.com/triton-inference-server/backend#backends. - public native @StdString BytePointer backend_dir_(); public native ServerOptions backend_dir_(BytePointer setter); - // The global directory searched for repository agent shared libraries. - // Default is "/opt/tritonserver/repoagents". See here for more information: - // https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/repository_agents.md. - public native @StdString BytePointer repo_agent_dir_(); public native ServerOptions repo_agent_dir_(BytePointer setter); - // If set, disables the triton and backends from auto completing model - // configuration files. Model configuration files must be provided and - // all required configuration settings must be specified. Default is false. - // See here for more information: - // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration. - public native @Cast("bool") boolean disable_auto_complete_config_(); public native ServerOptions disable_auto_complete_config_(boolean setter); - // Specify the mode for model management. Options are "NONE", "POLL" and - // "EXPLICIT". Default is "NONE". See here for more information: - // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md. - public native @Cast("triton::developer_tools::server::ModelControlMode") int model_control_mode_(); public native ServerOptions model_control_mode_(int setter); - // Interval in seconds between each poll of the model repository to check for - // changes. Valid only when 'model_control_mode_' is set to "POLL". Default - // is 15. - public native int repository_poll_secs_(); public native ServerOptions repository_poll_secs_(int setter); - // Specify the the models to be loaded on server startup. This will only take - // effect if 'model_control_mode_' is set to 'EXPLICIT'. - public native @ByRef StringSet startup_models_(); public native ServerOptions startup_models_(StringSet setter); - // The number of resources available to the server. Rate limiting is disabled - // by default, and can be enabled once 'rate_limit_resource_' is set. See the - // 'RateLimitResource' structure for more information. - public native @StdVector RateLimitResource rate_limit_resource_(); public native ServerOptions rate_limit_resource_(RateLimitResource setter); - // The total byte size that can be allocated as pinned system memory. If GPU - // support is enabled, the server will allocate pinned system memory to - // accelerate data transfer between host and devices until it exceeds the - // specified byte size. If 'NUMA_NODE' is configured via 'host_policy_', the - // pinned system memory of the pool size will be allocated on each numa node. - // This option will not affect the allocation conducted by the backend - // frameworks. Default is 256 MB. - public native @Cast("int64_t") long pinned_memory_pool_byte_size_(); public native ServerOptions pinned_memory_pool_byte_size_(long setter); - // The total byte size that can be allocated as CUDA memory for the GPU - // device. See the 'CUDAMemoryPoolByteSize' structure for more information. - public native @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size_(); public native ServerOptions cuda_memory_pool_byte_size_(CUDAMemoryPoolByteSize setter); - // The size in bytes to allocate for a request/response cache. When non-zero, - // Triton allocates the requested size in CPU memory and shares the cache - // across all inference requests and across all models. For a given model to - // use request caching, the model must enable request caching in the model - // configuration. See here for more information: - // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#response-cache. - // By default, no model uses request caching even if the - // 'response_cache_byte_size_' is set. Default is 0. - public native @Cast("uint64_t") long response_cache_byte_size_(); public native ServerOptions response_cache_byte_size_(long setter); - // The minimum supported CUDA compute capability. GPUs that don't support this - // compute capability will not be used by the server. Default is 0. - public native double min_cuda_compute_capability_(); public native ServerOptions min_cuda_compute_capability_(double setter); - // If set, exit the inference server when an error occurs during - // initialization. Default is true. - public native @Cast("bool") boolean exit_on_error_(); public native ServerOptions exit_on_error_(boolean setter); - // Timeout (in seconds) when exiting to wait for in-flight inferences to - // finish. After the timeout expires the server exits even if inferences are - // still in flight. Default is 30 secs. - public native int exit_timeout_secs_(); public native ServerOptions exit_timeout_secs_(int setter); - // The number of threads used to accelerate copies and other operations - // required to manage input and output tensor contents. Default is 0. - public native int buffer_manager_thread_count_(); public native ServerOptions buffer_manager_thread_count_(int setter); - // The number of threads used to concurrently load models in model - // repositories. Default is 2*. - public native @Cast("uint32_t") int model_load_thread_count_(); public native ServerOptions model_load_thread_count_(int setter); - // The GPU limit of model loading. See the 'ModelLoadGPULimit' structure for - // more information. - public native @StdVector ModelLoadGPULimit model_load_gpu_limit_(); public native ServerOptions model_load_gpu_limit_(ModelLoadGPULimit setter); - // The host policy setting. See the 'HostPolicy' structure for more - // information. - public native @StdVector HostPolicy host_policy_(); public native ServerOptions host_policy_(HostPolicy setter); - // The global trace setting. Default is nullptr, meaning that tracing is not - // enabled. See the 'Trace' structure for more information. - public native @SharedPtr Trace trace_(); public native ServerOptions trace_(Trace setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java deleted file mode 100644 index d01cc4cdd17..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringSet.java +++ /dev/null @@ -1,36 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Name("std::set") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class StringSet extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public StringSet(Pointer p) { super(p); } - public StringSet() { allocate(); } - private native void allocate(); - public native @Name("operator =") @ByRef StringSet put(@ByRef StringSet x); - - public boolean empty() { return size() == 0; } - public native long size(); - - public native void insert(@StdString BytePointer value); - public native void erase(@StdString BytePointer value); - public native @ByVal Iterator begin(); - public native @ByVal Iterator end(); - @NoOffset @Name("iterator") public static class Iterator extends Pointer { - public Iterator(Pointer p) { super(p); } - public Iterator() { } - - public native @Name("operator ++") @ByRef Iterator increment(); - public native @Name("operator ==") boolean equals(@ByRef Iterator it); - public native @Name("operator *") @StdString BytePointer get(); - } -} - diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java deleted file mode 100644 index 242088d67dd..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/StringVector.java +++ /dev/null @@ -1,99 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Name("std::vector") @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class StringVector extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public StringVector(Pointer p) { super(p); } - public StringVector(BytePointer value) { this(1); put(0, value); } - public StringVector(BytePointer ... array) { this(array.length); put(array); } - public StringVector(String value) { this(1); put(0, value); } - public StringVector(String ... array) { this(array.length); put(array); } - public StringVector() { allocate(); } - public StringVector(long n) { allocate(n); } - private native void allocate(); - private native void allocate(@Cast("size_t") long n); - public native @Name("operator =") @ByRef StringVector put(@ByRef StringVector x); - - public boolean empty() { return size() == 0; } - public native long size(); - public void clear() { resize(0); } - public native void resize(@Cast("size_t") long n); - - @Index(function = "at") public native @StdString BytePointer get(@Cast("size_t") long i); - public native StringVector put(@Cast("size_t") long i, BytePointer value); - @ValueSetter @Index(function = "at") public native StringVector put(@Cast("size_t") long i, @StdString String value); - - public native @ByVal Iterator insert(@ByVal Iterator pos, @StdString BytePointer value); - public native @ByVal Iterator erase(@ByVal Iterator pos); - public native @ByVal Iterator begin(); - public native @ByVal Iterator end(); - @NoOffset @Name("iterator") public static class Iterator extends Pointer { - public Iterator(Pointer p) { super(p); } - public Iterator() { } - - public native @Name("operator ++") @ByRef Iterator increment(); - public native @Name("operator ==") boolean equals(@ByRef Iterator it); - public native @Name("operator *") @StdString BytePointer get(); - } - - public BytePointer[] get() { - BytePointer[] array = new BytePointer[size() < Integer.MAX_VALUE ? (int)size() : Integer.MAX_VALUE]; - for (int i = 0; i < array.length; i++) { - array[i] = get(i); - } - return array; - } - @Override public String toString() { - return java.util.Arrays.toString(get()); - } - - public BytePointer pop_back() { - long size = size(); - BytePointer value = get(size - 1); - resize(size - 1); - return value; - } - public StringVector push_back(BytePointer value) { - long size = size(); - resize(size + 1); - return put(size, value); - } - public StringVector put(BytePointer value) { - if (size() != 1) { resize(1); } - return put(0, value); - } - public StringVector put(BytePointer ... array) { - if (size() != array.length) { resize(array.length); } - for (int i = 0; i < array.length; i++) { - put(i, array[i]); - } - return this; - } - - public StringVector push_back(String value) { - long size = size(); - resize(size + 1); - return put(size, value); - } - public StringVector put(String value) { - if (size() != 1) { resize(1); } - return put(0, value); - } - public StringVector put(String ... array) { - if (size() != array.length) { resize(array.length); } - for (int i = 0; i < array.length; i++) { - put(i, array[i]); - } - return this; - } -} - diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java deleted file mode 100644 index 574cfc7ef4f..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Backend.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Backend extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Backend() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Backend(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java deleted file mode 100644 index 2da20895083..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_BackendAttribute.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_BackendAttribute extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_BackendAttribute() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_BackendAttribute(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java deleted file mode 100644 index 3e627e2f39c..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Batcher.java +++ /dev/null @@ -1,23 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - - -/// -/// -/// -/// -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Batcher extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Batcher() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Batcher(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java deleted file mode 100644 index 73812b97187..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Input.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Input extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Input() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Input(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java deleted file mode 100644 index 1cf35b87293..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_MemoryManager.java +++ /dev/null @@ -1,20 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -// #endif -// #endif - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_MemoryManager extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_MemoryManager() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_MemoryManager(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java deleted file mode 100644 index 783af23deba..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Model.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Model extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Model() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Model(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java deleted file mode 100644 index 2319222a839..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ModelInstance.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_ModelInstance extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_ModelInstance() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_ModelInstance(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java deleted file mode 100644 index 2645a2d3a82..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Output.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Output extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Output() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Output(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java deleted file mode 100644 index 1e7e5af8b3f..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Request.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Request extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Request() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Request(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java deleted file mode 100644 index a60df8359c4..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_Response.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_Response extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_Response() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_Response(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java deleted file mode 100644 index 14ef5ac2cbd..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_ResponseFactory.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_ResponseFactory extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_ResponseFactory() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_ResponseFactory(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java deleted file mode 100644 index ad77aa48b96..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONBACKEND_State.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONBACKEND_State extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONBACKEND_State() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONBACKEND_State(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java deleted file mode 100644 index 8379f8e4dbc..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_Agent.java +++ /dev/null @@ -1,20 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -// #endif -// #endif - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONREPOAGENT_Agent extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONREPOAGENT_Agent() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONREPOAGENT_Agent(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java deleted file mode 100644 index 74d8a32d3ab..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONREPOAGENT_AgentModel.java +++ /dev/null @@ -1,23 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - - -/// -/// -/// -/// -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONREPOAGENT_AgentModel extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONREPOAGENT_AgentModel() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONREPOAGENT_AgentModel(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java deleted file mode 100644 index 427fb6aaf21..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_BufferAttributes.java +++ /dev/null @@ -1,20 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -// #endif -// #endif - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_BufferAttributes extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_BufferAttributes() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_BufferAttributes(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java deleted file mode 100644 index 0278510d8bd..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Error.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Error extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Error() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Error(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java deleted file mode 100644 index 8b80120a13f..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceRequest extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_InferenceRequest() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceRequest(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java deleted file mode 100644 index 3d5b2484589..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceRequestReleaseFn_t.java +++ /dev/null @@ -1,46 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for inference request release callback function. The callback - * indicates what type of release is being performed on the request - * and for some of these the callback function takes ownership of the - * TRITONSERVER_InferenceRequest object. The 'userp' data is the data - * provided as 'request_release_userp' in the call to - * TRITONSERVER_InferenceRequestSetReleaseCallback. - * - * One or more flags will be specified when the callback is invoked, - * and the callback must take the following actions: - * - * - TRITONSERVER_REQUEST_RELEASE_ALL: The entire inference request - * is being released and ownership is passed to the callback - * function. Triton will not longer access the 'request' object - * itself nor any input tensor data associated with the - * request. The callback should free or otherwise manage the - * 'request' object and all associated tensor data. - * - * Note that currently TRITONSERVER_REQUEST_RELEASE_ALL should always - * be set when the callback is invoked but in the future that may - * change, so the callback should explicitly check for the flag - * before taking ownership of the request object. - * */ - -/// -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceRequestReleaseFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceRequestReleaseFn_t(Pointer p) { super(p); } - protected TRITONSERVER_InferenceRequestReleaseFn_t() { allocate(); } - private native void allocate(); - public native void call( - TRITONSERVER_InferenceRequest request, @Cast("const uint32_t") int flags, Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java deleted file mode 100644 index 3f3e0c4f65e..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponse.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceResponse extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_InferenceResponse() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceResponse(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java deleted file mode 100644 index caada929b0c..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceResponseCompleteFn_t.java +++ /dev/null @@ -1,41 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for callback function indicating that an inference response - * has completed. The callback function takes ownership of the - * TRITONSERVER_InferenceResponse object. The 'userp' data is the - * data provided as 'response_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * - * One or more flags may be specified when the callback is invoked: - * - * - TRITONSERVER_RESPONSE_COMPLETE_FINAL: Indicates that no more - * responses will be generated for a given request (more - * specifically, that no more responses will be generated for the - * inference request that set this callback and 'userp'). When - * this flag is set 'response' may be a response object or may be - * nullptr. If 'response' is not nullptr, then 'response' is the - * last response that Triton will produce for the request. If - * 'response' is nullptr then Triton is indicating that no more - * responses will be produced for the request. */ - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceResponseCompleteFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceResponseCompleteFn_t(Pointer p) { super(p); } - protected TRITONSERVER_InferenceResponseCompleteFn_t() { allocate(); } - private native void allocate(); - public native void call( - TRITONSERVER_InferenceResponse response, @Cast("const uint32_t") int flags, - Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java deleted file mode 100644 index 307baec38c8..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTrace.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceTrace extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_InferenceTrace() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceTrace(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java deleted file mode 100644 index 4190dbe2f65..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceActivityFn_t.java +++ /dev/null @@ -1,29 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for trace timeline activity callback function. This callback function - * is used to report activity occurring for a trace. This function - * does not take ownership of 'trace' and so any information needed - * from that object must be copied before returning. The 'userp' data - * is the same as what is supplied in the call to - * TRITONSERVER_InferenceTraceNew. */ -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceTraceActivityFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceTraceActivityFn_t(Pointer p) { super(p); } - protected TRITONSERVER_InferenceTraceActivityFn_t() { allocate(); } - private native void allocate(); - public native void call( - TRITONSERVER_InferenceTrace trace, - @Cast("TRITONSERVER_InferenceTraceActivity") int activity, @Cast("uint64_t") long timestamp_ns, - Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java deleted file mode 100644 index 29df3a4476d..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceReleaseFn_t.java +++ /dev/null @@ -1,30 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for trace release callback function. This callback function - * is called when all activity for the trace has completed. The - * callback function takes ownership of the - * TRITONSERVER_InferenceTrace object. The 'userp' data is the same - * as what is supplied in the call to TRITONSERVER_InferenceTraceNew. */ - -/// -/// -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceTraceReleaseFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceTraceReleaseFn_t(Pointer p) { super(p); } - protected TRITONSERVER_InferenceTraceReleaseFn_t() { allocate(); } - private native void allocate(); - public native void call( - TRITONSERVER_InferenceTrace trace, Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java deleted file mode 100644 index 913d36cd6d0..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java +++ /dev/null @@ -1,31 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for trace tensor activity callback function. This callback function - * is used to report tensor activity occurring for a trace. This function - * does not take ownership of 'trace' and so any information needed - * from that object must be copied before returning. The 'userp' data - * is the same as what is supplied in the call to - * TRITONSERVER_InferenceTraceTensorNew. */ -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_InferenceTraceTensorActivityFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_InferenceTraceTensorActivityFn_t(Pointer p) { super(p); } - protected TRITONSERVER_InferenceTraceTensorActivityFn_t() { allocate(); } - private native void allocate(); - public native void call( - TRITONSERVER_InferenceTrace trace, - @Cast("TRITONSERVER_InferenceTraceActivity") int activity, String name, - @Cast("TRITONSERVER_DataType") int datatype, @Const Pointer base, @Cast("size_t") long byte_size, - @Cast("const int64_t*") LongPointer shape, @Cast("uint64_t") long dim_count, - @Cast("TRITONSERVER_MemoryType") int memory_type, @Cast("int64_t") long memory_type_id, Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java deleted file mode 100644 index cc2537c5153..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Message.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Message extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Message() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Message(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java deleted file mode 100644 index 242a48e645e..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metric.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Metric extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Metric() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Metric(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java deleted file mode 100644 index f8aa886086a..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_MetricFamily.java +++ /dev/null @@ -1,23 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - - -/// -/// -/// -/// -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_MetricFamily extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_MetricFamily() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_MetricFamily(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java deleted file mode 100644 index 8c7f8157d19..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Metrics.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Metrics extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Metrics() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Metrics(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java deleted file mode 100644 index 52ddd9ee2f5..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Parameter.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Parameter extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Parameter() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Parameter(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java deleted file mode 100644 index d58b84f1534..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocator.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocator extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_ResponseAllocator() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocator(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java deleted file mode 100644 index 99ea92db9d7..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java +++ /dev/null @@ -1,63 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** TRITONSERVER_ResponseAllocator - * - * Object representing a memory allocator for output tensors in an - * inference response. - * -

- * Type for allocation function that allocates a buffer to hold an - * output tensor. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param byte_size The size of the buffer to allocate. - * @param memory_type The type of memory that the caller prefers for - * the buffer allocation. - * @param memory_type_id The ID of the memory that the caller prefers - * for the buffer allocation. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Returns a pointer to the allocated memory. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @param actual_memory_type Returns the type of memory where the - * allocation resides. May be different than the type of memory - * requested by 'memory_type'. - * @param actual_memory_type_id Returns the ID of the memory where - * the allocation resides. May be different than the ID of the memory - * requested by 'memory_type_id'. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocatorAllocFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorAllocFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorAllocFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id, Pointer userp, @Cast("void**") PointerPointer buffer, @Cast("void**") PointerPointer buffer_userp, - @Cast("TRITONSERVER_MemoryType*") IntPointer actual_memory_type, - @Cast("int64_t*") LongPointer actual_memory_type_id); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java deleted file mode 100644 index 123a31eb2a2..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java +++ /dev/null @@ -1,47 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for allocation function that allocates a buffer to hold an - * output tensor with buffer attributes. The callback function must fill in the - * appropriate buffer attributes information related to this buffer. If set, - * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t - * function. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param buffer_attributes The buffer attributes associated with the buffer. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocatorBufferAttributesFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorBufferAttributesFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorBufferAttributesFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - TRITONSERVER_BufferAttributes buffer_attributes, Pointer userp, - Pointer buffer_userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java deleted file mode 100644 index 8b2af5b67aa..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java +++ /dev/null @@ -1,49 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for function that is called to query the allocator's preferred memory - * type and memory type ID. As much as possible, the allocator should attempt - * to return the same memory_type and memory_type_id values that will be - * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. - * But the allocator is not required to do so. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor. This is optional - * and it should be set to nullptr to indicate that the tensor name has - * not determined. - * @param byte_size The expected size of the buffer. This is optional - * and it should be set to nullptr to indicate that the byte size has - * not determined. - * @param memory_type Acts as both input and output. On input gives - * the memory type preferred by the caller. Returns memory type preferred - * by the allocator, taken account of the caller preferred type. - * @param memory_type_id Acts as both input and output. On input gives - * the memory type ID preferred by the caller. Returns memory type ID preferred - * by the allocator, taken account of the caller preferred type ID. - * @return a TRITONSERVER_Error object if a failure occurs. */ - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocatorQueryFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorQueryFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorQueryFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer userp, - String tensor_name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java deleted file mode 100644 index bd07e12eddc..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java +++ /dev/null @@ -1,42 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for function that is called when the server no longer holds - * any reference to a buffer allocated by - * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function - * is typically called when the response object associated with the - * buffer is deleted by TRITONSERVER_InferenceResponseDelete. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Pointer to the buffer to be freed. - * @param buffer_userp The user-specified value associated - * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. - * @param byte_size The size of the buffer. - * @param memory_type The type of memory holding the buffer. - * @param memory_type_id The ID of the memory holding the buffer. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting the release. If an error is returned Triton will not - * attempt to release the buffer again. */ - -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocatorReleaseFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorReleaseFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorReleaseFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, - @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java deleted file mode 100644 index c4f4ff05455..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ResponseAllocatorStartFn_t.java +++ /dev/null @@ -1,37 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -/** Type for function that is called to indicate that subsequent - * allocation requests will refer to a new response. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @return a TRITONSERVER_Error object if a failure occurs. */ - -/// -/// -/// -/// -/// -/// -@Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ResponseAllocatorStartFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorStartFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorStartFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer userp); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java deleted file mode 100644 index 8c2b4c140fd..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_Server.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_Server extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_Server() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_Server(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java deleted file mode 100644 index 2ddd5858995..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TRITONSERVER_ServerOptions.java +++ /dev/null @@ -1,17 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - -@Opaque @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TRITONSERVER_ServerOptions extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TRITONSERVER_ServerOptions() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ServerOptions(Pointer p) { super(p); } -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java deleted file mode 100644 index 1fbd27c8812..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Tensor.java +++ /dev/null @@ -1,81 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold information of a tensor. This object is used for adding - * input/requested output to an inference request, and retrieving the output - * result from inference result. - * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class Tensor extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public Tensor(Pointer p) { super(p); } - - public Tensor( - @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - public Tensor( - @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - public Tensor( - @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, - @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - - public Tensor( - @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - public Tensor( - @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - public Tensor( - @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } - private native void allocate( - @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, - @Cast("const int64_t") long memory_type_id); - - // The pointer to the start of the buffer. - public native @Cast("char*") BytePointer buffer_(); public native Tensor buffer_(BytePointer setter); - // The size of buffer in bytes. - public native @Cast("size_t") long byte_size_(); public native Tensor byte_size_(long setter); - // The data type of the tensor. - public native @Cast("triton::developer_tools::server::DataType") int data_type_(); public native Tensor data_type_(int setter); - // The shape of the tensor. - public native @Cast("int64_t*") @StdVector LongPointer shape_(); public native Tensor shape_(LongPointer setter); - // The memory type of the tensor. Valid memory types are "CPU", "CPU_PINNED" - // and "GPU". - public native @Cast("triton::developer_tools::server::MemoryType") int memory_type_(); public native Tensor memory_type_(int setter); - // The ID of the memory for the tensor. (e.g. '0' is the memory type id of - // 'GPU-0') - public native @Cast("int64_t") long memory_type_id_(); public native Tensor memory_type_id_(long setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java deleted file mode 100644 index b532c293c78..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/Trace.java +++ /dev/null @@ -1,66 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -/** Structure to hold global trace setting for 'ServerOptions' and - * model-specific trace setting for 'InferOptions'. See here for more - * information: - * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class Trace extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public Trace(Pointer p) { super(p); } - - /** enum class triton::developer_tools::server::Trace::Level */ - public static final int OFF = 0, TIMESTAMPS = 1, TENSORS = 2; - - public Trace(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } - private native void allocate(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level); - public Trace(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } - private native void allocate(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level); - - public Trace( - @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, - int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } - private native void allocate( - @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, - int count, @Cast("const uint32_t") int log_frequency); - public Trace( - @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, - int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } - private native void allocate( - @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, - int count, @Cast("const uint32_t") int log_frequency); - - // The file where trace output will be saved. If 'log-frequency' is also - // specified, this argument value will be the prefix of the files to save the - // trace output. - public native @StdString BytePointer file_(); public native Trace file_(BytePointer setter); - // Specify a trace level. OFF to disable tracing, TIMESTAMPS to trace - // timestamps, TENSORS to trace tensors. - public native @Cast("triton::developer_tools::server::Trace::Level") int level_(); public native Trace level_(int setter); - // The trace sampling rate. The value represents how many requests will one - // trace be sampled from. For example, if the trace rate is "1000", 1 trace - // will be sampled for every 1000 requests. Default is 1000. - public native @Cast("uint32_t") int rate_(); public native Trace rate_(int setter); - // The number of traces to be sampled. If the value is -1, the number of - // traces to be sampled will not be limited. Default is -1. - public native int count_(); public native Trace count_(int setter); - // The trace log frequency. If the value is 0, Triton will only log the trace - // output to 'file_' when shutting down. Otherwise, Triton will log the trace - // output to 'file_.' when it collects the specified number of traces. - // For example, if the log frequency is 100, when Triton collects the 100-th - // trace, it logs the traces to file 'file_.0', and when it collects the - // 200-th trace, it logs the 101-th to the 200-th traces to file file_.1'. - // Default is 0. - public native @Cast("uint32_t") int log_frequency_(); public native Trace log_frequency_(int setter); -} diff --git a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java b/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java deleted file mode 100644 index b147ef88e9f..00000000000 --- a/tritondevelopertoolsserver/src/gen/java/org/bytedeco/tritondevelopertoolsserver/tritondevelopertoolsserver/TritonException.java +++ /dev/null @@ -1,29 +0,0 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; - - -//============================================================================== -// TritonException -// -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritondevelopertoolsserver.presets.tritondevelopertoolsserver.class) -public class TritonException extends Pointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TritonException(Pointer p) { super(p); } - - public TritonException(@StdString BytePointer message) { super((Pointer)null); allocate(message); } - private native void allocate(@StdString BytePointer message); - public TritonException(@StdString String message) { super((Pointer)null); allocate(message); } - private native void allocate(@StdString String message); - - public native String what(); - - public native @StdString BytePointer message_(); public native TritonException message_(BytePointer setter); -} diff --git a/tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java b/tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java deleted file mode 100644 index f30b7d91d3f..00000000000 --- a/tritondevelopertoolsserver/src/main/java/org/bytedeco/tritondevelopertoolsserver/presets/tritondevelopertoolsserver.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2023 - * - * Licensed either under the Apache License, Version 2.0, or (at your option) - * under the terms of the GNU General Public License as published by - * the Free Software Foundation (subject to the "Classpath" exception), - * either version 2, or any later version (collectively, the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * http://www.gnu.org/licenses/ - * http://www.gnu.org/software/classpath/license.html - * - * or as provided in the LICENSE.txt file that accompanied this code. - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.bytedeco.tritondevelopertoolsserver.presets; - -import java.util.List; -import org.bytedeco.javacpp.ClassProperties; -import org.bytedeco.javacpp.LoadEnabled; -import org.bytedeco.javacpp.Loader; -import org.bytedeco.javacpp.annotation.Platform; -import org.bytedeco.javacpp.annotation.Properties; -import org.bytedeco.javacpp.tools.Info; -import org.bytedeco.javacpp.tools.InfoMap; -import org.bytedeco.javacpp.tools.InfoMapper; - -/** - * - * @author Katherine Yang - */ -@Properties( - value = { - @Platform( - value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"common.h", "generic_server_wrapper.h", "tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, - link = {"tritondevelopertoolsserver", "tritonserver"}, - includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, - linkpath = {"/opt/tritonserver/lib/"} - ), - }, - target = "org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver", - global = "org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver" -) -public class tritondevelopertoolsserver implements InfoMapper { - static { Loader.checkVersion("org.bytedeco", "tritondevelopertoolsserver"); } - public void map(InfoMap infoMap) { - infoMap.putFirst(new Info().enumerate(false)) - .put(new Info("bool").cast().valueTypes("boolean").pointerTypes("boolean[]", "BoolPointer")) - .put(new Info("const char").pointerTypes("String", "@Cast(\"const char*\") BytePointer")) - .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) - .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", - "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) - .put(new Info("std::set").pointerTypes("StringSet").define()) - .put(new Info("std::vector").pointerTypes("StringVector").define()) - .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) - .put(new Info("TritonServer").purify(false).virtualize()); - } -} diff --git a/tritondevelopertoolsserver/src/main/java9/module-info.java b/tritondevelopertoolsserver/src/main/java9/module-info.java deleted file mode 100644 index 0ec0a919305..00000000000 --- a/tritondevelopertoolsserver/src/main/java9/module-info.java +++ /dev/null @@ -1,6 +0,0 @@ -module org.bytedeco.tritondevelopertoolsserver { - requires transitive org.bytedeco.javacpp; - exports org.bytedeco.tritondevelopertoolsserver.global; - exports org.bytedeco.tritondevelopertoolsserver.presets; - exports org.bytedeco.tritondevelopertoolsserver; -} diff --git a/tritondevelopertoolsserver/samples/simple/Simple.java b/tritonserver/samples/simplecpp/SimpleCPP.java similarity index 95% rename from tritondevelopertoolsserver/samples/simple/Simple.java rename to tritonserver/samples/simplecpp/SimpleCPP.java index 9187ece0f80..348737d695f 100644 --- a/tritondevelopertoolsserver/samples/simple/Simple.java +++ b/tritonserver/samples/simplecpp/SimpleCPP.java @@ -29,10 +29,10 @@ import java.util.concurrent.*; import com.google.gson.*; import org.bytedeco.javacpp.*; -import org.bytedeco.tritondevelopertoolsserver.tritondevelopertoolsserver.*; -import static org.bytedeco.tritondevelopertoolsserver.global.tritondevelopertoolsserver.*; +import org.bytedeco.tritonserver.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritonserver.*; -public class Simple { +public class SimpleCPP { // Helper functions static void FAIL(String MSG) { System.err.println("Failure: " + MSG); @@ -45,7 +45,7 @@ static void Usage(String msg) System.err.println(msg); } - System.err.println("Usage: java " + Simple.class.getSimpleName() +" [options]"); + System.err.println("Usage: java " + SimpleCPP.class.getSimpleName() +" [options]"); System.err.println("\t-v Enable verbose logging"); System.err.println("\t-r [model repository absolute path]"); System.exit(1); diff --git a/tritondevelopertoolsserver/samples/simple/pom.xml b/tritonserver/samples/simplecpp/pom.xml similarity index 59% rename from tritondevelopertoolsserver/samples/simple/pom.xml rename to tritonserver/samples/simplecpp/pom.xml index 325ee49f7db..89ee1bed827 100644 --- a/tritondevelopertoolsserver/samples/simple/pom.xml +++ b/tritonserver/samples/simplecpp/pom.xml @@ -1,20 +1,25 @@ 4.0.0 - org.bytedeco.tritondevelopertoolsserver - simple + org.bytedeco.tritonserver + simplecpp 1.5.9-SNAPSHOT - Simple + SimpleCPP 1.8 1.8 org.bytedeco - tritondevelopertoolsserver-platform + tritonserver-platform 2.35-1.5.9-SNAPSHOT shaded + + com.google.code.gson + gson + 2.9.0 + . diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index ebed413d826..086ac763d64 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Jack He, Samuel Audet + * Copyright (C) 2021-2023 Jack He, Samuel Audet, Katherine Yang, Baojun Liu * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -34,23 +34,17 @@ /** * - * @author Jack He + * @author Katherine Yang, Jack He, Baojun Liu */ @Properties( value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, - link = "tritonserver", - includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include"}, + include = {"common.h", "generic_server_wrapper.h", "tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, + link = {"tritondevelopertoolsserver", "tritonserver"}, + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), - @Platform( - value = "windows-x86_64", - includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", - linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", - preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" - ) }, target = "org.bytedeco.tritonserver.tritonserver", global = "org.bytedeco.tritonserver.global.tritonserver" @@ -65,6 +59,9 @@ public void map(InfoMap infoMap) { .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) - ; + .put(new Info("std::set").pointerTypes("StringSet").define()) + .put(new Info("std::vector").pointerTypes("StringVector").define()) + .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) + .put(new Info("TritonServer").purify(false).virtualize()); } } From f8ea23230360e640bc559e758bba64970864c42a Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 31 May 2023 16:41:14 -0700 Subject: [PATCH 26/50] reorder dependencies --- .../java/org/bytedeco/tritonserver/presets/tritonserver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 086ac763d64..08d9d562f3c 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -40,8 +40,8 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"common.h", "generic_server_wrapper.h", "tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, - link = {"tritondevelopertoolsserver", "tritonserver"}, + include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h", "common.h", "generic_server_wrapper.h"}, + link = {"tritonserver", "tritondevelopertoolsserver"}, includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), From 3cc20e57e7fb76e105dab7b2c629e0b1965137ab Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 31 May 2023 17:12:43 -0700 Subject: [PATCH 27/50] add C++ bindings to build --- .../tritonserver/global/tritonserver.java | 199 +++++++++++++++++- .../tritonserver/tritonserver/Allocator.java | 37 ++++ .../tritonserver/BackendConfig.java | 43 ++++ .../tritonserver/CUDAMemoryPoolByteSize.java | 32 +++ .../tritonserver/GenericInferRequest.java | 57 +++++ .../tritonserver/GenericInferResult.java | 71 +++++++ .../tritonserver/GenericTritonServer.java | 135 ++++++++++++ .../tritonserver/tritonserver/HostPolicy.java | 47 +++++ .../tritonserver/InferOptions.java | 109 ++++++++++ .../tritonserver/LoggingOptions.java | 70 ++++++ .../tritonserver/MetricsOptions.java | 49 +++++ .../tritonserver/ModelLoadGPULimit.java | 30 +++ .../tritonserver/NewModelRepo.java | 50 +++++ .../tritonserver/OutputBufferReleaseFn_t.java | 21 ++ .../tritonserver/RateLimitResource.java | 43 ++++ .../tritonserver/RepositoryIndex.java | 50 +++++ .../ResponseAllocatorAllocFn_t.java | 26 +++ .../ResponseAllocatorStartFn_t.java | 21 ++ .../tritonserver/ServerOptions.java | 191 +++++++++++++++++ .../tritonserver/tritonserver/StringSet.java | 36 ++++ .../tritonserver/StringVector.java | 99 +++++++++ .../tritonserver/tritonserver/Tensor.java | 81 +++++++ .../tritonserver/tritonserver/Trace.java | 66 ++++++ .../tritonserver/TritonException.java | 29 +++ 24 files changed, 1588 insertions(+), 4 deletions(-) create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 6459d02d86d..decd65593a8 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -11,6 +11,12 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver { static { Loader.load(); } +// Targeting ../tritonserver/StringSet.java + + +// Targeting ../tritonserver/StringVector.java + + // Parsed from tritonserver.h // Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -130,7 +136,7 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver public static final int TRITONSERVER_API_VERSION_MAJOR = 1; /// -public static final int TRITONSERVER_API_VERSION_MINOR = 21; +public static final int TRITONSERVER_API_VERSION_MINOR = 22; /** Get the TRITONBACKEND API version supported by the Triton shared * library. This value can be compared against the @@ -1432,9 +1438,11 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetStringPa /// public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( - TRITONSERVER_InferenceRequest request, String key, @Cast("const int64_t") long value); + TRITONSERVER_InferenceRequest request, String key, + @Cast("const int64_t") long value); public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetIntParameter( - TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, @Cast("const int64_t") long value); + TRITONSERVER_InferenceRequest request, @Cast("const char*") BytePointer key, + @Cast("const int64_t") long value); /** Set a boolean parameter in the request. * @@ -2330,7 +2338,6 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetBackendConf * @param value The setting value. * @return a TRITONSERVER_Error indicating success or failure. */ -/// /// public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( TRITONSERVER_ServerOptions options, String policy_name, @@ -2339,6 +2346,24 @@ public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetHostPolicy( TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer policy_name, @Cast("const char*") BytePointer setting, @Cast("const char*") BytePointer value); +/** Set a configuration setting for metrics in server options. + * + * @param options The server options object. + * @param name The name of the configuration group. An empty string indicates + * a global configuration option. + * @param setting The name of the setting. + * @param value The setting value. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( + TRITONSERVER_ServerOptions options, String name, String setting, + String value); +public static native TRITONSERVER_Error TRITONSERVER_ServerOptionsSetMetricsConfig( + TRITONSERVER_ServerOptions options, @Cast("const char*") BytePointer name, @Cast("const char*") BytePointer setting, + @Cast("const char*") BytePointer value); + /** TRITONSERVER_Server * * An inference server. @@ -5573,4 +5598,170 @@ public static native TRITONSERVER_Error TRITONREPOAGENT_ModelAction( // #endif +// Parsed from common.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include + +//============================================================================== +/** enum classes + * */ +/** enum class triton::developer_tools::server::ModelControlMode */ +public static final int NONE = 0, POLL = 1, EXPLICIT = 2; +/** enum class triton::developer_tools::server::MemoryType */ +public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; +/** enum class triton::developer_tools::server::DataType */ +public static final int + INVALID = 0, + BOOL = 1, + UINT8 = 2, + UINT16 = 3, + UINT32 = 4, + UINT64 = 5, + INT8 = 6, + INT16 = 7, + INT32 = 8, + INT64 = 9, + FP16 = 10, + FP32 = 11, + FP64 = 12, + BYTES = 13, + BF16 = 14; +/** enum class triton::developer_tools::server::ModelReadyState */ +public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; +// Targeting ../tritonserver/TritonException.java + + +// Targeting ../tritonserver/ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritonserver/OutputBufferReleaseFn_t.java + + +// Targeting ../tritonserver/ResponseAllocatorStartFn_t.java + + +// Targeting ../tritonserver/LoggingOptions.java + + +// Targeting ../tritonserver/MetricsOptions.java + + +// Targeting ../tritonserver/RateLimitResource.java + + +// Targeting ../tritonserver/ModelLoadGPULimit.java + + +// Targeting ../tritonserver/Allocator.java + + +// Targeting ../tritonserver/BackendConfig.java + + +// Targeting ../tritonserver/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritonserver/HostPolicy.java + + +// Targeting ../tritonserver/Trace.java + + +// Targeting ../tritonserver/ServerOptions.java + + +// Targeting ../tritonserver/RepositoryIndex.java + + +// Targeting ../tritonserver/Tensor.java + + +// Targeting ../tritonserver/NewModelRepo.java + + +// Targeting ../tritonserver/InferOptions.java + + + + // namespace triton::developer_tools::server + + +// Parsed from generic_server_wrapper.h + +// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include +// #include "../src/infer_requested_output.h" +// #include "../src/tracer.h" +// #include "common.h" + +/// +// Targeting ../tritonserver/GenericTritonServer.java + + +// Targeting ../tritonserver/GenericInferResult.java + + +// Targeting ../tritonserver/GenericInferRequest.java + + + + // namespace triton::developer_tools::server + + } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java new file mode 100644 index 00000000000..617e4f3626f --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java @@ -0,0 +1,37 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + + +//============================================================================== +/** Custom Allocator object for providing custom functions for allocator. + * If there is no custom allocator provided, will use the default allocator. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class Allocator extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Allocator(Pointer p) { super(p); } + + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/) { super((Pointer)null); allocate(alloc_fn, release_fn, start_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn, + ResponseAllocatorStartFn_t start_fn/*=nullptr*/); + public Allocator( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn) { super((Pointer)null); allocate(alloc_fn, release_fn); } + private native void allocate( + ResponseAllocatorAllocFn_t alloc_fn, OutputBufferReleaseFn_t release_fn); + + public native ResponseAllocatorAllocFn_t AllocFn(); + public native OutputBufferReleaseFn_t ReleaseFn(); + public native ResponseAllocatorStartFn_t StartFn(); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java new file mode 100644 index 00000000000..b9387191f13 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold backend configuration for setting 'ServerOptions'. + * Different Triton-supported backends have different backend configuration + * options. Please refer to the 'Command line options' section in the + * documentation of each backend to see the options (e.g. Tensorflow Backend: + * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class BackendConfig extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public BackendConfig(Pointer p) { super(p); } + + public BackendConfig( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer setting, + @StdString BytePointer value); + public BackendConfig( + @StdString String name, @StdString String setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @StdString String setting, + @StdString String value); + + // The name of the backend. + public native @StdString BytePointer name_(); public native BackendConfig name_(BytePointer setter); + // The name of the setting. + public native @StdString BytePointer setting_(); public native BackendConfig setting_(BytePointer setter); + // The setting value. + public native @StdString BytePointer value_(); public native BackendConfig value_(BytePointer setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java new file mode 100644 index 00000000000..ab8cd68c875 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java @@ -0,0 +1,32 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold CUDA memory pool byte size for setting 'ServerOptions'. + * If GPU support is enabled, the server will allocate CUDA memory to minimize + * data transfer between host and devices until it exceeds the specified byte + * size. This will not affect the allocation conducted by the backend + * frameworks. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class CUDAMemoryPoolByteSize extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public CUDAMemoryPoolByteSize(Pointer p) { super(p); } + + public CUDAMemoryPoolByteSize(int gpu_device, @Cast("const uint64_t") long size) { super((Pointer)null); allocate(gpu_device, size); } + private native void allocate(int gpu_device, @Cast("const uint64_t") long size); + + // The GPU device ID to allocate the memory pool. + public native int gpu_device_(); public native CUDAMemoryPoolByteSize gpu_device_(int setter); + // The CUDA memory pool byte size that the server can allocate on given GPU + // device. Default is 64 MB. + public native @Cast("uint64_t") long size_(); public native CUDAMemoryPoolByteSize size_(long setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java new file mode 100644 index 00000000000..28e02e964a2 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java @@ -0,0 +1,57 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Object that describes an inflight inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class GenericInferRequest extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferRequest(Pointer p) { super(p); } + + /** Create an InferRequest instance. */ + public static native @UniquePtr GenericInferRequest Create( + @Const @ByRef InferOptions infer_options); + + /** Add an input tensor to be sent within an InferRequest object. The input + * data buffer within the 'Tensor' object must not be modified until + * inference is completed and result is returned. + * @param name The name of the input tensor. + * @param input A Tensor object that describes an input tensor. */ + public native @NoException(true) void AddInput( + @StdString BytePointer name, @Const @ByRef Tensor input); + public native @NoException(true) void AddInput( + @StdString String name, @Const @ByRef Tensor input); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. Pre-allocated buffer for each output should be specified within + * the 'Tensor' object. + * @param name The name of the output tensor. + * @param output A Tensor object that describes an output tensor containing + * its pre-allocated buffer. */ + public native void AddRequestedOutput(@StdString BytePointer name, @ByRef Tensor output); + public native void AddRequestedOutput(@StdString String name, @ByRef Tensor output); + + /** Add a requested output to be sent within an InferRequest object. + * Calling this function is optional. If no output(s) are specifically + * requested then all outputs defined by the model will be calculated and + * returned. + * @param name The name of the output tensor. */ + public native void AddRequestedOutput(@StdString BytePointer name); + public native void AddRequestedOutput(@StdString String name); + + /** Clear inputs and outputs of the request. This allows users to reuse the + * InferRequest object if needed. */ + public native void Reset(); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java new file mode 100644 index 00000000000..bafa00f9de4 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java @@ -0,0 +1,71 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** An interface for InferResult object to interpret the response to an + * inference request. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class GenericInferResult extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericInferResult(Pointer p) { super(p); } + + + /** Get the name of the model which generated this response. + * @return Returns the name of the model. */ + public native @StdString @NoException(true) BytePointer ModelName(); + + /** Get the version of the model which generated this response. + * @return Returns the version of the model. */ + public native @StdString @NoException(true) BytePointer ModelVersion(); + + /** Get the id of the request which generated this response. + * @return Returns the id of the request. */ + public native @StdString @NoException(true) BytePointer Id(); + + /** Get the output names from the infer result + * @return Vector of output names */ + public native @ByVal StringVector OutputNames(); + + /** Get the result output as a shared pointer of 'Tensor' object. The 'buffer' + * field of the output is owned by the returned 'Tensor' object itself. Note + * that for string data, need to use 'StringData' function for string data + * result. + * @param name The name of the output tensor to be retrieved. + * @return Returns the output result as a shared pointer of 'Tensor' object. */ + public native @SharedPtr Tensor Output(@StdString BytePointer name); + public native @SharedPtr Tensor Output(@StdString String name); + + /** Get the result data as a vector of strings. The vector will + * receive a copy of result data. An exception will be thrown if + * the data type of output is not 'BYTES'. + * @param output_name The name of the output to get result data. + * @return Returns the result data represented as a vector of strings. The + * strings are stored in the row-major order. */ + public native @ByVal StringVector StringData( + @StdString BytePointer output_name); + public native @ByVal StringVector StringData( + @StdString String output_name); + + /** Return the complete response as a user friendly string. + * @return The string describing the complete response. */ + public native @StdString BytePointer DebugString(); + + /** Return if there is an error within this result. + * @return True if this 'GenericInferResult' object has an error, false if no + * error. */ + public native @Cast("bool") boolean HasError(); + + /** Return the error message of the error. + * @return The messsage for the error. Empty if no error. */ + public native @StdString BytePointer ErrorMsg(); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java new file mode 100644 index 00000000000..5ea9df800d2 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java @@ -0,0 +1,135 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Object that encapsulates in-process C API functionalities. + * */ +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class GenericTritonServer extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public GenericTritonServer(Pointer p) { super(p); } + + /** Create a GenericTritonServer instance. */ + public static native @UniquePtr GenericTritonServer Create( + @Const @ByRef ServerOptions server_options); + + /** Load the requested model or reload the model if it is already loaded. + * @param model_name The name of the model. */ + public native void LoadModel(@StdString BytePointer model_name); + public native void LoadModel(@StdString String model_name); + + /** Unload the requested model. Unloading a model that is not loaded + * on server has no affect. + * @param model_name The name of the model. */ + public native void UnloadModel(@StdString BytePointer model_name); + public native void UnloadModel(@StdString String model_name); + + /** Get the set of names of models that are loaded and ready for inference. + * @return Returns the set of names of models that are + * loaded and ready for inference. */ + public native @ByVal StringSet LoadedModels(); + + /** Get the index of model repository contents. + * @return Returns a vector of 'RepositoryIndex' object + * representing the repository index. */ + public native @StdVector RepositoryIndex ModelIndex(); + + /** Get the metrics of the server. + * @return Returns a string representing the metrics. */ + public native @StdString BytePointer ServerMetrics(); + + /** Get the inference statistics of the specified model. + * @param model_name The name of the model. + * @param model_version the version of the model requested. + * @return Returns a json string representing the model metrics. */ + public native @StdString BytePointer ModelStatistics( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version); + public native @StdString String ModelStatistics( + @StdString String model_name, @Cast("const int64_t") long model_version); + + /** Is the server live? + * @return Returns true if server is live, false otherwise. */ + public native @Cast("bool") boolean IsServerLive(); + + /** Is the server ready? + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsServerReady(); + + /** Stop a server object. A server can't be restarted once it is + * stopped. */ + public native void ServerStop(); + + /** Is the model ready? + * @param model_name The name of the model to get readiness for. + * @param model_version The version of the model to get readiness + * for. If -1 then the server will choose a version based on the + * model's policy. This field is optional, default is -1. + * @return Returns true if server is ready, false otherwise. */ + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString BytePointer model_name); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @Cast("bool") boolean IsModelReady( + @StdString String model_name); + + /** Get the configuration of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will + * choose a version based on the model and internal policy. This field is + * optional. @return Returns JSON representation of model configuration as a + * string. */ + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelConfig( + @StdString BytePointer model_name); + public native @StdString String ModelConfig( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelConfig( + @StdString String model_name); + + /** Get the metadata of the server. + * @return Returns JSON representation of server metadata as a string. */ + public native @StdString BytePointer ServerMetadata(); + + /** Get the metadata of specified model. + * @param model_name The name of the model. + * @param model_version The version of the model to get configuration. + * The default value is -1 which means then the server will choose a version + * based on the model and internal policy. This field is optional. + * @return Returns JSON representation of model metadata as a string. */ + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString BytePointer ModelMetadata( + @StdString BytePointer model_name); + public native @StdString String ModelMetadata( + @StdString String model_name, @Cast("const int64_t") long model_version/*=-1*/); + public native @StdString String ModelMetadata( + @StdString String model_name); + + /** Register a new model repository. This function is not available in polling + * mode. + * @param new_model_repo The 'NewModelRepo' object contains the info of the + * new model repo to be registered. */ + public native void RegisterModelRepo(@Const @ByRef NewModelRepo new_model_repo); + + /** Unregister a model repository. This function is not available in polling + * mode. + * @param repo_path The full path to the model repository. */ + public native void UnregisterModelRepo(@StdString BytePointer repo_path); + public native void UnregisterModelRepo(@StdString String repo_path); + + public native @UniquePtr GenericInferResult Infer( + @ByRef GenericInferRequest infer_request); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java new file mode 100644 index 00000000000..f067927320d --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java @@ -0,0 +1,47 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold host policy for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class HostPolicy extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public HostPolicy(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::HostPolicy::Setting */ + public static final int NUMA_NODE = 0, CPU_CORES = 1; + + public HostPolicy( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString BytePointer name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString BytePointer value); + public HostPolicy( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value) { super((Pointer)null); allocate(name, setting, value); } + private native void allocate( + @StdString String name, @Cast("const triton::developer_tools::server::HostPolicy::Setting") int setting, + @StdString String value); + + // The name of the policy. + public native @StdString BytePointer name_(); public native HostPolicy name_(BytePointer setter); + // The kind of the host policy setting. Currently supported settings are + // 'NUMA_NODE', 'CPU_CORES'. Note that 'NUMA_NODE' setting will affect pinned + // memory pool behavior, see the comments of 'pinned_memory_pool_byte_size_' + // in 'ServerOptions' for more detail. + public native @Cast("triton::developer_tools::server::HostPolicy::Setting") int setting_(); public native HostPolicy setting_(int setter); + // The setting value. + public native @StdString BytePointer value_(); public native HostPolicy value_(BytePointer setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java new file mode 100644 index 00000000000..7d31f1a6687 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java @@ -0,0 +1,109 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold options for Inference Request. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class InferOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public InferOptions(Pointer p) { super(p); } + + public InferOptions(@StdString BytePointer model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString BytePointer model_name); + public InferOptions(@StdString String model_name) { super((Pointer)null); allocate(model_name); } + private native void allocate(@StdString String model_name); + + public InferOptions( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString BytePointer model_name, @Cast("const int64_t") long model_version, + @StdString BytePointer request_id, @Cast("const uint64_t") long correlation_id, + @StdString BytePointer correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + public InferOptions( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace) { super((Pointer)null); allocate(model_name, model_version, request_id, correlation_id, correlation_id_str, sequence_start, sequence_end, priority, request_timeout, custom_allocator, trace); } + private native void allocate( + @StdString String model_name, @Cast("const int64_t") long model_version, + @StdString String request_id, @Cast("const uint64_t") long correlation_id, + @StdString String correlation_id_str, @Cast("const bool") boolean sequence_start, + @Cast("const bool") boolean sequence_end, @Cast("const uint64_t") long priority, + @Cast("const uint64_t") long request_timeout, + @SharedPtr Allocator custom_allocator, + @SharedPtr Trace trace); + + // The name of the model to run inference. + public native @StdString BytePointer model_name_(); public native InferOptions model_name_(BytePointer setter); + // The version of the model to use while running inference. The default + // value is "-1" which means the server will select the + // version of the model based on its internal policy. + public native @Cast("int64_t") long model_version_(); public native InferOptions model_version_(long setter); + // An identifier for the request. If specified will be returned + // in the response. Default value is an empty string which means no + // request_id will be used. + public native @StdString BytePointer request_id_(); public native InferOptions request_id_(BytePointer setter); + // The correlation ID of the inference request to be an unsigned integer. + // Should be used exclusively with 'correlation_id_str_'. + // Default is 0, which indicates that the request has no correlation ID. + public native @Cast("uint64_t") long correlation_id_(); public native InferOptions correlation_id_(long setter); + // The correlation ID of the inference request to be a string. + // Should be used exclusively with 'correlation_id_'. + // Default value is "". + public native @StdString BytePointer correlation_id_str_(); public native InferOptions correlation_id_str_(BytePointer setter); + // Indicates whether the request being added marks the start of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_start_(); public native InferOptions sequence_start_(boolean setter); + // Indicates whether the request being added marks the end of the + // sequence. Default value is False. This argument is ignored if + // 'sequence_id' is 0. + public native @Cast("bool") boolean sequence_end_(); public native InferOptions sequence_end_(boolean setter); + // Indicates the priority of the request. Priority value zero + // indicates that the default priority level should be used + // (i.e. same behavior as not specifying the priority parameter). + // Lower value priorities indicate higher priority levels. Thus + // the highest priority level is indicated by setting the parameter + // to 1, the next highest is 2, etc. If not provided, the server + // will handle the request using default setting for the model. + public native @Cast("uint64_t") long priority_(); public native InferOptions priority_(long setter); + // The timeout value for the request, in microseconds. If the request + // cannot be completed within the time by the server can take a + // model-specific action such as terminating the request. If not + // provided, the server will handle the request using default setting + // for the model. + public native @Cast("uint64_t") long request_timeout_(); public native InferOptions request_timeout_(long setter); + // User-provided custom reponse allocator object. Default is nullptr. + // If using custom allocator, the lifetime of this 'Allocator' object should + // be long enough until `InferResult` object goes out of scope as we need + // this `Allocator` object to call 'ResponseAllocatorReleaseFn_t' for + // releasing the response. + public native @SharedPtr Allocator custom_allocator_(); public native InferOptions custom_allocator_(Allocator setter); + // Update trace setting for the specified model. If not set, will use global + // trace setting in 'ServerOptions' for tracing if tracing is enabled in + // 'ServerOptions'. Default is nullptr. + public native @SharedPtr Trace trace_(); public native InferOptions trace_(Trace setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java new file mode 100644 index 00000000000..24710b5ea6b --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java @@ -0,0 +1,70 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold logging options for setting 'ServerOptions'. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class LoggingOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public LoggingOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public LoggingOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public LoggingOptions position(long position) { + return (LoggingOptions)super.position(position); + } + @Override public LoggingOptions getPointer(long i) { + return new LoggingOptions((Pointer)this).offsetAddress(i); + } + + // The range of VerboseLevel is [0, INT_MAX]. + /** enum class triton::developer_tools::server::LoggingOptions::VerboseLevel */ + public static final int OFF = 0, MIN = 1, MAX = Integer.MAX_VALUE; + /** enum class triton::developer_tools::server::LoggingOptions::LogFormat */ + public static final int DEFAULT = 0, ISO8601 = 1; + + public LoggingOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString BytePointer log_file); + public LoggingOptions( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file) { super((Pointer)null); allocate(verbose, info, warn, error, format, log_file); } + private native void allocate( + @Cast("const triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose, @Cast("const bool") boolean info, @Cast("const bool") boolean warn, + @Cast("const bool") boolean error, @Cast("const triton::developer_tools::server::LoggingOptions::LogFormat") int format, @StdString String log_file); + + public native void SetVerboseLevel(int verbose_level); + + // Verbose logging level. Default is OFF. + public native @Cast("triton::developer_tools::server::LoggingOptions::VerboseLevel") int verbose_(); public native LoggingOptions verbose_(int setter); + // Enable or disable info logging level. Default is true. + public native @Cast("bool") boolean info_(); public native LoggingOptions info_(boolean setter); + // Enable or disable warn logging level. Default is true. + public native @Cast("bool") boolean warn_(); public native LoggingOptions warn_(boolean setter); + // Enable or disable error logging level. Default is true. + public native @Cast("bool") boolean error_(); public native LoggingOptions error_(boolean setter); + // The format of logging. For "DEFAULT", the log severity (L) and + // timestamp will be logged as "LMMDD hh:mm:ss.ssssss". For "ISO8601", the + // log format will be "YYYY-MM-DDThh:mm:ssZ L". Default is 'DEFAULT'. + public native @Cast("triton::developer_tools::server::LoggingOptions::LogFormat") int format_(); public native LoggingOptions format_(int setter); + // Logging output file. If specified, log outputs will be saved to this file. + // If not specified, log outputs will stream to the console. Default is an + // empty string. + public native @StdString BytePointer log_file_(); public native LoggingOptions log_file_(BytePointer setter); // logging output file +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java new file mode 100644 index 00000000000..bb5e97381fa --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java @@ -0,0 +1,49 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold metrics options for setting 'ServerOptions'. + * See here for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class MetricsOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public MetricsOptions(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public MetricsOptions(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public MetricsOptions position(long position) { + return (MetricsOptions)super.position(position); + } + @Override public MetricsOptions getPointer(long i) { + return new MetricsOptions((Pointer)this).offsetAddress(i); + } + + public MetricsOptions() { super((Pointer)null); allocate(); } + private native void allocate(); + + public MetricsOptions( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms) { super((Pointer)null); allocate(allow_metrics, allow_gpu_metrics, allow_cpu_metrics, metrics_interval_ms); } + private native void allocate( + @Cast("const bool") boolean allow_metrics, @Cast("const bool") boolean allow_gpu_metrics, + @Cast("const bool") boolean allow_cpu_metrics, @Cast("const uint64_t") long metrics_interval_ms); + + // Enable or disable metrics. Default is true. + public native @Cast("bool") boolean allow_metrics_(); public native MetricsOptions allow_metrics_(boolean setter); + // Enable or disable GPU metrics. Default is true. + public native @Cast("bool") boolean allow_gpu_metrics_(); public native MetricsOptions allow_gpu_metrics_(boolean setter); + // Enable or disable CPU metrics. Default is true. + public native @Cast("bool") boolean allow_cpu_metrics_(); public native MetricsOptions allow_cpu_metrics_(boolean setter); + // The interval for metrics collection. Default is 2000. + public native @Cast("uint64_t") long metrics_interval_ms_(); public native MetricsOptions metrics_interval_ms_(long setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java new file mode 100644 index 00000000000..cddc46f3cdc --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java @@ -0,0 +1,30 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold GPU limit of model loading for setting 'ServerOptions'. + * The limit on GPU memory usage is specified as a fraction. If model loading + * on the device is requested and the current memory usage exceeds the limit, + * the load will be rejected. If not specified, the limit will not be set. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class ModelLoadGPULimit extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ModelLoadGPULimit(Pointer p) { super(p); } + + public ModelLoadGPULimit(int device_id, double fraction) { super((Pointer)null); allocate(device_id, fraction); } + private native void allocate(int device_id, double fraction); + + // The GPU device ID. + public native int device_id_(); public native ModelLoadGPULimit device_id_(int setter); + // The limit on memory usage as a fraction. + public native double fraction_(); public native ModelLoadGPULimit fraction_(double setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java new file mode 100644 index 00000000000..b8e23f0b3d0 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold the full path to the model repository to be registered and + * the mapping from the original model name to the overriden one. This object + * is used for calling 'TritonServer::RegisterModelRepo' for registering + * model repository. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class NewModelRepo extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public NewModelRepo(Pointer p) { super(p); } + + public NewModelRepo(@StdString BytePointer path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString BytePointer path); + public NewModelRepo(@StdString String path) { super((Pointer)null); allocate(path); } + private native void allocate(@StdString String path); + + public NewModelRepo( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString BytePointer path, @StdString BytePointer original_name, + @StdString BytePointer override_name); + public NewModelRepo( + @StdString String path, @StdString String original_name, + @StdString String override_name) { super((Pointer)null); allocate(path, original_name, override_name); } + private native void allocate( + @StdString String path, @StdString String original_name, + @StdString String override_name); + + // The full path to the model repository. + public native @StdString BytePointer path_(); public native NewModelRepo path_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer original_name_(); public native NewModelRepo original_name_(BytePointer setter); + // The original name of the model. This field is optional when there is no + // name mapping needed. + public native @StdString BytePointer override_name_(); public native NewModelRepo override_name_(BytePointer setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java new file mode 100644 index 00000000000..7cd2be10c4c --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java @@ -0,0 +1,21 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class OutputBufferReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public OutputBufferReleaseFn_t(Pointer p) { super(p); } + protected OutputBufferReleaseFn_t() { allocate(); } + private native void allocate(); + public native void call( + Pointer buffer, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java new file mode 100644 index 00000000000..e6c595daae1 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java @@ -0,0 +1,43 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold rate limit resource for setting 'ServerOptions'. See here + * for more information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class RateLimitResource extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RateLimitResource(Pointer p) { super(p); } + + public RateLimitResource(@StdString BytePointer name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString BytePointer name, int count); + public RateLimitResource(@StdString String name, int count) { super((Pointer)null); allocate(name, count); } + private native void allocate(@StdString String name, int count); + + public RateLimitResource(@StdString BytePointer name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString BytePointer name, int count, int device); + public RateLimitResource(@StdString String name, int count, int device) { super((Pointer)null); allocate(name, count, device); } + private native void allocate(@StdString String name, int count, int device); + + // The name of the resource. + public native @StdString BytePointer name_(); public native RateLimitResource name_(BytePointer setter); + // The count of the resource. + public native int count_(); public native RateLimitResource count_(int setter); + // The device identifier for the resource. This field is optional and if not + // specified will be applied to every device. The device value is ignored for + // a global resource. The server will use the rate limiter configuration + // specified for instance groups in model config to determine whether resource + // is global. In case of conflicting resource type in different model + // configurations, server will raise an appropriate error while loading model. + public native int device_(); public native RateLimitResource device_(int setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java new file mode 100644 index 00000000000..de3317d60f7 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java @@ -0,0 +1,50 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold repository index for 'ModelIndex' function. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class RepositoryIndex extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RepositoryIndex(Pointer p) { super(p); } + + public RepositoryIndex( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString BytePointer name, @StdString BytePointer version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + public RepositoryIndex( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state) { super((Pointer)null); allocate(name, version, state); } + private native void allocate( + @StdString String name, @StdString String version, + @Cast("const triton::developer_tools::server::ModelReadyState") int state); + + // The name of the model. + public native @StdString BytePointer name_(); public native RepositoryIndex name_(BytePointer setter); + // The version of the model. + public native @StdString BytePointer version_(); public native RepositoryIndex version_(BytePointer setter); + // The state of the model. The states are + // * UNKNOWN: The model is in an unknown state. The model is not available for + // inferencing. + // * READY: The model is ready and available for inferencing. + // * UNAVAILABLE: The model is unavailable, indicating that the model failed + // to load or has been implicitly or explicitly unloaded. The model is not + // available for inferencing. + // * LOADING: The model is being loaded by the inference server. The model is + // not available for inferencing. + // * UNLOADING: The model is being unloaded by the inference server. The model + // is not available for inferencing. + public native @Cast("triton::developer_tools::server::ModelReadyState") int state_(); public native RepositoryIndex state_(int setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java new file mode 100644 index 00000000000..20d84c3dc83 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java @@ -0,0 +1,26 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Custom Response Allocator Callback function signatures. + * */ +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class ResponseAllocatorAllocFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorAllocFn_t(Pointer p) { super(p); } + protected ResponseAllocatorAllocFn_t() { allocate(); } + private native void allocate(); + public native void call( + String tensor_name, @Cast("size_t") long byte_size, @Cast("triton::developer_tools::server::MemoryType") int preferred_memory_type, + @Cast("int64_t") long preferred_memory_type_id, @Cast("void**") PointerPointer buffer, + @Cast("triton::developer_tools::server::MemoryType*") IntPointer actual_memory_type, @Cast("int64_t*") LongPointer actual_memory_type_id); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java new file mode 100644 index 00000000000..2ee9681a8d1 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java @@ -0,0 +1,21 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class ResponseAllocatorStartFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ResponseAllocatorStartFn_t(Pointer p) { super(p); } + protected ResponseAllocatorStartFn_t() { allocate(); } + private native void allocate(); + public native void call(Pointer userp); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java new file mode 100644 index 00000000000..493f4e2612a --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java @@ -0,0 +1,191 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Server options that are used to initialize Triton Server. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class ServerOptions extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ServerOptions(Pointer p) { super(p); } + + public ServerOptions(@Const @ByRef StringVector model_repository_paths) { super((Pointer)null); allocate(model_repository_paths); } + private native void allocate(@Const @ByRef StringVector model_repository_paths); + + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString BytePointer server_id, + @StdString BytePointer backend_dir, @StdString BytePointer repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + public ServerOptions( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace) { super((Pointer)null); allocate(model_repository_paths, logging, metrics, be_config, server_id, backend_dir, repo_agent_dir, disable_auto_complete_config, model_control_mode, repository_poll_secs, startup_models, rate_limit_resource, pinned_memory_pool_byte_size, cuda_memory_pool_byte_size, response_cache_byte_size, min_cuda_compute_capability, exit_on_error, exit_timeout_secs, buffer_manager_thread_count, model_load_thread_count, model_load_gpu_limit, host_policy, trace); } + private native void allocate( + @Const @ByRef StringVector model_repository_paths, + @Const @ByRef LoggingOptions logging, @Const @ByRef MetricsOptions metrics, + @StdVector BackendConfig be_config, @StdString String server_id, + @StdString String backend_dir, @StdString String repo_agent_dir, + @Cast("const bool") boolean disable_auto_complete_config, + @Cast("const triton::developer_tools::server::ModelControlMode") int model_control_mode, + int repository_poll_secs, + @Const @ByRef StringSet startup_models, + @StdVector RateLimitResource rate_limit_resource, + @Cast("const int64_t") long pinned_memory_pool_byte_size, + @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size, + @Cast("const uint64_t") long response_cache_byte_size, + double min_cuda_compute_capability, @Cast("const bool") boolean exit_on_error, + int exit_timeout_secs, + int buffer_manager_thread_count, + @Cast("const uint32_t") int model_load_thread_count, + @StdVector ModelLoadGPULimit model_load_gpu_limit, + @StdVector HostPolicy host_policy, @SharedPtr Trace trace); + + public native void SetLoggingOptions(@Const @ByRef LoggingOptions logging); + + // Paths to model repository directory. Note that if a model is not unique + // across all model repositories at any time, the model will not be available. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md. + public native @ByRef StringVector model_repository_paths_(); public native ServerOptions model_repository_paths_(StringVector setter); + // Logging options. See the 'LoggingOptions' structure for more information. + public native @ByRef LoggingOptions logging_(); public native ServerOptions logging_(LoggingOptions setter); + // Metrics options. See the 'MetricsOptions' structure for more information. + public native @ByRef MetricsOptions metrics_(); public native ServerOptions metrics_(MetricsOptions setter); + // Backend configuration. See the 'BackendConfig' structure for more + // information. + public native @StdVector BackendConfig be_config_(); public native ServerOptions be_config_(BackendConfig setter); + // The ID of the server. + public native @StdString BytePointer server_id_(); public native ServerOptions server_id_(BytePointer setter); + // The global directory searched for backend shared libraries. Default is + // "/opt/tritonserver/backends". See here for more information: + // https://github.com/triton-inference-server/backend#backends. + public native @StdString BytePointer backend_dir_(); public native ServerOptions backend_dir_(BytePointer setter); + // The global directory searched for repository agent shared libraries. + // Default is "/opt/tritonserver/repoagents". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/repository_agents.md. + public native @StdString BytePointer repo_agent_dir_(); public native ServerOptions repo_agent_dir_(BytePointer setter); + // If set, disables the triton and backends from auto completing model + // configuration files. Model configuration files must be provided and + // all required configuration settings must be specified. Default is false. + // See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration. + public native @Cast("bool") boolean disable_auto_complete_config_(); public native ServerOptions disable_auto_complete_config_(boolean setter); + // Specify the mode for model management. Options are "NONE", "POLL" and + // "EXPLICIT". Default is "NONE". See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md. + public native @Cast("triton::developer_tools::server::ModelControlMode") int model_control_mode_(); public native ServerOptions model_control_mode_(int setter); + // Interval in seconds between each poll of the model repository to check for + // changes. Valid only when 'model_control_mode_' is set to "POLL". Default + // is 15. + public native int repository_poll_secs_(); public native ServerOptions repository_poll_secs_(int setter); + // Specify the the models to be loaded on server startup. This will only take + // effect if 'model_control_mode_' is set to 'EXPLICIT'. + public native @ByRef StringSet startup_models_(); public native ServerOptions startup_models_(StringSet setter); + // The number of resources available to the server. Rate limiting is disabled + // by default, and can be enabled once 'rate_limit_resource_' is set. See the + // 'RateLimitResource' structure for more information. + public native @StdVector RateLimitResource rate_limit_resource_(); public native ServerOptions rate_limit_resource_(RateLimitResource setter); + // The total byte size that can be allocated as pinned system memory. If GPU + // support is enabled, the server will allocate pinned system memory to + // accelerate data transfer between host and devices until it exceeds the + // specified byte size. If 'NUMA_NODE' is configured via 'host_policy_', the + // pinned system memory of the pool size will be allocated on each numa node. + // This option will not affect the allocation conducted by the backend + // frameworks. Default is 256 MB. + public native @Cast("int64_t") long pinned_memory_pool_byte_size_(); public native ServerOptions pinned_memory_pool_byte_size_(long setter); + // The total byte size that can be allocated as CUDA memory for the GPU + // device. See the 'CUDAMemoryPoolByteSize' structure for more information. + public native @StdVector CUDAMemoryPoolByteSize cuda_memory_pool_byte_size_(); public native ServerOptions cuda_memory_pool_byte_size_(CUDAMemoryPoolByteSize setter); + // The size in bytes to allocate for a request/response cache. When non-zero, + // Triton allocates the requested size in CPU memory and shares the cache + // across all inference requests and across all models. For a given model to + // use request caching, the model must enable request caching in the model + // configuration. See here for more information: + // https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#response-cache. + // By default, no model uses request caching even if the + // 'response_cache_byte_size_' is set. Default is 0. + public native @Cast("uint64_t") long response_cache_byte_size_(); public native ServerOptions response_cache_byte_size_(long setter); + // The minimum supported CUDA compute capability. GPUs that don't support this + // compute capability will not be used by the server. Default is 0. + public native double min_cuda_compute_capability_(); public native ServerOptions min_cuda_compute_capability_(double setter); + // If set, exit the inference server when an error occurs during + // initialization. Default is true. + public native @Cast("bool") boolean exit_on_error_(); public native ServerOptions exit_on_error_(boolean setter); + // Timeout (in seconds) when exiting to wait for in-flight inferences to + // finish. After the timeout expires the server exits even if inferences are + // still in flight. Default is 30 secs. + public native int exit_timeout_secs_(); public native ServerOptions exit_timeout_secs_(int setter); + // The number of threads used to accelerate copies and other operations + // required to manage input and output tensor contents. Default is 0. + public native int buffer_manager_thread_count_(); public native ServerOptions buffer_manager_thread_count_(int setter); + // The number of threads used to concurrently load models in model + // repositories. Default is 2*. + public native @Cast("uint32_t") int model_load_thread_count_(); public native ServerOptions model_load_thread_count_(int setter); + // The GPU limit of model loading. See the 'ModelLoadGPULimit' structure for + // more information. + public native @StdVector ModelLoadGPULimit model_load_gpu_limit_(); public native ServerOptions model_load_gpu_limit_(ModelLoadGPULimit setter); + // The host policy setting. See the 'HostPolicy' structure for more + // information. + public native @StdVector HostPolicy host_policy_(); public native ServerOptions host_policy_(HostPolicy setter); + // The global trace setting. Default is nullptr, meaning that tracing is not + // enabled. See the 'Trace' structure for more information. + public native @SharedPtr Trace trace_(); public native ServerOptions trace_(Trace setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java new file mode 100644 index 00000000000..5836732ef4d --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java @@ -0,0 +1,36 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + +@Name("std::set") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class StringSet extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringSet(Pointer p) { super(p); } + public StringSet() { allocate(); } + private native void allocate(); + public native @Name("operator =") @ByRef StringSet put(@ByRef StringSet x); + + public boolean empty() { return size() == 0; } + public native long size(); + + public native void insert(@StdString BytePointer value); + public native void erase(@StdString BytePointer value); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } +} + diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java new file mode 100644 index 00000000000..7590a66ad75 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java @@ -0,0 +1,99 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + +@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class StringVector extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public StringVector(Pointer p) { super(p); } + public StringVector(BytePointer value) { this(1); put(0, value); } + public StringVector(BytePointer ... array) { this(array.length); put(array); } + public StringVector(String value) { this(1); put(0, value); } + public StringVector(String ... array) { this(array.length); put(array); } + public StringVector() { allocate(); } + public StringVector(long n) { allocate(n); } + private native void allocate(); + private native void allocate(@Cast("size_t") long n); + public native @Name("operator =") @ByRef StringVector put(@ByRef StringVector x); + + public boolean empty() { return size() == 0; } + public native long size(); + public void clear() { resize(0); } + public native void resize(@Cast("size_t") long n); + + @Index(function = "at") public native @StdString BytePointer get(@Cast("size_t") long i); + public native StringVector put(@Cast("size_t") long i, BytePointer value); + @ValueSetter @Index(function = "at") public native StringVector put(@Cast("size_t") long i, @StdString String value); + + public native @ByVal Iterator insert(@ByVal Iterator pos, @StdString BytePointer value); + public native @ByVal Iterator erase(@ByVal Iterator pos); + public native @ByVal Iterator begin(); + public native @ByVal Iterator end(); + @NoOffset @Name("iterator") public static class Iterator extends Pointer { + public Iterator(Pointer p) { super(p); } + public Iterator() { } + + public native @Name("operator ++") @ByRef Iterator increment(); + public native @Name("operator ==") boolean equals(@ByRef Iterator it); + public native @Name("operator *") @StdString BytePointer get(); + } + + public BytePointer[] get() { + BytePointer[] array = new BytePointer[size() < Integer.MAX_VALUE ? (int)size() : Integer.MAX_VALUE]; + for (int i = 0; i < array.length; i++) { + array[i] = get(i); + } + return array; + } + @Override public String toString() { + return java.util.Arrays.toString(get()); + } + + public BytePointer pop_back() { + long size = size(); + BytePointer value = get(size - 1); + resize(size - 1); + return value; + } + public StringVector push_back(BytePointer value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(BytePointer value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(BytePointer ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } + + public StringVector push_back(String value) { + long size = size(); + resize(size + 1); + return put(size, value); + } + public StringVector put(String value) { + if (size() != 1) { resize(1); } + return put(0, value); + } + public StringVector put(String ... array) { + if (size() != array.length) { resize(array.length); } + for (int i = 0; i < array.length; i++) { + put(i, array[i]); + } + return this; + } +} + diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java new file mode 100644 index 00000000000..cf05514516e --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java @@ -0,0 +1,81 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold information of a tensor. This object is used for adding + * input/requested output to an inference request, and retrieving the output + * result from inference result. + * */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class Tensor extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Tensor(Pointer p) { super(p); } + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongPointer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector LongBuffer shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, data_type, shape, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::DataType") int data_type, + @Cast("int64_t*") @StdVector long[] shape, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + public Tensor( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") BytePointer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") ByteBuffer buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + public Tensor( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id) { super((Pointer)null); allocate(buffer, byte_size, memory_type, memory_type_id); } + private native void allocate( + @Cast("char*") byte[] buffer, @Cast("const size_t") long byte_size, @Cast("const triton::developer_tools::server::MemoryType") int memory_type, + @Cast("const int64_t") long memory_type_id); + + // The pointer to the start of the buffer. + public native @Cast("char*") BytePointer buffer_(); public native Tensor buffer_(BytePointer setter); + // The size of buffer in bytes. + public native @Cast("size_t") long byte_size_(); public native Tensor byte_size_(long setter); + // The data type of the tensor. + public native @Cast("triton::developer_tools::server::DataType") int data_type_(); public native Tensor data_type_(int setter); + // The shape of the tensor. + public native @Cast("int64_t*") @StdVector LongPointer shape_(); public native Tensor shape_(LongPointer setter); + // The memory type of the tensor. Valid memory types are "CPU", "CPU_PINNED" + // and "GPU". + public native @Cast("triton::developer_tools::server::MemoryType") int memory_type_(); public native Tensor memory_type_(int setter); + // The ID of the memory for the tensor. (e.g. '0' is the memory type id of + // 'GPU-0') + public native @Cast("int64_t") long memory_type_id_(); public native Tensor memory_type_id_(long setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java new file mode 100644 index 00000000000..253bd8c1a72 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java @@ -0,0 +1,66 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +/** Structure to hold global trace setting for 'ServerOptions' and + * model-specific trace setting for 'InferOptions'. See here for more + * information: + * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class Trace extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Trace(Pointer p) { super(p); } + + /** enum class triton::developer_tools::server::Trace::Level */ + public static final int OFF = 0, TIMESTAMPS = 1, TENSORS = 2; + + public Trace(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + public Trace(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level) { super((Pointer)null); allocate(file, level); } + private native void allocate(@StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level); + + public Trace( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString BytePointer file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + public Trace( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency) { super((Pointer)null); allocate(file, level, rate, count, log_frequency); } + private native void allocate( + @StdString String file, @Cast("const triton::developer_tools::server::Trace::Level") int level, @Cast("const uint32_t") int rate, + int count, @Cast("const uint32_t") int log_frequency); + + // The file where trace output will be saved. If 'log-frequency' is also + // specified, this argument value will be the prefix of the files to save the + // trace output. + public native @StdString BytePointer file_(); public native Trace file_(BytePointer setter); + // Specify a trace level. OFF to disable tracing, TIMESTAMPS to trace + // timestamps, TENSORS to trace tensors. + public native @Cast("triton::developer_tools::server::Trace::Level") int level_(); public native Trace level_(int setter); + // The trace sampling rate. The value represents how many requests will one + // trace be sampled from. For example, if the trace rate is "1000", 1 trace + // will be sampled for every 1000 requests. Default is 1000. + public native @Cast("uint32_t") int rate_(); public native Trace rate_(int setter); + // The number of traces to be sampled. If the value is -1, the number of + // traces to be sampled will not be limited. Default is -1. + public native int count_(); public native Trace count_(int setter); + // The trace log frequency. If the value is 0, Triton will only log the trace + // output to 'file_' when shutting down. Otherwise, Triton will log the trace + // output to 'file_.' when it collects the specified number of traces. + // For example, if the log frequency is 100, when Triton collects the 100-th + // trace, it logs the traces to file 'file_.0', and when it collects the + // 200-th trace, it logs the 101-th to the 200-th traces to file file_.1'. + // Default is 0. + public native @Cast("uint32_t") int log_frequency_(); public native Trace log_frequency_(int setter); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java new file mode 100644 index 00000000000..279ec2ac07a --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java @@ -0,0 +1,29 @@ +// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +//============================================================================== +// TritonException +// +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TritonException extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TritonException(Pointer p) { super(p); } + + public TritonException(@StdString BytePointer message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString BytePointer message); + public TritonException(@StdString String message) { super((Pointer)null); allocate(message); } + private native void allocate(@StdString String message); + + public native String what(); + + public native @StdString BytePointer message_(); public native TritonException message_(BytePointer setter); +} From 264ab6343e89c4149379c5165c169d0c40c5dda3 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 31 May 2023 19:10:20 -0700 Subject: [PATCH 28/50] add cppbuild --- tritonserver/cppbuild.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 64f033033ff..a3324413984 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -9,13 +9,13 @@ fi case $PLATFORM in linux-arm64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then echo "Please make sure library and include files exist" exit 1 fi ;; linux-x86_64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then echo "Please make sure library and include files exist" exit 1 fi From ae6bb7e3d50af2e27a2133f156a04788ce6bdfa2 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 1 Jun 2023 14:22:48 -0700 Subject: [PATCH 29/50] adding bindings build --- tritonserver/cppbuild.sh | 41 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index a3324413984..76908664e89 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -7,15 +7,52 @@ if [[ -z "$PLATFORM" ]]; then exit fi +if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then + CMAKE_VERSION=${CMAKE_VERSION:="3.21.1"} + TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} + TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} + TRITON_HOME="/opt/tritonserver" + BUILD_HOME="$PWD"/tritonbuild + mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME} + # install cmake and rapidjson + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ + gpg --dearmor - | \ + tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + cmake-data=${CMAKE_VERSION}-0kitware1ubuntu20.04.1 \ + cmake=${CMAKE_VERSION}-0kitware1ubuntu20.04.1 \ + rapidjson-dev + + git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} + cd developer_tools/server + rm -r build + mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_BUILD_STATIC_LIBRARY=OFF .. + make -j"$(grep -c ^processor /proc/cpuinfo)" install + # Copy dynamic library to triton home + cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. + + rm -r ${TRITON_HOME}/include/triton/developer_tools + mkdir -p ${TRITON_HOME}/include/triton/developer_tools/src + cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/common.h ${TRITON_HOME}/include/triton/developer_tools/. + cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/generic_server_wrapper.h ${TRITON_HOME}/include/triton/developer_tools/. + cp ${BUILD_HOME}/developer_tools/server/src/infer_requested_output.h ${TRITON_HOME}/include/triton/developer_tools/src/. + cp ${BUILD_HOME}/developer_tools/server/src/tracer.h ${TRITON_HOME}/include/triton/developer_tools/src/ + cd ${BUILD_HOME}/.. + rm -r ${BUILD_HOME} +fi + case $PLATFORM in linux-arm64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then echo "Please make sure library and include files exist" exit 1 fi ;; linux-x86_64) - if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]] && [[ ! -d "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then + if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then echo "Please make sure library and include files exist" exit 1 fi From 51e7d1dce59c9fac92bdd9aa9f3fc71fc5d08999 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 1 Jun 2023 14:50:41 -0700 Subject: [PATCH 30/50] remove extra removes --- tritonserver/cppbuild.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 76908664e89..7e22ca8cf7c 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -27,14 +27,12 @@ if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrap git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} cd developer_tools/server - rm -r build mkdir build && cd build cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_BUILD_STATIC_LIBRARY=OFF .. make -j"$(grep -c ^processor /proc/cpuinfo)" install # Copy dynamic library to triton home cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. - rm -r ${TRITON_HOME}/include/triton/developer_tools mkdir -p ${TRITON_HOME}/include/triton/developer_tools/src cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/common.h ${TRITON_HOME}/include/triton/developer_tools/. cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/generic_server_wrapper.h ${TRITON_HOME}/include/triton/developer_tools/. From 1b1416676436bda5ed92b38688cb35d91494c9ff Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 1 Jun 2023 17:54:46 -0700 Subject: [PATCH 31/50] add back windows includepath --- .../org/bytedeco/tritonserver/presets/tritonserver.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 08d9d562f3c..1409f1d76db 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -45,6 +45,12 @@ includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), + @Platform( + value = "windows-x86_64", + includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", + linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", + preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" + ) }, target = "org.bytedeco.tritonserver.tritonserver", global = "org.bytedeco.tritonserver.global.tritonserver" From 313720a509758175ced799faeaf82ad4ba9275ef Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Mon, 5 Jun 2023 12:07:04 -0700 Subject: [PATCH 32/50] updated versioning --- .github/workflows/tritonserver.yml | 2 +- README.md | 2 +- platform/pom.xml | 2 +- tritonserver/README.md | 10 +++++----- tritonserver/platform/pom.xml | 2 +- tritonserver/platform/redist/pom.xml | 2 +- tritonserver/pom.xml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tritonserver.yml b/.github/workflows/tritonserver.yml index 81407e6a0f3..cfcfcc33b71 100644 --- a/.github/workflows/tritonserver.yml +++ b/.github/workflows/tritonserver.yml @@ -19,6 +19,6 @@ env: jobs: linux-x86_64: runs-on: ubuntu-20.04 - container: nvcr.io/nvidia/tritonserver:23.04-py3 + container: nvcr.io/nvidia/tritonserver:23.05-py3 steps: - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions diff --git a/README.md b/README.md index e754ac30489..d9ccbeed188 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ Each child module in turn relies by default on the included [`cppbuild.sh` scrip * TensorFlow 1.15.x https://github.com/tensorflow/tensorflow * TensorFlow Lite 2.12.x https://github.com/tensorflow/tensorflow * TensorRT 8.6.x https://developer.nvidia.com/tensorrt - * Triton Inference Server 2.33.x https://developer.nvidia.com/nvidia-triton-inference-server + * Triton Inference Server 2.34.x https://developer.nvidia.com/nvidia-triton-inference-server * The Arcade Learning Environment 0.8.x https://github.com/mgbellemare/Arcade-Learning-Environment * DepthAI 2.21.x https://github.com/luxonis/depthai-core * ONNX 1.14.x https://github.com/onnx/onnx diff --git a/platform/pom.xml b/platform/pom.xml index 44907ba6a54..a5b8d895599 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -311,7 +311,7 @@ org.bytedeco tritonserver-platform - 2.33-${project.version} + 2.34-${project.version} org.bytedeco diff --git a/tritonserver/README.md b/tritonserver/README.md index 32e7e5ec030..9d5608486b4 100644 --- a/tritonserver/README.md +++ b/tritonserver/README.md @@ -23,7 +23,7 @@ Introduction ------------ This directory contains the JavaCPP Presets module for: - * Triton Inference Server 2.33.0 https://github.com/triton-inference-server/server + * Triton Inference Server 2.34.0 https://github.com/triton-inference-server/server Please refer to the parent README.md file for more detailed information about the JavaCPP Presets. @@ -51,9 +51,9 @@ This sample intends to show how to call the Java-mapped C API of Triton to execu 1. Get the source code of Triton Inference Server to prepare the model repository: ```bash - $ wget https://github.com/triton-inference-server/server/archive/refs/tags/v2.33.0.tar.gz - $ tar zxvf v2.33.0.tar.gz - $ cd server-2.33.0/docs/examples/model_repository + $ wget https://github.com/triton-inference-server/server/archive/refs/tags/v2.34.0.tar.gz + $ tar zxvf v2.34.0.tar.gz + $ cd server-2.34.0/docs/examples/model_repository $ mkdir models $ cd models; cp -a ../simple . ``` @@ -61,7 +61,7 @@ Now, this `models` directory will be our model repository. 2. Start the Docker container to run the sample (assuming we are under the `models` directory created above): ```bash - $ docker run -it --gpus=all -v $(pwd):/workspace nvcr.io/nvidia/tritonserver:23.04-py3 bash + $ docker run -it --gpus=all -v $(pwd):/workspace nvcr.io/nvidia/tritonserver:23.05-py3 bash $ apt update $ apt install -y openjdk-11-jdk $ wget https://archive.apache.org/dist/maven/maven-3/3.8.4/binaries/apache-maven-3.8.4-bin.tar.gz diff --git a/tritonserver/platform/pom.xml b/tritonserver/platform/pom.xml index 836b4ca334e..d9f96e38297 100644 --- a/tritonserver/platform/pom.xml +++ b/tritonserver/platform/pom.xml @@ -12,7 +12,7 @@ org.bytedeco tritonserver-platform - 2.33-${project.parent.version} + 2.34-${project.parent.version} JavaCPP Presets Platform for Triton Inference Server diff --git a/tritonserver/platform/redist/pom.xml b/tritonserver/platform/redist/pom.xml index e9b1410d580..a34483df74e 100644 --- a/tritonserver/platform/redist/pom.xml +++ b/tritonserver/platform/redist/pom.xml @@ -12,7 +12,7 @@ org.bytedeco tritonserver-platform-redist - 2.33-${project.parent.version} + 2.34-${project.parent.version} JavaCPP Presets Platform Redist for Triton Inference Server diff --git a/tritonserver/pom.xml b/tritonserver/pom.xml index dd07cc91c9c..70eadb4560d 100644 --- a/tritonserver/pom.xml +++ b/tritonserver/pom.xml @@ -11,7 +11,7 @@ org.bytedeco tritonserver - 2.33-${project.parent.version} + 2.34-${project.parent.version} JavaCPP Presets for Triton Inference Server From 4689c330e4ad3e73748d419ffb83e84332649dca Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Mon, 5 Jun 2023 13:03:23 -0700 Subject: [PATCH 33/50] fixing ci --- .github/actions/deploy-ubuntu/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index 75f5275f961..0aba82bfb92 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -158,6 +158,11 @@ runs: ln -sf /usr/local/TensorRT* /usr/local/tensorrt fi + if [[ "$CI_DEPLOY_MODULE" == "tritonserver" ]]; then + echo for tritonserver all mentions of python is python3 + apt install python-is-python3 + fi + if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then echo Installing FlyCapture amdhf python3 -m gdown.cli https://drive.google.com/uc?id=16NuUBs2MXQpVYqzDCEr9KdMng-6rHuDI From 20f0954afdfb45bd6265b0b76360e0db28deb384 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Mon, 5 Jun 2023 15:08:17 -0700 Subject: [PATCH 34/50] fix build --- .github/actions/deploy-ubuntu/action.yml | 8 ++++---- .github/workflows/tritonserver.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index 0aba82bfb92..a477af88278 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -157,10 +157,10 @@ runs: tar -hxvf TensorRT-8.6.1.6.Ubuntu-20.04.aarch64-gnu.cuda-12.0.tar.gz -C /usr/local/ ln -sf /usr/local/TensorRT* /usr/local/tensorrt fi - - if [[ "$CI_DEPLOY_MODULE" == "tritonserver" ]]; then - echo for tritonserver all mentions of python is python3 - apt install python-is-python3 + + if [[ "$CI_DEPLOY_MODULE" == "tritonserver" ]]; then + echo For tritonserver all mentions of python is python3 + ln -s /usr/bin/python3 /usr/bin/python fi if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then diff --git a/.github/workflows/tritonserver.yml b/.github/workflows/tritonserver.yml index cfcfcc33b71..c1383bb384c 100644 --- a/.github/workflows/tritonserver.yml +++ b/.github/workflows/tritonserver.yml @@ -21,4 +21,4 @@ jobs: runs-on: ubuntu-20.04 container: nvcr.io/nvidia/tritonserver:23.05-py3 steps: - - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions + - uses: ./.github/actions/deploy-ubuntu From 3cea6cada00fbe69c6085a7a011e9f99881d957b Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Mon, 5 Jun 2023 18:49:57 -0700 Subject: [PATCH 35/50] adding local repo by adding actions/checkout --- .github/workflows/tritonserver.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tritonserver.yml b/.github/workflows/tritonserver.yml index c1383bb384c..66402e860b0 100644 --- a/.github/workflows/tritonserver.yml +++ b/.github/workflows/tritonserver.yml @@ -21,4 +21,5 @@ jobs: runs-on: ubuntu-20.04 container: nvcr.io/nvidia/tritonserver:23.05-py3 steps: + - uses: actions/checkout@v3 - uses: ./.github/actions/deploy-ubuntu From 22ae05b5ce4b346ef837ab83ff58643eef35c7a6 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Tue, 6 Jun 2023 11:03:26 -0700 Subject: [PATCH 36/50] move comment to earlier --- .github/actions/deploy-ubuntu/action.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index a477af88278..cfcaeffbe47 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -63,6 +63,12 @@ runs: echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME-security main restricted universe multiverse >> /etc/apt/sources.list TOOLCHAIN="gcc-$PREFIX g++-$PREFIX gfortran-$PREFIX linux-libc-dev-$ARCH-cross binutils-multiarch" fi + + if [[ "$CI_DEPLOY_MODULE" == "tritonserver" ]]; then + echo For tritonserver all mentions of python is python3 for Ubuntu 20.04 + ln -s /usr/bin/python3 /usr/bin/python + fi + echo deb [arch=amd64] http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $CODENAME main >> /etc/apt/sources.list echo deb [arch=amd64] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu $CODENAME main >> /etc/apt/sources.list sed -i 's/deb http/deb [arch=i386,amd64] http/g' /etc/apt/sources.list @@ -158,11 +164,6 @@ runs: ln -sf /usr/local/TensorRT* /usr/local/tensorrt fi - if [[ "$CI_DEPLOY_MODULE" == "tritonserver" ]]; then - echo For tritonserver all mentions of python is python3 - ln -s /usr/bin/python3 /usr/bin/python - fi - if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then echo Installing FlyCapture amdhf python3 -m gdown.cli https://drive.google.com/uc?id=16NuUBs2MXQpVYqzDCEr9KdMng-6rHuDI From 77c5aebf5a415622f9c96a9a59bffd8fc1701b69 Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Wed, 7 Jun 2023 09:32:47 +0900 Subject: [PATCH 37/50] Update action, workflow, version, and CHANGELOG.md --- .github/workflows/tritonserver.yml | 3 +-- CHANGELOG.md | 3 +++ tritonserver/platform/pom.xml | 2 +- tritonserver/platform/redist/pom.xml | 2 +- tritonserver/pom.xml | 2 +- tritonserver/samples/simple/pom.xml | 4 ++-- tritonserver/samples/simplecpp/pom.xml | 4 ++-- tritonserver/samples/unsupported/pom.xml | 8 ++++---- .../org/bytedeco/tritonserver/global/tritonserver.java | 2 +- .../org/bytedeco/tritonserver/tritonserver/Allocator.java | 2 +- .../bytedeco/tritonserver/tritonserver/BackendConfig.java | 2 +- .../tritonserver/tritonserver/CUDAMemoryPoolByteSize.java | 2 +- .../tritonserver/tritonserver/GenericInferRequest.java | 2 +- .../tritonserver/tritonserver/GenericInferResult.java | 2 +- .../tritonserver/tritonserver/GenericTritonServer.java | 2 +- .../bytedeco/tritonserver/tritonserver/HostPolicy.java | 2 +- .../bytedeco/tritonserver/tritonserver/InferOptions.java | 2 +- .../tritonserver/tritonserver/LoggingOptions.java | 2 +- .../tritonserver/tritonserver/MetricsOptions.java | 2 +- .../tritonserver/tritonserver/ModelLoadGPULimit.java | 2 +- .../bytedeco/tritonserver/tritonserver/NewModelRepo.java | 2 +- .../tritonserver/OutputBufferReleaseFn_t.java | 2 +- .../tritonserver/tritonserver/RateLimitResource.java | 2 +- .../tritonserver/tritonserver/RepositoryIndex.java | 2 +- .../tritonserver/ResponseAllocatorAllocFn_t.java | 2 +- .../tritonserver/ResponseAllocatorStartFn_t.java | 2 +- .../bytedeco/tritonserver/tritonserver/ServerOptions.java | 2 +- .../org/bytedeco/tritonserver/tritonserver/StringSet.java | 2 +- .../bytedeco/tritonserver/tritonserver/StringVector.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Backend.java | 2 +- .../tritonserver/TRITONBACKEND_BackendAttribute.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Batcher.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Input.java | 2 +- .../tritonserver/TRITONBACKEND_MemoryManager.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Model.java | 2 +- .../tritonserver/TRITONBACKEND_ModelInstance.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Output.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Request.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_Response.java | 2 +- .../tritonserver/TRITONBACKEND_ResponseFactory.java | 2 +- .../tritonserver/tritonserver/TRITONBACKEND_State.java | 2 +- .../tritonserver/tritonserver/TRITONREPOAGENT_Agent.java | 2 +- .../tritonserver/TRITONREPOAGENT_AgentModel.java | 2 +- .../tritonserver/TRITONSERVER_BufferAttributes.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Error.java | 2 +- .../tritonserver/TRITONSERVER_InferenceRequest.java | 2 +- .../TRITONSERVER_InferenceRequestReleaseFn_t.java | 2 +- .../tritonserver/TRITONSERVER_InferenceResponse.java | 2 +- .../TRITONSERVER_InferenceResponseCompleteFn_t.java | 2 +- .../tritonserver/TRITONSERVER_InferenceTrace.java | 2 +- .../TRITONSERVER_InferenceTraceActivityFn_t.java | 2 +- .../TRITONSERVER_InferenceTraceReleaseFn_t.java | 2 +- .../TRITONSERVER_InferenceTraceTensorActivityFn_t.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Message.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Metric.java | 2 +- .../tritonserver/TRITONSERVER_MetricFamily.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Metrics.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Parameter.java | 2 +- .../tritonserver/TRITONSERVER_ResponseAllocator.java | 2 +- .../TRITONSERVER_ResponseAllocatorAllocFn_t.java | 2 +- ...RITONSERVER_ResponseAllocatorBufferAttributesFn_t.java | 2 +- .../TRITONSERVER_ResponseAllocatorQueryFn_t.java | 2 +- .../TRITONSERVER_ResponseAllocatorReleaseFn_t.java | 2 +- .../TRITONSERVER_ResponseAllocatorStartFn_t.java | 2 +- .../tritonserver/tritonserver/TRITONSERVER_Server.java | 2 +- .../tritonserver/TRITONSERVER_ServerOptions.java | 2 +- .../org/bytedeco/tritonserver/tritonserver/Tensor.java | 2 +- .../org/bytedeco/tritonserver/tritonserver/Trace.java | 2 +- .../tritonserver/tritonserver/TritonException.java | 2 +- 69 files changed, 76 insertions(+), 74 deletions(-) diff --git a/.github/workflows/tritonserver.yml b/.github/workflows/tritonserver.yml index 66402e860b0..cfcfcc33b71 100644 --- a/.github/workflows/tritonserver.yml +++ b/.github/workflows/tritonserver.yml @@ -21,5 +21,4 @@ jobs: runs-on: ubuntu-20.04 container: nvcr.io/nvidia/tritonserver:23.05-py3 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/deploy-ubuntu + - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions diff --git a/CHANGELOG.md b/CHANGELOG.md index b71bbb77310..e59c4fef93d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ + * Map new higher-level C++ API of Triton Inference Server ([pull #1361](https://github.com/bytedeco/javacpp-presets/pull/1361)) + * Upgrade presets for Triton Inference Server 2.34.0 + ### June 6, 2023 version 1.5.9 * Virtualize `nvinfer1::IGpuAllocator` from TensorRT to allow customization ([pull #1367](https://github.com/bytedeco/javacpp-presets/pull/1367)) * Add new `SampleJpegEncoder` code for nvJPEG module of CUDA ([pull #1365](https://github.com/bytedeco/javacpp-presets/pull/1365)) diff --git a/tritonserver/platform/pom.xml b/tritonserver/platform/pom.xml index d8b9ee95dc1..b55d3024caa 100644 --- a/tritonserver/platform/pom.xml +++ b/tritonserver/platform/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.9 + 1.5.10-SNAPSHOT ../../ diff --git a/tritonserver/platform/redist/pom.xml b/tritonserver/platform/redist/pom.xml index e4b1d2b66c3..e5c0166c62f 100644 --- a/tritonserver/platform/redist/pom.xml +++ b/tritonserver/platform/redist/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.9 + 1.5.10-SNAPSHOT ../../../ diff --git a/tritonserver/pom.xml b/tritonserver/pom.xml index 268826248f7..63814515a60 100644 --- a/tritonserver/pom.xml +++ b/tritonserver/pom.xml @@ -6,7 +6,7 @@ org.bytedeco javacpp-presets - 1.5.9 + 1.5.10-SNAPSHOT org.bytedeco diff --git a/tritonserver/samples/simple/pom.xml b/tritonserver/samples/simple/pom.xml index c9353ae43b1..69045de1082 100644 --- a/tritonserver/samples/simple/pom.xml +++ b/tritonserver/samples/simple/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.bytedeco.tritonserver simple - 1.5.9 + 1.5.10-SNAPSHOT Simple 1.8 @@ -12,7 +12,7 @@ org.bytedeco tritonserver-platform - 2.33-1.5.9 + 2.34-1.5.10-SNAPSHOT shaded diff --git a/tritonserver/samples/simplecpp/pom.xml b/tritonserver/samples/simplecpp/pom.xml index 89ee1bed827..57e53303f3f 100644 --- a/tritonserver/samples/simplecpp/pom.xml +++ b/tritonserver/samples/simplecpp/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.bytedeco.tritonserver simplecpp - 1.5.9-SNAPSHOT + 1.5.10-SNAPSHOT SimpleCPP 1.8 @@ -12,7 +12,7 @@ org.bytedeco tritonserver-platform - 2.35-1.5.9-SNAPSHOT + 2.34-1.5.10-SNAPSHOT shaded diff --git a/tritonserver/samples/unsupported/pom.xml b/tritonserver/samples/unsupported/pom.xml index 4a6a1cd627a..b5d3cdd4ed7 100644 --- a/tritonserver/samples/unsupported/pom.xml +++ b/tritonserver/samples/unsupported/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.bytedeco.tritonserver simplegpu - 1.5.9 + 1.5.10-SNAPSHOT SimpleGPU 1.8 @@ -13,17 +13,17 @@ org.bytedeco cuda-platform - 12.1-8.9-1.5.9 + 12.1-8.9-1.5.10-SNAPSHOT org.bytedeco tensorrt-platform - 8.6-1.5.9 + 8.6-1.5.10-SNAPSHOT org.bytedeco tritonserver-platform - 2.33-1.5.9 + 2.34-1.5.10-SNAPSHOT shaded diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index d3da0713474..bc44aaff9a2 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.global; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java index 617e4f3626f..3ff35673a1e 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java index b9387191f13..bbbb2325c75 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java index ab8cd68c875..80cff8a981a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java index 28e02e964a2..555b31918b5 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java index bafa00f9de4..9b53a86f0bb 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java index 5ea9df800d2..764eb7129a6 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java index f067927320d..ad3ff654813 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java index 7d31f1a6687..bf7e7dd97d8 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java index 24710b5ea6b..57b1ec711e9 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java index bb5e97381fa..34edb4d3d35 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java index cddc46f3cdc..b7b514e9985 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java index b8e23f0b3d0..eecd324399d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java index 7cd2be10c4c..15d3bf6d70e 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java index e6c595daae1..d03d03a9ffe 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java index de3317d60f7..86201c4d0b2 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java index 20d84c3dc83..e2301e79e0f 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java index 2ee9681a8d1..f56efb93efe 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java index 493f4e2612a..f3071467190 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java index 5836732ef4d..c9d0c663050 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java index 7590a66ad75..bf4795fde54 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Backend.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Backend.java index 3940be4c52b..5f0926cf923 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Backend.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Backend.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java index 9ac4d65a5d7..a91e0d52d77 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_BackendAttribute.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Batcher.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Batcher.java index 263b6749076..642040632fe 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Batcher.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Batcher.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Input.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Input.java index 2401529c4b9..8dd4f051bdc 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Input.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Input.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_MemoryManager.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_MemoryManager.java index c19dbc45a84..eca7818c8e7 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_MemoryManager.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_MemoryManager.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Model.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Model.java index 9b61ef4733d..637636c3b46 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Model.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Model.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java index 54fdcbe1f91..9c3f17caa4a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ModelInstance.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Output.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Output.java index d61b4faaca2..c5945f7b2a4 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Output.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Output.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Request.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Request.java index 5577e1dacfd..ed870561f5c 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Request.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Request.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Response.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Response.java index 0017384dfc9..6e1f0123d53 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Response.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_Response.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ResponseFactory.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ResponseFactory.java index 7afbad74e14..3b6d3017e3a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ResponseFactory.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_ResponseFactory.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_State.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_State.java index 7de0e4a81b1..e0b160e925d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_State.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONBACKEND_State.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_Agent.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_Agent.java index 310005ab105..61570aa6807 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_Agent.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_Agent.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_AgentModel.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_AgentModel.java index 3b5a67959d0..0dd3cd9dfca 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_AgentModel.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONREPOAGENT_AgentModel.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java index d031da05a48..b2d910deab6 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java index 1368bcb4246..ba435fddfc9 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java index 9fa50a45d1a..822f6fefed3 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequestReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequestReleaseFn_t.java index 956f0c1721e..e6f2fe1bae8 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequestReleaseFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequestReleaseFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java index d9f9e16473f..51b3f7d2f76 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponseCompleteFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponseCompleteFn_t.java index 4b1b8b9be02..0d5545548b9 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponseCompleteFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponseCompleteFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java index 90efafaaad8..cea3a1d29eb 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceActivityFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceActivityFn_t.java index fa2cb97bf58..98eaac1da4d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceActivityFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceActivityFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceReleaseFn_t.java index 688e5217238..6e1472eab1a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceReleaseFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceReleaseFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java index dff0d4a17df..853371b792f 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTraceTensorActivityFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java index 00fefb183d8..fd93930d5f1 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java index 7f0d0b2f942..80802b35316 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java index abc9ccb3423..8da148fda60 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java index b16bb2b2f36..628ccf828db 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java index 9eeae2c4fb2..6d5ca985417 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java index a8f996c77c6..a8f485e6058 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java index 750ce765c1a..d63a92b782b 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java index 12fb96e3e7d..35373cba1c9 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java index 7f630e6eba5..c1d38109c66 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java index 6bff932dbbb..559c8b6f31b 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java index 38628f0ad15..c0c08a84e2b 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java index bc129556ed9..f842acf8270 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java index 3e18767f450..27c38af6cdd 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java index cf05514516e..8fdbcd3d93e 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java index 253bd8c1a72..d8aec4c436d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java index 279ec2ac07a..9ed68a09a0d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java @@ -1,4 +1,4 @@ -// Targeted by JavaCPP version 1.5.9-SNAPSHOT: DO NOT EDIT THIS FILE +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE package org.bytedeco.tritonserver.tritonserver; From 6d15aefd91cf651ba9743a4ed97cf62d1142b562 Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Wed, 7 Jun 2023 10:32:03 +0900 Subject: [PATCH 38/50] Fix up action and build script --- .github/actions/deploy-ubuntu/action.yml | 27 ++++++++++-------------- tritonserver/cppbuild.sh | 16 ++------------ 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index ff4c5128311..5609c91a7fa 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -7,10 +7,13 @@ runs: env: GITHUB_EVENT_HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | - SUDO=sudo - if [[ "$USER" == "root" ]]; then - SUDO= - cd /root + export SUDO=$(which sudo) + echo "SUDO=$SUDO" >> $GITHUB_ENV + if [[ -z "$SUDO" ]]; then + echo "Fixing HOME to /root (was '$HOME')" + export HOME=/root + echo "HOME=$HOME" >> $GITHUB_ENV + cd $HOME fi mkdir -p .ccache @@ -78,14 +81,13 @@ runs: $SUDO echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME-security main restricted universe multiverse | $SUDO tee -a /etc/apt/sources.list TOOLCHAIN="gcc-$PREFIX g++-$PREFIX gfortran-$PREFIX linux-libc-dev-$ARCH-cross binutils-multiarch" fi - # $SUDO echo deb [arch=amd64] http://ppa.launchpad.net/openjdk-r/ppa/ubuntu $CODENAME main | $SUDO tee -a /etc/apt/sources.list # $SUDO echo deb [arch=amd64] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu $CODENAME main | $SUDO tee -a /etc/apt/sources.list # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB9B1D8886F44E2A # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA6932366A755776 $SUDO apt-get update $SUDO apt-get -y install gcc-multilib g++-multilib python3 python2.7 python3-minimal python2.7-minimal rpm libasound2-dev:$ARCH freeglut3-dev:$ARCH libfontconfig-dev:$ARCH libgtk2.0-dev:$ARCH libusb-dev:$ARCH libusb-1.0-0-dev:$ARCH libffi-dev:$ARCH libbz2-dev:$ARCH zlib1g-dev:$ARCH libxcb1-dev:$ARCH - $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk-headless ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev + $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk-headless ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev ln -sf /usr/lib/jvm/java-8-openjdk-amd64 /usr/lib/jvm/default-java find /usr/lib/jvm/default-java/ @@ -194,7 +196,7 @@ runs: curl -LO https://registrationcenter-download.intel.com/akdlm/IRC_NAS/cd17b7fe-500e-4305-a89b-bd5b42bfd9f8/l_onemkl_p_2023.1.0.46342_offline.sh $SUDO bash l_onemkl_p_2023.1.0.46342_offline.sh -s -a -s --eula accept fi - + if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then echo Installing FlyCapture amdhf python3 -m gdown.cli https://drive.google.com/uc?id=16NuUBs2MXQpVYqzDCEr9KdMng-6rHuDI @@ -263,17 +265,13 @@ runs: export MAKEJ=$(getconf _NPROCESSORS_ONLN) echo Fetching $GITHUB_REPOSITORY@$GITHUB_SHA + git config --global --add safe.directory '*' git init git fetch --depth 1 https://github.com/$GITHUB_REPOSITORY $GITHUB_SHA git checkout $GITHUB_SHA git submodule update --init --recursive git submodule foreach --recursive 'git reset --hard' - if [[ "$USER" == "root" ]]; then - echo "Fixing HOME to /root (was '$HOME')" - export HOME=/root - fi - if [[ -n ${CI_DEPLOY_NEED_GRADLE:-} ]]; then echo Executing Gradle $GRADLE_TASK ${{ matrix.options }} $CI_DEPLOY_OPTIONS on $MAKEJ processors export GRADLE_OPTIONS="-Dorg.gradle.jvmargs=-Xmx2048m -PjavacppPlatform=$CI_DEPLOY_PLATFORM -PjavacppPlatformExtension=${{ matrix.ext }} ${{ matrix.options }} $CI_DEPLOY_OPTIONS --info" @@ -307,7 +305,4 @@ runs: - name: Clean up shell: bash run: | - if [[ "$USER" == "root" ]]; then - cd /root - fi - rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*') + rm -Rf $(find $HOME/.m2/repository/ -name '*SNAPSHOT*') diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 7e22ca8cf7c..454ad96a089 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -8,27 +8,15 @@ if [[ -z "$PLATFORM" ]]; then fi if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then - CMAKE_VERSION=${CMAKE_VERSION:="3.21.1"} TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} TRITON_HOME="/opt/tritonserver" BUILD_HOME="$PWD"/tritonbuild mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME} - # install cmake and rapidjson - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - cmake-data=${CMAKE_VERSION}-0kitware1ubuntu20.04.1 \ - cmake=${CMAKE_VERSION}-0kitware1ubuntu20.04.1 \ - rapidjson-dev - - git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} + git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} cd developer_tools/server mkdir build && cd build - cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_BUILD_STATIC_LIBRARY=OFF .. + cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_BUILD_STATIC_LIBRARY=OFF .. make -j"$(grep -c ^processor /proc/cpuinfo)" install # Copy dynamic library to triton home cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. From 4f38b543a8aa45d6971f2f689cebb3b8d7a92395 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 9 Jun 2023 17:24:09 -0700 Subject: [PATCH 39/50] add developer tools into tritonserver --- .../presets/tritondevelopertoolsserver.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java new file mode 100644 index 00000000000..3a3089328db --- /dev/null +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2023 Katherine Yang, Baojun Liu, Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.tritonserver.presets; + +import java.util.List; +import org.bytedeco.javacpp.ClassProperties; +import org.bytedeco.javacpp.LoadEnabled; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; + +/** + * + * @author Katherine Yang, Baojun Liu, Samuel Audet + */ +@Properties( + value = { + @Platform( + value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, + include = {"common.h", "generic_server_wrapper.h"}, + link = {"tritondevelopertoolsserver"}, + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, + linkpath = {"/opt/tritonserver/lib/"} + ), + @Platform( + value = "windows-x86_64", + includepath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/include/triton/core/", + linkpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/lib/", + preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" + ) + }, + target = "org.bytedeco.tritonserver.tritonserver", + global = "org.bytedeco.tritonserver.global.tritonserver" +) +public class tritonserver implements InfoMapper { + static { Loader.checkVersion("org.bytedeco", "tritonserver"); } + public void map(InfoMap infoMap) { + infoMap.putFirst(new Info().enumerate(false)) + .put(new Info("bool").cast().valueTypes("boolean").pointerTypes("boolean[]", "BoolPointer")) + .put(new Info("const char").pointerTypes("String", "@Cast(\"const char*\") BytePointer")) + .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) + .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", + "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); + .put(new Info("std::set").pointerTypes("StringSet").define()) + .put(new Info("std::vector").pointerTypes("StringVector").define()) + .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) + .put(new Info("TritonServer").purify(false).virtualize()); + } +} From 63431c6b96b5a60e784a85c44a75811a396e9021 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 9 Jun 2023 17:24:23 -0700 Subject: [PATCH 40/50] add developer tools into tritonserver --- tritonserver/cppbuild.sh | 2 +- .../global/tritondevelopertoolsserver.java | 187 ++++++++ .../tritonserver/global/tritonserver.java | 417 +++++++++--------- .../Allocator.java | 6 +- .../BackendConfig.java | 6 +- .../CUDAMemoryPoolByteSize.java | 6 +- .../GenericInferRequest.java | 6 +- .../GenericInferResult.java | 6 +- .../GenericTritonServer.java | 6 +- .../HostPolicy.java | 6 +- .../InferOptions.java | 6 +- .../LoggingOptions.java | 6 +- .../MetricsOptions.java | 6 +- .../ModelLoadGPULimit.java | 6 +- .../NewModelRepo.java | 6 +- .../OutputBufferReleaseFn_t.java | 6 +- .../RateLimitResource.java | 6 +- .../RepositoryIndex.java | 6 +- .../ResponseAllocatorAllocFn_t.java | 6 +- .../ResponseAllocatorStartFn_t.java | 6 +- .../ServerOptions.java | 6 +- .../StringSet.java | 6 +- .../StringVector.java | 6 +- .../Tensor.java | 6 +- .../Trace.java | 6 +- .../TritonException.java | 6 +- ...ITONSERVER_ResponseAllocatorAllocFn_t.java | 63 --- ...ResponseAllocatorBufferAttributesFn_t.java | 47 -- ...ITONSERVER_ResponseAllocatorQueryFn_t.java | 49 -- ...ONSERVER_ResponseAllocatorReleaseFn_t.java | 42 -- ...ITONSERVER_ResponseAllocatorStartFn_t.java | 37 -- .../presets/tritondevelopertoolsserver.java | 10 +- .../tritonserver/presets/tritonserver.java | 12 +- tritonserver/src/main/java9/module-info.java | 3 + 34 files changed, 485 insertions(+), 522 deletions(-) create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/Allocator.java (87%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/BackendConfig.java (89%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/CUDAMemoryPoolByteSize.java (86%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/GenericInferRequest.java (92%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/GenericInferResult.java (93%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/GenericTritonServer.java (96%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/HostPolicy.java (91%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/InferOptions.java (96%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/LoggingOptions.java (95%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/MetricsOptions.java (92%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/ModelLoadGPULimit.java (85%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/NewModelRepo.java (91%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/OutputBufferReleaseFn_t.java (75%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/RateLimitResource.java (91%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/RepositoryIndex.java (91%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/ResponseAllocatorAllocFn_t.java (82%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/ResponseAllocatorStartFn_t.java (70%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/ServerOptions.java (98%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/StringSet.java (86%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/StringVector.java (94%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/Tensor.java (96%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/Trace.java (94%) rename tritonserver/src/gen/java/org/bytedeco/tritonserver/{tritonserver => tritondevelopertoolsserver}/TritonException.java (82%) delete mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java delete mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java delete mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java delete mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java delete mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 454ad96a089..d6f84d4d791 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -7,7 +7,7 @@ if [[ -z "$PLATFORM" ]]; then exit fi -if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]]; then +if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -ne 1 ]; then TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} TRITON_HOME="/opt/tritonserver" diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java new file mode 100644 index 00000000000..302d63cfcba --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java @@ -0,0 +1,187 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.global; + +import org.bytedeco.tritonserver.tritondevelopertoolsserver.*; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +public class tritondevelopertoolsserver extends org.bytedeco.tritonserver.presets.tritondevelopertoolsserver { + static { Loader.load(); } + +// Targeting ../tritondevelopertoolsserver/StringSet.java + + +// Targeting ../tritondevelopertoolsserver/StringVector.java + + +// Parsed from common.h + +// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include + +//============================================================================== +/** enum classes + * */ +/** enum class triton::developer_tools::server::ModelControlMode */ +public static final int NONE = 0, POLL = 1, EXPLICIT = 2; +/** enum class triton::developer_tools::server::MemoryType */ +public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; +/** enum class triton::developer_tools::server::DataType */ +public static final int + INVALID = 0, + BOOL = 1, + UINT8 = 2, + UINT16 = 3, + UINT32 = 4, + UINT64 = 5, + INT8 = 6, + INT16 = 7, + INT32 = 8, + INT64 = 9, + FP16 = 10, + FP32 = 11, + FP64 = 12, + BYTES = 13, + BF16 = 14; +/** enum class triton::developer_tools::server::ModelReadyState */ +public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; +// Targeting ../tritondevelopertoolsserver/TritonException.java + + +// Targeting ../tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritondevelopertoolsserver/OutputBufferReleaseFn_t.java + + +// Targeting ../tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java + + +// Targeting ../tritondevelopertoolsserver/LoggingOptions.java + + +// Targeting ../tritondevelopertoolsserver/MetricsOptions.java + + +// Targeting ../tritondevelopertoolsserver/RateLimitResource.java + + +// Targeting ../tritondevelopertoolsserver/ModelLoadGPULimit.java + + +// Targeting ../tritondevelopertoolsserver/Allocator.java + + +// Targeting ../tritondevelopertoolsserver/BackendConfig.java + + +// Targeting ../tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java + + +// Targeting ../tritondevelopertoolsserver/HostPolicy.java + + +// Targeting ../tritondevelopertoolsserver/Trace.java + + +// Targeting ../tritondevelopertoolsserver/ServerOptions.java + + +// Targeting ../tritondevelopertoolsserver/RepositoryIndex.java + + +// Targeting ../tritondevelopertoolsserver/Tensor.java + + +// Targeting ../tritondevelopertoolsserver/NewModelRepo.java + + +// Targeting ../tritondevelopertoolsserver/InferOptions.java + + + + // namespace triton::developer_tools::server + + +// Parsed from generic_server_wrapper.h + +// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #pragma once +// #include +// #include +// #include +// #include +// #include "../src/infer_requested_output.h" +// #include "../src/tracer.h" +// #include "common.h" + +/// +// Targeting ../tritondevelopertoolsserver/GenericTritonServer.java + + +// Targeting ../tritondevelopertoolsserver/GenericInferResult.java + + +// Targeting ../tritondevelopertoolsserver/GenericInferRequest.java + + + + // namespace triton::developer_tools::server + + +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index bc44aaff9a2..3da718c7005 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -11,12 +11,6 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver { static { Loader.load(); } -// Targeting ../tritonserver/StringSet.java - - -// Targeting ../tritonserver/StringVector.java - - // Parsed from tritonserver.h // Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -136,7 +130,7 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver public static final int TRITONSERVER_API_VERSION_MAJOR = 1; /// -public static final int TRITONSERVER_API_VERSION_MINOR = 22; +public static final int TRITONSERVER_API_VERSION_MINOR = 23; /** Get the TRITONBACKEND API version supported by the Triton shared * library. This value can be compared against the @@ -450,21 +444,119 @@ public static native String TRITONSERVER_ErrorCodeString( /// public static native String TRITONSERVER_ErrorMessage( TRITONSERVER_Error error); -// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java - - -// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java - - -// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java - - -// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java - - -// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java +/** TRITONSERVER_ResponseAllocator + * + * Object representing a memory allocator for output tensors in an + * inference response. + * +

+ * Type for allocation function that allocates a buffer to hold an + * output tensor. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param byte_size The size of the buffer to allocate. + * @param memory_type The type of memory that the caller prefers for + * the buffer allocation. + * @param memory_type_id The ID of the memory that the caller prefers + * for the buffer allocation. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Returns a pointer to the allocated memory. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @param actual_memory_type Returns the type of memory where the + * allocation resides. May be different than the type of memory + * requested by 'memory_type'. + * @param actual_memory_type_id Returns the ID of the memory where + * the allocation resides. May be different than the ID of the memory + * requested by 'memory_type_id'. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/** Type for allocation function that allocates a buffer to hold an + * output tensor with buffer attributes. The callback function must fill in the + * appropriate buffer attributes information related to this buffer. If set, + * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t + * function. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param buffer_attributes The buffer attributes associated with the buffer. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/** Type for function that is called to query the allocator's preferred memory + * type and memory type ID. As much as possible, the allocator should attempt + * to return the same memory_type and memory_type_id values that will be + * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. + * But the allocator is not required to do so. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor. This is optional + * and it should be set to nullptr to indicate that the tensor name has + * not determined. + * @param byte_size The expected size of the buffer. This is optional + * and it should be set to nullptr to indicate that the byte size has + * not determined. + * @param memory_type Acts as both input and output. On input gives + * the memory type preferred by the caller. Returns memory type preferred + * by the allocator, taken account of the caller preferred type. + * @param memory_type_id Acts as both input and output. On input gives + * the memory type ID preferred by the caller. Returns memory type ID preferred + * by the allocator, taken account of the caller preferred type ID. + * @return a TRITONSERVER_Error object if a failure occurs. */ +/** Type for function that is called when the server no longer holds + * any reference to a buffer allocated by + * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function + * is typically called when the response object associated with the + * buffer is deleted by TRITONSERVER_InferenceResponseDelete. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Pointer to the buffer to be freed. + * @param buffer_userp The user-specified value associated + * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. + * @param byte_size The size of the buffer. + * @param memory_type The type of memory holding the buffer. + * @param memory_type_id The ID of the memory holding the buffer. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting the release. If an error is returned Triton will not + * attempt to release the buffer again. */ + +/** Type for function that is called to indicate that subsequent + * allocation requests will refer to a new response. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @return a TRITONSERVER_Error object if a failure occurs. */ /** Create a new response allocator object. * @@ -520,14 +612,14 @@ public static native String TRITONSERVER_ErrorMessage( /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( @Cast("TRITONSERVER_ResponseAllocator**") PointerPointer allocator, - TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - TRITONSERVER_ResponseAllocatorStartFn_t start_fn); + @ByVal TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + @ByVal TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + @ByVal TRITONSERVER_ResponseAllocatorStartFn_t start_fn); public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( @ByPtrPtr TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - TRITONSERVER_ResponseAllocatorStartFn_t start_fn); + @ByVal TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + @ByVal TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + @ByVal TRITONSERVER_ResponseAllocatorStartFn_t start_fn); /** Set the buffer attributes function for a response allocator object. * The function will be called after alloc_fn to set the buffer attributes @@ -545,7 +637,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); + @ByVal TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); /** Set the query function to a response allocator object. Usually the * function will be called before alloc_fn to understand what is the @@ -563,7 +655,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferA /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetQueryFunction( TRITONSERVER_ResponseAllocator allocator, - TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); + @ByVal TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); /** Delete a response allocator. * @@ -1119,6 +1211,7 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelat * @param correlation_id The correlation ID. * @return a TRITONSERVER_Error indicating success or failure. */ +/// /// public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( TRITONSERVER_InferenceRequest inference_request, @@ -1127,7 +1220,9 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelat TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer correlation_id); -/** Get the priority for a request. The default is 0 indicating that +/** Deprecated. See TRITONSERVER_InferenceRequestPriorityUInt64 instead. + * + * Get the priority for a request. The default is 0 indicating that * the request does not specify a priority and so will use the * model's default priority. * @@ -1143,7 +1238,29 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] priority); -/** Set the priority for a request. The default is 0 indicating that +/** Get the priority for a request. The default is 0 indicating that + * the request does not specify a priority and so will use the + * model's default priority. + * + * @param inference_request The request object. + * @param priority Returns the priority level. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( + TRITONSERVER_InferenceRequest inference_request, + @Cast("uint64_t*") LongPointer priority); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( + TRITONSERVER_InferenceRequest inference_request, + @Cast("uint64_t*") LongBuffer priority); +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( + TRITONSERVER_InferenceRequest inference_request, + @Cast("uint64_t*") long[] priority); + +/** Deprecated. See TRITONSERVER_InferenceRequestSetPriorityUInt64 instead. + * + * Set the priority for a request. The default is 0 indicating that * the request does not specify a priority and so will use the * model's default priority. * @@ -1154,6 +1271,18 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( /// public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriority( TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int priority); + +/** Set the priority for a request. The default is 0 indicating that + * the request does not specify a priority and so will use the + * model's default priority. + * + * @param inference_request The request object. + * @param priority The priority level. + * @return a TRITONSERVER_Error indicating success or failure. */ + +/// +public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriorityUInt64( + TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long priority); /** Get the timeout for a request, in microseconds. The default is 0 * which indicates that the request has no timeout. @@ -3089,7 +3218,7 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; /// -public static final int TRITONBACKEND_API_VERSION_MINOR = 12; +public static final int TRITONBACKEND_API_VERSION_MINOR = 13; /** Get the TRITONBACKEND API version supported by Triton. This value * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and @@ -4512,13 +4641,39 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelState( * @param state The user state, or nullptr if no user state. * @return a TRITONSERVER_Error indicating success or failure. */ +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( + TRITONBACKEND_Model model, Pointer state); + +/** Report the memory usage of the model that will be released on + * TRITONBACKEND_ModelFinalize. The backend may call this function within the + * lifecycle of the TRITONBACKEND_Model object (between + * TRITONBACKEND_ModelInitialize and TRITONBACKEND_ModelFinalize) to report the + * latest usage. To report the memory usage of a model instance, + * see TRITONBACKEND_ModelInstanceReportMemoryUsage. + * + * @param model The model. + * @param usage The list of buffer attributes that records the memory usage, + * each entry should record the total memory usage of a given memory type and + * id. For example, if the model itself occupies 64 bytes on each of + * CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries + * set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the + * other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and + * may be released after the function returns. + * @param usage_size The number of entries in 'usage'. + * @return a TRITONSERVER_Error indicating success or failure. */ + /// /// /// /// -public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( - TRITONBACKEND_Model model, Pointer state); +public static native TRITONSERVER_Error TRITONBACKEND_ModelReportMemoryUsage( + TRITONBACKEND_Model model, @Cast("TRITONSERVER_BufferAttributes**") PointerPointer usage, + @Cast("uint32_t") int usage_size); +public static native TRITONSERVER_Error TRITONBACKEND_ModelReportMemoryUsage( + TRITONBACKEND_Model model, @ByPtrPtr TRITONSERVER_BufferAttributes usage, + @Cast("uint32_t") int usage_size); /** * TRITONBACKEND_ModelInstance @@ -4728,12 +4883,38 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( * @return a TRITONSERVER_Error indicating success or failure. */ /// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( + TRITONBACKEND_ModelInstance instance, Pointer state); + +/** Report the memory usage of the model instance that will be released on + * TRITONBACKEND_ModelInstanceFinalize. The backend may call this function + * within the lifecycle of the TRITONBACKEND_Model object (between + * TRITONBACKEND_ModelInstanceInitialize and + * TRITONBACKEND_ModelInstanceFinalize) to report the latest usage. To report + * the memory usage of the model, see TRITONBACKEND_ModelReportMemoryUsage. + * + * @param instance The model instance. + * @param usage The list of buffer attributes that records the memory usage, + * each entry should record the total memory usage of a given memory type and + * id. For example, if the instance itself occupies 64 bytes on each of + * CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries + * set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the + * other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and + * may be released after the function returns. + * @param usage_size The number of entries in 'usage'. + * @return a TRITONSERVER_Error indicating success or failure. */ + /// /// /// /// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( - TRITONBACKEND_ModelInstance instance, Pointer state); +/// +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportMemoryUsage( + TRITONBACKEND_ModelInstance instance, + @Cast("TRITONSERVER_BufferAttributes**") PointerPointer usage, @Cast("uint32_t") int usage_size); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportMemoryUsage( + TRITONBACKEND_ModelInstance instance, + @ByPtrPtr TRITONSERVER_BufferAttributes usage, @Cast("uint32_t") int usage_size); /** Record statistics for an inference request. * @@ -5598,170 +5779,4 @@ public static native TRITONSERVER_Error TRITONREPOAGENT_ModelAction( // #endif -// Parsed from common.h - -// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include - -//============================================================================== -/** enum classes - * */ -/** enum class triton::developer_tools::server::ModelControlMode */ -public static final int NONE = 0, POLL = 1, EXPLICIT = 2; -/** enum class triton::developer_tools::server::MemoryType */ -public static final int CPU = 0, CPU_PINNED = 1, GPU = 2; -/** enum class triton::developer_tools::server::DataType */ -public static final int - INVALID = 0, - BOOL = 1, - UINT8 = 2, - UINT16 = 3, - UINT32 = 4, - UINT64 = 5, - INT8 = 6, - INT16 = 7, - INT32 = 8, - INT64 = 9, - FP16 = 10, - FP32 = 11, - FP64 = 12, - BYTES = 13, - BF16 = 14; -/** enum class triton::developer_tools::server::ModelReadyState */ -public static final int UNKNOWN = 0, READY = 1, UNAVAILABLE = 2, LOADING = 3, UNLOADING = 4; -// Targeting ../tritonserver/TritonException.java - - -// Targeting ../tritonserver/ResponseAllocatorAllocFn_t.java - - -// Targeting ../tritonserver/OutputBufferReleaseFn_t.java - - -// Targeting ../tritonserver/ResponseAllocatorStartFn_t.java - - -// Targeting ../tritonserver/LoggingOptions.java - - -// Targeting ../tritonserver/MetricsOptions.java - - -// Targeting ../tritonserver/RateLimitResource.java - - -// Targeting ../tritonserver/ModelLoadGPULimit.java - - -// Targeting ../tritonserver/Allocator.java - - -// Targeting ../tritonserver/BackendConfig.java - - -// Targeting ../tritonserver/CUDAMemoryPoolByteSize.java - - -// Targeting ../tritonserver/HostPolicy.java - - -// Targeting ../tritonserver/Trace.java - - -// Targeting ../tritonserver/ServerOptions.java - - -// Targeting ../tritonserver/RepositoryIndex.java - - -// Targeting ../tritonserver/Tensor.java - - -// Targeting ../tritonserver/NewModelRepo.java - - -// Targeting ../tritonserver/InferOptions.java - - - - // namespace triton::developer_tools::server - - -// Parsed from generic_server_wrapper.h - -// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// #pragma once -// #include -// #include -// #include -// #include -// #include "../src/infer_requested_output.h" -// #include "../src/tracer.h" -// #include "common.h" - -/// -// Targeting ../tritonserver/GenericTritonServer.java - - -// Targeting ../tritonserver/GenericInferResult.java - - -// Targeting ../tritonserver/GenericInferRequest.java - - - - // namespace triton::developer_tools::server - - } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Allocator.java similarity index 87% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Allocator.java index 3ff35673a1e..77058f039c3 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Allocator.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Allocator.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; @@ -14,7 +14,7 @@ /** Custom Allocator object for providing custom functions for allocator. * If there is no custom allocator provided, will use the default allocator. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class Allocator extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/BackendConfig.java similarity index 89% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/BackendConfig.java index bbbb2325c75..8bc1181ae07 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/BackendConfig.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/BackendConfig.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * options. Please refer to the 'Command line options' section in the * documentation of each backend to see the options (e.g. Tensorflow Backend: * https://github.com/triton-inference-server/tensorflow_backend#command-line-options) */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class BackendConfig extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java similarity index 86% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java index 80cff8a981a..681f5237cec 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/CUDAMemoryPoolByteSize.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/CUDAMemoryPoolByteSize.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * data transfer between host and devices until it exceeds the specified byte * size. This will not affect the allocation conducted by the backend * frameworks. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class CUDAMemoryPoolByteSize extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferRequest.java similarity index 92% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferRequest.java index 555b31918b5..5f8b3329052 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferRequest.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferRequest.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Object that describes an inflight inference request. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class GenericInferRequest extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferResult.java similarity index 93% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferResult.java index 9b53a86f0bb..05499c34034 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericInferResult.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericInferResult.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** An interface for InferResult object to interpret the response to an * inference request. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class GenericInferResult extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericTritonServer.java similarity index 96% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericTritonServer.java index 764eb7129a6..e2d50b071f1 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/GenericTritonServer.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/GenericTritonServer.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Object that encapsulates in-process C API functionalities. * */ -@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class GenericTritonServer extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/HostPolicy.java similarity index 91% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/HostPolicy.java index ad3ff654813..f07ff2f93b2 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/HostPolicy.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/HostPolicy.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold host policy for setting 'ServerOptions'. * See here for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/optimization.md#host-policy. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class HostPolicy extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/InferOptions.java similarity index 96% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/InferOptions.java index bf7e7dd97d8..35f6d120e5d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/InferOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/InferOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold options for Inference Request. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class InferOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/LoggingOptions.java similarity index 95% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/LoggingOptions.java index 57b1ec711e9..465205a4813 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/LoggingOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/LoggingOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold logging options for setting 'ServerOptions'. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class LoggingOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/MetricsOptions.java similarity index 92% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/MetricsOptions.java index 34edb4d3d35..26ea776db9a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/MetricsOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/MetricsOptions.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold metrics options for setting 'ServerOptions'. * See here for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class MetricsOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ModelLoadGPULimit.java similarity index 85% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ModelLoadGPULimit.java index b7b514e9985..9f4d968ff2d 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ModelLoadGPULimit.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ModelLoadGPULimit.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * The limit on GPU memory usage is specified as a fraction. If model loading * on the device is requested and the current memory usage exceeds the limit, * the load will be rejected. If not specified, the limit will not be set. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class ModelLoadGPULimit extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/NewModelRepo.java similarity index 91% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/NewModelRepo.java index eecd324399d..c7ad8e9bc8c 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/NewModelRepo.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/NewModelRepo.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -15,7 +15,7 @@ * is used for calling 'TritonServer::RegisterModelRepo' for registering * model repository. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class NewModelRepo extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java similarity index 75% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java index 15d3bf6d70e..ccc66eef07c 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/OutputBufferReleaseFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/OutputBufferReleaseFn_t.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class OutputBufferReleaseFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RateLimitResource.java similarity index 91% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RateLimitResource.java index d03d03a9ffe..b0b6107fabe 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RateLimitResource.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RateLimitResource.java @@ -1,19 +1,19 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold rate limit resource for setting 'ServerOptions'. See here * for more information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/rate_limiter.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class RateLimitResource extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RepositoryIndex.java similarity index 91% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RepositoryIndex.java index 86201c4d0b2..01809738633 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/RepositoryIndex.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/RepositoryIndex.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Structure to hold repository index for 'ModelIndex' function. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class RepositoryIndex extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java similarity index 82% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java index e2301e79e0f..9566fd0a4c5 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorAllocFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorAllocFn_t.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Custom Response Allocator Callback function signatures. * */ -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class ResponseAllocatorAllocFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java similarity index 70% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java index f56efb93efe..f31bf3ab144 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ResponseAllocatorStartFn_t.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ResponseAllocatorStartFn_t.java @@ -1,16 +1,16 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; /// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class ResponseAllocatorStartFn_t extends FunctionPointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ServerOptions.java similarity index 98% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ServerOptions.java index f3071467190..478b4f7a495 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/ServerOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/ServerOptions.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== /** Server options that are used to initialize Triton Server. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class ServerOptions extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringSet.java similarity index 86% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringSet.java index c9d0c663050..a8309dcb15f 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringSet.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringSet.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; -@Name("std::set") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("std::set") @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class StringSet extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringVector.java similarity index 94% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringVector.java index bf4795fde54..3914d216f41 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/StringVector.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/StringVector.java @@ -1,14 +1,14 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; -@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("std::vector") @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class StringVector extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Tensor.java similarity index 96% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Tensor.java index 8fdbcd3d93e..b3d143dbd3a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Tensor.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Tensor.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * input/requested output to an inference request, and retrieving the output * result from inference result. * */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class Tensor extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Trace.java similarity index 94% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Trace.java index d8aec4c436d..49f3af12ce8 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/Trace.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/Trace.java @@ -1,12 +1,12 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== @@ -14,7 +14,7 @@ * model-specific trace setting for 'InferOptions'. See here for more * information: * https://github.com/triton-inference-server/server/blob/main/docs/user_guide/trace.md. */ -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class Trace extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/TritonException.java similarity index 82% rename from tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java rename to tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/TritonException.java index 9ed68a09a0d..1db2c7aa1dc 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TritonException.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritondevelopertoolsserver/TritonException.java @@ -1,18 +1,18 @@ // Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE -package org.bytedeco.tritonserver.tritonserver; +package org.bytedeco.tritonserver.tritondevelopertoolsserver; import java.nio.*; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.annotation.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; //============================================================================== // TritonException // -@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Namespace("triton::developer_tools::server") @NoOffset @Properties(inherit = org.bytedeco.tritonserver.presets.tritondevelopertoolsserver.class) public class TritonException extends Pointer { static { Loader.load(); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java deleted file mode 100644 index d63a92b782b..00000000000 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java +++ /dev/null @@ -1,63 +0,0 @@ -// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserver.tritonserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/** TRITONSERVER_ResponseAllocator - * - * Object representing a memory allocator for output tensors in an - * inference response. - * -

- * Type for allocation function that allocates a buffer to hold an - * output tensor. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param byte_size The size of the buffer to allocate. - * @param memory_type The type of memory that the caller prefers for - * the buffer allocation. - * @param memory_type_id The ID of the memory that the caller prefers - * for the buffer allocation. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Returns a pointer to the allocated memory. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @param actual_memory_type Returns the type of memory where the - * allocation resides. May be different than the type of memory - * requested by 'memory_type'. - * @param actual_memory_type_id Returns the ID of the memory where - * the allocation resides. May be different than the ID of the memory - * requested by 'memory_type_id'. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) -public class TRITONSERVER_ResponseAllocatorAllocFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorAllocFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorAllocFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id, Pointer userp, @Cast("void**") PointerPointer buffer, @Cast("void**") PointerPointer buffer_userp, - @Cast("TRITONSERVER_MemoryType*") IntPointer actual_memory_type, - @Cast("int64_t*") LongPointer actual_memory_type_id); -} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java deleted file mode 100644 index 35373cba1c9..00000000000 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java +++ /dev/null @@ -1,47 +0,0 @@ -// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserver.tritonserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/** Type for allocation function that allocates a buffer to hold an - * output tensor with buffer attributes. The callback function must fill in the - * appropriate buffer attributes information related to this buffer. If set, - * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t - * function. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param buffer_attributes The buffer attributes associated with the buffer. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) -public class TRITONSERVER_ResponseAllocatorBufferAttributesFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorBufferAttributesFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorBufferAttributesFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, String tensor_name, - TRITONSERVER_BufferAttributes buffer_attributes, Pointer userp, - Pointer buffer_userp); -} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java deleted file mode 100644 index c1d38109c66..00000000000 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java +++ /dev/null @@ -1,49 +0,0 @@ -// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserver.tritonserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/** Type for function that is called to query the allocator's preferred memory - * type and memory type ID. As much as possible, the allocator should attempt - * to return the same memory_type and memory_type_id values that will be - * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. - * But the allocator is not required to do so. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor. This is optional - * and it should be set to nullptr to indicate that the tensor name has - * not determined. - * @param byte_size The expected size of the buffer. This is optional - * and it should be set to nullptr to indicate that the byte size has - * not determined. - * @param memory_type Acts as both input and output. On input gives - * the memory type preferred by the caller. Returns memory type preferred - * by the allocator, taken account of the caller preferred type. - * @param memory_type_id Acts as both input and output. On input gives - * the memory type ID preferred by the caller. Returns memory type ID preferred - * by the allocator, taken account of the caller preferred type ID. - * @return a TRITONSERVER_Error object if a failure occurs. */ - -/// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) -public class TRITONSERVER_ResponseAllocatorQueryFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorQueryFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorQueryFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer userp, - String tensor_name, @Cast("size_t*") SizeTPointer byte_size, - @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); -} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java deleted file mode 100644 index 559c8b6f31b..00000000000 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java +++ /dev/null @@ -1,42 +0,0 @@ -// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserver.tritonserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/** Type for function that is called when the server no longer holds - * any reference to a buffer allocated by - * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function - * is typically called when the response object associated with the - * buffer is deleted by TRITONSERVER_InferenceResponseDelete. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Pointer to the buffer to be freed. - * @param buffer_userp The user-specified value associated - * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. - * @param byte_size The size of the buffer. - * @param memory_type The type of memory holding the buffer. - * @param memory_type_id The ID of the memory holding the buffer. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting the release. If an error is returned Triton will not - * attempt to release the buffer again. */ - -/// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) -public class TRITONSERVER_ResponseAllocatorReleaseFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorReleaseFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorReleaseFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, - @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, - @Cast("int64_t") long memory_type_id); -} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java deleted file mode 100644 index c0c08a84e2b..00000000000 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java +++ /dev/null @@ -1,37 +0,0 @@ -// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE - -package org.bytedeco.tritonserver.tritonserver; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.bytedeco.tritonserver.global.tritonserver.*; - - -/** Type for function that is called to indicate that subsequent - * allocation requests will refer to a new response. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @return a TRITONSERVER_Error object if a failure occurs. */ - -/// -/// -/// -/// -/// -/// -@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) -public class TRITONSERVER_ResponseAllocatorStartFn_t extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TRITONSERVER_ResponseAllocatorStartFn_t(Pointer p) { super(p); } - protected TRITONSERVER_ResponseAllocatorStartFn_t() { allocate(); } - private native void allocate(); - public native TRITONSERVER_Error call( - TRITONSERVER_ResponseAllocator allocator, Pointer userp); -} diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java index 3a3089328db..a1efb2a8c02 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java @@ -41,7 +41,7 @@ @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, include = {"common.h", "generic_server_wrapper.h"}, - link = {"tritondevelopertoolsserver"}, + link = {"tritonserver", "tritondevelopertoolsserver"}, includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, linkpath = {"/opt/tritonserver/lib/"} ), @@ -52,10 +52,10 @@ preloadpath = "C:/Program Files/NVIDIA GPU Computing Toolkit/TritonServer/bin/" ) }, - target = "org.bytedeco.tritonserver.tritonserver", - global = "org.bytedeco.tritonserver.global.tritonserver" + target = "org.bytedeco.tritonserver.tritondevelopertoolsserver", + global = "org.bytedeco.tritonserver.global.tritondevelopertoolsserver" ) -public class tritonserver implements InfoMapper { +public class tritondevelopertoolsserver implements InfoMapper { static { Loader.checkVersion("org.bytedeco", "tritonserver"); } public void map(InfoMap infoMap) { infoMap.putFirst(new Info().enumerate(false)) @@ -64,7 +64,7 @@ public void map(InfoMap infoMap) { .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) .put(new Info("std::set").pointerTypes("StringSet").define()) .put(new Info("std::vector").pointerTypes("StringVector").define()) .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 1409f1d76db..3e5508e1887 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -40,9 +40,9 @@ value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, - include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h", "common.h", "generic_server_wrapper.h"}, - link = {"tritonserver", "tritondevelopertoolsserver"}, - includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include", "/opt/tritonserver/include/triton/developer_tools", "/opt/tritonserver/include/triton/developer_tools/src"}, + include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, + link = {"tritonserver"}, + includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include"}, linkpath = {"/opt/tritonserver/lib/"} ), @Platform( @@ -64,10 +64,6 @@ public void map(InfoMap infoMap) { .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) - .put(new Info("std::set").pointerTypes("StringSet").define()) - .put(new Info("std::vector").pointerTypes("StringVector").define()) - .put(new Info("INT_MAX").javaNames("Integer.MAX_VALUE").define()) - .put(new Info("TritonServer").purify(false).virtualize()); + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); } } diff --git a/tritonserver/src/main/java9/module-info.java b/tritonserver/src/main/java9/module-info.java index 6a1b64a9d46..199f9393e24 100644 --- a/tritonserver/src/main/java9/module-info.java +++ b/tritonserver/src/main/java9/module-info.java @@ -3,4 +3,7 @@ exports org.bytedeco.tritonserver.global; exports org.bytedeco.tritonserver.presets; exports org.bytedeco.tritonserver; + + exports org.bytedeco.tritonserver.tritondevelopertoolsserver; + exports org.bytedeco.tritonserver.tritonserver; } From 43f46cb361b82d50fb4eac5bb630dbbb0e0a9645 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 9 Jun 2023 18:00:44 -0700 Subject: [PATCH 41/50] fix merge issue --- .github/actions/deploy-ubuntu/action.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index 01fd3f1159a..c0a405d06f4 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -86,11 +86,9 @@ runs: # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA6932366A755776 $SUDO apt-get update $SUDO apt-get -y install gcc-multilib g++-multilib python3 python2.7 python3-minimal python2.7-minimal rpm libasound2-dev:$ARCH freeglut3-dev:$ARCH libfontconfig-dev:$ARCH libgtk2.0-dev:$ARCH libusb-dev:$ARCH libusb-1.0-0-dev:$ARCH libffi-dev:$ARCH libbz2-dev:$ARCH zlib1g-dev:$ARCH libxcb1-dev:$ARCH -<<<<<<< HEAD - $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk-headless ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev -======= - $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev ->>>>>>> master + $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 + python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake + make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV @@ -308,9 +306,5 @@ runs: - name: Clean up shell: bash run: | -<<<<<<< HEAD - rm -Rf $(find $HOME/.m2/repository/ -name '*SNAPSHOT*') -======= cd $HOME - rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*') ->>>>>>> master + rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*') \ No newline at end of file From e90149b0d58d92353cd3b32666391a6bedb5e3ae Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 9 Jun 2023 19:39:27 -0700 Subject: [PATCH 42/50] adding pre-processing for struct for structs --- .github/actions/deploy-ubuntu/action.yml | 6 +- .../tritonserver/global/tritonserver.java | 105 ++---------------- .../TRITONSERVER_BufferAttributes.java | 2 +- .../tritonserver/TRITONSERVER_Error.java | 2 +- .../TRITONSERVER_InferenceRequest.java | 2 +- .../TRITONSERVER_InferenceResponse.java | 2 +- .../TRITONSERVER_InferenceTrace.java | 2 +- .../tritonserver/TRITONSERVER_Message.java | 2 +- .../tritonserver/TRITONSERVER_Metric.java | 2 +- .../TRITONSERVER_MetricFamily.java | 2 +- .../tritonserver/TRITONSERVER_Metrics.java | 2 +- .../tritonserver/TRITONSERVER_Parameter.java | 2 +- .../TRITONSERVER_ResponseAllocator.java | 2 +- .../tritonserver/TRITONSERVER_Server.java | 2 +- .../TRITONSERVER_ServerOptions.java | 2 +- .../tritonserver/presets/tritonserver.java | 15 ++- 16 files changed, 37 insertions(+), 115 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index c0a405d06f4..6c55bec0aaa 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -86,9 +86,7 @@ runs: # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA6932366A755776 $SUDO apt-get update $SUDO apt-get -y install gcc-multilib g++-multilib python3 python2.7 python3-minimal python2.7-minimal rpm libasound2-dev:$ARCH freeglut3-dev:$ARCH libfontconfig-dev:$ARCH libgtk2.0-dev:$ARCH libusb-dev:$ARCH libusb-1.0-0-dev:$ARCH libffi-dev:$ARCH libbz2-dev:$ARCH zlib1g-dev:$ARCH libxcb1-dev:$ARCH - $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 - python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake - make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev + $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake make libtool bison flex perl nasm curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV @@ -307,4 +305,4 @@ runs: shell: bash run: | cd $HOME - rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*') \ No newline at end of file + rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*') diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 3da718c7005..9a8a54e45f5 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -130,7 +130,7 @@ public class tritonserver extends org.bytedeco.tritonserver.presets.tritonserver public static final int TRITONSERVER_API_VERSION_MAJOR = 1; /// -public static final int TRITONSERVER_API_VERSION_MINOR = 23; +public static final int TRITONSERVER_API_VERSION_MINOR = 22; /** Get the TRITONBACKEND API version supported by the Triton shared * library. This value can be compared against the @@ -1211,7 +1211,6 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelat * @param correlation_id The correlation ID. * @return a TRITONSERVER_Error indicating success or failure. */ -/// /// public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelationIdString( TRITONSERVER_InferenceRequest inference_request, @@ -1220,9 +1219,7 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetCorrelat TRITONSERVER_InferenceRequest inference_request, @Cast("const char*") BytePointer correlation_id); -/** Deprecated. See TRITONSERVER_InferenceRequestPriorityUInt64 instead. - * - * Get the priority for a request. The default is 0 indicating that +/** Get the priority for a request. The default is 0 indicating that * the request does not specify a priority and so will use the * model's default priority. * @@ -1238,29 +1235,7 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriority( TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t*") int[] priority); -/** Get the priority for a request. The default is 0 indicating that - * the request does not specify a priority and so will use the - * model's default priority. - * - * @param inference_request The request object. - * @param priority Returns the priority level. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( - TRITONSERVER_InferenceRequest inference_request, - @Cast("uint64_t*") LongPointer priority); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( - TRITONSERVER_InferenceRequest inference_request, - @Cast("uint64_t*") LongBuffer priority); -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUInt64( - TRITONSERVER_InferenceRequest inference_request, - @Cast("uint64_t*") long[] priority); - -/** Deprecated. See TRITONSERVER_InferenceRequestSetPriorityUInt64 instead. - * - * Set the priority for a request. The default is 0 indicating that +/** Set the priority for a request. The default is 0 indicating that * the request does not specify a priority and so will use the * model's default priority. * @@ -1271,18 +1246,6 @@ public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestPriorityUIn /// public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriority( TRITONSERVER_InferenceRequest inference_request, @Cast("uint32_t") int priority); - -/** Set the priority for a request. The default is 0 indicating that - * the request does not specify a priority and so will use the - * model's default priority. - * - * @param inference_request The request object. - * @param priority The priority level. - * @return a TRITONSERVER_Error indicating success or failure. */ - -/// -public static native TRITONSERVER_Error TRITONSERVER_InferenceRequestSetPriorityUInt64( - TRITONSERVER_InferenceRequest inference_request, @Cast("uint64_t") long priority); /** Get the timeout for a request, in microseconds. The default is 0 * which indicates that the request has no timeout. @@ -3218,7 +3181,7 @@ public static native TRITONSERVER_Error TRITONSERVER_GetMetricKind( public static final int TRITONBACKEND_API_VERSION_MAJOR = 1; /// -public static final int TRITONBACKEND_API_VERSION_MINOR = 13; +public static final int TRITONBACKEND_API_VERSION_MINOR = 12; /** Get the TRITONBACKEND API version supported by Triton. This value * can be compared against the TRITONBACKEND_API_VERSION_MAJOR and @@ -4641,39 +4604,13 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelState( * @param state The user state, or nullptr if no user state. * @return a TRITONSERVER_Error indicating success or failure. */ -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( - TRITONBACKEND_Model model, Pointer state); - -/** Report the memory usage of the model that will be released on - * TRITONBACKEND_ModelFinalize. The backend may call this function within the - * lifecycle of the TRITONBACKEND_Model object (between - * TRITONBACKEND_ModelInitialize and TRITONBACKEND_ModelFinalize) to report the - * latest usage. To report the memory usage of a model instance, - * see TRITONBACKEND_ModelInstanceReportMemoryUsage. - * - * @param model The model. - * @param usage The list of buffer attributes that records the memory usage, - * each entry should record the total memory usage of a given memory type and - * id. For example, if the model itself occupies 64 bytes on each of - * CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries - * set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the - * other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and - * may be released after the function returns. - * @param usage_size The number of entries in 'usage'. - * @return a TRITONSERVER_Error indicating success or failure. */ - /// /// /// /// -public static native TRITONSERVER_Error TRITONBACKEND_ModelReportMemoryUsage( - TRITONBACKEND_Model model, @Cast("TRITONSERVER_BufferAttributes**") PointerPointer usage, - @Cast("uint32_t") int usage_size); -public static native TRITONSERVER_Error TRITONBACKEND_ModelReportMemoryUsage( - TRITONBACKEND_Model model, @ByPtrPtr TRITONSERVER_BufferAttributes usage, - @Cast("uint32_t") int usage_size); +public static native TRITONSERVER_Error TRITONBACKEND_ModelSetState( + TRITONBACKEND_Model model, Pointer state); /** * TRITONBACKEND_ModelInstance @@ -4882,39 +4819,13 @@ public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceState( * @param state The user state, or nullptr if no user state. * @return a TRITONSERVER_Error indicating success or failure. */ -/// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( - TRITONBACKEND_ModelInstance instance, Pointer state); - -/** Report the memory usage of the model instance that will be released on - * TRITONBACKEND_ModelInstanceFinalize. The backend may call this function - * within the lifecycle of the TRITONBACKEND_Model object (between - * TRITONBACKEND_ModelInstanceInitialize and - * TRITONBACKEND_ModelInstanceFinalize) to report the latest usage. To report - * the memory usage of the model, see TRITONBACKEND_ModelReportMemoryUsage. - * - * @param instance The model instance. - * @param usage The list of buffer attributes that records the memory usage, - * each entry should record the total memory usage of a given memory type and - * id. For example, if the instance itself occupies 64 bytes on each of - * CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries - * set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the - * other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and - * may be released after the function returns. - * @param usage_size The number of entries in 'usage'. - * @return a TRITONSERVER_Error indicating success or failure. */ - /// /// /// /// /// -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportMemoryUsage( - TRITONBACKEND_ModelInstance instance, - @Cast("TRITONSERVER_BufferAttributes**") PointerPointer usage, @Cast("uint32_t") int usage_size); -public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceReportMemoryUsage( - TRITONBACKEND_ModelInstance instance, - @ByPtrPtr TRITONSERVER_BufferAttributes usage, @Cast("uint32_t") int usage_size); +public static native TRITONSERVER_Error TRITONBACKEND_ModelInstanceSetState( + TRITONBACKEND_ModelInstance instance, Pointer state); /** Record statistics for an inference request. * diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java index b2d910deab6..eeff007a65a 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_BufferAttributes.java @@ -11,7 +11,7 @@ // #endif // #endif -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_BufferAttributes") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_BufferAttributes extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_BufferAttributes() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java index ba435fddfc9..17fea63dc8f 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Error.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Error") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Error extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Error() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java index 822f6fefed3..61f4b6223eb 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceRequest.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_InferenceRequest") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_InferenceRequest extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_InferenceRequest() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java index 51b3f7d2f76..e7e711d015b 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceResponse.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_InferenceResponse") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_InferenceResponse extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_InferenceResponse() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java index cea3a1d29eb..ac4f54cdaeb 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_InferenceTrace.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_InferenceTrace") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_InferenceTrace extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_InferenceTrace() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java index fd93930d5f1..8783e13f100 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Message.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Message") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Message extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Message() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java index 80802b35316..9e7c563793e 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metric.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Metric") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Metric extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Metric() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java index 8da148fda60..fe027a8d217 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_MetricFamily.java @@ -14,7 +14,7 @@ /// /// /// -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_MetricFamily") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_MetricFamily extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_MetricFamily() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java index 628ccf828db..b612287abb0 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Metrics.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Metrics") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Metrics extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Metrics() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java index 6d5ca985417..e797669d377 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Parameter.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Parameter") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Parameter extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Parameter() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java index a8f485e6058..0cb3c0f73d9 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocator.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_ResponseAllocator") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_ResponseAllocator extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_ResponseAllocator() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java index f842acf8270..75f65f56ecf 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_Server.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_Server") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_Server extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_Server() { super((Pointer)null); } diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java index 27c38af6cdd..52aa8888165 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ServerOptions.java @@ -8,7 +8,7 @@ import static org.bytedeco.tritonserver.global.tritonserver.*; -@Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +@Name("struct TRITONSERVER_ServerOptions") @Opaque @Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) public class TRITONSERVER_ServerOptions extends Pointer { /** Empty constructor. Calls {@code super((Pointer)null)}. */ public TRITONSERVER_ServerOptions() { super((Pointer)null); } diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 3e5508e1887..2639995be6c 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -64,6 +64,19 @@ public void map(InfoMap infoMap) { .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + .put(new Info("struct TRITONSERVER_BufferAttributes").cppText("TRITONSERVER_BufferAttributes")) + .put(new Info("struct TRITONSERVER_Error").cppText("TRITONSERVER_Error")) + .put(new Info("struct TRITONSERVER_InferenceRequest").cppText("TRITONSERVER_InferenceRequest")) + .put(new Info("struct TRITONSERVER_InferenceResponse").cppText("TRITONSERVER_InferenceResponse")) + .put(new Info("struct TRITONSERVER_InferenceTrace").cppText("TRITONSERVER_InferenceTrace")) + .put(new Info("struct TRITONSERVER_Message").cppText("TRITONSERVER_Message")) + .put(new Info("struct TRITONSERVER_Metrics").cppText("TRITONSERVER_Metrics")) + .put(new Info("struct TRITONSERVER_Parameter").cppText("TRITONSERVER_Parameter")) + .put(new Info("struct TRITONSERVER_ResponseAllocator").cppText("TRITONSERVER_ResponseAllocator")) + .put(new Info("struct TRITONSERVER_Server").cppText("TRITONSERVER_Server")) + .put(new Info("struct TRITONSERVER_ServerOptions").cppText("TRITONSERVER_ServerOptions")) + .put(new Info("struct TRITONSERVER_Metric").cppText("TRITONSERVER_Metric")) + .put(new Info("struct TRITONSERVER_MetricFamily").cppText("TRITONSERVER_MetricFamily")); } } From 43e85d0805b5209b7fdc13994d2cea01272b0451 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 14 Jun 2023 16:59:28 -0700 Subject: [PATCH 43/50] update cppbuild --- tritonserver/cppbuild.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index d6f84d4d791..9a171c64616 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -7,7 +7,9 @@ if [[ -z "$PLATFORM" ]]; then exit fi -if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -ne 1 ]; then +INCLUDE_DEVELOPER_TOOLS_SERVER=${INCLUDE_DEVELOPER_TOOLS_SERVER:=1} + +if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 1 ]; then TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} TRITON_HOME="/opt/tritonserver" From 8abd627412c1cc905dfeec6b43ea8da244d4c433 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 14 Jun 2023 20:00:35 -0700 Subject: [PATCH 44/50] recompile after merging main and fixing cppbuild --- tritonserver/samples/simple/Simple.java | 1 + tritonserver/samples/simplecpp/SimpleCPP.java | 109 +++++++------- .../global/tritondevelopertoolsserver.java | 2 +- .../tritonserver/global/tritonserver.java | 140 +++--------------- ...ITONSERVER_ResponseAllocatorAllocFn_t.java | 63 ++++++++ ...ResponseAllocatorBufferAttributesFn_t.java | 47 ++++++ ...ITONSERVER_ResponseAllocatorQueryFn_t.java | 49 ++++++ ...ONSERVER_ResponseAllocatorReleaseFn_t.java | 42 ++++++ ...ITONSERVER_ResponseAllocatorStartFn_t.java | 37 +++++ 9 files changed, 313 insertions(+), 177 deletions(-) create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java create mode 100644 tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java diff --git a/tritonserver/samples/simple/Simple.java b/tritonserver/samples/simple/Simple.java index 4813daf5fc3..769db7903f0 100644 --- a/tritonserver/samples/simple/Simple.java +++ b/tritonserver/samples/simple/Simple.java @@ -370,6 +370,7 @@ output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data), "getting Triton API version"); if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) || (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) { + System.out.println("MAJOR " + api_version_major[0] + ", MINOR " + api_version_minor[0]); FAIL("triton server API version mismatch"); } diff --git a/tritonserver/samples/simplecpp/SimpleCPP.java b/tritonserver/samples/simplecpp/SimpleCPP.java index 348737d695f..81bffc1f0a2 100644 --- a/tritonserver/samples/simplecpp/SimpleCPP.java +++ b/tritonserver/samples/simplecpp/SimpleCPP.java @@ -29,18 +29,16 @@ import java.util.concurrent.*; import com.google.gson.*; import org.bytedeco.javacpp.*; -import org.bytedeco.tritonserver.tritonserver.*; -import static org.bytedeco.tritonserver.global.tritonserver.*; - +import org.bytedeco.tritonserver.tritondevelopertoolsserver.*; +import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; public class SimpleCPP { // Helper functions static void FAIL(String MSG) { - System.err.println("Failure: " + MSG); - System.exit(1); + System.err.println("Failure: " + MSG); + System.exit(1); } - static void Usage(String msg) - { + static void Usage(String msg) { if (msg != null) { System.err.println(msg); } @@ -55,8 +53,7 @@ static void Usage(String msg) CompareResult( String output0_name, String output1_name, IntPointer input0, IntPointer input1, IntPointer output0, - IntPointer output1) - { + IntPointer output1) { for (int i = 0; i < 16; ++i) { System.out.println(input0.get(i) + " + " + input1.get(i) + " = " + output0.get(i)); @@ -74,8 +71,7 @@ static void Usage(String msg) static void GenerateInputData( - IntPointer[] input0_data, IntPointer[] input1_data) - { + IntPointer[] input0_data, IntPointer[] input1_data) { input0_data[0] = new IntPointer(16); input1_data[0] = new IntPointer(16); for (int i = 0; i < 16; ++i) { @@ -85,52 +81,51 @@ static void Usage(String msg) } static int RunInference(int verbose_level, String model_repository_path, String model_name) { - StringVector model_repository_paths = new StringVector(model_repository_path); - ServerOptions options = new ServerOptions(model_repository_paths); - LoggingOptions logging_options = options.logging_(); - logging_options.SetVerboseLevel(verbose_level); - options.SetLoggingOptions(logging_options); - - GenericTritonServer server = GenericTritonServer.Create(options); - StringSet loaded_models = server.LoadedModels(); - System.out.println("Loaded_models count : " + loaded_models.size()); - - InferOptions infer_options = new InferOptions(model_name); - GenericInferRequest request = GenericInferRequest.Create(infer_options); - - BytePointer input0_data; - BytePointer input1_data; - IntPointer[] p0 = {null}, p1 = {null}; - GenerateInputData(p0, p1); - input0_data = p0[0].getPointer(BytePointer.class); - input1_data = p1[0].getPointer(BytePointer.class); - - LongPointer shape0 = new LongPointer(2); - LongPointer shape1 = new LongPointer(2); - shape0.put(0, 1); - shape0.put(1, 16); - shape1.put(0, 1); - shape1.put(1, 16); - Tensor tensor0 = new Tensor(input0_data, 4 * 16, 8, shape0, 0, 1); - Tensor tensor1 = new Tensor(input1_data, 4 * 16, 8, shape1, 0, 1); - request.AddInput("INPUT0", tensor0); - request.AddInput("INPUT1", tensor1); - GenericInferResult result = server.Infer(request); - - Tensor output = result.Output("OUTPUT0"); - BytePointer buffer = output.buffer_(); - - System.out.println("buffer to string : " + buffer.toString()); - - System.out.println("output val at index 0 : " + buffer.getInt(0)); - System.out.println("output val at index 1 : " + buffer.getInt(1 * 4)); - System.out.println("output val at index 2 : " + buffer.getInt(2 * 4)); - System.out.println("output val at index 3 : " + buffer.getInt(3 * 4)); - System.out.println("output val at index 4 : " + buffer.getInt(4 * 4)); - System.out.println("output val at index 5 : " + buffer.getInt(5 * 4)); - System.out.println("output val at index 6 : " + buffer.getInt(6 * 4)); - System.out.println("output val at index 7 : " + buffer.getInt(7 * 4)); - return 0; + StringVector model_repository_paths = new StringVector(model_repository_path); + ServerOptions options = new ServerOptions(model_repository_paths); + LoggingOptions logging_options = options.logging_(); + logging_options.SetVerboseLevel(verbose_level); + options.SetLoggingOptions(logging_options); + + GenericTritonServer server = GenericTritonServer.Create(options); + StringSet loaded_models = server.LoadedModels(); + System.out.println("Loaded_models count : " + loaded_models.size()); + + InferOptions infer_options = new InferOptions(model_name); + GenericInferRequest request = GenericInferRequest.Create(infer_options); + + BytePointer input0_data; + BytePointer input1_data; + IntPointer[] p0 = {null}, p1 = {null}; + GenerateInputData(p0, p1); + input0_data = p0[0].getPointer(BytePointer.class); + input1_data = p1[0].getPointer(BytePointer.class); + + LongPointer shape0 = new LongPointer(2); + LongPointer shape1 = new LongPointer(2); + shape0.put(0, 1); + shape0.put(1, 16); + shape1.put(0, 1); + shape1.put(1, 16); + Tensor tensor0 = new Tensor(input0_data, 4 * 16, 8, shape0, 0, 1); + Tensor tensor1 = new Tensor(input1_data, 4 * 16, 8, shape1, 0, 1); + request.AddInput("INPUT0", tensor0); + request.AddInput("INPUT1", tensor1); + GenericInferResult result = server.Infer(request); + + Tensor output = result.Output("OUTPUT0"); + BytePointer buffer = output.buffer_(); + + System.out.println("buffer to string : " + buffer.toString()); + System.out.println("output val at index 0 : " + buffer.getInt(0)); + System.out.println("output val at index 1 : " + buffer.getInt(1 * 4)); + System.out.println("output val at index 2 : " + buffer.getInt(2 * 4)); + System.out.println("output val at index 3 : " + buffer.getInt(3 * 4)); + System.out.println("output val at index 4 : " + buffer.getInt(4 * 4)); + System.out.println("output val at index 5 : " + buffer.getInt(5 * 4)); + System.out.println("output val at index 6 : " + buffer.getInt(6 * 4)); + System.out.println("output val at index 7 : " + buffer.getInt(7 * 4)); + return 0; } public static void diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java index 302d63cfcba..d2cfd5bcf26 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritondevelopertoolsserver.java @@ -19,7 +19,7 @@ public class tritondevelopertoolsserver extends org.bytedeco.tritonserver.preset // Parsed from common.h -// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java index 9a8a54e45f5..1653e118b32 100644 --- a/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/global/tritonserver.java @@ -444,119 +444,21 @@ public static native String TRITONSERVER_ErrorCodeString( /// public static native String TRITONSERVER_ErrorMessage( TRITONSERVER_Error error); +// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java + + +// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java + + +// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java + + +// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java + + +// Targeting ../tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java -/** TRITONSERVER_ResponseAllocator - * - * Object representing a memory allocator for output tensors in an - * inference response. - * -

- * Type for allocation function that allocates a buffer to hold an - * output tensor. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param byte_size The size of the buffer to allocate. - * @param memory_type The type of memory that the caller prefers for - * the buffer allocation. - * @param memory_type_id The ID of the memory that the caller prefers - * for the buffer allocation. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Returns a pointer to the allocated memory. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @param actual_memory_type Returns the type of memory where the - * allocation resides. May be different than the type of memory - * requested by 'memory_type'. - * @param actual_memory_type_id Returns the ID of the memory where - * the allocation resides. May be different than the ID of the memory - * requested by 'memory_type_id'. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/** Type for allocation function that allocates a buffer to hold an - * output tensor with buffer attributes. The callback function must fill in the - * appropriate buffer attributes information related to this buffer. If set, - * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t - * function. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor to allocate for. - * @param buffer_attributes The buffer attributes associated with the buffer. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer_userp Returns a user-specified value to associate - * with the buffer, or nullptr if no user-specified value should be - * associated with the buffer. This value will be provided in the - * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer - * is released and will also be returned by - * TRITONSERVER_InferenceResponseOutput. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting an allocation. If an error is returned all other return - * values will be ignored. */ - -/** Type for function that is called to query the allocator's preferred memory - * type and memory type ID. As much as possible, the allocator should attempt - * to return the same memory_type and memory_type_id values that will be - * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. - * But the allocator is not required to do so. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param tensor_name The name of the output tensor. This is optional - * and it should be set to nullptr to indicate that the tensor name has - * not determined. - * @param byte_size The expected size of the buffer. This is optional - * and it should be set to nullptr to indicate that the byte size has - * not determined. - * @param memory_type Acts as both input and output. On input gives - * the memory type preferred by the caller. Returns memory type preferred - * by the allocator, taken account of the caller preferred type. - * @param memory_type_id Acts as both input and output. On input gives - * the memory type ID preferred by the caller. Returns memory type ID preferred - * by the allocator, taken account of the caller preferred type ID. - * @return a TRITONSERVER_Error object if a failure occurs. */ -/** Type for function that is called when the server no longer holds - * any reference to a buffer allocated by - * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function - * is typically called when the response object associated with the - * buffer is deleted by TRITONSERVER_InferenceResponseDelete. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param buffer Pointer to the buffer to be freed. - * @param buffer_userp The user-specified value associated - * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. - * @param byte_size The size of the buffer. - * @param memory_type The type of memory holding the buffer. - * @param memory_type_id The ID of the memory holding the buffer. - * @return a TRITONSERVER_Error object if a failure occurs while - * attempting the release. If an error is returned Triton will not - * attempt to release the buffer again. */ - -/** Type for function that is called to indicate that subsequent - * allocation requests will refer to a new response. - * - * @param allocator The allocator that is provided in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @param userp The user data pointer that is provided as - * 'response_allocator_userp' in the call to - * TRITONSERVER_InferenceRequestSetResponseCallback. - * @return a TRITONSERVER_Error object if a failure occurs. */ /** Create a new response allocator object. * @@ -612,14 +514,14 @@ public static native String TRITONSERVER_ErrorMessage( /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( @Cast("TRITONSERVER_ResponseAllocator**") PointerPointer allocator, - @ByVal TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - @ByVal TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - @ByVal TRITONSERVER_ResponseAllocatorStartFn_t start_fn); + TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + TRITONSERVER_ResponseAllocatorStartFn_t start_fn); public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( @ByPtrPtr TRITONSERVER_ResponseAllocator allocator, - @ByVal TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, - @ByVal TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, - @ByVal TRITONSERVER_ResponseAllocatorStartFn_t start_fn); + TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, + TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, + TRITONSERVER_ResponseAllocatorStartFn_t start_fn); /** Set the buffer attributes function for a response allocator object. * The function will be called after alloc_fn to set the buffer attributes @@ -637,7 +539,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorNew( /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( TRITONSERVER_ResponseAllocator allocator, - @ByVal TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); + TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); /** Set the query function to a response allocator object. Usually the * function will be called before alloc_fn to understand what is the @@ -655,7 +557,7 @@ public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetBufferA /// public static native TRITONSERVER_Error TRITONSERVER_ResponseAllocatorSetQueryFunction( TRITONSERVER_ResponseAllocator allocator, - @ByVal TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); + TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); /** Delete a response allocator. * diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java new file mode 100644 index 00000000000..d63a92b782b --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorAllocFn_t.java @@ -0,0 +1,63 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/** TRITONSERVER_ResponseAllocator + * + * Object representing a memory allocator for output tensors in an + * inference response. + * +

+ * Type for allocation function that allocates a buffer to hold an + * output tensor. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param byte_size The size of the buffer to allocate. + * @param memory_type The type of memory that the caller prefers for + * the buffer allocation. + * @param memory_type_id The ID of the memory that the caller prefers + * for the buffer allocation. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Returns a pointer to the allocated memory. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @param actual_memory_type Returns the type of memory where the + * allocation resides. May be different than the type of memory + * requested by 'memory_type'. + * @param actual_memory_type_id Returns the ID of the memory where + * the allocation resides. May be different than the ID of the memory + * requested by 'memory_type_id'. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONSERVER_ResponseAllocatorAllocFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorAllocFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorAllocFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id, Pointer userp, @Cast("void**") PointerPointer buffer, @Cast("void**") PointerPointer buffer_userp, + @Cast("TRITONSERVER_MemoryType*") IntPointer actual_memory_type, + @Cast("int64_t*") LongPointer actual_memory_type_id); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java new file mode 100644 index 00000000000..35373cba1c9 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorBufferAttributesFn_t.java @@ -0,0 +1,47 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/** Type for allocation function that allocates a buffer to hold an + * output tensor with buffer attributes. The callback function must fill in the + * appropriate buffer attributes information related to this buffer. If set, + * this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t + * function. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor to allocate for. + * @param buffer_attributes The buffer attributes associated with the buffer. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer_userp Returns a user-specified value to associate + * with the buffer, or nullptr if no user-specified value should be + * associated with the buffer. This value will be provided in the + * call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer + * is released and will also be returned by + * TRITONSERVER_InferenceResponseOutput. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting an allocation. If an error is returned all other return + * values will be ignored. */ + +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONSERVER_ResponseAllocatorBufferAttributesFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorBufferAttributesFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorBufferAttributesFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, String tensor_name, + TRITONSERVER_BufferAttributes buffer_attributes, Pointer userp, + Pointer buffer_userp); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java new file mode 100644 index 00000000000..c1d38109c66 --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorQueryFn_t.java @@ -0,0 +1,49 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/** Type for function that is called to query the allocator's preferred memory + * type and memory type ID. As much as possible, the allocator should attempt + * to return the same memory_type and memory_type_id values that will be + * returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. + * But the allocator is not required to do so. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param tensor_name The name of the output tensor. This is optional + * and it should be set to nullptr to indicate that the tensor name has + * not determined. + * @param byte_size The expected size of the buffer. This is optional + * and it should be set to nullptr to indicate that the byte size has + * not determined. + * @param memory_type Acts as both input and output. On input gives + * the memory type preferred by the caller. Returns memory type preferred + * by the allocator, taken account of the caller preferred type. + * @param memory_type_id Acts as both input and output. On input gives + * the memory type ID preferred by the caller. Returns memory type ID preferred + * by the allocator, taken account of the caller preferred type ID. + * @return a TRITONSERVER_Error object if a failure occurs. */ + +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONSERVER_ResponseAllocatorQueryFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorQueryFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorQueryFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer userp, + String tensor_name, @Cast("size_t*") SizeTPointer byte_size, + @Cast("TRITONSERVER_MemoryType*") IntPointer memory_type, @Cast("int64_t*") LongPointer memory_type_id); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java new file mode 100644 index 00000000000..559c8b6f31b --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorReleaseFn_t.java @@ -0,0 +1,42 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/** Type for function that is called when the server no longer holds + * any reference to a buffer allocated by + * TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function + * is typically called when the response object associated with the + * buffer is deleted by TRITONSERVER_InferenceResponseDelete. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param buffer Pointer to the buffer to be freed. + * @param buffer_userp The user-specified value associated + * with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. + * @param byte_size The size of the buffer. + * @param memory_type The type of memory holding the buffer. + * @param memory_type_id The ID of the memory holding the buffer. + * @return a TRITONSERVER_Error object if a failure occurs while + * attempting the release. If an error is returned Triton will not + * attempt to release the buffer again. */ + +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONSERVER_ResponseAllocatorReleaseFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorReleaseFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorReleaseFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer buffer, Pointer buffer_userp, + @Cast("size_t") long byte_size, @Cast("TRITONSERVER_MemoryType") int memory_type, + @Cast("int64_t") long memory_type_id); +} diff --git a/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java new file mode 100644 index 00000000000..c0c08a84e2b --- /dev/null +++ b/tritonserver/src/gen/java/org/bytedeco/tritonserver/tritonserver/TRITONSERVER_ResponseAllocatorStartFn_t.java @@ -0,0 +1,37 @@ +// Targeted by JavaCPP version 1.5.10-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.tritonserver.tritonserver; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.tritonserver.global.tritonserver.*; + + +/** Type for function that is called to indicate that subsequent + * allocation requests will refer to a new response. + * + * @param allocator The allocator that is provided in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @param userp The user data pointer that is provided as + * 'response_allocator_userp' in the call to + * TRITONSERVER_InferenceRequestSetResponseCallback. + * @return a TRITONSERVER_Error object if a failure occurs. */ + +/// +/// +/// +/// +/// +/// +@Properties(inherit = org.bytedeco.tritonserver.presets.tritonserver.class) +public class TRITONSERVER_ResponseAllocatorStartFn_t extends FunctionPointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public TRITONSERVER_ResponseAllocatorStartFn_t(Pointer p) { super(p); } + protected TRITONSERVER_ResponseAllocatorStartFn_t() { allocate(); } + private native void allocate(); + public native TRITONSERVER_Error call( + TRITONSERVER_ResponseAllocator allocator, Pointer userp); +} From 0efe1067413c3fa259908211c28990b291c97c8a Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Wed, 21 Jun 2023 19:18:43 -0700 Subject: [PATCH 45/50] update build script --- tritonserver/cppbuild.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 9a171c64616..c16a8ac0716 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -7,27 +7,35 @@ if [[ -z "$PLATFORM" ]]; then exit fi -INCLUDE_DEVELOPER_TOOLS_SERVER=${INCLUDE_DEVELOPER_TOOLS_SERVER:=1} +INCLUDE_DEVELOPER_TOOLS_SERVER=${INCLUDE_DEVELOPER_TOOLS_SERVER:=0} -if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 1 ]; then +if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [[ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 0 ]]; then TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} - TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} + TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="r23.05"} + TRITON_CORE_REPO=${TRITON_CORE_REPO:="https://github.com/triton-inference-server/core.git"} + TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG="r23.05"} TRITON_HOME="/opt/tritonserver" BUILD_HOME="$PWD"/tritonbuild mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME} + git clone --single-branch --depth=1 -b ${TRITON_CORE_REPO} ${TRITON_CORE_REPO_TAG} + cp core/include/triton/core/* ${TRITON_HOME}/include/triton/core/. + + # Build CPP library and copy over dependencies git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} cd developer_tools/server mkdir build && cd build - cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_BUILD_STATIC_LIBRARY=OFF .. + cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} -DTRITON_BUILD_STATIC_LIBRARY=OFF .. make -j"$(grep -c ^processor /proc/cpuinfo)" install # Copy dynamic library to triton home - cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. - - mkdir -p ${TRITON_HOME}/include/triton/developer_tools/src - cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/common.h ${TRITON_HOME}/include/triton/developer_tools/. - cp ${BUILD_HOME}/developer_tools/server/include/triton/developer_tools/generic_server_wrapper.h ${TRITON_HOME}/include/triton/developer_tools/. - cp ${BUILD_HOME}/developer_tools/server/src/infer_requested_output.h ${TRITON_HOME}/include/triton/developer_tools/src/. - cp ${BUILD_HOME}/developer_tools/server/src/tracer.h ${TRITON_HOME}/include/triton/developer_tools/src/ + cp ${BUILD_HOME}/developer_tools/server/build/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. + BUILD_INCLUDE_REPO=${BUILD_HOME}/developer_tools/server/include/triton/developer_tools + BUILD_SRC_REPO=${BUILD_HOME}/developer_tools/server/src + TRITON_INCLUDE_REPO=${TRITON_HOME}/include/triton/developer_tools + mkdir -p ${TRITON_INCLUDE_REPO}/src + cp ${BUILD_INCLUDE_REPO}/common.h ${TRITON_INCLUDE_REPO}/. + cp ${BUILD_INCLUDE_REPO}/generic_server_wrapper.h ${TRITON_INCLUDE_REPO}/. + cp ${BUILD_SRC_REPO}/infer_requested_output.h ${TRITON_INCLUDE_REPO}/src/. + cp ${BUILD_SRC_REPO}/tracer.h ${TRITON_INCLUDE_REPO}/src/. cd ${BUILD_HOME}/.. rm -r ${BUILD_HOME} fi From 6cd07b3daf69e0741a64f69c2d0d3be1cd9aefd4 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 22 Jun 2023 17:06:54 -0700 Subject: [PATCH 46/50] correct cppbuild clone command --- tritonserver/cppbuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index c16a8ac0716..59251f65d16 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -17,7 +17,7 @@ if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrap TRITON_HOME="/opt/tritonserver" BUILD_HOME="$PWD"/tritonbuild mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME} - git clone --single-branch --depth=1 -b ${TRITON_CORE_REPO} ${TRITON_CORE_REPO_TAG} + git clone --single-branch --depth=1 -b ${TRITON_CORE_REPO_TAG} ${TRITON_CORE_REPO} cp core/include/triton/core/* ${TRITON_HOME}/include/triton/core/. # Build CPP library and copy over dependencies From 1f1d64d18cd8b97b3d96b7cb95fb9a73c5aad2f2 Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Thu, 22 Jun 2023 17:54:54 -0700 Subject: [PATCH 47/50] ci From b02482d56bc10541e0df7d7885770963e5fae64e Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 23 Jun 2023 14:46:51 -0700 Subject: [PATCH 48/50] get ci to pass --- tritonserver/cppbuild.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 59251f65d16..8efb63d78e3 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -11,7 +11,7 @@ INCLUDE_DEVELOPER_TOOLS_SERVER=${INCLUDE_DEVELOPER_TOOLS_SERVER:=0} if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [[ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 0 ]]; then TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"} - TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="r23.05"} + TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"} TRITON_CORE_REPO=${TRITON_CORE_REPO:="https://github.com/triton-inference-server/core.git"} TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG="r23.05"} TRITON_HOME="/opt/tritonserver" @@ -27,7 +27,7 @@ if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrap cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} -DTRITON_BUILD_STATIC_LIBRARY=OFF .. make -j"$(grep -c ^processor /proc/cpuinfo)" install # Copy dynamic library to triton home - cp ${BUILD_HOME}/developer_tools/server/build/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. + cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/. BUILD_INCLUDE_REPO=${BUILD_HOME}/developer_tools/server/include/triton/developer_tools BUILD_SRC_REPO=${BUILD_HOME}/developer_tools/server/src TRITON_INCLUDE_REPO=${TRITON_HOME}/include/triton/developer_tools From e1ab915ee7df6ebeb928409c24cd41b958e29f0a Mon Sep 17 00:00:00 2001 From: Katherine Yang Date: Fri, 23 Jun 2023 18:16:21 -0700 Subject: [PATCH 49/50] address comments --- .../tritonserver/presets/tritonserver.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 2639995be6c..3e5508e1887 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -64,19 +64,6 @@ public void map(InfoMap infoMap) { .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) - .put(new Info("struct TRITONSERVER_BufferAttributes").cppText("TRITONSERVER_BufferAttributes")) - .put(new Info("struct TRITONSERVER_Error").cppText("TRITONSERVER_Error")) - .put(new Info("struct TRITONSERVER_InferenceRequest").cppText("TRITONSERVER_InferenceRequest")) - .put(new Info("struct TRITONSERVER_InferenceResponse").cppText("TRITONSERVER_InferenceResponse")) - .put(new Info("struct TRITONSERVER_InferenceTrace").cppText("TRITONSERVER_InferenceTrace")) - .put(new Info("struct TRITONSERVER_Message").cppText("TRITONSERVER_Message")) - .put(new Info("struct TRITONSERVER_Metrics").cppText("TRITONSERVER_Metrics")) - .put(new Info("struct TRITONSERVER_Parameter").cppText("TRITONSERVER_Parameter")) - .put(new Info("struct TRITONSERVER_ResponseAllocator").cppText("TRITONSERVER_ResponseAllocator")) - .put(new Info("struct TRITONSERVER_Server").cppText("TRITONSERVER_Server")) - .put(new Info("struct TRITONSERVER_ServerOptions").cppText("TRITONSERVER_ServerOptions")) - .put(new Info("struct TRITONSERVER_Metric").cppText("TRITONSERVER_Metric")) - .put(new Info("struct TRITONSERVER_MetricFamily").cppText("TRITONSERVER_MetricFamily")); + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); } } From 2ca0f6eb9245030f12b6dd3009c0183deca517fe Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Tue, 27 Jun 2023 08:24:53 +0900 Subject: [PATCH 50/50] Fix nits --- CHANGELOG.md | 3 +-- tritonserver/cppbuild.sh | 2 +- tritonserver/samples/simplecpp/SimpleCPP.java | 10 +++++----- .../presets/tritondevelopertoolsserver.java | 2 +- .../bytedeco/tritonserver/presets/tritonserver.java | 9 +++++---- tritonserver/src/main/java9/module-info.java | 2 -- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b753d806b2..6bf7788507d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,6 @@ * Map new higher-level C++ API of Triton Inference Server ([pull #1361](https://github.com/bytedeco/javacpp-presets/pull/1361)) - * Upgrade presets for Triton Inference Server 2.34.0 - * Upgrade presets for DNNL 3.1.1, CPython 3.11.4, NumPy 1.25.0, SciPy 1.11.0, LLVM 16.0.6, ONNX Runtime 1.15.1, and their dependencies + * Upgrade presets for DNNL 3.1.1, CPython 3.11.4, NumPy 1.25.0, SciPy 1.11.0, LLVM 16.0.6, Triton Inference Server 2.34.0, ONNX Runtime 1.15.1, and their dependencies ### June 6, 2023 version 1.5.9 * Virtualize `nvinfer1::IGpuAllocator` from TensorRT to allow customization ([pull #1367](https://github.com/bytedeco/javacpp-presets/pull/1367)) diff --git a/tritonserver/cppbuild.sh b/tritonserver/cppbuild.sh index 8efb63d78e3..d4353da8eb2 100755 --- a/tritonserver/cppbuild.sh +++ b/tritonserver/cppbuild.sh @@ -19,7 +19,7 @@ if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrap mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME} git clone --single-branch --depth=1 -b ${TRITON_CORE_REPO_TAG} ${TRITON_CORE_REPO} cp core/include/triton/core/* ${TRITON_HOME}/include/triton/core/. - + # Build CPP library and copy over dependencies git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH} cd developer_tools/server diff --git a/tritonserver/samples/simplecpp/SimpleCPP.java b/tritonserver/samples/simplecpp/SimpleCPP.java index 81bffc1f0a2..a6421d44bbc 100644 --- a/tritonserver/samples/simplecpp/SimpleCPP.java +++ b/tritonserver/samples/simplecpp/SimpleCPP.java @@ -31,6 +31,7 @@ import org.bytedeco.javacpp.*; import org.bytedeco.tritonserver.tritondevelopertoolsserver.*; import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*; + public class SimpleCPP { // Helper functions static void FAIL(String MSG) { @@ -48,7 +49,7 @@ static void Usage(String msg) { System.err.println("\t-r [model repository absolute path]"); System.exit(1); } - + static void CompareResult( String output0_name, String output1_name, @@ -90,12 +91,12 @@ static int RunInference(int verbose_level, String model_repository_path, String GenericTritonServer server = GenericTritonServer.Create(options); StringSet loaded_models = server.LoadedModels(); System.out.println("Loaded_models count : " + loaded_models.size()); - + InferOptions infer_options = new InferOptions(model_name); GenericInferRequest request = GenericInferRequest.Create(infer_options); BytePointer input0_data; - BytePointer input1_data; + BytePointer input1_data; IntPointer[] p0 = {null}, p1 = {null}; GenerateInputData(p0, p1); input0_data = p0[0].getPointer(BytePointer.class); @@ -147,7 +148,7 @@ static int RunInference(int verbose_level, String model_repository_path, String break; } } - + // We use a simple model that takes 2 input tensors of 16 strings // each and returns 2 output tensors of 16 strings each. The input // strings must represent integers. One output tensor is the @@ -162,5 +163,4 @@ static int RunInference(int verbose_level, String model_repository_path, String System.exit(0); } - } diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java index a1efb2a8c02..1c9185641e6 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java @@ -34,7 +34,7 @@ /** * - * @author Katherine Yang, Baojun Liu, Samuel Audet + * @author Katherine Yang, Baojun Liu */ @Properties( value = { diff --git a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java index 3e5508e1887..ebed413d826 100644 --- a/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java +++ b/tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritonserver.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Jack He, Samuel Audet, Katherine Yang, Baojun Liu + * Copyright (C) 2021 Jack He, Samuel Audet * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -34,14 +34,14 @@ /** * - * @author Katherine Yang, Jack He, Baojun Liu + * @author Jack He */ @Properties( value = { @Platform( value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"}, include = {"tritonserver.h", "tritonbackend.h", "tritonrepoagent.h"}, - link = {"tritonserver"}, + link = "tritonserver", includepath = {"/opt/tritonserver/include/triton/core/", "/opt/tritonserver/include/", "/usr/include"}, linkpath = {"/opt/tritonserver/lib/"} ), @@ -64,6 +64,7 @@ public void map(InfoMap infoMap) { .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) .put(new Info("TRITONSERVER_EXPORT", "TRITONSERVER_DECLSPEC", "TRITONBACKEND_DECLSPEC", "TRITONBACKEND_ISPEC", - "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()); + "TRITONREPOAGENT_DECLSPEC", "TRITONREPOAGENT_ISPEC").cppTypes().annotations()) + ; } } diff --git a/tritonserver/src/main/java9/module-info.java b/tritonserver/src/main/java9/module-info.java index 199f9393e24..f74a0ecbf0c 100644 --- a/tritonserver/src/main/java9/module-info.java +++ b/tritonserver/src/main/java9/module-info.java @@ -2,8 +2,6 @@ requires transitive org.bytedeco.javacpp; exports org.bytedeco.tritonserver.global; exports org.bytedeco.tritonserver.presets; - exports org.bytedeco.tritonserver; - exports org.bytedeco.tritonserver.tritondevelopertoolsserver; exports org.bytedeco.tritonserver.tritonserver; }