From e8793b4aca7961f8ee0b4a93ef9303566e8553cc Mon Sep 17 00:00:00 2001 From: dingpei Date: Tue, 6 Aug 2024 14:36:44 +0800 Subject: [PATCH] google ai support vision request --- jcommon/ai/google/pom.xml | 2 +- .../java/run/mone/ai/google/CloudeClient.java | 41 +++++-------------- .../google/bo/multiModal/GVisionContent.java | 19 +++++++++ .../ai/google/bo/multiModal/GVisionMsg.java | 23 +++++++++++ .../google/bo/multiModal/GVisionRequest.java | 32 +++++++++++++++ .../google/bo/multiModal/GVisionSource.java | 20 +++++++++ 6 files changed, 106 insertions(+), 31 deletions(-) create mode 100644 jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionContent.java create mode 100644 jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionMsg.java create mode 100644 jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionRequest.java create mode 100644 jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionSource.java diff --git a/jcommon/ai/google/pom.xml b/jcommon/ai/google/pom.xml index 6213e1dbc..74d000330 100644 --- a/jcommon/ai/google/pom.xml +++ b/jcommon/ai/google/pom.xml @@ -10,7 +10,7 @@ google - 1.4-jdk8-SNAPSHOT + 1.5-jdk8-SNAPSHOT 8 diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/CloudeClient.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/CloudeClient.java index 0c3722ff7..73ae7a864 100644 --- a/jcommon/ai/google/src/main/java/run/mone/ai/google/CloudeClient.java +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/CloudeClient.java @@ -12,6 +12,7 @@ import run.mone.ai.google.bo.Content; import run.mone.ai.google.bo.RequestPayload; import run.mone.ai.google.bo.ResponsePayload; +import run.mone.ai.google.bo.multiModal.GVisionRequest; import java.io.FileInputStream; import java.io.IOException; @@ -38,7 +39,6 @@ public class CloudeClient { private static Gson gson = new Gson(); - @SneakyThrows public String token(String model) { GoogleCredentials credentials = GoogleCredentials.fromStream( @@ -52,42 +52,23 @@ public String token(String model) { return this.token; } + public ResponsePayload visionCall(String url, String token, GVisionRequest GVisionRequest) { + return baseCall(url, token, gson.toJson(GVisionRequest)); + } public ResponsePayload call(String token, RequestPayload requestPayload) { - OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build(); - MediaType mediaType = MediaType.parse("application/json; charset=utf-8"); - RequestBody body = RequestBody.create(mediaType, new Gson().toJson(requestPayload)); - Request request = new Request.Builder() - .url(url + projectId + "/locations/us-central1/publishers/anthropic/models/" + model + ":streamRawPredict") - .post(body) - .addHeader("Authorization", "Bearer " + token) - .addHeader("Content-Type", "application/json; charset=utf-8") - .build(); + String callUrl = url + projectId + "/locations/us-central1/publishers/anthropic/models/" + model + ":streamRawPredict"; + return call(callUrl, token, requestPayload); + } - try (Response response = client.newCall(request).execute()) { - if (response.code() == 429) { - ResponsePayload res = new ResponsePayload(); - Content content = new Content(); - content.setText(gson.toJson(ImmutableMap.of("message", "被claude3限流了", "code", "429"))); - log.info("claude res:{}", content.getText()); - res.setContent(Lists.newArrayList(content)); - return res; - } - if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); - // Handle the response - String res = response.body().string(); - log.info("claude3 res:{}", res); - return new Gson().fromJson(res, ResponsePayload.class); - } catch (Throwable e) { - log.error(e.getMessage(), e); - } - return null; + public ResponsePayload call(String url, String token, RequestPayload requestPayload) { + return baseCall(url, token, gson.toJson(requestPayload)); } - public ResponsePayload call(String url ,String token, RequestPayload requestPayload) { + private ResponsePayload baseCall(String url, String token, String bodyStr) { OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build(); MediaType mediaType = MediaType.parse("application/json; charset=utf-8"); - RequestBody body = RequestBody.create(mediaType, new Gson().toJson(requestPayload)); + RequestBody body = RequestBody.create(mediaType, bodyStr); Request request = new Request.Builder() .url(url) .post(body) diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionContent.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionContent.java new file mode 100644 index 000000000..bff182901 --- /dev/null +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionContent.java @@ -0,0 +1,19 @@ +package run.mone.ai.google.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GVisionContent { + + @SerializedName("type") + private String type; + + @SerializedName("text") + private String text; + + @SerializedName("source") + private GVisionSource source; +} diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionMsg.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionMsg.java new file mode 100644 index 000000000..6dc58d0f9 --- /dev/null +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionMsg.java @@ -0,0 +1,23 @@ +package run.mone.ai.google.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2023/5/25 14:16 + */ +@Data +@Builder +public class GVisionMsg implements Serializable { + + @SerializedName("role") + private String role; + + @SerializedName("content") + private List content; +} diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionRequest.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionRequest.java new file mode 100644 index 000000000..81a8600d7 --- /dev/null +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionRequest.java @@ -0,0 +1,32 @@ +package run.mone.ai.google.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2024/4/9 16:36 + */ +@Data +@Builder +public class GVisionRequest { + + + @SerializedName("anthropic_version") + private String anthropicVersion; + + @SerializedName("messages") + private List messages; + + @SerializedName("max_tokens") + private int maxTokens; + + @SerializedName("stream") + private boolean stream; + + + +} diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionSource.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionSource.java new file mode 100644 index 000000000..277aa9d08 --- /dev/null +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionSource.java @@ -0,0 +1,20 @@ +package run.mone.ai.google.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GVisionSource { + + @SerializedName("type") + private String type; + + @SerializedName("media_type") + private String mediaType; + + @SerializedName("data") + private String data; + +}