From 58040e4edb88dba0a8645e40b2b0fbfa7b7faea1 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Fri, 17 May 2024 15:29:26 +0530 Subject: [PATCH 01/10] feat: [COR-916] Make zimbraHttpProxyURL single cardinal --- .../common/account/ZAttrProvisioning.java | 4 +- .../com/zimbra/cs/account/ZAttrConfig.java | 88 ++++--------------- .../com/zimbra/cs/account/ZAttrServer.java | 88 ++++--------------- store/src/main/resources/conf/attrs/attrs.xml | 4 +- 4 files changed, 34 insertions(+), 150 deletions(-) diff --git a/common/src/main/java/com/zimbra/common/account/ZAttrProvisioning.java b/common/src/main/java/com/zimbra/common/account/ZAttrProvisioning.java index 4e6cef36b09..b054a585a80 100644 --- a/common/src/main/java/com/zimbra/common/account/ZAttrProvisioning.java +++ b/common/src/main/java/com/zimbra/common/account/ZAttrProvisioning.java @@ -8530,8 +8530,8 @@ public static TwoFactorAuthSecretEncoding fromString(String s) throws ServiceExc public static final String A_zimbraHttpOutputBufferSize = "zimbraHttpOutputBufferSize"; /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) */ @ZAttr(id=388) public static final String A_zimbraHttpProxyURL = "zimbraHttpProxyURL"; diff --git a/store/src/main/java/com/zimbra/cs/account/ZAttrConfig.java b/store/src/main/java/com/zimbra/cs/account/ZAttrConfig.java index 394f834bf08..0d3d923eda7 100644 --- a/store/src/main/java/com/zimbra/cs/account/ZAttrConfig.java +++ b/store/src/main/java/com/zimbra/cs/account/ZAttrConfig.java @@ -24347,106 +24347,48 @@ public Map unsetHttpOutputBufferSize(Map attrs) { } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * - * @return zimbraHttpProxyURL, or empty array if unset + * @return zimbraHttpProxyURL, or null if unset */ @ZAttr(id=388) - public String[] getHttpProxyURL() { - return getMultiAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, true, true); + public String getHttpProxyURL() { + return getAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, null, true); } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param zimbraHttpProxyURL new value * @throws com.zimbra.common.service.ServiceException if error during update */ @ZAttr(id=388) - public void setHttpProxyURL(String[] zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { + public void setHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { HashMap attrs = new HashMap<>(); attrs.put(ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); getProvisioning().modifyAttrs(this, attrs); } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param zimbraHttpProxyURL new value * @param attrs existing map to populate, or null to create a new map * @return populated map to pass into Provisioning.modifyAttrs */ @ZAttr(id=388) - public Map setHttpProxyURL(String[] zimbraHttpProxyURL, Map attrs) { + public Map setHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { if (attrs == null) attrs = new HashMap<>(); attrs.put(ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); return attrs; } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL new to add to existing values - * @throws com.zimbra.common.service.ServiceException if error during update - */ - @ZAttr(id=388) - public void addHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { - HashMap attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "+" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - getProvisioning().modifyAttrs(this, attrs); - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL new to add to existing values - * @param attrs existing map to populate, or null to create a new map - * @return populated map to pass into Provisioning.modifyAttrs - */ - @ZAttr(id=388) - public Map addHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { - if (attrs == null) attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "+" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - return attrs; - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL existing value to remove - * @throws com.zimbra.common.service.ServiceException if error during update - */ - @ZAttr(id=388) - public void removeHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { - HashMap attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "-" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - getProvisioning().modifyAttrs(this, attrs); - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL existing value to remove - * @param attrs existing map to populate, or null to create a new map - * @return populated map to pass into Provisioning.modifyAttrs - */ - @ZAttr(id=388) - public Map removeHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { - if (attrs == null) attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "-" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - return attrs; - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @throws com.zimbra.common.service.ServiceException if error during update */ @@ -24458,8 +24400,8 @@ public void unsetHttpProxyURL() throws com.zimbra.common.service.ServiceExceptio } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param attrs existing map to populate, or null to create a new map * @return populated map to pass into Provisioning.modifyAttrs diff --git a/store/src/main/java/com/zimbra/cs/account/ZAttrServer.java b/store/src/main/java/com/zimbra/cs/account/ZAttrServer.java index 387e3a13517..9ad571e5de5 100644 --- a/store/src/main/java/com/zimbra/cs/account/ZAttrServer.java +++ b/store/src/main/java/com/zimbra/cs/account/ZAttrServer.java @@ -10633,106 +10633,48 @@ public Map unsetHttpOutputBufferSize(Map attrs) { } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * - * @return zimbraHttpProxyURL, or empty array if unset + * @return zimbraHttpProxyURL, or null if unset */ @ZAttr(id=388) - public String[] getHttpProxyURL() { - return getMultiAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, true, true); + public String getHttpProxyURL() { + return getAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, null, true); } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param zimbraHttpProxyURL new value * @throws com.zimbra.common.service.ServiceException if error during update */ @ZAttr(id=388) - public void setHttpProxyURL(String[] zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { + public void setHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { HashMap attrs = new HashMap<>(); attrs.put(ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); getProvisioning().modifyAttrs(this, attrs); } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param zimbraHttpProxyURL new value * @param attrs existing map to populate, or null to create a new map * @return populated map to pass into Provisioning.modifyAttrs */ @ZAttr(id=388) - public Map setHttpProxyURL(String[] zimbraHttpProxyURL, Map attrs) { + public Map setHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { if (attrs == null) attrs = new HashMap<>(); attrs.put(ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); return attrs; } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL new to add to existing values - * @throws com.zimbra.common.service.ServiceException if error during update - */ - @ZAttr(id=388) - public void addHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { - HashMap attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "+" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - getProvisioning().modifyAttrs(this, attrs); - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL new to add to existing values - * @param attrs existing map to populate, or null to create a new map - * @return populated map to pass into Provisioning.modifyAttrs - */ - @ZAttr(id=388) - public Map addHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { - if (attrs == null) attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "+" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - return attrs; - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL existing value to remove - * @throws com.zimbra.common.service.ServiceException if error during update - */ - @ZAttr(id=388) - public void removeHttpProxyURL(String zimbraHttpProxyURL) throws com.zimbra.common.service.ServiceException { - HashMap attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "-" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - getProvisioning().modifyAttrs(this, attrs); - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) - * - * @param zimbraHttpProxyURL existing value to remove - * @param attrs existing map to populate, or null to create a new map - * @return populated map to pass into Provisioning.modifyAttrs - */ - @ZAttr(id=388) - public Map removeHttpProxyURL(String zimbraHttpProxyURL, Map attrs) { - if (attrs == null) attrs = new HashMap<>(); - StringUtil.addToMultiMap(attrs, "-" + ZAttrProvisioning.A_zimbraHttpProxyURL, zimbraHttpProxyURL); - return attrs; - } - - /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @throws com.zimbra.common.service.ServiceException if error during update */ @@ -10744,8 +10686,8 @@ public void unsetHttpProxyURL() throws com.zimbra.common.service.ServiceExceptio } /** - * external socks proxy URL to connect to when making outgoing - * connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + * URL of the external HTTP proxy to be used for outgoing connections + * (warning: only used in certain use cases, see HttpProxyUtil usages) * * @param attrs existing map to populate, or null to create a new map * @return populated map to pass into Provisioning.modifyAttrs diff --git a/store/src/main/resources/conf/attrs/attrs.xml b/store/src/main/resources/conf/attrs/attrs.xml index 960e973db3f..8ab93685bfc 100755 --- a/store/src/main/resources/conf/attrs/attrs.xml +++ b/store/src/main/resources/conf/attrs/attrs.xml @@ -2070,8 +2070,8 @@ TODO - add support for multi-line values in globalConfigValue and defaultCOSValu per RFC 3834 no out of office notifications are sent if recipients address is not directly specified in the To/CC headers - for this check, we check to see if To/CC contained accounts address, aliases, canonical address. But when external accounts are forwarded to Zimbra, and you want notifications sent to messages that contain their external address in To/Cc, add those address, then you can specify those external addresses here. - - external socks proxy URL to connect to when making outgoing connections (eg.Zimlet proxy, RSS/ATOM feeds, etc) + + URL of the external HTTP proxy to be used for outgoing connections (warning: only used in certain use cases, see HttpProxyUtil usages) From 6c113ba7136ae92beb1ce9fe38bf5d8745cdc7a2 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 13:55:37 +0530 Subject: [PATCH 02/10] refactor: HttpProxyUtil, added logging, performed cleanup --- .../zimbra/cs/httpclient/HttpProxyUtil.java | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java b/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java index b3ca95dfc59..9c995fc8d0c 100644 --- a/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java +++ b/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java @@ -5,9 +5,12 @@ package com.zimbra.cs.httpclient; +import com.zimbra.common.account.ZAttrProvisioning; +import com.zimbra.common.service.ServiceException; +import com.zimbra.common.util.ZimbraLog; +import com.zimbra.cs.account.Provisioning; import java.net.URI; import java.net.URISyntaxException; - import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -16,53 +19,56 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; -import com.zimbra.common.service.ServiceException; -import com.zimbra.common.util.ZimbraLog; -import com.zimbra.cs.account.Provisioning; - public class HttpProxyUtil { - private static String sProxyUrl = null; - private static URI sProxyUri = null; - private static AuthScope sProxyAuthScope = null; - private static UsernamePasswordCredentials sProxyCreds = null; + private HttpProxyUtil() { + throw new IllegalStateException("Utility class"); + } - public static synchronized void configureProxy(HttpClientBuilder clientBuilder) { - try { - String url = Provisioning.getInstance().getLocalServer().getAttr(Provisioning.A_zimbraHttpProxyURL, null); - if (url == null) return; + public static synchronized void configureProxy(HttpClientBuilder clientBuilder) { + try { + final var httpProxyUrl = Provisioning.getInstance().getLocalServer().getAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, null); + if (httpProxyUrl == null || httpProxyUrl.isEmpty()) { + ZimbraLog.misc.info("HttpProxyUtil.configureProxy 'zimbraHttpProxyURL' is null or empty, not using proxy."); + return; + } - // need to initializae all the statics - if (!url.equals(sProxyUrl)) { - sProxyUrl = url; - sProxyUri = new URI(url); - sProxyAuthScope = null; - sProxyCreds = null; - String userInfo = sProxyUri.getUserInfo(); - if (userInfo != null) { - int i = userInfo.indexOf(':'); - if (i != -1) { - sProxyAuthScope = new AuthScope(sProxyUri.getHost(), sProxyUri.getPort(), null); - sProxyCreds = new UsernamePasswordCredentials(userInfo.substring(0, i), userInfo.substring(i+1)); - } - } - } - if (ZimbraLog.misc.isDebugEnabled()) { - ZimbraLog.misc.debug("setting proxy: "+url); - } + var uri = new URI(httpProxyUrl); + var proxyHost = uri.getHost(); + var proxyPort = uri.getPort(); - HttpHost proxy = new HttpHost(sProxyUri.getHost(), sProxyUri.getPort()); - RequestConfig config = RequestConfig.custom() - .setProxy(proxy) - .build(); - clientBuilder.setDefaultRequestConfig(config); - if (sProxyAuthScope != null && sProxyCreds != null) { - CredentialsProvider cred = new BasicCredentialsProvider(); - cred.setCredentials(sProxyAuthScope, sProxyCreds); - clientBuilder.setDefaultCredentialsProvider(cred); - } - } catch (ServiceException | URISyntaxException e) { - ZimbraLog.misc.warn("Unable to configureProxy: "+e.getMessage(), e); + var userInfo = uri.getUserInfo(); + String username = null; + String password = null; + if (userInfo != null) { + var credentials = userInfo.split(":"); + if (credentials.length == 2) { + username = credentials[0]; + password = credentials[1]; } + } + + if (username != null && password != null) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + new AuthScope(proxyHost, proxyPort), + new UsernamePasswordCredentials(username, password) + ); + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + + var proxy = new HttpHost(proxyHost, proxyPort); + var config = RequestConfig.custom() + .setProxy(proxy) + .build(); + clientBuilder.setDefaultRequestConfig(config); + + if (ZimbraLog.misc.isDebugEnabled()) { + ZimbraLog.misc.debug("setting proxy: " + httpProxyUrl); + } + + } catch (ServiceException | URISyntaxException e) { + ZimbraLog.misc.warn("Unable to configureProxy: " + e.getMessage(), e); } + } } From fba7d67557f44852d2c7a50361342e58d6ffc59b Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 13:57:01 +0530 Subject: [PATCH 03/10] feat: move ProxyServlet out from zimlet stack + cleanup --- store/conf/web.xml.production | 16 + .../service/servlet/proxy/ProxyServlet.java | 381 ++++++++++++++++++ .../com/zimbra/cs/zimlet/ProxyServlet.java | 351 ---------------- 3 files changed, 397 insertions(+), 351 deletions(-) create mode 100644 store/src/main/java/com/zimbra/cs/service/servlet/proxy/ProxyServlet.java delete mode 100644 store/src/main/java/com/zimbra/cs/zimlet/ProxyServlet.java diff --git a/store/conf/web.xml.production b/store/conf/web.xml.production index 09194ca600f..9d993d53648 100755 --- a/store/conf/web.xml.production +++ b/store/conf/web.xml.production @@ -393,6 +393,17 @@ 7 + + ProxyServlet + com.zimbra.cs.service.servlet.proxy.ProxyServlet + true + + allowed.ports + %%zimbraMailPort%%, %%zimbraMailSSLPort%%, 7070 + + 8 + + DavServlet com.zimbra.cs.dav.service.DavServlet @@ -555,6 +566,11 @@ /collectldapconfig/* + + ProxyServlet + /proxy/* + + DavServlet /dav/* diff --git a/store/src/main/java/com/zimbra/cs/service/servlet/proxy/ProxyServlet.java b/store/src/main/java/com/zimbra/cs/service/servlet/proxy/ProxyServlet.java new file mode 100644 index 00000000000..18da05df066 --- /dev/null +++ b/store/src/main/java/com/zimbra/cs/service/servlet/proxy/ProxyServlet.java @@ -0,0 +1,381 @@ +// SPDX-FileCopyrightText: 2022 Synacor, Inc. +// SPDX-FileCopyrightText: 2022 Zextras +// +// SPDX-License-Identifier: GPL-2.0-only + +package com.zimbra.cs.service.servlet.proxy; + +import com.zimbra.common.account.Key.AccountBy; +import com.zimbra.common.account.ZAttrProvisioning; +import com.zimbra.common.httpclient.HttpClientUtil; +import com.zimbra.common.localconfig.LC; +import com.zimbra.common.mime.ContentDisposition; +import com.zimbra.common.mime.ContentType; +import com.zimbra.common.service.ServiceException; +import com.zimbra.common.util.ByteUtil; +import com.zimbra.common.util.ZimbraLog; +import com.zimbra.cs.account.Account; +import com.zimbra.cs.account.AuthToken; +import com.zimbra.cs.account.AuthTokenException; +import com.zimbra.cs.account.Provisioning; +import com.zimbra.cs.httpclient.HttpProxyUtil; +import com.zimbra.cs.mailbox.MailServiceException; +import com.zimbra.cs.service.AuthProvider; +import com.zimbra.cs.service.FileUploadServlet; +import com.zimbra.cs.service.FileUploadServlet.Upload; +import com.zimbra.cs.servlet.ZimbraServlet; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.DefaultRedirectStrategy; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; + +/** + * @author jylee + */ +public class ProxyServlet extends ZimbraServlet { + + private static final String TARGET_PARAM = "target"; + private static final String UPLOAD_PARAM = "upload"; + private static final String FILENAME_PARAM = "filename"; + private static final String FORMAT_PARAM = "fmt"; + + private static final String USER_PARAM = "user"; + private static final String PASS_PARAM = "pass"; + private static final String AUTH_PARAM = "auth"; + private static final String AUTH_BASIC = "basic"; + private static final String DEFAULT_CONTENT_TYPE_HEADER_VALUE = "text/xml"; + + private Set getAllowedDomains(AuthToken auth) throws ServiceException { + Provisioning prov = Provisioning.getInstance(); + Account acct = prov.get(AccountBy.id, auth.getAccountId(), auth); + Set allowedDomains = prov.getCOS(acct).getMultiAttrSet(ZAttrProvisioning.A_zimbraProxyAllowedDomains); + ZimbraLog.misc.debug("get allowedDomains result: " + allowedDomains); + return allowedDomains; + } + + private boolean checkPermissionOnTarget(URL target, AuthToken auth) { + String host = target.getHost().toLowerCase(); + ZimbraLog.misc.debug("checking allowedDomains permission on target host: " + host); + Set domains; + try { + domains = getAllowedDomains(auth); + } catch (ServiceException se) { + ZimbraLog.misc.info("error getting allowedDomains: " + se.getMessage()); + return false; + } + for (String domain : domains) { + if (domain.charAt(0) == '*') { + domain = domain.substring(1); + if (host.endsWith(domain)) { + return true; + } + } else if (host.equals(domain)) { + return true; + } + } + return false; + } + + private boolean canProxyHeader(String header) { + if (header == null) { + return false; + } + header = header.toLowerCase(); + return !header.startsWith("accept") && + !"content-length".equals(header) && + !"connection".equals(header) && + !"keep-alive".equals(header) && + !"pragma".equals(header) && + !"host".equals(header) && + !"cache-control".equals(header) && + !"cookie".equals(header) && + !"transfer-encoding".equals(header); + } + + private byte[] copyPostedData(HttpServletRequest req) throws IOException { + int size = req.getContentLength(); + if ("GET".equalsIgnoreCase(req.getMethod()) || size <= 0) { + return new byte[0]; + } + try (InputStream is = req.getInputStream(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(size)) { + byte[] buffer = new byte[8192]; + int num; + while ((num = is.read(buffer)) != -1) { + byteArrayOutputStream.write(buffer, 0, num); + } + return byteArrayOutputStream.toByteArray(); + } + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + handle(req, resp); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + handle(req, resp); + } + + @Override + protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws IOException { + handle(req, resp); + } + + @Override + protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws IOException { + handle(req, resp); + } + + private void handle(HttpServletRequest req, HttpServletResponse resp) { + try { + doProxy(req, resp); + } catch (IOException e) { + ZimbraLog.misc.info(e.getMessage(), e); + } + } + + @Override + protected boolean isAdminRequest(HttpServletRequest req) { + return req.getServerPort() == LC.zimbra_admin_service_port.intValue(); + } + + private void doProxy(HttpServletRequest req, HttpServletResponse resp) throws IOException { + ZimbraLog.clearContext(); + boolean isAdmin = isAdminRequest(req); + + try { + AuthToken authToken = getAuthTokenFrom(req, resp); + if (authToken == null) { + return; + } + + // get the posted body before the server read and parse them. + byte[] body = copyPostedData(req); + + // sanity check + final String target = req.getParameter(TARGET_PARAM); + if (target == null) { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + + // check for permission + final URL url = new URL(target); + if (!isAdmin && !checkPermissionOnTarget(url, authToken)) { + resp.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + + HttpClientBuilder clientBuilder = HttpClients.custom(); + HttpProxyUtil.configureProxy(clientBuilder); + final HttpRequestBase method = configureHttpMethod(req, target, body); + if (method == null) { + ZimbraLog.misc.info("unsupported request method"); + resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); + } else { + setBasicAuth(req, resp, clientBuilder); + proxyHeaders(req, method); + handleHttpResponse(req, resp, method, authToken, target); + } + } catch (IOException e) { + ZimbraLog.misc.info(e.getMessage(), e); + } + } + + private void setBasicAuth(HttpServletRequest req, HttpServletResponse resp, + HttpClientBuilder clientBuilder) throws IOException { + String auth = req.getParameter(AUTH_PARAM); + String user = req.getParameter(USER_PARAM); + String pass = req.getParameter(PASS_PARAM); + if (auth != null && user != null && pass != null) { + if (!auth.equals(AUTH_BASIC)) { + ZimbraLog.misc.info("unsupported auth type: " + auth); + resp.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + CredentialsProvider provider = new BasicCredentialsProvider(); + provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, pass)); + clientBuilder.setDefaultCredentialsProvider(provider); + } + } + + private void proxyHeaders(HttpServletRequest req, HttpRequestBase method) { + Enumeration headers = req.getHeaderNames(); + while (headers.hasMoreElements()) { + String hdr = headers.nextElement(); + ZimbraLog.misc.debug("incoming: " + hdr + ": " + req.getHeader(hdr)); + if (canProxyHeader(hdr)) { + ZimbraLog.misc.debug("outgoing: " + hdr + ": " + req.getHeader(hdr)); + method.addHeader(hdr, req.getHeader(hdr)); + } + } + } + + private HttpRequestBase configureHttpMethod(HttpServletRequest req, String target, byte[] body) { + String reqMethod = req.getMethod(); + HttpRequestBase method = null; + if ("GET".equalsIgnoreCase(reqMethod)) { + method = new HttpGet(target); + } else if ("POST".equalsIgnoreCase(reqMethod)) { + HttpPost post = new HttpPost(target); + post.setEntity(new ByteArrayEntity(body, org.apache.http.entity.ContentType.create(req.getContentType()))); + method = post; + } else if ("PUT".equalsIgnoreCase(reqMethod)) { + HttpPut put = new HttpPut(target); + put.setEntity(new ByteArrayEntity(body, org.apache.http.entity.ContentType.create(req.getContentType()))); + method = put; + } else if ("DELETE".equalsIgnoreCase(reqMethod)) { + method = new HttpDelete(target); + } + return method; + } + + private AuthToken getAuthTokenFrom(HttpServletRequest req, HttpServletResponse resp) throws IOException { + boolean isAdmin = isAdminRequest(req); + AuthToken authToken = isAdmin ? + getAdminAuthTokenFromCookie(req, resp, true) : getAuthTokenFromCookie(req, resp, true); + if (authToken == null) { + String zAuthTokenFromQueryParam = req.getParameter(QP_ZAUTHTOKEN); + if (zAuthTokenFromQueryParam != null) { + try { + authToken = AuthProvider.getAuthToken(zAuthTokenFromQueryParam); + if (authToken != null && authToken.isExpired()) { + resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken expired"); + return null; + } + } catch (AuthTokenException e) { + resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "unable to parse authtoken"); + return null; + } + } + } + if (authToken == null) { + resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "no authtoken cookie"); + return null; + } + if (!authToken.isRegistered()) { + resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken is invalid"); + return null; + } + if (isAdmin && !authToken.isAdmin()) { + resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "permission denied"); + return null; + } + return authToken; + } + + + private void handleHttpResponse(HttpServletRequest req, HttpServletResponse resp, HttpRequestBase method, + AuthToken authToken, String targetUrl) throws IOException { + HttpResponse httpResp; + try { + HttpClientBuilder clientBuilder = HttpClients.custom(); + HttpProxyUtil.configureProxy(clientBuilder); + if (!("POST".equalsIgnoreCase(req.getMethod()) || "PUT".equalsIgnoreCase(req.getMethod()))) { + clientBuilder.setRedirectStrategy(new DefaultRedirectStrategy()); + } + + HttpClient client = clientBuilder.build(); + httpResp = HttpClientUtil.executeMethod(client, method); + } catch (HttpException | IOException ex) { + ZimbraLog.misc.info("exception while proxying " + targetUrl, ex); + resp.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + int status = httpResp.getStatusLine() == null ? HttpServletResponse.SC_INTERNAL_SERVER_ERROR + : httpResp.getStatusLine().getStatusCode(); + + Header ctHeader = httpResp.getFirstHeader("Content-Type"); + String contentType = + ctHeader == null || ctHeader.getValue() == null ? DEFAULT_CONTENT_TYPE_HEADER_VALUE : ctHeader.getValue(); + + InputStream targetResponseBody = null; + HttpEntity targetResponseEntity = httpResp.getEntity(); + if (targetResponseEntity != null) { + targetResponseBody = targetResponseEntity.getContent(); + } + + String uploadParam = req.getParameter(UPLOAD_PARAM); + boolean asUpload = ("1".equals(uploadParam) || "true".equalsIgnoreCase(uploadParam)); + + if (asUpload) { + handleUploadResponse(req, resp, status, targetResponseBody, authToken, contentType, httpResp); + } else { + handleInlineResponse(resp, status, targetResponseBody, contentType, httpResp); + } + } + + private void handleUploadResponse(HttpServletRequest req, HttpServletResponse resp, int status, + InputStream targetResponseBody, AuthToken authToken, String contentType, HttpResponse httpResp) + throws IOException { + String filename = req.getParameter(FILENAME_PARAM); + if (filename == null || "".equals(filename)) { + filename = new ContentType(contentType).getParameter("name"); + } + if ((filename == null || "".equals(filename)) && httpResp.getFirstHeader("Content-Disposition") != null) { + filename = new ContentDisposition(httpResp.getFirstHeader("Content-Disposition").getValue()).getParameter( + FILENAME_PARAM); + } + if (filename == null || "".equals(filename)) { + filename = "unknown"; + } + + List uploads = null; + + if (targetResponseBody != null) { + try { + Upload up = FileUploadServlet.saveUpload(targetResponseBody, filename, contentType, authToken.getAccountId()); + uploads = List.of(up); + } catch (ServiceException e) { + if (e.getCode().equals(MailServiceException.UPLOAD_REJECTED)) { + status = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; + } else { + status = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + } + } + } + + resp.setStatus(status); + FileUploadServlet.sendResponse(resp, status, req.getParameter(FORMAT_PARAM), null, uploads, null); + } + + private void handleInlineResponse(HttpServletResponse resp, int status, InputStream targetResponseBody, + String contentType, HttpResponse httpResp) throws IOException { + resp.setStatus(status); + resp.setContentType(contentType); + for (Header h : httpResp.getAllHeaders()) { + if (canProxyHeader(h.getName())) { + resp.addHeader(h.getName(), h.getValue()); + } + } + if (targetResponseBody != null) { + ByteUtil.copy(targetResponseBody, true, resp.getOutputStream(), true); + } + } +} diff --git a/store/src/main/java/com/zimbra/cs/zimlet/ProxyServlet.java b/store/src/main/java/com/zimbra/cs/zimlet/ProxyServlet.java deleted file mode 100644 index 075cb20f3fc..00000000000 --- a/store/src/main/java/com/zimbra/cs/zimlet/ProxyServlet.java +++ /dev/null @@ -1,351 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Synacor, Inc. -// SPDX-FileCopyrightText: 2022 Zextras -// -// SPDX-License-Identifier: GPL-2.0-only - -package com.zimbra.cs.zimlet; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.List; -import java.util.Set; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.DefaultRedirectStrategy; -import org.apache.http.impl.client.HttpClientBuilder; - -import com.zimbra.common.account.Key.AccountBy; -import com.zimbra.common.httpclient.HttpClientUtil; -import com.zimbra.common.localconfig.LC; -import com.zimbra.common.mime.ContentDisposition; -import com.zimbra.common.mime.ContentType; -import com.zimbra.common.service.ServiceException; -import com.zimbra.common.util.ByteUtil; -import com.zimbra.common.util.ZimbraHttpConnectionManager; -import com.zimbra.common.util.ZimbraLog; -import com.zimbra.cs.account.Account; -import com.zimbra.cs.account.AuthToken; -import com.zimbra.cs.account.AuthTokenException; -import com.zimbra.cs.account.Cos; -import com.zimbra.cs.account.Provisioning; -import com.zimbra.cs.httpclient.HttpProxyUtil; -import com.zimbra.cs.mailbox.MailServiceException; -import com.zimbra.cs.service.AuthProvider; -import com.zimbra.cs.service.FileUploadServlet; -import com.zimbra.cs.service.FileUploadServlet.Upload; -import com.zimbra.cs.servlet.ZimbraServlet; -/** - * @author jylee - */ -@SuppressWarnings("serial") -public class ProxyServlet extends ZimbraServlet { - - private static final String TARGET_PARAM = "target"; - - private static final String UPLOAD_PARAM = "upload"; - private static final String FILENAME_PARAM = "filename"; - private static final String FORMAT_PARAM = "fmt"; - - private static final String USER_PARAM = "user"; - private static final String PASS_PARAM = "pass"; - private static final String AUTH_PARAM = "auth"; - private static final String AUTH_BASIC = "basic"; - - - private Set getAllowedDomains(AuthToken auth) throws ServiceException { - Provisioning prov = Provisioning.getInstance(); - Account acct = prov.get(AccountBy.id, auth.getAccountId(), auth); - - Cos cos = prov.getCOS(acct); - - Set allowedDomains = cos.getMultiAttrSet(Provisioning.A_zimbraProxyAllowedDomains); - - ZimbraLog.zimlet.debug("get allowedDomains result: "+allowedDomains); - - return allowedDomains; - } - - private boolean checkPermissionOnTarget(URL target, AuthToken auth) { - String host = target.getHost().toLowerCase(); - ZimbraLog.zimlet.debug("checking allowedDomains permission on target host: " + host); - Set domains; - try { - domains = getAllowedDomains(auth); - } catch (ServiceException se) { - ZimbraLog.zimlet.info("error getting allowedDomains: " + se.getMessage()); - return false; - } - for (String domain : domains) { - if (domain.charAt(0) == '*') { - domain = domain.substring(1); - if (host.endsWith(domain)) { - return true; - } - } - else if (host.equals(domain)) { - return true; - } - } - return false; - } - - private boolean canProxyHeader(String header) { - if (header == null) return false; - header = header.toLowerCase(); - if (header.startsWith("accept") || - header.equals("content-length") || - header.equals("connection") || - header.equals("keep-alive") || - header.equals("pragma") || - header.equals("host") || - //header.equals("user-agent") || - header.equals("cache-control") || - header.equals("cookie") || - header.equals("transfer-encoding")) { - return false; - } - return true; - } - - private byte[] copyPostedData(HttpServletRequest req) throws IOException { - int size = req.getContentLength(); - if (req.getMethod().equalsIgnoreCase("GET") || size <= 0) { - return null; - } - InputStream is = req.getInputStream(); - ByteArrayOutputStream baos = null; - try { - if (size < 0) - size = 0; - baos = new ByteArrayOutputStream(size); - byte[] buffer = new byte[8192]; - int num; - while ((num = is.read(buffer)) != -1) { - baos.write(buffer, 0, num); - } - return baos.toByteArray(); - } finally { - ByteUtil.closeStream(baos); - } - } - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - doProxy(req, resp); - } - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { - doProxy(req, resp); - } - - @Override - protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws IOException { - doProxy(req, resp); - } - - @Override - protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws IOException { - doProxy(req, resp); - } - - @Override - protected boolean isAdminRequest(HttpServletRequest req) { - return req.getServerPort() == LC.zimbra_admin_service_port.intValue(); - } - - private static final String DEFAULT_CTYPE = "text/xml"; - - private void doProxy(HttpServletRequest req, HttpServletResponse resp) throws IOException { - ZimbraLog.clearContext(); - boolean isAdmin = isAdminRequest(req); - AuthToken authToken = isAdmin ? - getAdminAuthTokenFromCookie(req, resp, true) : getAuthTokenFromCookie(req, resp, true); - if (authToken == null) { - String zAuthToken = req.getParameter(QP_ZAUTHTOKEN); - if (zAuthToken != null) { - try { - authToken = AuthProvider.getAuthToken(zAuthToken); - if (authToken.isExpired()) { - resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken expired"); - return; - } - } catch (AuthTokenException e) { - resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "unable to parse authtoken"); - return; - } - } - } - if (authToken == null) { - resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "no authtoken cookie"); - return; - } - if (!authToken.isRegistered()) { - resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken is invalid"); - return; - } - if (isAdmin && !authToken.isAdmin()) { - resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "permission denied"); - return; - } - - // get the posted body before the server read and parse them. - byte[] body = copyPostedData(req); - - // sanity check - String target = req.getParameter(TARGET_PARAM); - if (target == null) { - resp.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - - // check for permission - URL url = new URL(target); - if (!isAdmin && !checkPermissionOnTarget(url, authToken)) { - resp.sendError(HttpServletResponse.SC_FORBIDDEN); - return; - } - - // determine whether to return the target inline or store it as an upload - String uploadParam = req.getParameter(UPLOAD_PARAM); - boolean asUpload = uploadParam != null && (uploadParam.equals("1") || uploadParam.equalsIgnoreCase("true")); - - HttpRequestBase method = null; - try { - HttpClientBuilder clientBuilder = ZimbraHttpConnectionManager.getExternalHttpConnMgr().newHttpClient(); - HttpProxyUtil.configureProxy(clientBuilder); - String reqMethod = req.getMethod(); - if (reqMethod.equalsIgnoreCase("GET")) { - method = new HttpGet(target); - } else if (reqMethod.equalsIgnoreCase("POST")) { - HttpPost post = new HttpPost(target); - if (body != null) - post.setEntity(new ByteArrayEntity(body, org.apache.http.entity.ContentType.create(req.getContentType()))); - method = post; - } else if (reqMethod.equalsIgnoreCase("PUT")) { - HttpPut put = new HttpPut(target); - if (body != null) - put.setEntity(new ByteArrayEntity(body, org.apache.http.entity.ContentType.create(req.getContentType()))); - method = put; - } else if (reqMethod.equalsIgnoreCase("DELETE")) { - method = new HttpDelete(target); - } else { - ZimbraLog.zimlet.info("unsupported request method: " + reqMethod); - resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); - return; - } - - // handle basic auth - String auth, user, pass; - auth = req.getParameter(AUTH_PARAM); - user = req.getParameter(USER_PARAM); - pass = req.getParameter(PASS_PARAM); - if (auth != null && user != null && pass != null) { - if (!auth.equals(AUTH_BASIC)) { - ZimbraLog.zimlet.info("unsupported auth type: " + auth); - resp.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - CredentialsProvider provider = new BasicCredentialsProvider(); - provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, pass)); - clientBuilder.setDefaultCredentialsProvider(provider); - } - - Enumeration headers = req.getHeaderNames(); - while (headers.hasMoreElements()) { - String hdr = (String) headers.nextElement(); - ZimbraLog.zimlet.debug("incoming: " + hdr + ": " + req.getHeader(hdr)); - if (canProxyHeader(hdr)) { - ZimbraLog.zimlet.debug("outgoing: " + hdr + ": " + req.getHeader(hdr)); - method.addHeader(hdr, req.getHeader(hdr)); - } - } - - HttpResponse httpResp = null; - try { - if (!(reqMethod.equalsIgnoreCase("POST") || reqMethod.equalsIgnoreCase("PUT"))) { - clientBuilder.setRedirectStrategy(new DefaultRedirectStrategy()); - } - - HttpClient client = clientBuilder.build(); - httpResp = HttpClientUtil.executeMethod(client, method); - } catch (HttpException ex) { - ZimbraLog.zimlet.info("exception while proxying " + target, ex); - resp.sendError(HttpServletResponse.SC_NOT_FOUND); - return; - } - - int status = httpResp.getStatusLine() == null ? HttpServletResponse.SC_INTERNAL_SERVER_ERROR :httpResp.getStatusLine().getStatusCode(); - - // workaround for Alexa Thumbnails paid web service, which doesn't bother to return a content-type line - Header ctHeader = httpResp.getFirstHeader("Content-Type"); - String contentType = ctHeader == null || ctHeader.getValue() == null ? DEFAULT_CTYPE : ctHeader.getValue(); - - // getEntity may return null if no response body (e.g. HTTP 204) - InputStream targetResponseBody = null; - HttpEntity targetResponseEntity = httpResp.getEntity(); - if (targetResponseEntity != null) { - targetResponseBody = targetResponseEntity.getContent(); - } - - if (asUpload) { - String filename = req.getParameter(FILENAME_PARAM); - if (filename == null || filename.equals("")) - filename = new ContentType(contentType).getParameter("name"); - if ((filename == null || filename.equals("")) && httpResp.getFirstHeader("Content-Disposition") != null) - filename = new ContentDisposition(httpResp.getFirstHeader("Content-Disposition").getValue()).getParameter("filename"); - if (filename == null || filename.equals("")) - filename = "unknown"; - - List uploads = null; - - if (targetResponseBody != null) { - try { - Upload up = FileUploadServlet.saveUpload(targetResponseBody, filename, contentType, authToken.getAccountId()); - uploads = Arrays.asList(up); - } catch (ServiceException e) { - if (e.getCode().equals(MailServiceException.UPLOAD_REJECTED)) - status = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; - else - status = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; - } - } - - resp.setStatus(status); - FileUploadServlet.sendResponse(resp, status, req.getParameter(FORMAT_PARAM), null, uploads, null); - } else { - resp.setStatus(status); - resp.setContentType(contentType); - for (Header h : httpResp.getAllHeaders()) - if (canProxyHeader(h.getName())) - resp.addHeader(h.getName(), h.getValue()); - if (targetResponseBody != null) - ByteUtil.copy(targetResponseBody, true, resp.getOutputStream(), true); - } - } finally { - if (method != null) - method.releaseConnection(); - } - } -} From 82d462ebab5c7ec55417140831ef4c1cd5965f45 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 14:04:07 +0530 Subject: [PATCH 04/10] feat: use proxy config in PostHogTracking --- .../com/zextras/mailbox/tracking/PostHogTracking.java | 6 +++++- .../zextras/mailbox/tracking/PostHogTrackingTest.java | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java b/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java index 572c092714c..7c03399a105 100644 --- a/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java +++ b/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java @@ -1,5 +1,6 @@ package com.zextras.mailbox.tracking; +import com.zimbra.cs.httpclient.HttpProxyUtil; import java.io.UnsupportedEncodingException; import java.net.URI; import org.apache.http.client.methods.HttpPost; @@ -18,7 +19,10 @@ public PostHogTracking(String endPoint) { @Override public void sendEventIgnoringFailure(Event event) { - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { + var clientBuilder = HttpClientBuilder.create(); + HttpProxyUtil.configureProxy(clientBuilder); + + try (CloseableHttpClient client = clientBuilder.build()) { client.execute(generateRequest(event)); } catch (Exception ignored) { } diff --git a/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java b/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java index ba3b9299df7..e77fb36b554 100644 --- a/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java +++ b/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java @@ -6,7 +6,10 @@ import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; +import com.zextras.mailbox.util.MailboxTestUtil; import java.io.IOException; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,12 +24,14 @@ class PostHogTrackingTest { private final PostHogTracking postHogTracking = new PostHogTracking("http://localhost:" + POSTHOG_PORT); @BeforeEach - public void startUp() throws IOException { + public void startUp() throws Exception { + MailboxTestUtil.setUp(); postHog = startClientAndServer(POSTHOG_PORT); } @AfterEach - public void tearDown() throws IOException { + public void tearDown() throws Exception { + MailboxTestUtil.tearDown(); postHog.stop(); } @@ -62,6 +67,7 @@ private void mockSuccessResponse(HttpRequest request) { .respond(response().withStatusCode(200)); } + @SuppressWarnings("SameParameterValue") private HttpRequest createRequest(String uid, String action) { return request() .withMethod("POST") From 41b64cdc0a8d731a3413fdd6496d15369f1c71ac Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 14:04:38 +0530 Subject: [PATCH 05/10] chore: no need to use proxy here in PreviewServlet since it will make internal connections --- store/src/main/java/com/zimbra/cs/service/PreviewServlet.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/store/src/main/java/com/zimbra/cs/service/PreviewServlet.java b/store/src/main/java/com/zimbra/cs/service/PreviewServlet.java index 59a86548d8d..83030a81edc 100644 --- a/store/src/main/java/com/zimbra/cs/service/PreviewServlet.java +++ b/store/src/main/java/com/zimbra/cs/service/PreviewServlet.java @@ -22,7 +22,6 @@ import com.zimbra.common.util.ZimbraLog; import com.zimbra.cs.account.Account; import com.zimbra.cs.account.AuthToken; -import com.zimbra.cs.httpclient.HttpProxyUtil; import com.zimbra.cs.servlet.ZimbraServlet; import com.zimbra.cs.util.AccountUtil; import io.vavr.control.Try; @@ -259,7 +258,6 @@ private Try getAttachment(AuthToken authToken, String message () -> { HttpClientBuilder clientBuilder = ZimbraHttpConnectionManager.getInternalHttpConnMgr().newHttpClient(); - HttpProxyUtil.configureProxy(clientBuilder); HttpGet getRequest = new HttpGet(getContentServletResourceUrl(authToken, messageId, part)); HttpClient client = From 9d41d4c486cc0fd8948345a479db4219149b2b27 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 14:46:57 +0530 Subject: [PATCH 06/10] chore: add test for HttpProxyUtil + add IllegalArgumentException if the proxy url is malformed --- .../zimbra/cs/httpclient/HttpProxyUtil.java | 5 +- .../cs/httpclient/HttpProxyUtilTest.java | 128 ++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 store/src/test/java/com/zimbra/cs/httpclient/HttpProxyUtilTest.java diff --git a/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java b/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java index 9c995fc8d0c..8b637e807bc 100644 --- a/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java +++ b/store/src/main/java/com/zimbra/cs/httpclient/HttpProxyUtil.java @@ -27,7 +27,8 @@ private HttpProxyUtil() { public static synchronized void configureProxy(HttpClientBuilder clientBuilder) { try { - final var httpProxyUrl = Provisioning.getInstance().getLocalServer().getAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, null); + final var httpProxyUrl = Provisioning.getInstance().getLocalServer() + .getAttr(ZAttrProvisioning.A_zimbraHttpProxyURL, null); if (httpProxyUrl == null || httpProxyUrl.isEmpty()) { ZimbraLog.misc.info("HttpProxyUtil.configureProxy 'zimbraHttpProxyURL' is null or empty, not using proxy."); return; @@ -67,7 +68,7 @@ public static synchronized void configureProxy(HttpClientBuilder clientBuilder) ZimbraLog.misc.debug("setting proxy: " + httpProxyUrl); } - } catch (ServiceException | URISyntaxException e) { + } catch (ServiceException | URISyntaxException | IllegalArgumentException e) { ZimbraLog.misc.warn("Unable to configureProxy: " + e.getMessage(), e); } } diff --git a/store/src/test/java/com/zimbra/cs/httpclient/HttpProxyUtilTest.java b/store/src/test/java/com/zimbra/cs/httpclient/HttpProxyUtilTest.java new file mode 100644 index 00000000000..285c92f0dca --- /dev/null +++ b/store/src/test/java/com/zimbra/cs/httpclient/HttpProxyUtilTest.java @@ -0,0 +1,128 @@ +package com.zimbra.cs.httpclient; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import com.zextras.mailbox.util.MailboxTestUtil; +import com.zimbra.common.service.ServiceException; +import com.zimbra.cs.account.Provisioning; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +class HttpProxyUtilTest { + + @BeforeEach + public void setUp() throws Exception { + MailboxTestUtil.setUp(); + } + + @AfterEach + public void tearDown() throws Exception { + MailboxTestUtil.tearDown(); + } + + @Test + void testConstructor() throws NoSuchMethodException { + Constructor constructor = HttpProxyUtil.class.getDeclaredConstructor(); + constructor.setAccessible(true); + IllegalStateException thrown = assertThrows(IllegalStateException.class, () -> { + try { + constructor.newInstance(); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + + assertEquals("Utility class", thrown.getMessage()); + } + + @Test + void testConfigureProxy_noOrNullProxyUrl_doesNothing() throws ServiceException { + Provisioning.getInstance().getLocalServer().setHttpProxyURL(null); + + HttpClientBuilder httpClientBuilderSpy = Mockito.spy(HttpClientBuilder.create()); + HttpProxyUtil.configureProxy(httpClientBuilderSpy); + + verify(httpClientBuilderSpy, never()).setDefaultRequestConfig(Mockito.any(RequestConfig.class)); + verify(httpClientBuilderSpy, never()).setDefaultCredentialsProvider(Mockito.any(CredentialsProvider.class)); + } + + @Test + void testConfigureProxy_withProxyUrl_setsProxy() throws Exception { + Provisioning.getInstance().getLocalServer().setHttpProxyURL("http://user:pass@proxyhost:8080"); + + HttpClientBuilder httpClientBuilderSpy = Mockito.spy(HttpClientBuilder.create()); + HttpProxyUtil.configureProxy(httpClientBuilderSpy); + + ArgumentCaptor configCaptor = ArgumentCaptor.forClass(RequestConfig.class); + verify(httpClientBuilderSpy).setDefaultRequestConfig(configCaptor.capture()); + RequestConfig config = configCaptor.getValue(); + + HttpHost proxy = config.getProxy(); + Assertions.assertNotNull(proxy); + assertEquals("proxyhost", proxy.getHostName()); + assertEquals(8080, proxy.getPort()); + + ArgumentCaptor credentialsCaptor = ArgumentCaptor.forClass(CredentialsProvider.class); + verify(httpClientBuilderSpy).setDefaultCredentialsProvider(credentialsCaptor.capture()); + CredentialsProvider credentialsProvider = credentialsCaptor.getValue(); + + UsernamePasswordCredentials creds = (UsernamePasswordCredentials) credentialsProvider.getCredentials(AuthScope.ANY); + Assertions.assertNotNull(creds); + assertEquals("user", creds.getUserName()); + assertEquals("pass", creds.getPassword()); + } + + @Test + void testConfigureProxy_withProxyUrlWithoutCredentials_setsProxy() throws Exception { + Provisioning.getInstance().getLocalServer().setHttpProxyURL("http://proxyhost:8080"); + + HttpClientBuilder httpClientBuilderSpy = Mockito.spy(HttpClientBuilder.create()); + HttpProxyUtil.configureProxy(httpClientBuilderSpy); + + ArgumentCaptor configCaptor = ArgumentCaptor.forClass(RequestConfig.class); + verify(httpClientBuilderSpy).setDefaultRequestConfig(configCaptor.capture()); + RequestConfig config = configCaptor.getValue(); + + HttpHost proxy = config.getProxy(); + Assertions.assertNotNull(proxy); + assertEquals("proxyhost", proxy.getHostName()); + assertEquals(8080, proxy.getPort()); + } + + @Test + void testConfigureProxy_malformedUrl_doesNothing() throws ServiceException { + Provisioning.getInstance().getLocalServer().setHttpProxyURL("proxyhost/8080"); + + HttpClientBuilder httpClientBuilderSpy = Mockito.spy(HttpClientBuilder.create()); + HttpProxyUtil.configureProxy(httpClientBuilderSpy); + + verify(httpClientBuilderSpy, never()).setDefaultRequestConfig(Mockito.any(RequestConfig.class)); + verify(httpClientBuilderSpy, never()).setDefaultCredentialsProvider(Mockito.any(CredentialsProvider.class)); + } + + @Test + void testConfigureProxy_emptyProxyUrl_doesNothing() throws ServiceException { + Provisioning.getInstance().getLocalServer().setHttpProxyURL(""); + + HttpClientBuilder httpClientBuilderSpy = Mockito.spy(HttpClientBuilder.create()); + HttpProxyUtil.configureProxy(httpClientBuilderSpy); + + verify(httpClientBuilderSpy, never()).setDefaultRequestConfig(Mockito.any(RequestConfig.class)); + verify(httpClientBuilderSpy, never()).setDefaultCredentialsProvider(Mockito.any(CredentialsProvider.class)); + } +} \ No newline at end of file From efc29ef1da4d5cb42011cfa33fafa7f6e4fc3a61 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 16:06:09 +0530 Subject: [PATCH 07/10] feat: use proxy config in CallToHome --- store/src/main/java/com/zimbra/cs/util/CallToHome.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/com/zimbra/cs/util/CallToHome.java b/store/src/main/java/com/zimbra/cs/util/CallToHome.java index ef19eb54a13..fc726049adb 100644 --- a/store/src/main/java/com/zimbra/cs/util/CallToHome.java +++ b/store/src/main/java/com/zimbra/cs/util/CallToHome.java @@ -11,16 +11,20 @@ import com.zimbra.cs.account.Domain; import com.zimbra.cs.account.NamedEntry; import com.zimbra.cs.account.Provisioning; +import com.zimbra.cs.httpclient.HttpProxyUtil; import java.io.IOException; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.IOUtils; import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; import org.json.JSONException; import org.json.JSONObject; @@ -107,8 +111,9 @@ public void postDataToEndpoint(JSONObject data) throws IOException { ZimbraLog.mailbox.info( "CallToHome: started posting data: " + IOUtils.toString(post.getEntity().getContent())); } - CloseableHttpClient client = - ZimbraHttpConnectionManager.getInternalHttpConnMgr().newHttpClient().build(); + final HttpClientBuilder httpClientBuilder = HttpClients.custom(); + HttpProxyUtil.configureProxy(httpClientBuilder); + CloseableHttpClient client = httpClientBuilder.build(); try { CloseableHttpResponse httpResp = client.execute(post); int statusCode = httpResp.getStatusLine().getStatusCode(); From 57c748caad21b1e91ed53707ff640e588355d4a1 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 17:21:55 +0530 Subject: [PATCH 08/10] feat: CallToHome refactoring --- .../java/com/zimbra/cs/util/CallToHome.java | 209 ------------------ .../main/java/com/zimbra/cs/util/Zimbra.java | 6 +- .../zimbra/cs/util/calltohome/CallToHome.java | 117 ++++++++++ .../cs/util/calltohome/CallToHomeRunner.java | 67 ++++++ .../com/zimbra/cs/util/CallToHomeTest.java | 79 ------- .../util/calltohome/CallToHomeRunnerTest.java | 75 +++++++ 6 files changed, 262 insertions(+), 291 deletions(-) delete mode 100644 store/src/main/java/com/zimbra/cs/util/CallToHome.java create mode 100644 store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java create mode 100644 store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java delete mode 100644 store/src/test/java/com/zimbra/cs/util/CallToHomeTest.java create mode 100644 store/src/test/java/com/zimbra/cs/util/calltohome/CallToHomeRunnerTest.java diff --git a/store/src/main/java/com/zimbra/cs/util/CallToHome.java b/store/src/main/java/com/zimbra/cs/util/CallToHome.java deleted file mode 100644 index fc726049adb..00000000000 --- a/store/src/main/java/com/zimbra/cs/util/CallToHome.java +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Zextras -// -// SPDX-License-Identifier: GPL-2.0-only - -package com.zimbra.cs.util; - -import com.zimbra.common.service.ServiceException; -import com.zimbra.common.util.StringUtil; -import com.zimbra.common.util.ZimbraHttpConnectionManager; -import com.zimbra.common.util.ZimbraLog; -import com.zimbra.cs.account.Domain; -import com.zimbra.cs.account.NamedEntry; -import com.zimbra.cs.account.Provisioning; -import com.zimbra.cs.httpclient.HttpProxyUtil; -import java.io.IOException; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.json.JSONException; -import org.json.JSONObject; - -class CallToHomeRunner { - private static final long MILLIS_IN_DAY = 1000L * 60L * 60L * 24L; - private static final Timer timer = new Timer(true); - private static final boolean LOGGING = false; - private static boolean started = false; - - public static boolean isStarted() { - return started; - } - - public static void init() { - if (isStarted()) { - if (LOGGING) { - ZimbraLog.mailbox.info("CallToHome: already running"); - } - return; - } - Timer t = new Timer(); - // start service after 5 minutes - t.schedule( - new TimerTask() { - @Override - public void run() { - CallToHome task = new CallToHome(LOGGING); - timer.scheduleAtFixedRate(task, 0L, MILLIS_IN_DAY); - started = true; - if (LOGGING) { - ZimbraLog.mailbox.info("CallToHome: Started"); - } - t.cancel(); - } - }, - 300000L); - } - - public boolean isRunning() { - return started; - } - - public void stop() { - timer.cancel(); - started = false; - if (LOGGING) { - ZimbraLog.mailbox.info("CallToHome: Stopped"); - } - } -} - -public class CallToHome extends TimerTask { - - private static final String UPDATE_URL = "https://updates.zextras.com/openchat"; - private static boolean logging = false; - Provisioning prov = Provisioning.getInstance(); - - public CallToHome(boolean logging) { - CallToHome.logging = logging; - } - - @Override - public void run() { - if (logging) { - ZimbraLog.mailbox.info("CallToHome: Running"); - } - try { - postDataToEndpoint(getData()); - } catch (IOException e) { - if (logging) { - ZimbraLog.mailbox.info("CallToHome: response: " + e.getMessage()); - } - } - } - - public void postDataToEndpoint(JSONObject data) throws IOException { - if (logging) { - throw new IOException("logging is enabled preventing posting data"); - } - HttpPost post = new HttpPost(UPDATE_URL); - post.setHeader("Content-Type", "application/json; charset=UTF-8"); - post.setEntity(new StringEntity(data.toString(), "UTF-8")); - if (logging) { - ZimbraLog.mailbox.info( - "CallToHome: started posting data: " + IOUtils.toString(post.getEntity().getContent())); - } - final HttpClientBuilder httpClientBuilder = HttpClients.custom(); - HttpProxyUtil.configureProxy(httpClientBuilder); - CloseableHttpClient client = httpClientBuilder.build(); - try { - CloseableHttpResponse httpResp = client.execute(post); - int statusCode = httpResp.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { - throw new IOException( - "request returned with status " - + statusCode - + ":" - + httpResp.getStatusLine().getReasonPhrase(), - null); - } else { - if (logging) { - ZimbraLog.mailbox.info("CallToHome: OK"); - } - } - } catch (IOException e) { - throw new IOException("unexpected error during post operation: " + e.getMessage()); - } finally { - post.releaseConnection(); - } - } - - private JSONObject getData() { - JSONObject json = new JSONObject(); - try { - json.put("zimbraVersion", getCurrentZimbraVersion()); - json.put("isNetwork", false); - json.put("serverName", getLocalHostname()); - json.put("numAccounts", getAccountCounts(true)); - } catch (JSONException | ServiceException e) { - if (logging) { - ZimbraLog.mailbox.info("CallToHome: " + e.getMessage()); - } - } - return json; - } - - private String getLocalHostname() { - String localHostname = "unknown"; - try { - localHostname = prov.getLocalServer().getName(); - } catch (ServiceException e) { - if (logging) { - ZimbraLog.mailbox.info( - "CallToHome: Unable to determine local hostname using " - + localHostname - + " as localHostname", - e); - } - } - return localHostname; - } - - private String getCurrentZimbraVersion() { - String versionInfo = BuildInfo.VERSION; - if (StringUtil.isNullOrEmpty(versionInfo)) versionInfo = "unknown"; - return versionInfo; - } - - private int getAccountCounts(boolean usingVisitor) throws ServiceException { - if (logging) { - ZimbraLog.mailbox.info("CallToHome: counting using " + (usingVisitor ? "visitor" : "stream")); - } - AtomicInteger accountCount = new AtomicInteger(); - if (usingVisitor) { - NamedEntry.Visitor visitor = - entry -> { - Provisioning.CountAccountResult result = prov.countAccount((Domain) entry); - accountCount.addAndGet( - result.getCountAccountByCos().stream() - .mapToInt(c -> (int) c.getCount()) - .reduce(0, Integer::sum)); - }; - prov.getAllDomains(visitor, new String[] {Provisioning.A_zimbraId}); - } else { - prov.getAllDomains().parallelStream() - .forEach( - domain -> { - try { - accountCount.addAndGet( - prov.countAccount(domain).getCountAccountByCos().stream() - .mapToInt(c -> (int) c.getCount()) - .reduce(0, Integer::sum)); - } catch (ServiceException e) { - if (logging) { - ZimbraLog.mailbox.error("CallToHome: unable to get domains", e); - } - } - }); - } - return accountCount.get(); - } -} diff --git a/store/src/main/java/com/zimbra/cs/util/Zimbra.java b/store/src/main/java/com/zimbra/cs/util/Zimbra.java index cdb62417b9f..911ce507a2e 100644 --- a/store/src/main/java/com/zimbra/cs/util/Zimbra.java +++ b/store/src/main/java/com/zimbra/cs/util/Zimbra.java @@ -41,11 +41,13 @@ import com.zimbra.cs.session.WaitSetMgr; import com.zimbra.cs.stats.ZimbraPerf; import com.zimbra.cs.store.StoreManager; +import com.zimbra.cs.util.calltohome.CallToHomeRunner; import com.zimbra.znative.Util; import java.io.File; import java.io.IOException; import java.security.Security; import java.util.Timer; +import java.util.concurrent.TimeUnit; import org.apache.mina.core.buffer.IoBuffer; import org.dom4j.DocumentException; @@ -58,8 +60,6 @@ public final class Zimbra { private static boolean sIsMailboxd = false; private static final String HEAP_DUMP_JAVA_OPTION = "-xx:heapdumppath="; public static final Timer sTimer = new Timer("Timer-Zimbra", true); - private static final CallToHomeRunner c2hRunner = new CallToHomeRunner(); - private Zimbra() { throw new IllegalStateException("Utility class"); } @@ -359,7 +359,7 @@ private static synchronized void startup(boolean forMailboxd) throws ServiceExce ZimbraPerf.initialize(ZimbraPerf.ServerID.ZIMBRA); } - c2hRunner.init(); + CallToHomeRunner.getInstance().init(TimeUnit.MINUTES.toMillis(5)); } ExtensionUtil.postInitAll(); diff --git a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java new file mode 100644 index 00000000000..1ee700af00d --- /dev/null +++ b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2022 Zextras +// +// SPDX-License-Identifier: GPL-2.0-only + +package com.zimbra.cs.util.calltohome; + +import com.zimbra.common.account.ZAttrProvisioning; +import com.zimbra.common.service.ServiceException; +import com.zimbra.common.util.StringUtil; +import com.zimbra.common.util.ZimbraLog; +import com.zimbra.cs.account.Domain; +import com.zimbra.cs.account.NamedEntry; +import com.zimbra.cs.account.Provisioning; +import com.zimbra.cs.httpclient.HttpProxyUtil; +import com.zimbra.cs.util.BuildInfo; +import java.io.IOException; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClients; +import org.json.JSONException; +import org.json.JSONObject; + +public class CallToHome extends TimerTask { + + private static final String UPDATE_URL = "https://updates.zextras.com/openchat"; + private final Provisioning prov = Provisioning.getInstance(); + + @Override + public void run() { + ZimbraLog.misc.debug("CallToHome: Running..."); + try { + postDataToEndpoint(getData()); + } catch (IOException e) { + ZimbraLog.misc.error("CallToHome: Error posting data", e); + } + } + + private void postDataToEndpoint(JSONObject data) throws IOException { + var post = new HttpPost(UPDATE_URL); + post.setHeader("Content-Type", "application/json; charset=UTF-8"); + post.setEntity(new StringEntity(data.toString(), "UTF-8")); + ZimbraLog.misc.debug("CallToHome: Posting data: " + data); + + var httpClientBuilder = HttpClients.custom(); + HttpProxyUtil.configureProxy(httpClientBuilder); + try (var client = httpClientBuilder.build(); + var httpResp = client.execute(post)) { + var statusCode = httpResp.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + var reason = httpResp.getStatusLine().getReasonPhrase(); + ZimbraLog.misc.error("CallToHome: Request returned with status " + statusCode + ": " + reason); + throw new IOException("Request failed with status " + statusCode + ": " + reason); + } + + ZimbraLog.misc.debug("CallToHome: Data posted successfully"); + + } catch (IOException e) { + ZimbraLog.misc.error("CallToHome: Unexpected error during post operation", e); + throw e; + } + } + + private JSONObject getData() { + var json = new JSONObject(); + try { + json.put("zimbraVersion", getCurrentZimbraVersion()); + json.put("isNetwork", false); + json.put("serverName", getLocalHostname()); + json.put("numAccounts", getAccountCounts()); + } catch (JSONException | ServiceException e) { + ZimbraLog.misc.error("CallToHome: Error gathering data", e); + } + return json; + } + + private String getLocalHostname() { + try { + return prov.getLocalServer().getName(); + } catch (ServiceException e) { + ZimbraLog.misc.error("CallToHome: Unable to determine local hostname", e); + return "unknown"; + } + } + + private String getCurrentZimbraVersion() { + var versionInfo = BuildInfo.VERSION; + if (StringUtil.isNullOrEmpty(versionInfo)) { + versionInfo = "unknown"; + } + return versionInfo; + } + + private int getAccountCounts() throws ServiceException { + ZimbraLog.misc.debug("CallToHome: Counting accounts..."); + var accountCount = new AtomicInteger(); + + var visitor = (NamedEntry.Visitor) entry -> { + try { + var result = prov.countAccount((Domain) entry); + accountCount.addAndGet(result.getCountAccountByCos().stream() + .mapToInt(c -> (int) c.getCount()) + .sum()); + } catch (ServiceException e) { + ZimbraLog.misc.error("CallToHome: Error counting accounts for domain: " + entry.getName(), e); + } + }; + + prov.getAllDomains(visitor, new String[]{ZAttrProvisioning.A_zimbraId}); + ZimbraLog.misc.debug("CallToHome: Counting accounts done!"); + + return accountCount.get(); + } +} + diff --git a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java new file mode 100644 index 00000000000..c30ec0c78bc --- /dev/null +++ b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2022 Zextras +// +// SPDX-License-Identifier: GPL-2.0-only + +package com.zimbra.cs.util.calltohome; + +import com.zimbra.common.util.ZimbraLog; +import java.util.Timer; +import java.util.TimerTask; + +public class CallToHomeRunner { + + private static final long MILLIS_IN_DAY = 1000L * 60L * 60L * 24L; + private static Timer timer = new Timer(true); + private static volatile boolean started = false; + + private CallToHomeRunner() { + } + + public static CallToHomeRunner getInstance() { + return Holder.INSTANCE; + } + + public synchronized boolean isStarted() { + return started; + } + + public synchronized void init(long startupDelay) { + if (isStarted()) { + ZimbraLog.misc.debug("CallToHome: already running"); + return; + }else{ + ZimbraLog.misc.debug("CallToHome: init(), scheduled to start in " + startupDelay + " milliseconds"); + } + + timer.schedule( + new TimerTask() { + @Override + public void run() { + CallToHome task = new CallToHome(); + timer.scheduleAtFixedRate(task, 0L, MILLIS_IN_DAY); + started = true; + ZimbraLog.misc.debug("CallToHome: Started"); + } + }, + startupDelay + ); + } + + public synchronized void stop() { + if (!isStarted()) { + ZimbraLog.misc.debug("CallToHome: not running"); + return; + } + + timer.cancel(); + timer = new Timer(true); + started = false; + ZimbraLog.misc.debug("CallToHome: Stopped"); + } + + private static class Holder { + + private static final CallToHomeRunner INSTANCE = new CallToHomeRunner(); + } +} + diff --git a/store/src/test/java/com/zimbra/cs/util/CallToHomeTest.java b/store/src/test/java/com/zimbra/cs/util/CallToHomeTest.java deleted file mode 100644 index 900863ecf5e..00000000000 --- a/store/src/test/java/com/zimbra/cs/util/CallToHomeTest.java +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Zextras -// -// SPDX-License-Identifier: GPL-2.0-only - -package com.zimbra.cs.util; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.zimbra.common.localconfig.LC; -import com.zimbra.cs.account.MockProvisioning; -import com.zimbra.cs.account.Provisioning; -import com.zimbra.cs.mailbox.MailboxTestUtil; -import org.apache.commons.io.IOUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import java.io.IOException; - -public class CallToHomeTest { - - private static final String UPDATE_URL = "https://updates.zextras.com/openchat"; - - @BeforeAll - public static void init() throws Exception { - MailboxTestUtil.initServer(); - LC.zimbra_attrs_directory.setDefault(MailboxTestUtil.getZimbraServerDir("") + "conf/attrs"); - MockProvisioning prov = new MockProvisioning(); - prov.getLocalServer().setSmtpPort(25); - Provisioning.setInstance(prov); - } - - private JSONObject fakeData() throws JSONException { - JSONObject json = new JSONObject(); - json.put("zimbraVersion", "8.8.15_GA_1001"); - json.put("isNetwork", false); - json.put("serverName", "test_server_name"); - json.put("numAccounts", 0); - return json; - } - - @Test - void whenGetServerName() throws Exception { - - Provisioning provisioning = Provisioning.getInstance(); - assertEquals( - "localhost", provisioning.getLocalServer().getName(), "localserver name is "); - } - - @Test - void whenGetData() throws JSONException { - JSONObject parsed = new JSONObject(fakeData().toString()); - assertEquals(fakeData().toString(), parsed.toString(), "parsed json is valid"); - } - - @Test - void whenPostDataToEndpoint() throws IOException, JSONException { - JSONObject data = fakeData(); - HttpPost post = new HttpPost(UPDATE_URL); - post.setHeader("Content-Type", "application/json; charset=UTF-8"); - StringEntity stringEntity = new StringEntity(data.toString()); - post.setEntity(stringEntity); - CloseableHttpClient client = HttpClients.createDefault(); - try { - CloseableHttpResponse httpResp = client.execute(post); - int statusCode = httpResp.getStatusLine().getStatusCode(); - assertEquals(200, statusCode, "status code is not 200"); - } catch (Exception e) { - throw new IOException("unexpected error during post operation: " + e.getMessage()); - } finally { - post.releaseConnection(); - } - } -} diff --git a/store/src/test/java/com/zimbra/cs/util/calltohome/CallToHomeRunnerTest.java b/store/src/test/java/com/zimbra/cs/util/calltohome/CallToHomeRunnerTest.java new file mode 100644 index 00000000000..2fed04149f5 --- /dev/null +++ b/store/src/test/java/com/zimbra/cs/util/calltohome/CallToHomeRunnerTest.java @@ -0,0 +1,75 @@ +package com.zimbra.cs.util.calltohome; + +import com.zextras.mailbox.util.MailboxTestUtil; +import com.zimbra.common.service.ServiceException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; + +class CallToHomeRunnerTest { + + private CallToHomeRunner callToHomeRunner; + + @BeforeEach + public void setUp() throws Exception { + MailboxTestUtil.setUp(); + callToHomeRunner = CallToHomeRunner.getInstance(); + } + + @AfterEach + public void tearDown() throws ServiceException { + MailboxTestUtil.tearDown(); + callToHomeRunner.stop(); + } + + @Test + void testInitAndStop() throws InterruptedException { + assertFalse(callToHomeRunner.isStarted(), "Initially should not be started"); + + callToHomeRunner.init(TimeUnit.SECONDS.toMillis(1)); + TimeUnit.SECONDS.sleep(2); + + assertTrue(callToHomeRunner.isStarted(), "After init, should be started"); + + callToHomeRunner.stop(); + assertFalse(callToHomeRunner.isStarted(), "After stop, should not be started"); + } + + @Test + void testInitTwice() throws InterruptedException { + assertFalse(callToHomeRunner.isStarted(), "Initially should not be started"); + + callToHomeRunner.init(TimeUnit.SECONDS.toMillis(1)); + TimeUnit.SECONDS.sleep(2); + + assertTrue(callToHomeRunner.isStarted(), "After first init, should be started"); + + callToHomeRunner.init(TimeUnit.SECONDS.toMillis(1)); + assertTrue(callToHomeRunner.isStarted(), "After second init, should still be started"); + } + + @Test + void testStopWithoutInit() { + assertFalse(callToHomeRunner.isStarted(), "Initially should not be started"); + + callToHomeRunner.stop(); + assertFalse(callToHomeRunner.isStarted(), "After stop without init, should still not be started"); + } + + @Test + void testInitAndStopDelay() throws InterruptedException { + assertFalse(callToHomeRunner.isStarted(), "Initially should not be started"); + + callToHomeRunner.init(TimeUnit.SECONDS.toMillis(1)); + TimeUnit.SECONDS.sleep(2); + + assertTrue(callToHomeRunner.isStarted(), "After init, should be started"); + + callToHomeRunner.stop(); + assertFalse(callToHomeRunner.isStarted(), "After stop, should not be started"); + } +} From a1c8e3063c3caf02a930995eb5153135cddef0e3 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 18:18:25 +0530 Subject: [PATCH 09/10] feat: introduce HttpClientFactory --- .../cs/httpclient/HttpClientFactory.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 store/src/main/java/com/zimbra/cs/httpclient/HttpClientFactory.java diff --git a/store/src/main/java/com/zimbra/cs/httpclient/HttpClientFactory.java b/store/src/main/java/com/zimbra/cs/httpclient/HttpClientFactory.java new file mode 100644 index 00000000000..cdc62cbb0d9 --- /dev/null +++ b/store/src/main/java/com/zimbra/cs/httpclient/HttpClientFactory.java @@ -0,0 +1,27 @@ +package com.zimbra.cs.httpclient; + +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +/** + * Factory class for creating instances of {@link CloseableHttpClient}. + * This factory provides methods to create HTTP clients with various configurations. + */ +public class HttpClientFactory { + + /** + * Creates a {@link CloseableHttpClient} instance configured with proxy settings. + *

+ * This method uses the {@link HttpProxyUtil} class to configure proxy settings + * for the HTTP client builder before building the {@link CloseableHttpClient}. + *

+ * + * @return a {@link CloseableHttpClient} instance configured with proxy settings + */ + public CloseableHttpClient createWithProxy() { + final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + HttpProxyUtil.configureProxy(httpClientBuilder); + return httpClientBuilder.build(); + } +} + From 35f5db14ea98df4e61727f022b2119325a6b0f56 Mon Sep 17 00:00:00 2001 From: Keshav Bhatt Date: Tue, 21 May 2024 18:18:49 +0530 Subject: [PATCH 10/10] chore: use HttpClientFactory to create http client with proxy setting --- .../mailbox/tracking/PostHogTracking.java | 14 ++++++-------- .../zimbra/cs/service/mail/MailService.java | 3 ++- .../zimbra/cs/util/calltohome/CallToHome.java | 13 ++++++++----- .../cs/util/calltohome/CallToHomeRunner.java | 3 ++- .../mailbox/tracking/PostHogTrackingTest.java | 19 +++++++++++-------- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java b/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java index 7c03399a105..9e3f72ebb00 100644 --- a/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java +++ b/store/src/main/java/com/zextras/mailbox/tracking/PostHogTracking.java @@ -1,28 +1,26 @@ package com.zextras.mailbox.tracking; -import com.zimbra.cs.httpclient.HttpProxyUtil; +import com.zimbra.cs.httpclient.HttpClientFactory; import java.io.UnsupportedEncodingException; import java.net.URI; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; public class PostHogTracking implements Tracking { static final String SITE_KEY = "phc_egpFZ14OKByQMK51wCTzYp8tLrg0VA8wa2QDagXCjDG"; private final String endPoint; - public PostHogTracking(String endPoint) { + private final HttpClientFactory httpClientFactory; + + public PostHogTracking(String endPoint, HttpClientFactory httpClientFactory) { this.endPoint = endPoint; + this.httpClientFactory = httpClientFactory; } @Override public void sendEventIgnoringFailure(Event event) { - var clientBuilder = HttpClientBuilder.create(); - HttpProxyUtil.configureProxy(clientBuilder); - - try (CloseableHttpClient client = clientBuilder.build()) { + try (var client = httpClientFactory.createWithProxy()) { client.execute(generateRequest(event)); } catch (Exception ignored) { } diff --git a/store/src/main/java/com/zimbra/cs/service/mail/MailService.java b/store/src/main/java/com/zimbra/cs/service/mail/MailService.java index 7f8af776809..d76fa7c76bf 100644 --- a/store/src/main/java/com/zimbra/cs/service/mail/MailService.java +++ b/store/src/main/java/com/zimbra/cs/service/mail/MailService.java @@ -11,6 +11,7 @@ import com.zextras.mailbox.tracking.Tracking; import com.zimbra.common.soap.MailConstants; import com.zimbra.cs.account.Provisioning; +import com.zimbra.cs.httpclient.HttpClientFactory; import com.zimbra.cs.service.MailboxAttachmentService; import com.zimbra.soap.DocumentDispatcher; import com.zimbra.soap.DocumentService; @@ -249,7 +250,7 @@ protected Provisioning getProvisioning() { } protected Tracking getTracking() { - return new PostHogTracking("https://eu.posthog.com"); + return new PostHogTracking("https://eu.posthog.com", new HttpClientFactory()); } protected FilesClient getFilesClient() { diff --git a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java index 1ee700af00d..e1ffa1dbbce 100644 --- a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java +++ b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHome.java @@ -11,7 +11,7 @@ import com.zimbra.cs.account.Domain; import com.zimbra.cs.account.NamedEntry; import com.zimbra.cs.account.Provisioning; -import com.zimbra.cs.httpclient.HttpProxyUtil; +import com.zimbra.cs.httpclient.HttpClientFactory; import com.zimbra.cs.util.BuildInfo; import java.io.IOException; import java.util.TimerTask; @@ -19,7 +19,6 @@ import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.HttpClients; import org.json.JSONException; import org.json.JSONObject; @@ -28,6 +27,12 @@ public class CallToHome extends TimerTask { private static final String UPDATE_URL = "https://updates.zextras.com/openchat"; private final Provisioning prov = Provisioning.getInstance(); + private final HttpClientFactory httpClientFactory; + + public CallToHome(HttpClientFactory httpClientFactory) { + this.httpClientFactory = httpClientFactory; + } + @Override public void run() { ZimbraLog.misc.debug("CallToHome: Running..."); @@ -44,9 +49,7 @@ private void postDataToEndpoint(JSONObject data) throws IOException { post.setEntity(new StringEntity(data.toString(), "UTF-8")); ZimbraLog.misc.debug("CallToHome: Posting data: " + data); - var httpClientBuilder = HttpClients.custom(); - HttpProxyUtil.configureProxy(httpClientBuilder); - try (var client = httpClientBuilder.build(); + try (var client = httpClientFactory.createWithProxy(); var httpResp = client.execute(post)) { var statusCode = httpResp.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { diff --git a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java index c30ec0c78bc..62925412169 100644 --- a/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java +++ b/store/src/main/java/com/zimbra/cs/util/calltohome/CallToHomeRunner.java @@ -5,6 +5,7 @@ package com.zimbra.cs.util.calltohome; import com.zimbra.common.util.ZimbraLog; +import com.zimbra.cs.httpclient.HttpClientFactory; import java.util.Timer; import java.util.TimerTask; @@ -37,7 +38,7 @@ public synchronized void init(long startupDelay) { new TimerTask() { @Override public void run() { - CallToHome task = new CallToHome(); + CallToHome task = new CallToHome(new HttpClientFactory()); timer.scheduleAtFixedRate(task, 0L, MILLIS_IN_DAY); started = true; ZimbraLog.misc.debug("CallToHome: Started"); diff --git a/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java b/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java index e77fb36b554..a47819562cb 100644 --- a/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java +++ b/store/src/test/java/com/zextras/mailbox/tracking/PostHogTrackingTest.java @@ -1,15 +1,15 @@ package com.zextras.mailbox.tracking; import static com.zextras.mailbox.tracking.PostHogTracking.SITE_KEY; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; -import com.zextras.mailbox.util.MailboxTestUtil; -import java.io.IOException; -import org.junit.Before; -import org.junit.BeforeClass; +import com.zimbra.cs.httpclient.HttpClientFactory; +import org.apache.http.impl.client.HttpClients; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,17 +21,20 @@ class PostHogTrackingTest { private ClientAndServer postHog; private static final int POSTHOG_PORT = 5000; - private final PostHogTracking postHogTracking = new PostHogTracking("http://localhost:" + POSTHOG_PORT); + + private final HttpClientFactory httpClientFactoryMock = mock(HttpClientFactory.class); + + private final PostHogTracking postHogTracking = new PostHogTracking("http://localhost:" + POSTHOG_PORT, + httpClientFactoryMock); @BeforeEach public void startUp() throws Exception { - MailboxTestUtil.setUp(); + when(httpClientFactoryMock.createWithProxy()).thenReturn(HttpClients.createMinimal()); postHog = startClientAndServer(POSTHOG_PORT); } @AfterEach public void tearDown() throws Exception { - MailboxTestUtil.tearDown(); postHog.stop(); }