Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

userdata: fix append scenarios #7741

Merged
merged 7 commits into from
Jul 19, 2023
Merged

userdata: fix append scenarios #7741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3bc530e
userdata: fix append scenarios
shwstppr Jul 12, 2023
bc1ac0f
fix vm manual userdata formatting, refactor
shwstppr Jul 14, 2023
6d7b8ba
license
shwstppr Jul 14, 2023
d774e3f
newline
shwstppr Jul 14, 2023
d81d7d9
fix tests
shwstppr Jul 14, 2023
356c354
fix adding same type userdata type to multipart
shwstppr Jul 14, 2023
0db0f43
test: properly check multipart userdata
shwstppr Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@
// under the License.
package org.apache.cloudstack.userdata;

import com.cloud.utils.component.Manager;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.framework.config.Configurable;

import com.cloud.utils.component.Manager;

public interface UserDataManager extends Manager, Configurable {
String concatenateUserData(String userdata1, String userdata2, String userdataProvider);
String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod);
}
5 changes: 5 additions & 0 deletions engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.Map;
import java.util.Set;

import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO;

import com.cloud.dc.ClusterVO;
Expand Down Expand Up @@ -59,6 +60,10 @@ public interface ConfigurationManager {
public static final String MESSAGE_CREATE_VLAN_IP_RANGE_EVENT = "Message.CreateVlanIpRange.Event";
public static final String MESSAGE_DELETE_VLAN_IP_RANGE_EVENT = "Message.DeleteVlanIpRange.Event";

static final String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length";
static final ConfigKey<Integer> VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768",
"Max length of vm userdata after base64 decoding. Default is 32768 and maximum is 1048576", true);

/**
* @param offering
* @return
Expand Down
134 changes: 106 additions & 28 deletions ...ta/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
Expand All @@ -35,12 +35,14 @@
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;

import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.exception.CloudRuntimeException;
import com.sun.mail.util.BASE64DecoderStream;

public class CloudInitUserDataProvider extends AdapterBase implements UserDataProvider {

Expand Down Expand Up @@ -69,11 +71,11 @@ public String getName() {
return "cloud-init";
}

protected boolean isGZipped(String userdata) {
if (StringUtils.isEmpty(userdata)) {
protected boolean isGZipped(String encodedUserdata) {
if (StringUtils.isEmpty(encodedUserdata)) {
return false;
}
byte[] data = userdata.getBytes(StandardCharsets.ISO_8859_1);
byte[] data = Base64.decodeBase64(encodedUserdata);
if (data.length < 2) {
return false;
}
Expand All @@ -82,9 +84,6 @@ protected boolean isGZipped(String userdata) {
}

protected String extractUserDataHeader(String userdata) {
if (isGZipped(userdata)) {
throw new CloudRuntimeException("Gzipped user data can not be used together with other user data formats");
}
List<String> lines = Arrays.stream(userdata.split("\n"))
.filter(x -> (x.startsWith("#") && !x.startsWith("##")) || (x.startsWith("Content-Type:")))
.collect(Collectors.toList());
Expand Down Expand Up @@ -131,7 +130,7 @@ protected FormatType getUserDataFormatType(String userdata) {

private String getContentType(String userData, FormatType formatType) throws MessagingException {
if (formatType == FormatType.MIME) {
MimeMessage msg = new MimeMessage(session, new ByteArrayInputStream(userData.getBytes()));
NoIdMimeMessage msg = new NoIdMimeMessage(session, new ByteArrayInputStream(userData.getBytes()));
return msg.getContentType();
}
if (!formatContentTypeMap.containsKey(formatType)) {
Expand All @@ -141,15 +140,35 @@ private String getContentType(String userData, FormatType formatType) throws Mes
return formatContentTypeMap.get(formatType);
}

protected MimeBodyPart generateBodyPartMIMEMessage(String userData, FormatType formatType) throws MessagingException {
protected String getBodyPartContentAsString(BodyPart bodyPart) throws MessagingException, IOException {
Object content = bodyPart.getContent();
if (content instanceof BASE64DecoderStream) {
return new String(((BASE64DecoderStream)bodyPart.getContent()).readAllBytes());
} else if (content instanceof ByteArrayInputStream) {
return new String(((ByteArrayInputStream)bodyPart.getContent()).readAllBytes());
} else if (content instanceof String) {
return (String)bodyPart.getContent();
}
throw new CloudRuntimeException(String.format("Failed to get content for multipart data with content type: %s", getBodyPartContentType(bodyPart)));
}

private String getBodyPartContentType(BodyPart bodyPart) throws MessagingException {
String contentType = StringUtils.defaultString(bodyPart.getDataHandler().getContentType(), bodyPart.getContentType());
return contentType.contains(";") ? contentType.substring(0, contentType.indexOf(';')) : contentType;
}

protected MimeBodyPart generateBodyPartMimeMessage(String userData, String contentType) throws MessagingException {
MimeBodyPart bodyPart = new MimeBodyPart();
String contentType = getContentType(userData, formatType);
bodyPart.setContent(userData, contentType);
bodyPart.addHeader("Content-Transfer-Encoding", "base64");
return bodyPart;
}

private Multipart getMessageContent(MimeMessage message) {
protected MimeBodyPart generateBodyPartMimeMessage(String userData, FormatType formatType) throws MessagingException {
return generateBodyPartMimeMessage(userData, getContentType(userData, formatType));
}

private Multipart getMessageContent(NoIdMimeMessage message) {
Multipart messageContent;
try {
messageContent = (MimeMultipart) message.getContent();
Expand All @@ -159,40 +178,83 @@ private Multipart getMessageContent(MimeMessage message) {
return messageContent;
}

private void addBodyPartsToMessageContentFromUserDataContent(Multipart messageContent,
MimeMessage msgFromUserdata) throws MessagingException, IOException {
Multipart msgFromUserdataParts = (MimeMultipart) msgFromUserdata.getContent();
int count = msgFromUserdataParts.getCount();
int i = 0;
while (i < count) {
BodyPart bodyPart = msgFromUserdataParts.getBodyPart(0);
messageContent.addBodyPart(bodyPart);
i++;
private void addBodyPartToMultipart(Multipart existingMultipart, MimeBodyPart bodyPart) throws MessagingException, IOException {
boolean added = false;
final int existingCount = existingMultipart.getCount();
for (int j = 0; j < existingCount; ++j) {
MimeBodyPart existingBodyPart = (MimeBodyPart)existingMultipart.getBodyPart(j);
String existingContentType = getBodyPartContentType(existingBodyPart);
String newContentType = getBodyPartContentType(bodyPart);
if (existingContentType.equals(newContentType)) {
String existingContent = getBodyPartContentAsString(existingBodyPart);
String newContent = getBodyPartContentAsString(bodyPart);
// generating a combined content MimeBodyPart to replace
MimeBodyPart combinedBodyPart = generateBodyPartMimeMessage(
simpleAppendSameFormatTypeUserData(existingContent, newContent), existingContentType);
existingMultipart.removeBodyPart(j);
existingMultipart.addBodyPart(combinedBodyPart, j);
added = true;
break;
}
}
if (!added) {
existingMultipart.addBodyPart(bodyPart);
}
}

private void addBodyPartsToMessageContentFromUserDataContent(Multipart existingMultipart,
NoIdMimeMessage msgFromUserdata) throws MessagingException, IOException {
MimeMultipart newMultipart = (MimeMultipart)msgFromUserdata.getContent();
final int existingCount = existingMultipart.getCount();
final int newCount = newMultipart.getCount();
for (int i = 0; i < newCount; ++i) {
BodyPart bodyPart = newMultipart.getBodyPart(i);
if (existingCount == 0) {
existingMultipart.addBodyPart(bodyPart);
continue;
}
addBodyPartToMultipart(existingMultipart, (MimeBodyPart)bodyPart);
}
}

private MimeMessage createMultipartMessageAddingUserdata(String userData, FormatType formatType,
MimeMessage message) throws MessagingException, IOException {
MimeMessage newMessage = new MimeMessage(session);
private NoIdMimeMessage createMultipartMessageAddingUserdata(String userData, FormatType formatType,
NoIdMimeMessage message) throws MessagingException, IOException {
NoIdMimeMessage newMessage = new NoIdMimeMessage(session);
Multipart messageContent = getMessageContent(message);

if (formatType == FormatType.MIME) {
MimeMessage msgFromUserdata = new MimeMessage(session, new ByteArrayInputStream(userData.getBytes()));
NoIdMimeMessage msgFromUserdata = new NoIdMimeMessage(session, new ByteArrayInputStream(userData.getBytes()));
addBodyPartsToMessageContentFromUserDataContent(messageContent, msgFromUserdata);
} else {
MimeBodyPart part = generateBodyPartMIMEMessage(userData, formatType);
messageContent.addBodyPart(part);
MimeBodyPart part = generateBodyPartMimeMessage(userData, formatType);
addBodyPartToMultipart(messageContent, part);
}
newMessage.setContent(messageContent);
return newMessage;
}

private String simpleAppendSameFormatTypeUserData(String userData1, String userData2) {
return String.format("%s\n\n%s", userData1, userData2.substring(userData2.indexOf('\n')+1));
}

private void checkGzipAppend(String encodedUserData1, String encodedUserData2) {
if (isGZipped(encodedUserData1) || isGZipped(encodedUserData2)) {
throw new CloudRuntimeException("Gzipped user data can not be used together with other user data formats");
}
}

@Override
public String appendUserData(String userData1, String userData2) {
public String appendUserData(String encodedUserData1, String encodedUserData2) {
try {
checkGzipAppend(encodedUserData1, encodedUserData2);
String userData1 = new String(Base64.decodeBase64(encodedUserData1));
String userData2 = new String(Base64.decodeBase64(encodedUserData2));
FormatType formatType1 = getUserDataFormatType(userData1);
FormatType formatType2 = getUserDataFormatType(userData2);
MimeMessage message = new MimeMessage(session);
if (formatType1.equals(formatType2) && List.of(FormatType.CLOUD_CONFIG, FormatType.BASH_SCRIPT).contains(formatType1)) {
return simpleAppendSameFormatTypeUserData(userData1, userData2);
}
NoIdMimeMessage message = new NoIdMimeMessage(session);
message = createMultipartMessageAddingUserdata(userData1, formatType1, message);
message = createMultipartMessageAddingUserdata(userData2, formatType2, message);
ByteArrayOutputStream output = new ByteArrayOutputStream();
Expand All @@ -205,4 +267,20 @@ public String appendUserData(String userData1, String userData2) {
throw new CloudRuntimeException(msg, e);
}
}

/* This is a wrapper class just to remove Message-ID header from the resultant
multipart data which may contain server details.
*/
private class NoIdMimeMessage extends MimeMessage {
NoIdMimeMessage (Session session) {
super(session);
}
NoIdMimeMessage (Session session, InputStream is) throws MessagingException {
super(session, is);
}
@Override
protected void updateMessageID() throws MessagingException {
removeHeader("Message-ID");
}
}
}
Loading