Skip to content

Commit

Permalink
Merge branch '#2011_group_name_as_to'
Browse files Browse the repository at this point in the history
  • Loading branch information
@lfcnassif
lfcnassif committed Aug 2, 2024
2 parents cd72b0f + c599d32 commit d9dbe20
Show file tree
Hide file tree
Showing 9 changed files with 68 additions and 67 deletions.
2 changes: 2 additions & 0 deletions iped-api/src/main/java/iped/properties/ExtraProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public class ExtraProperties {

public static final String GROUP_ID = "GroupID";

public static final String IS_GROUP_MESSAGE = "isGroupMessage";

public static final String MESSAGE_BODY = MESSAGE_PREFIX + "Body"; //$NON-NLS-1$

public static final String MESSAGE_IS_ATTACHMENT = Message.MESSAGE_PREFIX + "IsEmailAttachment"; //$NON-NLS-1$
Expand Down
42 changes: 17 additions & 25 deletions iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
Expand Down Expand Up @@ -1176,43 +1175,36 @@ private void processItem(Item item) throws SAXException {
}
}

private HashMap<String, String[]> toCache = new LinkedHashMap<String, String[]>(16, 0.75f, true) {

private static final long serialVersionUID = 1L;

@Override
protected boolean removeEldestEntry(Entry<String, String[]> eldest) {
return this.size() > 1000;
}
};

private Property toProperty = Property.internalText(ExtraProperties.COMMUNICATION_TO);

private void fillMissingInfo(Item item) {
String from = item.getMetadata().get(ExtraProperties.COMMUNICATION_FROM);
String to = item.getMetadata().get(ExtraProperties.COMMUNICATION_TO);
String[] to = item.getMetadata().getValues(ExtraProperties.COMMUNICATION_TO);
boolean fromOwner = Boolean.valueOf(item.getMetadata().get(ExtraProperties.UFED_META_PREFIX + "fromOwner"));
if (to == null) {
if (to == null || to.length != 1) {
if (item.getMediaType() != null
&& MediaTypes.isInstanceOf(item.getMediaType(), MediaTypes.UFED_MESSAGE_MIME)) {
// we have seen ufed messages without parent chat
if (itemSeq.size() == 0)
return;
IItem parentChat = itemSeq.get(itemSeq.size() - 1);
String[] parties = parentChat.getMetadata()
.getValues(ExtraProperties.UFED_META_PREFIX + "Participants");
ArrayList<String> toList = new ArrayList<>();
for (String party : parties) {
if ((from != null && !party.equals(from)) || (fromOwner && !ownerParties.contains(party)))
toList.add(party);
List<String> toList = new ArrayList<>();
if (to != null && to.length > 0) {
toList = Arrays.asList(to);
} else {
String[] parties = parentChat.getMetadata().getValues(ExtraProperties.UFED_META_PREFIX + "Participants");
for (String party : parties) {
if ((from != null && !party.equals(from)) || (fromOwner && !ownerParties.contains(party)))
toList.add(party);
}
}
String key = toList.toString();
String[] val = toCache.get(key);
if (val == null) {
val = toList.toArray(new String[toList.size()]);
toCache.put(key, val);
if (toList.size() == 1) {
item.getMetadata().set(toProperty, toList.get(0));
} else if (toList.size() > 1) {
item.getMetadata().set(toProperty, parentChat.getName());
item.getMetadata().set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}
item.getMetadata().set(toProperty, val);

}
}
if (!msisdns.isEmpty() && (MediaTypes.UFED_CALL_MIME.equals(item.getMediaType())
Expand Down
11 changes: 10 additions & 1 deletion iped-engine/src/main/java/iped/engine/graph/GraphTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,10 @@ private NodeValues getPhoneNodeValues(String value) {
return null;
}

private NodeValues getGroupNodeValues(String value) {
return new NodeValues(DynLabel.label(GraphConfiguration.CONTACT_GROUP_LABEL), BasicProps.NAME, value.trim().toLowerCase());
}

private NodeValues getGenericNodeValues(String value) {
return new NodeValues(DynLabel.label("GENERIC"), "entity", value.trim().toLowerCase());
}
Expand Down Expand Up @@ -469,7 +473,12 @@ private void processCommunicationMetadata(IItem evidence) throws IOException {
recipients.addAll(Arrays.asList(metadata.getValues(Message.MESSAGE_BCC)));

for (String recipient : recipients) {
NodeValues nv2 = getNodeValues(recipient, evidence.getMetadata(), detectPhones);
NodeValues nv2;
if (Boolean.valueOf(metadata.get(ExtraProperties.IS_GROUP_MESSAGE))) {
nv2 = getGroupNodeValues(recipient);
} else {
nv2 = getNodeValues(recipient, metadata, detectPhones);
}
graphFileWriter.writeNode(nv2.label, nv2.propertyName, nv2.propertyValue, nv2.props);
graphFileWriter.writeRelationship(nv1.label, nv1.propertyName, nv1.propertyValue, nv2.label,
nv2.propertyName, nv2.propertyValue, relationshipType, relProps);
Expand Down
20 changes: 11 additions & 9 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@

import iped.data.IItemReader;
import iped.parsers.browsers.chrome.CacheIndexParser;
import iped.parsers.discord.cache.CacheAddr.InputStreamNotAvailable;
import iped.parsers.discord.cache.Index;
import iped.parsers.discord.json.DiscordAttachment;
import iped.parsers.discord.json.DiscordAuthor;
Expand Down Expand Up @@ -324,15 +323,18 @@ private void extractMessages(String chatName, List<DiscordRoot> discordRoot, Con
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, d.getAuthor().getFullUsername());
meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString());

// Add "Message-TO" field
for (String participant : participants) {
if (participant.length() <= 1) {
// In cases where only one participant sends messages, it is not possible to
// determine the participants as only the participant list of the calls are
// cached.
} else if (!participant.equals(d.getAuthor().getFullUsername())) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant);
// Add "Message-TO" field.
// In cases where only one participant sends messages, it is not possible to
// determine the participants as only the participant list of the calls are cached.
if (participants.size() <= 2) {
for (String participant : participants) {
if (!participant.equals(d.getAuthor().getFullUsername())) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant);
}
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, chatName);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

for (DiscordAttachment da : d.getAttachments()) {
Expand Down
10 changes: 2 additions & 8 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -253,13 +252,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
tMetadata.set(ExtraProperties.MESSAGE_DATE, t.getStart());
tMetadata.set(ExtraProperties.MESSAGE_BODY, name);
tMetadata.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, t.getFrom()));
if (t.getType() == 3 && t.getConversation().getParticipantes() != null) {
for (Iterator<String> iterator = t.getConversation().getParticipantes().iterator(); iterator
.hasNext();) {
String recip = formatSkypeName(contactMap, iterator.next());
if (recip != null)
tMetadata.add(Metadata.MESSAGE_TO, recip);
}
if (t.getType() == 3 && t.getConversation().getParticipantes() != null && t.getConversation().getParticipantes().size() > 1) {
tMetadata.set(Metadata.MESSAGE_TO, t.getConversation().getTitle());
} else {
tMetadata.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, t.getTo()));
}
Expand Down
8 changes: 4 additions & 4 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,10 @@ private void extractMessages(String chatName, List<Message> messages, Contact ac
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, m.getFrom().toString());
if (m.getChat().isGroupOrChannel()) {
ChatGroup groupChat = (ChatGroup) m.getChat();
for (Long id : groupChat.getMembers()) {
if (id != m.getFrom().getId())
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, e.getContact(id).toString());
}
String to = groupChat.isGroup() ? "Group " : "Channel ";
to += groupChat.getName() + " (id:" + groupChat.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}
if (meta.get(org.apache.tika.metadata.Message.MESSAGE_TO) == null) {
if (m.getToId() != 0) {
Expand Down
17 changes: 9 additions & 8 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,13 +331,14 @@ private ThreemaAccount getUserAccount(IItemSearcher searcher) {
return account;
}

private void fillGroupRecipients(Metadata meta, Chat c, String from) {
for (ThreemaContact member : c.getParticipants()) {
String gmb = member.getFullId();
if (!gmb.equals(from)) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, gmb);
}
private void fillGroupRecipients(Metadata meta, Chat c) {
String to = "Group";
if (c.getSubject() != null) {
to += " " + c.getSubject().strip();
}
to += " (id:" + c.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

private void extractMessages(String chatName, Chat c, List<Message> messages, ThreemaAccount account, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor) throws SAXException, IOException {
Expand All @@ -360,14 +361,14 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, Th
if (m.isFromMe()) {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, local);
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote);
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, remote);
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local);
}
Expand Down
23 changes: 12 additions & 11 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -889,13 +889,14 @@ private String formatContact(WAContact contact, Map<String, String> cache) {
return result;
}

private void fillGroupRecipients(Metadata meta, Chat c, String from, Map<String, String> cache) {
for (WAContact member : c.getGroupMembers()) {
String gmb = formatContact(member, cache);
if (!gmb.equals(from)) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, gmb);
}
}
private void fillGroupRecipients(Metadata meta, Chat c) {
String to = c.isChannelChat() ? "Channel " : "Group ";
if (c.getSubject() != null) {
to += c.getSubject().strip();
}
to += " (id:" + c.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

private void extractMessages(String chatName, Chat c, List<Message> messages, WAAccount account,
Expand Down Expand Up @@ -926,15 +927,15 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, WA
}
if (m.isFromMe()) {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, local, cache);
if (c.isGroupOrChannelChat()) {
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote);
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, remote, cache);
if (c.isGroupOrChannelChat()) {
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local);
}
Expand Down
2 changes: 1 addition & 1 deletion iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void testTelegramParser() throws IOException, SAXException, TikaException

assertEquals("Telegram (phone: 42777)", telegramtracker.messageto.get(0));
assertEquals("Nickerida (phone: 5561983125151)", telegramtracker.messageto.get(1));
assertEquals("Guilherme Andreúce (phone: 5561986143035)", telegramtracker.messageto.get(150));
assertEquals("Group mixirica e noronhe-se (id:-136232058)", telegramtracker.messageto.get(150));

assertTrue(telegramtracker.messagebody.get(0).contains(
"Código de login: 73632. Não envie esse código para ninguém, nem mesmo que eles digam que são do Telegram!"));
Expand Down

0 comments on commit d9dbe20

Please sign in to comment.