Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improve] mongodb connector v2 add source query capability #3697

Merged
merged 14 commits into from
Dec 26, 2022
Merged

[Improve] mongodb connector v2 add source query capability #3697

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
20a284f
[Improve] mongodb connector v2 add source query capability
MonsterChenzhuo Dec 12, 2022
441f9cf
fix
MonsterChenzhuo Dec 12, 2022
fdc3df7
fix
MonsterChenzhuo Dec 12, 2022
c733ef0
fix
MonsterChenzhuo Dec 12, 2022
028fe2a
Merge branch 'dev' into mongodb
MonsterChenzhuo Dec 12, 2022
152b7ed
fix
MonsterChenzhuo Dec 12, 2022
ba3b46d
fix
MonsterChenzhuo Dec 13, 2022
ab03bc3
add e2e
MonsterChenzhuo Dec 13, 2022
2f50c23
fix
MonsterChenzhuo Dec 13, 2022
b34a20f
fix
MonsterChenzhuo Dec 13, 2022
7c57a73
Update MongodbMatchQueryIT.java
MonsterChenzhuo Dec 13, 2022
fc30e5e
fix e2e
MonsterChenzhuo Dec 13, 2022
e7d8623
fix e2e
MonsterChenzhuo Dec 13, 2022
72f4213
fix e2e
MonsterChenzhuo Dec 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/config/MongodbConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ public class MongodbConfig implements Serializable {
.noDefaultValue()
.withDescription("MongoDB collection");

public static final Option<String> MATCHQUERY =
Options.key("matchQuery")
.stringType()
.noDefaultValue()
.withDescription("matchQuery MongoDB collection");

// Don't use now
public static final String FORMAT = "format";

Expand Down
2 changes: 2 additions & 0 deletions ...main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/config/MongodbParameters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,6 @@ public class MongodbParameters implements Serializable {

private String collection;

private String matchQuery;

}
3 changes: 2 additions & 1 deletion ...n/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/MongodbSourceFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import static org.apache.seatunnel.connectors.seatunnel.mongodb.config.MongodbConfig.COLLECTION;
import static org.apache.seatunnel.connectors.seatunnel.mongodb.config.MongodbConfig.DATABASE;
import static org.apache.seatunnel.connectors.seatunnel.mongodb.config.MongodbConfig.MATCHQUERY;
import static org.apache.seatunnel.connectors.seatunnel.mongodb.config.MongodbConfig.URI;

import org.apache.seatunnel.api.configuration.util.OptionRule;
Expand All @@ -37,6 +38,6 @@ public String factoryIdentifier() {

@Override
public OptionRule optionRule() {
return OptionRule.builder().required(URI, DATABASE, COLLECTION, SeaTunnelSchema.SCHEMA).build();
return OptionRule.builder().required(URI, DATABASE, COLLECTION, MATCHQUERY, SeaTunnelSchema.SCHEMA).build();
}
}
4 changes: 3 additions & 1 deletion ...in/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/MongodbSourceReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Projections;
import lombok.extern.slf4j.Slf4j;
import org.bson.BsonDocument;
import org.bson.Document;
import org.bson.conversions.Bson;

import java.io.IOException;
import java.util.Optional;

@Slf4j
public class MongodbSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
Expand Down Expand Up @@ -86,7 +88,7 @@ public void close() throws IOException {
public void pollNext(Collector<SeaTunnelRow> output) throws Exception {
try (MongoCursor<Document> mongoCursor = client.getDatabase(params.getDatabase())
.getCollection(params.getCollection())
.find()
.find(Optional.ofNullable(params.getMatchQuery()).isPresent() ? BsonDocument.parse(params.getMatchQuery()) : new BsonDocument())
.projection(projectionFields)
.iterator()) {
while (mongoCursor.hasNext()) {
Expand Down