Skip to content

Commit

Permalink
fix(interactive): support a general Intersect implementation in GIE…
Browse files Browse the repository at this point in the history 
… Runtime (#3689)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?

<!-- Please give a short brief about these changes. -->

As titled.

This pr mainly includes:

1. Refactor the convert logic of `Intersect` in
`GraphRelToProtoConverter` in Compiler
2. Refactor the parsing and processing logic of `Intersect` in assembly
in Runtime
3. Support a more general `Intersect` that can preserve the edges during
intersection, in Runtime
4. Add a micro benchmark framework, and testing for general_intersection
v.s. optimized_intersection. The results shows that, on ldbc dataset
(sf=1), to find matches for a triangle (a,b,c knows each other),
general_intersection's time cost is about 1.7\times of
optimized_intersection's, as the general_intersection further preserves
all matched edges during the intersection computation.

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes #3685 #3745

---------

Co-authored-by: xiao.zl <xiaolei.zl@alibaba-inc.com>
Co-authored-by: Longbin Lai <longbin.lailb@alibaba-inc.com>
  • Loading branch information
@BingqingLyu @zhanglei1949 @longbinlai
3 people authored May 14, 2024
1 parent fda099f commit 348a2b7
Show file tree
Hide file tree
Showing 20 changed files with 3,315 additions and 316 deletions.
8 changes: 1 addition & 7 deletions flex/codegen/src/hqps_generator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,18 +518,12 @@ class QueryGenerator {

case physical::PhysicalOpr::Operator::kIntersect: {
LOG(INFO) << "Found a intersect operator";
// a intersect op must be followed by a unfold op
CHECK(i + 1 < size) << " intersect op must be followed by a unfold op";
auto& next_op = plan_.plan(i + 1).opr();
CHECK(next_op.op_kind_case() ==
physical::PhysicalOpr::Operator::kUnfold)
<< "intersect op must be followed by a unfold op";
// Note that intersect operator will not be followed by unfold anymore.
auto& intersect_op = opr.intersect();
auto intersect_opt_code = BuildIntersectOp<LabelT>(ctx_, intersect_op);
for (auto& line : intersect_opt_code) {
ss << line << std::endl;
}
i += 1; // skip unfold
break;
}

Expand Down
59 changes: 4 additions & 55 deletions ...rc/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,10 +841,6 @@ public RelNode visit(MultiJoin multiJoin) {
GraphAlgebraPhysical.PhysicalOpr.newBuilder();
GraphAlgebraPhysical.Intersect.Builder intersectBuilder =
GraphAlgebraPhysical.Intersect.newBuilder();
GraphAlgebraPhysical.PhysicalOpr.Builder unfoldOprBuilder =
GraphAlgebraPhysical.PhysicalOpr.newBuilder();
GraphAlgebraPhysical.Unfold.Builder unfoldBuilder =
GraphAlgebraPhysical.Unfold.newBuilder();

List<RexNode> conditions = RelOptUtil.conjunctions(multiJoin.getJoinFilter());
int intersectKey = -1;
Expand All @@ -867,66 +863,19 @@ public RelNode visit(MultiJoin multiJoin) {
Preconditions.checkArgument(intersectKey != -1, "intersect key should be set");
intersectBuilder.setKey(intersectKey);

// then, process operators in the intersect branches;
// currently, there are some cases:
// case 1: PhysicalExpand;
// case 2: PhysicalExpand + PhysicalGetV(filter);
// case 3: EdgeExpand + GetV; (not supported yet)
// case 4: PathExpand + GetV;
// TODO(bingqing): This should be refactored. Directly add these cases as subplans in the
// intersect.
// Currently, we process these cases in a consistent way with the previous ir-core
// implementation.
GraphPhysicalGetV auxiliaFilter = null;
// then, build subplans for intersect
for (RelNode input : multiJoin.getInputs()) {
GraphAlgebraPhysical.PhysicalPlan.Builder subPlanBuilder =
GraphAlgebraPhysical.PhysicalPlan.newBuilder();
// specifically, if it is PhysicalGetV(filter), we build an auxilia node after
// the intersect.
if (input instanceof GraphPhysicalGetV
&& !ObjectUtils.isEmpty(((GraphPhysicalGetV) input).getFilters())) {
auxiliaFilter = (GraphPhysicalGetV) input;
auxiliaFilter
.getInput()
.accept(
new GraphRelToProtoConverter(
isColumnId,
graphConfig,
subPlanBuilder,
this.relToCommons,
depth + 1));
} else if (input instanceof GraphLogicalGetV) {
throw new UnsupportedOperationException(
"Unsupported of LogicalEdgeEdge + LogicalGetV in Intersect yet");
} else {
input.accept(
new GraphRelToProtoConverter(
isColumnId,
graphConfig,
subPlanBuilder,
this.relToCommons,
depth + 1));
}
input.accept(
new GraphRelToProtoConverter(
isColumnId, graphConfig, subPlanBuilder, this.relToCommons, depth + 1));
intersectBuilder.addSubPlans(subPlanBuilder);
}
intersectOprBuilder.setOpr(
GraphAlgebraPhysical.PhysicalOpr.Operator.newBuilder()
.setIntersect(intersectBuilder));
physicalBuilder.addPlan(intersectOprBuilder.build());

// after intersect, we need to unfold the result.
unfoldBuilder.setTag(Utils.asAliasId(intersectKey));
unfoldBuilder.setAlias(Utils.asAliasId(intersectKey));
unfoldOprBuilder.setOpr(
GraphAlgebraPhysical.PhysicalOpr.Operator.newBuilder().setUnfold(unfoldBuilder));

physicalBuilder.addPlan(unfoldOprBuilder.build());

// if have filters, we need to add a auxilia node after intersect.
if (auxiliaFilter != null) {
addAuxilia(physicalBuilder, auxiliaFilter);
}

return multiJoin;
}

Expand Down
121 changes: 121 additions & 0 deletions .../main/java/com/alibaba/graphscope/gremlin/integration/suite/pattern/PatternQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,16 @@ public abstract class PatternQueryTest extends AbstractGremlinProcessTest {

public abstract Traversal<Vertex, Long> get_pattern_4_test();

public abstract Traversal<Vertex, Long> get_pattern_4b_test();

public abstract Traversal<Vertex, Long> get_pattern_4c_test();

public abstract Traversal<Vertex, Long> get_pattern_5_test();

public abstract Traversal<Vertex, Long> get_pattern_5b_test();

public abstract Traversal<Vertex, Long> get_pattern_5c_test();

public abstract Traversal<Vertex, Long> get_pattern_6_test();

public abstract Traversal<Vertex, Long> get_pattern_7_test();
Expand All @@ -46,12 +54,16 @@ public abstract class PatternQueryTest extends AbstractGremlinProcessTest {

public abstract Traversal<Vertex, Long> get_pattern_9_test();

public abstract Traversal<Vertex, Long> get_pattern_9b_test();

public abstract Traversal<Vertex, Long> get_pattern_10_test();

public abstract Traversal<Vertex, Long> get_pattern_11_test();

public abstract Traversal<Vertex, Long> get_pattern_12_test();

public abstract Traversal<Vertex, Long> get_pattern_12b_test();

public abstract Traversal<Vertex, Long> get_pattern_13_test();

public abstract Traversal<Vertex, Long> get_pattern_14_test();
Expand Down Expand Up @@ -94,13 +106,41 @@ public void run_pattern_4_test() {
Assert.assertEquals(23286L, traversal.next().longValue());
}

@Test
public void run_pattern_4b_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_4b_test();
this.printTraversalForm(traversal);
Assert.assertEquals(23286L, traversal.next().longValue());
}

@Test
public void run_pattern_4c_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_4c_test();
this.printTraversalForm(traversal);
Assert.assertEquals(23286L, traversal.next().longValue());
}

@Test
public void run_pattern_5_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_5_test();
this.printTraversalForm(traversal);
Assert.assertEquals(5596L, traversal.next().longValue());
}

@Test
public void run_pattern_5b_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_5b_test();
this.printTraversalForm(traversal);
Assert.assertEquals(5596L, traversal.next().longValue());
}

@Test
public void run_pattern_5c_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_5c_test();
this.printTraversalForm(traversal);
Assert.assertEquals(5596L, traversal.next().longValue());
}

@Test
public void run_pattern_6_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_6_test();
Expand Down Expand Up @@ -129,6 +169,13 @@ public void run_pattern_9_test() {
Assert.assertEquals(23286L, traversal.next().longValue());
}

@Test
public void run_pattern_9b_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_9b_test();
this.printTraversalForm(traversal);
Assert.assertEquals(23286L, traversal.next().longValue());
}

@Test
public void run_pattern_10_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_10_test();
Expand All @@ -150,6 +197,13 @@ public void run_pattern_12_test() {
Assert.assertEquals(232854L, traversal.next().longValue());
}

@Test
public void run_pattern_12b_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_12b_test();
this.printTraversalForm(traversal);
Assert.assertEquals(232854L, traversal.next().longValue());
}

@Test
public void run_pattern_13_test() {
Traversal<Vertex, Long> traversal = this.get_pattern_13_test();
Expand Down Expand Up @@ -255,6 +309,28 @@ public Traversal<Vertex, Long> get_pattern_4_test() {
.count();
}

// PM5
@Override
public Traversal<Vertex, Long> get_pattern_4b_test() {
return g.V().match(
__.as("a").outE("KNOWS").as("d").inV().as("b"),
__.as("b").out("KNOWS").as("c"),
__.as("a").out("KNOWS").as("c"))
.select("d")
.count();
}

// PM5
@Override
public Traversal<Vertex, Long> get_pattern_4c_test() {
return g.V().match(
__.as("a").outE("KNOWS").as("d").inV().as("b"),
__.as("b").outE("KNOWS").as("e").inV().as("c"),
__.as("a").outE("KNOWS").as("f").inV().as("c"))
.select("d", "e", "f")
.count();
}

// PM7
@Override
public Traversal<Vertex, Long> get_pattern_5_test() {
Expand All @@ -265,6 +341,28 @@ public Traversal<Vertex, Long> get_pattern_5_test() {
.count();
}

// PM7
@Override
public Traversal<Vertex, Long> get_pattern_5b_test() {
return g.V().match(
__.as("a").has("gender", "male").outE("KNOWS").as("d").inV().as("b"),
__.as("b").has("gender", "female").out("KNOWS").as("c"),
__.as("a").out("KNOWS").as("c"))
.select("d")
.count();
}

// PM7
@Override
public Traversal<Vertex, Long> get_pattern_5c_test() {
return g.V().match(
__.as("a").has("gender", "male").outE("KNOWS").as("d").inV().as("b"),
__.as("b").has("gender", "female").outE("KNOWS").as("e").inV().as("c"),
__.as("a").outE("KNOWS").as("f").inV().as("c"))
.select("d", "e", "f")
.count();
}

// PM12
@Override
public Traversal<Vertex, Long> get_pattern_6_test() {
Expand Down Expand Up @@ -317,6 +415,18 @@ public Traversal<Vertex, Long> get_pattern_9_test() {
.count();
}

// PM5-path
@Override
public Traversal<Vertex, Long> get_pattern_9b_test() {
return g.V().match(
((IrCustomizedTraversal) __.as("a").out("2..3", "KNOWS"))
.endV()
.as("c"),
__.as("a").outE("KNOWS").as("d").inV().as("c"))
.select("d")
.count();
}

// PM11-path
@Override
public Traversal<Vertex, Long> get_pattern_10_test() {
Expand Down Expand Up @@ -351,6 +461,17 @@ public Traversal<Vertex, Long> get_pattern_12_test() {
.count();
}

// fuzzy pattern
@Override
public Traversal<Vertex, Long> get_pattern_12b_test() {
return g.V().match(
__.as("a").outE("KNOWS", "LIKES").as("d").inV().as("b"),
__.as("b").outE("KNOWS", "LIKES").as("e").inV().as("c"),
__.as("a").outE("KNOWS", "LIKES").as("f").inV().as("c"))
.select("d", "e", "f")
.count();
}

// support both
@Override
public Traversal<Vertex, Long> get_pattern_13_test() {
Expand Down
85 changes: 85 additions & 0 deletions .../compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/GraphRelToProtoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1123,6 +1123,75 @@ public void intersect_test() throws Exception {
FileUtils.readJsonFromResource("proto/intersect_test.json"),
plan.explain().trim());
}

try (PhysicalBuilder protoBuilder =
new GraphRelProtoPhysicalBuilder(
getMockPartitionedCBOConfig(), getMockCBOMeta(), new LogicalPlan(after))) {
PhysicalPlan plan = protoBuilder.build();
Assert.assertEquals(
FileUtils.readJsonFromResource("proto/partitioned_intersect_test.json"),
plan.explain().trim());
}
}

@Test
public void intersect_test_02() throws Exception {
GraphRelOptimizer optimizer = getMockCBO();
IrMeta irMeta = getMockCBOMeta();
GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta);
RelNode before =
com.alibaba.graphscope.cypher.antlr4.Utils.eval(
"Match (message:COMMENT|POST)-[e1:HASCREATOR]->(person:PERSON), \n"
+ " (message:COMMENT|POST)-[e2:HASTAG]->(tag:TAG), \n"
+ " (person:PERSON)-[e3:HASINTEREST]->(tag:TAG)\n"
+ "Return count(person);",
builder)
.build();
RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta));
Assert.assertEquals(
"root:\n"
+ "GraphLogicalAggregate(keys=[{variables=[], aliases=[]}],"
+ " values=[[{operands=[person], aggFunction=COUNT, alias='$f0',"
+ " distinct=false}]])\n"
+ " MultiJoin(joinFilter=[=(tag, tag)], isFullOuterJoin=[false],"
+ " joinTypes=[[INNER, INNER]], outerJoinConditions=[[NULL, NULL]],"
+ " projFields=[[ALL, ALL]])\n"
+ " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[TAG]}], alias=[tag],"
+ " opt=[END])\n"
+ " GraphLogicalExpand(tableConfig=[[EdgeLabel(HASTAG, COMMENT, TAG),"
+ " EdgeLabel(HASTAG, POST, TAG)]], alias=[e2], startAlias=[message],"
+ " opt=[OUT])\n"
+ " CommonTableScan(table=[[common#-676410541]])\n"
+ " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[TAG]}], alias=[tag],"
+ " opt=[END])\n"
+ " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASINTEREST]}],"
+ " alias=[e3], startAlias=[person], opt=[OUT])\n"
+ " CommonTableScan(table=[[common#-676410541]])\n"
+ "common#-676410541:\n"
+ "GraphLogicalGetV(tableConfig=[{isAll=false, tables=[POST, COMMENT]}],"
+ " alias=[message], opt=[START])\n"
+ " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASCREATOR]}],"
+ " alias=[e1], startAlias=[person], opt=[IN])\n"
+ " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}],"
+ " alias=[person], opt=[VERTEX])",
com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim());

try (PhysicalBuilder protoBuilder =
new GraphRelProtoPhysicalBuilder(
getMockCBOConfig(), getMockCBOMeta(), new LogicalPlan(after))) {
PhysicalPlan plan = protoBuilder.build();
Assert.assertEquals(
FileUtils.readJsonFromResource("proto/intersect_test_2.json"),
plan.explain().trim());
}
try (PhysicalBuilder protoBuilder =
new GraphRelProtoPhysicalBuilder(
getMockPartitionedCBOConfig(), getMockCBOMeta(), new LogicalPlan(after))) {
PhysicalPlan plan = protoBuilder.build();
Assert.assertEquals(
FileUtils.readJsonFromResource("proto/partitioned_intersect_test_2.json"),
plan.explain().trim());
}
}

private Configs getMockCBOConfig() {
Expand All @@ -1139,6 +1208,22 @@ private Configs getMockCBOConfig() {
"target/test-classes/statistics/ldbc30_hierarchy_statistics.txt"));
}

private Configs getMockPartitionedCBOConfig() {
return new Configs(
ImmutableMap.of(
"graph.planner.is.on",
"true",
"graph.planner.opt",
"CBO",
"graph.planner.rules",
"FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule,"
+ " ExpandGetVFusionRule",
"graph.planner.cbo.glogue.schema",
"target/test-classes/statistics/ldbc30_hierarchy_statistics.txt",
"pegasus.hosts",
"host1,host2"));
}

private GraphRelOptimizer getMockCBO() {
return new GraphRelOptimizer(getMockCBOConfig());
}
Expand Down
Loading

0 comments on commit 348a2b7

Please sign in to comment.