Skip to content

Commit

Permalink
8302673: [SuperWord] MaxReduction and MinReduction should vectorize f…
Browse files Browse the repository at this point in the history 
…or int

Co-authored-by: Jatin Bhateja <jbhateja@openjdk.org>
Reviewed-by: epeter, kvn
  • Loading branch information
@robcasloz
robcasloz and Jatin Bhateja committed Jun 5, 2023
1 parent 22a9a86 commit 3fa776d
Show file tree
Hide file tree
Showing 5 changed files with 409 additions and 177 deletions.
279 changes: 113 additions & 166 deletions src/hotspot/share/opto/addnode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1030,6 +1030,14 @@ const Type* XorLNode::Value(PhaseGVN* phase) const {
return AddNode::Value(phase);
}

Node* build_min_max_int(Node* a, Node* b, bool is_max) {
if (is_max) {
return new MaxINode(a, b);
} else {
return new MinINode(a, b);
}
}

Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, const Type* t, PhaseGVN& gvn) {
bool is_int = gvn.type(a)->isa_int();
assert(is_int || gvn.type(a)->isa_long(), "int or long inputs");
Expand All @@ -1044,13 +1052,7 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co
}
Node* res = nullptr;
if (is_int && !is_unsigned) {
Node* res_new = nullptr;
if (is_max) {
res_new = new MaxINode(a, b);
} else {
res_new = new MinINode(a, b);
}
res = gvn.transform(res_new);
res = gvn.transform(build_min_max_int(a, b, is_max));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
} else {
Node* cmp = nullptr;
Expand Down Expand Up @@ -1096,185 +1098,130 @@ Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const
return res;
}

//=============================================================================
//------------------------------add_ring---------------------------------------
// Supplied function returns the sum of the inputs.
const Type *MaxINode::add_ring( const Type *t0, const Type *t1 ) const {
const TypeInt *r0 = t0->is_int(); // Handy access
const TypeInt *r1 = t1->is_int();

// Otherwise just MAX them bits.
return TypeInt::make( MAX2(r0->_lo,r1->_lo), MAX2(r0->_hi,r1->_hi), MAX2(r0->_widen,r1->_widen) );
}

// Check if addition of an integer with type 't' and a constant 'c' can overflow
// Check if addition of an integer with type 't' and a constant 'c' can overflow.
static bool can_overflow(const TypeInt* t, jint c) {
jint t_lo = t->_lo;
jint t_hi = t->_hi;
return ((c < 0 && (java_add(t_lo, c) > t_lo)) ||
(c > 0 && (java_add(t_hi, c) < t_hi)));
}

// Ideal transformations for MaxINode
Node* MaxINode::Ideal(PhaseGVN* phase, bool can_reshape) {
// Force a right-spline graph
Node* l = in(1);
Node* r = in(2);
// Transform MaxI1(MaxI2(a, b), c) into MaxI1(a, MaxI2(b, c))
// to force a right-spline graph for the rest of MaxINode::Ideal().
if (l->Opcode() == Op_MaxI) {
assert(l != l->in(1), "dead loop in MaxINode::Ideal");
r = phase->transform(new MaxINode(l->in(2), r));
l = l->in(1);
set_req_X(1, l, phase);
set_req_X(2, r, phase);
return this;
}

// Get left input & constant
Node* x = l;
jint x_off = 0;
if (x->Opcode() == Op_AddI && // Check for "x+c0" and collect constant
x->in(2)->is_Con()) {
const Type* t = x->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
x_off = t->is_int()->get_con();
x = x->in(1);
}

// Scan a right-spline-tree for MAXs
Node* y = r;
jint y_off = 0;
// Check final part of MAX tree
if (y->Opcode() == Op_AddI && // Check for "y+c1" and collect constant
y->in(2)->is_Con()) {
const Type* t = y->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}
if (x->_idx > y->_idx && r->Opcode() != Op_MaxI) {
swap_edges(1, 2);
return this;
// Let <x, x_off> = x_operands and <y, y_off> = y_operands.
// If x == y and neither add(x, x_off) nor add(y, y_off) overflow, return
// add(x, op(x_off, y_off)). Otherwise, return nullptr.
Node* MaxNode::extract_add(PhaseGVN* phase, ConstAddOperands x_operands, ConstAddOperands y_operands) {
Node* x = x_operands.first;
Node* y = y_operands.first;
int opcode = Opcode();
assert(opcode == Op_MaxI || opcode == Op_MinI, "Unexpected opcode");
const TypeInt* tx = phase->type(x)->isa_int();
jint x_off = x_operands.second;
jint y_off = y_operands.second;
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
jint c = opcode == Op_MinI ? MIN2(x_off, y_off) : MAX2(x_off, y_off);
return new AddINode(x, phase->intcon(c));
}
return nullptr;
}

const TypeInt* tx = phase->type(x)->isa_int();
// Try to cast n as an integer addition with a constant. Return:
// <x, C>, if n == add(x, C), where 'C' is a non-TOP constant;
// <nullptr, 0>, if n == add(x, C), where 'C' is a TOP constant; or
// <n, 0>, otherwise.
static ConstAddOperands as_add_with_constant(Node* n) {
if (n->Opcode() != Op_AddI) {
return ConstAddOperands(n, 0);
}
Node* x = n->in(1);
Node* c = n->in(2);
if (!c->is_Con()) {
return ConstAddOperands(n, 0);
}
const Type* c_type = c->bottom_type();
if (c_type == Type::TOP) {
return ConstAddOperands(nullptr, 0);
}
return ConstAddOperands(x, c_type->is_int()->get_con());
}

if (r->Opcode() == Op_MaxI) {
assert(r != r->in(2), "dead loop in MaxINode::Ideal");
y = r->in(1);
// Check final part of MAX tree
if (y->Opcode() == Op_AddI &&// Check for "y+c1" and collect constant
y->in(2)->is_Con()) {
const Type* t = y->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
Node* MaxNode::IdealI(PhaseGVN* phase, bool can_reshape) {
int opcode = Opcode();
assert(opcode == Op_MinI || opcode == Op_MaxI, "Unexpected opcode");
// Try to transform the following pattern, in any of its four possible
// permutations induced by op's commutativity:
// op(op(add(inner, inner_off), inner_other), add(outer, outer_off))
// into
// op(add(inner, op(inner_off, outer_off)), inner_other),
// where:
// op is either MinI or MaxI, and
// inner == outer, and
// the additions cannot overflow.
for (uint inner_op_index = 1; inner_op_index <= 2; inner_op_index++) {
if (in(inner_op_index)->Opcode() != opcode) {
continue;
}

if (x->_idx > y->_idx)
return new MaxINode(r->in(1), phase->transform(new MaxINode(l, r->in(2))));

// Transform MAX2(x + c0, MAX2(x + c1, z)) into MAX2(x + MAX2(c0, c1), z)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new MaxINode(phase->transform(new AddINode(x, phase->intcon(MAX2(x_off, y_off)))), r->in(2));
Node* outer_add = in(inner_op_index == 1 ? 2 : 1);
ConstAddOperands outer_add_operands = as_add_with_constant(outer_add);
if (outer_add_operands.first == nullptr) {
return nullptr; // outer_add has a TOP input, no need to continue.
}
} else {
// Transform MAX2(x + c0, y + c1) into x + MAX2(c0, c1)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new AddINode(x, phase->intcon(MAX2(x_off, y_off)));
// One operand is a MinI/MaxI and the other is an integer addition with
// constant. Test the operands of the inner MinI/MaxI.
for (uint inner_add_index = 1; inner_add_index <= 2; inner_add_index++) {
Node* inner_op = in(inner_op_index);
Node* inner_add = inner_op->in(inner_add_index);
ConstAddOperands inner_add_operands = as_add_with_constant(inner_add);
if (inner_add_operands.first == nullptr) {
return nullptr; // inner_add has a TOP input, no need to continue.
}
// Try to extract the inner add.
Node* add_extracted = extract_add(phase, inner_add_operands, outer_add_operands);
if (add_extracted == nullptr) {
continue;
}
Node* add_transformed = phase->transform(add_extracted);
Node* inner_other = inner_op->in(inner_add_index == 1 ? 2 : 1);
return build_min_max_int(add_transformed, inner_other, opcode == Op_MaxI);
}
}
return nullptr;
// Try to transform
// op(add(x, x_off), add(y, y_off))
// into
// add(x, op(x_off, y_off)),
// where:
// op is either MinI or MaxI, and
// inner == outer, and
// the additions cannot overflow.
ConstAddOperands xC = as_add_with_constant(in(1));
ConstAddOperands yC = as_add_with_constant(in(2));
if (xC.first == nullptr || yC.first == nullptr) return nullptr;
return extract_add(phase, xC, yC);
}

// Ideal transformations for MaxINode
Node* MaxINode::Ideal(PhaseGVN* phase, bool can_reshape) {
return IdealI(phase, can_reshape);
}

//=============================================================================
//------------------------------add_ring---------------------------------------
// Supplied function returns the sum of the inputs.
const Type *MaxINode::add_ring( const Type *t0, const Type *t1 ) const {
const TypeInt *r0 = t0->is_int(); // Handy access
const TypeInt *r1 = t1->is_int();

// Otherwise just MAX them bits.
return TypeInt::make( MAX2(r0->_lo,r1->_lo), MAX2(r0->_hi,r1->_hi), MAX2(r0->_widen,r1->_widen) );
}

//=============================================================================
//------------------------------Idealize---------------------------------------
// MINs show up in range-check loop limit calculations. Look for
// "MIN2(x+c0,MIN2(y,x+c1))". Pick the smaller constant: "MIN2(x+c0,y)"
Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) {
Node *progress = nullptr;
// Force a right-spline graph
Node *l = in(1);
Node *r = in(2);
// Transform MinI1( MinI2(a,b), c) into MinI1( a, MinI2(b,c) )
// to force a right-spline graph for the rest of MinINode::Ideal().
if( l->Opcode() == Op_MinI ) {
assert( l != l->in(1), "dead loop in MinINode::Ideal" );
r = phase->transform(new MinINode(l->in(2),r));
l = l->in(1);
set_req_X(1, l, phase);
set_req_X(2, r, phase);
return this;
}

// Get left input & constant
Node *x = l;
jint x_off = 0;
if( x->Opcode() == Op_AddI && // Check for "x+c0" and collect constant
x->in(2)->is_Con() ) {
const Type *t = x->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
x_off = t->is_int()->get_con();
x = x->in(1);
}

// Scan a right-spline-tree for MINs
Node *y = r;
jint y_off = 0;
// Check final part of MIN tree
if( y->Opcode() == Op_AddI && // Check for "y+c1" and collect constant
y->in(2)->is_Con() ) {
const Type *t = y->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}
if( x->_idx > y->_idx && r->Opcode() != Op_MinI ) {
swap_edges(1, 2);
return this;
}

const TypeInt* tx = phase->type(x)->isa_int();

if( r->Opcode() == Op_MinI ) {
assert( r != r->in(2), "dead loop in MinINode::Ideal" );
y = r->in(1);
// Check final part of MIN tree
if( y->Opcode() == Op_AddI &&// Check for "y+c1" and collect constant
y->in(2)->is_Con() ) {
const Type *t = y->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}

if( x->_idx > y->_idx )
return new MinINode(r->in(1),phase->transform(new MinINode(l,r->in(2))));

// Transform MIN2(x + c0, MIN2(x + c1, z)) into MIN2(x + MIN2(c0, c1), z)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new MinINode(phase->transform(new AddINode(x, phase->intcon(MIN2(x_off, y_off)))), r->in(2));
}
} else {
// Transform MIN2(x + c0, y + c1) into x + MIN2(c0, c1)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new AddINode(x,phase->intcon(MIN2(x_off,y_off)));
}
}
return nullptr;
Node* MinINode::Ideal(PhaseGVN* phase, bool can_reshape) {
return IdealI(phase, can_reshape);
}

//------------------------------add_ring---------------------------------------
Expand Down
4 changes: 4 additions & 0 deletions src/hotspot/share/opto/addnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@
#include "opto/node.hpp"
#include "opto/opcodes.hpp"
#include "opto/type.hpp"
#include "utilities/pair.hpp"

// Portions of code courtesy of Clifford Click

class PhaseTransform;
typedef const Pair<Node*, jint> ConstAddOperands;

//------------------------------AddNode----------------------------------------
// Classic Add functionality. This covers all the usual 'add' behaviors for
Expand Down Expand Up @@ -252,12 +254,14 @@ class MaxNode : public AddNode {
private:
static Node* build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, const Type* t, PhaseGVN& gvn);
static Node* build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const Type* t, PhaseGVN& gvn);
Node* extract_add(PhaseGVN* phase, ConstAddOperands x_operands, ConstAddOperands y_operands);

public:
MaxNode( Node *in1, Node *in2 ) : AddNode(in1,in2) {}
virtual int Opcode() const = 0;
virtual int max_opcode() const = 0;
virtual int min_opcode() const = 0;
Node* IdealI(PhaseGVN* phase, bool can_reshape);

static Node* unsigned_max(Node* a, Node* b, const Type* t, PhaseGVN& gvn) {
return build_min_max(a, b, true, true, t, gvn);
Expand Down
Loading

1 comment on commit 3fa776d

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.