Skip to content

Commit

Permalink
FExpr literals (#2567)
Browse files Browse the repository at this point in the history 
Old `Expr`s corresponding to various python literals are hereby converted into the new `FExpr` format.

WIP for #2562
  • Loading branch information
@st-pasha
st-pasha authored Aug 14, 2020
1 parent c5aa9db commit a7816ca
Show file tree
Hide file tree
Showing 26 changed files with 947 additions and 747 deletions.
106 changes: 9 additions & 97 deletions src/core/expr/expr.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
#include "expr/head_frame.h"
#include "expr/head_func.h"
#include "expr/head_list.h"
#include "expr/head_literal.h"
#include "expr/workframe.h"
#include "expr/eval_context.h"
#include "datatable.h"
Expand All @@ -43,36 +42,30 @@ OldExpr::OldExpr(py::robj src)
: FExpr()
{
if (src.is_dtexpr()) _init_from_dtexpr(src);
else if (src.is_int()) _init_from_int(src);
else if (src.is_string()) _init_from_string(src);
else if (src.is_float()) _init_from_float(src);
else if (src.is_bool()) _init_from_bool(src);
else if (src.is_slice()) _init_from_slice(src);
// else if (src.is_int()) _init_from_int(src);
// else if (src.is_string()) _init_from_string(src);
// else if (src.is_float()) _init_from_float(src);
// else if (src.is_bool()) _init_from_bool(src);
// else if (src.is_slice()) _init_from_slice(src);
else if (src.is_list_or_tuple()) _init_from_list(src);
else if (src.is_dict()) _init_from_dictionary(src);
else if (src.is_anytype()) _init_from_type(src);
// else if (src.is_anytype()) _init_from_type(src);
else if (src.is_generator()) _init_from_iterable(src);
else if (src.is_none()) _init_from_none();
// else if (src.is_none()) _init_from_none();
else if (src.is_frame()) _init_from_frame(src);
else if (src.is_range()) _init_from_range(src);
// else if (src.is_range()) _init_from_range(src);
else if (src.is_pandas_frame() ||
src.is_pandas_series()) _init_from_pandas(src);
else if (src.is_numpy_array() ||
src.is_numpy_marray()) _init_from_numpy(src);
else if (src.is_ellipsis()) _init_from_ellipsis();
// else if (src.is_ellipsis()) _init_from_ellipsis();
else {
throw TypeError() << "An object of type " << src.typeobj()
<< " cannot be used in an Expr";
}
}


void OldExpr::_init_from_bool(py::robj src) {
int8_t t = src.to_bool_strict();
head = ptrHead(new Head_Literal_Bool(t));
}


void OldExpr::_init_from_dictionary(py::robj src) {
strvec names;
for (auto kv : src.to_pydict()) {
Expand All @@ -98,37 +91,12 @@ void OldExpr::_init_from_dtexpr(py::robj src) {
}


void OldExpr::_init_from_ellipsis() {
head = ptrHead(new Head_Literal_SliceAll);
}


void OldExpr::_init_from_float(py::robj src) {
double x = src.to_double();
head = ptrHead(new Head_Literal_Float(x));
}


void OldExpr::_init_from_frame(py::robj src) {
head = Head_Frame::from_datatable(src);
}


void OldExpr::_init_from_int(py::robj src) {
py::oint src_int = src.to_pyint();
int overflow;
int64_t x = src_int.ovalue<int64_t>(&overflow);
if (overflow) {
// If overflow occurs here, the returned value will be +/-Inf,
// which is exactly what we need.
double xx = src_int.ovalue<double>(&overflow);
head = ptrHead(new Head_Literal_Float(xx));
} else {
head = ptrHead(new Head_Literal_Int(x));
}
}


void OldExpr::_init_from_iterable(py::robj src) {
for (auto elem : src.to_oiter()) {
inputs.emplace_back(as_fexpr(elem));
Expand All @@ -147,11 +115,6 @@ void OldExpr::_init_from_list(py::robj src) {
}


void OldExpr::_init_from_none() {
head = ptrHead(new Head_Literal_None);
}


void OldExpr::_init_from_numpy(py::robj src) {
head = Head_Frame::from_numpy(src);
}
Expand All @@ -162,37 +125,6 @@ void OldExpr::_init_from_pandas(py::robj src) {
}


void OldExpr::_init_from_range(py::robj src) {
py::orange rr = src.to_orange();
head = ptrHead(new Head_Literal_Range(std::move(rr)));
}


void OldExpr::_init_from_slice(py::robj src) {
auto src_as_slice = src.to_oslice();
if (src_as_slice.is_trivial()) {
head = ptrHead(new Head_Literal_SliceAll);
}
else if (src_as_slice.is_numeric()) {
head = ptrHead(new Head_Literal_SliceInt(src_as_slice));
}
else if (src_as_slice.is_string()) {
head = ptrHead(new Head_Literal_SliceStr(src_as_slice));
}
else {
throw TypeError() << src << " is neither integer- nor string- valued";
}
}


void OldExpr::_init_from_string(py::robj src) {
head = ptrHead(new Head_Literal_String(src));
}


void OldExpr::_init_from_type(py::robj src) {
head = ptrHead(new Head_Literal_Type(src));
}



Expand Down Expand Up @@ -250,19 +182,6 @@ RiGb OldExpr::evaluate_iby(EvalContext& ctx) const {
}


bool OldExpr::evaluate_bool() const {
auto boolhead = dynamic_cast<Head_Literal_Bool*>(head.get());
xassert(boolhead);
return boolhead->get_value();
}


py::oobj OldExpr::evaluate_pystr() const {
auto strhead = dynamic_cast<Head_Literal_String*>(head.get());
xassert(strhead);
return strhead->evaluate_pystr();
}


std::shared_ptr<FExpr> OldExpr::unnegate_column() const {
auto unaryfn_head = dynamic_cast<Head_Func_Unary*>(head.get());
Expand All @@ -274,13 +193,6 @@ std::shared_ptr<FExpr> OldExpr::unnegate_column() const {
}


int64_t OldExpr::evaluate_int() const {
auto inthead = dynamic_cast<Head_Literal_Int*>(head.get());
xassert(inthead);
return inthead->get_value();
}


int OldExpr::precedence() const noexcept {
return 0;
}
Expand Down
11 changes: 0 additions & 11 deletions src/core/expr/expr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,27 +117,16 @@ class OldExpr : public FExpr {
void prepare_by(EvalContext&, Workframe&, std::vector<SortFlag>&) const override;

std::shared_ptr<FExpr> unnegate_column() const override;
bool evaluate_bool() const override;
int64_t evaluate_int() const override;
py::oobj evaluate_pystr() const override;

private:
// Construction helpers
void _init_from_bool(py::robj);
void _init_from_dictionary(py::robj);
void _init_from_dtexpr(py::robj);
void _init_from_ellipsis();
void _init_from_float(py::robj);
void _init_from_frame(py::robj);
void _init_from_int(py::robj);
void _init_from_iterable(py::robj);
void _init_from_list(py::robj);
void _init_from_none();
void _init_from_numpy(py::robj);
void _init_from_pandas(py::robj);
void _init_from_range(py::robj);
void _init_from_slice(py::robj);
void _init_from_string(py::robj);
void _init_from_type(py::robj);
};

Expand Down
57 changes: 43 additions & 14 deletions src/core/expr/fexpr.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,38 +20,41 @@
// IN THE SOFTWARE.
//------------------------------------------------------------------------------
#include "expr/fexpr.h"
#include "expr/expr.h" // OldExpr
#include "expr/fexpr_literal.h"
#include "expr/expr.h" // OldExpr
#include "python/obj.h"
#include "utils/exceptions.h"
namespace dt {
namespace expr {



//------------------------------------------------------------------------------
// dt::expr::FExpr class
//------------------------------------------------------------------------------

ptrExpr FExpr::unnegate_column() const {
return nullptr;
}


bool FExpr::evaluate_bool() const {
throw RuntimeError();
}
throw RuntimeError(); // LCOV_EXCL_LINE
} // LCOV_EXCL_LINE


int64_t FExpr::evaluate_int() const {
throw RuntimeError();
}
throw RuntimeError(); // LCOV_EXCL_LINE
} // LCOV_EXCL_LINE


py::oobj FExpr::evaluate_pystr() const {
throw RuntimeError();
}
throw RuntimeError(); // LCOV_EXCL_LINE
} // LCOV_EXCL_LINE


void FExpr::prepare_by(EvalContext&, Workframe&, std::vector<SortFlag>&) const {
throw RuntimeError() << "Unexpected prepare_by() call"; // LCOV_EXCL_LINE
} // LCOV_EXCL_LINE
throw RuntimeError(); // LCOV_EXCL_LINE
} // LCOV_EXCL_LINE



Expand All @@ -60,12 +63,38 @@ void FExpr::prepare_by(EvalContext&, Workframe&, std::vector<SortFlag>&) const {
// as_fexpr()
//------------------------------------------------------------------------------

// TODO: subsume functionality of OldExpr::OldExpr(py::robj)
static ptrExpr extract_fexpr(py::robj src) {
xassert(src.is_fexpr());
auto fexpr = reinterpret_cast<py::FExpr*>(src.to_borrowed_ref());
return fexpr->get_expr();
}


ptrExpr as_fexpr(py::robj src) {
if (src.is_fexpr()) {
auto fexpr = reinterpret_cast<py::FExpr*>(src.to_borrowed_ref());
return fexpr->get_expr();
if (src.is_fexpr()) return extract_fexpr(src);
else if (src.is_dtexpr()) ;
else if (src.is_int()) return FExpr_Literal_Int::make(src);
else if (src.is_string()) return FExpr_Literal_String::make(src);
else if (src.is_float()) return FExpr_Literal_Float::make(src);
else if (src.is_bool()) return FExpr_Literal_Bool::make(src);
else if (src.is_slice()) return FExpr_Literal_Slice::make(src);
else if (src.is_list_or_tuple()) ;
else if (src.is_dict()) ;
else if (src.is_anytype()) return FExpr_Literal_Type::make(src);
else if (src.is_generator()) ;
else if (src.is_none()) return FExpr_Literal_None::make();
else if (src.is_frame()) ;
else if (src.is_range()) return FExpr_Literal_Range::make(src);
else if (src.is_pandas_frame() ||
src.is_pandas_series()) ;
else if (src.is_numpy_array() ||
src.is_numpy_marray()) ;
else if (src.is_ellipsis()) return ptrExpr(new FExpr_Literal_SliceAll());
else {
throw TypeError() << "An object of type " << src.typeobj()
<< " cannot be used in an FExpr";
}
// TODO: remove this
return std::make_shared<OldExpr>(src);
}

Expand Down
43 changes: 30 additions & 13 deletions src/core/expr/head_literal.cc → src/core/expr/fexpr_literal.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,48 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//------------------------------------------------------------------------------
#include "expr/head_literal.h"
#include "column/const.h"
#include "expr/fexpr_literal.h"
#include "expr/eval_context.h"
#include "expr/workframe.h"
#include "utils/assert.h"
namespace dt {
namespace expr {


ptrExpr FExpr_Literal_Slice::make(py::robj src) {
auto ss = src.to_oslice();
if (ss.is_trivial()) {
return ptrExpr(new FExpr_Literal_SliceAll);
}
else if (ss.is_numeric()) {
return ptrExpr(new FExpr_Literal_SliceInt(ss));
}
else if (ss.is_string()) {
return ptrExpr(new FExpr_Literal_SliceStr(ss));
}
else {
throw TypeError() << src << " is neither integer- nor string- valued";
}
}

//------------------------------------------------------------------------------
// Head_Literal
//------------------------------------------------------------------------------

Workframe Head_Literal::_wrap_column(EvalContext& ctx, Column&& col) {
Workframe outputs(ctx);
outputs.add_column(std::move(col), std::string(), Grouping::SCALAR);
return outputs;

Workframe FExpr_Literal_Slice::evaluate_n(EvalContext&) const {
throw TypeError() << "A slice expression cannot appear in this context";
}


Workframe Head_Literal::evaluate_r(const vecExpr& args, EvalContext& ctx,
const sztvec&) const
{
return evaluate_n(args, ctx);
Workframe FExpr_Literal_Slice::evaluate_r(EvalContext&, const sztvec&) const {
throw TypeError() << "A slice expression cannot appear in this context";
}


int FExpr_Literal_Slice::precedence() const noexcept {
return 0;
}





}} // namespace dt::expr
Loading

0 comments on commit a7816ca

Please sign in to comment.