Skip to content

Commit

Permalink
remove the stateful parse_next API
Browse files Browse the repository at this point in the history
  • Loading branch information
@JeffBezanson
JeffBezanson committed Jan 17, 2016
1 parent b992fe6 commit 63277f4
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 163 deletions.
123 changes: 71 additions & 52 deletions src/ast.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -694,23 +694,85 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
return result;
}

jl_ast_context_t *jl_start_parsing_file(const char *fname)
// parse and eval a whole file, possibly reading from a string (`content`)
jl_value_t *jl_parse_eval_all(const char *fname, size_t len,
const char *content, size_t contentlen)
{
jl_ast_context_t *ctx = jl_ast_ctx_enter();
fl_context_t *fl_ctx = &ctx->fl;
value_t s = cvalue_static_cstring(fl_ctx, fname);
if (fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), s) == fl_ctx->F) {
value_t f, ast;
f = cvalue_static_cstring(fl_ctx, fname);
fl_gc_handle(fl_ctx, &f);
if (content != NULL) {
value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
fl_gc_handle(fl_ctx, &t);
ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
fl_free_gc_handles(fl_ctx, 1);
}
else {
ast = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), f);
}
fl_free_gc_handles(fl_ctx, 1);
if (ast == fl_ctx->F) {
jl_ast_ctx_leave(ctx);
return NULL;
jl_errorf("could not open file %s", fname);
}
return ctx;
fl_gc_handle(fl_ctx, &ast);

int last_lineno = jl_lineno;
const char *last_filename = jl_filename;
jl_lineno = 0;
jl_filename = fname;
jl_array_t *roots = NULL;
jl_array_t **old_roots = ctx->roots;
ctx->roots = &roots;
jl_value_t *form=NULL, *result=jl_nothing;
int err = 0;
JL_GC_PUSH3(&roots, &form, &result);
JL_TRY {
assert(iscons(ast) && car_(ast) == symbol(fl_ctx,"toplevel"));
ast = cdr_(ast);
while (iscons(ast)) {
value_t expansion = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk")), car_(ast));
form = scm_to_julia(fl_ctx, expansion, 0);
jl_sym_t *head = NULL;
if (jl_is_expr(form)) head = ((jl_expr_t*)form)->head;
if (head == jl_incomplete_sym)
jl_errorf("syntax: %s", jl_string_data(jl_exprarg(form,0)));
else if (head == error_sym)
jl_interpret_toplevel_expr(form);
else if (head == line_sym)
jl_lineno = jl_unbox_long(jl_exprarg(form,0));
else
result = jl_toplevel_eval_flex(form, 1);
ast = cdr_(ast);
}
}
JL_CATCH {
form = jl_pchar_to_string(fname, len);
result = jl_box_long(jl_lineno);
err = 1;
}
jl_lineno = last_lineno;
jl_filename = last_filename;
fl_free_gc_handles(fl_ctx, 1);
ctx->roots = old_roots;
jl_ast_ctx_leave(ctx);
if (err) {
if (jl_loaderror_type == NULL)
jl_rethrow();
else
jl_rethrow_other(jl_new_struct(jl_loaderror_type, form, result,
jl_exception_in_transit));
}
JL_GC_POP();
return result;
}

void jl_stop_parsing(jl_ast_context_t *ctx)
JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
char *filename, size_t namelen)
{
fl_context_t *fl_ctx = &ctx->fl;
fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-close-stream")));
jl_ast_ctx_leave(ctx);
return jl_parse_eval_all(filename, namelen, text, len);
}

JL_DLLEXPORT int jl_parse_depwarn(int warn)
Expand All @@ -735,49 +797,6 @@ static int jl_parse_deperror(fl_context_t *fl_ctx, int err)
return prev == fl_ctx->T ? 1 : 0;
}

jl_value_t *jl_parse_next(jl_ast_context_t *ctx)
{
fl_context_t *fl_ctx = &ctx->fl;
JL_AST_PRESERVE_PUSH(ctx, roots, old_roots);
value_t c = fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-next")));
if (c == fl_ctx->FL_EOF) {
JL_AST_PRESERVE_POP(ctx, old_roots);
return NULL;
}
if (iscons(c)) {
if (cdr_(c) == fl_ctx->FL_EOF) {
JL_AST_PRESERVE_POP(ctx, old_roots);
return NULL;
}
value_t a = car_(c);
if (isfixnum(a)) {
jl_lineno = numval(a);
//jl_printf(JL_STDERR, " on line %d\n", jl_lineno);
c = cdr_(c);
}
}
// for error, get most recent line number
if (iscons(c) && car_(c) == jl_ast_ctx(fl_ctx)->error_sym)
jl_lineno = numval(fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-current-lineno"))));
jl_value_t *res = scm_to_julia(fl_ctx, c, 0);
JL_AST_PRESERVE_POP(ctx, old_roots);
return res;
}

JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
char *filename, size_t namelen)
{
jl_ast_context_t *ctx = jl_ast_ctx_enter();
fl_context_t *fl_ctx = &ctx->fl;
value_t t, f;
t = cvalue_static_cstrn(fl_ctx, text, len);
fl_gc_handle(fl_ctx, &t);
f = cvalue_static_cstrn(fl_ctx, filename, namelen);
fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
fl_free_gc_handles(fl_ctx, 1);
return jl_parse_eval_all(filename, namelen, ctx);
}

// returns either an expression or a thunk
JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr)
{
Expand Down
82 changes: 29 additions & 53 deletions src/jlfrontend.scm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,10 @@
(define (jl-parse-one-string s pos0 greedy)
(let ((inp (open-input-string s)))
(io.seek inp pos0)
(let ((expr
(parser-wrap (lambda ()
(if greedy
(julia-parse inp)
(julia-parse inp parse-atom))))))
(let ((expr (parser-wrap (lambda ()
(if greedy
(julia-parse inp)
(julia-parse inp parse-atom))))))
(cons expr (io.pos inp)))))

(define (jl-parse-string s)
Expand All @@ -158,39 +157,37 @@
(loop (nreconc (cdr expr) exprs))
(loop (cons expr exprs))))))))))

(define (jl-parse-all io filename)
(unwind-protect
(with-bindings ((current-filename (symbol filename)))
(let ((stream (make-token-stream io)))
(let loop ((exprs '()))
(let ((lineno (parser-wrap
(lambda ()
(skip-ws-and-comments (ts:port stream))
(input-port-line (ts:port stream))))))
(if (pair? lineno)
(cons 'toplevel (reverse! (cons lineno exprs)))
(let ((expr (parser-wrap
(lambda ()
(julia-parse stream)))))
(if (eof-object? expr)
(cons 'toplevel (reverse! exprs))
(let ((next (list* expr `(line ,lineno) exprs)))
(if (and (pair? expr) (eq? (car expr) 'error))
(cons 'toplevel (reverse! next))
(loop next))))))))))
(io.close io)))

;; parse file-in-a-string
(define (jl-parse-string-stream str filename)
(jl-parser-set-stream filename (open-input-string str)))
(jl-parse-all (open-input-string str) filename))

(define (jl-parse-file s)
(define (jl-parse-file filename)
(trycatch
(let ((b (buffer))
(f (open-input-file s)))
;; read whole file first to avoid problems with concurrent modification (issue #10497)
(io.copy b f)
(io.close f)
(io.seek b 0)
(begin (jl-parser-set-stream s b)
#t))
(jl-parse-all (open-input-file filename) filename)
(lambda (e) #f)))

(define *filename-stack* '())
(define *ts-stack* '())
(define current-token-stream #())

(define (jl-parser-set-stream name stream)
(set! *filename-stack* (cons current-filename *filename-stack*))
(set! *ts-stack* (cons current-token-stream *ts-stack*))
(set! current-filename (symbol name))
(set! current-token-stream (make-token-stream stream)))

(define (jl-parser-close-stream)
(io.close (ts:port current-token-stream))
(set! current-filename (car *filename-stack*))
(set! current-token-stream (car *ts-stack*))
(set! *filename-stack* (cdr *filename-stack*))
(set! *ts-stack* (cdr *ts-stack*)))

(define *depwarn* #t)
(define (jl-parser-depwarn w)
(let ((prev *depwarn*))
Expand All @@ -203,27 +200,6 @@
(set! *deperror* (eq? e #t))
prev))

(define (jl-parser-next)
(let* ((err (parser-wrap
(lambda ()
(skip-ws-and-comments (ts:port current-token-stream)))))
(lineno (input-port-line (ts:port current-token-stream))))
(cons lineno
(if (pair? err)
err
(parser-wrap
(lambda ()
(let ((e (julia-parse current-token-stream)))
(if (eof-object? e)
e
(if (and (pair? e) (or (eq? (car e) 'error)
(eq? (car e) 'continue)))
e
(expand-toplevel-expr e))))))))))

(define (jl-parser-current-lineno)
(input-port-line (ts:port current-token-stream)))

; expand a piece of raw surface syntax to an executable thunk
(define (jl-expand-to-thunk expr)
(parser-wrap (lambda ()
Expand Down
4 changes: 3 additions & 1 deletion src/julia_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,13 @@ typedef struct _jl_ast_context_t jl_ast_context_t;
jl_ast_context_t *jl_start_parsing_file(const char *fname);
void jl_stop_parsing(jl_ast_context_t *ctx);
jl_value_t *jl_parse_next(jl_ast_context_t *ctx);
jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast);

jl_lambda_info_t *jl_wrap_expr(jl_value_t *expr);
void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx);
jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e);
jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_ast_context_t *ctx);
jl_value_t *jl_parse_eval_all(const char *fname, size_t len,
const char *content, size_t contentlen);
jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam);
jl_value_t *jl_interpret_toplevel_thunk_with(jl_lambda_info_t *lam,
jl_value_t **loc, size_t nl);
Expand Down
58 changes: 1 addition & 57 deletions src/toplevel.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ jl_module_t *jl_old_base_module = NULL;
// the Main we started with, in case it is switched
jl_module_t *jl_internal_main_module = NULL;

jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast);

JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m)
{
assert(jl_base_module != NULL);
Expand Down Expand Up @@ -554,54 +552,6 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v)
return jl_toplevel_eval_flex(v, 1);
}

// repeatedly call jl_parse_next and eval everything
jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_ast_context_t *ctx)
{
//jl_printf(JL_STDERR, "***** loading %s\n", fname);
int last_lineno = jl_lineno;
const char *last_filename = jl_filename;
jl_lineno = 0;
jl_filename = fname;
jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *result=jl_nothing;
JL_GC_PUSH4(&fn, &ln, &form, &result);
JL_TRY {
// handle syntax error
while (1) {
form = jl_parse_next(ctx);
if (form == NULL)
break;
if (jl_is_expr(form)) {
if (((jl_expr_t*)form)->head == jl_incomplete_sym) {
jl_errorf("syntax: %s", jl_string_data(jl_exprarg(form,0)));
}
if (((jl_expr_t*)form)->head == error_sym) {
jl_interpret_toplevel_expr(form);
}
}
result = jl_toplevel_eval_flex(form, 1);
}
}
JL_CATCH {
jl_stop_parsing(ctx);
fn = jl_pchar_to_string(fname, len);
ln = jl_box_long(jl_lineno);
jl_lineno = last_lineno;
jl_filename = last_filename;
if (jl_loaderror_type == NULL) {
jl_rethrow();
}
else {
jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln,
jl_exception_in_transit));
}
}
jl_stop_parsing(ctx);
jl_lineno = last_lineno;
jl_filename = last_filename;
JL_GC_POP();
return result;
}

JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len)
{
if (jl_current_module->istopmod) {
Expand All @@ -615,13 +565,7 @@ JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len)
if (jl_stat(fpath, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) {
jl_errorf("could not open file %s", fpath);
}
jl_ast_context_t *ctx = jl_start_parsing_file(fpath);
if (!ctx) {
jl_errorf("could not open file %s", fpath);
}
jl_value_t *result = jl_parse_eval_all(fpath, len, ctx);
if (fpath != fname) free(fpath);
return result;
return jl_parse_eval_all(fpath, len, NULL, 0);
}

// load from filename given as a ByteString object
Expand Down

0 comments on commit 63277f4

Please sign in to comment.