Merge pull request #1185 from Thirumalai-Shaktivel/compile_packages

Compile Simple Python package

integration_tests/CMakeLists.txt

@@ -194,6 +194,7 @@ RUN(NAME test_dict_05        LABELS cpython llvm)
 RUN(NAME test_for_loop       LABELS cpython llvm c)
 RUN(NAME modules_01          LABELS cpython llvm)
 RUN(NAME modules_02          LABELS cpython llvm)
+RUN(NAME test_import_01      LABELS cpython llvm)
 RUN(NAME test_math           LABELS cpython llvm)
 RUN(NAME test_numpy_01       LABELS cpython llvm c)
 RUN(NAME test_numpy_02       LABELS cpython llvm c)

integration_tests/test_import/__init__.py

@@ -0,0 +1,2 @@
+from .test_import_1 import print_a, print_b
+from .test_import_2 import print_c

integration_tests/test_import/test_import_1.py

@@ -0,0 +1,5 @@
+def print_a() -> str:
+    return "A"
+
+def print_b() -> str:
+    return "B"

integration_tests/test_import/test_import_2.py

@@ -0,0 +1,2 @@
+def print_c() -> str:
+    return "C"

integration_tests/test_import_01.py

@@ -0,0 +1,6 @@
+import test_import
+from test_import import print_a
+
+print(print_a())
+print(test_import.print_b())
+print(test_import.print_c())

src/bin/lpython.cpp

@@ -164,7 +164,8 @@ int emit_asr(const std::string &infile,
     diagnostics.diagnostics.clear();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     std::cerr << diagnostics.render(input, lm, compiler_options);
     if (!r.ok) {
         LFORTRAN_ASSERT(diagnostics.has_error())
@@ -207,7 +208,8 @@ int emit_cpp(const std::string &infile,
     diagnostics.diagnostics.clear();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     std::cerr << diagnostics.render(input, lm, compiler_options);
     if (!r1.ok) {
         LFORTRAN_ASSERT(diagnostics.has_error())
@@ -248,7 +250,8 @@ int emit_c(const std::string &infile,
     diagnostics.diagnostics.clear();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     std::cerr << diagnostics.render(input, lm, compiler_options);
     if (!r1.ok) {
         LFORTRAN_ASSERT(diagnostics.has_error())
@@ -289,7 +292,8 @@ int emit_wat(const std::string &infile,
     diagnostics.diagnostics.clear();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     std::cerr << diagnostics.render(input, lm, compiler_options);
     if (!r1.ok) {
         LFORTRAN_ASSERT(diagnostics.has_error())
@@ -333,7 +337,8 @@ int get_symbols (const std::string &infile,
            LFortran::LPython::AST::ast_t* ast = r1.result;
            LFortran::Result<LFortran::ASR::TranslationUnit_t*>
                x = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-                       compiler_options.disable_main, compiler_options.symtab_only, infile);
+                       compiler_options.disable_main, compiler_options.symtab_only,
+                       infile, compiler_options.import_path);
            if (!x.ok) {
                std::cout << "{}\n";
                return 0;
@@ -426,7 +431,8 @@ int get_errors (const std::string &infile,
             LFortran::LPython::AST::ast_t* ast = r1.result;
             LFortran::Result<LFortran::ASR::TranslationUnit_t*>
                 r = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-                        compiler_options.disable_main, compiler_options.symtab_only, infile);
+                        compiler_options.disable_main, compiler_options.symtab_only,
+                        infile, compiler_options.import_path);
         }
         std::vector<LFortran::LPython::error_highlight> diag_lists;
         LFortran::LPython::error_highlight h;
@@ -538,7 +544,8 @@ int emit_llvm(const std::string &infile,
     diagnostics.diagnostics.clear();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     std::cerr << diagnostics.render(input, lm, compiler_options);
     if (!r1.ok) {
         LFORTRAN_ASSERT(diagnostics.has_error())
@@ -596,7 +603,8 @@ int compile_python_to_object_file(
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics,
             !(arg_c && compiler_options.disable_main),
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     auto ast_to_asr_end = std::chrono::high_resolution_clock::now();
     times.push_back(std::make_pair("AST to ASR", std::chrono::duration<double, std::milli>(ast_to_asr_end - ast_to_asr_start).count()));
     std::cerr << diagnostics.render(input, lm, compiler_options);
@@ -679,7 +687,8 @@ int compile_to_binary_wasm(
     auto ast_to_asr_start = std::chrono::high_resolution_clock::now();
     LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         r1 = LFortran::LPython::python_ast_to_asr(al, *ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, infile);
+            compiler_options.disable_main, compiler_options.symtab_only, infile,
+            compiler_options.import_path);
     auto ast_to_asr_end = std::chrono::high_resolution_clock::now();
     times.push_back(std::make_pair("AST to ASR", std::chrono::duration<double, std::milli>(ast_to_asr_end - ast_to_asr_start).count()));
     std::cerr << diagnostics.render(input, lm, compiler_options);
@@ -927,7 +936,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_asr_from_source(char *input) {
         auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
         LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, "input");
+            compiler_options.disable_main, compiler_options.symtab_only, "input",
+            compiler_options.import_path);
         out = diagnostics.render(input, lm, compiler_options);
         if (asr.ok) {
             out += LFortran::pickle(*asr.result, compiler_options.use_colors, compiler_options.indent,
@@ -946,7 +956,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_wat_from_source(char *input) {
         auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
         LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, "input");
+            compiler_options.disable_main, compiler_options.symtab_only, "input",
+            compiler_options.import_path);
         out = diagnostics.render(input, lm, compiler_options);
         if (asr.ok) {
             LFortran::Result<LFortran::Vec<uint8_t>>
@@ -974,7 +985,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_cpp_from_source(char *input) {
         auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
         LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, "input");
+            compiler_options.disable_main, compiler_options.symtab_only, "input",
+            compiler_options.import_path);
         out = diagnostics.render(input, lm, compiler_options);
         if (asr.ok) {
             auto res = LFortran::asr_to_cpp(al, *asr.result, diagnostics,
@@ -1013,7 +1025,8 @@ EMSCRIPTEN_KEEPALIVE char* emit_wasm_from_source(char *input) {
         auto casted_ast = (LFortran::LPython::AST::ast_t*)ast.result;
         LFortran::Result<LFortran::ASR::TranslationUnit_t*>
         asr = LFortran::LPython::python_ast_to_asr(al, *casted_ast, diagnostics, true,
-            compiler_options.disable_main, compiler_options.symtab_only, "input");
+            compiler_options.disable_main, compiler_options.symtab_only, "input",
+            compiler_options.import_path);
         out = diagnostics.render(input, lm, compiler_options);
         if (asr.ok) {
             LFortran::Result<LFortran::Vec<uint8_t>>
@@ -1114,7 +1127,8 @@ int main(int argc, char *argv[])
         // app.add_flag("-E", arg_E, "Preprocess only; do not compile, assemble or link");
         // app.add_option("-l", arg_l, "Link library option");
         // app.add_option("-L", arg_L, "Library path option");
-        // app.add_option("-I", arg_I, "Include path")->allow_extra_args(false);
+        app.add_option("-I", compiler_options.import_path, "Specify the path"
+            "to look for the module")->allow_extra_args(false);
         // app.add_option("-J", arg_J, "Where to save mod files");
         // app.add_flag("-g", arg_g, "Compile with debugging information");
         // app.add_option("-D", compiler_options.c_preprocessor_defines, "Define <macro>=<value> (or 1 if <value> omitted)")->allow_extra_args(false);

src/libasr/utils.h

@@ -37,6 +37,7 @@ struct CompilerOptions {
     bool implicit_typing = false;
     bool implicit_interface = false;
     std::string target = "";
+    std::string import_path = "";
     Platform platform;
 
     CompilerOptions () : platform{get_platform()} {};

src/lpython/semantics/python_ast_to_asr.cpp

@@ -254,7 +254,7 @@ ASR::TranslationUnit_t* compile_module_till_asr(Allocator& al,
     LFortran::LocationManager lm;
     lm.in_filename = infile;
     Result<ASR::TranslationUnit_t*> r2 = python_ast_to_asr(al, *ast,
-        diagnostics, false, true, false, infile);
+        diagnostics, false, true, false, infile, "");
     // TODO: Uncomment once a check is added for ensuring
     // that module.py file hasn't changed between
     // builds.
@@ -315,6 +315,7 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
     bool compile_module = true;
     ASR::TranslationUnit_t* mod1 = nullptr;
     std::string input;
+    std::string module_dir_name = "";
     bool found = set_module_path(infile0c, rl_path, infile,
                                  path_used, input, ltypes, enum_py);
     if( !found ) {
@@ -332,7 +333,8 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
         return nullptr;
     }
     if (!found) {
-        err("Could not find the module '" + infile0 + "'", loc);
+        err("Could not find the module '" + infile0 + "'. If an import path "
+            "is available, please use the `-I` option to specify it", loc);
     }
     if (ltypes) return nullptr;
 
@@ -342,7 +344,15 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
 
     // insert into `symtab`
     std::vector<std::pair<std::string, ASR::Module_t*>> children_modules;
-    ASRUtils::extract_module_python(*mod1, children_modules, module_name);
+    if (module_name == "__init__") {
+        // remove `__init__.py`
+        module_dir_name = infile.substr(0, infile.find_last_of('/'));
+        // assign module directory name
+        module_dir_name = module_dir_name.substr(module_dir_name.find_last_of('/') + 1);
+        ASRUtils::extract_module_python(*mod1, children_modules, module_dir_name);
+    } else {
+        ASRUtils::extract_module_python(*mod1, children_modules, module_name);
+    }
     ASR::Module_t* mod2 = nullptr;
     for( auto& a: children_modules ) {
         std::string a_name = a.first;
@@ -351,6 +361,10 @@ ASR::Module_t* load_module(Allocator &al, SymbolTable *symtab,
             a_mod->m_name = s2c(al, module_name);
             a_mod->m_intrinsic = intrinsic;
             mod2 = a_mod;
+        } else if (a_name == module_dir_name) {
+            a_mod->m_name = s2c(al, module_dir_name);
+            a_mod->m_intrinsic = intrinsic;
+            mod2 = a_mod;
         }
         symtab->add_symbol(a_name, (ASR::symbol_t*)a_mod);
         a_mod->m_symtab->parent = symtab;
@@ -436,9 +450,18 @@ ASR::symbol_t* import_from_module(Allocator &al, ASR::Module_t *m, SymbolTable *
             ASR::accessType::Public
             );
         return ASR::down_cast<ASR::symbol_t>(v);
+    } else if (ASR::is_a<ASR::ExternalSymbol_t>(*t)) {
+        ASR::ExternalSymbol_t *es = ASR::down_cast<ASR::ExternalSymbol_t>(t);
+        SymbolTable *symtab = current_scope;
+        while (symtab->parent != nullptr) symtab = symtab->parent;
+        ASR::symbol_t *sym = symtab->get_symbol(es->m_module_name);
+        ASR::Module_t *m = ASR::down_cast<ASR::Module_t>(sym);
+
+        return import_from_module(al, m, symtab, es->m_name,
+                            cur_sym_name, new_sym_name, loc);
     } else {
-        throw SemanticError("Only Subroutines, Functions and Variables are currently supported in 'import'",
-            loc);
+        throw SemanticError("Only Subroutines, Functions, Variables and "
+            "ExternalSymbol are currently supported in 'import'", loc);
     }
     LFORTRAN_ASSERT(false);
     return nullptr;
@@ -475,6 +498,7 @@ class CommonVisitor : public AST::BaseVisitor<Struct> {
     IntrinsicNodeHandler intrinsic_node_handler;
     std::map<int, ASR::symbol_t*> &ast_overload;
     std::string parent_dir;
+    std::string import_path;
     Vec<ASR::stmt_t*> *current_body;
     ASR::ttype_t* ann_assign_target_type;
 
@@ -485,9 +509,10 @@ class CommonVisitor : public AST::BaseVisitor<Struct> {
 
     CommonVisitor(Allocator &al, SymbolTable *symbol_table,
             diag::Diagnostics &diagnostics, bool main_module,
-            std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
+            std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
+            std::string import_path)
         : diag{diagnostics}, al{al}, current_scope{symbol_table}, main_module{main_module},
-            ast_overload{ast_overload}, parent_dir{parent_dir},
+            ast_overload{ast_overload}, parent_dir{parent_dir}, import_path{import_path},
             current_body{nullptr}, ann_assign_target_type{nullptr} {
         current_module_dependencies.reserve(al, 4);
     }
@@ -2912,8 +2937,10 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
 
     SymbolTableVisitor(Allocator &al, SymbolTable *symbol_table,
         diag::Diagnostics &diagnostics, bool main_module,
-        std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
-      : CommonVisitor(al, symbol_table, diagnostics, main_module, ast_overload, parent_dir), is_derived_type{false} {}
+        std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
+        std::string import_path)
+      : CommonVisitor(al, symbol_table, diagnostics, main_module, ast_overload,
+            parent_dir, import_path), is_derived_type{false} {}
 
 
     ASR::symbol_t* resolve_symbol(const Location &loc, const std::string &sub_name) {
@@ -3206,6 +3233,24 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
                 st = st->parent;
             }
             bool ltypes, enum_py;
+            if (msym != "ltypes") {
+                if (import_path != "" &&
+                        !path_exits(paths[0] + '/' + msym + ".py")) {
+                    paths = {import_path};
+                    if (parent_dir != "") paths[0] += '/' + parent_dir;
+                    if(is_directory(paths[0])) {
+                        paths[0] += '/' + msym;
+                        msym = "__init__";
+                    }
+                } else if (is_directory(msym)) {
+                    paths = {rl_path, msym};
+                    msym = "__init__";
+                } else if (paths[1] != ""
+                        && is_directory(paths[1] + '/' + msym)) {
+                    paths[1] += '/' + msym;
+                    msym = "__init__";
+                }
+            }
             t = (ASR::symbol_t*)(load_module(al, st,
                 msym, x.base.base.loc, false, paths, ltypes, enum_py,
                 [&](const std::string &msg, const Location &loc) { throw SemanticError(msg, loc); }
@@ -3251,6 +3296,24 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
         }
         for (auto &mod_sym : mods) {
             bool ltypes, enum_py;
+            if (mod_sym != "ltypes") {
+                if (import_path != "" &&
+                        !path_exits(paths[0] + '/' + mod_sym + ".py")) {
+                    paths = {import_path};
+                    if (parent_dir != "") paths[0] += '/' + parent_dir;
+                    if(is_directory(paths[0])) {
+                        paths[0] += '/' + mod_sym;
+                        mod_sym = "__init__";
+                    }
+                } else if (is_directory(mod_sym)) {
+                    paths = {rl_path, mod_sym};
+                    mod_sym = "__init__";
+                } else if (paths[1] != ""
+                        && is_directory(paths[1] + '/' + mod_sym)) {
+                    paths[1] += '/' + mod_sym;
+                    mod_sym = "__init__";
+                }
+            }
             t = (ASR::symbol_t*)(load_module(al, st,
                 mod_sym, x.base.base.loc, false, paths, ltypes, enum_py,
                 [&](const std::string &msg, const Location &loc) { throw SemanticError(msg, loc); }
@@ -3355,9 +3418,11 @@ class SymbolTableVisitor : public CommonVisitor<SymbolTableVisitor> {
 
 Result<ASR::asr_t*> symbol_table_visitor(Allocator &al, const AST::Module_t &ast,
         diag::Diagnostics &diagnostics, bool main_module,
-        std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir)
+        std::map<int, ASR::symbol_t*> &ast_overload, std::string parent_dir,
+        std::string import_path)
 {
-    SymbolTableVisitor v(al, nullptr, diagnostics, main_module, ast_overload, parent_dir);
+    SymbolTableVisitor v(al, nullptr, diagnostics, main_module, ast_overload,
+        parent_dir, import_path);
     try {
         v.visit_Module(ast);
     } catch (const SemanticError &e) {
@@ -3383,8 +3448,8 @@ class BodyVisitor : public CommonVisitor<BodyVisitor> {
 
     BodyVisitor(Allocator &al, ASR::asr_t *unit, diag::Diagnostics &diagnostics,
          bool main_module, std::map<int, ASR::symbol_t*> &ast_overload)
-         : CommonVisitor(al, nullptr, diagnostics, main_module, ast_overload, ""), asr{unit},
-         gotoids{0}
+         : CommonVisitor(al, nullptr, diagnostics, main_module, ast_overload, "", ""),
+         asr{unit}, gotoids{0}
          {}
 
     // Transforms statements to a list of ASR statements
@@ -5524,15 +5589,16 @@ std::string get_parent_dir(const std::string &path) {
 
 Result<ASR::TranslationUnit_t*> python_ast_to_asr(Allocator &al,
     AST::ast_t &ast, diag::Diagnostics &diagnostics, bool main_module,
-    bool disable_main, bool symtab_only, std::string file_path)
+    bool disable_main, bool symtab_only, std::string file_path,
+    std::string import_path)
 {
     std::map<int, ASR::symbol_t*> ast_overload;
     std::string parent_dir = get_parent_dir(file_path);
     AST::Module_t *ast_m = AST::down_cast2<AST::Module_t>(&ast);
 
     ASR::asr_t *unit;
     auto res = symbol_table_visitor(al, *ast_m, diagnostics, main_module,
-        ast_overload, parent_dir);
+        ast_overload, parent_dir, import_path);
     if (res.ok) {
         unit = res.result;
     } else {