Implementing extracting symbols from load()s and adding them to stubs.

This change adds LoadExtractor, which walks the AST of the .bzl file and
extracts information from the load() statements about symbols that are loaded
from other .bzl files. These symbols are then added to the set of global stubs
in RuleDocExtractor so that, as a first step, if any of these symbols loaded
from other .bzl files are referenced in the global scope, Skydoc would not
raise a NameError.

skydoc/BUILD

@@ -22,6 +22,19 @@ py_test(
     ],
 )
 
+py_library(
+    name = "load_extractor",
+    srcs = ["load_extractor.py"],
+)
+
+py_test(
+    name = "load_extractor_test",
+    srcs = ["load_extractor_test.py"],
+    deps = [
+        ":load_extractor",
+    ],
+)
+
 py_library(
     name = "macro_extractor",
     srcs = ["macro_extractor.py"],
@@ -42,6 +55,7 @@ py_library(
     srcs = ["rule_extractor.py"],
     deps = [
         ":common",
+        ":load_extractor",
         "//skydoc/stubs",
     ],
 )
@@ -51,6 +65,7 @@ py_test(
     srcs = ["rule_extractor_test.py"],
     deps = [
         ":build_pb_py",
+        ":load_extractor",
         ":rule_extractor",
     ],
 )

skydoc/load_extractor.py

@@ -0,0 +1,110 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Extracts information about symbols loaded from other .bzl files."""
+
+import ast
+from collections import namedtuple
+
+LoadSymbol = namedtuple('LoadSymbol', ['label', 'symbol', 'alias'])
+"""Information about a symbol loaded from another .bzl file."""
+
+
+class LoadExtractorError(Exception):
+  """Error raised by LoadExtractor"""
+  pass
+
+
+class LoadExtractor(object):
+  """Extracts information on symbols load()ed from other .bzl files."""
+
+  def _extract_loads(self, bzl_file):
+    """Walks the AST and extracts information on loaded symbols."""
+    load_symbols = []
+    try:
+      tree = ast.parse(open(bzl_file).read(), bzl_file)
+      key = None
+      for node in ast.iter_child_nodes(tree):
+        if not isinstance(node, ast.Expr):
+          continue
+        call = node.value
+        if (not isinstance(call, ast.Call) or
+            not isinstance(call.func, ast.Name) or
+            call.func.id != 'load'):
+          continue
+
+        args = []
+        for arg in call.args:
+          if not isinstance(arg, ast.Str):
+            raise LoadExtractorError(
+                'Only string literals in load statments are supported.')
+          args.append(arg.s)
+        kwargs = {}
+        for keyword in call.keywords:
+          if not isinstance(keyword.value, ast.Str):
+            raise LoadExtractorError(
+                'Only string literals in load statments are supported.')
+          kwargs[keyword.arg] = keyword.value.s
+
+        label = args[0]
+        for arg in args[1:]:
+          load_symbol = LoadSymbol(label, arg, None)
+          load_symbols.append(load_symbol)
+        for alias, symbol in kwargs.iteritems():
+          load_symbol = LoadSymbol(label, symbol, alias)
+          load_symbols.append(load_symbol)
+
+    except IOError:
+      print("Failed to parse {0}: {1}".format(bzl_file, e.strerror))
+      pass
+
+    return load_symbols
+
+  def _validate_loads(self, load_symbols):
+    """Checks that there are no collisions from the extracted symbols."""
+    symbols = set()
+    for load in load_symbols:
+      if load.alias:
+        if load.alias in symbols:
+          raise LoadExtractorError(
+              "Load symbol conflict: %s (aliased from %s) loaded from %s" %
+              (load.alias, load.symbol, load.label))
+        else:
+          symbols.add(load.alias)
+      elif load.symbol in symbols:
+        raise LoadExtractorError(
+            "Load symbol conflict: %s loaded from %s" %
+            (load.alias, load.label))
+      else:
+        symbols.add(load.symbol)
+
+  def extract(self, bzl_file):
+    """Extracts symbols loaded from other .bzl files.
+
+    Walks the AST of the .bzl files and extracts information about symbols
+    loaded from other .bzl files from load() calls. Then, validate the
+    extracted symbols to check that all symbols are unique.
+
+    Note that only load() calls where all arguments are string literals
+    (ast.Str) are supported.
+
+    Args:
+      bzl_file: The .bzl file to extract load symbols from.
+
+    Returns:
+      List of LoadSymbol objects.
+    """
+    load_symbols = self._extract_loads(bzl_file)
+    self._validate_loads(load_symbols)
+    return load_symbols

skydoc/load_extractor_test.py

@@ -0,0 +1,90 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import textwrap
+import unittest
+
+from skydoc import load_extractor
+
+class LoadExtractorTest(unittest.TestCase):
+
+  def check_symbols(self, src, expected):
+    with tempfile.NamedTemporaryFile() as tf:
+      tf.write(src)
+      tf.flush()
+
+      extractor = load_extractor.LoadExtractor()
+      load_symbols = extractor.extract(tf.name)
+
+      self.assertEqual(expected, load_symbols)
+
+  def test_load(self):
+    src = textwrap.dedent("""\
+      load("//foo/bar:bar.bzl", "foo_library")
+      load("//foo/bar:baz.bzl", "foo_test", orig_foo_binary = "foo_binary")
+      """)
+    expected = [
+      load_extractor.LoadSymbol('//foo/bar:bar.bzl', 'foo_library', None),
+      load_extractor.LoadSymbol('//foo/bar:baz.bzl', 'foo_test', None),
+      load_extractor.LoadSymbol('//foo/bar:baz.bzl', 'foo_binary',
+                                'orig_foo_binary'),
+    ]
+    self.check_symbols(src, expected)
+
+  def raises_error(self, src):
+    with tempfile.NamedTemporaryFile() as tf:
+      tf.write(src)
+      tf.flush()
+
+      extractor = load_extractor.LoadExtractor()
+      self.assertRaises(load_extractor.LoadExtractorError,
+                        extractor.extract, tf.name)
+
+  def test_invalid_non_string_literal_in_label(self):
+    src = textwrap.dedent("""\
+      load(load_label, "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_non_string_literal_in_keywords(self):
+    src = textwrap.dedent("""\
+        load("//foo/bar:bar.bzl", loaded_symbol)
+      """)
+    self.raises_error(src)
+
+  def test_invalid_symbol_conflict(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", "foo_binary", "foo_library")
+        load("//foo:baz.bzl", "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_symbol_alias_conflict(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", foo_library="some_foo_library")
+        load("//foo:baz.bzl", "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_duplicate_symbol_loaded(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", "foo_library", "foo_library")
+      """)
+    self.raises_error(src)
+
+
+if __name__ == '__main__':
+  unittest.main()

skydoc/main.py

@@ -26,6 +26,7 @@
 import zipfile
 
 from skydoc import common
+from skydoc import load_extractor
 from skydoc import macro_extractor
 from skydoc import rule
 from skydoc import rule_extractor
@@ -280,11 +281,21 @@ def main(argv):
     sys.exit(1)
 
   rulesets = []
+  load_sym_extractor = load_extractor.LoadExtractor()
   for bzl_file in bzl_files:
+    load_symbols = []
+    try:
+      load_symbols = load_sym_extractor.extract(bzl_file)
+    except load_extractor.LoadExtractorError as e:
+      print("ERROR: Error extracting loaded symbols from %s: %s" %
+            (bzl_file, str(e)))
+      sys.exit(2)
+
+    # TODO(dzc): Make MacroDocExtractor and RuleDocExtractor stateless.
     macro_doc_extractor = macro_extractor.MacroDocExtractor()
     rule_doc_extractor = rule_extractor.RuleDocExtractor()
     macro_doc_extractor.parse_bzl(bzl_file)
-    rule_doc_extractor.parse_bzl(bzl_file)
+    rule_doc_extractor.parse_bzl(bzl_file, load_symbols)
     merged_language = merge_languages(macro_doc_extractor.proto(),
                                       rule_doc_extractor.proto())
     rulesets.append(

skydoc/rule_extractor.py

@@ -43,15 +43,45 @@
 """Stubs for Skylark globals to be used to evaluate the .bzl file."""
 
 
+SKYLARK_GLOBAL_SYMBOLS = set(SKYLARK_STUBS.keys())
+
+
+def create_stubs(skylark_stubs, load_symbols):
+  """Combines Skylark stubs with loaded symbols.
+
+  This function creates a copy of the global Skylark stubs and combines them
+  with symbols from the list of load_extractor.LoadSymbol, which contain
+  information about symbols extracted from other .bzl files. The stubs created
+  for the loaded symbols are global variables set to the empty string.
+
+  Args:
+    skylark_stubs: Dict containing the Skylark global stubs.
+    load_symbols: List of load_extractor.LoadSymbol objects containing
+      information about symbols extracted from other .bzl files.
+
+  Returns:
+    Dictionary containing both the Skylark global stubs and stubs created for
+    the loaded symbols.
+  """
+  stubs = dict(skylark_stubs)
+  for load_symbol in load_symbols:
+    if load_symbol.alias:
+      stubs[load_symbol.alias] = ""
+    else:
+      stubs[load_symbol.symbol] = ""
+  return stubs
+
+
 class RuleDocExtractor(object):
   """Extracts documentation for rules from a .bzl file."""
 
   def __init__(self):
     """Inits RuleDocExtractor with a new BuildLanguage proto"""
     self.__language = build_pb2.BuildLanguage()
     self.__extracted_rules = {}
+    self.__load_symbols = []
 
-  def _process_skylark(self, bzl_file):
+  def _process_skylark(self, bzl_file, load_symbols):
     """Evaluates the Skylark code in the .bzl file.
 
     This function evaluates the Skylark code in the .bzl file as Python against
@@ -60,15 +90,19 @@ def _process_skylark(self, bzl_file):
 
     Args:
       bzl_file: The .bzl file to evaluate.
+      load_symbols: List of load_extractor.LoadSymbol objects containing info
+        about symbols load()ed from other .bzl files.
     """
+    compiled = compile(open(bzl_file).read(), bzl_file, 'exec')
     skylark_locals = {}
-    compiled = compile(open(bzl_file).read(), bzl_file, "exec")
-    exec(compiled) in SKYLARK_STUBS, skylark_locals
+    global_stubs = create_stubs(SKYLARK_STUBS, load_symbols)
+    exec(compiled) in global_stubs, skylark_locals
 
     for name, obj in skylark_locals.iteritems():
-      if hasattr(obj, "is_rule") and not name.startswith("_"):
-        obj.attrs["name"] = attr.AttrDescriptor(
-            type=build_pb2.Attribute.UNKNOWN, mandatory=True, name="name")
+      if (isinstance(obj, skylark_globals.RuleDescriptor) and
+          not name.startswith('_')):
+        obj.attrs['name'] = attr.AttrDescriptor(
+            type=build_pb2.Attribute.UNKNOWN, mandatory=True, name='name')
         self.__extracted_rules[name] = obj
 
   def _add_rule_doc(self, name, doc):
@@ -134,7 +168,7 @@ def _assemble_protos(self):
     """Builds the BuildLanguage protos for the extracted rule documentation.
 
     Iterates through the map of extracted rule documentation and builds a
-    BuildLanguage proto containing the documentation for publid rules extracted
+    BuildLanguage proto containing the documentation for public rules extracted
     from the .bzl file.
     """
     rules = []
@@ -173,7 +207,13 @@ def _assemble_protos(self):
         output.template = template
         output.documentation = doc
 
-  def parse_bzl(self, bzl_file):
+    for load_symbol in self.__load_symbols:
+      load = self.__language.load.add()
+      load.label = load_symbol.label
+      load.symbol = load_symbol.symbol
+      load.alias = load_symbol.alias
+
+  def parse_bzl(self, bzl_file, load_symbols):
     """Extracts the documentation for all public rules from the given .bzl file.
 
     The Skylark code is first evaluated against stubs to extract rule and
@@ -184,11 +224,10 @@ def parse_bzl(self, bzl_file):
     Args:
       bzl_file: The .bzl file to extract rule documentation from.
     """
-    self._process_skylark(bzl_file)
+    self._process_skylark(bzl_file, load_symbols)
     self._extract_docstrings(bzl_file)
     self._assemble_protos()
 
   def proto(self):
     """Returns the proto containing the macro documentation."""
     return self.__language
-