Implementing extracting symbols from load()s and adding them to stubs.

skydoc/BUILD

@@ -22,6 +22,19 @@ py_test(
     ],
 )
 
+py_library(
+    name = "load_extractor",
+    srcs = ["load_extractor.py"],
+)
+
+py_test(
+    name = "load_extractor_test",
+    srcs = ["load_extractor_test.py"],
+    deps = [
+        ":load_extractor",
+    ],
+)
+
 py_library(
     name = "macro_extractor",
     srcs = ["macro_extractor.py"],
@@ -42,6 +55,7 @@ py_library(
     srcs = ["rule_extractor.py"],
     deps = [
         ":common",
+        ":load_extractor",
         "//skydoc/stubs",
     ],
 )
@@ -51,6 +65,7 @@ py_test(
     srcs = ["rule_extractor_test.py"],
     deps = [
         ":build_pb_py",
+        ":load_extractor",
         ":rule_extractor",
     ],
 )

skydoc/load_extractor.py

@@ -0,0 +1,110 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Extracts information about symbols loaded from other .bzl files."""
+
+import ast
+from collections import namedtuple
+
+LoadSymbol = namedtuple('LoadSymbol', ['label', 'symbol', 'alias'])
+"""Information about a symbol loaded from another .bzl file."""
+
+
+class LoadExtractorError(Exception):
+  """Error raised by LoadExtractor"""
+  pass
+
+
+class LoadExtractor(object):
+  """Extracts information on symbols load()ed from other .bzl files."""
+
+  def _extract_loads(self, bzl_file):
+    """Walks the AST and extracts information on loaded symbols."""
+    load_symbols = []
+    try:
+      tree = ast.parse(open(bzl_file).read(), bzl_file)
+      key = None
+      for node in ast.iter_child_nodes(tree):
+        if not isinstance(node, ast.Expr):
+          continue
+        call = node.value
+        if (not isinstance(call, ast.Call) or
+            not isinstance(call.func, ast.Name) or
+            call.func.id != 'load'):
+          continue
+
+        args = []
+        for arg in call.args:
+          if not isinstance(arg, ast.Str):
+            raise LoadExtractorError(
+                'Only string literals in load statments are supported.')
+          args.append(arg.s)
+        kwargs = {}
+        for keyword in call.keywords:
+          if not isinstance(keyword.value, ast.Str):
+            raise LoadExtractorError(
+                'Only string literals in load statments are supported.')
+          kwargs[keyword.arg] = keyword.value.s
+
+        label = args[0]
+        for arg in args[1:]:
+          load_symbol = LoadSymbol(label, arg, None)
+          load_symbols.append(load_symbol)
+        for alias, symbol in kwargs.iteritems():
+          load_symbol = LoadSymbol(label, symbol, alias)
+          load_symbols.append(load_symbol)
+
+    except IOError:
+      print("Failed to parse {0}: {1}".format(bzl_file, e.strerror))
+      pass
+
+    return load_symbols
+
+  def _validate_loads(self, load_symbols):
+    """Checks that there are no collisions from the extracted symbols."""
+    symbols = set()
+    for load in load_symbols:
+      if load.alias:
+        if load.alias in symbols:
+          raise LoadExtractorError(
+              "Load symbol conflict: %s (aliased from %s) loaded from %s" %
+              (load.alias, load.symbol, load.label))
+        else:
+          symbols.add(load.alias)
+      elif load.symbol in symbols:
+        raise LoadExtractorError(
+            "Load symbol conflict: %s loaded from %s" %
+            (load.alias, load.label))
+      else:
+        symbols.add(load.symbol)
+
+  def extract(self, bzl_file):
+    """Extracts symbols loaded from other .bzl files.
+
+    Walks the AST of the .bzl files and extracts information about symbols
+    loaded from other .bzl files from load() calls. Then, validate the
+    extracted symbols to check that all symbols are unique.
+
+    Note that only load() calls where all arguments are string literals
+    (ast.Str) are supported.
+
+    Args:
+      bzl_file: The .bzl file to extract load symbols from.
+
+    Returns:
+      List of LoadSymbol objects.
+    """
+    load_symbols = self._extract_loads(bzl_file)
+    self._validate_loads(load_symbols)
+    return load_symbols

skydoc/load_extractor_test.py

@@ -0,0 +1,90 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import textwrap
+import unittest
+
+from skydoc import load_extractor
+
+class LoadExtractorTest(unittest.TestCase):
+
+  def check_symbols(self, src, expected):
+    with tempfile.NamedTemporaryFile() as tf:
+      tf.write(src)
+      tf.flush()
+
+      extractor = load_extractor.LoadExtractor()
+      load_symbols = extractor.extract(tf.name)
+
+      self.assertEqual(expected, load_symbols)
+
+  def test_load(self):
+    src = textwrap.dedent("""\
+      load("//foo/bar:bar.bzl", "foo_library")
+      load("//foo/bar:baz.bzl", "foo_test", orig_foo_binary = "foo_binary")
+      """)
+    expected = [
+      load_extractor.LoadSymbol('//foo/bar:bar.bzl', 'foo_library', None),
+      load_extractor.LoadSymbol('//foo/bar:baz.bzl', 'foo_test', None),
+      load_extractor.LoadSymbol('//foo/bar:baz.bzl', 'foo_binary',
+                                'orig_foo_binary'),
+    ]
+    self.check_symbols(src, expected)
+
+  def raises_error(self, src):
+    with tempfile.NamedTemporaryFile() as tf:
+      tf.write(src)
+      tf.flush()
+
+      extractor = load_extractor.LoadExtractor()
+      self.assertRaises(load_extractor.LoadExtractorError,
+                        extractor.extract, tf.name)
+
+  def test_invalid_non_string_literal_in_label(self):
+    src = textwrap.dedent("""\
+      load(load_label, "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_non_string_literal_in_keywords(self):
+    src = textwrap.dedent("""\
+        load("//foo/bar:bar.bzl", loaded_symbol)
+      """)
+    self.raises_error(src)
+
+  def test_invalid_symbol_conflict(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", "foo_binary", "foo_library")
+        load("//foo:baz.bzl", "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_symbol_alias_conflict(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", foo_library="some_foo_library")
+        load("//foo:baz.bzl", "foo_library")
+      """)
+    self.raises_error(src)
+
+  def test_invalid_duplicate_symbol_loaded(self):
+    src = textwrap.dedent("""\
+        load("//foo:bar.bzl", "foo_library", "foo_library")
+      """)
+    self.raises_error(src)
+
+
+if __name__ == '__main__':
+  unittest.main()

skydoc/main.py

@@ -26,6 +26,7 @@
 import zipfile
 
 from skydoc import common
+from skydoc import load_extractor
 from skydoc import macro_extractor
 from skydoc import rule
 from skydoc import rule_extractor
@@ -280,11 +281,21 @@ def main(argv):
     sys.exit(1)
 
   rulesets = []
+  load_sym_extractor = load_extractor.LoadExtractor()
   for bzl_file in bzl_files:
+    load_symbols = []
+    try:
+      load_symbols = load_sym_extractor.extract(bzl_file)
+    except load_extractor.LoadExtractorError as e:
+      print("ERROR: Error extracting loaded symbols from %s: %s" %
+            (bzl_file, str(e)))
+      sys.exit(2)
+
+    # TODO(dzc): Make MacroDocExtractor and RuleDocExtractor stateless.
     macro_doc_extractor = macro_extractor.MacroDocExtractor()
     rule_doc_extractor = rule_extractor.RuleDocExtractor()
     macro_doc_extractor.parse_bzl(bzl_file)
-    rule_doc_extractor.parse_bzl(bzl_file)
+    rule_doc_extractor.parse_bzl(bzl_file, load_symbols)
     merged_language = merge_languages(macro_doc_extractor.proto(),
                                       rule_doc_extractor.proto())
     rulesets.append(

skydoc/rule_extractor.py

@@ -43,15 +43,45 @@
 """Stubs for Skylark globals to be used to evaluate the .bzl file."""
 
 
+SKYLARK_GLOBAL_SYMBOLS = set(SKYLARK_STUBS.keys())
+
+
+def create_stubs(skylark_stubs, load_symbols):
+  """Combines Skylark stubs with loaded symbols.
+
+  This function creates a copy of the global Skylark stubs and combines them
+  with symbols from the list of load_extractor.LoadSymbol, which contain
+  information about symbols extracted from other .bzl files. The stubs created
+  for the loaded symbols are global variables set to the empty string.
+
+  Args:
+    skylark_stubs: Dict containing the Skylark global stubs.
+    load_symbols: List of load_extractor.LoadSymbol objects containing
+      information about symbols extracted from other .bzl files.
+
+  Returns:
+    Dictionary containing both the Skylark global stubs and stubs created for
+    the loaded symbols.
+  """
+  stubs = dict(skylark_stubs)
+  for load_symbol in load_symbols:
+    if load_symbol.alias:
+      stubs[load_symbol.alias] = ""
+    else:
+      stubs[load_symbol.symbol] = ""
+  return stubs
+
+
 class RuleDocExtractor(object):
   """Extracts documentation for rules from a .bzl file."""
 
   def __init__(self):
     """Inits RuleDocExtractor with a new BuildLanguage proto"""
     self.__language = build_pb2.BuildLanguage()
     self.__extracted_rules = {}
+    self.__load_symbols = []
 
-  def _process_skylark(self, bzl_file):
+  def _process_skylark(self, bzl_file, load_symbols):
     """Evaluates the Skylark code in the .bzl file.
 
     This function evaluates the Skylark code in the .bzl file as Python against
@@ -60,15 +90,19 @@ def _process_skylark(self, bzl_file):
 
     Args:
       bzl_file: The .bzl file to evaluate.
+      load_symbols: List of load_extractor.LoadSymbol objects containing info
+        about symbols load()ed from other .bzl files.
     """
+    compiled = compile(open(bzl_file).read(), bzl_file, 'exec')
     skylark_locals = {}
-    compiled = compile(open(bzl_file).read(), bzl_file, "exec")
-    exec(compiled) in SKYLARK_STUBS, skylark_locals
+    global_stubs = create_stubs(SKYLARK_STUBS, load_symbols)
+    exec(compiled) in global_stubs, skylark_locals
 
     for name, obj in skylark_locals.iteritems():
-      if hasattr(obj, "is_rule") and not name.startswith("_"):
-        obj.attrs["name"] = attr.AttrDescriptor(
-            type=build_pb2.Attribute.UNKNOWN, mandatory=True, name="name")
+      if (isinstance(obj, skylark_globals.RuleDescriptor) and
+          not name.startswith('_')):
+        obj.attrs['name'] = attr.AttrDescriptor(
+            type=build_pb2.Attribute.UNKNOWN, mandatory=True, name='name')
         self.__extracted_rules[name] = obj
 
   def _add_rule_doc(self, name, doc):
@@ -134,7 +168,7 @@ def _assemble_protos(self):
     """Builds the BuildLanguage protos for the extracted rule documentation.
 
     Iterates through the map of extracted rule documentation and builds a
-    BuildLanguage proto containing the documentation for publid rules extracted
+    BuildLanguage proto containing the documentation for public rules extracted
     from the .bzl file.
     """
     rules = []
@@ -173,7 +207,13 @@ def _assemble_protos(self):
         output.template = template
         output.documentation = doc
 
-  def parse_bzl(self, bzl_file):
+    for load_symbol in self.__load_symbols:
+      load = self.__language.load.add()
+      load.label = load_symbol.label
+      load.symbol = load_symbol.symbol
+      load.alias = load_symbol.alias
+
+  def parse_bzl(self, bzl_file, load_symbols):
     """Extracts the documentation for all public rules from the given .bzl file.
 
     The Skylark code is first evaluated against stubs to extract rule and
@@ -184,11 +224,10 @@ def parse_bzl(self, bzl_file):
     Args:
       bzl_file: The .bzl file to extract rule documentation from.
     """
-    self._process_skylark(bzl_file)
+    self._process_skylark(bzl_file, load_symbols)
     self._extract_docstrings(bzl_file)
     self._assemble_protos()
 
   def proto(self):
     """Returns the proto containing the macro documentation."""
     return self.__language
-