Add native.rule(NAME), which returns the attributes of a previously d…

…efined rule.

Add native.rules(), which returns all previously defined rules.

These primitives can be used to write Skylark extensions that aggregate over the contents of a BUILD file, eg.

   def instantiate_if_needed(name):
      n = name + "_wrapped"
      if not native.rule(n):
         py_test(name = n , ... )

   def archive_cc_src_files(tag):
     all_src = []
     for r in native.rules().values():
       if tag in r["tags"] and r["kind"] == "cc_library":
         all_src.append(r["srcs"])
     native.genrule(cmd = "zip $@ $^", srcs = all_src, outs = ["out.zip"])


RELNOTES: Support aggregation over existing rules in Skylark extensions
through native.rules and native.rule.

--
MOS_MIGRATED_REVID=112249050

site/docs/skylark/cookbook.md

@@ -59,6 +59,76 @@ load("/pkg/extension", "macro")
 macro(name = "myrule")
 ```
 
+## <a name="conditional-instantiation"></a>Conditional instantiation.</a>
+
+Macros can look at previously instantiated rules. This is done with
+`native.rule`, which returns information on a single rule defined in the same
+`BUILD` file, eg.,
+
+```python
+native.rule("descriptor_proto")
+```
+
+This is useful to avoid instantiating the same rule twice, which is an
+error. For example, the following rule will simulate a test suite, instantiating
+tests for diverse flavors of the same test.
+
+`extension.bzl`:
+
+```python
+def system_test(test_file, flavor):
+  n = "system_test_%s_%s_test" % (test_file, flavor)
+  if native.rule(n) == None:
+    native.py_test(
+      name = n,
+      srcs = [ "test_driver.py", test_file ],
+      args = [ "--flavor=" + flavor])
+  return n
+
+def system_test_suite(name, flavors=["default"], test_files):
+  ts = []
+  for flavor in flavors:
+    for test in test_files:
+      ts.append(system_test(name, flavor, test))
+  native.test_suite(name = name, tests = ts) 
+```
+
+In the following BUILD file, note how `(fast, basic_test.py)` is emitted for
+both the `smoke` test suite and the `thorough` test suite.
+
+```python
+load("/pkg/extension", "system_test_suite")
+
+# Run all files through the 'fast' flavor.
+system_test_suite("smoke", flavors=["fast"], glob(["*_test.py"]))
+
+# Run the basic test through all flavors.
+system_test_suite("thorough", flavors=["fast", "debug", "opt"], ["basic_test.py"])
+```
+
+
+## <a name="aggregation"></a>Aggregating over the BUILD file.</a>
+
+Macros can collect information from the BUILD file as processed so far.  We call
+this aggregation. The typical example is collecting data from all rules of a
+certain kind.  This is done by calling `native.rules`, which returns a
+dictionary representing all rules defined so far in the current BUILD file. The
+dictionary has entries of the form `name` => `rule`, with the values using the
+same format as `native.rule`.
+
+```python
+def archive_cc_src_files(tag):
+  """Create an archive of all C++ sources that have the given tag."""
+  all_src = []
+  for r in native.rules().values():
+    if tag in r["tags"] and r["kind"] == "cc_library":
+      all_src.append(r["srcs"])
+  native.genrule(cmd = "zip $@ $^", srcs = all_src, outs = ["out.zip"])
+```
+
+Since `native.rules` constructs a potentially large dictionary, you should avoid
+calling it repeatedly within BUILD file.
+
 ## <a name="empty"></a>Empty rule
 
 Minimalist example of a rule that does nothing. If you build it, the target will

src/main/java/com/google/devtools/build/lib/packages/Package.java

@@ -49,6 +49,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
 /**
  * A package, which is a container of {@link Rule}s, each of
  * which contains a dictionary of named attributes.
@@ -1045,6 +1047,11 @@ public Collection<Target> getTargets() {
       return Package.getTargets(targets);
     }
 
+    @Nullable
+    public Target getTarget(String name) {
+      return targets.get(name);
+    }
+
     /**
      * Returns an (immutable, unordered) view of all the targets belonging to
      * this package which are instances of the specified class.

src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java

@@ -66,9 +66,12 @@
 import com.google.devtools.build.lib.vfs.UnixGlob;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -828,6 +831,112 @@ public Runtime.NoneType invoke(String name, SkylarkList packages, SkylarkList in
         }
       };
 
+  @Nullable
+  static Map<String, Object> callGetRuleFunction(
+      String name, FuncallExpression ast, Environment env)
+      throws EvalException, ConversionException {
+    PackageContext context = getContext(env, ast);
+    Target target = context.pkgBuilder.getTarget(name);
+
+    return targetDict(target);
+  }
+
+  @Nullable
+  private static Map<String, Object> targetDict(Target target) {
+    if (target == null && !(target instanceof Rule)) {
+      return null;
+    }
+    Map<String, Object> values = new TreeMap<>();
+
+    Rule rule = (Rule) target;
+    AttributeContainer cont = rule.getAttributeContainer();
+    for (Attribute attr : rule.getAttributes()) {
+      if (!Character.isAlphabetic(attr.getName().charAt(0))) {
+        continue;
+      }
+
+      Object val = skylarkifyValue(cont.getAttr(attr.getName()), target.getPackage());
+      if (val == null) {
+        continue;
+      }
+      values.put(attr.getName(), val);
+    }
+
+    values.put("name", rule.getName());
+    values.put("kind", rule.getRuleClass());
+    return values;
+  }
+
+  /**
+   * Converts back to type that will work in BUILD and skylark,
+   * such as string instead of label, SkylarkList instead of List,
+   * Returns null if we don't want to export the value.
+   *
+   * <p>All of the types returned are immutable. If we want, we can change this to
+   * immutable in the future, but this is the safe choice for now.
+   */
+  private static Object skylarkifyValue(Object val, Package pkg) {
+    if (val == null) {
+      return null;
+    }
+    if (val instanceof Integer) {
+      return val;
+    }
+    if (val instanceof String) {
+      return val;
+    }
+    if (val instanceof Label) {
+      Label l = (Label) val;
+      if (l.getPackageName().equals(pkg.getName())) {
+        return ":" + l.getName();
+      }
+      return l.getCanonicalForm();
+    }
+    if (val instanceof List) {
+      List<Object> l = new ArrayList<>();
+      for (Object o : (List) val) {
+        l.add(skylarkifyValue(o, pkg));
+      }
+
+      return SkylarkList.Tuple.copyOf(l);
+    }
+    if (val instanceof Map) {
+      Map<Object, Object> m = new TreeMap<>();
+      for (Map.Entry<?, ?> e : ((Map<?, ?>) val).entrySet()) {
+        m.put(skylarkifyValue(e.getKey(), pkg), skylarkifyValue(e.getValue(), pkg));
+      }
+      return m;
+    }
+    if (val.getClass().isAnonymousClass()) {
+      // Computed defaults. They will be represented as
+      // "deprecation": com.google.devtools.build.lib.analysis.BaseRuleClasses$2@6960884a,
+      // Filter them until we invent something more clever.
+      return null;
+    }
+
+    // Add any types we want to allow through here.
+    return null;
+  }
+
+  static Map callGetRulesFunction(FuncallExpression ast, Environment env) throws EvalException {
+
+    PackageContext context = getContext(env, ast);
+    Collection<Target> targets = context.pkgBuilder.getTargets();
+
+    // Sort by name.
+    Map<String, Map<String, Object>> rules = new TreeMap<>();
+    for (Target t : targets) {
+      if (t instanceof Rule) {
+        Map<String, Object> m = targetDict(t);
+        Preconditions.checkNotNull(m);
+
+        rules.put(t.getName(), m);
+      }
+    }
+
+    return rules;
+  }
+
   static Runtime.NoneType callPackageFunction(String name, Object packagesO, Object includesO,
       FuncallExpression ast, Environment env) throws EvalException, ConversionException {
     PackageContext context = getContext(env, ast);
@@ -1082,7 +1191,7 @@ public Package createPackageForTesting(
   }
 
   /**
-   * Same as {@link #createPackage}, but does the required validation of "packageName" first,
+   * Same as createPackage, but does the required validation of "packageName" first,
    * throwing a {@link NoSuchPackageException} if the name is invalid.
    */
   @VisibleForTesting

src/main/java/com/google/devtools/build/lib/packages/SkylarkNativeModule.java

@@ -26,6 +26,8 @@
 import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor;
 import com.google.devtools.build.lib.syntax.Type.ConversionException;
 
+import java.util.Map;
+
 /**
  * A class for the Skylark native module.
  */
@@ -39,34 +41,111 @@
 public class SkylarkNativeModule {
 
   // TODO(bazel-team): shouldn't we return a SkylarkList instead?
-  @SkylarkSignature(name = "glob", objectType = SkylarkNativeModule.class,
-      returnType = SkylarkList.class,
-      doc = "Glob returns a list of every file in the current package that:<ul>\n"
-          + "<li>Matches at least one pattern in <code>include</code>.</li>\n"
-          + "<li>Does not match any of the patterns in <code>exclude</code> "
-          + "(default <code>[]</code>).</li></ul>\n"
-          + "If the <code>exclude_directories</code> argument is enabled (set to <code>1</code>), "
-          + "files of type directory will be omitted from the results (default <code>1</code>).",
-      mandatoryPositionals = {
-      @Param(name = "include", type = SkylarkList.class, generic1 = String.class,
-          defaultValue = "[]", doc = "The list of glob patterns to include.")},
-      optionalPositionals = {
-      @Param(name = "exclude", type = SkylarkList.class, generic1 = String.class,
-          defaultValue = "[]", doc = "The list of glob patterns to exclude."),
+  @SkylarkSignature(
+    name = "glob",
+    objectType = SkylarkNativeModule.class,
+    returnType = SkylarkList.class,
+    doc =
+        "Glob returns a list of every file in the current package that:<ul>\n"
+            + "<li>Matches at least one pattern in <code>include</code>.</li>\n"
+            + "<li>Does not match any of the patterns in <code>exclude</code> "
+            + "(default <code>[]</code>).</li></ul>\n"
+            + "If the <code>exclude_directories</code> argument is enabled (set to <code>1</code>), "
+            + "files of type directory will be omitted from the results (default <code>1</code>).",
+    mandatoryPositionals = {
+      @Param(
+        name = "include",
+        type = SkylarkList.class,
+        generic1 = String.class,
+        defaultValue = "[]",
+        doc = "The list of glob patterns to include."
+      )
+    },
+    optionalPositionals = {
+      @Param(
+        name = "exclude",
+        type = SkylarkList.class,
+        generic1 = String.class,
+        defaultValue = "[]",
+        doc = "The list of glob patterns to exclude."
+      ),
       // TODO(bazel-team): accept booleans as well as integers? (and eventually migrate?)
-      @Param(name = "exclude_directories", type = Integer.class, defaultValue = "1",
-          doc = "A flag whether to exclude directories or not.")},
-      useAst = true, useEnvironment = true)
-  private static final BuiltinFunction glob = new BuiltinFunction("glob") {
-      public SkylarkList invoke(
-          SkylarkList include, SkylarkList exclude,
-          Integer excludeDirectories, FuncallExpression ast, Environment env)
-          throws EvalException, ConversionException, InterruptedException {
-        env.checkLoadingPhase("native.glob", ast.getLocation());
-        return PackageFactory.callGlob(
-            null, false, include, exclude, excludeDirectories != 0, ast, env);
-    }
-  };
+      @Param(
+        name = "exclude_directories",
+        type = Integer.class,
+        defaultValue = "1",
+        doc = "A flag whether to exclude directories or not."
+      )
+    },
+    useAst = true,
+    useEnvironment = true
+  )
+  private static final BuiltinFunction glob =
+      new BuiltinFunction("glob") {
+        public SkylarkList invoke(
+            SkylarkList include,
+            SkylarkList exclude,
+            Integer excludeDirectories,
+            FuncallExpression ast,
+            Environment env)
+            throws EvalException, ConversionException, InterruptedException {
+          env.checkLoadingPhase("native.glob", ast.getLocation());
+          return PackageFactory.callGlob(
+              null, false, include, exclude, excludeDirectories != 0, ast, env);
+        }
+      };
+
+  @SkylarkSignature(
+    name = "rule",
+    objectType = SkylarkNativeModule.class,
+    returnType = Object.class,
+    doc =
+        "Returns a dictionary representing the attributes of a previously defined rule, "
+            + "or None if the rule does not exist.",
+    mandatoryPositionals = {
+      @Param(name = "name", type = String.class, doc = "The name of the rule.")
+    },
+    useAst = true,
+    useEnvironment = true
+  )
+  private static final BuiltinFunction getRule =
+      new BuiltinFunction("rule") {
+        public Object invoke(String name, FuncallExpression ast, Environment env)
+            throws EvalException, InterruptedException {
+          env.checkLoadingPhase("native.rule", ast.getLocation());
+          Map<String, Object> rule = PackageFactory.callGetRuleFunction(name, ast, env);
+          if (rule != null) {
+            return rule;
+          }
+
+          return Runtime.NONE;
+        }
+      };
+
+  /*
+    If necessary, we could allow filtering by tag (anytag, alltags), name (regexp?), kind ?
+    For now, we ignore this, since users can implement it in Skylark.
+  */
+  @SkylarkSignature(
+    name = "rules",
+    objectType = SkylarkNativeModule.class,
+    returnType = Map.class,
+    doc =
+        "Returns a dict containing all the rules instantiated so far. "
+            + "The map key is the name of the rule. The map value is equivalent to the "
+            + "get_rule output for that rule.",
+    mandatoryPositionals = {},
+    useAst = true,
+    useEnvironment = true
+  )
+  private static final BuiltinFunction getRules =
+      new BuiltinFunction("rules") {
+        public Map invoke(FuncallExpression ast, Environment env)
+            throws EvalException, InterruptedException {
+          env.checkLoadingPhase("native.rules", ast.getLocation());
+          return PackageFactory.callGetRulesFunction(ast, env);
+        }
+      };
 
   @SkylarkSignature(name = "package_group", objectType = SkylarkNativeModule.class,
       returnType = Runtime.NoneType.class,