diff --git a/site/docs/skylark/cookbook.md b/site/docs/skylark/cookbook.md index 4952d91a20c7e8..c4f718faea4a0f 100644 --- a/site/docs/skylark/cookbook.md +++ b/site/docs/skylark/cookbook.md @@ -59,6 +59,76 @@ load("/pkg/extension", "macro") macro(name = "myrule") ``` +## Conditional instantiation. + +Macros can look at previously instantiated rules. This is done with +`native.rule`, which returns information on a single rule defined in the same +`BUILD` file, eg., + +```python +native.rule("descriptor_proto") +``` + +This is useful to avoid instantiating the same rule twice, which is an +error. For example, the following rule will simulate a test suite, instantiating +tests for diverse flavors of the same test. + +`extension.bzl`: + +```python +def system_test(test_file, flavor): + n = "system_test_%s_%s_test" % (test_file, flavor) + if native.rule(n) == None: + native.py_test( + name = n, + srcs = [ "test_driver.py", test_file ], + args = [ "--flavor=" + flavor]) + return n + +def system_test_suite(name, flavors=["default"], test_files): + ts = [] + for flavor in flavors: + for test in test_files: + ts.append(system_test(name, flavor, test)) + native.test_suite(name = name, tests = ts) +``` + +In the following BUILD file, note how `(fast, basic_test.py)` is emitted for +both the `smoke` test suite and the `thorough` test suite. + +```python +load("/pkg/extension", "system_test_suite") + +# Run all files through the 'fast' flavor. +system_test_suite("smoke", flavors=["fast"], glob(["*_test.py"])) + +# Run the basic test through all flavors. +system_test_suite("thorough", flavors=["fast", "debug", "opt"], ["basic_test.py"]) +``` + + +## Aggregating over the BUILD file. + +Macros can collect information from the BUILD file as processed so far. We call +this aggregation. The typical example is collecting data from all rules of a +certain kind. This is done by calling `native.rules`, which returns a +dictionary representing all rules defined so far in the current BUILD file. The +dictionary has entries of the form `name` => `rule`, with the values using the +same format as `native.rule`. + +```python +def archive_cc_src_files(tag): + """Create an archive of all C++ sources that have the given tag.""" + all_src = [] + for r in native.rules().values(): + if tag in r["tags"] and r["kind"] == "cc_library": + all_src.append(r["srcs"]) + native.genrule(cmd = "zip $@ $^", srcs = all_src, outs = ["out.zip"]) +``` + +Since `native.rules` constructs a potentially large dictionary, you should avoid +calling it repeatedly within BUILD file. + ## Empty rule Minimalist example of a rule that does nothing. If you build it, the target will diff --git a/src/main/java/com/google/devtools/build/lib/packages/Package.java b/src/main/java/com/google/devtools/build/lib/packages/Package.java index bc7804475cb349..ce49383f206bc0 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/Package.java +++ b/src/main/java/com/google/devtools/build/lib/packages/Package.java @@ -49,6 +49,8 @@ import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; + /** * A package, which is a container of {@link Rule}s, each of * which contains a dictionary of named attributes. @@ -1045,6 +1047,11 @@ public Collection getTargets() { return Package.getTargets(targets); } + @Nullable + public Target getTarget(String name) { + return targets.get(name); + } + /** * Returns an (immutable, unordered) view of all the targets belonging to * this package which are instances of the specified class. diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java index 1de065eed8d318..f738a0d252bf73 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java @@ -66,9 +66,12 @@ import com.google.devtools.build.lib.vfs.UnixGlob; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -828,6 +831,112 @@ public Runtime.NoneType invoke(String name, SkylarkList packages, SkylarkList in } }; + @Nullable + static Map callGetRuleFunction( + String name, FuncallExpression ast, Environment env) + throws EvalException, ConversionException { + PackageContext context = getContext(env, ast); + Target target = context.pkgBuilder.getTarget(name); + + return targetDict(target); + } + + @Nullable + private static Map targetDict(Target target) { + if (target == null && !(target instanceof Rule)) { + return null; + } + Map values = new TreeMap<>(); + + Rule rule = (Rule) target; + AttributeContainer cont = rule.getAttributeContainer(); + for (Attribute attr : rule.getAttributes()) { + if (!Character.isAlphabetic(attr.getName().charAt(0))) { + continue; + } + + Object val = skylarkifyValue(cont.getAttr(attr.getName()), target.getPackage()); + if (val == null) { + continue; + } + values.put(attr.getName(), val); + } + + values.put("name", rule.getName()); + values.put("kind", rule.getRuleClass()); + return values; + } + + /** + * Converts back to type that will work in BUILD and skylark, + * such as string instead of label, SkylarkList instead of List, + * Returns null if we don't want to export the value. + * + *

All of the types returned are immutable. If we want, we can change this to + * immutable in the future, but this is the safe choice for now. + */ + private static Object skylarkifyValue(Object val, Package pkg) { + if (val == null) { + return null; + } + if (val instanceof Integer) { + return val; + } + if (val instanceof String) { + return val; + } + if (val instanceof Label) { + Label l = (Label) val; + if (l.getPackageName().equals(pkg.getName())) { + return ":" + l.getName(); + } + return l.getCanonicalForm(); + } + if (val instanceof List) { + List l = new ArrayList<>(); + for (Object o : (List) val) { + l.add(skylarkifyValue(o, pkg)); + } + + return SkylarkList.Tuple.copyOf(l); + } + if (val instanceof Map) { + Map m = new TreeMap<>(); + for (Map.Entry e : ((Map) val).entrySet()) { + m.put(skylarkifyValue(e.getKey(), pkg), skylarkifyValue(e.getValue(), pkg)); + } + return m; + } + if (val.getClass().isAnonymousClass()) { + // Computed defaults. They will be represented as + // "deprecation": com.google.devtools.build.lib.analysis.BaseRuleClasses$2@6960884a, + // Filter them until we invent something more clever. + return null; + } + + // Add any types we want to allow through here. + return null; + } + + static Map callGetRulesFunction(FuncallExpression ast, Environment env) throws EvalException { + + PackageContext context = getContext(env, ast); + Collection targets = context.pkgBuilder.getTargets(); + + // Sort by name. + Map> rules = new TreeMap<>(); + for (Target t : targets) { + if (t instanceof Rule) { + Map m = targetDict(t); + Preconditions.checkNotNull(m); + + rules.put(t.getName(), m); + } + } + + return rules; + } + static Runtime.NoneType callPackageFunction(String name, Object packagesO, Object includesO, FuncallExpression ast, Environment env) throws EvalException, ConversionException { PackageContext context = getContext(env, ast); @@ -1082,7 +1191,7 @@ public Package createPackageForTesting( } /** - * Same as {@link #createPackage}, but does the required validation of "packageName" first, + * Same as createPackage, but does the required validation of "packageName" first, * throwing a {@link NoSuchPackageException} if the name is invalid. */ @VisibleForTesting diff --git a/src/main/java/com/google/devtools/build/lib/packages/SkylarkNativeModule.java b/src/main/java/com/google/devtools/build/lib/packages/SkylarkNativeModule.java index be648b022e9676..ad73b169c9ba02 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/SkylarkNativeModule.java +++ b/src/main/java/com/google/devtools/build/lib/packages/SkylarkNativeModule.java @@ -26,6 +26,8 @@ import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor; import com.google.devtools.build.lib.syntax.Type.ConversionException; +import java.util.Map; + /** * A class for the Skylark native module. */ @@ -39,34 +41,111 @@ public class SkylarkNativeModule { // TODO(bazel-team): shouldn't we return a SkylarkList instead? - @SkylarkSignature(name = "glob", objectType = SkylarkNativeModule.class, - returnType = SkylarkList.class, - doc = "Glob returns a list of every file in the current package that:
    \n" - + "
  • Matches at least one pattern in include.
  • \n" - + "
  • Does not match any of the patterns in exclude " - + "(default []).
\n" - + "If the exclude_directories argument is enabled (set to 1), " - + "files of type directory will be omitted from the results (default 1).", - mandatoryPositionals = { - @Param(name = "include", type = SkylarkList.class, generic1 = String.class, - defaultValue = "[]", doc = "The list of glob patterns to include.")}, - optionalPositionals = { - @Param(name = "exclude", type = SkylarkList.class, generic1 = String.class, - defaultValue = "[]", doc = "The list of glob patterns to exclude."), + @SkylarkSignature( + name = "glob", + objectType = SkylarkNativeModule.class, + returnType = SkylarkList.class, + doc = + "Glob returns a list of every file in the current package that:
    \n" + + "
  • Matches at least one pattern in include.
  • \n" + + "
  • Does not match any of the patterns in exclude " + + "(default []).
\n" + + "If the exclude_directories argument is enabled (set to 1), " + + "files of type directory will be omitted from the results (default 1).", + mandatoryPositionals = { + @Param( + name = "include", + type = SkylarkList.class, + generic1 = String.class, + defaultValue = "[]", + doc = "The list of glob patterns to include." + ) + }, + optionalPositionals = { + @Param( + name = "exclude", + type = SkylarkList.class, + generic1 = String.class, + defaultValue = "[]", + doc = "The list of glob patterns to exclude." + ), // TODO(bazel-team): accept booleans as well as integers? (and eventually migrate?) - @Param(name = "exclude_directories", type = Integer.class, defaultValue = "1", - doc = "A flag whether to exclude directories or not.")}, - useAst = true, useEnvironment = true) - private static final BuiltinFunction glob = new BuiltinFunction("glob") { - public SkylarkList invoke( - SkylarkList include, SkylarkList exclude, - Integer excludeDirectories, FuncallExpression ast, Environment env) - throws EvalException, ConversionException, InterruptedException { - env.checkLoadingPhase("native.glob", ast.getLocation()); - return PackageFactory.callGlob( - null, false, include, exclude, excludeDirectories != 0, ast, env); - } - }; + @Param( + name = "exclude_directories", + type = Integer.class, + defaultValue = "1", + doc = "A flag whether to exclude directories or not." + ) + }, + useAst = true, + useEnvironment = true + ) + private static final BuiltinFunction glob = + new BuiltinFunction("glob") { + public SkylarkList invoke( + SkylarkList include, + SkylarkList exclude, + Integer excludeDirectories, + FuncallExpression ast, + Environment env) + throws EvalException, ConversionException, InterruptedException { + env.checkLoadingPhase("native.glob", ast.getLocation()); + return PackageFactory.callGlob( + null, false, include, exclude, excludeDirectories != 0, ast, env); + } + }; + + @SkylarkSignature( + name = "rule", + objectType = SkylarkNativeModule.class, + returnType = Object.class, + doc = + "Returns a dictionary representing the attributes of a previously defined rule, " + + "or None if the rule does not exist.", + mandatoryPositionals = { + @Param(name = "name", type = String.class, doc = "The name of the rule.") + }, + useAst = true, + useEnvironment = true + ) + private static final BuiltinFunction getRule = + new BuiltinFunction("rule") { + public Object invoke(String name, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + env.checkLoadingPhase("native.rule", ast.getLocation()); + Map rule = PackageFactory.callGetRuleFunction(name, ast, env); + if (rule != null) { + return rule; + } + + return Runtime.NONE; + } + }; + + /* + If necessary, we could allow filtering by tag (anytag, alltags), name (regexp?), kind ? + For now, we ignore this, since users can implement it in Skylark. + */ + @SkylarkSignature( + name = "rules", + objectType = SkylarkNativeModule.class, + returnType = Map.class, + doc = + "Returns a dict containing all the rules instantiated so far. " + + "The map key is the name of the rule. The map value is equivalent to the " + + "get_rule output for that rule.", + mandatoryPositionals = {}, + useAst = true, + useEnvironment = true + ) + private static final BuiltinFunction getRules = + new BuiltinFunction("rules") { + public Map invoke(FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + env.checkLoadingPhase("native.rules", ast.getLocation()); + return PackageFactory.callGetRulesFunction(ast, env); + } + }; @SkylarkSignature(name = "package_group", objectType = SkylarkNativeModule.class, returnType = Runtime.NoneType.class, diff --git a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleContextTest.java b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleContextTest.java index a2af3e771ac0de..4ed30baf18cce2 100644 --- a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleContextTest.java +++ b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleContextTest.java @@ -353,6 +353,75 @@ public void testGetRuleAttributeListType() throws Exception { assertThat(result).isInstanceOf(SkylarkList.class); } + @Test + public void testGetRule() throws Exception { + scratch.file("test/skylark/BUILD"); + scratch.file( + "test/skylark/rulestr.bzl", + "def rule_dict(name):", + " return native.rule(name)", + "def rules_dict():", + " return native.rules()", + "def nop(ctx):", + " pass", + "nop_rule = rule(attrs = {'x': attr.label()}, implementation = nop)", + "consume_rule = rule(attrs = {'s': attr.string_list()}, implementation = nop)"); + + scratch.file( + "test/getrule/BUILD", + "load('/test/skylark/rulestr', 'rules_dict', 'rule_dict', 'nop_rule', 'consume_rule')", + "genrule(name = 'a', outs = ['a.txt'], tools = [ '//test:bla' ], cmd = 'touch $@')", + "nop_rule(name = 'c', x = ':a')", + "rlist= rules_dict()", + "consume_rule(name = 'all_str', s = [rlist['a']['kind'], rlist['a']['name'], ", + " rlist['c']['kind'], rlist['c']['name']])", + "adict = rule_dict('a')", + "cdict = rule_dict('c')", + "consume_rule(name = 'a_str', ", + " s = [adict['kind'], adict['name'], adict['outs'][0], adict['tools'][0]])", + "consume_rule(name = 'genrule_attr', ", + " s = adict.keys())", + "consume_rule(name = 'c_str', s = [cdict['kind'], cdict['name'], cdict['x']])"); + + SkylarkRuleContext allContext = createRuleContext("//test/getrule:all_str"); + Object result = evalRuleContextCode(allContext, "ruleContext.attr.s"); + assertEquals( + new SkylarkList.MutableList(ImmutableList.of("genrule", "a", "nop_rule", "c")), + result); + + result = evalRuleContextCode(createRuleContext("//test/getrule:a_str"), "ruleContext.attr.s"); + assertEquals( + new SkylarkList.MutableList( + ImmutableList.of("genrule", "a", ":a.txt", "//test:bla")), + result); + + result = evalRuleContextCode(createRuleContext("//test/getrule:c_str"), "ruleContext.attr.s"); + assertEquals( + new SkylarkList.MutableList(ImmutableList.of("nop_rule", "c", ":a")), result); + + result = + evalRuleContextCode(createRuleContext("//test/getrule:genrule_attr"), "ruleContext.attr.s"); + assertEquals( + new SkylarkList.MutableList( + ImmutableList.of( + "cmd", + "compatible_with", + "features", + "generator_function", + "generator_location", + "generator_name", + "kind", + "message", + "name", + "outs", + "restricted_to", + "srcs", + "tags", + "tools", + "visibility")), + result); + } + @Test public void testGetRuleAttributeListValue() throws Exception { SkylarkRuleContext ruleContext = createRuleContext("//foo:foo");