Adds Skylark Remote Repository documentation

Fixes bazelbuild#1043

--
MOS_MIGRATED_REVID=118039426

site/docs/skylark/repository_rules.md

@@ -0,0 +1,78 @@
+---
+layout: documentation
+title: Skylark Repository Rules
+---
+# Repository Rules
+
+**Status: Experimental**. We may make breaking changes to the API, but we will
+  announce them and help you update your code.
+
+An [external repository](/docs/external.md) is a rule that can be used only
+in the `WORKSPACE` file and enable non-hermetic operation at the loading phase
+of Bazel. Each external repository rule creates its own workspace, with its
+own BUILD files and artifacts. They can be used to depend on third-party
+libraries (such as Maven packaged libraries) but also to generate BUILD files
+specific to the host Bazel is running on.
+
+## Repository Rule creation
+
+In a Skylark extension, use the
+[repository_rule](lib/globals.html#repository_rule) function to create a new
+repository rule and store it in a global variable.
+
+A custom repository rule can be used just like a native repository rule. It
+has a mandatory `name` attribute and every target present in its build files
+can be refered as `@<name>//package:target` where `<name>` is the value of the
+`name` attribute.
+
+The rule is loaded when you explictly build it, or if it is a dependency of
+the build. In this case, Bazel will execute its `implementation` function. This
+function describe how to creates the repository, its content and BUILD files.
+
+## Attributes
+
+An attribute is a rule argument, such as `url` or `sha256`. You must list
+the attributes and their types when you define a repository rule.
+
+```python
+local_repository = repository_rule(
+    implementation=_impl,
+    local=True,
+    attrs={"path": attr.string(mandatory=True)})
+```
+
+`name` attributes are implicitely defined for all `repository_rule`s.
+To access an attribute, use `repository_ctx.attr.<attribute_name>`.
+The name of a repository rule is accessible with `repository_ctx.name`.
+
+If an attribute name starts with `_` it is private and users cannot set it.
+
+## Implementation function
+
+Every repository rule requires an `implementation` function. It contains the
+actual logic of the rule and is executed strictly in the Loading Phase.
+The function has exactly one input parameter, `repository_ctx`, and should
+always returns `None`. The input parameter `repository_ctx` can be used to
+access attribute values, and non-hermetic functions (finding a binary,
+exuting a binary, creating a file in the repository or downloading a file
+from the Internet). See [the library](lib/repository_ctx.html) for more
+context. Example:
+
+```python
+def _impl(repository_ctx):
+  repository_ctx.symlink(repository_ctx.attr.path, "")
+
+local_repository = repository_rule(
+    implementation=_impl,
+    ...)
+```
+
+## Examples
+
+For now, we only have one full example of usage of the `repository_rule`:
+[C++ auto-configured toolchain](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L288).
+
+This example uses a Skylark repository rule to automatically create the
+C++ configuration files for Bazel by looking for the local C++ compiler, the
+environment and the flags the C++ compiler supports.
+

src/main/java/com/google/devtools/build/docgen/BUILD

@@ -39,6 +39,7 @@ java_binary(
     name = "skydoc_bin",
     srcs = ["SkylarkDocumentationGenerator.java"],
     main_class = "com.google.devtools.build.docgen.SkylarkDocumentationGenerator",
+    runtime_deps = ["//src/main/java/com/google/devtools/build/lib:bazel-repository"],
     deps = [":docgen_javalib"],
 )
 

src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationCollector.java

@@ -69,10 +69,10 @@ public static SkylarkModule getTopLevelModule() {
    * Collects the documentation for all Skylark modules and returns a map that maps Skylark
    * module name to the module documentation.
    */
-  public static Map<String, SkylarkModuleDoc> collectModules() {
+  public static Map<String, SkylarkModuleDoc> collectModules(String... clazz) {
     Map<String, SkylarkModuleDoc> modules = new TreeMap<>();
-    Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules();
-    Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects();
+    Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules(clazz);
+    Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects(clazz);
 
     modules.putAll(builtinModules);
     for (SkylarkModuleDoc builtinObject : builtinModules.values()) {
@@ -137,13 +137,21 @@ static void collectJavaObjects(SkylarkModule firstModule, Class<?> firstClass,
     }
   }
 
-  private static Map<String, SkylarkModuleDoc> collectBuiltinModules() {
+  private static Map<String, SkylarkModuleDoc> collectBuiltinModules(String... clazz) {
     Map<String, SkylarkModuleDoc> modules = new HashMap<>();
     collectBuiltinDoc(modules, Runtime.class.getDeclaredFields());
     collectBuiltinDoc(modules, MethodLibrary.class.getDeclaredFields());
     for (Class<?> moduleClass : SkylarkModules.MODULES) {
       collectBuiltinDoc(modules, moduleClass.getDeclaredFields());
     }
+    for (String c : clazz) {
+      try {
+        collectBuiltinDoc(modules,
+            SkylarkDocumentationCollector.class.getClassLoader().loadClass(c).getDeclaredFields());
+      } catch (ClassNotFoundException e) {
+        System.err.println("SkylarkModule class " + c + " could not be found, ignoring...");
+      }
+    }
     return modules;
   }
 
@@ -171,7 +179,7 @@ private static void collectBuiltinDoc(Map<String, SkylarkModuleDoc> modules, Fie
     }
   }
 
-  private static Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() {
+  private static Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects(String ...clazz) {
     Map<SkylarkModule, Class<?>> modules = new HashMap<>();
     collectBuiltinModule(modules, SkylarkRuleContext.class);
     collectBuiltinModule(modules, TransitiveInfoCollection.class);
@@ -183,6 +191,14 @@ private static Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() {
     collectBuiltinModule(modules, JavaSkylarkApiProvider.class);
     collectBuiltinModule(modules, JavaRuleOutputJarsProvider.OutputJar.class);
     collectBuiltinModule(modules, AndroidSkylarkApiProvider.class);
+    for (String c : clazz) {
+      try {
+        collectBuiltinModule(modules,
+            SkylarkDocumentationCollector.class.getClassLoader().loadClass(c));
+      } catch (ClassNotFoundException e) {
+        System.err.println("SkylarkModule class " + c + " could not be found, ignoring...");
+      }
+    }
     return modules;
   }
 

src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java

@@ -14,14 +14,16 @@
 package com.google.devtools.build.docgen;
 
 
+import java.util.Arrays;
+
 /**
  * The main class for the skylark documentation generator.
  */
 public class SkylarkDocumentationGenerator {
 
   private static boolean checkArgs(String[] args) {
     if (args.length < 1) {
-      System.err.println("There has to be one input parameter\n"
+      System.err.println("There has to be at least one input parameter\n"
           + " - an output file.");
       return false;
     }
@@ -40,7 +42,9 @@ public static void main(String[] args) {
     if (checkArgs(args)) {
       System.out.println("Generating Skylark documentation...");
       try {
-        SkylarkDocumentationProcessor.generateDocumentation(args[0]);
+        SkylarkDocumentationProcessor.generateDocumentation(
+            args[0],
+            Arrays.copyOfRange(args, 1, args.length));
       } catch (Throwable e) {
         fail(e, true);
       }

src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java

@@ -32,9 +32,9 @@ private SkylarkDocumentationProcessor() {}
   /**
    * Generates the Skylark documentation to the given output directory.
    */
-  public static void generateDocumentation(String outputDir) throws IOException,
+  public static void generateDocumentation(String outputDir, String... clazz) throws IOException,
       BuildEncyclopediaDocException {
-    Map<String, SkylarkModuleDoc> modules = SkylarkDocumentationCollector.collectModules();
+    Map<String, SkylarkModuleDoc> modules = SkylarkDocumentationCollector.collectModules(clazz);
     List<SkylarkModuleDoc> navModules = new ArrayList<>();
 
     // Generate the top level module first in the doc

src/main/java/com/google/devtools/build/lib/BUILD

@@ -958,12 +958,22 @@ genrule(
     ],
 )
 
+# The skylark repository classes are passed as parameter of the Skylark documentation generator.
+SKYLARK_REPOSITORY_CLASSES = [
+    "com.google.devtools.build.lib.bazel.repository.skylark.SkylarkExecutionResult",
+    "com.google.devtools.build.lib.bazel.repository.skylark.SkylarkOS",
+    "com.google.devtools.build.lib.bazel.repository.skylark.SkylarkPath",
+    "com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryContext",
+    "com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryModule",
+]
+
 genrule(
     name = "gen_skylarklibrary",
     outs = ["skylark-library.zip"],
     cmd = "mkdir -p $(@D)/skylark-lib &&" +
-          "$(location //src/main/java/com/google/devtools/build/docgen:skydoc_bin) $(@D)/skylark-lib &&" +
-          "zip -qj $@ $(@D)/skylark-lib/*",
+          "$(location //src/main/java/com/google/devtools/build/docgen:skydoc_bin) $(@D)/skylark-lib " +
+          " ".join(SKYLARK_REPOSITORY_CLASSES) +
+          " && zip -qj $@ $(@D)/skylark-lib/*",
     tools = ["//src/main/java/com/google/devtools/build/docgen:skydoc_bin"],
 )
 

src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryModule.java

@@ -60,8 +60,8 @@ public class SkylarkRepositoryModule {
         type = BaseFunction.class,
         doc =
             "the function implementing this rule, has to have exactly one parameter: "
-                + "<code>repository_ctx</code>. The function is called during analysis phase for "
-                + "each instance of the rule."
+                + "<code><a href=\"repository_ctx.html\">repository_ctx</a></code>. The function "
+                + "is called during loading phase for each instance of the rule."
       )
     },
     optionalNamedOnly = {
@@ -72,7 +72,7 @@ public class SkylarkRepositoryModule {
         defaultValue = "None",
         doc =
             "dictionary to declare all the attributes of the rule. It maps from an attribute "
-                + "name to an attribute object (see <a href=\"#modules.attr\">attr</a> "
+                + "name to an attribute object (see <a href=\"attr.html\">attr</a> "
                 + "module). Attributes starting with <code>_</code> are private, and can be "
                 + "used to add an implicit dependency on a label to a file (a repository "
                 + "rule cannot depend on a generated artifact). The attribute "

tools/cpp/cc_configure.bzl

@@ -285,7 +285,7 @@ def _impl(repository_ctx):
     })
 
 
-cc_autoconf = repository_rule(_impl, local=True)
+cc_autoconf = repository_rule(implementation=_impl, local=True)
 
 
 def cc_configure():