From b781ac26f8bbbe26cdf0ce4845a56f7ea73a2a62 Mon Sep 17 00:00:00 2001 From: "larry hosken (Twitter)" Date: Tue, 28 Oct 2014 16:04:19 -0700 Subject: [PATCH] Revert "convert more .rst files to .md" This reverts commit b7967f52c1b70aa0c7c297633438fafbfd23164a. This commit broke CI :-( Submitting this reversion TBR Reviewed at https://rbcommons.com/s/twitter/r/1235/ --- build-support/bin/publish_docs_new.sh | 3 +- .../java/com/pants/examples/3rdparty_jvm.md | 223 --------------- examples/src/java/com/pants/examples/BUILD | 27 +- .../src/java/com/pants/examples/from_maven.md | 79 ------ examples/src/java/com/pants/examples/page.md | 177 ------------ .../src/java/com/pants/examples/readme.md | 19 +- examples/src/python/example/3rdparty_py.md | 69 ----- examples/src/python/example/BUILD | 18 +- examples/src/python/example/pex_design.md | 154 ----------- examples/src/python/example/python_old.md | 14 +- examples/src/python/example/readme.md | 13 +- examples/src/thrift/com/pants/examples/BUILD | 6 - .../src/thrift/com/pants/examples/readme.md | 197 ------------- src/docs/3rdparty.md | 34 --- src/docs/BUILD | 121 +------- src/docs/announce_201409.md | 98 ------- src/docs/build_files.md | 7 +- src/docs/first_concepts.md | 148 ---------- src/docs/first_tutorial.md | 260 ------------------ src/docs/index.md | 70 ----- src/docs/install.md | 61 ---- src/docs/invoking.md | 69 ----- src/docs/publish.md | 26 +- src/docs/setup_repo.md | 254 ----------------- src/docs/target_addresses.md | 125 --------- src/docs/tshoot.md | 74 ----- src/docs/with_emacs.md | 20 -- src/docs/with_intellij.md | 23 -- src/python/pants/docs/BUILD | 103 ------- src/python/pants/docs/credits.md | 22 -- src/python/pants/docs/dev_tasks.md | 168 ----------- .../pants/docs/dev_tasks_publish_extras.md | 86 ------ src/python/pants/docs/docs.md | 86 ------ src/python/pants/docs/docsite.css | 15 +- src/python/pants/docs/docsite.json | 88 +----- src/python/pants/docs/howto_ask.md | 34 --- src/python/pants/docs/howto_contribute.md | 220 --------------- src/python/pants/docs/howto_customize.md | 19 -- src/python/pants/docs/howto_develop.md | 241 ---------------- src/python/pants/docs/howto_plugin.md | 23 -- src/python/pants/docs/intellij.md | 112 -------- src/python/pants/docs/internals.md | 157 ----------- src/python/pants/docs/readme.md | 20 -- src/python/pants/docs/release.md | 96 ------- 44 files changed, 77 insertions(+), 3802 deletions(-) delete mode 100644 examples/src/java/com/pants/examples/3rdparty_jvm.md delete mode 100644 examples/src/java/com/pants/examples/from_maven.md delete mode 100644 examples/src/java/com/pants/examples/page.md delete mode 100644 examples/src/python/example/3rdparty_py.md delete mode 100644 examples/src/python/example/pex_design.md delete mode 100644 examples/src/thrift/com/pants/examples/BUILD delete mode 100644 examples/src/thrift/com/pants/examples/readme.md delete mode 100644 src/docs/3rdparty.md delete mode 100644 src/docs/announce_201409.md delete mode 100644 src/docs/first_concepts.md delete mode 100644 src/docs/first_tutorial.md delete mode 100644 src/docs/index.md delete mode 100644 src/docs/install.md delete mode 100644 src/docs/invoking.md delete mode 100644 src/docs/setup_repo.md delete mode 100644 src/docs/target_addresses.md delete mode 100644 src/docs/tshoot.md delete mode 100644 src/docs/with_emacs.md delete mode 100644 src/docs/with_intellij.md delete mode 100644 src/python/pants/docs/BUILD delete mode 100644 src/python/pants/docs/credits.md delete mode 100644 src/python/pants/docs/dev_tasks.md delete mode 100644 src/python/pants/docs/dev_tasks_publish_extras.md delete mode 100644 src/python/pants/docs/docs.md delete mode 100644 src/python/pants/docs/howto_ask.md delete mode 100644 src/python/pants/docs/howto_contribute.md delete mode 100644 src/python/pants/docs/howto_customize.md delete mode 100644 src/python/pants/docs/howto_develop.md delete mode 100644 src/python/pants/docs/howto_plugin.md delete mode 100644 src/python/pants/docs/intellij.md delete mode 100644 src/python/pants/docs/internals.md delete mode 100644 src/python/pants/docs/readme.md delete mode 100644 src/python/pants/docs/release.md diff --git a/build-support/bin/publish_docs_new.sh b/build-support/bin/publish_docs_new.sh index 2567e364188..dfd540a1c85 100755 --- a/build-support/bin/publish_docs_new.sh +++ b/build-support/bin/publish_docs_new.sh @@ -56,7 +56,8 @@ function do_open() { # generate some markdown as fodder for prototype doc site generator ${PANTS_EXE} goal markdown --print-exception-stacktrace \ - --markdown-fragment examples:: src:: || \ + --markdown-fragment src:: examples:: src/docs:: //:readme \ + testprojects/src/java/com/pants/testproject/page:readme || \ die "Failed to generate HTML from markdown'." # invoke doc site generator. diff --git a/examples/src/java/com/pants/examples/3rdparty_jvm.md b/examples/src/java/com/pants/examples/3rdparty_jvm.md deleted file mode 100644 index 2e88a670774..00000000000 --- a/examples/src/java/com/pants/examples/3rdparty_jvm.md +++ /dev/null @@ -1,223 +0,0 @@ -JVM 3rdparty Pattern -==================== - -In general, we use the -[[3rdparty idiom|pants('src/docs:3rdparty')]] to organize -dependencies on code from outside the source tree. This document -describes how to make this work for JVM (Java or Scala) code. - -Your JVM code can pull in code written elsewhere. Pants uses -[Ivy](http://ant.apache.org/ivy/), a tool based on Maven's jar-sharing. -You should know the ([Maven/Ivy groupId, artifactId, and -version](http://maven.apache.org/guides/mini/guide-central-repository-upload.html)) -you want to use. - -The 3rdparty pattern described here eases avoiding diamond dependency -problems and version conflicts. If your code depends on artifacts `foo` -and `bar`; and if `foo` and `bar` depend on different versions of the -`baz` artifact; then some code will be linked together with a version of -`baz` it didn't "expect." Tracking versioned dependencies in one place -makes it easier to reason about them. - -3rdparty/jvm ------------- - -**The JVM part of 3rdparty is organized by org (Maven groupId)** Under -there, see if there's already a `3rdparty/jvm/path/to/org/BUILD` file. -If there isn't, then you want to create one. E.g., to import -`com.sun.jersey-apache-client`, look in `3rdparty/jvm/com/sun` for a -likely-looking `BUILD` file--in this example, -`3rdparty/jvm/com/google/sun/jersey/BUILD`. - -In the appropriate `BUILD` file, you want to find a -`jar_library` -with the `jar`s you want: - -!inc[start-at=junit&end-before=specs](../../../../../../3rdparty/BUILD) - -Here, the -`jar_library`'s name -defines a target address that -other build targets can refer to. The -`jar`s refer to jars known to -your Ivy resolver. - -If there's already a `jar` importing the code you want but with a -*different* version, then you probably want to talk to other folks in -your organization to agree on one version. (If there's already a `jar` -importing the code you want with the version you want, then great. Leave -it there.) - -(You don't *need* a tree of `BUILD` files; you could instead have, e.g., -one `3rdparty/jvm/BUILD` file. In a large organization, a tree can ease -some things. For example, `git log` quickly answers questions like "Who -set up this dependency? Who cares if I bump the version?") - -Additionally, some families of jars have different groupId's but are -logically part of the same project, or need to have their rev's kept in -sync. For example, (`com.fasterxml.jackson.core`, -`com.fasterxml.jackson.dataformat`). Sometimes it makes sense to define -these in a single build file, such as -`3rdparty/jvm/com/fasterxml/jackson/BUILD` for the jackson family of -jars. - -Your Code's BUILD File ----------------------- - -To set up your code to import the external jar, you add a dependency to -the appropriate Java target[s] in your `BUILD` file and add `import` -statements in your Java code. - -For example, your `BUILD` file might have - -!inc[start-after=sources&end-before=src/java](../../../../../tests/java/com/pants/examples/hello/greet/BUILD) - -And your Java code might have: - - :::java - import org.junit.Test; - -"Round Trip" Dependencies -------------------------- - -Depending on your workspace's relation with the rest of the world, you -might want to look out for "round trip" dependencies. You can publish an -artifact *near* generated from your workspace's source code and consume -a third-party artifact *far* that depends on *near*. If you're not -careful, you might depend on two versions of the *near* code: the local -source code and an artifact you published a while ago. When consuming -such third-party artifacts, exclude dependencies that "collide" with -source code and depend on local source: - - :::python - jar_library(name='far', - jars=[ - # exclude conflicting dep: - jar(org='org.archie', name='far', rev='0.0.18').with_sources() - .exclude(org='org.archimedes', name='near') - ] - dependencies=[ - # and re-include local version of source manually: - 'util/near', - ] - ) - -Troubleshooting a JVM Dependencies Problem ------------------------------------------- - -If you're working in JVM (Java or Scala) and suspect you're pulling in -different versions of some package, you can dump your dependency "tree" -with versions with an Ivy resolve report. To generate a report for a -target such as the `hello/main` example: - - :::bash - $ ./pants goal resolve examples/src/java/com/pants/examples/hello/main --ivy-open - -Ivy's report shows which things depend on which versions. You can see -which package is pulling in the package-version you didn't expect. (It -might not be clear which version you want to use; but at least you'll -know what's causing the problem.) - -**If you notice a small number of wrong-version things,** then in a JVM -target, you can depend on a `jar` that specifies a version and sets -`force=True` to *force* using that version: - - :::python - scala_library( - name = "loadtest", - dependencies = [ - '3rdparty/bijection:bijection-scrooge', - # our 3rdparty/BUILD still has 6.1.4 as the default version, but - # finagle-[core|thrift] version 6.1.4 is superceded (evicted) by - # version 6.4.1 - # Force inclusion of version 6.1.4, until we're bumped to finagle 6.4.1+ - jar(org='com.twitter', name='iago', rev='0.6.3', force=True), - jar(org='com.twitter', name='finagle-core', rev='6.1.4', force=True), - jar(org='com.twitter', name='finagle-thrift', rev='6.1.4', force=True), - ], - sources = ["LoadTestRecordProcessor.scala"]) - -**If you notice that one "foreign" dependency pulls in mostly wrong -things,** tell Pants not to pull in its dependencies. In your -`3rdparty/.../BUILD` file, call the `jar`'s `intransitive` method; then -carefully add hand-picked versions: - - :::python - jar_library(name="retro-naming-factory", - jars=[ - jar(org='retro', name='retro-factory', rev='5.0.18').intransitive(), - ], - dependencies=[ - # Don't use retro's expected (old, incompatible) common-logging - # version, yipe; use the same version we use everywhere else: - '3rdparty/jvm/common-logging', - ]) - -**If you notice a small number of transitive dependencies to exclude** -Rather than mark the `jar` intransitive, you can `exclude` some -transitive dependencies from JVM targets: - - :::python - java_library(name = 'loadtest', - dependencies = [ - '3rdparty/storm:storm', - ], - sources = globs('*.java'), - excludes = [ - exclude('org.sonatype.sisu.inject', 'cglib') - ] - ) - -**If you notice a missing dependency**, check for a naming conflict. -When bringing in multiple jars with the same org, name, and version, -only the first reference will win, and subsequent references will be -silently discarded. One way that this can occur is with dependencies -that use a classifier to differentiate themselves. Consider this -example: - - :::python - jar_library(name = 'stanford-corenlp', - jars = [ - jar(org = 'edu.stanford.nlp', name = 'stanford-corenlp', rev = '3.3.1').with_sources(), - jar(org = 'edu.stanford.nlp', name = 'stanford-corenlp', rev = '3.3.1', classifier='models') - ] - ) - -In the above example, the -`edu.stanford.nlp.stanford-corenlp-3.3.1-models.jar` will be silently -skipped by pants. To bring both jars in, use the `.with_artifacts()` -method of the bdict\_jar. Using this method, the above example would be -transformed into: - - :::python - jar_library(name = 'stanford-corenlp', - jars = [ - jar(org = 'edu.stanford.nlp', name = 'stanford-corenlp', rev = '3.3.1') - .with_sources().with_artifact(classifier='models').with_artifact(classifier=''), - ] - ) - -And as a result, both jars will now be brought into the target's -classpath. - - - -Using a SNAPSHOT JVM Dependency -------------------------------- - -Sometimes your code depends on a buggy external JVM dependency. You -think you've fixed the external code, but want to test locally before -uploading it to make sure. To do this, in the `jar` dependency for the -artifact, specify the `url` attribute to point to the local file and -change the `rev`. If you are actively making changes to the dependency, -you can also use the `mutable` jar attribute to re-import the file each -time pants is run (otherwise, pants will cache it): - - :::python - jar_library(name='checkstyle', - jars = [ - jar(org='com.puppycrawl.tools', name='checkstyle', rev='5.5-SNAPSHOT', - url='file:///Users/pantsdev/Src/checkstyle/checkstyle.jar', - mutable=True), - ], - ) diff --git a/examples/src/java/com/pants/examples/BUILD b/examples/src/java/com/pants/examples/BUILD index 777fb578e46..164ac5658d7 100644 --- a/examples/src/java/com/pants/examples/BUILD +++ b/examples/src/java/com/pants/examples/BUILD @@ -2,31 +2,6 @@ page( name='readme', source='readme.md', dependencies=[ - ':3rdparty_jvm', - ':from_maven', 'src/docs:publish', - 'src/docs:first_concepts', - 'src/docs:first_tutorial', ] -) - -page( - name='from_maven', - source='from_maven.md', - dependencies=[ - ':3rdparty_jvm', - ], -) - -page( - name='3rdparty_jvm', - source='3rdparty_jvm.md', - dependencies=[ - 'src/docs:3rdparty', - ], -) - -page( - name='page', - source='page.md', -) +) \ No newline at end of file diff --git a/examples/src/java/com/pants/examples/from_maven.md b/examples/src/java/com/pants/examples/from_maven.md deleted file mode 100644 index b09a2c37680..00000000000 --- a/examples/src/java/com/pants/examples/from_maven.md +++ /dev/null @@ -1,79 +0,0 @@ -Pants for Maven Experts -======================= - -If you're used to Maven and learning Pants, you're part of a growing -crowd. Here are some things that helped other folks come up to speed. - -The good news is that Pants and Maven are pretty similar. Both tools use -several configuration-snippet files near in source code directories to -specify how to build those directories' source code. Both tools use the -configuration-snippet files to build up a model of your source code, -then execute tasks in a lifecycle over that model. Pants targets tend to -be finer-grained than Maven's projects; but if you use subprojects in -Maven, Pants targets might feel familiar. Both expect code to be laid -out in directories in a consistent way. If you're used to Maven's -commands, many of Pants' goals will feel eerily familiar. - -Pants uses Ivy to manage artifact fetching and publishing; Ivy's -behavior here is pretty similar to Maven. - -Three Pants features that especially confuse Maven experts as they move -to pants are - -- Pants has a first-class mechanism for targets depending on other - targets on the local file system -- Pants targets do not specify version numbers; versions are only - determined during release -- BUILD files are python code that pants evaluates dynamically. - -The first two points are a significant departure from Maven's handling -of inter-project dependencies. The last point isn't necessary for -understanding how to read and write most BUILD files, but is helpful to -be aware of. - -Folks switching a Maven-built codebase to Pants often encounter another -source of confusion: they uncover lurking jar-dependency version -conflicts. JVM projects can inadvertently end up relying on classpath -order for correctness; any two build tools will order their classpaths -differently. If your project depends on two versions of the same jar -(all too easy to do with transitive dependencies), then your Maven build -chose one version, but Pants might end up choosing another: Pants is -likely to generate a differently-ordered `CLASSPATH` than Maven did. You -can fix these, making your build configuration more robust along the -way; see -[[JVM 3rdparty Pattern|pants('examples/src/java/com/pants/examples:3rdparty_jvm')]] -for advice. - -Pants Equivalents ------------------ - -`exec:java` run a binary
-`goal run` - -`-Xdebug` run a binary in the debugger
-`goal run --jvm-run-debug` - -`-Dtest=com.foo.BarSpec -Dmaven.surefire.debug=true test` run one test in the debugger
-`goal test --test-junit-debug --test-junit-test=com.foo.BarSpec` or -`goal test --test-specs-debug --test-specs-test=com.foo.BarSpec` - -Depending on Source, not Jars ------------------------------ - -Pants arose in an environment of a big multi-project repo. Several teams -contributed code to the same source tree; projects depended on each -other. Getting those dependencies to work with Maven was tricky. As the -number of engineers grew, it wasn't so easy to have one team ask another -team to release a new jar. Using snapshot dependencies mostly worked, -but it wasn't always clear what needed rebuilding when pulling fresh -code from origin; if you weren't sure and didn't want to investigate, -the safe thing was to rebuild everything your project depended upon. -Alas, for a big tree of Scala code, that might take 45 minutes. - -Pants has a first-class concept of "depend on whatever version of this -project is defined on disk," and caches targets based on their -fingerprints (i.e. SHAs of the contents of the files and command line -options used to build the target). When code changes (e.g., after a git -pull), pants recompiles only those targets whose source files have -differing contents. - diff --git a/examples/src/java/com/pants/examples/page.md b/examples/src/java/com/pants/examples/page.md deleted file mode 100644 index 27b232b4181..00000000000 --- a/examples/src/java/com/pants/examples/page.md +++ /dev/null @@ -1,177 +0,0 @@ -README Files and Markdown -========================= - -You can write program documentation in the popular Markdown format; -Pants eases publishing your docs to places where your users can read it. -E.g., that `README.md` file in your source code is handy for editing; -but you might want to generate a web page from that so folks can decide -whether they want to look at your source code. - -Markdown to HTML ----------------- - -Pants uses the Python `Markdown` module; thus, in addition to the usual -Gruber `Markdown` syntax, there are [other -features](http://pythonhosted.org/Markdown/) Pants uses Python -Markdown's `codehilite`, `extra`, `tables`, and `toc` extensions. - -To tell Pants about your Markdown file, use a -`page` -target in a `BUILD` file as in this excerpt from -[examples/src/java/com/pants/examples/hello/main/BUILD](https://github.com/pantsbuild/pants/blob/master/examples/src/java/com/pants/examples/hello/main/BUILD): - -!inc[start-after=README page](hello/main/BUILD) - -To render the page as HTML, use the -markdown goal. For example, to view -`examples/src/java/com/pants/examples/hello/main/README.md` as HTML in -your browser, - - :::bash - $ ./pants goal markdown --markdown-open examples/src/java/com/pants/examples/hello/main:readme - -Pants generates the HTML files in the `dist/markdown/` directory tree. - -Link to Another `page` ----------------------- - -One `page` can link to another. Regular Markdown-link syntax works for -regular links; but if you use `page`s to generate both `.html` files and -wiki pages, it's not clear what to link to: the `.html` file or the wiki -address. You can use a Pants-specific syntax to make links that work -with generated HTML or wiki pages. - -To set up a page `source.md` that contains a link to `dest.md`, you make -a `dependencies` relation and use the special `pants(...)` syntax in the -markdown. - -In the `BUILD` file: - - :::python - page(name='source', - source='source.md', - dependencies=[':dest'], # enables linking - provides=[...publishing info...], - ) - - page(name='dest', - source='dest.md', - provides=[...publishing info...], - ) - -To set up the links in `source.md` that point to `dest.md` or an anchor -therein: - - For more information about this fascinating topic, - please see [[Destinations|pants('path/to:dest')]], - especially the - [[Addendum section|pants('path/to:dest)'#addendum]], - -Pants replaces the `pants('path/to:dest')` with the appropriate link. - -Include a File Snippet ----------------------- - -Sometimes the best way to explain `HelloWorld.java` is to show an -excerpt from `HelloWorld.java`. You can use the `!inc` markdown to do -this. Specify a file to include and (optionally) regexps at which to -start copying or stop copying. For example, to include an excerpt from -the file `HelloMain.java`, starting with the first line matching the -pattern `void main` and stopping before a subsequent line matching -`private HelloMain`: - - !inc[start-at=void main&end-before=private HelloMain](HelloMain.java) - -To include *all* of `HelloMain.java`: - - !inc(HelloMain.java) - -To include most of `HelloMain.java`, starting after license boilerplate: - - !inc[start-after=Licensed under the Apache](HelloMain.java) - -It accepts the following optional parameters, separated by ampersands -(&): - -start-at=*substring*
-When excerpting the file to include, start at the first line containing -*substring*. - -start-after=*substring*
-When excerpting the file to include, start after the first line -containing *substring*. - -end-before=*substring*
-When excerpting the file to include, stop before a line containing -*substring*. - -end-at=*substring*
-When excerpting the file to include, stop at a line containing -*substring*. - -Publishing ----------- - -You can tell Pants to publish a page. So far, there's only one way to -publish: as a page in an Atlassian Confluence wiki. (You can add other -doc-publish backends to Pants; -[[send us a patch|pants('src/python/pants/docs:howto_contribute')]]!) - -To specify the "address" to which to publish a page, give it a -`provides` parameter. - -Once you've done this, you can publish the page by invoking the goal -set up in your workspace. For example, if -the goal was set up with the name "confluence", you invoke: - - :::bash - $ ./pants goal confluence examples/src/java/com/pants/examples/hello/main:readme - -To specify that a page should be published to a Confluence wiki page, -set its `provides` to something like: - - :::python - page(... - provides=[ - wiki_artifact(wiki='//:confluence', - space='ENG', - title='Pants Hello World Example', - parent='Examples', - ) - ],) - -...assuming your workspace is set up with a wiki target named -`confluence` in a top-level `BUILD` file (as described below). - - - -**Setting up your workspace for Confluence publish** - -That `wiki` specifies some information about your wiki -server. So far, the only kind of thing you can publish to is a -Confluence wiki. You want to specify its target. Do this in one of the -`BUILD` files that's always processed (probably a top-directory `BUILD` -file). If your Confluence server lived at wiki.archie.org, the target -would probably look something like: - - :::python - import urllib - - def confluence_url_builder(page, config): - return config['title'], 'https://wiki.archie.org/display/%s/%s' % ( - config['space'], - urllib.quote_plus(config['title'])) - - # Use this wiki target's address for the wiki= param in your wiki_artifacts - confluence = wiki(name="confluence", - url_builder=confluence_url_builder) - -You need to install a goal to enable publishing a doc to confluence. To -do this, create a Pants plugin that installs a goal that subclasses -ConfluencePublish. - -In your `pants.ini` file, add a section with the url of your wiki -server. E.g., if your server is at wiki.archie.org, it would look like: - - [confluence-publish] - url: https://wiki.archie.org diff --git a/examples/src/java/com/pants/examples/readme.md b/examples/src/java/com/pants/examples/readme.md index 110b2707d6f..bb338944773 100644 --- a/examples/src/java/com/pants/examples/readme.md +++ b/examples/src/java/com/pants/examples/readme.md @@ -1,17 +1,24 @@ JVM Projects with Pants ======================= + Assuming you know the basic -[[Pants concepts|pants('src/docs:first_concepts')]] -and have gone through the -[[first Tutorial|pants('src/docs:first_tutorial')]], + +[Pants concepts](http://pantsbuild.github.io/first_concepts.html) and have +gone through the + +[first Tutorial](http://pantsbuild.github.io/first_tutorial.html), you've made a great start towards using Pants to work with Java and Scala code. This page goes into some of the details. If you are accustomed to the Maven tool and contemplating moving to Pants, you are not alone; -[[Pants for Maven Experts|pants('examples/src/java/com/pants/examples:from_maven')]] + +[Pants for Maven Experts](http://pantsbuild.github.io/from_maven.html) has some advice. Relevant Goals and Targets @@ -55,8 +62,8 @@ especially relevant. > > To use pre-built `.jar`s, a JVM target can depend on a `jar`, a > reference to published code; these `jar`s normally live in a -> directory called -> [[3rdparty|pants('examples/src/java/com/pants/examples:3rdparty_jvm')]]. +> +> directory called [3rdparty](http://pantsbuild.github.io/3rdparty_jvm.html). > > Pants can `publish` a JVM library so code in other repos can use it; > if the `*_library` target has a `provides` parameter, that specifies diff --git a/examples/src/python/example/3rdparty_py.md b/examples/src/python/example/3rdparty_py.md deleted file mode 100644 index 6afc8a42f0e..00000000000 --- a/examples/src/python/example/3rdparty_py.md +++ /dev/null @@ -1,69 +0,0 @@ -Python 3rdparty Pattern -======================= - -In general, we use the -[[3rdparty idiom|pants('src/docs:3rdparty')]] to organize -dependencies on code from outside the source tree. This document -describes how to make this work for Python code. - -Your Python code can pull in code written elsewhere. Pants fetches code -via a library that uses pip-style specifications (name and -version-range). - -3rdparty/python ---------------- - -To keep all of your code depending on the same versions of third-party -Python artifacts, you might use the idiom of keeping them in a directory -tree under `3rdparty/python`. If your organization has many such -dependencies, you might arrange them in several directories: this can -ease later "git detective work" when finding out who changed a version. - -**pip-style requirements.txt:** - -To define some third-party dependencies, use a -python_requirements in your `BUILD` -file and make a pip `requirements.txt` file in the same directory. - -E.g, your `3rdparty/python/BUILD` file might look like: - -!inc[start-after=Licensed under&end-before=target](../../../../3rdparty/python/BUILD) - -...with `3rdparty/python/requirements.txt` like: - -!inc[end-before=mox](../../../../3rdparty/python/requirements.txt) - -`python_requirements` defines a named target for each line in the -`requirements.txt` line. For example, a line like `ansicolors==1.0.2` in -`requirements.txt` defines a target named `ansicolors` that pulls in -ansicolors version 1.0.2. - -**python\_requirement\_library and python\_requirement:** - -A `BUILD` file can also define requirements without a `requirements.txt` -file. Set up a -`python_requirement_library` -with one or more -`python_requirement`s -like: - - :::python - python_requirement_library( - name='beautifulsoup', - requirements=[ - python_requirement(name='beautifulsoup', - requirement='BeautifulSoup==3.2.0'), - ]) - -Your Code's BUILD File ----------------------- - -In your code's `BUILD` file, introduce a dependency on the `3rdparty` -target: - -!inc[start-after=Like Hello](hello/greet/BUILD) - -Then in your Python code, you can `import` from that package: - - :::python - from colors import green diff --git a/examples/src/python/example/BUILD b/examples/src/python/example/BUILD index a21cd31272a..42a11a1e274 100644 --- a/examples/src/python/example/BUILD +++ b/examples/src/python/example/BUILD @@ -3,26 +3,10 @@ page( source='readme.md', dependencies=[ ':python_old', - 'src/docs:first_concepts', ] ) -page( - name='3rdparty_py', - source='3rdparty_py.md', - dependencies=[ - 'src/docs:3rdparty']) - -page( - name='pex_design', - source='pex_design.md' -) - page( name='python_old', - source='python_old.md', - dependencies=[ - 'src/docs:first_concepts', - 'src/docs:install', - ] + source='python_old.md' ) \ No newline at end of file diff --git a/examples/src/python/example/pex_design.md b/examples/src/python/example/pex_design.md deleted file mode 100644 index a34a4275017..00000000000 --- a/examples/src/python/example/pex_design.md +++ /dev/null @@ -1,154 +0,0 @@ -PEX Design -========== - -But why another system? - -Alternatives ------------- - -There are several solutions for package management in Python. Almost -everyone is familiar with running `sudo easy_install PackageXYZ`. This -leaves a lot to be desired. Over time, your Python installation will -collect dozens of packages, become annoyingly slow or even broken, and -reinstalling it will invariably break a number of the applications that -you were using. - -A marked improvement over the sudo `easy_install` model is -[virtualenv](http://www.virtualenv.org) to isolate Python environments -on a project by project basis. This is useful for development but does -not directly solve any problems related to deployment, whether it be to -a production environment or to your peers. It is also challenging to -explain to a Python non-expert. - -A different solution altogether, [zc.buildout](http://www.buildout.org/) -attempts to provide a framework and recipes for many common development -environments. It has arguably gone the farthest for automating -environment reproducibility amongst the popular tools, but shares the -same complexity problems as all the other abovementioned solutions. - -Most solutions leave deployment as an afterthought. Why not make the -development and deployment environments the same by taking the -environment along with you? - -Pants and PEX -------------- - -The lingua franca of Pants is the PEX file (PEX itself does not stand -for anything in particular, though in spirit you can think of it as a -"Python EXecutable".) - -**PEX files are single-file lightweight virtual Python environments.** - -The only difference is no virtualenv setup instructions or -pip install foo bar baz. PEX files are self-bootstrapping Python -environments with no strings attached and no side-effects. Just a simple -mechanism that unifies both your development and your deployment. - -### How PEX files work - -the utility of zipimport and `__main__.py` ------------------------------------------- - -As an aside, in Python, you may not know that you can import code from -directories: - - :::bash - $ mkdir -p foo - $ touch foo/__init__.py - $ echo "print 'spam'" > foo/bar.py - $ python -c 'import foo.bar' - spam - -All that is necessary is the presence of `__init__.py` to signal to -the importer that we are dealing with a package. Similarly, a directory -can be made "executable": - - :::bash - $ echo "print 'i like flowers'" > foo/__main__.py - $ python foo - i like flowers - -And because the zipimport module now provides a default import hook for -Pythons \>= 2.4, if the Python import framework sees a zip file, with -the inclusion of a proper `__init__.py`, it can be treated similarly -to a directory. But since a directory can be executable, if we just drop -a `__main__.py` into a zip file, it suddenly becomes executable: - - :::bash - $ pushd foo && zip /tmp/flower.zip __main__.py && popd - /tmp/foo /tmp - adding: __main__.py (stored 0%) - /tmp - $ python flower.zip - i like flowers - -And since zip files don't actually start until the zip magic number, you -can embed arbitrary strings at the beginning of them and they're still -valid zips. Hence simple PEX files are born: - - :::bash - $ echo '#!/usr/bin/env python2.6' > flower.pex && cat flower.zip >> flower.pex - $ chmod +x flower.pex - $ ./flower.pex - i like flowers - -Remember pants.pex? - - :::bash - $ unzip -l pants.pex | tail -2 - warning [pants.pex]: 25 extra bytes at beginning or within zipfile - (attempting to process anyway) - -------- ------- - 7900812 543 files - - $ head -c 25 pants.pex - #!/usr/bin/env python2.6 - -PEX `__main__.py` ------------------ - -The `__main__.py` in a real PEX file is somewhat special: - - :::python - import os - import sys - - __entry_point__ = None - if '__file__' in locals() and __file__ is not None: - __entry_point__ = os.path.dirname(__file__) - elif '__loader__' in locals(): - from pkgutil import ImpLoader - if hasattr(__loader__, 'archive'): - __entry_point__ = __loader__.archive - elif isinstance(__loader__, ImpLoader): - __entry_point__ = os.path.dirname(__loader__.get_filename()) - - if __entry_point__ is None: - sys.stderr.write('Could not launch python executable!\n') - sys.exit(2) - - sys.path.insert(0, os.path.join(__entry_point__, '.bootstrap')) - - from twitter.common.python.importer import monkeypatch - monkeypatch() - del monkeypatch - - from twitter.common.python.pex import PEX - PEX(__entry_point__).execute() - -PEX is just a class that manages requirements (often embedded within PEX -files as egg distributions in the .deps directory) and autoimports them -into the sys.path, then executes a prescribed entry point. - -If you read the code closely, you'll notice that it relies upon -monkeypatching zipimport. Inside the twitter.common.python library we've -provided a recursive zip importer derived from Google's [pure Python -zipimport](http://code.google.com/appengine/articles/django10_zipimport.html) -module that allows for depending upon eggs within eggs or zips (and so -forth) so that PEX files need not extract egg dependencies to disk a -priori. This even extends to C extensions (.so and .dylib files) which -are written to disk long enough to be dlopened before being unlinked. - -Strictly speaking this monkeypatching is not necessary and we may -consider making that optional. - diff --git a/examples/src/python/example/python_old.md b/examples/src/python/example/python_old.md index bd8830c5394..9de544564c7 100644 --- a/examples/src/python/example/python_old.md +++ b/examples/src/python/example/python_old.md @@ -1,5 +1,9 @@ Using Pants for Python development, 2014 and Earlier ==================================================== + Why use Pants for Python development? ------------------------------------- @@ -52,10 +56,12 @@ Getting started --------------- First it is necessary to install Pants. See -[[Installing Pants|pants('src/docs:install')]]. + +[Installing Pants](http://pantsbuild.github.io/install.html). It is also helpful to read the -[[Pants Conceptual Overview|pants('src/docs:first_concepts')]]. + +[Pants Conceptual Overview](http://pantsbuild.github.io/first_concepts.html). TL;DR - 'Hello world!' with Pants Python ---------------------------------------- @@ -254,13 +260,13 @@ format of pkg\_resources.EntryPoint, which is something akin to some.module.name:my.attr which means run the function pointed by my.attr inside the module some.module inside the environment. The :my.attr component can be omitted and the module is executed directly (presuming -it has a `__main__.py`.) +it has a \_\_main\_\_.py.) #### python\_requirement A python\_requirement target describes an external dependency as understood by easy\_install or pip. It takes only a single non-keyword -argument of the Requirement-style string, e.g., +argument of the Requirement-style string, e.g. : python_requirement('django-celery') python_requirement('tornado==2.2') diff --git a/examples/src/python/example/readme.md b/examples/src/python/example/readme.md index 720416fb01e..a5b0f575d2a 100644 --- a/examples/src/python/example/readme.md +++ b/examples/src/python/example/readme.md @@ -1,9 +1,14 @@ Python Projects with Pants ========================== + Pants makes the manipulation and distribution of hermetically sealed Python environments painless. You can organize your code in the -[[Pants way|pants('src/docs:first_concepts')]] + +[Pants way](http://pantsbuild.github.io/first_concepts.html) with targets for binaries, libraries, and tests. Pants builds Python code into [PEXes](https://github.com/twitter/commons/blob/master/src/python/twitter/common/python/README.md). @@ -13,7 +18,8 @@ you have some "common code" used by more than one PEX, Pants makes it easy to manage the dependencies. This page assumes that you've already read -[[the Pants Tutorial|pants('src/docs:first_tutorial')]]. + +[the Pants Tutorial](http://pantsbuild.github.io/first_tutorial.html). Pants' Python support changed in 2014. Depending on how old your Pants is, Pants goals might not work with Python code. If you use older pants, @@ -310,7 +316,7 @@ More About Python Tests Pants runs Python tests with `pytest`. You can pass CLI options to `pytest` with `--test-pytest-options`. For example, to only run tests -whose names match the pattern `*foo*`, you could run: +whose names match the pattern `*foo*`, you could run : :::bash $ ./pants goal test examples/tests/python/example_test/hello/greet --test-pytest-options='-k foo' @@ -427,3 +433,4 @@ You can then combine the coverage files by running `PEX_MODULE=coverage` `my_pex.pex` report. Since PEX files are just zip files, coverage is able to understand and extract source and line numbers from them in order to produce coverage reports. + diff --git a/examples/src/thrift/com/pants/examples/BUILD b/examples/src/thrift/com/pants/examples/BUILD deleted file mode 100644 index 7dd56672866..00000000000 --- a/examples/src/thrift/com/pants/examples/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -page( - name='readme', - source='readme.md', - dependencies=[ - 'src/docs:publish', - ]) \ No newline at end of file diff --git a/examples/src/thrift/com/pants/examples/readme.md b/examples/src/thrift/com/pants/examples/readme.md deleted file mode 100644 index c8fbd7d656c..00000000000 --- a/examples/src/thrift/com/pants/examples/readme.md +++ /dev/null @@ -1,197 +0,0 @@ -Using Pants with Thrift Dependencies -==================================== - -[Apache Thrift](http://thrift.apache.org/) is a popular framework for -working with data types and service interfaces. It uses an Interface -Definition Language (IDL) to define these types and interfaces. There -are tools to generate code in "real" programming languages from Thrift -IDL files. Two programs, perhaps in different programming languages, -should be able to communicate over Thrift interfaces by using this -generated code. - -Pants knows Thrift. For each Thrift file you use, your codebase has some -`BUILD` targets that represent "real" code generated from IDL code. You -can write code in your favorite language that imports the generated -code. To make the import work, your code's `BUILD` target depends on the -appropriate Thrift `BUILD` target. - -Generating Code ---------------- - -You have some Thrift; you want to use it from your "regular" programming -language. Normally, to make, e.g., Java code usable, you set up a -`java_library` target with sources `*.java` and then depend on that -target; Thrift works similarly, but you use a different target type that -generates Java code from `*.thrift`. You can define Java, Python, or -Scala library targets whose code is Thrift-generated by setting up -*lang*\_thrift\_library targets. (Scala is tricky; you use a -`java_thrift_library` with carefully-chosen parameters.) Other targets -can depend on a *lang*\_thrift\_library and their code can then import -the generated code. - -### Target Example - -This example sets up a `java_thrift_library` target; its source is -Thrift; it generates Java. - - :::python - # Target defined in src/thrift/com/twitter/mybird/BUILD: - java_thrift_library(name='mybird', - # Specify dependencies for thrift IDL file includes. - dependencies=[ - 'src/thrift/com/twitter/otherbird', - ], - sources=globs('*.thrift') - ) - -Pants knows that before it compiles such a target, it must first -generate Java code from the Thrift IDL files. Users can depend on this -target like any other internal target. In this case, users would add a -dependency on `'src/thrift/com/twitter/mybird'`. - -One *lang*\_thrift\_library can depend on another; use this if one of -your Thrift files `include`s a Thrift file that lives in another target. - -### Configuring - -Here are some popular `*_thrift_library` configurations: - -**Java** - -Use Apache Thrift compiler (the default): - - :::python - java_thrift_library(...) - -...or Scrooge: - - :::python - java_thrift_library( - compiler='scrooge') - -**Python** - - :::python - python_thrift_library(...) - -**Scala** - - :::python - java_thrift_library( # Yes, a "java" library to generate Scala - compiler='scrooge', # default compiler does not gen Scala; Scrooge does - language='scala', - # maybe set an rpc_style - ) - -Thrift Example --------------- - -Let's look at some sample code that puts all of this together. - -- Thrift IDL code (`.thrift` files) -- `BUILD` targets for the Thrift IDL code -- Java code that `import`s code generated from Thrift -- `BUILD` target dependencies that allow that `import` - -### Thrift IDL - -Our example uses two Thrift files, one of which `include`s the other. -They look pretty ordinary. The include-d Thrift, -`examples/src/thrift/com/pants/examples/distance/distance.thrift`, is -regular Thrift (albeit with a `#namespace` comment used for Thrift that -will be compiled with both Apache Thrift and Scrooge): - -!inc(distance/distance.thrift) - -The include-ing Thrift, -`examples/src/thrift/com/pants/examples/precipitation/precipitation.thrift`, -also looks ordinary. (The include path is a little tricky: it's based on -source roots. Thus, if your source tree has more than one root `foo` and -`bar` and has Thrift in both, code in foo can `include` code from `bar` -without mentioning `bar` in the include path.): - -!inc(precipitation/precipitation.thrift) - -### BUILD Targets - -In a `BUILD` file, we use a `java_thrift_library` or -`python_thrift_library` to generate "real" code from Thrift. Our example -just uses Java; thus, the `BUILD` file for `distance.thrift` looks like - -!inc[start-after=cd ../precipitation](distance/BUILD) - -Notice the target type is -java_thrift_library, -and this target staked its claim to our distance thrift IDL file. -JVM library targets -(e.g.: -`java_library`, -`scala_library`) that depend on this -target will simply see generated code from the IDL. Since no additional -options are specified we use the defaults; however, if we need more -control over how code is generated we control that through arguments -provided by -java_thrift_library. - -> **note** -> -> While the name `java_thrift_library` might make you think it generates -> Java, it can also generate other target languages via the `language` -> parameter (scala for example). For Python code, however, use -> python_thrift_library. - -As with "regular" languages, for one target's code to include another's, -a target should have the other in its `dependencies`. Thus, to allow -`precipitation.thrift` to depend on `distance.thrift`, we set up -`.../precipitation/BUILD` like so: - -!inc[start-after=includes other thrift](precipitation/BUILD) - -### Using in "Regular" Code - -We want to use the Thrift-generated interface from "regular" code. In -this Java example, we want to `import` the generated code. In our Java, -the `import` statements use the names from the `.thrift` files' -`namespace`s: - -!inc[start-after=from Java.](../../../../../tests/java/com/pants/examples/usethrift/UseThriftTest.java) - -As usual, for code in one target to use code from another, one target -needs to depend on the other. Thus, our Java code's target has the -`*_thrift_library` target whose code it uses in its dependencies: - -!inc[start-after=using Thrift from Java, though.](../../../../../tests/java/com/pants/examples/usethrift/BUILD) - -Publishing ----------- - -Publishing a *lang*\_thrift\_library is like -[[publishing a "regular" library|pants('src/docs:publish')]] -The targets use `provides` -parameters. It might look something like: - - :::python - java_thrift_library(name='eureka-java', - sources=['eureka.thrift'], - dependencies=[ - 'src/thrift/org/archimedes/volume:volume-java', - ], - language='java', - provides=artifact( - org='org.archimedes', - name='eureka-java', - repo='BUILD.archimedes:jar-public', - )) - - java_thrift_library(name='eureka-scala', - sources=['eureka.thrift'], - dependencies=[ - 'src/thrift/org/archimedes/volume:volume-scala', - ], - compiler='scrooge', - language='scala', - provides=artifact( - org='org.archimedes', - name='eureka-scala', - repo='BUILD.archimedes:jar-public', - )) diff --git a/src/docs/3rdparty.md b/src/docs/3rdparty.md deleted file mode 100644 index 7db4fcf5f30..00000000000 --- a/src/docs/3rdparty.md +++ /dev/null @@ -1,34 +0,0 @@ -Third-Party Dependencies -======================== - -Despite your best recruiting efforts, most software is still written by -people outside your organization. Your code can import some of this -*third-party* code. In the usual Pants way, a build target *depends* on -something to enable importing its code. Special dependencies represent -third-party code. - -To help all your code depend on the same version of third-party code, -it's handy to keep these special dependencies in one place in your -source tree. By convention, Pants-using source trees use a `3rdparty/` -directory to hold these dependencies. - -If two parts of your code depend on two versions of the same package, -some tool will pick one version to use. The behavior depends on the -tool, but you can be sure that one part of your code is *not* using the -version it expects. This is known as a *diamond dependencies problem*, -*dependencies version conflict*, or *dependency hell*; you don't want -it. - -By keeping external dependencies in one place, you make it easier for -all your code to depend on the same version and avoid surprises. - -Beware: some version dependencies "hide." You depend on an external -packages; an external package itself depends on others and "knows" what -versions of those packages it depends on. Even though all your code -depends on the version specified in `3rdparty/`, you might depend on -something which, in turn, depends on some other version. - -* For see how this works in JVM languages, see - [[Python 3rdparty Pattern|pants('examples/src/java/com/pants/examples:3rdparty_jvm')]] -* For see how this works in Python, see - [[Python 3rdparty Pattern|pants('examples/src/python/example:3rdparty_py')]] diff --git a/src/docs/BUILD b/src/docs/BUILD index a8b4a80e9f3..abae29ffa88 100644 --- a/src/docs/BUILD +++ b/src/docs/BUILD @@ -1,124 +1,9 @@ -page( - name='3rdparty', - source='3rdparty.md', - dependencies=[ - 'examples/src/java/com/pants/examples:3rdparty_jvm', - 'examples/src/python/example:3rdparty_py', - ], -) - -page( - name='announce_201409', - source='announce_201409.md', -) - page( name='build_files', - source='build_files.md', - dependencies=[ - ':first_tutorial', - ] -) - -page( - name='first_concepts', - source='first_concepts.md', - dependencies=[ - ':first_tutorial', - ] -) - -page( - name='first_tutorial', - source='first_tutorial.md', - dependencies=[ - ':first_concepts', - 'examples/src/java/com/pants/examples:readme', - 'examples/src/python/example:readme', - ] -) - -page( - name='index', - source='index.md', - dependencies=[ - ':3rdparty', - ':announce_201409', - ':build_files', - ':first_concepts', - ':first_tutorial', - ':install', - ':invoking', - ':publish', - ':setup_repo', - ':target_addresses', - ':tshoot', - ':with_emacs', - ':with_intellij', - 'examples/src/java/com/pants/examples:page', - 'examples/src/java/com/pants/examples:readme', - 'examples/src/python/example:readme', - 'examples/src/thrift/com/pants/examples:readme', - 'src/python/pants/docs:readme', - ]) - -page( - name='invoking', - source='invoking.md', - dependencies=[ - ':first_tutorial', - ':target_addresses', - ] -) - -page( - name='install', - source='install.md', - dependencies=[ - ':setup_repo', - ] + source='build_files.md' ) page( name='publish', - source='publish.md', - dependencies=[ - ':3rdparty', - ] -) - -page( - name='setup_repo', - source='setup_repo.md', - dependencies=[ - ':first_concepts', - ':publish', - 'src/python/pants/docs:dev_tasks', - 'src/python/pants/docs:dev_tasks_publish_extras', - 'src/python/pants/docs:howto_contribute', - 'src/python/pants/docs:howto_plugin', - ] -) - -page( - name='target_addresses', - source='target_addresses.md' -) - -page( - name='tshoot', - source='tshoot.md', - dependencies=[ - 'src/python/pants/docs:howto_ask', - ] -) - -page( - name='with_emacs', - source='with_emacs.md' -) - -page( - name='with_intellij', - source='with_intellij.md' -) + source='publish.md' +) \ No newline at end of file diff --git a/src/docs/announce_201409.md b/src/docs/announce_201409.md deleted file mode 100644 index 59981e57c43..00000000000 --- a/src/docs/announce_201409.md +++ /dev/null @@ -1,98 +0,0 @@ -Hello Pants Build -================= - -*Posted September 2014* - -As code bases grow, they become increasingly difficult to work with. -Builds get ever slower and existing tooling doesn’t scale. One solution -is to keep splitting the code into more and more independent -repositories. However, as growth continues you end up with hundreds of -free-floating codebases with hard-to-manage dependencies. This makes it -hard to discover, navigate and share code which can impact developer -productivity. - -Another solution is to have a single large, unified code base. We’ve -found that this promotes better engineering team cohesion and -collaboration which results in greater productivity and happiness. -However, tooling for such structured code bases has been lacking which -is why we developed [Pants](http://pantsbuild.github.io/), an open -source build system written in Python. - -Pants models code modules (known as “targets”) and their dependencies in -`BUILD` files—in a manner similar to Google's [internal build -system](http://google-engtools.blogspot.com/2011/08/build-in-cloud-how-build-system-works.html). -This allows it to only build the parts of the codebase you actually -need, ignoring the rest of the code. This is a key requirement for -scaling large, unified code bases. - -Pants started out in 2010 as an internal tool at -[Twitter](https://twitter.com/) and was originally just a frontend to -generate `build.xml` files for the Ant build tool, hence the name (a -contraction of “Python Ant”). Pants grew in capability and complexity, -and became the build tool for the -[twitter/commons](https://github.com/twitter/commons/) open source -libraries, and hence became open source itself. - -In 2012, [Foursquare](https://foursquare.com/) began using Pants -internally, and Foursquare engineers picked up the Pants development -mantle, adding Scala support, build artifact caching and many other -features. - -Since then several other engineering teams, including those at [Urban -Compass](https://www.urbancompass.com/) and -[Oscar](https://www.hioscar.com/?locale=en), have integrated Pants into -their codebases. Most recently, [Square](https://squareup.com/) began to -use Pants and has also contributed significantly to its development. - -As a result, Pants is a true independent open source project with -collaborators across companies and a growing development community. It -now lives in a standalone repo at . - -Among Pants’s current strengths are: - -- Builds [Java](http://pantsbuild.github.io/JVMProjects.html), Scala, - and [Python](http://pantsbuild.github.io/python-readme.html). -- Adding support for new languages is straightforward. -- Supports code generation: - [thrift](http://pantsbuild.github.io/ThriftDeps.html), protocol - buffers, custom code generators. -- Resolves external JVM and Python dependencies. -- Runs tests. -- Spawns Python and Scala REPLs with appropriate load paths. -- Creates deployable packages. -- Scales to large repos with many interdependent modules. -- Designed for incremental builds. -- Support for local and distributed caching. -- Especially fast for Scala builds, compared to alternatives. -- Builds standalone python executables ([PEX - files](http://pex.readthedocs.org/)) -- Has a plugin system to add custom features and override stock - behavior. -- Runs on Linux and Mac OS X. - -Since moving Pants to its own GitHub organization, our [commit -rate](https://github.com/pantsbuild/pants/graphs/contributors) has grown -and we’ve welcomed more committers to the project. - -If your codebase is growing beyond your toolchain’s ability to scale, -but you’re reluctant to split it up, you might want to give Pants a try. -It may be of particular interest if you have complex dependencies, -generated code and custom build steps. - -Pants is still a young and evolving open source project. We constantly -strive to make it easier to use. If you’re interested in using or -learning from Pants, our advice is to reach out to the community on the -[developer mailing -list](http://pantsbuild.github.io/howto_contribute.html) and follow -[@pantsbuild](https://twitter.com/pantsbuild) on Twitter for updates. - -**Organization Perspectives:** - -- [Introducing Pants: a build system for large-scale codebases like - Foursquare’s](http://engineering.foursquare.com/2014/09/16/introducing-pants-a-build-system-for-large-scale-codebases-like-foursquares/), - Foursquare Engineering Blog -- [Trying on - Pants](http://corner.squareup.com/2014/09/trying-on-pants.html), - Eric Ayers, Infrastructure/Shared Systems team engineer / Square - Engineering Blog - diff --git a/src/docs/build_files.md b/src/docs/build_files.md index 4f43cf0f877..1790bb34b7e 100644 --- a/src/docs/build_files.md +++ b/src/docs/build_files.md @@ -1,5 +1,9 @@ BUILD files =========== + To tell Pants about your source code, you have files named `BUILD` in directories of your source tree. These files define build-able targets @@ -8,7 +12,8 @@ and specify source code layout. This page goes into some detail about `BUILD` files (`java_library`, `python_binary`, etc.), please see the [BUILD Dictionary](build_dictionary.html). If you want less detail-y information about `BUILD` files, -[[the Tutorial|pants('src/docs:first_tutorial')]] + +[the Tutorial](http://pantsbuild.github.io/first_tutorial.html) is a good place to start. `BUILD` files are little Python scripts with some diff --git a/src/docs/first_concepts.md b/src/docs/first_concepts.md deleted file mode 100644 index b588a536eae..00000000000 --- a/src/docs/first_concepts.md +++ /dev/null @@ -1,148 +0,0 @@ -Pants Conceptual Overview -========================= - -Pants is a build system for software. It works particularly well for a -source code repository that contains many distinct but interdependent -pieces. - -Pants is similar to `make`, `maven`, `ant`, `gradle`, `sbt`, etc.; but -pants pursues different design goals. Pants optimizes for - -- building multiple, dependent things from source -- building code in a variety of languages -- speed of build execution - -A Pants build "sees" only the target it's building and the transitive -dependencies of that target. This approach works well for a big -repository containing several things; a tool that builds everything -would bog down. - -Goals and Targets ------------------ - -To use Pants, you must understand a few concepts: - -**Goals** are the "verbs" of Pants.
-When you invoke Pants, you name goals on the command line to say what -Pants should do. For example, to run tests, you would invoke Pants with -the `test` goal. To create a bundle--an archive containing a runnable -binary and resource files--you would invoke Pants with the `bundle` -goal. These goals are built into Pants. - -**Targets** are the "nouns" of Pants, things pants can act upon.
-You annotate your source code with `BUILD` files to define these -targets. For example, if your `tests/com/twitter/mybird/` directory -contains JUnit tests, you have a `tests/com/twitter/mybird/BUILD` file -with a `junit_tests` target definition. As you change your source code, -you'll occasionally change the set of Targets by editing `BUILD` files. -E.g., if you refactor some code, moving part of it to a new directory, -you'll probably set up a new `BUILD` file with a target to build that -new directory's code. - -When you invoke Pants, you specify goals and targets: the actions to -take, and the things to carry out those actions upon. Together, your -chosen goals and targets determine what Pants produces. Invoking the -`bundle` goal produces an archive; invoking the `test` goal displays -test results on the console. Assuming you didn't duplicate code between -folders, targets in `tests/com/twitter/mybird/` will have different code -than those in `tests/com/twitter/otherbird/`. - -Goals can "depend" on other goals. For example, there are `test` and -`compile` goals. If you invoke Pants with the `test` goal, Pants "knows" -it must compile tests before it can run them, and does so. (This can be -confusing: you can invoke the `test` goal on a target that isn't -actually a test. You might think this would be a no-op. But since Pants -knows it must compile things before it tests them, it will compile the -target.) - -Targets can "depend" on other targets. For example, if your `foo` code -imports code from another target `bar`, then `foo` depends on `bar`. You -specify this dependency in `foo`'s target definition in its `BUILD` -file. If you invoke Pants to compile `foo`, it "knows" it also needs to -compile `bar`, and does so. - -Target Types ------------- - -Each Pants build target has a *type*, such as `java_library` or -`python_binary`. Pants uses the type to determine how to apply goals to -that target. - -**Library Targets**
-To define an "importable" thing, you want a library target type, such as -`java_library` or `python_library`. Another target whose code imports a -library target's code should list the library target in its -`dependencies`. - -**Binary Targets**
-To define a "runnable" thing, you want a `jvm_binary` or `python_binary` -target. A binary probably has a `main` and dependencies. (We encourage a -binary's main to be separate from the libraries it uses to run, if any.) - -**External Dependencies**
-Not everything's source code is in your repository. Your targets can -depend on `.jar`s or `.eggs`s from elsewhere. - -**Test Targets**
-To define a collection of tests, you want a `junit_tests`, -`scala_specs`, `python_tests`, or `python_test_suite` target. The test -target depends upon the targets whose code it tests. This isn't just -logical, it's handy, too: you can compute dependencies to figure out -what tests to run if you change some target's code. - -For a list of all Target types (and other things that can go in `BUILD` -files), see the BUILD Dictionary. - -What Pants Does ---------------- - -When you invoke Pants, you specify goals (actions to take) and targets -(things to act upon). - -**Pants plans a list of goals.** You specify one or more goals on the -command line. Pants knows that some goals depend on others. If you -invoke Pants with, say, the `test` goal to test some code, Pants knows -it must first compile code; before it can compile code, it needs to -resolve artifact dependencies and generate code from IDL files (e.g., -Thrift). Pants thus generates a topologically-sorted list of goals, a -*build execution plan*. This plan might look something like - -> resolve-idl -\> gen -\> resolve -\> compile -\> resources -\> test - -Pants does *not* consider targets while planning; some of these goals -might thus turn out to be no-ops. E.g., Pants might plan a `gen` -(generate code) goal even if you don't, in fact, use any generated code. - -**Pants computes a target dependencies graph.** It starts with the -target[s] you specify on the command line. It notes which targets they -depend on, which targets those targets depend on, which targets *those* -targets depend on, and so on. - -**Pants then attempts to carry out its planned goals.** It proceeds goal -by goal. If it has a problem carrying out one goal, it does not continue -to the other goals. (Thus, if you attempt to test targets *A* and *B*, -but there's a compilation error in *A*, then Pants won't test *B* even -if it compiled fine.) - -For each goal, Pants attempts to apply that goal to all targets in its -computed dependency tree[s]. It starts with depended-upon targets and -works its way up to depending targets. Each Pants target has a type; -Pants uses this to determine how to apply a goal to that target. In many -cases, applying a goal to a target is a no-op. In the more interesting -cases, Pants does something. It probably invokes other tools. For -example, depending on the code in the relevant targets, that "compile" -goal might invoke `javac` a few times and `scalac`. - -Pants caches things it builds. Thus, if you change one source file and -re-build, Pants probably doesn't "build the world." It just builds a few -things. Pants keys its cache on hashed file contents. This is a -straightforward way to build the right things after some files' contents -change. (It *can* surprise you if you `touch` a file, start a compile, -and nothing happens. If you want to, e.g., see `Foo.java`'s compile -warnings again, instead of using `touch`, you might append a newline.) - -Next Step ---------- - -If you're ready to give Pants a try, go to -[[First Tutorial|pants('src/docs:first_tutorial')]]. diff --git a/src/docs/first_tutorial.md b/src/docs/first_tutorial.md deleted file mode 100644 index 760f8e94fb0..00000000000 --- a/src/docs/first_tutorial.md +++ /dev/null @@ -1,260 +0,0 @@ -First Tutorial -============== - -This tutorial walks you through some first steps with Pants build: -invoking commands, looking at the files that define build-able things. -It assumes you're already familiar with -[[basic Pants build concepts|pants('src/docs:first_concepts')]]. -It assumes you're -working in a source tree that already has `pants` installed (such as -Pants's own repo: -[pantsbuild/pants](https://github.com/pantsbuild/pants)). - -The first time you run `pants`, try it without arguments. This makes -Pants "bootstrap" itself, downloading and compiling things it needs: - - :::bash - $ ./pants - -Now you're ready to invoke pants for more useful things. - -You invoke pants with *goals* (like `test` or `bundle`) and the *build -targets* to use (like -`examples/tests/java/com/pants/examples/hello/greet:greet`). For -example, - - :::bash - $ ./pants goal test examples/tests/java/com/pants/examples/hello/greet:greet - -Goals (the "verbs" of Pants) produce new files from Targets (the -"nouns"). - -As a code author, you define your code's build targets in BUILD files. A -build target might produce some output file[s]; it might have sources -and/or depend on other build targets. There might be several BUILD files -in the codebase; a target in one can depend on a target in another. -Typically, a directory's BUILD file defines the target[s] whose sources -are files in that directory. - -Invoking Pants --------------- - -Pants knows about goals ("verbs" like `bundle` and `test`) and targets -(build-able things in your source code). A typical pants command-line -invocation looks like - - :::bash - $ ./pants goal test examples/tests/java/com/pants/examples/hello/greet:greet - -Looking at the pieces of this we see - -`./pants`
-That `./` isn't a typo. A source tree that's been set up with Pants -build has a `pants` executable in its top-level directory. - -The first time you run `./pants`, it might take a while: it will -probably auto-update by downloading the latest version. - -`goal`
-Magic word that you use on most Pants command lines. We hope that -someday we won't need to use this magic word anymore. As a rule of -thumb, if you work with JVM code, you need `goal`; if you work with -Python code, you leave it out. - -`test`
-`test` is a *goal*, a "verb" that Pants knows about. The `test` goal -runs tests and reports results. (When working with Python code, you -don't normally specify a goal on the command line. Instead Pants -"figures out" what to do based on the targets. E.g., it runs tests on -test targets.) - -Some goals are `gen` (generate code from Thrift, Antlr, Protocol -Buffer), `compile`, and `eclipse` (generate an Eclipse project). Pants -knows that some of these goals depend on each other. E.g., in this -example, before it run tests, it must compile the code. - -You can specify more than one goal on a command line. E.g., to generate -an Eclipse project *and* run tests, we could have said `eclipse tests`. - -`examples/tests/java/com/pants/examples/hello/greet:greet`
-This is a *build target*, a "build-able" thing in your source code. To -define these, you set up configuration files named `BUILD` in your -source code file tree. (You'll see more about these later.) - -Targets can depend on other targets. E.g., a test suite target normally -depends on another target containing the "library" code to test; to -build the test code, Pants also must build the library code. - -You can specify more than one target on a command line. Pants will carry -out its goals on all specified targets. E.g., you might use this to -generate an Eclipse project based on Hello World's source and tests. - -### Output - -Pants produces files, both build outputs and intermediate files -generated "along the way". These files live in directories under the -top-level directory: - -`dist/`
-By default, build outputs go in the `dist/` directory. So far, you've -just run the `test` goal, which doesn't output a file. But if you'd -instead invoked, for example, the `bundle` goal on a `jvm_app` target, -Pants would have populated this directory with many JVM `.jar` files. - -`.pants.d/`
-Intermediate files go in the `.pants.d/` directory. You don't want to -rely on files in there; if the Pants implementation changes, it's likely -to change how it uses intermediate files. You don't want to edit/delete -files in there; you may confuse Pants. But if you want to peek at some -generated code, the code is probably in here somewhere. - -### Multiple Goals, Multiple Targets - -You can specify multiple goals and multiple targets. Pants applies all -the goals to all the targets, skipping things that wouldn't make sense. -E.g., you could - -- Invoke `eclipse` and `test` goals to both generate an Eclipse - project and run tests. -- Specify both test-suite and "library" targets so Eclipse sees all - the source code. - -In this example, it doesn't make sense to run library code as a test, so -Pants doesn't do that. Since pants knows that the `test` goal depends on -the `compile` goal, it *does* compile the library. - - - -*Goal-Target Mismatch* - -One tricky side effect of this is accidental *goal-target mismatch*: You -can invoke a goal that doesn't make sense for a target. E.g., you can -invoke the `test` goal on a target that's not a test suite. Pants won't -complain. It knows that it should compile code before it tests it; it -will happily compile the build targets. If you're not watching closely, -you might see a lot of output scrolling past and think it was running -tests. - -### Help - -To get help about a Pants goal, invoke ./pants goal *goalname* -h. -This lists the many command-line options you can pass for that goal. -E.g., - - :::bash - $ ./pants goal test -h - - test: Test compiled code. - - test options: - --ng-daemons, --no-ng-daemons - [True] Use nailgun daemons to execute java tasks. - - gen:thrift options: - ... - --test-specs-color, --no-test-specs-color - [True] Emit test result with ANSI terminal color - codes. - -For a list of available goals, `./pants goal goals`. - -For help with things that aren't goals, use - - :::bash - $ ./pants -h - -If you want help diagnosing some strange Pants behavior, you might -verbose output. To get this, instead of just invoking `./pants`, set -some environment variables: -`PEX_VERBOSE=1 PANTS_VERBOSE=1 PYTHON_VERBOSE=1 ./pants`. - -BUILD Files ------------ - -We told pants what target to build, but where are these defined? -Scattered around the source tree are `BUILD` files. These `BUILD` files -define targets. For example, this code snippet of -`java/com/pants/examples/hello/main/BUILD` defines two targets: the app -we ran and the binary that contains its code. These targets are named -`main` (of type `jvm_app`) and and `main-bin` (of type `jvm_binary`): - -That `dependencies` is interesting. The `main-bin` build target depends -on other build targets; its `dependencies` lists those. To build a -runnable Java binary, we need to first compile its dependencies. The -`main-bin` binary has two dependencies, -`'examples/src/java/com/pants/examples/hello/greet'` is the *address* of -another target. Addresses look, roughly, like `path/to/dir:targetname`. -We can see this build target in the `.../hello/greet/BUILD` file: - -!inc[start-after=see LICENSE](../../examples/src/java/com/pants/examples/hello/greet/BUILD) - -Pants uses dependency information to figure out how to build your code. -You might find it useful for other purposes, too. For example, if you -change a library's code, you might want to know which test targets -depend on that library: you might want to run those tests to make sure -they still work. - -### Anatomy of a `BUILD` Target - -A target definition in a `BUILD` file looks something like - - :::python - scala_library( - name='util', - dependencies = ['3rdparty:commons-math', - '3rdparty:thrift', - 'src/main/scala/com/foursquare/auth', - ':base'], - sources=globs('*.scala'), - ) - -Here, `scala_library` is the target's *type*. Different target types -support different arguments. The following arguments are pretty common: - -**name**
-We use a target's name to refer to the target. This argument isn't just -"pretty common," it's required. You use names on the command line to -specify which targets to operate on. You also use names in `BUILD` files -when one target refers to another, e.g. in `dependencies`: - -**dependencies**
-List of things this target depends upon. If this target's code imports -code that "lives" in other targets, list those targets here. If this -target imports code that "lives" in `.jar`s/`.egg`s from elsewhere, -refer to them here. - -**sources**
-List of source files. The `globs` function is handy here. - -The Usual Commands ------------------- - -**Make sure code compiles and tests pass:**
-Use the `test` goal with the targets you're interested in. If they are -test targets, Pants runs the tests. If they aren't test targets, Pants -will still compile them since it knows it must compile before it can -test. - - :::bash - $ pants goal test src/java/com/myorg/myproject tests/java/com/myorg/myproject - -**Get Help**
-Get the list of goals: - - :::bash - $ ./pants goal goals - -Get help for one goal: - - :::bash - $ ./pants goal help onegoal - -Next ----- - -To learn more about working with Python projects, see the -[[Python Tutorial|pants('examples/src/python/example:readme')]]. - -To learn more about working with Java/JVM projects, see the -[[Java Tutorial|pants('examples/src/java/com/pants/examples:readme')]] - diff --git a/src/docs/index.md b/src/docs/index.md deleted file mode 100644 index 96752f366a1..00000000000 --- a/src/docs/index.md +++ /dev/null @@ -1,70 +0,0 @@ -Welcome to the Pants build system. -================================== - -Pants is a build system for software projects in a variety of languages. -It works particularly well for a source code repository that contains -many distinct projects. - -Getting started using Pants ---------------------------- - -Tutorials and basic concepts. How to use Pants to build things. How to -configure build-able things in BUILD files. - -+ [[Pants Conceptual Overview|pants('src/docs:first_concepts')]] -+ [[First Tutorial|pants('src/docs:first_tutorial')]] -+ [[Target Addresses|pants('src/docs:target_addresses')]] -+ [[JVM Projects|pants('examples/src/java/com/pants/examples:readme')]] -+ [[Python Projects|pants('examples/src/python/example:readme')]] -+ [[README Files and Markdown|pants('examples/src/java/com/pants/examples:page')]] -+ [[Pants Conceptual Overview|pants('src/docs:build_files')]] -+ [[Invoking Pants Build|pants('src/docs:invoking')]] -+ [[Troubleshooting|pants('src/docs:tshoot')]] - -Troubleshooting ---------------- - -+ Something that usually works just failed? See - [[Troubleshooting|pants('src/docs:tshoot')]] -+ Publishing can fail in more ways. See - [[Publishing Artifacts|pants('src/docs:publish')]]. - -Pants Patterns --------------- - -Common Pants build idioms. - -+ [[Third-Party Dependencies|pants('src/docs:3rdparty')]] -+ [[Thrift|pants('examples/src/thrift/com/pants/examples:readme')]] -+ [[Publishing Artifacts|pants('src/docs:publish')]] - -Using Pants With... -------------------- - -+ [[Emacs and Ensime|pants('src/docs:with_emacs')]] -+ [[IntelliJ IDEA|pants('src/docs:with_intellij')]] - -News ----- - -+ [[2014-09-16 Announcement|pants('src/docs:announce_201409')]] - "Hello Pants Build" - -Advanced Documentation ----------------------- - -+ [[Set up your Source Tree for Pants|pants('src/docs:announce_201409')]] -+ [[Installing Pants|pants('src/docs:install')]] - -Pants Reference Documentation ------------------------------ - -+ BUILD Dictionary -+ Goals Reference - -Contributing to Pants ---------------------- - -How to develop Pants itself and contribute your changes. - -+ [[Pants Developer Center|pants('src/python/pants/docs:readme')]] diff --git a/src/docs/install.md b/src/docs/install.md deleted file mode 100644 index 0be495005b1..00000000000 --- a/src/docs/install.md +++ /dev/null @@ -1,61 +0,0 @@ -Installing Pants -================ - -**As of September 2014, alas, Pants is not something you can just -install and use.** To be precise: you can install it, but unless you've -also -[[Set up your code workspace to work with Pants|pants('src/docs:setup_repo')]], -it won't work. You can use it in a workspace in which some Pants expert has -set it up. - -We're fixing this problem, but we're not done yet. - -If want to try out Pants and no Pants expert has set it up for you, you -might try . -( also uses Pants to build, but -there tends to be a lot of "churn".) - -If you're reading this in an organization that already uses Pants, ask -your neighbor where your source code is. - -There are a few ways to get a runnable version of Pants into a -developer's workspace. - -Virtualenv-based Installation ------------------------------ - -[Virtualenv](http://www.virtualenv.org/) is a tool for creating isolated -Python environments. This is the recommended way of installing pants -locally as it does not modify the system Python libraries. - - :::bash - $ virtualenv /tmp/pants - $ source /tmp/pants/bin/activate - $ pip install pantsbuild.pants - $ pants - -To simplify a virtualenv-based installation, add a wrapper script to -your repo. For an example, see the -[`twitter/commons` script `./pants`](https://github.com/twitter/commons/blob/master/pants), -and its helper scripts. - -System-wide Installation ------------------------- - -To install pants for all users on your system: - - :::bash - $ pip install pantsbuild.pants - -This installs pants (and its dependencies) into your Python distribution -site-packages, making it available to all users on your system. This -installation method requires root access and may cause dependency -conflicts with other pip-installed applications. - -PEX-based Installation ----------------------- - -To support hermetic builds and not depend on a local pants installation -(e.g.: CI machines may prohibit software installation), some sites fetch -a pre-build pants.pex whose version is checked-into pants.ini. - diff --git a/src/docs/invoking.md b/src/docs/invoking.md deleted file mode 100644 index 2208ba6cac0..00000000000 --- a/src/docs/invoking.md +++ /dev/null @@ -1,69 +0,0 @@ -Invoking Pants Build -==================== - -This page discusses some advanced features of invoking the Pants build -tool on the command line. We assume you already know the -[[basic command-line structure|pants('src/docs:first_tutorial')]], -something like: - - :::bash - $ ./pants goal test bundle path/to/target path/to/another/target - -For a full description of specifying target addresses, see -[[Target Addresses|pants('src/docs:target_addresses')]]. -For a list of Pants' goals and their options, see the -Goals Reference. - -rc files --------- - -(As of November 2014, rc file support was changing. Expect this -info to be obsolete soon:) - -If there's a command line flag that you always (or nearly always) use, -you might set up a configuration file to ease this. A typical Pants -installation looks for machine-specific settings in `/etc/pantsrc` and -personal settings in `~/.pants.rc`, with personal settings overriding -machine-specific settings. - -For example, suppose that every time you invoke Pants to compile Java -code, you pass flags -`--compile-javac-args=-source --compile-javac-args=7 --compile-javac-args=-target --compile-javac-args=7`. -Instead of passing them on the command line each time, you could set up -a `~/.pants.rc` file: - - [javac] - options: - --compile-javac-args=-source --compile-javac-args=7 - --compile-javac-args=-target --compile-javac-args=7 - -With this configuration, Pants will have these flags on by default. - -`--compile-javac-*` flags go in the `[javac]` section; generally, -`--compile`-*foo*-\* flags go in the `[foo]` section. `--test-junit-*` -flags go in the `[junit]` section; generally, `--test`-*bar*-\* flags go -in the `[bar]` section. `--idea-*` flags go in the `[idea]` section. - -If you know the Pants internals well enough to know the name of a `Task` -class, you can use that class' name as a category to set command-line -options affecting it: - - [pants.tasks.nailgun_task.NailgunTask] - # Don't spawn compilation daemons on this shared build server - options: --no-ng-daemons - -Although `/etc/pantsrc` and `~/.pants.rc` are the typical places for -this configuration, you can check pants.ini \ to find -out what your source tree uses. - - # excerpt from pants.ini - [DEFAULT] - # Look for these rcfiles - they need not exist on the system - rcfiles: ['/etc/pantsrc', '~/.pants.CUSTOM.rc'] # different .rc name! - -In this list, later files override earlier ones. - -These files are formatted as [Python config -files](http://docs.python.org/install/index.html#inst-config-syntax), -parsed by -[ConfigParser](http://docs.python.org/library/configparser.html). diff --git a/src/docs/publish.md b/src/docs/publish.md index 615fd75ef99..32d93e1d836 100644 --- a/src/docs/publish.md +++ b/src/docs/publish.md @@ -1,5 +1,9 @@ Publishing Artifacts ==================== + A library owner/maintainer can *publish* versioned artifacts that folks elsewhere can fetch and import. In the JVM world, these are jars (with @@ -10,7 +14,8 @@ This page talks about publishing artifacts. We assume you already know enough about Pants to *build* the library that underlies an artifact. To *use* an artifact that has already been published from some other source tree, see -[[Third-Party Dependencies|pants('src/docs:3rdparty')]]. + +[Third-Party Dependencies](http://pantsbuild.github.io/3rdparty.html) (To use a artifact that has been published from *your own* source tree... don't do that. Instead, depend on the `*_library` build target.) @@ -72,7 +77,7 @@ Things can go wrong; you can recover: - Uploading the artifact can fail for another reason: that artifact+version already exists on the server. *In theory*, this shouldn't happen: Pants bumps the version it found in the pushdb. - But in practice, this can happen. + But in practice, this can happen. : Exception in thread "main" java.io.IOException: destination file exists and overwrite == false ... @@ -130,8 +135,9 @@ Restricting to "Release Branch" Your organization might have a notion of a special "release branch": you want all publishing to happen on this source control branch, which you maintain extra-carefully. You can -configure your repo so that the -`publish` goal only allows `publish`-ing from this special branch. + +[configure your repo](http://pantsbuild.github.io/setup_repo.html#setup-publish-restrict-branch) so the +`publish` goal only allows `publish`-ing from this special branch. Authenticating to the Artifact Repository ----------------------------------------- @@ -149,7 +155,7 @@ files](http://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-Fil to get a user's username and password on an artifact repository machine. To make this work: -- Each user needs a `~/.netrc` file with a section that looks like: +- Each user needs a `~/.netrc` file with a section that looks like : machine maven.twttr.com login sandy @@ -183,7 +189,7 @@ To make this work: If you need to implement some other kind of authentication, you might look at [the Netrc -implementation](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/authentication/netrc_util.py) +implementation](https://github.com/pantsbuild/pants/blob/master/src/python/pants/authentication/netrc_util.py) and the `credentials` target type for inspiration. Troubleshooting @@ -287,7 +293,8 @@ coordinates from the target's `provides` parameter. Thus, if you try to publish a target that has no `provides`, Pants doesn't try. If the target depends on *other* targets that *do* provide artifacts, Pants might publish those. This is a case of -goal-target mismatch + +[goal-target mismatch](http://pantsbuild.github.io/first_tutorial.html#tut-goal-target-mismatch) To fix this, set `provides` correctly. Want to Publish Something? Publish Many Things @@ -347,6 +354,7 @@ Setting Up Your Workspace ------------------------- To get Pants publish working in the first place, someone needs to -configure your codebase -to register one or more + +[configure your codebase](http://pantsbuild.github.io/setup_repo.html#setup-publish) to register one or more artifact repositories and, optionally, enable some features. + diff --git a/src/docs/setup_repo.md b/src/docs/setup_repo.md deleted file mode 100644 index 93e63e4ab0d..00000000000 --- a/src/docs/setup_repo.md +++ /dev/null @@ -1,254 +0,0 @@ -Set up Your Source Tree for Pants -================================= - -**As of September 2014, this is only possible for Pants experts.** **The -Pants community is actively working to improve it.** If you're setting -up the Pants build tool to work in your source tree, you need to -configure Pants' behavior. (Once it's set up, most folks should be able -[[to use Pants as normal|pants('src/docs:first_concepts')]] -and not worry about these things.) - -Configuring with `pants.ini` ----------------------------- - -Pants Build is very configurable. Your source tree's top-level directory -should contain a `pants.ini` file that sets many, many options. You can -modify a broad range of settings here, including specific binaries to -use in your toolchain, arguments to pass to tools, etc. - -These files are formatted as [Python config -files](http://docs.python.org/install/index.html#inst-config-syntax), -parsed by -[ConfigParser](http://docs.python.org/library/configparser.html). Thus, -they look something like: - - [section] - setting1: value1 - setting2: value2 - -The `[DEFAULT]` section is special: its values are available in other -sections. It's thus handy for defining values that will be used in -several contexts, as in these excerpts that define/use `thrift_workdir`: - - [DEFAULT] - thrift_workdir: %(pants_workdir)s/thrift - - [thrift-gen] - workdir: %(thrift_workdir)s - - [java-compile] - args: [ - '-C-Tnowarnprefixes', '-C%(thrift_workdir)s', - ] - -It's also handy for defining values that are used in several contexts, -since these values will be available in all those contexts. The code -that combines DEFAULT values with others is in Pants' -[base/config.py](https://github.com/pantsbuild/pants/blob/master/src/python/pants/base/config.py). - -Configure Code Layout with `source_root` ----------------------------------------- - -Maybe someday all the world's programmers will agree on the one true -directory structure for source code. Until then, you'll want some -`source_root` -rules to specify which directories hold your code. A -typical programming language has a notion of *base paths* for imports; -you configure pants to tell it those base paths. - -If your project's source tree is laid out for Maven, there's a shortcut -function -`maven_layout` -that configures source roots for Maven's expected -source code tree structure. See -[`testprojects/maven_layout`](https://github.com/pantsbuild/pants/tree/master/testprojects/maven_layout) -for examples of using this style source tree. - -### Organized by Language - -If your top-level `BUILD` file is `top/BUILD` and your main Java code is -in `top/src/java/com/foo/` and your Java tests are in -`top/src/javatest/com/foo/`, then your top-level BUILD file might look -like: - - :::python - # top/BUILD - source_root('src/java') - source_root('src/javatest') - ... - -Pants can optionally enforce that only certain target types are allowed -under each source root: - - :::python - # top/BUILD - source_root('src/java', annotation_processor, doc, jvm_binary, java_library, page) - source_root('src/javatest', doc, java_library, java_tests, page) - ... - -### Organized by Project - -If your top-level `BUILD` file is `top/BUILD` and the Java code for your -Theodore and Hank projects live in `top/theodore/src/java/com/foo/`, -then your top-level BUILD file might not contain any `source_root` -statements. Instead, `theodore/BUILD` and `hank/BUILD` might look like: - - :::python - # top/(project)/BUILD - source_root('src/java') - source_root('src/javatest') - ... - -Or: - - :::python - # top/(project)/BUILD - source_root('src/java', annotation_processor, doc, jvm_binary, java_library, page) - source_root('src/javatest', doc, java_library, java_tests, page) - ... - -#### BUILD.\* and environment-specific config - -When we said BUILD files were named `BUILD`, we really meant `BUILD` or -BUILD.*something*. If you have some rules that make sense for folks in -one environment but not others, you might put them into a separate BUILD -file named BUILD.*something*. - -Top-level BUILD.\* for tree-global config ------------------------------------------ - -When you invoke `./pants goal something src/foo:foo` it processes the -code in src/foo/BUILD and the code in ./BUILD *and* ./BUILD.\*. If you -distribute code to different organizations, and want different -configuration for them, you might put the relevant config code in -./BUILD.something. You can give that file to some people and not-give it -to others. - -Integrate New Tools via a Pants Plugin --------------------------------------- - -Pants knows how to build many things, but maybe you need it to learn a -new tool. Maybe your organization has a custom linter, a custom code -generator, or some other custom tool. Maybe your organization uses a -tool that, while not custom, has not yet been integrated with Pants. - -- If your organization has some custom tools to integrate, - set up a [[Pants plugin|pants('src/python/pants/docs:howto_plugin')]]. -- If you want to integrate with a not-custom tool, you still want to - set up a Pants plugin (or perhaps add abilities to an existing - plugin), but it might make sense to - [[get your changes in upstream|pants('src/python/pants/docs:howto_contribute')]]. - -BUILD.\* in the source tree for special targets ------------------------------------------------ - -If you distribute code to different organizations, you might want to -expose some targets to one organization but not to another. You can do -this by defining those targets in a BUILD.\* file. You can give that -file to some people and not-give it to others. This code will be -processed by people invoking pants on this directory only if they have -the file. - -For example, you might work at the Foo Corporation, which maintains a -fleet of machines to run big test jobs. You might define a humungous -test job as a convenient way to send many many tests to the fleet: - - :::python - # src/javatest/com/foo/BUILD.foo - - # many-many test: Run this on the fleet, not your workstation - # (unless you want to wait a few hours for results) - junit_tests(name='many-many', - dependencies = [ - 'bar:all', - 'baz:all', - 'garply:all', - ]) - -If you don't want to make this test definition available to the public -(lest they complain about how long it takes), you might put this in a -BUILD.foo file and hold back this file when mirroring for the public -repository. - - - -Enabling Pants Publish ----------------------- - -Pants can ease -[["publishing"|pants('src/docs:publish')]]: uploading versioned compiled -artifacts. There are some special things to set up to enable and -customize publishing. - -### Tell Pants about your Artifact Repository - -To tell Pants which artifact repsitory to publish to, - -[[Create a plugin|pants('src/python/pants/docs:howto_plugin')]] -if you haven't already. Register it with Pants. - -In the plugin, define and register a `Repository` in a BUILD file alias -as shown in -[`src/python/internal_backend/repositories/register.py`](https://github.com/pantsbuild/pants/blob/master/src/python/internal_backend/repositories/register.py). - -`BUILD` targets can use this Repository's alias as the `repo` parameter -to an `artifact`. For example, the -`src/java/com/pants/examples/hello/greet/BUILD` refers to the `public` -repostiory defined above. (Notice it's a Python object, not a string.) - - :::python - provides = artifact(org='com.pants.examples', - name='hello-greet', - repo=public,) - -If you get an error that the repo name (here, `public`) isn't defined, -your plugin didn't register with Pants successfully. Make sure you -bootstrap Pants in a way that loads your `register.py`. - -In your config file (usually `pants.ini`), set up a `[jar-publish]` -section. In that section, create a `dict` called `repos`. It should -contain a section for each Repository: - - repos: { - 'public': { # must match the alias above - 'resolver': 'maven.twttr.com', # must match URL above and name - # in ivysettings.xml - 'confs': ['default', 'sources', 'docs', 'changelog'], - # 'auth': 'build-support:netrc', - # 'help': 'Configure your ~/.netrc for artifactory access. - }, - 'testing': { # this key must match the alias name above - 'resolver': 'maven.twttr.com', - 'confs': ['default', 'sources', 'docs', 'changelog'], - # 'auth': 'build-support:netrc', - # 'help': 'Configure your ~/.netrc for artifactory access. - }, - } - - - -### Restricting Publish to "Release Branch" - -Your organization might have a notion of a special "release branch": you -want -[[artifact publishing|pants('src/docs:publish')]] -to happen on this source control -branch, which you maintain extra-carefully. You can set this branch -using the `restrict_push_branches` option of the `[jar-publish]` section -of your config file (usually `pants.ini`). - -### Task to Publish "Extra" Artifacts - -Pants supports "publish plugins", which allow end-users to add -additional, arbitrary files to be published along with the primary -artifact. For example, let's say that along with publishing your jar -full of class files, you would also like to publish a companion file -that contains some metadata -- code coverage info, source git -repository, java version that created the jar, etc. By -[[developing a task|pants('src/python/pants/docs:dev_tasks')]] -in a [[plugin|pants('src/python/pants/docs:howto_plugin')]], -you give Pants a new ability. -[[Develop a Task to Publish "Extra" Artifacts|pants('src/python/pants/docs:dev_tasks_publish_extras')]] -to find out how to -develop a special Task to include "extra" data with published artifacts. - diff --git a/src/docs/target_addresses.md b/src/docs/target_addresses.md deleted file mode 100644 index feeb14ccfe2..00000000000 --- a/src/docs/target_addresses.md +++ /dev/null @@ -1,125 +0,0 @@ -Target Addresses -================ - -Knowing how to specify exactly the targets you need is a critical skill -when using pants. This document describes target addresses and a -multitude of ways to specify them. - -Pants targets ("nouns" of the build) have an `address`, a combination of -the `BUILD` file path and target name. Addresses are used to reference -targets in `BUILD` files, and from the command-line to specify what -targets to perform the given actions on. - -You can specify targets several ways. Some are most useful when writing -`BUILD` targets, while others are useful when invoking pants on the -command-line. Most specify a single target, but globs are available too. - -The following target addresses all specify the same single target. - -- Fully qualified target address is the BUILD file path plus target - name: - - :::bash - $ ./pants goal list examples/src/java/com/pants/examples/hello/main:main - examples/src/java/com/pants/examples/hello/main:main - -- Specify the default target, which matches the parent directory name: - - :::bash - $ ./pants goal list examples/src/java/com/pants/examples/hello/main - examples/src/java/com/pants/examples/hello/main:main - -- Relative paths and trailing forward slashes are ignored on the - command-line to accommodate tab completion: - - :::bash - $ ./pants goal list ./examples/src/java/com/pants/examples/hello/main/ - examples/src/java/com/pants/examples/hello/main:main - - Absolute paths are also allowed to support flexibility in scripting - and command line use: - - :::bash - $ pants goal list $REPO_ROOT/src/java/com/pants/examples/hello/main - src/java/com/pants/examples/hello/main:main - - *NB: Neither the `./` or any other relative or absolute path forms nor - the trailing slash are allowed in target addresses written down in - BUILD files - these affordances are just for ease of command line - specification of target addresses.* - -As a convenience, targets can be referenced relatively within the same -BUILD file: - - :::python - java_library(name='application', ...) - java_library(name='mybird', - dependencies=[':application'], - ) - -To refer to a target in a top-level BUILD file, prefix the target name -with `//:`. (You can prefix any absolute path with //, but it's mainly -useful for top-level targets since ":target" is relative.) - - :::python - java_library(name='application', ...) - java_library(name='mybird', - dependencies=['//:application'], - ) - -Here, `//:application` refers to the application target in the root level -BUILD file and *not* to the application target defined just above -mybird. On the command line you could reference the root level -application target with either of: - -- `$ ./pants goal list :application` - -- `$ ./pants goal list //:application` - -Pants supports two globbing target selectors. These globbing selectors -are provided as a convenience on the command-line. For target -dependencies, explicit target names are used. - -A trailing single colon specifies a glob of targets at the specified -location: - - :::bash - $ ./pants goal list tests/python/pants_test/: - tests/python/pants_test:base-test - tests/python/pants_test:test_maven_layout - tests/python/pants_test:test_thrift_util - tests/python/pants_test:all - -A trailing double colon specifies a recursive glob of targets at the -specified location: - - :::bash - $ ./pants goal list tests/python/pants_test/:: - tests/python/pants_test:base-test - tests/python/pants_test:test_maven_layout - tests/python/pants_test:all - tests/python/pants_test/base:base-test - tests/python/pants_test/base:all - tests/python/pants_test/base:base - ... - tests/python/pants_test/tasks:sorttargets - tests/python/pants_test/tasks:targets_help - tests/python/pants_test/tasks:what_changed - tests/python/pants_test/testutils:testutils - -A leading caret specifies that this target or set of targets should be -excluded from the list of targets: - - :::bash - $ ./pants goal list tests/python/pants_test/:: ^tests/python/pants_test/base:: - tests/python/pants_test:base-test - tests/python/pants_test:test_maven_layout - tests/python/pants_test:all - tests/python/pants_test/cache:cache - tests/python/pants_test/commands:commands - tests/python/pants_test/commands:test_goal - ... - tests/python/pants_test/tasks:sorttargets - tests/python/pants_test/tasks:targets_help - tests/python/pants_test/tasks:what_changed - tests/python/pants_test/testutils:testutils diff --git a/src/docs/tshoot.md b/src/docs/tshoot.md deleted file mode 100644 index cfcac926e41..00000000000 --- a/src/docs/tshoot.md +++ /dev/null @@ -1,74 +0,0 @@ -Troubleshooting -=============== - -Sometimes Pants doesn't do what you hoped. -Pants has good error messages for common errors, but -some messages are not so useful. -(Please report these when you see them. -We want Pants' error messages to be useful.) -Sometimes Pants just plain doesn't work. (Please report these, too.) The -following workarounds can get you up and running again. - - - -Maximum Verbosity ------------------ - -To run a Pants command so that it outputs much much more information to -stdout, you can set some environment variables and pass the `-ldebug` -flag (along with the parameters you meant to pass): - - :::bash - $ PEX_VERBOSE=1 PANTS_VERBOSE=1 PYTHON_VERBOSE=1 ./pants goal -ldebug ... - -This can be especially useful if you're trying to figure out what Pants -was "looking at" when it crashed. - - - -Scrub the Environment ---------------------- - -If you suspect that Pants has cached some corrupt data somewhere, but -don't want to track down exactly what, you can reset your state: - -- **Clean many cached files:** `./build-support/python/clean.sh` -- **Clean more cached files:** If your source tree lives under source - control, use your source control tool to clean up more files. For - example with `git`, you might do something like: - - :::bash - $ git status # look for untracked files - $ git add path/to/file1 path/to/file2 # preserve untracked files you don't want deleted - $ git clean -fdx # delete all untracked files - -- **Stop background processes:** - - :::bash - $ ./pants goal ng-killall --ng-killall-everywhere - -Nailgun 5 seconds ------------------ - -If Pants fails with a stack trace that ends with something like - - :::bash - File "pants/tasks/nailgun_task.py", line 255, in _spawn_nailgun_server - File "pants/tasks/nailgun_task.py", line 226, in _await_nailgun_server - pants.java.nailgun_client.NailgunError: Failed to read ng output after 5 seconds - -Pants uses a program called nailgun to run some JVM jobs. Pants runs -nailgun as a server in the background and then sends requests to it. If -nailgun runs into problems, it might not respond. - -To debug this, look in `./pants.d/ng/*/*`: these files should be named -`stdout` and `stderr`. - -One typical cause behind this symptom: if you removed your machine's Ivy -cache, Pants may try to use symbolic links to files that have gone away. -To recover from this, scrub the environment. - -Questions, Issues, Bug Reports ------------------------------- - -See [[How to Ask|pants('src/python/pants/docs:howto_ask')]] diff --git a/src/docs/with_emacs.md b/src/docs/with_emacs.md deleted file mode 100644 index 365c0856f71..00000000000 --- a/src/docs/with_emacs.md +++ /dev/null @@ -1,20 +0,0 @@ -Emacs With Pants -================ - -Emacs doesn't have much special support for Pants; but if you -also use Scala, you might like: - -Ensime with Pants ------------------ - -Ensime is ENhanced Scala Interaction Mode for GNU Emacs providing code -navigation, type inspection, a lot of things that modern IDE should -support. To install ensime please follow the instructions at - - -To generate ensime project file for `src/java/com/archie/path/to:target` -one just need to run this command: - - :::bash - $ ./pants goal ensime src/java/com/archie/path/to:target - diff --git a/src/docs/with_intellij.md b/src/docs/with_intellij.md deleted file mode 100644 index 20d6d3a3064..00000000000 --- a/src/docs/with_intellij.md +++ /dev/null @@ -1,23 +0,0 @@ -Using Pants with IntelliJ IDEA -============================== - -This page documents how to use Pants with [IntelliJ -IDEA](http://www.jetbrains.com/idea/). (To use IntelliJ to work on Pants -*itself*, see intellij.) - -Pants Build Plugin for IntelliJ -------------------------------- - -For an installation walkthrough and quick demo, see - - -Generating IDEA Project ------------------------ - -To generate an IDEA project for the code in -`src/java/com/archie/path/to:target` and -`src/java/com/archie/another:target`, use the -idea goal: - - :::bash - $ ./pants goal idea src/java/com/archie/path/to:target src/java/com/archie/another:target diff --git a/src/python/pants/docs/BUILD b/src/python/pants/docs/BUILD deleted file mode 100644 index 9c31ba4a5dc..00000000000 --- a/src/python/pants/docs/BUILD +++ /dev/null @@ -1,103 +0,0 @@ -page( - name='readme', - source='readme.md', - dependencies=[ - ':credits', - ':dev_tasks', - ':docs', - ':howto_ask', - ':howto_contribute', - ':howto_customize', - ':howto_develop', - ':howto_plugin', - ':intellij', - ':internals', - ':release', - ] -) - -page( - name='credits', - source='credits.md', -) - -page( - name='dev_tasks', - source='dev_tasks.md', -) - -page( - name='dev_tasks_publish_extras', - source='dev_tasks_publish_extras.md', -) - -page( - name='docs', - source='docs.md', -) - -page( - name='howto_ask', - source='howto_ask.md', -) - -page( - name='howto_contribute', - source='howto_contribute.md', - dependencies=[ - ':docs', - ':howto_ask', - ':howto_develop', - ':internals', - ] -) - -page( - name='howto_customize', - source='howto_customize.md', - dependencies=[ - ':howto_contribute', - ':howto_plugin' - ] -) - -page( - name='howto_develop', - source='howto_develop.md', - dependencies=[ - ':howto_contribute', - 'src/docs:first_tutorial', - ] -) - -page( - name='howto_plugin', - source='howto_plugin.md', - dependencies=[ - ':docs', - ':howto_develop', - ] -) - -page( - name='intellij', - source='intellij.md', - dependencies=[ - 'src/docs:with_intellij', - ]) - -page( - name='internals', - source='internals.md', - dependencies=[ - ':dev_tasks', - 'src/docs:first_concepts', - 'src/python/pants/docs:dev_tasks', - ] -) - -page( - name='release', - source='release.md', -) - diff --git a/src/python/pants/docs/credits.md b/src/python/pants/docs/credits.md deleted file mode 100644 index 8e49371c35e..00000000000 --- a/src/python/pants/docs/credits.md +++ /dev/null @@ -1,22 +0,0 @@ -Credits -======= - -Pants was originally written by John Sirois. - -Major contributors in alphabetical order: - -- Alec Thomas -- Benjy Weinberger -- Bill Farner -- Brian Wickman -- David Buchfuhrer -- Eric Ayers -- Ity Kaul -- John Sirois -- Larry Hosken -- Mark McBride -- Tejal Desai -- Travis Crawford - -If you are a contributor, please add your name to the list! - diff --git a/src/python/pants/docs/dev_tasks.md b/src/python/pants/docs/dev_tasks.md deleted file mode 100644 index 0fdab410eeb..00000000000 --- a/src/python/pants/docs/dev_tasks.md +++ /dev/null @@ -1,168 +0,0 @@ -Task Developer's Guide -====================== - -In Pants, code that does "real build work" (e.g., downloads prebuilt -artifacts, compiles Java code, runs tests) lives in *Tasks*. To add a -feature to Pants so that it can, e.g., compile a new language, you want -to write a new Task. - -This page documents how to develop a Pants Task, enabling you to teach -pants how to do things it does not already know how to do today. To see -the Tasks that are built into Pants, look over -`src/python/pants/backend/*/tasks/*.py`. The code makes more sense if -you know the concepts from internals. The rest of this page introduces -some concepts especially useful when defining a Task. - -Hello Task ----------- - -To implement a Task, you define a subclass of -[pants.backend.core.tasks.task.Task](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/core/tasks/task.py) -and define an `execute` method for that class. The `execute` method does -the work. - -The Task can see (and affect) the state of the build via its `.context` -member, a -[pants.goal.context.Context.](https://github.com/pantsbuild/pants/blob/master/src/python/pants/goal/context.py) - -**Which targets to act on?** A typical Task wants to act on all "in -play" targets that match some predicate. Here, "'in play' targets" means -those targets the user specified on the command line, the targets needed -to build those targets, the targets needed to build *those* targets, -etc. Call `self.context.targets()` to get these. This method takes an -optional parameter, a predicate function; this is useful for filtering -just those targets that match some criteria. - -Task Installation: Associate Task with Goal[s] ----------------------------------------------- - -Defining a Task is nice, but doesn't hook it up so users can get to it. -*Install* a task to make it available to users. To do this, you register -it with Pants, associating it with a goal. A plugin's `register.py` -registers goals in its `register_goals` function. Here's an excerpt from -[Pants' own JVM -backend](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/register.py): - -!inc[start-after=pants/issues/604 register_goals&end-before=Compilation](../backend/jvm/register.py) - -That `task(...)` is a name for -`pants.goal.task_registrar.TaskRegistrar`. Calling its `install` method -installs the task in a goal with the same name. To install a task in -goal `foo`, use `Goal.by_name('foo').install`. You can install more than -one task in a goal; e.g., there are separate tasks to run Java tests and -Python tests; but both are in the `test` goal. - -`product_types` and `require_data`: Why "test" comes after "compile" --------------------------------------------------------------------- - -It might only make sense to run your Task after some other Task has -finished. E.g., Pants has separate tasks to compile Java code and run -Java tests; it only makes sense to run those tests after compiling. To -tell Pants about these inter-task dependencies... - -The "early" task class defines a `product_types` class method that -returns a list of strings: - -!inc[start-after=pants/issues/604 product_types start&end-before=pants/issues/604 product_types finish](../backend/jvm/tasks/ivy_imports.py) - -The "late" task defines a `prepare` method that calls -`round_manager.require_data` to "require" one of those same strings: - -!inc[start-after=pants/issues/604 prep start&end-before=pants/issues/604 prep finish](../backend/codegen/tasks/protobuf_gen.py) - -Pants uses this information to determine which tasks must run frist to -prepare data required by other tasks. (If one task requires data that no -task provides, Pants errors out.) - -Products Map: how one task uses products of another ---------------------------------------------------- - -One task might need the products of another. E.g., the Java test runner -task uses Java `.class` files that the Java compile task produces. Pants -tasks keep track of this in a -[pants.goal.products.ProductMapping.](https://github.com/pantsbuild/pants/blob/master/src/python/pants/goal/products.py) - -The `ProductMapping` is basically a dict. Calling -`self.context.products.get('jar_dependencies')` looks up -`jar_dependencies` in that dict. Tasks can set/change the value stored -at that key; later tasks can read (and perhaps further change) that -value. That value might be, say, a dictionary that maps target specs to -file paths. - -Task Configuration ------------------- - -Tasks may be configured in two ways: - -- a configuration file -- command-line flags - -The configuration file is normally called `pants.ini` and is a standard -`ini` file loaded with `ConfigParser`. During instantiation, tasks have -access to a `pants.base.config.Config` to read these settings. - - :::python - # Let's read mykey from the mytask pants.ini section. - self.context.config.get('mytask', 'mykey') - -To define a command-line flag, handle your Task's `register_options` -class method and call the passed-in `register` function: - -!inc[start-after=ListGoals&end-before=console_output](../backend/core/tasks/list_goals.py) - -Option values are available via `self.get_options()`: - - :::python - # Did user pass in the --all CLI flag (or set it in .ini)? - if self.get_options().all: - -GroupTask ---------- - -Some `Task`s are grouped together under a parent `GroupTask`. -Specifically, the JVM compile tasks: - - :::python - jvm_compile = GroupTask.named( - 'jvm-compilers', - product_type=['classes_by_target', 'classes_by_source'], - flag_namespace=['compile']) - - jvm_compile.add_member(ScalaCompile) - jvm_compile.add_member(AptCompile) - jvm_compile.add_member(JavaCompile) - -A `GroupTask` allows its constituent tasks to 'claim' targets for -processing, and can iterate between those tasks until all work is done. -This allows, e.g., Java code to depend on Scala code which itself -depends on some other Java code. - -JVM Tool Bootstrapping ----------------------- - -If you want to integrate an existing JVM-based tool with a pants task, -Pants must be able to bootstrap it. That is, a running Pants will need -to fetch the tool and create a classpath with which to run it. - -Your job as a task developer is to set up the arguments passed to your -tool (e.g.: source file names to compile) and do something useful after -the tool has run. For example, a code generation tool would identify -targets that own IDL sources, pass those sources as arguments to the -code generator, create targets of the correct type to own generated -sources, and mutate the targets graph rewriting dependencies on targets -owning IDL sources to point at targets that own the generated code. - - -The [Scalastyle](http://www.scalastyle.org/) tool enforces style -policies for scala code. The [Pants Scalastyle -task](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/tasks/scalastyle.py) -shows some useful idioms for JVM tasks. - -- Inherit `NailgunTask` to avoid starting up a new JVM. -- Specify the tool executable as a Pants `jar`; Pants knows how to - download and run those. -- Let organizations/users override the jar in `pants.ini`; it makes it - easy to use/test a new version. - diff --git a/src/python/pants/docs/dev_tasks_publish_extras.md b/src/python/pants/docs/dev_tasks_publish_extras.md deleted file mode 100644 index d0eb18e1ace..00000000000 --- a/src/python/pants/docs/dev_tasks_publish_extras.md +++ /dev/null @@ -1,86 +0,0 @@ -Develop a Task to Publish "Extra" Artifacts -=========================================== - -Pants supports "publish plugins", which allow end-users to add -additional, arbitrary files to be published along with the primary -artifact. For example, let's say that along with publishing your jar -full of class files, you would also like to publish a companion file -that contains some metadata -- code coverage info, source git -repository, java version that created the jar, etc. To accomplish this, -you'll first need to write a custom task, which creates any additional -files (jar or otherwise) that you would like to publish. Next, you'll -create a `publish_extras` section under `[jar-publish]` in pants.ini, -and add a key for the new product type. Your custom task will create the -extra file(s) that you want to publish, and write the path to the -products map under the key that you have defined in pants.ini. The -publishing code will loop over all keys found in pants.ini, and consult -the product map. If pants finds a file for the current key, it will -gather it up, and bundle it in with the rest of the files being -published. - -An example of a custom task is supplied in the -`examples/src/python/example/pants_publish_plugin` directory. To use it, -add the following to your pants.ini: - - [jar-publish] - publish_extras: { - 'extra_test_jar_example': { - 'override_name': '{target_provides_name}-extra_example', - 'classifier': 'classy', - 'extension': 'jar', - }, - } - - [backends] - packages: [ - 'example.pants_publish_plugin', - ] - -In the above configuration example, the string -`'extra_test_jar_example'` is a key into the product map. In this case, -the example task will add additional files for publishing to the product -map under this key. And `jar_publish.py` will examine the product map, -looking for all keys defined here, and publishing any additional files -found. - -Constructing a name for your extra artifact: --------------------------------------------- - -By default, pants uses the following scheme when publishing artifacts: - - [artifactId]-[version](-[classifier]).[ext] - -The pants plugin publishing system allows a customization of the -artifact identifier, classifier, and file extension. To customize the -name of your extra object, you can supply some extra parameters in the -`pants.ini` file: - -> - `override_name` -- allows customization of the name (`artifactId`) -> of the additional file published. Specifying a string will -> completely override the name, or include -> '`{target_provides_name}`' to tack an addition on to the -> pre-existing artifact name. Defaults to the pre-existing artifact -> name. -> - `classifier` -- the maven classifier. Can be any arbitrary string, -> or leave this unspecified for nothing. -> - `extension` -- the filename extension. Defaults to "jar". - -**Note:** You must supply a non-default value for at least one of the -above parameters, otherwise your extra publish artifact won't have a -unique name. With the above config in your pants.ini, invoke pants like -this, to do a test publish: - - WRAPPER_SRCPATH=examples/src/python PANTS_DEV=1 ./pants goal publish examples/src/java/com/pants/examples/hello/greet --no-publish-dryrun --publish-local=~/tmp - -Now if you examine the `/tmp` directory, you'll notice that an extra jar -has been published for the `greet` target: - - :::bash - $ ls -1 /tmp/com/pants/examples/hello-greet/0.0.1-SNAPSHOT/|grep example - hello-greet-extra_example-0.0.1-SNAPSHOT-classy.jar - hello-greet-extra_example-0.0.1-SNAPSHOT-classy.jar.md5 - hello-greet-extra_example-0.0.1-SNAPSHOT-classy.jar.sha1 - -This example task should provide a rough starting guide, and can be -tailored to suit a more specific situation. - diff --git a/src/python/pants/docs/docs.md b/src/python/pants/docs/docs.md deleted file mode 100644 index 091f62fbb89..00000000000 --- a/src/python/pants/docs/docs.md +++ /dev/null @@ -1,86 +0,0 @@ -About the documentation -======================= - -Pants documentation is generated from markdown sources and some -reflection code by Pants itself. - -Generating the site -------------------- - -A script encapsulates all that needs to be done. To generate and preview -changes, just: - - :::bash - # This publishes the docs **locally** and opens (-o) them in your browser for review - ./build-support/bin/publish_docs.sh -o - -Publishing the site -------------------- - -Use the same script as for generating the site, but request it also be -published. Don't worry - you'll get a chance to abort the publish just -before its comitted remotely: - - :::bash - # This publishes the docs locally and opens (-o) them in your browser for review - # and then prompts you to confirm you want to publish these docs remotely before - # proceeding to publish to http://pantsbuild.github.io - ./build-support/bin/publish_docs.sh -op - -If you'd like to publish remotely for others to preview your changes -easily, there is a -d option that will create a copy of the site in a -subdir of : - - # This publishes the docs locally and opens (-o) them in your browser for review - # and then prompts you to confirm you want to publish these docs remotely before - # proceeding to publish to http://pantsbuild.github.io/sirois-test-site - ./build-support/bin/publish_docs.sh -opd sirois-test-site - -X-References ------------- - -If your doc has a -link like `java_library`, it links to -the BUILD Dictionary entry for `java_library`. To set up a short-hand -link like this... - -Define the destination of the link with an `xmark` anchor, e.g., -` `. The `xmark` attribute (here, -`bdict_java_library`) must be unique within the doc set. - -Link to the destination with an `xref`, e.g., -`java_library`. - -Doc Site Config ---------------- - -The site generator -takes "raw" `.html` files, "wraps" them in a template with some -navigation UI, and writes out the resulting `.html` files. - -You configure this with `src/python/pants/docs/docsite.json`: - -`sources`:
-Map of pages to the `.html` files they're generated from. E.g., -`"build_dictionary": "dist/builddict/build_dictionary.html",` means to -generate the site's /build\_dictionary.html page, the site generator -should get the "raw" file `dist/builddict/build_dictionary.html` and -apply the template to it. - -`tree`:
-Outline structure of the site. Each node of the tree is a dict. Each -node-dict can have a `page`, a page defined in `sources` above. Each -node-dict can have a `children`, a list of more nodes. - -`template`:
-Path to mustache template to apply to each page. - -`extras`:
-Map of "extra" files to copy over. Handy for graphics, stylesheets, and -such. - -`outdir`:
-Path to which to write the generated site. - -To add a page and have it show up in the side navigation UI, add the -page to the `sources` dict and to the `tree` hierarchy. diff --git a/src/python/pants/docs/docsite.css b/src/python/pants/docs/docsite.css index a59b5824565..40629e8f055 100644 --- a/src/python/pants/docs/docsite.css +++ b/src/python/pants/docs/docsite.css @@ -5,7 +5,7 @@ body { - background-color: #e5e7f3; + background-color: #eee; color: #111; font-family: sans-serif; } @@ -112,14 +112,14 @@ form.search input.query { form.search .button { width: 20%; padding: 0.5em 1em; - background-color: #e5e7f3; + background-color: #eee; color: #258; border: 1px solid #147; border-radius: 2px; } nav.bignav .site-toc { - background-color: #e5e7f3; + background-color: #eee; } .site-toc ul { @@ -136,7 +136,7 @@ nav.bignav .site-toc { color: white; } .site-toc li.toc-here:hover { - background-color: #e5e7f3; + background-color: #eee; color: black; } .site-toc li:hover a { @@ -169,7 +169,6 @@ nav.pagetoc { float: right; margin: 0px -1em 0px 1em; padding-left: 1em; - padding-right: 1em; background-color: #ddd; max-width: 24%; border: 3px solid #38c; @@ -181,6 +180,12 @@ nav.pagetoc ul { list-style-type: none; } +nav.pagetoc li { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + nav.pagetoc li:hover { background-color: black; color: white; diff --git a/src/python/pants/docs/docsite.json b/src/python/pants/docs/docsite.json index 8c54755a239..3e2cbbad7b7 100644 --- a/src/python/pants/docs/docsite.json +++ b/src/python/pants/docs/docsite.json @@ -1,53 +1,15 @@ { "sources": { - "3rdparty": "dist/markdown/html/src/docs/3rdparty.html", - "3rdparty_jvm": "dist/markdown/html/examples/src/java/com/pants/examples/3rdparty_jvm.html", - "3rdparty_py": "dist/markdown/html/examples/src/python/example/3rdparty_py.html", - "announce_201409": "dist/markdown/html/src/docs/announce_201409.html", "build_dictionary": "dist/builddict/build_dictionary.html", "build_files": "dist/markdown/html/src/docs/build_files.html", - "credits": "dist/markdown/html/src/python/pants/docs/credits.html", - "dev": "dist/markdown/html/src/python/pants/docs/readme.html", - "dev_tasks": "dist/markdown/html/src/python/pants/docs/dev_tasks.html", - "dev_tasks_publish_extras": "dist/markdown/html/src/python/pants/docs/dev_tasks_publish_extras.html", - "docs": "dist/markdown/html/src/python/pants/docs/docs.html", - "first_concepts": "dist/markdown/html/src/docs/first_concepts.html", - "first_tutorial": "dist/markdown/html/src/docs/first_tutorial.html", - "from_maven": "dist/markdown/html/examples/src/java/com/pants/examples/from_maven.html", "goals_reference": "dist/builddict/goals_reference.html", - "howto_ask": "dist/markdown/html/src/python/pants/docs/howto_ask.html", - "howto_contribute": "dist/markdown/html/src/python/pants/docs/howto_contribute.html", - "howto_customize": "dist/markdown/html/src/python/pants/docs/howto_customize.html", - "howto_develop": "dist/markdown/html/src/python/pants/docs/howto_develop.html", - "howto_plugin": "dist/markdown/html/src/python/pants/docs/howto_plugin.html", - "index": "dist/markdown/html/src/docs/index.html", - "install": "dist/markdown/html/src/docs/install.html", - "intellij": "dist/markdown/html/src/python/pants/docs/intellij.html", - "internals": "dist/markdown/html/src/python/pants/docs/internals.html", - "invoking": "dist/markdown/html/src/docs/invoking.html", + "index": "dist/markdown/html/README.html", "JVMProjects": "dist/markdown/html/examples/src/java/com/pants/examples/readme.html", - "page": "dist/markdown/html/examples/src/java/com/pants/examples/page.html", - "pex_design": "dist/markdown/html/examples/src/python/example/pex_design.html", "publish": "dist/markdown/html/src/docs/publish.html", "python_old": "dist/markdown/html/examples/src/python/example/python_old.html", - "python-readme": "dist/markdown/html/examples/src/python/example/readme.html", - "release": "dist/markdown/html/src/python/pants/docs/release.html", - "setup_repo": "dist/markdown/html/src/docs/setup_repo.html", - "target_addresses": "dist/markdown/html/src/docs/target_addresses.html", - "tshoot": "dist/markdown/html/src/docs/tshoot.html", - "ThriftDeps": "dist/markdown/html/examples/src/thrift/com/pants/examples/readme.html", - "with_emacs": "dist/markdown/html/src/docs/with_emacs.html", - "with_intellij": "dist/markdown/html/src/docs/with_intellij.html" + "python-readme": "dist/markdown/html/examples/src/python/example/readme.html" }, "extras": { - "images/intellij-configure-tests.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-new-project-1.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-new-project-2.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-new-pythonsdk.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-project-structure-modules-sources.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-project-structure-project.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-run.png": "src/python/pants/docs/images/intellij-configure-tests.png", - "images/intellij-select-venv.png": "src/python/pants/docs/images/intellij-configure-tests.png", "logo.ico": "src/python/pants/docs/pants-logo.ico", "s.js": "src/python/pants/docs/docsite.js", "s.css": "src/python/pants/docs/docsite.css" @@ -55,58 +17,16 @@ "tree": [ { "page": "index", "children": [ - { "page": "first_concepts" }, - { "page": "first_tutorial" }, - { "page": "target_addresses" }, - { "page": "JVMProjects", - "children": [ - { "page": "3rdparty_jvm" }, - { "page": "from_maven" } - ] - }, + { "page": "JVMProjects" }, { "page": "python-readme", "children": [ - { "page": "3rdparty_py" }, - { "page": "pex_design" }, { "page": "python_old" } ] }, - { "page": "page" }, { "page": "build_files" }, - { "page": "invoking" }, - { "page": "tshoot" }, - { "page": "3rdparty", - "children": [ - { "page": "3rdparty_jvm" }, - { "page": "3rdparty_py" } - ] - }, - { "page": "ThriftDeps" }, { "page": "publish" }, - { "page": "with_emacs" }, - { "page": "with_intellij" }, - { "page": "install" }, - { "page": "setup_repo" }, { "page": "build_dictionary" }, - { "page": "goals_reference" }, - { "page": "dev", - "children": [ - { "page": "howto_contribute" }, - { "page": "howto_ask" }, - { "page": "howto_develop" }, - { "page": "dev_tasks", - "children": [ - { "page": "dev_tasks_publish_extras" } - ] - }, - { "page": "howto_plugin" }, - { "page": "howto_customize" }, - { "page": "intellij" }, - { "page": "internals" }, - { "page": "release" }, - { "page": "docs" }, - { "page": "credits" } - ]} + { "page": "goals_reference" } ] } ], diff --git a/src/python/pants/docs/howto_ask.md b/src/python/pants/docs/howto_ask.md deleted file mode 100644 index 92d92c0d6e0..00000000000 --- a/src/python/pants/docs/howto_ask.md +++ /dev/null @@ -1,34 +0,0 @@ -How To Ask -========== - -Sometimes you have a question about Pants' behavior. Sometimes you want -to report a Pants bug. You want to tell other people about something -strange that Pants is doing. Since those people can't look over your -shoulder, you must provide details about your environment and Pants' -behavior. - -- What command[s] did you enter? -- What was the output? -- What output did you expect instead? -- If you are on the `pantsbuild/pants` git repo, what is your branch - and sha? Please report the output of these commands: - - :::bash - $ git branch - $ git log -1 - -- If you're on your own branch, push it to origin so that other - people can see it and perhaps try to reproduce the behavior. (If - you're using `pantsbuild/pants` itself and don't have permission - to push to that repo, you might need to clone the repo first.) - Please report the branch's location, as shown by the output of - these commands (where myusername\_myproblem is a branch name you - choose): - - :::bash - $ git push origin myusername_myproblem - $ git remote -v | grep origin - -**Post this information** as a new thread on the [pants-devel Google -Group](https://groups.google.com/forum/#!forum/pants-devel) - diff --git a/src/python/pants/docs/howto_contribute.md b/src/python/pants/docs/howto_contribute.md deleted file mode 100644 index 4555d956e55..00000000000 --- a/src/python/pants/docs/howto_contribute.md +++ /dev/null @@ -1,220 +0,0 @@ -Pants Contributors Guide -======================== - -This page documents how to make contributions to Pants. If you've -[[developed a change to Pants|pants('src/python/pants/docs:howto_develop')]], -it passes all tests, and -you'd like to "send it upstream", here's what to do: - -Questions, Issues, Bug Reports ------------------------------- - -See [[How To Ask|pants('src/python/pants/docs:howto_ask')]] - -Join the Conversation ---------------------- - -Join the [pants-devel Google -Group](https://groups.google.com/forum/#!forum/pants-devel) to keep in -touch with other pants developers. - -Join the [pants-reviews Google -Group](https://groups.google.com/forum/#!forum/pants-reviews) to see -many code reviews. - -Watch the [pantsbuild/pants Github -project](https://github.com/pantsbuild/pants) for notifications of new -issues, etc. - -Follow [@pantsbuild on Twitter](https://twitter.com/pantsbuild) for -occasional announcements. - -Find out when the CI tests go red/green by adding your email address to -[.travis.yml](https://github.com/pantsbuild/pants/blob/master/.travis.yml). - -Life of a Change ----------------- - -Let's walk through the process of making a change to pants. At a high -level we'll do the following: - -- Identify the change you'd like to make (e.g.: fix a bug, add a - feature). -- Get the code. -- Make your change on a branch. -- Get a code review. -- Commit your change to master. - -Overall it's quite straightforward. Please note - despite being hosted -on GitHub - we do not use just pull requests. This is because we prefer a linear -commit history and doing code reviews with Review Board. - -### Identify the change - -It's a good idea to make sure the work you'll be embarking on is -generally agreed to be in a useful direction for the project before -getting too far along. - -If there is a pre-existing github issue filed and un-assigned, feel free -to grab it and ask any clarifying questions needed on -[pants-devel](https://groups.google.com/forum/#!forum/pants-devel). If -there is an issue you'd like to work on that's assigned and stagnant, -please ping the assignee and finally -[pants-devel](https://groups.google.com/forum/#!forum/pants-devel) -before taking over ownership for the issue. - -If you have an idea for new work that's not yet been discussed on -[pants-devel](https://groups.google.com/forum/#!forum/pants-devel), then -start a conversation there to vet the proposal. Once the group agrees -it's worth a spike you can file a github issue and assign it to -yourself. - - - -### Getting Pants Source Code - -If you just want to compile and look at the source code, the easiest way -is to clone the repo. - - :::bash - $ git clone https://github.com/pantsbuild/pants - -If you would like to start developing patches and contributing them -back, you will want to create a fork of the repo using the [instructions -on github.com](https://help.github.com/articles/fork-a-repo/). With this -setup, you can push branches and run Travis-CI before your change is -committed. - -If you've already cloned your repo without forking, you don't have to -re-checkout your repo. First, create the fork on github. Make a note the -clone url of your fork. Then run the following commands: - - :::bash - $ git remote remove origin - $ git remote add origin - $ git remote add upstream https://github.com/pantsbuild/pants - -After this change, `git push` and `git pull` will go to your fork. You -can get the latest changes from the `pantsbuild/pants` repo's master -branch using the [syncing a -fork](https://help.github.com/articles/syncing-a-fork/) instructions on -github. - -**Pro tip:** If you want your local master branch to be an exact copy of -the `pantsbuild/pants` repo's master branch, use these commands: - - :::bash - $ git co master - $ git fetch upstream - $ git reset --hard upstream/master - -### Making the Change - -You might want to familiarize yourself with the -[[Pants Internals|pants('src/python/pants/docs:internals')]] or -[[Pants Developers Guide|pants('src/python/pants/docs:howto_develop')]]. - -Create a new branch off master and make changes. - - :::bash - $ git checkout -b $FEATURE_BRANCH - -Does your change alter Pants' behavior in a way users will notice? If -so, then along with changing code... - -+ Update `src/python/pants/CHANGELOG.rst`. -+ Consider updating the - [[user documentation|pants('src/python/pants/docs:docs')]]. - -### Run the CI Tests - -Before posting a review but certainly before the branch ships you should -run relevant tests. If you're not sure what those are, -run all the tests. - -### Code Review - -Now that your change is complete, we'll post it for review. We use - to host code reviews. - -#### Posting the First Draft - -**Before posting your first review,** you must create an account at - . To create one, visit - and click "Create one now." - -To set up local tools, run `./rbt help`. (`./rbt` is a wrapper around -the usual RBTools [rbt](http://www.reviewboard.org/docs/rbtools/dev/) -script.) The first time this runs it will bootstrap: you'll see a lot of -building info. - -Post your change for review: - - :::bash - $ ./rbt post -o -g - -The first time you `post`, rbt asks you to log in. Subsequent runs use -your cached login credentials. - -This `post` creates a new review, but does not yet publish it. - -At the provided URL, there's a web form. To get your change reviewed, -you must fill in the change description, reviewers, testing done, etc. -To make sure it gets seen, add `pants-reviews` to the Groups field and a -specific reviewer to the People field. If you have created a -pull request on github to run Travis-CI, -put the pull -request number in the Bug field and your git branch name in the Branch -field. - -When the review looks good, publish it. An email will be sent to the -`pants-reviews` mailing list and the reviewers will take a look. (For -your first review, double-check that the mail got sent; rbcommons tries -to "spoof" mail from you and it doesn't work for everybody's email -address. If your address doesn't work, you might want to use another -one.) - -#### Iterating - -If reviewers have feedback, there might be a few iterations before -finally getting a Ship It. As reviewers enter feedback, the rbcommons -page updates; it should also send you mail (but sometimes its "spoof" -fails). - -If those reviews inspire you to change some code, great. Change some -code, commit locally. To update the code review with the new diff where -`` is a review number like `123`: - - :::bash - $ ./rbt post -o -r - -Look over the fields in the web form; perhaps some could use updating. -Press the web form's Publish button. - -### Commit Your Change - -At this point you've made a change, had it reviewed and are ready to -complete things by getting your change in master. (If you're not a -committer, please ask one to do this section for you.) - - :::bash - $ cd /path/to/pants/repo - $ ./build-support/bin/ci.sh - $ git checkout master - $ git pull - $ ./rbt patch -c - -Here, ensure that the commit message generated from the review summary -is accurate, and that the resulting commit contains the changes you -expect. (If `rbt` gives mysterious errors, pass `--debug` for more info. -If that doesn't clarify the problem, mail pants-devel (and include that -`--debug` output).) - -Finally, - - :::bash - $ git push origin master - -The very last step is closing the review as "Submitted". The change is -now complete. Huzzah! - diff --git a/src/python/pants/docs/howto_customize.md b/src/python/pants/docs/howto_customize.md deleted file mode 100644 index 1821fae3cdf..00000000000 --- a/src/python/pants/docs/howto_customize.md +++ /dev/null @@ -1,19 +0,0 @@ -Custom Pants for Your Site -========================== - -*As of September 2014, this process is new and still evolving;* *expect -it to change plenty.* - -You might want to tailor Pants' behavior for your organization by -defining some special functionality. If your changes are -generally-useful, you could -[[add them to Pants proper|pants('src/python/pants/docs:howto_contribute')]] -But perhaps your -organization has, e.g., some custom code generation that wouldn't be of -use to other organizations. You want to add some functionality to Pants -in a maintainable way. - -At a high level, you want to -[[develop a Pants plugin|pants('src/python/pants/docs:howto_plugin')]] -and create an altered Pants that registers your Plugin in addition to -the regular ones. diff --git a/src/python/pants/docs/howto_develop.md b/src/python/pants/docs/howto_develop.md deleted file mode 100644 index 9069d0e0e7b..00000000000 --- a/src/python/pants/docs/howto_develop.md +++ /dev/null @@ -1,241 +0,0 @@ -Pants Developers Guide -====================== - -This page describes the developer workflow when changing Pants itself. -(If you wanted instructions for using Pants to develop other programs, -please see [[First Tutorial|pants('src/docs:first_tutorial')]].) - -These instructions assume you've already -downloaded the source code. - -Running from sources --------------------- - -As pants is implemented in python it can be run directly from sources. - - $ PANTS_DEV=1 ./pants goal goals - *** running pants in dev mode from ./src/python/pants/bin/pants_exe.py *** - - -Notice this invocation specifies the `PANTS_DEV` environment variable. -By defining `PANTS_DEV` pants will be run from sources. - -Building a Pants PEX for Testing --------------------------------- - -The `./pants` wrapper provides a convenient way to produce a `.pex` file -for testing pants on your local workstation. If you call it without the -`PANTS_DEV=1` environment described above, it - -> - Checks the source tree's top directory for a `pants.pex` and runs -> it if it exists. Otherwise `./pants`... -> - Builds a new `pants.pex`, moves it to the source tree's top -> directory, and runs that. - -It looks something like: - - $ rm pants.pex - $ ./pants goal my-new-feature - Building pants.pex to /Users/zundel/Src/Pants/pants.pex... - ... - Build operating on top level addresses: set([BuildFileAddress(/Users/pantsdev/Src/pants/src/python/pants/bin/BUILD, pants_local_binary)]) - Building PythonBinary PythonBinary(BuildFileAddress(/Users/pantsdev/Src/pants/src/python/pants/bin/BUILD, pants_local_binary)): - Wrote /Users/pantsdev/Src/pants/dist/pants_local_binary.pex - /Users/pantsdev/Src/Pants/dist/pants_local_binary.pex -> /Users/pantsdev/Src/Pants/pants.pex - AMAZING NEW FEATURE PRINTS HERE - $ ls pants.pex # gets moved here, though originally "Wrote" to ./dist/ - pants.pex - $ ./pants goal my-new-feature - AMAZING NEW FEATURE PRINTS HERE - -Using `./pants` to launch Pants thus gives a handy workflow: generate -`pants.pex`. Go back and forth between trying the generated `pants.pex` -and fixing source code as inspired by its misbehaviors. When the fixed -source code is in a consistent state, remove `pants.pex` so that it will -get replaced on the next `./pants` run. - -Building a Pants PEX for Production ------------------------------------ - -Most of the time, you will probably want to use an official published -version of pants. But what if you want to let some of your internal -users try out the latest and greatest unreleased code? What if you want -to create a custom build of pants with some unpublished patches? In that -case, you want to build a production ready version of pants including -dependencies for all platforms, not just your development environment. - -The following command will create a locally built `pants.pex` for all -platforms: - - $ ./pants goal binary src/python/pants/bin:pants - ... - SUCCESS - -The resulting `pants.pex` will be in the `dist/` directory: - - $ ls -l dist/pants.pex - -rwxr-xr-x 1 pantsdev pantsdev 5561254 Oct 8 09:52 dist/pants.pex - -You can see that the pex contains bundled dependencies for both mac and -linux: - - $ unzip -l dist/pants.pex | grep -e 'macos\|linux' - -You can distribute the resulting `pants.pex` file to your users via your -favorite method. A user can just copy this pex to the top of their Pants -workspace and use it: - - $ cp /mnt/fd0/pants.pex . - $ ./pants.pex goal test examples/tests/java/com/pants/examples/hello/greet: - -There are some parameters in `src/python/pants/bin/BUILD` that you may -want to tweak for your production distribution. For example, you may -want to update the list of supported platforms: - - platforms=[ - 'linux-x86_64', - 'macosx-10.4-x86_64', - -> ], - -Or you may want to force the Python interpreter to be a specific -version: - - PANTS_COMPATIBILITY = 'CPython>=2.7,<2.8' - -Testing -------- - - - -### Running Tests - -Pants has many tests. There are BUILD targets to run those tests. We try -to keep them passing. [A Travis-CI -job](https://travis-ci.org/pantsbuild/pants) runs tests on each sha -pushed to origin on `github.com/pantsbuild/pants`. - -Most test are runnable as regular Pants test targets. To find tests that -work with a particular feature, you might explore -`tests/python/pants_tests/.../BUILD`. - -Typically, you're not sure precisely which tests you need to run, so you -run all of them. - -To run all tests, - - ./pants goal test tests:: - -To run just Pants' *unit* tests (skipping the can-be-slow integration -tests), use the `tests/python/pants_test:all` target: - - ./pants goal test tests/python/pants_test:all - -To bring up the `pdb` debugger when Python tests fail, pass the `--pdb` -flag. - -Before -[[contributing a change to Pants|pants('src/python/pants/docs:howto_contribute')]], -make sure -it passes **all** of our continuous integration (CI) tests: everything -builds, all tests pass. To try all the CI tests in a few configurations, -you can run the same script that our Travis CI does. This can take a -while, but it's a good idea to run it before you contribute a change or -merge it to master: - - :::bash - $ ./build-support/bin/ci.sh - -You can run your code through the Travis-CI before you submit a change. -Travis-CI is integrated with the pull requests for the -`pantsbuild/pants` repo. Travis-CI will test it soon after the pull -request is created. It will queue up a new job every time you -subsequently push your branch. - -To kick off a new CI-build, push a branch to -your fork of `pantsbuild/pants`. -Create a pull -request on the `pantsbuild/pants` -[repo](https://github.com/pantsbuild/pants), not your fork. If you are -posting a review request, put the pull request number into the Bug -field. Then, when you close the request, you can navigate from the bug -number to easily close the pull request. - -Debugging ---------- - -To run Pants under `pdb` and set a breakpoint, you can typically add - - import pdb; pdb.set_trace() - -...where you first want to break. If the code is in a test, instead use -: - - import pytest; pytest.set_trace() - -To run tests and bring up `pdb` for failing tests, you can instead pass -`--pdb`: - - $ ./pants tests/python/pants_test/tasks: --pdb - ... plenty of test output ... - tests/python/pants_test/tasks/test_targets_help.py E - >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> traceback >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - - cls = - - @classmethod - def setUpClass(cls): - > super(TargetsHelpTest, cls).setUpClass() - E AttributeError: 'super' object has no attribute 'setUpClass' - - tests/python/pants_test/tasks/test_targets_help.py:24: AttributeError - >>>>>>>>>>>>>>>>>>>>>>>>>>>>> entering PDB >>>>>>>>>>>>>>>>>>>>>>>>>>>>> - > /Users/lhosken/workspace/pants/tests/python/pants_test/tasks/test_targets_help.py(24)setUpClass() - -> super(TargetsHelpTest, cls).setUpClass() - (Pdb) - -Debug quickly; that test target will time out in a couple of minutes, -quitting you out. - -To start an interactive Python shell that can `import` Pants modules, -use the usual `./pants py` on a `python_library` target that builds (or -depends on) the modules you want: - - $ ./pants py src/python/pants/backend/core/targets:common - /Users/lhosken/workspace/pants src/python/pants/backend/core/targets:common - Python 2.6.8 (unknown, Mar 9 2014, 22:16:00) - [GCC 4.2.1 Compatible Apple LLVM 5.0 (clang-500.0.68)] on darwin - Type "help", "copyright", "credits" or "license" for more information. - (InteractiveConsole) - >>> from pants.backend.core.targets import repository - >>> - -Debugging a JVM Tool --------------------- - -Some Pants tools are imported as external JVM dependencies. If you need -to debug one of these tools and change code, see -Using a SNAPSHOT JVM Dependency -which describes how to specify the `url` and `mutable` attributes of a `jar` -dependency found on the local filesystem: - - :::python - jar_library(name='jmake', - jars=[ - jar(org='com.sun.tools', name='jmake', rev='1.3.8-4-SNAPSHOT', - url='file://squarepants/lib/jmake.jar', mutable=True), - ], - ) - -Append JVM args to turn on the debugger for the appropriate tool in -`pants.ini`: - - [jar-tool] - jvm_args: ['-Xmx300m', '-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=%(debug_port)s'] - -Note that some tools run under nailgun by default. The easiest way to -debug them is to disable nailgun by specifying the command line option -`--no-ng-daemons`. If you need to debug the tool under nailgun, make -sure you run `pants goal ng-killall` or `pants goal clean-all` so that -any running nailgun servers are restarted. - diff --git a/src/python/pants/docs/howto_plugin.md b/src/python/pants/docs/howto_plugin.md deleted file mode 100644 index af781d38690..00000000000 --- a/src/python/pants/docs/howto_plugin.md +++ /dev/null @@ -1,23 +0,0 @@ -Developing a Pants Plugin -========================= - -*As of September 2014, this process is new and still evolving;* *expect -it to change somewhat.* - -This page documents how to develop a Pants plugin, a set of code that -defines new Pants functionality. If you -[[develop a new task|pants('src/python/pants/docs:dev_tasks')]] -or target to add to Pants (or to -override an existing part of Pants), a plugin gives you a way to -register your code with Pants. - -Much of Pants' own functionality is organized in plugins; see them in -`src/python/pants/backend/*`. - -A plugin registers its functionality with Pants by defining some -functions in a `register.py` file in its top directory. For example, -Pants' `jvm` code registers in -[src/python/pants/backend/jvm/register.py](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/register.py) -Pants' backend-loader code assumes your plugin has a `register.py` file -there. - diff --git a/src/python/pants/docs/intellij.md b/src/python/pants/docs/intellij.md deleted file mode 100644 index e243fbba0d2..00000000000 --- a/src/python/pants/docs/intellij.md +++ /dev/null @@ -1,112 +0,0 @@ -Pants Development with IntelliJ IDEA -==================================== - -This page documents how to develop pants with [IntelliJ -IDEA](http://www.jetbrains.com/idea/). (To use IntelliJ with Pants, not -necessarily to develop Pants itself, see -[[Using Pants with IntelliJ IDEA|pants('src/docs:with_intellij')]]. - -IntelliJ Setup --------------- - -As pants is a python application, the "Ultimate" (aka paid-for) edition -of IntelliJ is required, as is the Python plugin. You'll need to: - -- Download "IntelliJ IDEA Ultimate Edition" from - . -- Within IntelliJ, install the Python plugin. - -Project Setup -------------- - -While Pants can generate IntelliJ IDEA projects for Java/Scala targets, -it cannot yet generate projects for Python targets. For this reason you -must manually create the project. This section walks you through that -process using IntelliJ IDEA 13.1.2. - -First you need to bootstrap pants in developer mode. This is generally -the way you want to run pants when iterating and it also prepares a -virtual environment suitable for IDEs: - - :::bash - $ PANTS_DEV=1 ./pants - -Next open IntelliJ and select "Create New Project". - -![image](images/intellij-new-project-1.png) - -In the "New Project" window, select "Python". - -![image](images/intellij-new-project-2.png) - -Then skip past the project templates screen and land at the Python -interpreter configuration screen. Click "Configure..." and add a "Python -SDK". - -![image](images/intellij-new-pythonsdk.png) - -This will be a "local" interpreter and you'll need to select the virtual -environment bootstrapped above; it's in -build-support/pants\_dev\_deps.venv (not -build-support/pants\_deps.venv). This is **important** because the dev -virtual environment is designed to have 3rdparty dependencies that IDEs -can handle. - -![image](images/intellij-select-venv.png) - -Next specify a "Project name" and "Project location". - -Now open the "File -\> Project Structure" window. For the "Project SDK" -choice, specify the python interpreter you configured above. - -![image](images/intellij-project-structure-project.png) - -In the "Modules" section, you need to mark "Sources" and "Tests". This -establishes the loose python source roots to add to the PYTHONPATH - the -parent directory of what you'll import. Click the "+ Add Content Root" -button on the right, and choose the directory where you cloned pants. -Next, mark the src/python directory as sources and tests/python -directory as test\_sources in this new content root. IntelliJ may -auto-select some sources and tests in directories such as src/java --you -may wish to unselect these, as these are example files used for tests -and/or documentation. - -![image](images/intellij-project-structure-modules-sources.png) - -Finally in "File -\> Settings -\> Python Integrated Tools" (or "IntelliJ -Idea -\> Preferences -\> Python Integrated Tools" if you are on Mac OS -X) set the default test runner to py.test - which is what pants uses to -drive python tests. - -![image](images/intellij-configure-tests.png) - -Now your project setup is complete! - -Running Pants within IntelliJ IDEA ----------------------------------- - -In addition to editing pants code in IntelliJ, pants itself can be -run/debugged from within the editor. This is particularly useful for -fast iteration on the pants code base -- both within the pants repo, and -running the development version of pants from sources against a -different repo. - -Open the "Run -\> Edit Configurations..." dialog box. - -- Add a new Python configuration. -- Set the "Script" to the location of your - src/python/pants/bin/pants\_exe.py file (example: - /home/jsirois/dev-pants/src/python/pants/bin/pants\_exe.py) -- Set the "Script parameters" to your pants command-line args, such as - goal goals. -- Set the "Working directory" to where you want to run pants from. - Note this could be an entirely different repo from where the pants - source code lives. This is very useful for making a pants change and - testing in the repo where you use pants. - -![image](images/intellij-run.png) - -After creating the run configuration, simply run or debug pants from -within the editor. This will allow you to use all the regular Python -IntelliJ features to speed your pants development. - diff --git a/src/python/pants/docs/internals.md b/src/python/pants/docs/internals.md deleted file mode 100644 index 12230f0539d..00000000000 --- a/src/python/pants/docs/internals.md +++ /dev/null @@ -1,157 +0,0 @@ -Pants Internals -=============== - -Pants is a build tool. This document describes some of its internals, -concentrating on how to tailor Pants to your needs, such as integrating -it with other tools. - -If you want to run Pants or to write BUILD files, you probably want the -[[Pants Conceptual Overview|pants('src/docs:first_concepts')]] instead. -But if you want to support a new tool or a new language, read on. - -How Some Base Classes Interrelate ---------------------------------- - -**Target**
-An addressable thing, such as a `python_binary` or `scala_tests`. To add -support for a new language, you probably want to add new target types to -represent things you can build with that language. Most Target instances -can depend on other Target instances. As a rule of thumb, if code in -Target \_A\_ imports code in Target *B*, then *A* depends on *B*. If *A* -depends on *B*, then when carrying out some goal on *A*, you can be sure -that goal has been carried out on *B*. - - -**Goal**
-A build verb, such as compile or test. Internally, a goal is implemented -as a set of Tasks. - -**Task**
-A Goal has one or more Tasks, which do the actual work of invoking -tools. A compile Goal, for example, could contain a Task for Java -compilation, a Task for Scala compilation, etc. If you want an existing -Goal to do something new (e.g., compile FooLang), instead of cramming -your code into an existing Task, you probably want to define a new Task -and install it in the existing Goal. A Task looks at the environment and -Targets, invokes some tool, generates things, and reports -success/failure. If you're giving Pants the ability to do something new, -you're probably adding a Task. See -[[Developing a Pants Task|pants('src/python/pants/docs:dev_tasks')]]. - -**Plugin (or "Backend")**
-Collection of Targets, Goals, Tasks, Commands to do something useful. At -Pants' core are the abstractions Target, and Task. These abstractions -don't do anything in particular. To build real code, you need to define -and register some more specific classes. For example, to build Java -code, you want the `JavaLibrary` Target, `JavaCompile` task (registered -in the `compile` goal), and many more. We organize this "real" code into -"plugins". A typical plugin defines several classes and registers them -with the Pants core. For a design discussion on registering plugins, see -the [Plugin -Engine](https://groups.google.com/forum/#!topic/pants-devel/uHGpR2K6FBI) -`pants-devel` thread. - -**Context**
-An API to the state of the world. A Task uses this to find out things -like the flags the user set on the command line, pants.ini config, and -the state of the build cache. The task uses context.products to -communicate results and requests for build results. - -Examining a Goal-Task Chain ---------------------------- - -It's not so easy to figure out in your head which Goals+Tasks are -invoked for some command line. The dependency relationships between -Goals and Tasks can get complex. The --explain flag helps here. Instead -of building something, it echoes a summary of the goals and tasks it -would use to build something. For example, you can find out what happens -on a `compile`: - - :::bash - $./pants goal compile --explain - Goal Execution Order: - - bootstrap -> imports -> gen -> check-exclusives -> resolve -> compile - - Goal [TaskRegistrar->Task] Order: - - bootstrap [bootstrap-jvm-tools->BootstrapJvmTools] - imports [ivy-imports->IvyImports] - gen [thrift->ApacheThriftGen, scrooge->ScroogeGen, protoc->ProtobufGen, antlr->AntlrGen, ragel->RagelGen, jaxb->JaxbGen, aapt->AaptGen] - check-exclusives [check-exclusives->CheckExclusives] - resolve [ivy->IvyResolve] - compile [jvm->SingletonGroupTask] - $ - -This tells you that the resolve goal comes before the compile goal, the -gen goal comes before that, etc. There is more than one Task registered -for the gen goal. In the gen [thrift-\>ApacheThriftGen,... text, thrift -is the name of a task and ApacheThriftGen is the name of the class that -implements it. - -Defining a Task ---------------- - -Defining a new Task tells Pants of some new action it can take. This -might be a new goal or adding new functionality in an existing goal -(e.g., telling the "gen" code-generation goal about some new way to -generate code). See -[[Developing a Pants Task|pants('src/python/pants/docs:dev_tasks')]]. - -Code Layout ------------ - -[./](https://github.com/pantsbuild/pants/tree/master/src/python/pants/base/)
-Top-level directory `__init__.py` Among other things, defines the -symbols visible in BUILD files. If you add a Target type, this file -should import it. `BUILD` Dogfood and/or recursion. `*.md` Docs too -important for docs/. - -[base](https://github.com/pantsbuild/pants/tree/master/src/python/pants/base/)
-Defines Target and other fundamental pieces/base classes. As a rule of -thumb, code in `base` shouldn't `import` anything in non-base Pants; but -many things in non-base Pants `import` from `base`. If you're editing -code in `base` and find yourself referring to the JVM (or other -target-language-specific things), you're probably editing the wrong -thing and want to look further up the inheritance tree. - -[bin](https://github.com/pantsbuild/pants/tree/master/src/python/pants/bin/)
-The "main" of Pants itself lives here. - -[commands](https://github.com/pantsbuild/pants/tree/master/src/python/pants/commands/)
-Before we had goals we had commands, and they lived here. **goal.py** -Many Goals and Tasks are defined here. - -[docs](https://github.com/pantsbuild/pants/tree/master/src/python/pants/docs/)
-Documentation. The source of this very document you're reading now lives -here. - -[goal](https://github.com/pantsbuild/pants/tree/master/src/python/pants/goal/)
-The source of Context and Goal (some important classes) lives here. If -you extend pants to work with other tools/languages, hopefully you won't -need to edit these; but you'll probably look at them to see the flow of -control. - -[java](https://github.com/pantsbuild/pants/tree/master/src/python/pants/java/)
-(TODO OMG bluffing) Utility classes useful to many things that work with -Java code. - -[python](https://github.com/pantsbuild/pants/tree/master/src/python/pants/backend/python/)
-(TODO OMG bluffing) Utility classes useful to many things that work with -Python code. - -[targets](https://github.com/pantsbuild/pants/tree/master/src/python/pants/targets/)
-Source of the Target classes; e.g., the code behind `jvm_binary` lives -here. If you define a new Target type, add its code here. - -[tasks](https://github.com/pantsbuild/pants/tree/master/src/python/pants/backend/core/tasks/)
-Source of the Task classes. E.g., `junit_run`, the code that invokes -JUnit if someone tests a `java_tests` target. - -[tests/.../pants](https://github.com/pantsbuild/pants/tree/master/tests/python/pants_test/)
-Tests for Pants. These tend to be `python_tests` exercising Pants -functions. `pants_test.base_build_root_test.BaseBuildRootTest` is a very -handy class; it has methods to set up and tear down little source trees -with `BUILD` files. - - diff --git a/src/python/pants/docs/readme.md b/src/python/pants/docs/readme.md deleted file mode 100644 index d11b8ccd71a..00000000000 --- a/src/python/pants/docs/readme.md +++ /dev/null @@ -1,20 +0,0 @@ -Pants Developer Center -====================== - -Welcome to the Pants Build developer center. This page is intended for -developers who wish to contribute changes to Pants itself or to create a -custom version of Pants. In the following guides you'll learn the -workflow for making changes, get your editor set up, how to build and -test pants, and how to get your change committed. - -+ [[How to Contribute|pants('src/python/pants/docs:howto_contribute']] -+ [[How to Ask|pants('src/python/pants/docs:howto_ask']] -+ [[Pants Developers Guide|pants('src/python/pants/docs:howto_develop')]] -+ [[Developing a Task|pants('src/python/pants/docs:dev_tasks')]] -+ [[Developing a Plugin|pants('src/python/pants/docs:howto_plugin')]] -+ [[How to Customize Pants for Your Site|pants('src/python/pants/docs:howto_customize')]] -+ [[Pants Development with IntelliJ IDEA|pants('src/python/pants/docs:intellij')]] -+ [[Pants Internals|pants('src/python/pants/docs:internals')]] -+ [[Release Process|pants('src/python/pants/docs:release')]] -+ [[Updating the Docs|pants('src/python/pants/docs:docs')]] -+ [[Credits|pants('src/python/pants/docs:credits')]] diff --git a/src/python/pants/docs/release.md b/src/python/pants/docs/release.md deleted file mode 100644 index a8f23777c06..00000000000 --- a/src/python/pants/docs/release.md +++ /dev/null @@ -1,96 +0,0 @@ -Release Process -=============== - -This page describes how to make a versioned release of Pants and and -other related packages to PyPi. - -At a high level, releasing pants involves: - -- Deciding what/when to release. At present this is ad-hoc, typically - when a change has been made and the author wants to use a version - incorporating that change. Things are likely to remain this way pre - 1.0.0. -- Preparing the release. -- (optional) Perform a release dry run. -- Publishing the release to PyPi. -- Announce the release on pants-devel. - -The current list of packages that are part of this release process: - -- pantsbuild.pants -- pantsbuild.pants.testinfra - -Prepare Release ---------------- - -Pants and the common libraries are published to the [Python Package -Index](https://pypi.python.org/pypi) per the Python community -convention. - -Although the build and publish are automated, the version bumping and -CHANGELOG management are not. You'll need to edit the version number in -[src/python/pants/version.py](https://github.com/pantsbuild/pants/tree/master/src/python/pants/version.py) -and add an entry in the CHANGELOG at -[src/python/pants/CHANGELOG.rst](https://github.com/pantsbuild/pants/tree/master/src/python/pants/CHANGELOG.rst) -then send this out for review. - -Dry Run -------- - -In order to publish the prepared release you'll need to be a -pantsbuild.pants package owner; otherwise you'll need to hand off to -someone who is. The list is on the [pantsbuild.pants package index -page](https://pypi.python.org/pypi/pantsbuild.pants) in the -Package Index Owner field: - - :::bash - $ curl -s "https://pypi.python.org`(curl -s https://pypi.python.org/pypi/pantsbuild.pants | grep -oE "/pypi/pantsbuild.pants/[0-9]*\.[0-9]*\.[0-9]*" | head -n1)`" | grep -A1 "Owner" - Package Index Owner: - john.sirois, benjyw, traviscrawford, ericzundel, ity - -All the other packages that are part of this simultaneous release -process must have the same owner list or the release script would break. - -Releases should only be published from master, so get on master and -ensure your version number commit is present. After confirming this, -publish locally and verify the release. - - :::bash - $ ./build-support/bin/release.sh -n - -This will perform a dry run local build of the pantsbuild.pants sdist -and other related package sdists, install them in a virtualenv and then -smoke test basic operations. - -Note that the release publish flow also performs a mandatory dry run so -executing a dry run separately is not required. - -Publish to PyPi ---------------- - -Now that we've smoke-tested this release, we can publish to PyPi: - - :::bash - $ ./build-support/bin/release.sh - -This also performs a dry run and then proceeds to upload the smoke -tested sdists to PyPi. - -Announce --------- - -Check PyPi to ensure everything looks good. The [pantsbuild.pants -package index page](https://pypi.python.org/pypi/pantsbuild.pants) -should display the package version you just uploaded. The same check -applies to other related package PyPi pages. - -To test the packages are installable: - - :::bash - $ ./build-support/bin/release.sh -t - -This will attempt to install the just-published packages from pypi and -then smoke test them. - -Finally, announce the release to pants-devel. -