Skip to content

Commit

Permalink
[LIVY-141][LIVY-175][DOCS] Update javadocs and scaladocs and include …
Browse files Browse the repository at this point in the history
…in Docs build

[LIVY-141](https://issues.apache.org/jira/browse/LIVY-141) [LIVY-175](https://issues.apache.org/jira/browse/LIVY-175)

Adding javadocs to Livy:
- Add ability to build Livy javadocs
- Update current javadoc comments to address build errors and warnings
- Add more javadoc comments to fully describe API
- Include public API javadocs and scaladocs in Livy Documentation

Noted Remaining Issues:
- Since only the public javadocs are build with the Livy Docs not all javadoc warnings in other modules have been addressed.
- There are still some warnings in the Scala API scaladocs build due to the docs not linking to an external lib for referenced scala lib classes, this does not break the build in any way.
- Scaladocs is not supported for Livy as a whole. With the update to Scala 2.11 many scaladoc warnings were upgraded to error and Livy fundamentally can't fix them.

Author: Alex Bozarth <[email protected]>

Closes apache#38 from ajbozarth/javadoc.
ajbozarth committed Jan 23, 2018
1 parent 3440857 commit fc22da9
Showing 18 changed files with 250 additions and 28 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
conf/*.conf
conf/*.properties
conf/*.sh
docs/api/
lib_managed/
logs/
src_managed/
47 changes: 38 additions & 9 deletions api/src/main/java/org/apache/livy/JobContext.java
Original file line number Diff line number Diff line change
@@ -33,25 +33,51 @@
*/
public interface JobContext {

/** The shared SparkContext instance. */
/**
* @return The shared SparkContext instance.
*/
JavaSparkContext sc();

/** The shared SQLContext instance. */
/**
* @return The shared SQLContext instance.
*/
SQLContext sqlctx();

/** The shared HiveContext instance. */
/**
* @return The shared HiveContext instance.
*/
HiveContext hivectx();

/** Returns the JavaStreamingContext which has already been created. */
/**
* @return The JavaStreamingContext which has already been created.
*/
JavaStreamingContext streamingctx();

/** Get shared object */
/**
* Get shared object
*
* @param <E> The type of the shared object
* @param name Name of the shared object to return
* @return The shared object matching name
*/
<E> E getSharedObject(String name) throws NoSuchElementException;

/** Set shared object, it will replace the old one if already existed */
/**
* Set shared object, it will replace the old one if already existed
*
* @param <E> The type of the shared object
* @param name Name of the shared object to be set
* @param object The object to be set
*/
<E> void setSharedObject(String name, E object);

/** Remove shared object from cache */
/**
* Remove shared object from cache
*
* @param <E> The type of the shared object
* @param name Name of the shared object to be removed
* @return The object that was removed
*/
<E> E removeSharedObject(String name);

/**
@@ -66,12 +92,15 @@ public interface JobContext {
void stopStreamingCtx();

/**
* Returns a local tmp dir specific to the context
* @return A local tmp dir specific to the context
*/
File getLocalTmpDir();

/**
* Returns SparkSession if it existed, otherwise throws Exception.
*
* @param <E> The type of the sparksession object
* @return The SparkSession if it exists
* @throws Exception If SparkSession does not exist
*/
<E> E sparkSession() throws Exception;
}
4 changes: 4 additions & 0 deletions api/src/main/java/org/apache/livy/JobHandle.java
Original file line number Diff line number Diff line change
@@ -27,6 +27,8 @@ public interface JobHandle<T> extends Future<T> {

/**
* Return the current state of the job.
*
* @return The current State of this job
*/
State getState();

@@ -60,6 +62,8 @@ static interface Listener<T> {
* Notifies when a job has been queued for execution on the remote context. Note that it is
* possible for jobs to bypass this state and got directly from the SENT state to the STARTED
* state.
*
* @param job The JobHandle for the queued job
*/
void onJobQueued(JobHandle<T> job);

9 changes: 5 additions & 4 deletions api/src/main/java/org/apache/livy/LivyClient.java
Original file line number Diff line number Diff line change
@@ -29,23 +29,24 @@ public interface LivyClient {
/**
* Submits a job for asynchronous execution.
*
* @param <T> The return type of the job
* @param job The job to execute.
* @return A handle that be used to monitor the job.
*/
<T> JobHandle<T> submit(Job<T> job);

/**
* Asks the remote context to run a job immediately.
* <p/>
* <p>
* Normally, the remote context will queue jobs and execute them based on how many worker
* threads have been configured. This method will run the submitted job in the same thread
* processing the RPC message, so that queueing does not apply.
* <p/>
* <p>
* It's recommended that this method only be used to run code that finishes quickly. This
* avoids interfering with the normal operation of the context.
* <p/>
* Note: the {@link JobContext#monitor()} functionality is not available when using this method.
* <p>
*
* @param <T> The return type of the job
* @param job The job to execute.
* @return A future to monitor the result of the job.
*/
6 changes: 6 additions & 0 deletions api/src/main/java/org/apache/livy/LivyClientBuilder.java
Original file line number Diff line number Diff line change
@@ -40,6 +40,8 @@ public final class LivyClientBuilder {
/**
* Creates a new builder that will automatically load the default Livy and Spark configuration
* from the classpath.
*
* @throws IOException If an error occurred when reading from the config files.
*/
public LivyClientBuilder() throws IOException {
this(true);
@@ -54,6 +56,10 @@ public LivyClientBuilder() throws IOException {
* application's classpath. Livy configuration takes precedence over Spark's (in case
* configuration entries are duplicated), and configuration set in this builder object will
* override the values in those files.
*
* @param loadDefaults Whether to load configs from spark-defaults.conf and livy-client.conf
* if they are found in the application's classpath.
* @throws IOException If an error occurred when reading from the config files.
*/
public LivyClientBuilder(boolean loadDefaults) throws IOException {
this.config = new Properties();
2 changes: 2 additions & 0 deletions api/src/main/java/org/apache/livy/LivyClientFactory.java
Original file line number Diff line number Diff line change
@@ -38,6 +38,8 @@ public interface LivyClientFactory {
* Instantiates a new client if the given URI is supported by the implementation.
*
* @param uri URI pointing at the livy backend to use.
* @param config Livy client configs.
* @return The newly created LivyClient or null if an unsupported URI
*/
LivyClient createClient(URI uri, Properties config);

21 changes: 21 additions & 0 deletions api/src/main/java/org/apache/livy/package-info.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/**
* Livy programmatic Java API
*/
package org.apache.livy;
Original file line number Diff line number Diff line change
@@ -52,10 +52,13 @@ public void addListener(Listener<T> l) {
* Changes the state of this job handle, making sure that illegal state transitions are ignored.
* Fires events appropriately.
*
* As a rule, state transitions can only occur if the current state is "higher" than the current
* As a rule, state transitions can only occur if the new state is "higher" than the current
* state (i.e., has a higher ordinal number) and is not a "final" state. "Final" states are
* CANCELLED, FAILED and SUCCEEDED, defined here in the code as having an ordinal number higher
* than the CANCELLED enum constant.
*
* @param newState The new state to change to.
* @return Whether the state changed.
*/
public boolean changeState(State newState) {
synchronized (listeners) {
Original file line number Diff line number Diff line change
@@ -40,12 +40,14 @@ public abstract class ClientConf<T extends ClientConf>

public static interface ConfEntry {

/** The key in the configuration file. */
/**
* @return The key in the configuration file.
*/
String key();

/**
* The default value, which also defines the type of the config. Supported types:
* Boolean, Integer, Long, String. <code>null</code> maps to String.
* @return The default value, which also defines the type of the config. Supported types:
* Boolean, Integer, Long, String. <code>null</code> maps to String.
*/
Object dflt();

@@ -226,10 +228,14 @@ private void logDeprecationWarning(String key) {
}
}

/** Maps valid key to DeprecatedConf with the deprecated key. */
/**
* @return A Map from a valid key to a DeprecatedConf with the deprecated key.
*/
protected abstract Map<String, DeprecatedConf> getConfigsWithAlternatives();

/** Maps deprecated key to DeprecatedConf with the same key. */
/**
* @return A Map from a deprecated key to a DeprecatedConf with the same key.
*/
protected abstract Map<String, DeprecatedConf> getDeprecatedConfigs();

private static class ConfPair {
@@ -261,13 +267,19 @@ private Map<String, ConfPair> allAlternativeKeys() {

public static interface DeprecatedConf {

/** The key in the configuration file. */
/**
* @return The key in the configuration file.
*/
String key();

/** The Livy version in which the key was deprecated. */
/**
* @return The Livy version in which the key was deprecated.
*/
String version();

/** Message to include in the deprecation warning for configs without alternatives */
/**
* @return Message to include in the deprecation warning for configs without alternatives
*/
String deprecationMessage();
}

17 changes: 16 additions & 1 deletion docs/README.md
Original file line number Diff line number Diff line change
@@ -46,12 +46,27 @@ Before opening a pull request, you can preview your contributions by running fro
3. Open http://localhost:4000
```

## API Docs (Scaladocs and Javadocs)

You can build just the Livy javadocs by running `mvn javadoc:aggregate` from the `LIVY_HOME`
directory, or the Livy scaladocs by running `mvn scala:doc` in certain modules. (Scaladocs do
not currently build in every module)

When you run `jekyll build` in the `docs` directory, it will also copy over the scaladocs and
javadocs for the public APIs into the `docs` directory (and then also into the `_site` directory).
We use a jekyll plugin to run the api builds before building the site so if you haven't run it
(recently) it may take some time as it generates all of the scaladoc and javadoc using Maven.

NOTE: To skip the step of building and copying over the Scala and Java API docs, run `SKIP_API=1
jekyll build`.


## Publishing Docs to [livy.incubator.apache.org]

1. Build Livy Docs (`cd docs` then `bundle exec jekyll build`).
2. Copy the contents of `docs/target/` excluding `assets/` into a new directory (eg. `0.4.0/`) and
move it into the `docs/` directory in your local fork of `apache/incubator-livy-website`.
3. If this is a major version, update the `latest` symlink to point to the new directory.
3. If nesesary, update the `latest` symlink to point to the new directory.
4. Open a pull request to `apache/incubator-livy-website` with the update.

Note: If you made any changes to files in the `assets/` directory you will need to replicate those
4 changes: 4 additions & 0 deletions docs/_data/navigation.yml
Original file line number Diff line number Diff line change
@@ -21,6 +21,10 @@ topnav:
url: rest-api.html
- title: Programmatic API
url: programmatic-api.html
- title: Java API Docs
url: api/java/index.html
- title: Scala API Docs
url: api/scala/index.html#org.apache.livy.scalaapi.package

- title: Apache
subcategories:
62 changes: 62 additions & 0 deletions docs/_plugins/build-apis.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

require 'fileutils'
include FileUtils

if not (ENV['SKIP_API'] == '1')
# Build Scaladoc for Scala API and Javadoc for Java API

puts "Moving to scala-api module and building Scala API docs."
cd("../scala-api")

puts "Running 'mvn scala:doc' from " + pwd + "; this may take a few minutes..."
system("mvn scala:doc") || raise("Scaladoc maven build failed")

puts "Moving to api module and building Java API docs."
cd("../api")

puts "Running 'mvn javadoc:javadoc -Ppublic-docs' from " + pwd + "; this may take a few minutes..."
system("mvn javadoc:javadoc -Ppublic-docs") || raise("Javadoc maven build failed")

puts "Moving back into docs dir."
cd("../docs")

puts "Removing old docs"
puts `rm -rf api`

# Copy over the ScalaDoc for the Scala API to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
source = "../scala-api/target/site/scaladocs"
dest = "api/scala"

puts "Making directory " + dest
mkdir_p dest

# From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't.
puts "cp -r " + source + "/. " + dest
cp_r(source + "/.", dest)

# Copy over the JavaDoc for the Java API to api/java.
source = "../api/target/site/apidocs"
dest = "api/java"

puts "Making directory " + dest
mkdir_p dest

puts "cp -r " + source + "/. " + dest
cp_r(source + "/.", dest)

end
Loading

0 comments on commit fc22da9

Please sign in to comment.