Skip to content

Commit

Permalink
[FLINK-12720][python][docs] Add the Python Table API Sphinx docs
Browse files Browse the repository at this point in the history
This closes apache#8774
  • Loading branch information
HuangXingBo authored and sunjincheng121 committed Jun 19, 2019
1 parent 8c57e5a commit d70e8e2
Show file tree
Hide file tree
Showing 24 changed files with 966 additions and 210 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ flink-runtime-web/web-dashboard/web/
flink-python/dist/
flink-python/build/
flink-python/pyflink.egg-info/
flink-python/docs/_build
flink-python/.tox/
flink-python/dev/download
flink-python/dev/.conda/
Expand Down
1 change: 1 addition & 0 deletions docs/_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ baseurl: //ci.apache.org/projects/flink/flink-docs-master
stable_baseurl: //ci.apache.org/projects/flink/flink-docs-stable

javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master
pythondocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master

# Flag whether this is a stable version or not. Used for the quickstart page.
is_stable: false
Expand Down
1 change: 1 addition & 0 deletions docs/_includes/sidenav.html
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@
<li class="divider"></li>
<li><a href="{{ site.javadocs_baseurl }}/api/java"><i class="fa fa-external-link title" aria-hidden="true"></i> Javadocs</a></li>
<li><a href="{{ site.javadocs_baseurl }}/api/scala/index.html#org.apache.flink.api.scala.package"><i class="fa fa-external-link title" aria-hidden="true"></i> Scaladocs</a></li>
<li><a href="{{ site.pythondocs_baseurl }}/api/python"><i class="fa fa-external-link title" aria-hidden="true"></i> Pythondocs</a></li>
<li><a href="http://flink.apache.org"><i class="fa fa-external-link title" aria-hidden="true"></i> Project Page</a></li>
</ul>

Expand Down
60 changes: 58 additions & 2 deletions flink-python/dev/lint-python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,28 @@ function install_flake8() {
fi
}

# Install sphinx.
# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
function install_sphinx() {
if [ -f "$SPHINX_PATH" ]; then
${CONDA_PATH} remove sphinx -y -q 2>&1 >/dev/null
if [ $? -ne 0 ]; then
echo "conda remove sphinx failed \
please try to exec the script again.\
if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
exit 1
fi
fi

${CONDA_PATH} install -c anaconda sphinx -y -q 2>&1 >/dev/null
if [ $? -ne 0 ]; then
echo "conda install sphinx failed \
please try to exec the script again.\
if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
exit 1
fi
}


# In this function, the script will prepare all kinds of python environments and checks.
function install_environment() {
Expand Down Expand Up @@ -304,6 +326,15 @@ function install_environment() {
fi
print_function "STEP" "install flake8... [SUCCESS]"

# step-6 install sphinx
print_function "STEP" "installing sphinx..."
if [ $STEP -lt 6 ]; then
install_sphinx
STEP=6
checkpoint_stage $STAGE $STEP
fi
print_function "STEP" "install sphinx... [SUCCESS]"

print_function "STAGE" "install environment... [SUCCESS]"
}

Expand Down Expand Up @@ -467,6 +498,28 @@ function flake8_check() {
print_function "STAGE" "flake8 checks... [SUCCESS]"
fi
}
# Sphinx check
function sphinx_check() {
export SPHINXBUILD=$SPHINX_PATH
# cd to $FLINK_PYTHON_DIR
pushd "$FLINK_PYTHON_DIR"/docs &> /dev/null
make clean
# the return value of a pipeline is the status of the last command to exit
# with a non-zero status or zero if no command exited with a non-zero status
set -o pipefail
(SPHINXOPTS="-a -W" make html) 2>&1 | tee -a $LOG_FILE
SPHINXBUILD_STATUS=$?
if [ $SPHINXBUILD_STATUS -ne 0 ]; then
print_function "STAGE" "sphinx checks... [FAILED]"
# Stop the running script.
exit 1;
else
print_function "STAGE" "sphinx checks... [SUCCESS]"
fi
}
###############################################################All Checks Definitions###############################################################
# CURRENT_DIR is "flink/flink-python/dev/"
Expand All @@ -485,6 +538,9 @@ TOX_PATH=$CURRENT_DIR/.conda/bin/tox
# flake8 path
FLAKE8_PATH=$CURRENT_DIR/.conda/bin/flake8
# sphinx path
SPHINX_PATH=$CURRENT_DIR/.conda/bin/sphinx-build
_OLD_PATH="$PATH"
SUPPORT_OS=("Darwin" "Linux")
Expand Down Expand Up @@ -515,7 +571,7 @@ echo >$LOG_FILE
CONDA_INSTALL_SH=$CURRENT_DIR/download/miniconda.sh
# stage "install" includes the num of steps.
STAGE_INSTALL_STEPS=5
STAGE_INSTALL_STEPS=6
# whether force to restart the script.
FORCE_START=0
Expand Down Expand Up @@ -550,7 +606,7 @@ while getopts "hfi:e:l" arg; do
EXCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' ))
;;
i)
SELECT_CHECKS=($(echo $OPTARG | tr ',' ' ' ))
INCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' ))
;;
l)
printf "current supported checks includes:\n"
Expand Down
137 changes: 137 additions & 0 deletions flink-python/docs/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

ifndef SPHINXBUILD
ifndef SPHINXPYTHON
SPHINXBUILD = sphinx-build
SPHINXPYTHON = python
endif
endif

# You can set SPHINXBUILD to specify the sphinx-build or SPHINXPYTHON to specify the python env to call the sphinx module
# If SPHINXBUILD is set, the command is 'sphinx-build'.
# Elseif SPHINXPYTHON is set, the command is 'python -m sphinx'.
ifdef SPHINXBUILD
# Check whether sphinx-build is installed
ifeq ($(shell hash $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The command '$(SPHINXBUILD)' was not found.If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
else
ifeq ($(shell $(SPHINXPYTHON) -c 'import sphinx' >/dev/null 2>&1; echo $$?), 1)
# Check whether sphinx module is installed in $(SPHINXPYTHON)
$(error '$(SPHINXPYTHON)' doesn't install sphinx module.You can use specified pip of $(SPHINXPYTHON) to install sphinx)
endif
SPHINXBUILD = $(SPHINXPYTHON) -m sphinx
endif

# The project depends on py4j, so py4j-*-src.zip should be set in PYTHONPATH.
py4j_lib = $(shell echo ../lib/py4j-*-src.zip)
export PYTHONPATH = $(realpath $(py4j_lib))

# You can set these variables from the command line.
SPHINXOPTS ?=
PAPER ?=
SOURCEDIR ?= .
BUILDDIR ?= _build

# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .

.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest

help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"

clean:
-rm -rf _build/*

html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html
@echo
@echo "Build finished. The HTML pages are in _build/html."

dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) _build/dirhtml
@echo
@echo "Build finished. The HTML pages are in _build/dirhtml."

pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) _build/pickle
@echo
@echo "Build finished; now you can process the pickle files."

json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) _build/json
@echo
@echo "Build finished; now you can process the JSON files."

htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in _build/htmlhelp."

qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) _build/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in _build/qthelp, like this:"
@echo "# qcollectiongenerator _build/qthelp/pyflinkdoc.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile _build/qthelp/pyflinkdoc.qhc"

latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex
@echo
@echo "Build finished; the LaTeX files are in _build/latex."
@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
"run these through (pdf)latex."

changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes
@echo
@echo "The overview file is in _build/changes."

linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in _build/linkcheck/output.txt."

doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in _build/doctest/output.txt."

pdf: latex
cd _build/latex && make all-pdf

pdf-html: pdf html
cp _build/latex/pyflinkdoc.pdf _build/html
116 changes: 116 additions & 0 deletions flink-python/docs/_static/pyflink.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

$(function () {

function startsWith(s, prefix) {
return s && s.indexOf(prefix) === 0;
}

// Constructs map stores modules
function buildModuleLinkMap() {
var moduleLinkMap = {};
$('div.sphinxsidebar a.reference.internal').each(function (_, a) {
var href = $(a).attr('href');
// Whether is a module.
if (startsWith(href, '#module-')) {
var moduleName = href.substr(8);
moduleLinkMap[moduleName] = [$(a), null];
}
});
return moduleLinkMap;
}

function getDivNoteMap(dd) {
//Stores the divs that includes 'Special note' such as 'Deprecated',Experimental.
var divNoteMap = {};
// <div class="admonition note">
// <p class="first admonition-title">Note</p>
// <p class="last">WARN: Flink Internal Use Only</p>
// </div>
// The structure will get 'WARN: Flink Internal Use Only'
dd.find('> div.admonition.note > p.last').each(function (_, p) {
var text = $(p).text();
// Whether is Deprecated.
if (!divNoteMap.deprecated && startsWith(text, 'Deprecated')) {
divNoteMap.deprecated = $(p).parent();
}
// Whether is Experimental.
if (!divNoteMap.experimental && startsWith(text, 'Experimental')) {
divNoteMap.experimental = $(p).parent();
}
});
return divNoteMap;
}

function getPackageName(name) {
var last_idx = name.lastIndexOf('.');
return last_idx === -1 ? '' : name.substr(0, last_idx);
}

function buildTag(text, cls, tooltip) {
return '<span class="pys-tag ' + cls + ' hasTooltip">' + text + '<span class="tooltip">'
+ tooltip + '</span></span>'
}


var moduleLinkMap = buildModuleLinkMap();

// The class field of dl label includes 'class','method','classmethod'.
$('dl.class, dl.function').each(function (_, dl) {

dl = $(dl);
var dt = dl.children('dt').eq(0);
var dd = dl.children('dd').eq(0);

// Name of the class(method,classmethod).
// e.g. <dt id="pyflink.table.TableEnvironment"> will get 'pyflink.table.TableEnvironment'
var name = dt.attr('id');
// Desc of the class(method,classmethod).
// e.g. <code class="descname">TableEnvironment</code> will get 'TableEnvironment'
var descName = dt.find('> .descname').text();
var divNoteMap = getDivNoteMap(dd);

if (name) {
// e.g. 'pyflink.table.TableEnvironment' will get 'pyflink.table'
var packageName = getPackageName(name);

var moduleLink = moduleLinkMap[packageName];
if (moduleLink) {
if (moduleLink[1] === null) {
moduleLink[1] = $('<ul/>');
moduleLink[0].parent().append(moduleLink[1]);
}
var tags = '';
if (divNoteMap.experimental) {
tags += buildTag('E', 'pys-tag-experimental', 'Experimental');
divNoteMap.experimental.addClass('pys-note pys-note-experimental');
}
if (divNoteMap.deprecated) {
tags += buildTag('D', 'pys-tag-deprecated', 'Deprecated');
divNoteMap.deprecated.addClass('pys-note pys-note-deprecated');
}
var li = $('<li/>');
var a = $('<a href="#' + name + '">' + descName + '</a>');
li.append(a);
li.append(tags);
moduleLink[1].append(li);
moduleLinkMap[name] = [a, null];
}
}
});
});
Loading

0 comments on commit d70e8e2

Please sign in to comment.