From daadc74839ccfb56e54e033739fc73dceb003b80 Mon Sep 17 00:00:00 2001 From: Ryota MIBU Date: Thu, 7 Jul 2016 16:38:19 +0900 Subject: [PATCH] migrate docs-build.sh from releng repo JIRA: DOCS-153 Change-Id: I568192dbb6653eb8fd8518abebf3558dc00dcee8 Signed-off-by: Ryota MIBU --- docs/how-to-use-docs/documentation-example.rst | 26 +-- etc/conf.py | 40 +++++ etc/opnfv-logo.png | Bin 0 -> 2829 bytes etc/requirements.txt | 6 + scripts/docs-build.sh | 234 +++++++++++++++++++++++++ 5 files changed, 293 insertions(+), 13 deletions(-) create mode 100644 etc/conf.py create mode 100644 etc/opnfv-logo.png create mode 100644 etc/requirements.txt create mode 100755 scripts/docs-build.sh diff --git a/docs/how-to-use-docs/documentation-example.rst b/docs/how-to-use-docs/documentation-example.rst index ca997d08f..33261eff4 100644 --- a/docs/how-to-use-docs/documentation-example.rst +++ b/docs/how-to-use-docs/documentation-example.rst @@ -25,7 +25,7 @@ your documents. Note: You may have "docs/how-to-use-docs/" in you project repo. You can delete it, -since it is sample and master version is stored in releng repo. +since it is sample and master version is stored in opnfvdocs repo. Note: During the document build process, 'docs_build' and 'docs_output' will be @@ -37,7 +37,6 @@ so that git can ignore built files. /docs_build/ /docs_output/ - /releng/ Index File ========== @@ -173,8 +172,8 @@ also used in document build jobs below: .. code-block:: bash $ cd /local/repo/path/to/project - $ git clone https://git.opnfv.org/releng - $ ./releng/utils/docs-build.sh + $ git clone https://git.opnfv.org/opnfvdocs docs_build/_opnfvdocs + $ ./docs_build/_opnfvdocs/scripts/docs-build.sh Then, you can see the docs in 'docs_output' directory if build succeeded. @@ -186,9 +185,10 @@ necessary packages are installed as follows: $ sudo pip install Sphinx==1.3.1 doc8 sphinxcontrib-httpdomain Note: -Developers are encouraged to use "ssh://@gerrit.opnfv.org:29418/releng" -instead of "https://git.opnfv.org/releng", so that you can quickly start -development in releng. +Developers are encouraged to use +"ssh://@gerrit.opnfv.org:29418/opnfvdocs" +instead of "https://git.opnfv.org/opnfvdocs", so that you can quickly start +development in opnfvdocs. See https://wiki.opnfv.org/display/DEV/Developer+Getting+Started for more detail. @@ -242,17 +242,17 @@ latest document always. Sphinx Extensions ================= -You can see available sphinx extension(s) in `docs/etc/requirements.txt`_. +You can see available sphinx extension(s) in `opnfvdocs/etc/requirements.txt`_. -.. _docs/etc/requirements.txt: - https://gerrit.opnfv.org/gerrit/gitweb?p=releng.git;a=blob;f=docs/etc/requirements.txt; +.. _opnfvdocs/etc/requirements.txt: + https://gerrit.opnfv.org/gerrit/gitweb?p=opnfvdocs.git;a=blob;f=etc/requirements.txt; You can use other sphinx extensions to improve your documents. To share such improvements, we encourage you to enable the extension in OPNFV infra by asking releng and opnfvdocs teams to add new sphinx extension via gerrit -(proposing change in `utils/docs-build.sh`_ and `docs/etc/requirements.txt`_). +(proposing change in `opnfvdocs/scripts/docs-build.sh`_ and `opnfvdocs/etc/requirements.txt`_). After quick sanity checks, we'll merge the patch to make it available in OPNFV document build. -.. _utils/docs-build.sh: - https://gerrit.opnfv.org/gerrit/gitweb?p=releng.git;a=blob;f=utils/docs-build.sh; +.. _opnfvdocs/scripts/docs-build.sh: + https://gerrit.opnfv.org/gerrit/gitweb?p=opnfvdocs.git;a=blob;f=scripts/docs-build.sh; diff --git a/etc/conf.py b/etc/conf.py new file mode 100644 index 000000000..c4cbae775 --- /dev/null +++ b/etc/conf.py @@ -0,0 +1,40 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +''' +Base configuration file for building OPNFV docs + +You can override this configuration by putting 'conf.py' in the document +directory (e.g. docs/how-to-use-docs/conf.py). If there is no 'conf.py' +in the document directory, this file will be copied to that directory +before the document builder jobs ('opnfv-docs-verify' and 'opnfv-docs-merge'). + +You may need python package installation for new sphinx extension. +Install python package with 'pip' in your machine and add the extension to +the 'extensions' list below to test the documentation build locally. +If you feel that your extensions would be useful for other projects too, +we encourage you to propose a change in the releng repository. + +For further guidance see the https://wiki.opnfv.org/documentation/tools page. +''' + +extensions = ['sphinxcontrib.httpdomain', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon'] + +needs_sphinx = '1.3' +master_doc = 'index' +pygments_style = 'sphinx' + +html_use_index = False +numfig = True +html_logo = 'opnfv-logo.png' + +latex_domain_indices = False +latex_logo = 'opnfv-logo.png' diff --git a/etc/opnfv-logo.png b/etc/opnfv-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..1519503eba764f7a00877b9b8bdbbd37fd3707c6 GIT binary patch literal 2829 zcmV+o3-a`dP)3)28sjxMC?FF++Bq$U6w!I|Q zuDk75d^``ll?O5~(CmjEDa)%lO*g>n_g{0#>mC}Gzk+T4rzf9>j53xroI0y~C_9nZ zRT`GLg0kl=wSC&o>6~=+jNzR71<#dp4To_2s0>t*!h`9cI?8$fX~Qs`rR1m400P?y z90%)qtm)id4ONiKpFsI}ZSK@iJ|1rbWk*!DU!d(zDwO-{0ftjgEE_w%;rV)uW5>yH z_exeI%QmujpMCMW4sNM^@?x)fLqOrhd{$9sTgQ`d%yCO<1Iu4R`O_B4UJp!A_62MX zv+dC{T~-}GLx+x;{ijz!sH5z=g`on8=PSHb0@Q#SaG3%t>jDJstyjW^fI6$;(C9VJ zM0+-BvHTu^?J(N@qDGm&V(fERuzosQRFSAa>*vWrfXJ0P?>aF7?>Ukt-*BC%M;idD zR})ZIu_ASxwrtR%$Ly%uwx-ywh|=kCy{aTtUB<3KB~XF4LO2cx3OQ&Lq?yxFVUVVp z0PEI0KSd4M#7I7+3(=OHPzP?sW6{8s*)F2yL=?p|oPJ~?`#D}}PV%!7PnCdm2ZMFk zW>V&_r_iP-9Vh7N0H%W`WFN{S(M)#H$rw-?2dl-hr#TA2Yp8sVw>mO79!D;2$j(C9 zzXy*$kkgBc&!cDaj~~4?%jjHLfvo_Dkm|Ke`S>rtaK&a?_58uB@A@?~_QQr_jA4r% zdo6L~(X+|q1$xR6wDO0o6BtGDy;|9Z8x6|M}356(@zSPXM%I z0Lusl0BW2ws#XdW$sw*J3o5uE4yIuhY3{5zlq-KbOpogcel|5sNqChP?l1?Huz=hO z3SQz!l7IoUeXMDtEbxn7>nNR~-uQD!D>F?@CkV2w46eFk7TlE;cNI&vfdHD!z_ z;-^n$4()d+RT3Nj{MYwKT=-mO$Li`vcN)ExjLy|D=p^S*?wC11I3%$KjPkiq9dic& zD1XFAINVJI`no{?+x)D;i*#V%#b0@^6+U-r5gc9;H_K&CRFHYaqspI@xpfrruA!jPbh-F;K8f!CJmzV zLU^o+ux2fL(BTYxUP&*N3GW6};H`XywCO?Ilt`dZ<@y1lkihEFYh6s!Cok30j1p@| zA*PCL7qX^OJS8L`Cu6zNLM4H9hooAZa`}@)!U0$bFcXy=K$sXnq37#@mx`sb0!4-h zfeLNEv8FIi;>F96e%vf)rl-oBrvk7d@g%V6_BVf7| zS9#?9)}X;>(XRIvx@+2+Chn!X=v9g*FSX8@n+oz`TlVDdw|;)?Qi7qA@?Piv_~kSp zz&N9Wpt$LZ@;^pTT&l=jY6Ntv!S~(!0Ta?)fhieeICo6MH?;1D1P$l8DC}dpX`5a~ zALHCbvB>*ip9$1haUlorIZDD~fXnDY0!+y*J);b_d;*^%R6~F(+Hr^uC68UD{MqKZ zyQ#Xn1?sk9Ky^fF%h*#DhDrm}69BE&;9s04v@Bh*TZP#G|!E+Ltr=`%PvYj2Zzz=`e#qSKRysVIIQ8#7S=4&P4ul z16;dqQw`0;M;E8cz{eLz0YHF%!Pmr+uvP zEG{6yZ@RHP70GX%dWv-m-sHwqFa7pYIh`H<7f$Y9Kl$`$%G98dx#an({(_lPu>y#%;AjQuVEuIY|;4s81w z#Z9+x0(6IfO6rah*C;7Q>2IAZ=A_a?j=YvVm5$`A=Xk(NjTP=x_bf}^Nv%B}fOYWx z-+$w4&SwN*t#-g;MKm(5?=ac1cqys-LMUztGL*=J3S^`(fvTgaU|;f5s$9OWDi9=I z1mFZYUJgA`4`A^D8XT2Wv*9PAp~W{FPBzFLX_p|XyY7n$pqd*&g_%8ysUC{WH7f9A z{3JO6UdmNy&)G-!?s6LASmZx|h}zZ%u#N#NN6`>B>G!PgVok2wT@V%84C-F@)68XM zGW#x-jYI{2;|u+SY#vZP0L5w;Oh|yY!hN&Wg1h=3z#0Nr=i>Rt>A6e%UKe}1!$6_^ zyY2!|KE8hcHS9CJx}hwxL|L2>K%N3nzR5u_jZQK!y-@we@hOeIt`Go*-YJIn>X1b) zcSps7k=C}`RwbZX)CLv6gAir@0^=XP_$PXk@XgyVD}%}x-+6=`IL7!&&~sY=n5Ri( zAYfsMQAU@D!Ifo|}iQURGX*Df~iL*>ymsNDu7Cs_;TR&=~wSP z9+T$^YHUmnj8!iS9oRm+UBb=MKy;;sk-xOF8-RYdS%B~4aLpauf1?pPG zRovzD3f;9+c%BL+9>i4$h%21{H=dY~eDumzpn~LuFa55oXP~o9(&iJ!@#?aOQNC#m ze6EpDkzx5k3v5joGO*Bjy-? /dev/null || echo draft) ($revision)" +project="$(basename $(git rev-parse --show-toplevel))" +html_notes=" Revision: $rev_full\n Build date: $(date -u +'%Y-%m-%d')" +default_conf="$OPNFVDOCS_DIR/etc/conf.py" +opnfv_logo="$OPNFVDOCS_DIR/etc/opnfv-logo.png" + +function check_rst_doc() { + _src="$1" + + # Note: This check may fail in many jobs for building project docs, since + # the old sample has lines more than 120. We ignore failures on this + # check right now, but these have to be fixed before OPNFV B release. + _out=$(doc8 --max-line-length 240 --ignore D000 "$_src") || { + _msg='Warning: rst validation (doc8) has failed, please fix the following error(s).' + _errs=$(echo "$_out" | sed -n -e "/^$_src/s/^/ /p") + echo + echo -e "$_msg\n$_errs" + echo + [[ -n "$GERRIT_COMMENT" ]] && echo -e "$_msg\n$_errs" >> "$GERRIT_COMMENT" + } +} + +function add_html_notes() { + _src="$1" + + find "$_src" -name '*.rst' | while read file + do + if grep -q -e ' _sha1_' "$file" ; then + # TODO: remove this, once old templates were removed from all repos. + echo + echo "Warn: '_sha1_' was found in [$file], use the latest document template." + echo " See https://wiki.opnfv.org/documentation/tools ." + echo + sed -i "s/ _sha1_/ $git_sha1/g" "$file" + fi + sed -i -e "\$a\\\n..\n$html_notes" "$file" + done +} + +function prepare_src_files() { + mkdir -p "$(dirname $SRC_DIR)" + + [[ -e "$SRC_DIR" ]] && rm -rf "$SRC_DIR" + cp -r "$DOCS_DIR" "$SRC_DIR" + add_html_notes "$SRC_DIR" +} + +function add_config() { + _conf="$1" + _param="$2" + _val="$3" + + if ! grep -q -e "^$_param = " "$_conf" ; then + echo "Adding '$_param' into $_conf ..." + echo "$_param = $_val" >> "$_conf" + fi +} + +function is_top_dir() { + [[ "$1" == "$DOCS_DIR" ]] +} + +function generate_name_for_top_dir() { + for suffix in '' '.top' '.all' '.master' '_' '__' '___' + do + _name="$(basename $DOCS_DIR)$suffix" + [[ -e "$DOCS_DIR/$_name" ]] && continue + echo "$_name" + return + done + + echo "Error: cannot find name for top directory [$DOCS_DIR]" + exit 1 +} + +function generate_name() { + _dir=$1 + + if is_top_dir "$_dir" ; then + _name=$(generate_name_for_top_dir $DOCS_DIR) + else + _name="${_dir#$DOCS_DIR/}" + fi + # Replace '/' by '_' + echo "${_name////_}" +} + + +check_rst_doc $DOCS_DIR + +if [[ ! -d "$OPNFVDOCS_DIR" ]] ; then + echo "Error: $OPNFVDOCS_DIR dir not found. See https://wiki.opnfv.org/documentation/tools ." + exit 1 +fi + +prepare_src_files + +if ! which virtualenv > /dev/null ; then + echo "Error: 'virtualenv' not found. Exec 'sudo pip install virtualenv' first." + exit 1 +fi + +virtualenv "$VENV_DIR" +source "$VENV_DIR/bin/activate" +pip install -r "$OPNFVDOCS_DIR/etc/requirements.txt" + +find $DOCS_DIR -name $INDEX_RST -printf '%h\n' | while read dir +do + name=$(generate_name $dir) + if is_top_dir "$dir" ; then + src="$SRC_DIR" + else + src="$SRC_DIR/${dir#$DOCS_DIR/}" + fi + build="$BUILD_DIR/$name" + output="$OUTPUT_DIR/$name" + conf="$src/conf.py" + + echo + echo "#################${dir//?/#}" + echo "Building DOCS in $dir" + echo "#################${dir//?/#}" + echo + + [[ ! -f "$conf" ]] && cp "$default_conf" "$conf" + title=$(cd $src; python -c "$get_title_script") + latex_conf="[('index', '$name.tex', \"$title\", 'OPNFV', 'manual'),]" + add_config "$conf" 'latex_documents' "$latex_conf" + add_config "$conf" 'release' "u'$version'" + add_config "$conf" 'version' "u'$version'" + add_config "$conf" 'project' "u'$project'" + add_config "$conf" 'copyright' "u'$(date +%Y), OPNFV'" + cp -f $opnfv_logo "$src/opnfv-logo.png" + + mkdir -p "$output" + + sphinx-build -b html -t html -E "$src" "$output" + + # Note: PDF creation may fail in project doc builds. + # We allow this build job to be marked as succeeded with + # failure in PDF creation, but leave message to fix it. + # Any failure has to be fixed before OPNFV B release. + { + sphinx-build -b latex -t pdf -E "$src" "$build" && \ + make -C "$build" LATEXOPTS='--interaction=nonstopmode' all-pdf + } && { + mv "$build/$name.pdf" "$output" + } || { + msg="Error: PDF creation for $dir has failed, please fix source rst file(s)." + echo + echo "$msg" + echo + [[ -n "$GERRIT_COMMENT" ]] && echo "$msg" >> "$GERRIT_COMMENT" + } + + # TODO: failures in ODT creation should be handled error and + # cause 'exit 1' before OPNFV B release. + tex=$(find $build -name '*.tex' | head -1) + odt="${tex%.tex}.odt" + if [[ -e $tex ]] && which pandoc > /dev/null ; then + ( + cd $(dirname $tex) + pandoc $(basename $tex) -o $(basename $odt) + ) && { + mv $odt $output/ + }|| { + msg="Error: ODT creation for $dir has failed." + echo + echo "$msg" + echo + } + else + echo "Warn: tex file and/or 'pandoc' are not found, skip ODT creation." + fi + + if is_top_dir "$dir" ; then + # NOTE: Having top level document (docs/index.rst) is not recommended. + # It may cause conflicts with other docs (mostly with HTML + # folders for contents in top level docs and other document + # folders). But, let's try merge of those contents into the top + # docs directory. + ( + cd $output + find . -type d -print | xargs -I d mkdir -p ../d + find . -type f -print | xargs -I f mv -b f ../f + ) + rm -rf "$output" + fi + +done + +deactivate -- 2.16.6