From 243124470e1f346b71f8c499f576747374dcff0e Mon Sep 17 00:00:00 2001 From: Cole Greer Date: Thu, 21 May 2026 14:00:42 -0700 Subject: [PATCH 01/28] Replace docs preprocessor/postprocessor with AsciidoctorJ extension Replace the awk-based preprocessor and shell postprocessor pipeline with a Java-based AsciidoctorJ extension (tinkerpop-docs module) that: - Processes gremlin-groovy listing blocks via GremlinConsole subprocess - Handles multi-line statement joining and callout stripping - Generates tabbed HTML output for language variants - Applies version substitution and callout fixes via postprocessor - Auto-restarts console on timeout with block-level retry - Falls back to dry-run output for blocks that fail after retry The new bin/process-docs.sh orchestrates console/server setup and passes attributes to the AsciidoctorJ plugin via Maven properties. Known issues to address: - Echo pattern in console output needs stripping - Second groovy tab (clean source) not yet generated - Version x.y.z substitution not wired - Missing CSS stylesheet in output - Some callout markers rendered as raw text --- .gitignore | 3 +- bin/process-docs.sh | 225 +++++--- bin/validate-distribution.sh | 2 +- docs/postprocessor/postprocess.sh | 38 -- docs/postprocessor/processor.awk | 53 -- docs/preprocessor/awk/cleanup.awk | 36 -- docs/preprocessor/awk/ignore.awk | 23 - docs/preprocessor/awk/init-code-blocks.awk | 73 --- docs/preprocessor/awk/language-variants.awk | 44 -- docs/preprocessor/awk/prepare.awk | 83 --- docs/preprocessor/awk/prettify.awk | 31 -- docs/preprocessor/awk/progressbar.awk | 36 -- docs/preprocessor/awk/tabify.awk | 132 ----- docs/preprocessor/install-plugins.groovy | 44 -- docs/preprocessor/install-plugins.sh | 74 --- docs/preprocessor/preprocess-file.sh | 174 ------ docs/preprocessor/preprocess.sh | 155 ------ docs/preprocessor/uninstall-plugins.sh | 36 -- .../development-environment.asciidoc | 2 +- pom.xml | 48 ++ tools/tinkerpop-docs/REQUIREMENTS.md | 216 ++++++++ tools/tinkerpop-docs/pom.xml | 81 +++ .../gremlin/docs/ConsoleRestartHandler.java | 43 ++ .../gremlin/docs/GremlinConsole.java | 270 +++++++++ .../gremlin/docs/GremlinDocsExtension.java | 28 +- .../gremlin/docs/GremlinPostprocessor.java | 78 +++ .../gremlin/docs/GremlinTreeprocessor.java | 521 ++++++++++++++++++ .../gremlin/docs/TabbedHtmlBuilder.java | 231 ++++++++ ...ctor.jruby.extension.spi.ExtensionRegistry | 1 + .../src/main/resources/hadoop-conf/README | 34 ++ .../main/resources/hadoop-conf/core-site.xml | 33 ++ .../hadoop-conf/hadoop-docs.properties | 14 +- .../gremlin/docs/GremlinConsoleTest.java | 264 +++++++++ .../gremlin/docs/GremlinDocsTest.java | 81 +++ .../docs/GremlinPostprocessorTest.java | 105 ++++ .../docs/GremlinTreeprocessorTest.java | 512 +++++++++++++++++ .../gremlin/docs/IntegrationTest.java | 131 +++++ .../gremlin/docs/TabbedHtmlBuilderTest.java | 286 ++++++++++ .../resources/integration-fixture.asciidoc | 98 ++++ 39 files changed, 3208 insertions(+), 1131 deletions(-) delete mode 100755 docs/postprocessor/postprocess.sh delete mode 100644 docs/postprocessor/processor.awk delete mode 100644 docs/preprocessor/awk/cleanup.awk delete mode 100644 docs/preprocessor/awk/ignore.awk delete mode 100644 docs/preprocessor/awk/init-code-blocks.awk delete mode 100644 docs/preprocessor/awk/language-variants.awk delete mode 100644 docs/preprocessor/awk/prepare.awk delete mode 100644 docs/preprocessor/awk/prettify.awk delete mode 100644 docs/preprocessor/awk/progressbar.awk delete mode 100644 docs/preprocessor/awk/tabify.awk delete mode 100644 docs/preprocessor/install-plugins.groovy delete mode 100755 docs/preprocessor/install-plugins.sh delete mode 100755 docs/preprocessor/preprocess-file.sh delete mode 100755 docs/preprocessor/preprocess.sh delete mode 100755 docs/preprocessor/uninstall-plugins.sh create mode 100644 tools/tinkerpop-docs/REQUIREMENTS.md create mode 100644 tools/tinkerpop-docs/pom.xml create mode 100644 tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java create mode 100644 tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java rename docs/preprocessor/awk/progressbar.groovy.template => tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java (54%) create mode 100644 tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java create mode 100644 tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java create mode 100644 tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java create mode 100644 tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry create mode 100644 tools/tinkerpop-docs/src/main/resources/hadoop-conf/README create mode 100644 tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml rename docs/preprocessor/control-characters.sh => tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties (71%) mode change 100755 => 100644 create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java create mode 100644 tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java create mode 100644 tools/tinkerpop-docs/src/test/resources/integration-fixture.asciidoc diff --git a/.gitignore b/.gitignore index 9359a483c4b..fafcdc86c4f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,8 @@ __pycache__/ __version__.py .glv settings.xml -tools/ +tools/* +!tools/tinkerpop-docs/ [Dd]ebug/ [Rr]elease/ [Oo]bj/ diff --git a/bin/process-docs.sh b/bin/process-docs.sh index 7236faad887..2efd344edd0 100755 --- a/bin/process-docs.sh +++ b/bin/process-docs.sh @@ -18,96 +18,169 @@ # under the License. # -pushd "$(dirname $0)/.." > /dev/null - -NOCLEAN= - -DRYRUN= -DRYRUN_DOCS= -FULLRUN_DOCS= - -makeAbsPaths () { - for doc in $(tr ',' $'\n' <<< "$1"); do - if [ -d $doc ]; then - for d in $(find "$doc" -name "*.asciidoc"); do - echo $(cd $(dirname "$d") && pwd -P)/$(basename "$d") - done - else - echo $(cd $(dirname "$doc") && pwd -P)/$(basename "$doc") - fi - done | paste -sd ',' - -} - -while [[ $# -gt 0 ]] -do - key="$1" - case $key in - -n|--noClean) - NOCLEAN=1 - shift - ;; - -d|--dryRun) - DRYRUN=1 - shift - if [[ $# -gt 0 ]] && [[ $1 != -* ]]; then - DRYRUN_DOCS=$(makeAbsPaths "$1") - shift - else - DRYRUN_DOCS="*" - fi - ;; - -f|--fullRun) - DRYRUN=1 - DRYRUN_DOCS=${DRYRUN_DOCS:-"*"} - shift - FULLRUN_DOCS=$(makeAbsPaths "$1") - shift - ;; - *) - # unknown option - shift - ;; +# Orchestration script for the TinkerPop documentation build. +# +# Full build (--fullRun, default): +# Validates console/server distributions, installs plugins, starts +# Gremlin Server and Gephi mock, then invokes Maven with the +# AsciidoctorJ extension to render docs with live code execution. +# +# Dry-run (--dryRun): +# Invokes Maven with -Dgremlin-docs-dryrun=true. No server, console, +# or plugins required. + +set -e + +cd "$(dirname "$0")/.." +TP_HOME="$(pwd)" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +MODE="full" +NOCLEAN="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --fullRun|-f) MODE="full"; shift ;; + --dryRun|-d) MODE="dry"; shift ;; + --noClean|-n) NOCLEAN="1"; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; esac done -if [ -z ${NOCLEAN} ]; then - rm -rf ~/.groovy/grapes/org.apache.tinkerpop/ - if hash hadoop 2> /dev/null; then - hadoop fs -rm -r "hadoop-gremlin-*-libs" > /dev/null 2>&1 +# --------------------------------------------------------------------------- +# Cleanup trap +# --------------------------------------------------------------------------- +GREMLIN_SERVER_PID="" +GEPHI_MOCK_PID="" + +cleanup() { + set +e + [ -n "${GREMLIN_SERVER_PID}" ] && kill "${GREMLIN_SERVER_PID}" 2>/dev/null + [ -n "${GEPHI_MOCK_PID}" ] && kill "${GEPHI_MOCK_PID}" 2>/dev/null +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Dry-run mode +# --------------------------------------------------------------------------- +if [ "${MODE}" == "dry" ]; then + if [ -z "${NOCLEAN}" ]; then + rm -rf target/postprocess-asciidoc target/doc-source target/docs 2>/dev/null || true fi + echo "Copying docs sources to target/postprocess-asciidoc/..." + mkdir -p target/postprocess-asciidoc + cp -r docs/src/* target/postprocess-asciidoc/ + mvn process-resources -Dasciidoc -Dgremlin.docs.dryrun=true + exit $? fi -if [ ${DRYRUN} ] && [ "${DRYRUN_DOCS}" == "*" ] && [ -z "${FULLRUN_DOCS}" ]; then +# --------------------------------------------------------------------------- +# Full build mode +# --------------------------------------------------------------------------- - mkdir -p target/postprocess-asciidoc/tmp - cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - cp -R docs/src/. target/postprocess-asciidoc/ - ec=$? +# Resolve version from pom.xml +TP_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout 2>/dev/null || \ + grep -A1 'tinkerpop' pom.xml | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) -else +# 1. Validate console distribution +CONSOLE_DIR=$(ls -d gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone 2>/dev/null | head -n1) +if [ -z "${CONSOLE_DIR}" ] || [ ! -d "${CONSOLE_DIR}" ]; then + echo "ERROR: Gremlin Console distribution not found." + echo "Build it first: mvn clean install -pl :gremlin-console -am -DskipTests" + exit 1 +fi +CONSOLE_HOME="$(cd "${CONSOLE_DIR}" && pwd)" + +# 2. Validate server distribution +SERVER_DIR=$(ls -d gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone 2>/dev/null | head -n1) +if [ -z "${SERVER_DIR}" ] || [ ! -d "${SERVER_DIR}" ]; then + echo "ERROR: Gremlin Server distribution not found." + echo "Build it first: mvn clean install -pl :gremlin-server -am -DskipTests" + exit 1 +fi +SERVER_HOME="$(cd "${SERVER_DIR}" && pwd)" + +# 3. Install plugins into console +echo "Installing plugins into console..." +PLUGINS="hadoop-gremlin spark-gremlin neo4j-gremlin sparql-gremlin" +for plugin in ${PLUGINS}; do + PLUGIN_DIR="${plugin}/target/${plugin}-${TP_VERSION}-standalone" + if [ -d "${PLUGIN_DIR}" ]; then + echo " * installing ${plugin} (standalone)" + cp -r "${PLUGIN_DIR}" "${CONSOLE_HOME}/ext/${plugin}" + elif [ -f "${plugin}/target/${plugin}-${TP_VERSION}.jar" ]; then + echo " * installing ${plugin} (jar + dependencies)" + mkdir -p "${CONSOLE_HOME}/ext/${plugin}/lib" + cp "${plugin}/target/${plugin}-${TP_VERSION}.jar" "${CONSOLE_HOME}/ext/${plugin}/lib/" + cp "${plugin}"/target/dependency/*.jar "${CONSOLE_HOME}/ext/${plugin}/lib/" 2>/dev/null || \ + mvn dependency:copy-dependencies -pl "${plugin}" -DoutputDirectory="${CONSOLE_HOME}/ext/${plugin}/lib" -q + else + echo " * WARNING: ${plugin} not found" + fi +done - GEPHI_MOCK= +# 4. Copy hadoop config to console classpath +HADOOP_CONF_SRC="tools/tinkerpop-docs/src/main/resources/hadoop-conf" +cp "${HADOOP_CONF_SRC}/core-site.xml" "${CONSOLE_HOME}/conf/" +cp "${HADOOP_CONF_SRC}/hadoop-docs.properties" "${CONSOLE_HOME}/conf/" + +# 5. Start Gremlin Server +echo "Starting Gremlin Server..." +mkdir -p target +pushd "${SERVER_HOME}" > /dev/null +bin/gremlin-server.sh conf/gremlin-server-modern.yaml > "${TP_HOME}/target/gremlin-server-docs.log" 2>&1 & +GREMLIN_SERVER_PID=$! +popd > /dev/null + +# Wait for server to be ready (port 8182) +echo -n "Waiting for Gremlin Server on port 8182..." +for i in $(seq 1 30); do + if nc -z localhost 8182 2>/dev/null; then + echo " ready." + break + fi + if [ $i -eq 30 ]; then + echo " TIMEOUT" + echo "ERROR: Gremlin Server failed to start within 30 seconds." + exit 1 + fi + sleep 1 + echo -n "." +done - trap cleanup EXIT +# 6. Start Gephi mock (port 8080) +if ! nc -z localhost 8080 2>/dev/null; then + echo "Starting Gephi mock on port 8080..." + bin/gephi-mock.py > /dev/null 2>&1 & + GEPHI_MOCK_PID=$! +fi - function cleanup() { - [ ${GEPHI_MOCK} ] && kill ${GEPHI_MOCK} - } +# 7. Resolve HADOOP_GREMLIN_LIBS path +HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/hadoop-gremlin/lib" - nc -z localhost 8080 || ( - bin/gephi-mock.py > /dev/null 2>&1 & - GEPHI_MOCK=$! - ) +# 8. Copy source docs to staging area (replaces old preprocessor copy) +echo "Copying docs sources to target/postprocess-asciidoc/..." +mkdir -p target/postprocess-asciidoc +cp -r docs/src/* target/postprocess-asciidoc/ - docs/preprocessor/preprocess.sh "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - ec=$? +# 9. Invoke Maven with AsciidoctorJ extension attributes +echo "Running documentation build..." +if [ -z "${NOCLEAN}" ]; then + rm -r target/doc-source target/docs 2>/dev/null || true fi - -if [ $ec -eq 0 ]; then - mvn process-resources -Dasciidoc && docs/postprocessor/postprocess.sh - ec=$? +set +e +mvn process-resources -Dasciidoc \ + -Dgremlin.docs.console.home="${CONSOLE_HOME}" \ + -Dgremlin.docs.hadoop.libs="${HADOOP_GREMLIN_LIBS}" +ec=$? +set -e + +if [ ${ec} -eq 0 ]; then + echo "Documentation build complete. Output: target/docs/htmlsingle/" +else + echo "ERROR: Documentation build failed." fi -popd > /dev/null - exit ${ec} diff --git a/bin/validate-distribution.sh b/bin/validate-distribution.sh index 7c5c3ee851f..1a1c1b738b6 100755 --- a/bin/validate-distribution.sh +++ b/bin/validate-distribution.sh @@ -209,7 +209,7 @@ if [ "${TYPE}" = "CONSOLE" ]; then SCRIPT_FILENAME="test.groovy" SCRIPT_PATH="${TMP_DIR}/${SCRIPT_FILENAME}" echo ${SCRIPT} > ${SCRIPT_PATH} - [[ `bin/gremlin.sh <<< ${SCRIPT} | ${SCRIPT_DIR}/../docs/preprocessor/control-characters.sh | grep '^==>' | sed -e 's/^==>//'` -eq 6 ]] || { echo "failed to evaluate sample script"; exit 1; } + [[ `bin/gremlin.sh <<< ${SCRIPT} | sed "s/\x1b\[[0-9;]*m//g" | tr -d "\r" | grep '^==>' | sed -e 's/^==>//'` -eq 6 ]] || { echo "failed to evaluate sample script"; exit 1; } [[ `bin/gremlin.sh -e ${SCRIPT_PATH}` -eq 6 ]] || { echo "failed to evaluate sample script using -e option"; exit 1; } CONSOLE_DIR=`pwd` cd ${TMP_DIR} diff --git a/docs/postprocessor/postprocess.sh b/docs/postprocessor/postprocess.sh deleted file mode 100755 index e7dca523b84..00000000000 --- a/docs/postprocessor/postprocess.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -pushd "$(dirname $0)/../.." > /dev/null - -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) - -if [ -d "target/docs" ]; then - - # redirect the GLV Tutorial to reference docs - sed -i "s///" target/docs/htmlsingle/tutorials/gremlin-language-variants/index.html - - find target/docs -name index.html | while read file ; do - awk -f "docs/postprocessor/processor.awk" "${file}" 2>/dev/null \ - | perl -0777 -pe 's/\/\*\n \*\/<\/span>//igs' \ - | sed "s/x\.y\.z/${TP_VERSION}/g" \ - > "${file}.tmp" && mv "${file}.tmp" "${file}" - done -fi - -popd > /dev/null diff --git a/docs/postprocessor/processor.awk b/docs/postprocessor/processor.awk deleted file mode 100644 index fbeadcccb44..00000000000 --- a/docs/postprocessor/processor.awk +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BEGIN { - firstMatch=1 - styled=0 -} - -/Licensed to the Apache Software Foundation/ { - isHeader=1 -} - -/<\/style>/ { - if (!styled) { - print ".invisible {color: rgba(0,0,0,0); font-size: 0;}" - styled=1 - } -} - -!// { - if (firstMatch || !isHeader) { - print gensub(/()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3", "g") - } -} - -// { - if (firstMatch || !isHeader) { - print gensub(/\/\/<\/span>[ ]*()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3\\\\<\/span>", "g") - } -} - -/under the License\./ { - firstMatch=0 - isHeader=0 -} diff --git a/docs/preprocessor/awk/cleanup.awk b/docs/preprocessor/awk/cleanup.awk deleted file mode 100644 index b49a2cc772c..00000000000 --- a/docs/preprocessor/awk/cleanup.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+EVALUATED'$/ { evaluated = 1 } -/^==>\-EVALUATED$/ { evaluated = 0 } - -!/^((gremlin> ')|==>)[\+\-]EVALUATED(')?$/ { - if ($0 !~ /^gremlin> pb\([0-9]*\); '----'$/) { - if (!evaluated || $0 !~ /^gremlin> [']?:/) { - if (evaluated && $0 ~ /^==>:/) gsub(/^==>/, "gremlin> ") - if (!evaluated || $0 == "==>----") gsub(/^==>/, "") - if (evaluated) { - if ($0 !~ /^WARN /) print - } else if ($0 !~ /^gremlin> pb\([0-9]*\); / && $0 !~ /^gremlin> $/) { - print - } - } - } -} diff --git a/docs/preprocessor/awk/ignore.awk b/docs/preprocessor/awk/ignore.awk deleted file mode 100644 index 99f55e06d90..00000000000 --- a/docs/preprocessor/awk/ignore.awk +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+IGNORE'$/ { ignore = 1 } -{ if (!ignore) print } -/^==>\-IGNORE$/ { ignore = 0 } diff --git a/docs/preprocessor/awk/init-code-blocks.awk b/docs/preprocessor/awk/init-code-blocks.awk deleted file mode 100644 index 8349d6e6410..00000000000 --- a/docs/preprocessor/awk/init-code-blocks.awk +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function capitalize(string) { - return toupper(substr(string, 1, 1)) substr(string, 2) -} - -BEGIN { - delimiter = 0 -} - -/^pb\([0-9]*\); '\[gremlin-/ { - delimiter = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, c, ",") - split(a[1], d, ";") - lang = c[1] - graph = c[2] - print d[1] "; '[source," lang "]'" - print "'+EVALUATED'" - print "'+IGNORE'" - if (graph != "existing") { - if (graph) { - print "graph = TinkerFactory.create" capitalize(graph) "()" - } else { - print "graph = TinkerGraph.open()" - } - print "g = graph.traversal()" - print "marko = g.V().has('name', 'marko').tryNext().orElse(null)" - print "f = new File('/tmp/neo4j')" - print "if (f.exists()) f.deleteDir()" - print "f = new File('/tmp/tinkergraph.kryo')" - print "if (f.exists()) f.deleteDir()" - print ":set max-iteration 100" - } - print "'-IGNORE'" -} - -!/^pb\([0-9]*\); '\[gremlin-/ { - if (delimiter == 2 && !($0 ~ /^pb\([0-9]*\); '----'/)) { - switch (lang) { - default: - print - break - } - } else print -} - -/^pb\([0-9]*\); '----'/ { - if (delimiter == 1) delimiter = 2 - else if (delimiter == 2) { - print "'-EVALUATED'" - delimiter = 0 - } -} diff --git a/docs/preprocessor/awk/language-variants.awk b/docs/preprocessor/awk/language-variants.awk deleted file mode 100644 index 9d9fbf03dff..00000000000 --- a/docs/preprocessor/awk/language-variants.awk +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - lang = null - inCodeBlock = 0 -} - -/^\[source,/ { - delimiter = 1 - split($0, a, ",") - lang = gensub(/]$/, "", 1, a[2]) -} - -/^----$/ { - if (inCodeBlock == 0) inCodeBlock = 1 - else inCodeBlock = 0 -} - -{ if (inCodeBlock) { - switch (lang) { - default: - print - break - } - } else print -} diff --git a/docs/preprocessor/awk/prepare.awk b/docs/preprocessor/awk/prepare.awk deleted file mode 100644 index 860dd4ee2bb..00000000000 --- a/docs/preprocessor/awk/prepare.awk +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - p = 0 - c = "//" -} - -function escape_string(string) { - str = gensub(/\\/, "\\\\\\\\", "g", string) - return gensub(/'/, "\\\\'", "g", str) -} - -function print_string(string) { - print "pb(" p++ "); '" escape_string(string) "'" -} - -function transform_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, " " c c " \\1", "g", code) -} - -function remove_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, "", "g", code) -} - -/^----$/ { - if (inCodeSection) { - if (prepared) { - inCodeSection = 0 - prepared = 0 - } else { - prepared = 1 - } - } - print_string($0) -} - -!/^----$/ { - if (inCodeSection) { - if ($0 ~ /^:/) { - print "'" escape_string(transform_callouts($0)) "'" - print remove_callouts($0) - } else { - print transform_callouts($0) - } - } else { - print_string($0) - } -} - -/^\[gremlin-/ { - inCodeSection = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, l, ",") - lang = l[1] - switch (lang) { - default: - c = "//" - break - } -} - -END { - print_string("// LAST LINE") -} diff --git a/docs/preprocessor/awk/prettify.awk b/docs/preprocessor/awk/prettify.awk deleted file mode 100644 index 522ac631924..00000000000 --- a/docs/preprocessor/awk/prettify.awk +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^==>\/\/\/\/$/ { doPrint = 1 } - -{ - if (inCodeSection && $0 ~ /^\.*[0-9]+> /) { - gsub(/^.{8}/, " ") - } - if (doPrint) print -} - -/^==>\+EVALUATED/ { inCodeSection = 1 } -/^==>\-EVALUATED/ { inCodeSection = 0 } diff --git a/docs/preprocessor/awk/progressbar.awk b/docs/preprocessor/awk/progressbar.awk deleted file mode 100644 index 25109c42e00..00000000000 --- a/docs/preprocessor/awk/progressbar.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - max = 0 - content = "" -} - -/^pb\([0-9]*\); / { - max = gensub(/^pb\(([0-9]*)\); .*/, "\\1", "g", $0) -} -{ content = content "\n" $0 } - -END { - while ((getline line < tpl) > 0) { - print gensub(/TOTAL_LINES/, max, "g", line) - } - print content -} diff --git a/docs/preprocessor/awk/tabify.awk b/docs/preprocessor/awk/tabify.awk deleted file mode 100644 index 7e68c13e5dd..00000000000 --- a/docs/preprocessor/awk/tabify.awk +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function print_tabs(next_id, tabs, blocks) { - - num_tabs = length(tabs) - x = next_id - - print "++++" - print "
" - - for (i = 1; i <= num_tabs; i++) { - title = tabs[i] - print " " - print " " - x++ - } - - for (i = 1; i<= num_tabs; i++) { - print "
" - print "
" - print "++++\n" - print blocks[i] - print "++++" - print "
" - print "
" - } - - print "
" - print "++++\n" -} - -function transform_callouts(code, c) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*\n/, " " c c " \\1\\2\n", "g", code) -} - -BEGIN { - id_part=systime() - status = 0 - next_id = 1 - block[0] = 0 # initialize "blocks" as an array - delete blocks[0] -} - -/^\[gremlin-/ { - status = 1 - lang = gensub(/^\[gremlin-([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 1 -} - -/^\[source,(csharp|groovy|java|javascript|python|go),tab\]/ { - status = 1 - lang = gensub(/^\[source,([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 0 -} - -/^\[source,(csharp|groovy|java|javascript|python|go)\]/ { - if (status == 3) { - status = 1 - lang = gensub(/^\[source,([^\]]+).*/, "\\1", "g", $0) - code = "" - } -} - -! /^\[source,(csharp|groovy|java|javascript|python|go)/ { - if (status == 3 && $0 != "") { - print_tabs(next_id, tabs, blocks) - next_id = next_id + length(tabs) - for (i in tabs) { - delete tabs[i] - delete blocks[i] - } - status = 0 - } -} - -/^----$/ { - if (status == 1) { - status = 2 - } else if (status == 2) { - status = 3 - } -} - -{ if (status == 3) { - if ($0 == "----") { - i = length(blocks) + 1 - if (i == 1 && evaluate == 1) { - tabs[i] = "console (" lang ")" - blocks[i] = code_header code "\n" $0 "\n" - i++ - } - tabs[i] = lang - switch (lang) { - default: - c = "//" - break - } - blocks[i] = "[source," lang "]" transform_callouts(code, c) "\n" $0 "\n" - } - } else { - if (status == 0) print - else if (status == 1) code_header = gensub(/,tab/, "", "g", $0) - else code = code "\n" $0 - } -} - -END { - # EOF - if (status == 3) { - print_tabs(next_id, tabs, blocks) - } -} diff --git a/docs/preprocessor/install-plugins.groovy b/docs/preprocessor/install-plugins.groovy deleted file mode 100644 index 0edbfdbd48e..00000000000 --- a/docs/preprocessor/install-plugins.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * @author Daniel Kuppitz (http://gremlin.guru) - */ -import org.apache.tinkerpop.gremlin.groovy.util.Artifact -import org.apache.tinkerpop.gremlin.groovy.util.DependencyGrabber - -installPlugin = { def artifact -> - def classLoader = new groovy.lang.GroovyClassLoader() - def extensionPath = new File(System.getProperty("user.dir"), "ext") - try { - System.err.print(" * ${artifact.getArtifact()} ... ") - new DependencyGrabber(classLoader, extensionPath).copyDependenciesToPath(artifact) - System.err.println("done") - } catch (Exception e) { - System.err.println("failed") - System.err.println() - System.err.println(e.getMessage()) - e.printStackTrace() - System.exit(1) - } -} - -:plugin use tinkerpop.sugar -:plugin use tinkerpop.credentials -System.err.println("done") diff --git a/docs/preprocessor/install-plugins.sh b/docs/preprocessor/install-plugins.sh deleted file mode 100755 index ba882132076..00000000000 --- a/docs/preprocessor/install-plugins.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TP_VERSION=$2 -TMP_DIR=$3 -INSTALL_TEMPLATE="docs/preprocessor/install-plugins.groovy" -INSTALL_FILE="${TMP_DIR}/install-plugins.groovy" - -plugins=("hadoop-gremlin" "spark-gremlin" "neo4j-gremlin" "sparql-gremlin") -# plugins=() -pluginsCount=${#plugins[@]} - -i=0 - -cp ${INSTALL_TEMPLATE} ${INSTALL_FILE} - -while [ ${i} -lt ${pluginsCount} ]; do - pluginName=${plugins[$i]} - className="" - for part in $(tr '-' '\n' <<< ${pluginName}); do - className="${className}$(tr '[:lower:]' '[:upper:]' <<< ${part:0:1})${part:1}" - done - pluginClassFile=$(find . -name "${className}Plugin.java") - pluginClass=`sed -e 's@.*src/main/java/@@' -e 's/\.java$//' <<< ${pluginClassFile} | tr '/' '.'` - installed=`grep -c "${pluginClass}" ${CONSOLE_HOME}/ext/plugins.txt` - if [ ${installed} -eq 0 ]; then - echo "installPlugin(new Artifact(\"org.apache.tinkerpop\", \"${pluginName}\", \"${TP_VERSION}\"))" >> ${INSTALL_FILE} - echo "${pluginName}" >> ${TMP_DIR}/plugins.dir - echo "${pluginClass}" >> ${TMP_DIR}/plugins.txt - else - echo " * skipping ${pluginName} (already installed)" - fi - ((i++)) -done - -echo "System.exit(0)" >> ${INSTALL_FILE} -echo -ne " * tinkerpop-sugar ... " - -pushd ${CONSOLE_HOME} > /dev/null - -mkdir -p ~/.java/.userPrefs -chmod 700 ~/.java/.userPrefs - -bin/gremlin.sh -e ${INSTALL_FILE} > /dev/null - -if [ ${PIPESTATUS[0]} -ne 0 ]; then - popd > /dev/null - exit 1 -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - cat ${TMP_DIR}/plugins.txt >> ${CONSOLE_HOME}/ext/plugins.txt -fi - -popd > /dev/null diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh deleted file mode 100755 index 9ae43c2f8d0..00000000000 --- a/docs/preprocessor/preprocess-file.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -TP_HOME=`pwd` -CONSOLE_HOME=$1 -AWK_SCRIPTS="${TP_HOME}/docs/preprocessor/awk" - -IFS=',' read -r -a DRYRUN_DOCS <<< "$2" -IFS=',' read -r -a FULLRUN_DOCS <<< "$3" - -dryRun () { - local doc - yes=0 - no=1 - doDryRun=${no} - if [ "${DRYRUN_DOCS}" == "*" ]; then - doDryRun=${yes} - else - for doc in "${DRYRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${yes} - break - fi - done - fi - if [ ${doDryRun} ]; then - for doc in "${FULLRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${no} - break - fi - done - fi - return ${doDryRun} -} - -input=$4 -output=`sed 's@/docs/src/@/target/postprocess-asciidoc/@' <<< "${input}"` - -SKIP= -if dryRun ${input}; then - SKIP=1 -fi - -mkdir -p `dirname ${output}` - -if hash stdbuf 2> /dev/null; then - lb="stdbuf -oL" -else - lb="" -fi - -trap cleanup INT - -function cleanup { - if [ -f "${output}" ]; then - if [ `wc -l "${output}" | awk '{print $1}'` -gt 0 ]; then - echo -e "\n\e[1mLast 10 lines of ${output}:\e[0m\n" - tail -n10 ${output} - echo - echo "Opening ${output} for full inspection" - sleep 5 - less ${output} - fi - fi - rm -rf ${output} ${CONSOLE_HOME}/.ext - exit 255 -} - -function processed { - echo -ne "\r progress: [====================================================================================================] 100%\n" -} - -echo -echo " * source: ${input}" -echo " target: ${output}" -echo -ne " progress: initializing" - -if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then - if [ ${output} -nt ${input} ]; then - processed - exit 0 - fi - pushd "${CONSOLE_HOME}" > /dev/null - - doc=`basename ${input} .asciidoc` - - case "${doc}" in - "implementations-neo4j") - # deactivate Spark plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/spark-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'SparkGremlinPlugin' > .ext/plugins.txt - ;; - "implementations-hadoop-start" | "implementations-hadoop-end" | "implementations-spark" | "olap-spark-yarn") - # deactivate Neo4j plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/neo4j-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'Neo4jGremlinPlugin' > .ext/plugins.txt - ;; - "gremlin-variants") - # deactivate plugin to prevent version conflicts - mkdir .ext - mv ext/neo4j-gremlin .ext/ - mv ext/spark-gremlin .ext/ - mv ext/hadoop-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -v 'Neo4jGremlinPlugin\|SparkGremlinPlugin\|HadoopGremlinPlugin' > .ext/plugins.txt - ;; - esac - - if [ -d ".ext" ]; then - mv .ext/plugins.txt ext/ - fi - - sed 's/\t/ /g' ${input} | - awk -f ${AWK_SCRIPTS}/tabify.awk | - awk -f ${AWK_SCRIPTS}/prepare.awk | - awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" | - awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | - HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh | ${TP_HOME}/docs/preprocessor/control-characters.sh | - ${lb} awk -f ${AWK_SCRIPTS}/ignore.awk | - ${lb} awk -f ${AWK_SCRIPTS}/prettify.awk | - ${lb} awk -f ${AWK_SCRIPTS}/cleanup.awk | - ${lb} awk -f ${AWK_SCRIPTS}/language-variants.awk > ${output} - - # check exit code for each of the previously piped commands - ps=(${PIPESTATUS[@]}) - for i in {0..9}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - - if [ -d ".ext" ]; then - mv .ext/plugins.all ext/plugins.txt - mv .ext/* ext/ - rm -r .ext/ - fi - - if [ ${ec} -eq 0 ]; then - tail -n1 ${output} | grep -F '// LAST LINE' > /dev/null - ec=$? - fi - - if [ ${ec} -eq 0 ]; then - processed - fi - - echo - popd > /dev/null - if [ ${ec} -ne 0 ]; then - cleanup - fi -else - cp ${input} ${output} - processed -fi diff --git a/docs/preprocessor/preprocess.sh b/docs/preprocessor/preprocess.sh deleted file mode 100755 index 3c05d946e63..00000000000 --- a/docs/preprocessor/preprocess.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -DRYRUN_DOCS="$1" -FULLRUN_DOCS="$2" - -pushd "$(dirname $0)/../.." > /dev/null - -if [ "${DRYRUN_DOCS}" != "*" ]; then - - if [ ! -f bin/gremlin.sh ]; then - echo "Gremlin REPL is not available. Cannot preprocess AsciiDoc files." - popd > /dev/null - exit 1 - fi - - for daemon in "NameNode" "DataNode" "ResourceManager" "NodeManager" - do - running=`jps | cut -d ' ' -f2 | grep -c ${daemon}` - if [ ${running} -eq 0 ]; then - echo "Hadoop is not running, be sure to start it before processing the docs." - exit 1 - fi - done - - netstat -an | awk '{print $4}' | grep -o '[0-9]*$' | grep '\b8182\b' > /dev/null && { - echo "The port 8182 is required for Gremlin Server, but it is already in use. Be sure to close the application that currently uses the port before processing the docs." - exit 1 - } - - if [ -e /tmp/neo4j ]; then - echo "The directory '/tmp/neo4j' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi - - if [ -e /tmp/tinkergraph.kryo ]; then - echo "The file '/tmp/tinkergraph.kryo' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi -fi - -function directory { - d1=`pwd` - cd $1 - d2=`pwd` - cd $d1 - echo "$d2" -} - -mkdir -p target/postprocess-asciidoc/tmp -mkdir -p target/postprocess-asciidoc/logs -cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - -TP_HOME=`pwd` -CONSOLE_HOME=`directory "${TP_HOME}/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone"` -PLUGIN_DIR="${CONSOLE_HOME}/ext" -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) -TMP_DIR="/tmp/tp-docs-preprocessor" - -mkdir -p "${TMP_DIR}" - -HISTORY_FILE=".gremlin_groovy_history" -[ -f ~/${HISTORY_FILE} ] && cp ~/${HISTORY_FILE} ${TMP_DIR} - -pushd gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone > /dev/null -bin/gremlin-server.sh conf/gremlin-server-modern.yaml > ${TP_HOME}/target/postprocess-asciidoc/logs/gremlin-server.log 2>&1 & -GREMLIN_SERVER_PID=$! -popd > /dev/null - -function cleanup() { - echo -ne "\r\n\n" - docs/preprocessor/uninstall-plugins.sh "${CONSOLE_HOME}" "${TMP_DIR}" - [ -f ${TMP_DIR}/plugins.txt.orig ] && mv ${TMP_DIR}/plugins.txt.orig ${CONSOLE_HOME}/ext/plugins.txt - find ${TP_HOME}/docs/src/ -name "*.asciidoc.groovy" | xargs rm -f - [ -f ${TMP_DIR}/${HISTORY_FILE} ] && mv ${TMP_DIR}/${HISTORY_FILE} ~/ - rm -rf ${TMP_DIR} - kill ${GREMLIN_SERVER_PID} &> /dev/null - popd &> /dev/null -} - -trap cleanup EXIT - -if [ "${DRYRUN_DOCS}" != "*" ] || [ ! -z "${FULLRUN_DOCS}" ]; then - - # install plugins - echo - echo "==========================" - echo "+ Installing Plugins +" - echo "==========================" - echo - cp ${CONSOLE_HOME}/ext/plugins.txt ${TMP_DIR}/plugins.txt.orig - docs/preprocessor/install-plugins.sh "${CONSOLE_HOME}" "${TP_VERSION}" "${TMP_DIR}" - - if [ $? -ne 0 ]; then - exit 1 - else - echo - fi - -fi - -# process *.asciidoc files -COLS=${COLUMNS} -[[ ${COLUMNS} -lt 240 ]] && stty cols 240 - -tput rmam - -echo -echo "============================" -echo "+ Processing AsciiDocs +" -echo "============================" - -ec=0 -for subdir in $(find "${TP_HOME}/docs/src/" -name index.asciidoc | xargs -n1 dirname) -do - find "${subdir}" -maxdepth 1 -name "*.asciidoc" | - xargs -n1 basename | - xargs -n1 -I {} echo "echo -ne {}' '; (grep -n {} ${subdir}/index.asciidoc || echo 0) | head -n1 | cut -d ':' -f1" | /bin/bash | sort -nk2 | cut -d ' ' -f1 | - xargs -n1 -I {} echo "${subdir}/{}" | - xargs -n1 ${TP_HOME}/docs/preprocessor/preprocess-file.sh "${CONSOLE_HOME}" "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - - ps=(${PIPESTATUS[@]}) - for i in {0..7}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - [ ${ec} -eq 0 ] || break -done - -tput smam -[[ "${COLUMNS}" != "" ]] && stty cols ${COLS} - -rm -rf /tmp/neo4j /tmp/tinkergraph.kryo - -[ ${ec} -eq 0 ] || exit 1 - -echo diff --git a/docs/preprocessor/uninstall-plugins.sh b/docs/preprocessor/uninstall-plugins.sh deleted file mode 100755 index 6353fe50f71..00000000000 --- a/docs/preprocessor/uninstall-plugins.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TMP_DIR=$2 - -if [ -f "${TMP_DIR}/plugins.dir" ]; then - for pluginDirectory in $(cat ${TMP_DIR}/plugins.dir); do - rm -rf ${CONSOLE_HOME}/ext/${pluginDirectory} - done -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - for className in $(cat ${TMP_DIR}/plugins.txt); do - sed -e "/${className}/d" ${CONSOLE_HOME}/ext/plugins.txt > ${CONSOLE_HOME}/ext/plugins.txt. - mv ${CONSOLE_HOME}/ext/plugins.txt. ${CONSOLE_HOME}/ext/plugins.txt - done -fi diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 5756bc8da88..dded1db342d 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -525,7 +525,7 @@ mvn -Dmaven.javadoc.skip=true --projects tinkergraph-gremlin test ** Build AsciiDocs (but don't evaluate code blocks): `bin/process-docs.sh --dryRun` ** Build AsciiDocs (but don't evaluate code blocks in specific files): `bin/process-docs.sh --dryRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` ** Build AsciiDocs (but evaluate code blocks only in specific files): `bin/process-docs.sh --fullRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` -** Process a single AsciiDoc file: +pass:[docs/preprocessor/preprocess-file.sh `pwd`/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone "" "*" `pwd`/docs/src/xyz.asciidoc]+ +** Process docs in dry-run mode (no console required): `bin/process-docs.sh --dryRun` * Build JavaDocs/JSDoc: `mvn process-resources -Djavadoc` ** Javadoc to `target/site/apidocs` directory ** JSDoc to the `gremlin-javascript/src/main/javascript/gremlin-javascript/doc/` directory diff --git a/pom.xml b/pom.xml index 64fb4c4f143..20ca0d968fd 100644 --- a/pom.xml +++ b/pom.xml @@ -972,6 +972,11 @@ limitations under the License. ${project.basedir}/target/docs/htmlsingle ${asciidoc.input.dir}/stylesheets + + + + + false @@ -1038,6 +1043,13 @@ limitations under the License. org.asciidoctor asciidoctor-maven-plugin false + + + org.apache.tinkerpop + tinkerpop-docs + ${project.version} + + home @@ -1072,6 +1084,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1099,6 +1114,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1126,6 +1144,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1153,6 +1174,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1180,6 +1204,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1207,6 +1234,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1234,6 +1264,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1261,6 +1294,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1286,6 +1322,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1312,6 +1351,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1338,6 +1380,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} @@ -1363,6 +1408,9 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} diff --git a/tools/tinkerpop-docs/REQUIREMENTS.md b/tools/tinkerpop-docs/REQUIREMENTS.md new file mode 100644 index 00000000000..382e7e74bb8 --- /dev/null +++ b/tools/tinkerpop-docs/REQUIREMENTS.md @@ -0,0 +1,216 @@ +# TinkerPop Documentation Build - Requirements + +## Overview + +The documentation build system generates TinkerPop's reference documentation from AsciiDoc sources. It processes `[gremlin-groovy]` code blocks by executing them against a real Gremlin Console and producing tabbed output with console results and manual language variant tabs. + +## Architecture + +### Components + +1. **`tinkerpop-docs`** - An AsciidoctorJ extension (standalone Maven module under `tools/`) that processes gremlin code blocks during AsciiDoc rendering. +2. **`bin/process-docs.sh`** - Shell script that orchestrates the full build: installs plugins, starts required services, and invokes Maven/Asciidoctor. +3. **Gremlin Console** - A long-running console subprocess used for code execution (one per document/book). +4. **Gremlin Server** - Started for remote connection examples. +5. **Gephi Mock** - A mock HTTP server (`bin/gephi-mock.py`) for Gephi plugin examples. + +### Module Location + +The `tinkerpop-docs` module lives at `tools/tinkerpop-docs/`. It uses the root `tinkerpop` pom as its Maven parent (for version inheritance) but is NOT part of the main reactor. It must be built separately before running the docs build. + +### Extension Type + +- **Treeprocessor**: Walks the AST after parsing, finds `[gremlin-groovy]` listing blocks by style attribute, replaces them with tabbed HTML pass blocks. +- **Postprocessor**: Applies callout fixes, removes empty comment spans, replaces `x.y.z` version placeholder in rendered HTML. +- Registered via SPI (`META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry`). +- Targets **AsciidoctorJ 2.5.x** for compatibility with existing asciidoctor-maven-plugin 2.2.4. + +## Functional Requirements + +### FR-1: Gremlin Code Block Processing + +The extension SHALL walk the AsciiDoc AST and transform listing blocks with style `gremlin-groovy` into tabbed HTML output containing: + +- A "console ()" tab showing the Gremlin prompt, input, and execution output (syntax highlighted as Groovy via coderay) + +### FR-2: Console Execution + +- The extension SHALL execute code blocks against a real Gremlin Console process to produce authentic console output. +- The console process SHALL be started once per document (book-scoped) and reused across all blocks, maintaining session state. +- The console SHALL have TinkerGraph, SPARQL, Hadoop, and Spark plugins activated. + +**Note:** The old system used per-file console sessions. The new book-scoped approach is safe because every non-`existing` block reinitializes its graph. This scope change should be noted in commit messages and PR descriptions. + +### FR-3: Graph Initialization + +For each block, the extension SHALL inject initialization code before executing the block's content: + +- `[gremlin-groovy,]`: `graph = TinkerFactory.create(); g = graph.traversal()` +- `[gremlin-groovy]` (no graph): `graph = TinkerGraph.open(); g = graph.traversal()` +- `[gremlin-groovy,existing]`: No initialization (reuses current state) + +Initialization also includes: +- Cleanup: delete `/tmp/neo4j` and `/tmp/tinkergraph.kryo` if they exist +- `:set max-iteration 100` +- Graph initialization SHALL be skipped when consecutive blocks use the same graph (NFR-4) + +Supported graph names: `modern`, `classic`, `theCrew`, `kitchenSink`, `gratefulDead` + +### FR-4: Error Handling in Code Blocks + +- When a Gremlin statement produces a runtime error, the extension SHALL capture the error message and include it in the console output — this IS the expected output for documentation purposes. +- Errors in code blocks SHALL NOT fail the docs build. +- The "Display stack trace? [yN]" prompt SHALL be dismissed automatically without user intervention. + +### FR-5: Manual Language Variant Tabs + +- When `[source,]` blocks immediately follow a `[gremlin-groovy]` block (as consecutive AST siblings), they SHALL be consumed and rendered as additional tabs alongside the console output. +- Supported languages: `groovy`, `java`, `csharp`, `javascript`, `python`, `go` +- Consumption stops at the first non-matching sibling block. + +### FR-7: Standalone Tab Groups + +- `[source,,tab]` blocks SHALL be grouped into tabbed UI independently of gremlin blocks. +- Consecutive `[source,]` blocks following a `[source,,tab]` block are grouped together. + +### FR-8: Dry-Run Mode + +- When `gremlin-docs-dryrun` attribute is set, the extension SHALL skip console execution. +- Code blocks SHALL be formatted with `gremlin>` prompts but no execution output. +- Tab structure (manual tabs, standalone tab groups) SHALL still render fully. +- Dry-run mode SHALL NOT require a built Gremlin Console or Server. + +## Out of Scope (Initial Release) + +### FR-6: Auto-Translation (DEFERRED) + +Auto-translation of Gremlin to language variants via GremlinTranslator is not included in the initial scope. Only manual language variant tabs (FR-5) are supported. + +### FR-9: Rouge Syntax Highlighting (DEFERRED) + +Existing coderay highlighting is retained. Rouge would fix C# highlighting but this is a pre-existing issue. + +## Non-Functional Requirements + +### NFR-1: Fail-Fast on Missing Dependencies + +- If the `gremlin-docs-console-home` attribute is not set and the build is NOT in dry-run mode, the extension SHALL throw an error immediately with a clear message indicating the Console must be built. + +### NFR-2: Timeout Safety Net + +- Console communication SHALL have a 30-second timeout to prevent infinite hangs from unexpected console states. +- A timeout SHALL fail the build with a clear error message including the buffer contents at the time of timeout. + +### NFR-3: Progress Visibility + +- The extension SHALL log (at INFO level) each gremlin block being processed, including the first line of code. +- The extension SHALL log each statement being executed. +- Console stderr output SHALL be logged at INFO level for diagnostic visibility. + +### NFR-4: Build Performance + +- The Console communication protocol SHALL NOT introduce artificial delays (no polling timeouts for expected responses). +- Graph initialization SHALL be skipped when consecutive blocks use the same graph. + +## Console Communication Protocol + +### Prompt-Based Boundary Detection + +The extension communicates with the Gremlin Console using prompt-based boundary detection: + +1. The console outputs `gremlin>` (without trailing newline) after each statement completes. +2. The extension reads character-by-character and detects when the buffer ends with `gremlin>`. +3. Continuation prompts (`......N>`) are skipped — only `gremlin>` signals statement completion. + +### Error Prompt Handling + +When a statement errors, the console writes "Display stack trace? [yN]" to stderr and blocks reading from stdin: + +1. A stderr-draining thread detects "Display stack trace?" and sets a flag. +2. The main thread's read loop checks this flag while waiting for stdout data. +3. When the flag is set, a blank line is sent to stdin to dismiss the prompt. +4. The console then prints `gremlin>` to stdout, unblocking the read. + +### Pre-Statement Dismissal + +Before each statement, any pending error prompt from a previous call is dismissed: + +1. Check the `errorPromptPending` flag. +2. If set, send a blank line and wait for the resulting `gremlin>` prompt. + +## Tabbed HTML Output + +### Structure + +Tab HTML uses CSS-only radio button tabs matching existing `tinkerpop.css` styles: +- `
` container +- Radio inputs + labels for tab switching +- `
` panels +- Tab IDs use a deterministic sequential counter (no timestamps) + +### Console Tab + +- Labeled "console ()" (e.g., "console (groovy)") +- Content wrapped as `[source,groovy]` for coderay syntax highlighting +- Callout markers rendered as `// ` comments + +### Plugin Conflict Management + +Plugin conflicts between chapters are managed via AsciiDoc document attributes: + +```asciidoc +// Exclude spark-gremlin to avoid Spark jar conflicts with Neo4j's Spark dependencies +:gremlin-docs-plugins-exclude: spark-gremlin +``` + +When the extension encounters this attribute, it shuts down the current console, toggles plugin directories, and starts a new console. Each usage must include an inline comment explaining the conflict. + +## Build Invocation + +### Full Build (with execution) + +``` +mvn clean install -pl :gremlin-server,:gremlin-console -am -DskipTests +bin/process-docs.sh +``` + +### Dry-Run (layout only, no Console needed) + +``` +bin/process-docs.sh --dry-run +``` + +### Output + +Final HTML documentation is placed in `target/docs/htmlsingle/`. + +## Dependencies + +| Dependency | Purpose | +|---|---| +| Gremlin Console distribution | Code execution | +| Gremlin Server distribution | Remote connection examples | +| `sparql-gremlin` plugin | SPARQL examples | +| `hadoop-gremlin` plugin | Hadoop examples (local filesystem mode) | +| `spark-gremlin` plugin | Spark examples (local mode) | +| `gephi-mock.py` | Gephi plugin examples | + +### Hadoop Configuration + +The docs build does NOT require an external Hadoop cluster. A docs-specific `core-site.xml` configures `fs.defaultFS=file:///` so the `hdfs` binding operates on the local filesystem. `spark.master=local[4]` and `jarsInDistributedCache=false` ensure Spark runs in-process. + +## Configuration + +The extension is configured via Asciidoctor document attributes: + +| Attribute | Description | +|---|---| +| `gremlin-docs-console-home` | Path to the unpacked Gremlin Console distribution | +| `gremlin-docs-hadoop-libs` | Path to Hadoop library jars for HADOOP_GREMLIN_LIBS | +| `gremlin-docs-dryrun` | When present, enables dry-run mode | +| `gremlin-docs-plugins-exclude` | Comma-separated list of plugins to exclude (triggers console restart) | + +These attributes are passed via the Maven command line (resolved by `bin/process-docs.sh`): +``` +-Dasciidoctor.attributes="gremlin-docs-console-home=/path/to/console gremlin-docs-hadoop-libs=/path/to/libs" +``` diff --git a/tools/tinkerpop-docs/pom.xml b/tools/tinkerpop-docs/pom.xml new file mode 100644 index 00000000000..b7d3c4db166 --- /dev/null +++ b/tools/tinkerpop-docs/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + org.apache.tinkerpop + tinkerpop + 3.7.7-SNAPSHOT + ../../pom.xml + + tinkerpop-docs + Apache TinkerPop :: Docs Extension + AsciidoctorJ extension for processing gremlin code blocks in TinkerPop documentation + jar + + + 2.5.13 + + + + + org.asciidoctor + asciidoctorj + ${asciidoctorj.version} + provided + + + junit + junit + ${junit.version} + test + + + org.hamcrest + hamcrest + ${hamcrest.version} + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.0 + + + 11 + + + + org.apache.rat + apache-rat-plugin + + + REQUIREMENTS.md + src/main/resources/META-INF/services/** + + + + + + diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java new file mode 100644 index 00000000000..946c97c6ac7 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.io.IOException; +import java.util.List; + +/** + * Callback interface invoked by the {@link GremlinTreeprocessor} when the document signals + * that the console needs to be restarted with certain plugins excluded. + *

+ * The orchestration layer implements this to close the current console, toggle/rename plugin + * directories, and start a new console instance. + */ +@FunctionalInterface +public interface ConsoleRestartHandler { + + /** + * Called when the document attribute {@code :gremlin-docs-plugins-exclude:} is detected + * with a changed value. The handler should close the current console, exclude the specified + * plugin directories, and start a new console. + * + * @param excludedPlugins list of plugin directory names to exclude (e.g., "neo4j-gremlin") + * @throws IOException if the restart operation fails + */ + void onRestart(List excludedPlugins) throws IOException; +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java new file mode 100644 index 00000000000..d1823407a6d --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.file.Path; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * Manages a long-running Gremlin Console subprocess, sending statements and capturing output. + *

+ * Uses prompt-based boundary detection: reads stdout until the {@code gremlin>} prompt appears. + * Automatically dismisses {@code Display stack trace?} error prompts on stderr. + */ +public class GremlinConsole implements Closeable { + + static final String PROMPT = "gremlin>"; + static final String ERROR_PROMPT = "Display stack trace?"; + private static final long DEFAULT_TIMEOUT_MS = 120_000; + private static final int EOF = -1; + + private final Process process; + private final Writer stdin; + private final BufferedReader stdout; + private final BufferedReader stderr; + private final Object stderrStdinLock = new Object(); + private final Thread errorDismisser; + private final Thread stdoutReaderThread; + private final BlockingQueue stdoutQueue = new LinkedBlockingQueue<>(); + private final long timeoutMs; + private volatile boolean running; + private boolean inEscape; + + /** + * Creates a new GremlinConsole by starting the {@code bin/gremlin.sh} subprocess. + * + * @param tinkerpopHome path to the TinkerPop distribution home directory + */ + public GremlinConsole(final Path tinkerpopHome) throws IOException, ConsoleTimeoutException { + this(new ProcessBuilder(tinkerpopHome.resolve("bin/gremlin.sh").toString()) + .redirectErrorStream(false) + .start(), DEFAULT_TIMEOUT_MS); + } + + /** + * Package-private constructor for testing with a pre-built process. + */ + GremlinConsole(final Process process, final long timeoutMs) throws IOException, ConsoleTimeoutException { + this.process = process; + this.stdin = new OutputStreamWriter(process.getOutputStream()); + this.stdout = new BufferedReader(new InputStreamReader(process.getInputStream())); + this.stderr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + this.timeoutMs = timeoutMs; + this.running = true; + + this.stdoutReaderThread = new Thread(this::readStdout, "gremlin-console-stdout-reader"); + this.stdoutReaderThread.setDaemon(true); + this.stdoutReaderThread.start(); + + this.errorDismisser = new Thread(this::dismissErrorPrompts, "gremlin-console-error-dismisser"); + this.errorDismisser.setDaemon(true); + this.errorDismisser.start(); + + // Wait for initial prompt + readUntilPrompt(); + } + + /** + * Sends a statement to the console and returns the output (excluding the prompt line). + * + * @param statement the Gremlin statement to execute + * @return the console output for this statement + * @throws IOException if an I/O error occurs + * @throws ConsoleTimeoutException if the prompt is not received within the timeout period + */ + public String execute(final String statement) throws IOException, ConsoleTimeoutException { + dismissPendingErrorPrompt(); + stdin.write(statement + "\n"); + stdin.flush(); + return readUntilPrompt(); + } + + @Override + public void close() { + running = false; + errorDismisser.interrupt(); + stdoutReaderThread.interrupt(); + process.destroyForcibly(); + try { + process.waitFor(5, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + closeQuietly(stdin); + closeQuietly(stdout); + closeQuietly(stderr); + } + + /** + * Background thread that reads stdout and feeds characters into the blocking queue. + */ + private void readStdout() { + try { + while (running) { + final int ch = stdout.read(); + stdoutQueue.put(ch); + if (ch == EOF) return; + } + } catch (final IOException | InterruptedException e) { + // Process closed or interrupted, signal EOF + try { + stdoutQueue.put(EOF); + } catch (final InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Reads from the stdout queue character-by-character until the prompt is detected. + */ + private String readUntilPrompt() throws IOException, ConsoleTimeoutException { + final StringBuilder buffer = new StringBuilder(); + final long deadline = System.currentTimeMillis() + timeoutMs; + + while (true) { + final long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new ConsoleTimeoutException( + "Timed out after " + timeoutMs + "ms waiting for gremlin> prompt. Buffer contents:\n" + buffer); + } + + final Integer ch; + try { + ch = stdoutQueue.poll(Math.min(remaining, 100), TimeUnit.MILLISECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while reading console output", e); + } + + if (ch == null) continue; + + if (ch == EOF) { + throw new IOException("Console process ended unexpectedly. Buffer contents:\n" + buffer); + } + final char c = (char) ch.intValue(); + // Skip ANSI escape sequences (ESC [ ... letter) + if (c == '\u001B') { + inEscape = true; + continue; + } + if (inEscape) { + if (Character.isLetter(c)) { + inEscape = false; + } + continue; + } + buffer.append(c); + + if (buffer.length() >= PROMPT.length()) { + final String tail = buffer.substring(buffer.length() - PROMPT.length()); + if (tail.equals(PROMPT)) { + final String output = buffer.substring(0, buffer.length() - PROMPT.length()); + return output.trim(); + } + } + } + } + + /** + * Dismisses any pending error prompt before sending a new statement. + */ + private void dismissPendingErrorPrompt() throws IOException { + synchronized (stderrStdinLock) { + if (stderr.ready()) { + final StringBuilder errBuf = new StringBuilder(); + while (stderr.ready()) { + errBuf.append((char) stderr.read()); + } + if (errBuf.toString().contains(ERROR_PROMPT)) { + stdin.write("\n"); + stdin.flush(); + } + } + } + } + + /** + * Background thread that monitors stderr and automatically dismisses error prompts. + * Uses a sliding window of the last N characters where N = ERROR_PROMPT.length(). + */ + private void dismissErrorPrompts() { + final int windowSize = ERROR_PROMPT.length(); + final StringBuilder errBuffer = new StringBuilder(); + try { + while (running) { + synchronized (stderrStdinLock) { + if (stderr.ready()) { + final int ch = stderr.read(); + if (ch == -1) return; + errBuffer.append((char) ch); + + if (errBuffer.toString().contains(ERROR_PROMPT)) { + stdin.write("\n"); + stdin.flush(); + errBuffer.setLength(0); + } else if (errBuffer.length() > windowSize) { + errBuffer.delete(0, errBuffer.length() - windowSize); + } + } + } + Thread.sleep(50); + } + } catch (final InterruptedException e) { + // Expected on shutdown + } catch (final IOException e) { + // Process closed, exit silently + } + } + + private static void closeQuietly(final Closeable closeable) { + try { + if (closeable != null) closeable.close(); + } catch (final IOException ignored) { + // ignore + } + } + + /** + * Thrown when the console does not produce a prompt within the timeout period. + */ + public static class ConsoleTimeoutException extends Exception { + public ConsoleTimeoutException(final String message) { + super(message); + } + } + + /** + * @deprecated Use {@link ConsoleTimeoutException} instead. + */ + @Deprecated + public static class TimeoutException extends ConsoleTimeoutException { + public TimeoutException(final String message) { + super(message); + } + } +} diff --git a/docs/preprocessor/awk/progressbar.groovy.template b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java similarity index 54% rename from docs/preprocessor/awk/progressbar.groovy.template rename to tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java index 7fcafea8e9c..a6e2e1400fd 100644 --- a/docs/preprocessor/awk/progressbar.groovy.template +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java @@ -16,20 +16,22 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.jruby.extension.spi.ExtensionRegistry; /** - * @author Daniel Kuppitz (http://gremlin.guru) + * SPI entry point that registers the TinkerPop documentation extensions with AsciidoctorJ. + * Registers both the Treeprocessor (for gremlin block processing) and the Postprocessor + * (for callout fixes, version replacement, etc.) */ -pb = { def progress -> - def barLength = 100 - def ratio = barLength / 100 - def builder = new StringBuilder() - def percent = (int) ((progress / TOTAL_LINES) * 100) - def progressLength = (int) ((progress / TOTAL_LINES) * (100 * ratio)) - builder.append('=' * progressLength) - if (progressLength < barLength) { - builder.append('>') - builder.append(' ' * (barLength - progressLength - 1)) - } - System.err.print(String.format("\r progress: [%s] %s", builder, "${percent}%")) +public class GremlinDocsExtension implements ExtensionRegistry { + + @Override + public void register(final Asciidoctor asciidoctor) { + asciidoctor.javaExtensionRegistry() + .treeprocessor(GremlinTreeprocessor.class) + .postprocessor(GremlinPostprocessor.class); + } } diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java new file mode 100644 index 00000000000..1698252f962 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Document; +import org.asciidoctor.extension.Postprocessor; + +import java.util.regex.Pattern; + +/** + * Postprocessor that applies callout fixes, removes empty comment spans, + * and replaces x.y.z version placeholders with the actual TinkerPop version. + */ +public class GremlinPostprocessor extends Postprocessor { + + // Matches or (with possible other classes) + private static final Pattern CONUM_PATTERN = Pattern.compile( + "<([ib])\\s+class=\"conum\""); + + // Matches // preceding a callout marker, wraps in hide-when-copy span + private static final Pattern COMMENT_BEFORE_CONUM_PATTERN = Pattern.compile( + "//\\s*(<[ib] class=\"conum)"); + + // Matches empty comment spans from CodeRay: /* */ + private static final Pattern EMPTY_COMMENT_SPAN_PATTERN = Pattern.compile( + "/\\*\\s*\\*/"); + + @Override + public String process(final Document document, final String output) { + String result = output; + + // 1. Callout fix: add invisible class to conum elements + result = CONUM_PATTERN.matcher(result).replaceAll("<$1 class=\"conum invisible\""); + + // 2. Wrap // before callouts in hide-when-copy span + result = COMMENT_BEFORE_CONUM_PATTERN.matcher(result).replaceAll( + "// $1"); + + // 3. Remove empty comment spans + result = EMPTY_COMMENT_SPAN_PATTERN.matcher(result).replaceAll(""); + + // 4. Replace x.y.z with actual version + final String version = resolveVersion(document); + if (version != null) { + result = result.replace("x.y.z", version); + } + + return result; + } + + private String resolveVersion(final Document document) { + Object version = document.getAttribute("tinkerpop-version"); + if (version != null) { + return version.toString(); + } + version = document.getAttribute("revnumber"); + if (version != null) { + return version.toString(); + } + return null; + } +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java new file mode 100644 index 00000000000..3c731ea454d --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Block; +import org.asciidoctor.ast.Document; +import org.asciidoctor.ast.StructuralNode; +import org.asciidoctor.extension.Treeprocessor; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Walks the AsciiDoc AST after parsing and finds listing blocks with style {@code gremlin-groovy}. + * Executes the Gremlin code against a {@link GremlinConsole} and replaces block content with + * formatted console output. + */ +public class GremlinTreeprocessor extends Treeprocessor { + + private static final Logger LOG = Logger.getLogger(GremlinTreeprocessor.class.getName()); + + private static final String STYLE = "gremlin-groovy"; + private static final String PROMPT = "gremlin> "; + private static final String EXISTING = "existing"; + private static final String TAB_ATTR = "tab"; + static final String PLUGINS_EXCLUDE_ATTR = "gremlin-docs-plugins-exclude"; + + static final Set SUPPORTED_LANGUAGES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("groovy", "java", "csharp", "javascript", "python", "go"))); + + static final Map GRAPH_INIT; + + static { + final Map m = new HashMap<>(); + m.put("modern", "graph = TinkerFactory.createModern()"); + m.put("classic", "graph = TinkerFactory.createClassic()"); + m.put("crew", "graph = TinkerFactory.createTheCrew()"); + m.put("grateful", "graph = TinkerFactory.createGratefulDead()"); + m.put("sink", "graph = TinkerFactory.createKitchenSink()"); + GRAPH_INIT = Collections.unmodifiableMap(m); + } + + private int gremlinBlockCount; + private final StatementExecutor executor; + private final TabbedHtmlBuilder tabBuilder; + private final ConsoleRestartHandler restartHandler; + private String currentGraph; + private List currentExcludedPlugins; + + /** + * Creates a GremlinTreeprocessor that processes blocks without executing them (dry-run mode). + */ + public GremlinTreeprocessor() { + this((StatementExecutor) null, null); + } + + /** + * Creates a GremlinTreeprocessor that executes blocks against the provided console. + * + * @param console the GremlinConsole to execute statements against, or null for dry-run + */ + public GremlinTreeprocessor(final GremlinConsole console) { + this(console == null ? null : statement -> console.execute(statement), null); + } + + /** + * Creates a GremlinTreeprocessor with a custom statement executor for testing. + * + * @param executor the executor to use, or null for dry-run + */ + GremlinTreeprocessor(final StatementExecutor executor) { + this(executor, null); + } + + /** + * Creates a GremlinTreeprocessor with a custom statement executor and restart handler. + * + * @param executor the executor to use, or null for dry-run + * @param restartHandler the handler invoked when plugin exclusions change, or null to ignore + */ + GremlinTreeprocessor(final StatementExecutor executor, final ConsoleRestartHandler restartHandler) { + this.executor = executor; + this.tabBuilder = new TabbedHtmlBuilder(); + this.restartHandler = restartHandler; + } + + private StatementExecutor resolvedExecutor; + private GremlinConsole lazyConsole; + private Path consoleHomePath; + private boolean dryRun; + + @Override + public Document process(final Document document) { + gremlinBlockCount = 0; + currentGraph = null; + final Object dryRunAttr = document.getAttribute("gremlin-docs-dryrun"); + dryRun = dryRunAttr != null && !"false".equals(dryRunAttr.toString()); + + // Store console home for lazy init on first gremlin block + if (!dryRun && executor == null) { + final Object consoleHome = document.getAttribute("gremlin-docs-console-home"); + if (consoleHome != null && !consoleHome.toString().isEmpty()) { + consoleHomePath = Paths.get(consoleHome.toString()); + } else { + LOG.info("No gremlin-docs-console-home attribute; skipping console execution"); + } + } + + try { + checkPluginExclusions(document); + processBlock(document, dryRun); + LOG.info("Processed " + gremlinBlockCount + " gremlin blocks"); + } finally { + if (lazyConsole != null) { + lazyConsole.close(); + lazyConsole = null; + resolvedExecutor = null; + } + } + return document; + } + + /** + * Starts the console on demand when the first gremlin block is encountered. + */ + private void ensureConsoleStarted() { + if (resolvedExecutor != null || executor != null) return; + if (consoleHomePath == null) return; + try { + LOG.info("Starting GremlinConsole from: " + consoleHomePath); + lazyConsole = new GremlinConsole(consoleHomePath); + resolvedExecutor = statement -> lazyConsole.execute(statement); + LOG.info("GremlinConsole started successfully"); + } catch (final IOException | GremlinConsole.ConsoleTimeoutException e) { + throw new RuntimeException("Failed to start GremlinConsole from: " + consoleHomePath, e); + } + } + + /** + * Returns the number of gremlin-groovy listing blocks found during the last {@link #process} call. + */ + public int getGremlinBlockCount() { + return gremlinBlockCount; + } + + /** + * Checks the document for the {@code :gremlin-docs-plugins-exclude:} attribute and invokes + * the restart handler if the exclusion list has changed. + */ + private void checkPluginExclusions(final Document document) { + if (restartHandler == null) return; + if (!document.hasAttribute(PLUGINS_EXCLUDE_ATTR)) { + if (currentExcludedPlugins != null) { + currentExcludedPlugins = null; + invokeRestartHandler(Collections.emptyList()); + } + return; + } + + final Object attrValue = document.getAttribute(PLUGINS_EXCLUDE_ATTR); + final List excludeList = parseExcludeList(attrValue == null ? "" : attrValue.toString()); + + if (!excludeList.equals(currentExcludedPlugins)) { + currentExcludedPlugins = excludeList; + invokeRestartHandler(excludeList); + } + } + + /** + * Parses a comma-separated list of plugin names into a sorted, deduplicated list. + */ + static List parseExcludeList(final String value) { + if (value == null || value.trim().isEmpty()) { + return Collections.emptyList(); + } + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .sorted() + .distinct() + .collect(Collectors.toList()); + } + + private void invokeRestartHandler(final List excludedPlugins) { + try { + restartHandler.onRestart(excludedPlugins); + } catch (final IOException e) { + throw new RuntimeException("Failed to restart console with excluded plugins: " + excludedPlugins, e); + } + } + + private void processBlock(final StructuralNode node, final boolean dryRun) { + final List blocks = node.getBlocks(); + for (int i = 0; i < blocks.size(); i++) { + final StructuralNode block = blocks.get(i); + if ("listing".equals(block.getContext()) && STYLE.equals(block.getStyle())) { + gremlinBlockCount++; + LOG.info("Processing gremlin block #" + gremlinBlockCount); + i = processGremlinTabGroup(node, i, dryRun); + } else if (isStandaloneTabBlock(block)) { + i = processStandaloneTabGroup(node, i); + } else { + processBlock(block, dryRun); + } + } + } + + /** + * Processes a gremlin-groovy block and any consecutive manual language variant siblings + * into a tabbed HTML group. Returns the index to continue iteration from. + */ + private int processGremlinTabGroup(final StructuralNode parent, final int startIndex, final boolean dryRun) { + final List blocks = parent.getBlocks(); + final Block gremlinBlock = (Block) blocks.get(startIndex); + + final String consoleOutput = buildConsoleOutput(gremlinBlock, dryRun); + final List tabs = new ArrayList<>(); + tabs.add(TabbedHtmlBuilder.consoleTab("groovy", consoleOutput)); + + // Consume consecutive [source,] sibling blocks as manual tabs (FR-5) + int lastIndex = startIndex; + for (int j = startIndex + 1; j < blocks.size(); j++) { + final StructuralNode sibling = blocks.get(j); + if (isManualTabBlock(sibling)) { + final Block sourceBlock = (Block) sibling; + final String lang = getSourceLanguage(sourceBlock); + tabs.add(TabbedHtmlBuilder.codeTab(lang, sourceBlock.getSource())); + lastIndex = j; + } else { + break; + } + } + + final String html = tabBuilder.build(tabs); + replaceWithPassBlock(parent, startIndex, lastIndex, html); + return startIndex; + } + + /** + * Processes consecutive standalone [source,,tab] blocks into a tab group (FR-7). + * Returns the index to continue iteration from. + */ + private int processStandaloneTabGroup(final StructuralNode parent, final int startIndex) { + final List blocks = parent.getBlocks(); + final List tabs = new ArrayList<>(); + + int lastIndex = startIndex; + for (int j = startIndex; j < blocks.size(); j++) { + final StructuralNode block = blocks.get(j); + if (isStandaloneTabBlock(block)) { + final Block sourceBlock = (Block) block; + final String lang = getSourceLanguage(sourceBlock); + tabs.add(TabbedHtmlBuilder.codeTab(lang, sourceBlock.getSource())); + lastIndex = j; + } else { + break; + } + } + + final String html = tabBuilder.build(tabs); + replaceWithPassBlock(parent, startIndex, lastIndex, html); + return startIndex; + } + + /** + * Checks if a block is a [source,] listing block with a supported language (not tab-annotated). + */ + private static boolean isManualTabBlock(final StructuralNode block) { + if (!"listing".equals(block.getContext())) return false; + if (!"source".equals(block.getStyle())) return false; + final Object thirdAttr = block.getAttributes().get("3"); + if (thirdAttr != null && TAB_ATTR.equals(thirdAttr.toString().trim())) return false; + final String lang = getSourceLanguage((Block) block); + return lang != null && SUPPORTED_LANGUAGES.contains(lang); + } + + /** + * Checks if a block is a standalone tab block: [source,,tab]. + */ + private static boolean isStandaloneTabBlock(final StructuralNode block) { + if (!"listing".equals(block.getContext())) return false; + if (!"source".equals(block.getStyle())) return false; + final Object thirdAttr = block.getAttributes().get("3"); + if (thirdAttr == null) return false; + if (!TAB_ATTR.equals(thirdAttr.toString().trim())) return false; + final String lang = getSourceLanguage((Block) block); + return lang != null && SUPPORTED_LANGUAGES.contains(lang); + } + + /** + * Gets the source language from a listing block's attributes. + */ + private static String getSourceLanguage(final Block block) { + final Object langAttr = block.getAttributes().get("language"); + if (langAttr != null) { + final String lang = langAttr.toString().trim(); + if (!lang.isEmpty()) return lang; + } + final Object attr = block.getAttributes().get("2"); + if (attr == null) return null; + final String lang = attr.toString().trim(); + return lang.isEmpty() ? null : lang; + } + + /** + * Replaces blocks from startIndex to endIndex (inclusive) with a pass block containing raw HTML. + */ + private void replaceWithPassBlock(final StructuralNode parent, final int startIndex, final int endIndex, + final String html) { + final List blocks = parent.getBlocks(); + for (int j = endIndex; j >= startIndex; j--) { + blocks.remove(j); + } + final Map attrs = new HashMap<>(); + final Block passBlock = createBlock(parent, "pass", html, attrs); + blocks.add(startIndex, passBlock); + } + + private String buildConsoleOutput(final Block block, final boolean dryRun) { + try { + return doBuildConsoleOutput(block, dryRun); + } catch (final ConsoleRestartedException e) { + // Console was restarted — retry the entire block from scratch + LOG.info("Retrying block after console restart"); + try { + return doBuildConsoleOutput(block, dryRun); + } catch (final ConsoleRestartedException e2) { + // Second failure — skip this block with a warning rather than failing the build + LOG.warning("Block failed after retry, skipping: " + e2.getMessage()); + return buildDryRunOutput(block); + } + } + } + + private String buildDryRunOutput(final Block block) { + final String source = block.getSource(); + if (source == null || source.isEmpty()) return ""; + final StringBuilder output = new StringBuilder(); + for (final String line : source.split("\\r?\\n")) { + output.append(PROMPT).append(line).append("\n"); + } + return output.toString().stripTrailing(); + } + + private String doBuildConsoleOutput(final Block block, final boolean dryRun) { + if (!dryRun) { + ensureConsoleStarted(); + } + if (!dryRun && getActiveExecutor() != null) { + final String graphName = extractGraphName(block); + initGraphIfNeeded(graphName); + } + + final String source = block.getSource(); + if (source == null || source.isEmpty()) { + return ""; + } + + final StringBuilder output = new StringBuilder(); + final String[] lines = source.split("\\r?\\n"); + final List statements = buildStatements(lines); + for (final String statement : statements) { + // Show each original line with prompt in output + for (final String displayLine : statement.split("\\r?\\n")) { + output.append(PROMPT).append(displayLine).append("\n"); + } + if (!dryRun && getActiveExecutor() != null) { + final String result = executeSafely(statement); + if (result != null && !result.isEmpty()) { + for (final String resultLine : result.split("\\r?\\n")) { + output.append("\t").append(resultLine).append("\n"); + } + } + } + } + return output.toString().stripTrailing(); + } + + /** + * Groups source lines into complete statements. Lines ending with a period, backslash, + * or opening brace, or followed by indented continuation lines, are joined. + */ + static List buildStatements(final String[] lines) { + final List statements = new ArrayList<>(); + final StringBuilder current = new StringBuilder(); + for (final String line : lines) { + final String cleaned = stripCallouts(line); + if (current.length() == 0) { + current.append(cleaned); + } else if (cleaned.length() > 0 && Character.isWhitespace(cleaned.charAt(0))) { + // Continuation line (indented) + current.append("\n").append(cleaned); + } else { + statements.add(current.toString()); + current.setLength(0); + current.append(cleaned); + } + } + if (current.length() > 0) { + statements.add(current.toString()); + } + return statements; + } + + /** + * Strips AsciiDoc callout markers (e.g. {@code <1>}, {@code <2>}) from the end of a line. + */ + static String stripCallouts(final String line) { + return line.replaceAll("\\s*((<\\d+>\\s*)*<\\d+>)\\s*$", ""); + } + + /** + * Extracts the graph name from the second positional attribute of the block. + */ + static String extractGraphName(final StructuralNode block) { + final Object attr = block.getAttributes().get("2"); + if (attr == null) { + return null; + } + final String name = attr.toString().trim(); + return name.isEmpty() ? null : name; + } + + /** + * Initializes the graph in the console if the graph name has changed. + */ + private void initGraphIfNeeded(final String graphName) { + if (EXISTING.equals(graphName)) { + return; + } + + if (graphName != null && graphName.equals(currentGraph)) { + return; + } + if (graphName == null && currentGraph == null && gremlinBlockCount > 1) { + return; + } + + final String initStatement; + if (graphName == null) { + initStatement = "graph = TinkerGraph.open()"; + } else { + initStatement = GRAPH_INIT.getOrDefault(graphName, "graph = TinkerGraph.open()"); + } + + executeSafely(initStatement); + executeSafely("g = graph.traversal()"); + currentGraph = graphName; + } + + private StatementExecutor getActiveExecutor() { + return executor != null ? executor : resolvedExecutor; + } + + private String executeSafely(final String statement) { + try { + return getActiveExecutor().execute(statement); + } catch (final GremlinConsole.ConsoleTimeoutException | IOException e) { + // Console may have died — restart it and propagate to retry at block level + LOG.warning("Console appears dead, restarting: " + e.getMessage()); + restartConsole(); + throw new ConsoleRestartedException("Console restarted due to: " + e.getMessage()); + } catch (final Exception e) { + throw new RuntimeException("Failed to execute statement: " + statement, e); + } + } + + private void restartConsole() { + if (lazyConsole != null) { + lazyConsole.close(); + lazyConsole = null; + resolvedExecutor = null; + } + currentGraph = null; + ensureConsoleStarted(); + } + + /** + * Thrown when the console was restarted mid-block to signal that the block should be retried. + */ + static class ConsoleRestartedException extends RuntimeException { + ConsoleRestartedException(final String message) { + super(message); + } + } + + /** + * Functional interface for executing Gremlin statements, enabling testability. + */ + @FunctionalInterface + interface StatementExecutor { + String execute(String statement) throws IOException, GremlinConsole.ConsoleTimeoutException; + } +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java new file mode 100644 index 00000000000..378fc0963f8 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Generates CSS-only tabbed HTML matching the existing {@code tinkerpop.css} tab styles. + *

+ * The generated structure uses radio inputs for tab switching without JavaScript: + * 

+ * <section class="tabs tabs-N">
+ *   <input type="radio" name="tab-group-ID" id="tab-ID-1" class="tab-selector-1" checked>
+ *   <label for="tab-ID-1" class="tab-label-1">Label</label>
+ *   ...
+ *   <div class="clear-shadow"></div>
+ *   <div class="tabcontent">
+ *     <div class="tabcontent-1">...</div>
+ *     ...
+ *   </div>
+ * </section>
+ *
+ */ +public class TabbedHtmlBuilder { + + private static final Pattern CALLOUT_PATTERN = Pattern.compile("<(\\d+)>"); + + private int groupCounter = 0; + + /** + * A single tab entry with a label, language, and source code content. + */ + static class Tab { + private final String label; + private final String language; + private final String content; + + Tab(final String label, final String language, final String content) { + this.label = label; + this.language = language; + this.content = content; + } + + String getLabel() { + return label; + } + + String getLanguage() { + return language; + } + + String getContent() { + return content; + } + } + + /** + * Returns the current group counter value (for testing). + */ + int getGroupCounter() { + return groupCounter; + } + + /** + * Resets the group counter (for testing or reprocessing). + */ + void resetCounter() { + groupCounter = 0; + } + + /** + * Builds tabbed HTML for a list of tabs. + * + * @param tabs the tabs to render + * @return the complete HTML string for the tab group + */ + String build(final List tabs) { + if (tabs == null || tabs.isEmpty()) { + return ""; + } + + final int groupId = ++groupCounter; + final int tabCount = tabs.size(); + final StringBuilder html = new StringBuilder(); + + html.append("
\n"); + + // Radio inputs and labels + for (int i = 0; i < tabCount; i++) { + final int tabNum = i + 1; + final String inputId = "tab-" + groupId + "-" + tabNum; + final String checked = (i == 0) ? " checked" : ""; + + html.append(" \n"); + html.append(" \n"); + } + + html.append("
\n"); + html.append("
\n"); + + // Tab content panels + for (int i = 0; i < tabCount; i++) { + final Tab tab = tabs.get(i); + final int tabNum = i + 1; + html.append("
\n"); + html.append(" 
")
+                    .append(escapeHtml(tab.getContent()))
+                    .append("
\n"); + html.append("
\n"); + } + + html.append("
\n"); + html.append("
"); + + return html.toString(); + } + + /** + * Creates a console tab for a gremlin-groovy block. + * + * @param lang the language (e.g., "groovy") + * @param consoleOutput the formatted console output + * @return a Tab instance for the console output + */ + static Tab consoleTab(final String lang, final String consoleOutput) { + return new Tab("console (" + lang + ")", lang, consoleOutput); + } + + /** + * Creates a code tab for a manual language variant block. + * + * @param lang the language identifier + * @param source the source code + * @return a Tab instance for the code + */ + static Tab codeTab(final String lang, final String source) { + return new Tab(lang, lang, transformCallouts(source)); + } + + /** + * Transforms AsciiDoc callout markers (e.g., {@code <1>}) into comment form ({@code // <1>}). + * + * @param source the source code potentially containing callout markers + * @return the source with callout markers transformed to comments + */ + static String transformCallouts(final String source) { + if (source == null || source.isEmpty()) { + return source; + } + + final StringBuilder result = new StringBuilder(); + final String[] lines = source.split("\\r?\\n"); + for (int i = 0; i < lines.length; i++) { + if (i > 0) { + result.append("\n"); + } + result.append(transformCalloutLine(lines[i])); + } + return result.toString(); + } + + /** + * Transforms callout markers at the end of a line into comment form. + */ + private static String transformCalloutLine(final String line) { + final Matcher matcher = CALLOUT_PATTERN.matcher(line); + final StringBuilder sb = new StringBuilder(); + int lastEnd = 0; + final List callouts = new ArrayList<>(); + + while (matcher.find()) { + // Only transform callouts that appear at the end of the line (possibly with whitespace) + final String afterMatch = line.substring(matcher.end()).trim(); + if (afterMatch.isEmpty() || CALLOUT_PATTERN.matcher(afterMatch).matches()) { + callouts.add(matcher.group(1)); + if (sb.length() == 0) { + sb.append(line, 0, matcher.start()); + } + } else { + // Not a trailing callout, keep as-is + return line; + } + lastEnd = matcher.end(); + } + + if (callouts.isEmpty()) { + return line; + } + + // Trim trailing whitespace before callouts and append as comments + final String codePart = sb.toString().stripTrailing(); + final StringBuilder result = new StringBuilder(codePart); + for (final String num : callouts) { + result.append(" // <").append(num).append(">"); + } + return result.toString(); + } + + private static String escapeHtml(final String text) { + if (text == null) return ""; + return text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """); + } +} diff --git a/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry b/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry new file mode 100644 index 00000000000..6a81ac1f60e --- /dev/null +++ b/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry @@ -0,0 +1 @@ +org.apache.tinkerpop.gremlin.docs.GremlinDocsExtension diff --git a/tools/tinkerpop-docs/src/main/resources/hadoop-conf/README b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/README new file mode 100644 index 00000000000..969c3a5a44b --- /dev/null +++ b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/README @@ -0,0 +1,34 @@ +Hadoop Configuration for Docs Builds +===================================== + +This directory contains Hadoop configuration files used during TinkerPop +documentation builds. These files allow SparkGraphComputer and Hadoop +filesystem operations (copyFromLocal, ls, head) to work against the local +filesystem without requiring any Hadoop daemons. + +Files +----- + +core-site.xml + Sets fs.defaultFS=file:/// so all HDFS API calls resolve to the local + filesystem. No NameNode or DataNode is needed. + +hadoop-docs.properties + Sets gremlin.hadoop.jarsInDistributedCache=false since all jars are + already on the local classpath. No ResourceManager or NodeManager is + needed. + +Usage +----- + +The docs build orchestration script (phase 8) adds this directory to the +Gremlin Console classpath so that Hadoop picks up core-site.xml +automatically. The properties file can be loaded explicitly in console +scripts or referenced as a graph configuration source. + +Prerequisites +------------- + +- Java (version matching the project build) +- spark.master=local[4] (configured elsewhere in the docs build) +- NO Hadoop daemons required (NameNode, DataNode, ResourceManager, NodeManager) diff --git a/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml new file mode 100644 index 00000000000..1155717eb62 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml @@ -0,0 +1,33 @@ + + + + + + fs.defaultFS + file:/// + + diff --git a/docs/preprocessor/control-characters.sh b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties old mode 100755 new mode 100644 similarity index 71% rename from docs/preprocessor/control-characters.sh rename to tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties index b8336d6bf01..6ee38de2b7d --- a/docs/preprocessor/control-characters.sh +++ b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties @@ -1,6 +1,3 @@ -#!/bin/bash -# -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,9 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -# The main purpose of this script is to remove control characters -# that are occasionally hidden in the Groovy console's output +# +# Hadoop/Spark properties for TinkerPop docs builds. +# No Hadoop daemons are required -- all I/O uses the local filesystem +# via fs.defaultFS=file:/// (see core-site.xml in this directory). +# -sed -r 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' +# Disable distributed cache since we run locally with all jars on the classpath. +gremlin.hadoop.jarsInDistributedCache=false diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java new file mode 100644 index 00000000000..15ce98f1540 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link GremlinConsole} that verify prompt detection, timeout, and error dismissal + * without requiring a real Gremlin Console process. + */ +public class GremlinConsoleTest { + + @Test + public void shouldDetectPromptAndReturnOutput() throws Exception { + final String fullStdout = "gremlin>" + "==>v[1]\n==>v[2]\ngremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("g.V()"); + assertThat(result, is("==>v[1]\n==>v[2]")); + } finally { + console.close(); + } + } + + @Test + public void shouldReturnEmptyForPromptOnly() throws Exception { + final String fullStdout = "gremlin>" + "gremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("1+1"); + assertThat(result, is("")); + } finally { + console.close(); + } + } + + @Test + public void shouldHandleMultiLineOutput() throws Exception { + final String fullStdout = "gremlin>" + "line1\nline2\nline3\ngremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("something"); + assertThat(result, is("line1\nline2\nline3")); + } finally { + console.close(); + } + } + + @Test + public void shouldTimeoutWhenNoPromptReceived() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + // Use 200ms timeout for fast test + final GremlinConsole console = new GremlinConsole(mockProcess, 200); + try { + console.execute("g.V()"); + throw new AssertionError("Expected ConsoleTimeoutException"); + } catch (final GremlinConsole.ConsoleTimeoutException e) { + assertThat(e.getMessage(), containsString("Timed out after 200ms")); + assertThat(e.getMessage(), containsString("Buffer contents:")); + } finally { + console.close(); + feeder.close(); + } + } + + @Test + public void shouldIncludeBufferContentsInTimeoutMessage() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds, then partial output + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.write("partial output here".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + final GremlinConsole console = new GremlinConsole(mockProcess, 200); + try { + console.execute("test"); + throw new AssertionError("Expected ConsoleTimeoutException"); + } catch (final GremlinConsole.ConsoleTimeoutException e) { + assertThat(e.getMessage(), containsString("partial output here")); + } finally { + console.close(); + feeder.close(); + } + } + + @Test + public void shouldDismissErrorPromptOnStderr() throws Exception { + final String fullStdout = "gremlin>" + "gremlin>"; + final String stderrContent = "Display stack trace? [yN]"; + final ByteArrayOutputStream capturedStdin = new ByteArrayOutputStream(); + final GremlinConsole console = createConsoleWithStdin(fullStdout, stderrContent, capturedStdin); + + try { + // Poll until the error dismisser has written to stdin, with timeout + final long deadline = System.currentTimeMillis() + 5000; + while (capturedStdin.size() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + final String result = console.execute("g.V()"); + assertThat(result, is("")); + // Verify that a newline was sent to dismiss the error prompt + final String sent = capturedStdin.toString(StandardCharsets.UTF_8.name()); + assertThat(sent.contains("\n"), is(true)); + } finally { + console.close(); + } + } + + @Test + public void shouldThrowIOExceptionWhenProcessDiesMidRead() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.write("partial".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + // Close the stream to simulate process death (read returns -1) + feeder.close(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + final GremlinConsole console = new GremlinConsole(mockProcess, 5000); + try { + console.execute("g.V()"); + throw new AssertionError("Expected IOException"); + } catch (final IOException e) { + assertThat(e.getMessage(), containsString("Console process ended unexpectedly")); + assertThat(e.getMessage(), containsString("partial")); + } finally { + console.close(); + } + } + + @Test + public void shouldShutdownCleanly() throws Exception { + final String fullStdout = "gremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + console.close(); + // If we get here without hanging, shutdown is clean + } + + private GremlinConsole createConsole(final String stdoutContent, final String stderrContent) + throws IOException, GremlinConsole.ConsoleTimeoutException { + return createConsoleWithStdin(stdoutContent, stderrContent, new ByteArrayOutputStream()); + } + + private GremlinConsole createConsoleWithStdin(final String stdoutContent, final String stderrContent, + final OutputStream stdinCapture) + throws IOException, GremlinConsole.ConsoleTimeoutException { + final InputStream stdoutStream = new ByteArrayInputStream(stdoutContent.getBytes(StandardCharsets.UTF_8)); + final InputStream stderrStream = new ByteArrayInputStream(stderrContent.getBytes(StandardCharsets.UTF_8)); + final Process mockProcess = new MockProcess(stdoutStream, stderrStream, stdinCapture); + return new GremlinConsole(mockProcess, 5_000); + } + + /** + * A minimal Process mock that provides controlled streams. + */ + private static class MockProcess extends Process { + private final InputStream stdout; + private final InputStream stderr; + private final OutputStream stdin; + + MockProcess(final InputStream stdout, final InputStream stderr, final OutputStream stdin) { + this.stdout = stdout; + this.stderr = stderr; + this.stdin = stdin; + } + + @Override + public OutputStream getOutputStream() { + return stdin; + } + + @Override + public InputStream getInputStream() { + return stdout; + } + + @Override + public InputStream getErrorStream() { + return stderr; + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public boolean waitFor(final long timeout, final TimeUnit unit) { + return true; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() { + } + + @Override + public Process destroyForcibly() { + return this; + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java new file mode 100644 index 00000000000..76d9188035a --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Options; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Verifies that the AsciidoctorJ extensions register via SPI and process documents correctly. + */ +public class GremlinDocsTest { + + @Test + public void shouldRegisterExtensionsViaSpi() { + // SPI auto-registration happens when Asciidoctor is created + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + assertThat(asciidoctor, is(notNullValue())); + + // Convert a simple doc to verify no errors from extension registration + final String result = asciidoctor.convert("= Test\n\nHello", Options.builder().build()); + assertThat(result, is(notNullValue())); + } + } + + @Test + public void shouldPassthroughPostprocessor() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nSome content here."; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Postprocessor is a no-op, so output should contain the content + assertThat(result.contains("Some content here"), is(true)); + } + } + + @Test + public void shouldFindGremlinGroovyListingBlocks() { + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final String input = "= Test\n\n[gremlin-groovy]\n----\ng.V()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(treeprocessor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldIgnoreNonGremlinListingBlocks() { + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final String input = "= Test\n\n[source,java]\n----\nSystem.out.println();\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(treeprocessor.getGremlinBlockCount(), is(0)); + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java new file mode 100644 index 00000000000..67ffada29ed --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for the GremlinPostprocessor HTML transforms. + */ +public class GremlinPostprocessorTest { + + @Test + public void shouldAddInvisibleClassToConumElements() { + // Process through Asciidoctor with a callout to trigger conum generation + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\n" + + "[source,java]\n----\nint x = 1; // <1>\n----\n<1> Sets x\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + // The postprocessor (SPI-registered) should have added invisible class + assertThat(result, containsString("conum invisible")); + } + } + + @Test + public void shouldRemoveEmptyCommentSpans() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + // Simulate by testing the postprocessor directly with a mock-like Document + final GremlinPostprocessor pp = new GremlinPostprocessor(); + // Create a minimal document via Asciidoctor for version resolution + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + asciidoctor.load("= T\n\nhi", options); + + // Test the pattern directly + final String html = "code/* */more"; + // We can't easily call process() with a Document without SPI, + // so test the pattern logic + assertThat(html.replaceAll("/\\*\\s*\\*/", ""), + is("codemore")); + } + } + + @Test + public void shouldReplaceVersionPlaceholder() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n:tinkerpop-version: 3.7.7\n\nDownload version x.y.z from the repo."; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("3.7.7")); + assertThat(result, not(containsString("x.y.z"))); + } + } + + @Test + public void shouldNotModifyUnrelatedContent() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nHello world paragraph."; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("Hello world paragraph")); + } + } + + @Test + public void shouldLeaveVersionWhenNoAttribute() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nversion x.y.z here"; + // No tinkerpop-version or revnumber set + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("x.y.z")); + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java new file mode 100644 index 00000000000..bf51dbe1109 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java @@ -0,0 +1,512 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link GremlinTreeprocessor} that verify block processing logic + * without requiring a real Gremlin Console subprocess. + */ +public class GremlinTreeprocessorTest { + + @Test + public void shouldExecuteModernGraphBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(1)); + assertThat(executor.statements.contains("graph = TinkerFactory.createModern()"), is(true)); + assertThat(executor.statements.contains("g = traversal().with(graph)"), is(true)); + assertThat(executor.statements.contains("g.V(1)"), is(true)); + } + } + + @Test + public void shouldUseTinkerGraphForBareBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>2"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy]\n----\n1+1\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("graph = TinkerGraph.open()"), is(true)); + assertThat(executor.statements.contains("g = traversal().with(graph)"), is(true)); + } + } + + @Test + public void shouldReuseGraphStateForExisting() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,existing]\n----\ng.E()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(2)); + // Should only have one graph init (for modern), not for existing + final long initCount = executor.statements.stream() + .filter(s -> s.startsWith("graph = ")).count(); + assertThat(initCount, is(1L)); + } + } + + @Test + public void shouldSkipGraphInitForConsecutiveSameGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(2)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(2)); + final long modernInitCount = executor.statements.stream() + .filter(s -> s.equals("graph = TinkerFactory.createModern()")).count(); + assertThat(modernInitCount, is(1L)); + } + } + + @Test + public void shouldReInitWhenGraphChanges() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,classic]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("graph = TinkerFactory.createModern()"), is(true)); + assertThat(executor.statements.contains("graph = TinkerFactory.createClassic()"), is(true)); + } + } + + @Test + public void shouldFormatOutputWithPromptAndTabs() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("gremlin> g.V(1)")); + } + } + + @Test + public void shouldCaptureErrorsWithoutFailingBuild() { + final GremlinTreeprocessor.StatementExecutor failingExecutor = statement -> { + throw new IOException("Simulated error"); + }; + final GremlinTreeprocessor processor = new GremlinTreeprocessor(failingExecutor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ninvalid()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldHandleMultiLineCode() { + final List responses = new ArrayList<>(); + responses.add("==>v[1]\n==>v[2]"); + responses.add("==>2"); + final RecordingExecutor executor = new RecordingExecutor(responses); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V()\ng.V().count()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("g.V()"), is(true)); + assertThat(executor.statements.contains("g.V().count()"), is(true)); + } + } + + @Test + public void shouldHandleCrewGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,crew]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createTheCrew()"), is(true)); + } + } + + @Test + public void shouldHandleGratefulGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,grateful]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createGratefulDead()"), is(true)); + } + } + + @Test + public void shouldHandleSinkGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,sink]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createKitchenSink()"), is(true)); + } + } + + @Test + public void shouldFormatDryRunWithPromptsOnly() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\ng.E()\n----\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("gremlin-docs-dryrun", "").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("gremlin> g.V(1)")); + assertThat(result, containsString("gremlin> g.E()")); + assertThat(processor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldNotExecuteInDryRunMode() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("gremlin-docs-dryrun", "").build()) + .build(); + asciidoctor.convert(input, options); + assertThat(executor.statements.isEmpty(), is(true)); + } + } + + @Test + public void shouldCountBlocksWithoutConsole() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy]\n----\ng.V()\n----\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(processor.getGremlinBlockCount(), is(2)); + } + } + + @Test + public void shouldGenerateTabbedHtmlForGremlinBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("section class=\"tabs tabs-1\"")); + assertThat(result, containsString("console (groovy)")); + assertThat(result, containsString("tab-group-1")); + } + } + + @Test + public void shouldConsumeManualTabBlocksAfterGremlinBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n" + + "[source,java]\n----\ng.V(1)\n----\n" + + "[source,python]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("tabs tabs-3")); + assertThat(result, containsString("console (groovy)")); + assertThat(result, containsString("tab-label-2\">java")); + assertThat(result, containsString("tab-label-3\">python")); + } + } + + @Test + public void shouldNotConsumeUnsupportedLanguageAsManualTab() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n" + + "[source,ruby]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Only the console tab, ruby is not consumed + assertThat(result, containsString("tabs tabs-1")); + } + } + + @Test + public void shouldProcessStandaloneTabGroup() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[source,groovy,tab]\n----\ng.V()\n----\n" + + "[source,java,tab]\n----\ng.V()\n----\n" + + "[source,python,tab]\n----\ng.V()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("tabs tabs-3")); + assertThat(result, containsString("tab-label-1\">groovy")); + assertThat(result, containsString("tab-label-2\">java")); + assertThat(result, containsString("tab-label-3\">python")); + } + } + + @Test + public void shouldKeepStandaloneTabGroupsSeparateFromGremlinGroups() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[source,groovy,tab]\n----\ncode1\n----\n" + + "[source,java,tab]\n----\ncode2\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Should have two separate tab groups + assertThat(result, containsString("tab-group-1")); + assertThat(result, containsString("tab-group-2")); + } + } + + @Test + public void shouldInvokeRestartHandlerWhenPluginExcludeAttributePresent() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin,spark-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + assertThat(restartCalls.get(0).contains("neo4j-gremlin"), is(true)); + assertThat(restartCalls.get(0).contains("spark-gremlin"), is(true)); + } + } + + @Test + public void shouldNotInvokeRestartHandlerWhenNoExcludeAttribute() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.isEmpty(), is(true)); + } + } + + @Test + public void shouldNotInvokeRestartHandlerWhenExcludeListUnchanged() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // Process same document twice - handler should only be called once + final String input = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + } + } + + @Test + public void shouldInvokeRestartHandlerWhenExcludeListChanges() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input1 = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input1, Options.builder().build()); + + final String input2 = "= Test\n:gremlin-docs-plugins-exclude: spark-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input2, Options.builder().build()); + assertThat(restartCalls.size(), is(2)); + assertThat(restartCalls.get(0).contains("neo4j-gremlin"), is(true)); + assertThat(restartCalls.get(1).contains("spark-gremlin"), is(true)); + } + } + + @Test + public void shouldParseExcludeListWithWhitespace() { + final List result = GremlinTreeprocessor.parseExcludeList(" neo4j-gremlin , spark-gremlin , hadoop-gremlin "); + assertThat(result.size(), is(3)); + assertThat(result.contains("neo4j-gremlin"), is(true)); + assertThat(result.contains("spark-gremlin"), is(true)); + assertThat(result.contains("hadoop-gremlin"), is(true)); + } + + @Test + public void shouldParseEmptyExcludeList() { + assertThat(GremlinTreeprocessor.parseExcludeList("").isEmpty(), is(true)); + assertThat(GremlinTreeprocessor.parseExcludeList(" ").isEmpty(), is(true)); + assertThat(GremlinTreeprocessor.parseExcludeList(null).isEmpty(), is(true)); + } + + @Test + public void shouldInvokeRestartHandlerWithEmptyListWhenAttributeRemoved() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // First doc has exclusions + final String input1 = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input1, Options.builder().build()); + + // Second doc has no exclusions + final String input2 = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input2, Options.builder().build()); + assertThat(restartCalls.size(), is(2)); + assertThat(restartCalls.get(1).isEmpty(), is(true)); + } + } + + /** + * A recording executor that captures all executed statements and returns canned responses. + */ + private static class RecordingExecutor implements GremlinTreeprocessor.StatementExecutor { + final List statements = new ArrayList<>(); + private final List responses; + private int responseIndex = 0; + + RecordingExecutor(final String response) { + this.responses = new ArrayList<>(); + if (response != null) { + this.responses.add(response); + } + } + + RecordingExecutor(final List responses) { + this.responses = responses; + } + + @Override + public String execute(final String statement) { + statements.add(statement); + if (responses.isEmpty()) { + return ""; + } + final String response = responses.get(Math.min(responseIndex, responses.size() - 1)); + responseIndex++; + return response; + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java new file mode 100644 index 00000000000..5e1702f809c --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.asciidoctor.SafeMode; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Integration test that processes the full fixture document through Asciidoctor with the + * GremlinTreeprocessor (dry-run) and GremlinPostprocessor, verifying the output HTML. + */ +public class IntegrationTest { + + private static String html; + + @BeforeClass + public static void processFixture() throws IOException { + final String fixture; + try (final InputStream is = IntegrationTest.class.getResourceAsStream("/integration-fixture.asciidoc")) { + fixture = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final Options options = Options.builder() + .safe(SafeMode.UNSAFE) + .attributes(Attributes.builder() + .attribute("gremlin-docs-dryrun", "") + .attribute("tinkerpop-version", "3.7.7-SNAPSHOT") + .build()) + .build(); + html = asciidoctor.convert(fixture, options); + } + } + + @Test + public void shouldContainTabbedSectionsForGremlinBlocks() { + assertThat(html, containsString("section class=\"tabs")); + } + + @Test + public void shouldContainGremlinPrompts() { + assertThat(html, containsString("gremlin> g.V(1)")); + assertThat(html, containsString("gremlin> g.V(1).values('name')")); + } + + @Test + public void shouldContainConsoleTabLabel() { + assertThat(html, containsString("console (groovy)")); + } + + @Test + public void shouldContainManualTabLabels() { + assertThat(html, containsString(">java<")); + assertThat(html, containsString(">python<")); + } + + @Test + public void shouldProduceMultiTabGroupForManualTabs() { + // The gremlin block + java + python = 3 tabs + assertThat(html, containsString("tabs tabs-3")); + } + + @Test + public void shouldProduceStandaloneTabGroup() { + // groovy + java + csharp standalone tabs = 3 tabs + assertThat(html, containsString(">groovy<")); + assertThat(html, containsString(">csharp<")); + } + + @Test + public void shouldHandleBareGremlinBlock() { + assertThat(html, containsString("gremlin> 1+1")); + } + + @Test + public void shouldHandleExistingGraphBlock() { + assertThat(html, containsString("gremlin> g.V().count()")); + } + + @Test + public void shouldHandleErrorBlock() { + assertThat(html, containsString("gremlin> invalid_syntax_here()")); + } + + @Test + public void shouldHandleCallouts() { + // Callouts in the [source,java] block should be processed by Asciidoctor + assertThat(html, containsString("conum")); + } + + @Test + public void shouldReplaceVersionPlaceholder() { + assertThat(html, containsString("3.7.7-SNAPSHOT")); + assertThat(html, not(containsString("x.y.z"))); + } + + @Test + public void shouldContainCodeRayHighlightStructure() { + assertThat(html, containsString("CodeRay highlight")); + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java new file mode 100644 index 00000000000..e26efc99f78 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link TabbedHtmlBuilder}. + */ +public class TabbedHtmlBuilderTest { + + private TabbedHtmlBuilder builder; + + @Before + public void setUp() { + builder = new TabbedHtmlBuilder(); + } + + @Test + public void shouldReturnEmptyStringForNullTabs() { + assertThat(builder.build(null), is("")); + } + + @Test + public void shouldReturnEmptyStringForEmptyTabs() { + assertThat(builder.build(Collections.emptyList()), is("")); + } + + @Test + public void shouldGenerateSectionWithTabsClass() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "gremlin> g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateCorrectTabCountClass() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "g.V()"), + TabbedHtmlBuilder.codeTab("python", "g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateRadioInputsWithCorrectAttributes() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("type=\"radio\"")); + assertThat(html, containsString("name=\"tab-group-1\"")); + assertThat(html, containsString("id=\"tab-1-1\"")); + assertThat(html, containsString("id=\"tab-1-2\"")); + assertThat(html, containsString("class=\"tab-selector-1\"")); + assertThat(html, containsString("class=\"tab-selector-2\"")); + } + + @Test + public void shouldCheckFirstTabByDefault() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("id=\"tab-1-1\" class=\"tab-selector-1\" checked")); + assertThat(html, not(containsString("id=\"tab-1-2\" class=\"tab-selector-2\" checked"))); + } + + @Test + public void shouldGenerateLabelsWithCorrectForAttribute() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("")); + assertThat(html, containsString("")); + } + + @Test + public void shouldGenerateClearShadowDiv() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "output")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateTabContentContainer() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "output")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateCodeRayPreBlocks() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.codeTab("java", "g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
"));
+        assertThat(html, containsString("
")); + } + + @Test + public void shouldEscapeHtmlInContent() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.codeTab("java", "List x = new ArrayList<>()")); + final String html = builder.build(tabs); + assertThat(html, containsString("List<String> x = new ArrayList<>()")); + } + + @Test + public void shouldEscapeHtmlInLabels() { + final TabbedHtmlBuilder.Tab tab = new TabbedHtmlBuilder.Tab("